remi 0.2.31 → 0.2.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c7c81a972d6fc604a4761b7fff19a26ec187065
-  data.tar.gz: f07165f263dee57adb1802182cf72551d3326ab4
+  metadata.gz: 465e12569ac09f43f5e9f20fcf40ac4ee9f2e14c
+  data.tar.gz: 2c81e6feda132e5c3bc3aea5259f907a56bf0f6f
 SHA512:
-  metadata.gz: d4ea4cde266110c374b75ef527f0ec71a674bba865c3da46107687fddb4871b3fda5aa601c44cd0fad5857038d8dff1c4dce297cf1d710ca335e2afafaa882dd
-  data.tar.gz: 7a25d3e85b4ae727998ff196b79395db40bc0d584c04c061069c4d0a457bd79530f7a66b4ce3480df739ebfcef6dda7ef011c5e8171f76bac2326ee392e1d01a
+  metadata.gz: 8632e44d75be366bc3323567f597c73b5efb5e88a8fd07f94e1fa69e09a9a9c5b0f021f9f4ac13d7ae94030957f559136dfd09ad6f2626eb92743108f25cf307
+  data.tar.gz: 8657740cdb339d3bd311ec87d3aab8bbb63e005bd8bf07e5cac565bb7276417940e5885014dc95b8c5e3b6d69412b467cdeb0acff69ab981590cf884faa8948f

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 gem 'daru', '0.1.2', git: 'git@github.com:inside-track/daru.git', branch: 'itk-master'
 gem 'restforce', '~> 2.1'
 gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
+gem 'aws-sdk', '~> 2.3'

data/Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.31)
+    remi (0.2.32)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
@@ -40,6 +40,12 @@ GEM
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
+    aws-sdk (2.3.5)
+      aws-sdk-resources (= 2.3.5)
+    aws-sdk-core (2.3.5)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.3.5)
+      aws-sdk-core (= 2.3.5)
     bond (0.5.1)
     builder (3.2.2)
     clbustos-rtf (0.4.2)
@@ -70,7 +76,10 @@ GEM
       mimemagic (~> 0.3)
       multi_json (~> 1.11)
       rbczmq (~> 1.7)
+    jmespath (1.2.4)
+      json_pure (>= 1.8.1)
     json (1.8.3)
+    json_pure (1.8.3)
     mimemagic (0.3.1)
     minitest (5.8.4)
     multi_json (1.11.2)
@@ -128,6 +137,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk (~> 2.3)
   daru (= 0.1.2)!
   iruby (= 0.2.7)
   remi!

data/features/step_definitions/remi_step.rb

@@ -334,6 +334,7 @@ end
 ### Transforms
 
 Then /^the target field '([^']+)' is a concatenation of the source fields '(.+)', delimited by "([^"]*)"$/ do |target_field, source_field_list, delimiter|
+  delimiter = delimiter.gsub(/(\\n|\\t)/, '\n' => "\n", '\t' => "t" )
   source_fields = "'#{source_field_list}'".gsub(' and ', ', ').split(',').map do |field_with_quotes|
     full_field_name = field_with_quotes.match(/'(.+)'/)[1]
 

data/lib/remi/data_subject/csv_file.rb

@@ -71,7 +71,7 @@ module Remi
 
     def extractor=(arg)
       case arg
-      when Extractor::SftpFile, Extractor::LocalFile
+      when Extractor::SftpFile, Extractor::LocalFile, Extractor::S3File
         @extractor = arg
       when String
         @extractor = Extractor::LocalFile.new(path: arg)

data/lib/remi/data_subject/salesforce.rb

@@ -110,11 +110,11 @@ module Remi
 
       df_as_array_of_hashes = df.to_a[0] # This probably wouldn't work with a non-Daru df
       if @operation == :update
-        Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, logger: @logger)
       elsif @operation == :create
-        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, logger: @logger)
       elsif @operation == :upsert
-        Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, df_as_array_of_hashes, external_id: @external_id, logger: @logger)
+        Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, external_id: @external_id, logger: @logger)
       else
         raise ArgumentError, "Unknown operation: #{@operation}"
       end
@@ -124,9 +124,10 @@ module Remi
 
     private
 
-    def init_salesforce(*args, object:, operation:, credentials:, external_id: 'Id', api: :bulk, **kargs, &block)
+    def init_salesforce(*args, object:, operation:, credentials:, batch_size: 5000, external_id: 'Id', api: :bulk, **kargs, &block)
       @sfo = object
       @operation = operation
+      @batch_size = batch_size
       @external_id = external_id
       @credentials = credentials
       @api = api

data/lib/remi/extractor/file_system.rb

@@ -0,0 +1,87 @@
+module Remi
+  module Extractor
+
+    class FileSystemEntry
+      def initialize(pathname:, create_time:, modified_time:, raw: nil)
+        @pathname = Pathname.new(pathname)
+        @create_time = create_time
+        @modified_time = modified_time
+        @raw = raw
+      end
+
+      attr_reader :pathname, :create_time, :modified_time, :raw
+
+      def name
+        @pathname.basename.to_s
+      end
+    end
+
+
+    class FileSystem
+
+      class FileNotFoundError < StandardError; end
+
+      def initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block)
+        @remote_path = Pathname.new(remote_path)
+        @pattern = pattern
+        @local_path = Pathname.new(local_path)
+        @most_recent_only = most_recent_only
+        @group_by = group_by
+        @most_recent_by = most_recent_by
+        @logger = logger
+      end
+
+      attr_reader :logger
+
+      # Public: Called to extract files from the source filesystem.
+      #
+      # Returns an array with containing the paths to all files extracted.
+      def extract
+        raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
+      end
+
+      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
+      # NOTE: all_entries is responsible for matching the path using @remote_path
+      def all_entries
+        raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
+      end
+
+      # Public: Returns just the entries that are to be extracted.
+      def entries
+        if @group_by
+          most_recent_matching_entry_in_group
+        elsif @most_recent_only
+          Array(most_recent_matching_entry)
+        else
+          matching_entries
+        end
+      end
+
+      def matching_entries
+        all_entries.select { |e| @pattern.match e.name }
+      end
+
+      def most_recent_matching_entry
+        matching_entries.sort_by { |e| e.send(@most_recent_by) }.reverse.first
+      end
+
+      def most_recent_matching_entry_in_group
+        entries_with_group = matching_entries.map do |entry|
+          match = entry.name.match(@group_by)
+          next unless match
+
+          group = match.to_a[1..-1]
+          { group: group, entry: entry }
+        end.compact
+        sorted_entries_with_group = entries_with_group.sort_by { |e| [e[:group], e[:entry].send(@most_recent_by)] }.reverse
+
+        last_group = nil
+        sorted_entries_with_group.map do |entry|
+          next unless entry[:group] != last_group
+          last_group = entry[:group]
+          entry[:entry]
+        end.compact
+      end
+    end
+  end
+end

data/lib/remi/extractor/s3_file.rb

@@ -0,0 +1,57 @@
+module Remi
+  module Extractor
+
+    class S3File < FileSystem
+
+      def initialize(*args, **kargs, &block)
+        super
+        init_s3_file(*args, **kargs, &block)
+      end
+
+      # Public: Called to extract files from the source filesystem.
+      #
+      # Returns an array with containing the paths to all files extracted.
+      def extract
+        entries.map do |entry|
+          local_file = File.join(@local_path, entry.name)
+          @logger.info "Downloading #{entry.pathname} from S3 to #{local_file}"
+          File.open(local_file, 'wb') { |file| entry.raw.get(response_target: file) }
+          local_file
+        end
+      end
+
+      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
+      def all_entries
+        @all_entries ||= all_entries!
+      end
+
+      def all_entries!
+        # S3 does not track anything like a create time, so use last modified for both
+        bucket.objects(prefix: @remote_path.to_s).map do |entry|
+          FileSystemEntry.new(
+            pathname: entry.key,
+            create_time: entry.last_modified,
+            modified_time: entry.last_modified,
+            raw: entry
+          )
+        end
+      end
+
+      def s3_client
+        @s3_client ||= Aws::S3::Client.new
+      end
+
+      private
+
+      def init_s3_file(*args, bucket:, **kargs)
+        @bucket_name = bucket
+      end
+
+      def bucket
+        @bucket ||= Aws::S3::Bucket.new(@bucket_name, client: s3_client)
+      end
+
+    end
+
+  end
+end

data/lib/remi/extractor/sftp_file_new.rb

@@ -0,0 +1,78 @@
+module Remi
+  module Extractor
+
+    class SftpFileNew < FileSystem
+
+      N_RETRY = 3
+
+      def initialize(*args, **kargs)
+        super
+        init_sftp_file(*args, **kargs)
+      end
+
+      # Public: Called to extract files from the source filesystem.
+      #
+      # Returns an array with containing the paths to all files extracted.
+      def extract
+        connection do |sftp|
+          entries.map do |entry|
+            local_file = File.join(@local_path, entry.name)
+            @logger.info "Downloading #{entry.name} to #{local_file}"
+            retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
+            local_file
+          end
+        end
+      end
+
+      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
+      def all_entries
+        @all_entries ||= all_entries!
+      end
+
+      def all_entries!
+        sftp_entries = connection { |sftp| sftp.dir.entries(@remote_path.dirname) }
+        sftp_entries.map do |entry|
+          # Early versions of the protocol don't support create time, fake it with modified time?
+          FileSystemEntry.new(
+            name: File.join(@remote_path.dirname, entry.name),
+            create_time: entry.respond_to?(:createtime) ? entry.createtime : entry.mtime,
+            modified_time: entry.mtime
+          )
+        end
+      end
+
+
+      private
+
+      def init_sftp_file(*args, credentials:, **kargs)
+        @host = credentials.fetch(:host)
+        @username = credentials.fetch(:username)
+        @password = credentials.fetch(:password)
+        @port = credentials.fetch(:port, '22')
+      end
+
+      def connection(&block)
+        result = nil
+        Net::SFTP.start(@host, @username, password: @password, port: @port) do |sftp|
+          result = yield sftp
+        end
+        result
+      end
+
+      def retry_download(&block)
+        1.upto(N_RETRY).each do |itry|
+          begin
+            block.call
+            break
+          rescue RuntimeError => err
+            raise err unless itry < ntry
+            @logger.error "Download failed with error: #{err.message}"
+            @logger.error "Retry attempt #{itry}/#{ntry-1}"
+            sleep(1)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.31'
+  VERSION = '0.2.32'
 end

data/lib/remi.rb

@@ -42,7 +42,10 @@ require 'remi/field_symbolizers'
 
 require 'remi/refinements/symbolizer'
 
-require 'remi/extractor/sftp_file'
+require 'remi/extractor/sftp_file' # deprecated
+require 'remi/extractor/file_system'
+require 'remi/extractor/s3_file'
+
 
 require 'remi/fields'
 require 'remi/data_frame'

data/spec/extractor/file_system_spec.rb

@@ -0,0 +1,138 @@
+require 'remi_spec'
+
+describe Extractor::FileSystem do
+  before do
+    now = Time.new
+
+    example_files = [
+      { pathname: "pdir/ApplicantsA-9.csv", create_time: now - 10.minutes },
+      { pathname: "pdir/ApplicantsA-3.csv", create_time: now - 5.minutes },
+      { pathname: "pdir/ApplicantsA-5.csv", create_time: now - 1.minutes },
+      { pathname: "pdir/ApplicantsB-7.csv", create_time: now - 10.minutes },
+      { pathname: "pdir/ApplicantsB-6.csv", create_time: now - 5.minutes },
+      { pathname: "pdir/ApplicantsB-2.csv", create_time: now - 1.minutes },
+      { pathname: "pdir/ApplicantsB-2.txt", create_time: now - 0.minutes },
+      { pathname: "pdir/Apples.csv",        createtime: now - 1.minutes },
+      { pathname: "otherdir/ApplicantsA-11.csv", createtime: now - 1.minutes },
+    ]
+
+    remote_path = 'pdir'
+    allow_any_instance_of(Extractor::FileSystem).to receive(:all_entries) do
+      example_files.map do |entry|
+        Extractor::FileSystemEntry.new(
+          pathname: entry[:pathname],
+          create_time: entry[:create_time],
+          modified_time: entry[:create_time]
+        ) if Pathname.new(entry[:pathname]).dirname.to_s == remote_path
+      end.compact
+    end
+
+    @params = { remote_path: remote_path }
+  end
+
+  let(:file_system) { Extractor::FileSystem.new(**@params) }
+
+
+
+  context 'extracting all files matching a pattern' do
+    before do
+      @params.merge!({
+        pattern: /ApplicantsA-\d+\.csv/
+      })
+    end
+
+    it 'does not extract non-matching files' do
+      expect(file_system.entries.map(&:name)).not_to include "Apples.csv"
+    end
+
+    it 'does not extract files not in the target directory' do
+      expect(file_system.entries.map(&:name)).not_to include "ApplicantsA-11.csv"
+    end
+
+    it 'extracts all matching files' do
+      expect(file_system.entries.map(&:name)).to match_array([
+       "ApplicantsA-9.csv",
+       "ApplicantsA-3.csv",
+       "ApplicantsA-5.csv"
+      ])
+    end
+  end
+
+
+  context 'extracting only the most recent matching a pattern' do
+    before do
+      @params.merge!({
+        pattern: /ApplicantsA-\d+\.csv/,
+        most_recent_only: true
+      })
+    end
+
+    it 'extracts only the most recent matching file' do
+      expect(file_system.entries.map(&:name)).to match_array([
+       "ApplicantsA-5.csv"
+      ])
+    end
+
+    context 'using filename instead of createtime' do
+      before do
+        @params.merge!({
+          most_recent_by: :name
+        })
+      end
+
+      it 'extracts only the most recent matching file' do
+        expect(file_system.entries.map(&:name)).to match_array([
+         "ApplicantsA-9.csv"
+        ])
+      end
+    end
+  end
+
+
+  context 'extracting files matching a pattern with a by group' do
+    before do
+      @params.merge!({
+        pattern: /^Applicants(A|B)-\d+\.csv/,
+        group_by: /^Applicants(A|B)/
+      })
+    end
+
+    it 'extracts the most recent file that matches a particular regex' do
+      expect(file_system.entries.map(&:name)).to match_array([
+       "ApplicantsA-5.csv",
+       "ApplicantsB-2.csv"
+      ])
+    end
+
+    context 'with a minimally selective pre-filter' do
+      before do
+        @params.merge!({
+          pattern: /^Applicants/
+        })
+      end
+
+      it 'extracts the most recent file that matches a particular regex' do
+        expect(file_system.entries.map(&:name)).to match_array([
+         "ApplicantsA-5.csv",
+         "ApplicantsB-2.txt"
+        ])
+      end
+    end
+
+    context 'using filename instead of createtime' do
+      before do
+        @params.merge!({
+          most_recent_by: :name
+        })
+      end
+
+      it 'extracts only the most recent matching file' do
+        expect(file_system.entries.map(&:name)).to match_array([
+         "ApplicantsA-9.csv",
+         "ApplicantsB-7.csv"
+        ])
+      end
+    end
+
+  end
+end

data/spec/extractor/s3_file_spec.rb

@@ -0,0 +1,25 @@
+require 'remi_spec'
+require 'aws-sdk'
+
+describe Extractor::S3File do
+
+  before do
+    Aws.config[:s3] = {
+      stub_responses: true
+    }
+
+    prefix = "the-best-prefix"
+    @s3_file = Extractor::S3File.new(bucket: 'the-best-bucket', remote_path: "#{prefix}")
+    @s3_file.s3_client.stub_responses(:list_objects, {
+      contents: [
+        { key: "#{prefix}/file1.csv" },
+        { key: "#{prefix}/file2.csv" }
+      ]
+    })
+
+  end
+
+  it 'returns all entries' do
+    expect(@s3_file.all_entries.map(&:name)).to eq ['file1.csv', 'file2.csv']
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.31
+  version: 0.2.32
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-10 00:00:00.000000000 Z
+date: 2016-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -231,7 +231,10 @@ files:
 - lib/remi/data_subject/postgres.rb
 - lib/remi/data_subject/salesforce.rb
 - lib/remi/data_subject/sftp_file.rb
+- lib/remi/extractor/file_system.rb
+- lib/remi/extractor/s3_file.rb
 - lib/remi/extractor/sftp_file.rb
+- lib/remi/extractor/sftp_file_new.rb
 - lib/remi/field_symbolizers.rb
 - lib/remi/fields.rb
 - lib/remi/job.rb
@@ -242,6 +245,8 @@ files:
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
+- spec/extractor/file_system_spec.rb
+- spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
@@ -291,6 +296,8 @@ test_files:
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
 - features/transforms/truthy.feature
+- spec/extractor/file_system_spec.rb
+- spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
 - spec/metadata_spec.rb
 - spec/remi_spec.rb