remi 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0de4e8f2de3129e2e4b93c3d22dc5f718a05b56a
-  data.tar.gz: d963548c553f1918b33bd391038bc3481ce4a5d8
+  metadata.gz: 051d6add4664343ee59a6c722a2abadc15ea4377
+  data.tar.gz: f7f438b794a08948617b767dfca83e58533300ad
 SHA512:
-  metadata.gz: d01e67e38c2a76784e65a22536d2d9cba7c9f56dc3686e8d0d23ea1e5176cb8495cd06977ba85357d336cbf7fd91641f79795d68df69542ce9e94b39bc85c6ec
-  data.tar.gz: 07cec77fc7c40299207081f5ea7390cdd3cf863ac49295e9de166e4736ff95ae5d57e642de2a10ea74d1d812aeeff2ed4ffdb6a93c232658efc7552332a9f1e9
+  metadata.gz: d0e46a405da1e48dc0b82afe9c350b26f0b436fe99aa1cdf9500d035ec5a01612257c679338063920d370ead1e09c68ec35f7de9d732338417874a396ed8634c
+  data.tar.gz: d39aaad0be382a3f70359c03eeaa1b13ef9a594967cd029014da40ca2882cffe53a81a534cdf4c8221399fb10636e5c1304ac591e6a0ef589ecd1d0fab1b07f2

data/Gemfile

@@ -7,4 +7,4 @@ gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1
 gem 'restforce', '~> 2.1'
 gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
 gem 'soapforce', '~> 0.5'
-gem 'aws-sdk', '~> 2.3'
+gem 'aws-sdk', '~> 2.10'

data/Gemfile.lock

@@ -18,7 +18,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.3.2)
+    remi (0.3.3)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
@@ -43,12 +43,14 @@ GEM
     akami (1.3.1)
       gyoku (>= 0.4.0)
       nokogiri
-    aws-sdk (2.3.5)
-      aws-sdk-resources (= 2.3.5)
-    aws-sdk-core (2.3.5)
+    aws-sdk (2.10.3)
+      aws-sdk-resources (= 2.10.3)
+    aws-sdk-core (2.10.3)
+      aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.3.5)
-      aws-sdk-core (= 2.3.5)
+    aws-sdk-resources (2.10.3)
+      aws-sdk-core (= 2.10.3)
+    aws-sigv4 (1.0.0)
     backports (3.6.8)
     bond (0.5.1)
     builder (3.2.2)
@@ -104,10 +106,8 @@ GEM
       mimemagic (~> 0.3)
       multi_json (~> 1.11)
       rbczmq (~> 1.7)
-    jmespath (1.2.4)
-      json_pure (>= 1.8.1)
+    jmespath (1.3.1)
     json (1.8.3)
-    json_pure (1.8.3)
     jwt (1.5.6)
     little-plugger (1.1.4)
     logging (2.1.0)
@@ -187,7 +187,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  aws-sdk (~> 2.3)
+  aws-sdk (~> 2.10)
   daru (= 0.1.4.1)!
   github-markup (~> 1.4)
   google-api-client (~> 0.9)
@@ -200,4 +200,4 @@ DEPENDENCIES
   yard (~> 0.9)
 
 BUNDLED WITH
-   1.14.3
+   1.15.1

data/features/s3_file_target_job.feature

@@ -0,0 +1,10 @@
+Feature: Tests targets that are S3 Files.
+
+  Background:
+    Given the job is 'S3 File Target'
+    And the job target 'Some File'
+
+  Scenario: Defining the remote path.
+    Given the target 'Some File'
+    Then the file is uploaded to the S3 bucket "the-big-one"
+    And the file is uploaded to the remote path "some_file_*Today: %Y%m%d*.csv"

data/features/step_definitions/remi_step.rb

@@ -69,6 +69,14 @@ Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
   expect(@brt.target.data_subject.loaders.map(&:remote_path)).to include expected_path
 end
 
+Then /^the file is uploaded to the S3 bucket "([^"]+)"$/ do |bucket_name|
+  expected_bucket_name = Remi::Testing::BusinessRules::ParseFormula.parse(bucket_name)
+  bucket_names = @brt.target.data_subject.loaders.map do |loader|
+    loader.bucket_name if loader.respond_to? :bucket_name
+  end
+  expect(bucket_names).to include expected_bucket_name
+end
+
 ## CSV Options
 
 Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
@@ -124,6 +132,16 @@ Given /^the (source|target) file contains all of the following headers in this o
   expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to eq @brt.send(st.to_sym).fields.field_names
 end
 
+Given /^the (source|target) file contains all of the following headers in no particular order:$/ do |st, table|
+  table.rows.each do |row|
+    field = row.first
+    step "the #{st} field '#{field}'"
+  end
+
+  @brt.run_transforms if st == 'target'
+  expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to match_array @brt.send(st.to_sym).fields.field_names
+end
+
 ### Source
 
 Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
@@ -260,6 +278,7 @@ Then /^the target field '([^']+)' has the label '([^']+)'$/ do |target_field, la
   data_field  = @brt.targets.fields.next
   expect(data_field.metadata[:label]).to eq label
   expect(data_field.name).to eq target_field
+
 end
 
 Then /^the target field '([^']+)' is copied from the source field$/ do |target_field|
@@ -780,3 +799,10 @@ Then /^the target '([[:alnum:]\s\-_]+)' has (\d+) record(?:s|) where '([[:alnum:
   @brt.run_transforms
   expect(@brt.targets[target_name].where_between(field_name, low_value, high_value).size).to eq nrecords.to_i
 end
+
+Then /^the target field '([^']+)' (?:has|is set to) the multiline value$/ do |target_field, value|
+  step "the target field '#{target_field}'"
+  @brt.run_transforms
+  target_name, target_field_name = @brt.targets.parse_full_field(target_field)
+  expect(@brt.targets[target_name].fields[target_field_name].value).to eq Remi::Testing::BusinessRules::ParseFormula.parse(value)
+end

data/jobs/s3_file_target_job.rb

@@ -0,0 +1,23 @@
+require_relative 'all_jobs_shared'
+require 'aws-sdk'
+
+class S3FileTargetJob < Remi::Job
+  target :some_file do
+    encoder Remi::Encoder::CsvFile.new
+    loader Remi::Loader::S3File.new(
+      credentials: {
+        aws_access_key_id: 'blort',
+        aws_secret_access_key: 'blerg',
+        region: 'us-west-2'
+      },
+      kms_opt: {
+        ciphertext: 'blergity'
+      },
+      bucket: 'the-big-one',
+      remote_path: "some_file_#{DateTime.current.strftime('%Y%m%d')}.csv"
+    )
+  end
+
+  transform :main do
+  end
+end

data/lib/remi/data_subject.rb

@@ -93,7 +93,11 @@ module Remi
       sttm = SourceToTargetMap.new(df, source_metadata: fields)
       fields.keys.each do |field|
         next unless (types.size == 0 || types.include?(fields[field][:type])) && df.vectors.include?(field)
-        sttm.source(field).target(field).transform(Remi::Transform::EnforceType.new).execute
+        begin
+          sttm.source(field).target(field).transform(Remi::Transform::EnforceType.new).execute
+        rescue StandardError => err
+          raise ArgumentError, "Field '#{field}': #{err.message}"
+        end
       end
 
       self

data/lib/remi/data_subjects/none.rb

@@ -8,10 +8,10 @@ module Remi
   end
 
 
-  # The None Parser just returns what it is given.
+  # The None Parser just returns an empty dataframe if it's not given any data
   class Parser::None < Parser
-    def parse(data)
-      data
+    def parse(data=nil)
+      data || Remi::DataFrame::Daru.new([], order: fields.keys)
     end
   end
 

data/lib/remi/data_subjects/s3_file.rb

@@ -1,13 +1,59 @@
 module Remi
 
+  module DataSubject::S3File
+    attr_accessor :region
+    attr_accessor :aws_credentials
+
+    def init_aws_credentials(credentials)
+      @aws_credentials = Aws::Credentials.new(
+        credentials.fetch(:aws_access_key_id, ENV['AWS_ACCESS_KEY_ID']),
+        credentials.fetch(:aws_secret_access_key, ENV['AWS_SECRET_ACCESS_KEY'])
+      )
+    end
+
+    def s3
+      @s3 ||= Aws::S3::Resource.new(
+        credentials: aws_credentials,
+        region: region
+      )
+    end
+
+    def encrypt_args
+      @kms_args || {}
+    end
+
+    def init_kms(opt)
+      return nil unless opt
+
+      kms = Aws::KMS::Client.new(
+        region: @region,
+        credentials: @aws_credentials
+      )
+
+      ciphertext = opt.fetch(:ciphertext)
+      algorithm = opt.fetch(:algorithm, 'AES256')
+      key = kms.decrypt(ciphertext_blob: Base64.decode64(ciphertext)).plaintext
+
+      @kms_args = {
+        sse_customer_algorithm: algorithm,
+        sse_customer_key: key
+      }
+    end
+  end
+
   # S3 File extractor
   # Used to extract files from Amazon S3
   #
-  # @example
+  # @example Standard use
   #
   #  class MyJob < Remi::Job
   #    source :some_file do
   #      extractor Remi::Extractor::S3File.new(
+  #        credentials: {
+  #          aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  #          aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+  #          region: 'us-west-2'
+  #        },
   #        bucket: 'my-awesome-bucket',
   #        remote_path: 'some_file-',
   #        most_recent_only: true
@@ -28,9 +74,40 @@ module Remi
   #  #          0          1     Albert
   #  #          1          2      Betsy
   #  #          2          3       Camu
+  #
+  # @example Using AWS KMS
+  # To use AWS KMS, supply a :ciphertext and optional :algorithm (default is AES256).
+  # The encrypted key stored in the ciphertext must be the same as that used when the file was written.
+  #
+  #  class MyJob < Remi::Job
+  #    source :some_file do
+  #      extractor Remi::Extractor::S3File.new(
+  #        credentials: {
+  #          aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  #          aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+  #          region: 'us-west-2'
+  #        },
+  #        bucket: 'my-awesome-bucket',
+  #        remote_path: 'some_file-',
+  #        most_recent_only: true,
+  #        kms_opt: {
+  #          ciphertext: '<base64-encoded ciphertext>'
+  #        }
+  #      )
+  #      parser Remi::Parser::CsvFile.new(
+  #        csv_options: {
+  #          headers: true,
+  #          col_sep: '|'
+  #        }
+  #      )
+  #    end
+  #  end
   class Extractor::S3File < Extractor::FileSystem
+    include Remi::DataSubject::S3File
 
-    # @param bucket_name [String] S3 bucket containing the files
+    # @param bucket [String] Name of S3 bucket containing the files
+    # @param kms_opt [Hash] Hash containing AWS KMS options
+    # @param credentials [Hash] Hash containing AWS credentials (must contain :aws_access_key_id, :aws_secret_access_key, :region)
     def initialize(*args, **kargs, &block)
       super
       init_s3_file(*args, **kargs, &block)
@@ -39,10 +116,12 @@ module Remi
     # Called to extract files from the source filesystem.
     # @return [Array<String>] An array of paths to a local copy of the files extacted
     def extract
+      init_kms(@kms_opt)
+
       entries.map do |entry|
         local_file = File.join(@local_path, entry.name)
         logger.info "Downloading #{entry.pathname} from S3 to #{local_file}"
-        File.open(local_file, 'wb') { |file| entry.raw.get(response_target: file) }
+        File.open(local_file, 'wb') { |file| entry.raw.get({ response_target: file }.merge(encrypt_args)) }
         local_file
       end
     end
@@ -55,7 +134,7 @@ module Remi
     # @return [Array<Extractor::FileSystemEntry>] List of objects in the bucket/prefix
     def all_entries!
       # S3 does not track anything like a create time, so use last modified for both
-      bucket.objects(prefix: @remote_path.to_s).map do |entry|
+      s3.bucket(@bucket_name).objects(prefix: @remote_path.to_s).map do |entry|
         Extractor::FileSystemEntry.new(
           pathname: entry.key,
           create_time: entry.last_modified,
@@ -65,20 +144,128 @@ module Remi
       end
     end
 
-    # @return [Aws::S3::Client] The S3 client used
-    def s3_client
-      @s3_client ||= Aws::S3::Client.new
-    end
-
     private
 
-    def init_s3_file(*args, bucket:, **kargs)
+    def init_s3_file(*args, credentials: {}, bucket:, kms_opt: nil, **kargs)
+      @region = credentials.fetch(:region, 'us-west-2')
+      @kms_opt = kms_opt
+      init_aws_credentials(credentials)
+
       @bucket_name = bucket
     end
+  end
+
+
+
+  # S3 File loader
+  # Used to post files to Amazon S3
+  #
+  # @example Standard use
+  #
+  #  class MyJob < Remi::Job
+  #    target :some_file do
+  #      encoder Remi::Encoder::CsvFile.new
+  #      loader Remi::Loader::S3File.new(
+  #        credentials: {
+  #          aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  #          aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+  #          region: 'us-west-2'
+  #        },
+  #        bucket: 'itk-de-archive',
+  #        remote_path: 'awesome.csv'
+  #      )
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.some_file.df = Daru::DataFrame.new(
+  #    {
+  #      numbers: [1,2,3],
+  #      words: ['one', 'two', 'three']
+  #    }
+  #  )
+  #  job.some_file.load
+  #
+  # @example Using AWS KMS
+  # To use AWS KMS, supply a :ciphertext and optional :algorithm (default is AES256).
+  # The encrypted key stored in the ciphertext must be the same as that used for reading the file.
+  #
+  #  class MyJob < Remi::Job
+  #    target :some_file do
+  #      encoder Remi::Encoder::CsvFile.new
+  #      loader Remi::Loader::S3File.new(
+  #        credentials: {
+  #          aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+  #          aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+  #          region: 'us-west-2'
+  #        },
+  #        bucket: 'itk-de-archive',
+  #        remote_path: 'awesome.csv',
+  #        kms_opt: {
+  #          ciphertext: '<base64-encoded ciphertext>'
+  #        }
+  #      )
+  #    end
+  #  end
+  #
+  # @example Generating a ciphertext
+  # A ciphertext can be generated using the AWS SDK
+  #
+  #  require 'aws-sdk'
+  #  require 'base64'
+  #
+  #  aws_credentials = Aws::Credentials.new(
+  #    ENV['AWS_ACCESS_KEY_ID'],
+  #    ENV['AWS_SECRET_ACCESS_KEY']
+  #  )
+  #
+  #  kms = Aws::KMS::Client.new(
+  #    region: 'us-west-2',
+  #    credentials: aws_credentials
+  #  )
+  #
+  #  # See AWS docs for creating keys: http://docs.aws.amazon.com/kms/latest/developerguide/create-keys.html
+  #  data_key = kms.generate_data_key(
+  #    key_id: 'alias/alias-of-kms-key',
+  #    key_spec: 'AES_256'
+  #  )
+  #
+  #  ciphertext = Base64.strict_encode64(data_key.ciphertext_blob)
+  #  #=> "AQIDAHjmmRVcBAdMHsA9VUoJKgbW8niK2qL1qPcQ2OWEUlh5XAFw0vfl+QIgawB8cbAZ2OqXAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMIUIFFh++2w4d9al7AgEQgDvSRXQCOPLSMOjRS/lM5uxuyRV47qInlKKBIezIaYzXuFu1sRU+L46HqRyS0XqR4flFJ/fc8yEj3pU1UA=="
+  class Loader::S3File < Loader
+    include Remi::DataSubject::S3File
+
+    # @param bucket [String] Name of S3 bucket containing the files
+    # @param kms_opt [Hash] Hash containing AWS KMS options
+    # @param credentials [Hash] Hash containing AWS credentials (must contain :aws_access_key_id, :aws_secret_access_key, :region)
+    def initialize(*args, **kargs, &block)
+      super
+      init_s3_loader(*args, **kargs, &block)
+    end
+
+    attr_reader :remote_path
+    attr_reader :bucket_name
 
-    def bucket
-      @bucket ||= Aws::S3::Bucket.new(@bucket_name, client: s3_client)
+    # Copies data to S3
+    # @param data [Object] The path to the file in the temporary work location
+    # @return [true] On success
+    def load(data)
+      init_kms(@kms_opt)
+
+      @logger.info "Writing file #{data} to S3 #{@bucket_name} as #{@remote_path}"
+      s3.bucket(@bucket_name).object(@remote_path).upload_file(data, encrypt_args)
+      true
     end
 
+    private
+
+    def init_s3_loader(*args, credentials:{}, bucket:, remote_path:, kms_opt: nil, **kargs, &block)
+      @region = credentials.fetch(:region, 'us-west-2')
+      @kms_opt = kms_opt
+      init_aws_credentials(credentials)
+
+      @bucket_name = bucket
+      @remote_path = remote_path
+    end
   end
 end

data/lib/remi/data_subjects/salesforce.rb

@@ -185,11 +185,11 @@ module Remi
       if @operation == :update
         Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
       elsif @operation == :create
-        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
+        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, data, batch_size: @batch_size, max_attempts: 1, logger: logger)
       elsif @operation == :upsert
         Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, data, batch_size: @batch_size, external_id: @external_id, logger: logger)
       elsif @operation == :delete
-        Remi::SfBulkHelper::SfBulkDelete.upsert(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
+        Remi::SfBulkHelper::SfBulkDelete.delete(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
       else
         raise ArgumentError, "Unknown operation: #{@operation}"
       end

data/lib/remi/data_subjects/salesforce_soap.rb

@@ -79,7 +79,9 @@ module Remi
           end
 
           merge_id = Array(row.delete(@merge_id_field))
-          soapforce_client.merge(@sfo, row, merge_id)
+          merge_row = row.select { |_, v| !v.blank? }
+          logger.info "Merging Id #{merge_id} into #{merge_row}"
+          soapforce_client.merge!(@sfo, merge_row, merge_id)
         end
       else
         raise ArgumentError, "Unknown soap operation: #{@operation}"

data/lib/remi/data_subjects/sftp_file.rb

@@ -1,4 +1,44 @@
 module Remi
+  module DataSubject::SftpFile
+
+    attr_reader :sftp_session
+
+    def sftp_retry(&block)
+      tries ||= @retries
+
+      block.call
+    rescue StandardError => err
+      if (tries -= 1) > 0
+        logger.error "Error: #{err.message}"
+        logger.error "Will retry #{tries} more times"
+        sleep(1)
+        retry
+      else
+        raise err
+      end
+    end
+
+    def begin_connection
+      sftp_retry do
+        Timeout.timeout(@timeout) do
+          @ssh_session = Net::SSH.start(@host, @username, password: @password, port: @port, number_of_password_prompts: 0)
+          @sftp_session = Net::SFTP::Session.new(@ssh_session)
+          @sftp_session.connect!
+        end
+      end
+    end
+
+    def end_connection
+      @sftp_session.close_channel unless @sftp_session.nil?
+      @ssh_session.close unless @ssh_session.nil?
+
+      Timeout.timeout(@timeout) do
+        sleep 1 until (@sftp_session.nil? || @sftp_session.closed?) && (@ssh_session.nil? || @ssh_session.closed?)
+      end
+    end
+  end
+
+
 
   # Sftp File extractor
   # Used to extract files from an SFTP server
@@ -35,13 +75,15 @@ module Remi
   #  #          1          2      Betsy
   #  #          2          3       Camu
   class Extractor::SftpFile < Extractor::FileSystem
-    N_RETRY = 3
+    include DataSubject::SftpFile
 
     # @param credentials [Hash] Options hash containing login credentials
     # @param credentials [String] :host SFTP host (e.g., coolserver.com)
     # @param credentials [String] :username SFTP username
     # @param credentials [String] :password SFTP password
     # @param credentials [String] :port SFTP port (default: 22)
+    # @param retries [Integer] Number of times a connection or operation will be retried (default: 3)
+    # @param timeout [Integer] Number of seconds to wait for establishing/closing a connection (default: 30)
     def initialize(*args, **kargs, &block)
       super
       init_sftp_extractor(*args, **kargs)
@@ -55,15 +97,16 @@ module Remi
     # Called to extract files from the source filesystem.
     # @return [Array<String>] An array of paths to a local copy of the files extacted
     def extract
-      connection do |sftp|
-        entries.map do |entry|
-          local_file = File.join(@local_path, entry.name)
-          logger.info "Downloading #{entry.name} to #{local_file}"
-          retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
-          local_file
+      begin_connection
 
-        end
+      entries.map do |entry|
+        local_file = File.join(@local_path, entry.name)
+        logger.info "Downloading #{entry.name} to #{local_file}"
+        sftp_retry { sftp_session.download!(File.join(@remote_path, entry.name), local_file) }
+        local_file
       end
+    ensure
+      end_connection
     end
 
     # @return [Array<Extractor::FileSystemEntry>] (Memoized) list of objects in the bucket/prefix
@@ -73,8 +116,7 @@ module Remi
 
     # @return [Array<Extractor::FileSystemEntry>] (Memoized) list of objects in the bucket/prefix
     def all_entries!
-      sftp_entries = connection { |sftp| sftp.dir.entries(@remote_path) }
-      sftp_entries.map do |entry|
+      sftp_session.dir.entries(@remote_path).map do |entry|
         # Early versions of the protocol don't support create time, fake it with modified time?
         FileSystemEntry.new(
           pathname: File.join(@remote_path, entry.name),
@@ -87,33 +129,13 @@ module Remi
 
     private
 
-    def init_sftp_extractor(*args, credentials:, **kargs)
+    def init_sftp_extractor(*args, credentials:, retries: 3, timeout: 30, **kargs)
       @host     = credentials.fetch(:host)
       @username = credentials.fetch(:username)
-      @password = credentials.fetch(:password)
+      @password = credentials.fetch(:password, nil)
       @port     = credentials.fetch(:port, '22')
-    end
-
-    def connection(&block)
-      result = nil
-      Net::SFTP.start(@host, @username, password: @password, port: @port) do |sftp|
-        result = yield sftp
-      end
-      result
-    end
-
-    def retry_download(&block)
-      1.upto(N_RETRY).each do |itry|
-        begin
-          block.call
-          break
-        rescue RuntimeError => err
-          raise err unless itry < N_RETRY
-          logger.error "Download failed with error: #{err.message}"
-          logger.error "Retry attempt #{itry}/#{N_RETRY-1}"
-          sleep(1)
-        end
-      end
+      @retries  = retries
+      @timeout  = timeout
     end
   end
 
@@ -143,8 +165,16 @@ module Remi
   #  job.my_target.df = my_df
   #  job.my_target.load
   class Loader::SftpFile < Loader
+    include DataSubject::SftpFile
 
+    # @param credentials [Hash] Options hash containing login credentials
+    # @param credentials [String] :host SFTP host (e.g., coolserver.com)
+    # @param credentials [String] :username SFTP username
+    # @param credentials [String] :password SFTP password
+    # @param credentials [String] :port SFTP port (default: 22)
     # @param remote_path [String, Pathname] Full path to the file to be created on the target filesystem
+    # @param retries [Integer] Number of times a connection or operation will be retried (default: 3)
+    # @param timeout [Integer] Number of seconds to wait for establishing/closing a connection (default: 30)
     def initialize(*args, **kargs, &block)
       super
       init_sftp_loader(*args, **kargs, &block)
@@ -156,42 +186,27 @@ module Remi
     # @param data [Object] The path to the file in the temporary work location
     # @return [true] On success
     def load(data)
-      logger.info "Uploading #{data} to #{@credentials[:username]}@#{@credentials[:host]}: #{@remote_path}"
-      connection do |sftp|
-        retry_upload { sftp.upload! data, @remote_path }
-      end
+      begin_connection
+
+      logger.info "Uploading #{data} to #{@username}@#{@host}: #{@remote_path}"
+      sftp_retry { sftp_session.upload! data, @remote_path }
 
       true
+    ensure
+      end_connection
     end
 
 
     private
 
-    def init_sftp_loader(*args, credentials:, remote_path:, **kargs, &block)
-      @credentials = credentials
+    def init_sftp_loader(*args, credentials:, remote_path:, retries: 3, timeout: 30, **kargs, &block)
+      @host        = credentials.fetch(:host)
+      @username    = credentials.fetch(:username)
+      @password    = credentials.fetch(:password, nil)
+      @port        = credentials.fetch(:port, '22')
       @remote_path = remote_path
-    end
-
-    def connection(&block)
-      result = nil
-      Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @credentials[:port] || '22') do |sftp|
-        result = yield sftp
-      end
-      result
-    end
-
-    def retry_upload(ntry=2, &block)
-      1.upto(ntry).each do |itry|
-        begin
-          block.call
-          break
-        rescue RuntimeError => err
-          raise err unless itry < ntry
-          logger.error "Upload failed with error: #{err.message}"
-          logger.error "Retry attempt #{itry}/#{ntry-1}"
-          sleep(1)
-        end
-      end
+      @retries     = retries
+      @timeout     = timeout
     end
   end
 end

data/lib/remi/job/transform.rb

@@ -15,6 +15,7 @@ module Remi
     #   end
     #   tform.execute
     class Transform
+      class IncompatibleTargetIndexError < StandardError; end
 
       FieldMap = Struct.new(:from_subject, :to_subject, :field_from_to)
 
@@ -152,6 +153,19 @@ module Remi
           sub_trans_ds = field_map.from_subject
           fields_to_map = field_map.field_from_to.keys
 
+          job_idx = job_ds.df.index.to_a
+          sub_idx = sub_trans_ds.df.index.to_a
+          diff = ((job_idx | sub_idx) - (job_idx & sub_idx))
+          if job_idx.size > 0 && diff.size > 0 then
+            msg = <<-EOT
+              Incompatible target index!
+              Sub transform target #{sub_trans_ds.name} index is #{sub_trans_ds.df.index.inspect}
+              Job transform target #{job_ds.name} index is #{job_ds.df.index.inspect}
+            EOT
+            raise IncompatibleTargetIndexError.new msg
+          end
+
+
           fields_to_map.each do |sub_trans_field|
             job_field = field_map.field_from_to[sub_trans_field]
             job_ds.fields[job_field].merge! sub_trans_ds.fields[sub_trans_field]

data/lib/remi/source_to_target_map/map.rb

@@ -128,10 +128,13 @@ module Remi
 
       # Private: Converts the transformed data into vectors in the target dataframe.
       def map_to_target_df
+        index = @target_df.index.size > 0 ? @target_df.index : @source_df.index
+
         result_hash_of_arrays.each do |vector, values|
-          @target_df[vector] = Daru::Vector.new(values, index: @source_df.index)
+          @target_df[vector] = Daru::Vector.new(values, index: index)
         end
 
+        @target_df.index = index
         @target_df
       end
 

data/lib/remi/transform.rb

@@ -550,6 +550,9 @@ module Remi
             raise ArgumentError, "Unknown type enforcement: #{type}"
           end
         end
+
+      rescue StandardError => err
+        raise ArgumentError, "Unable to convert value '#{value}' to type '#{type}': #{err.message}"
       end
     end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.3.2'
+  VERSION = '0.3.3'
 end

data/spec/data_subjects/none_spec.rb

@@ -14,7 +14,11 @@ describe Parser::None do
   let(:parser) { Parser::None.new }
 
   context '#parse' do
-    it 'returns what it is given' do
+    it 'returns an empty dataframe when given no data' do
+      expect(parser.parse.to_a).to eq Remi::DataFrame::Daru.new([]).to_a
+    end
+
+    it 'returns an what it was given' do
       expect(parser.parse('some data')).to eq 'some data'
     end
   end

data/spec/data_subjects/s3_file_spec.rb

@@ -9,14 +9,23 @@ describe Extractor::S3File do
     }
 
     prefix = "the-best-prefix"
-    @s3_file = Extractor::S3File.new(bucket: 'the-best-bucket', remote_path: "#{prefix}")
-    @s3_file.s3_client.stub_responses(:list_objects, {
+    credentials = {
+      aws_access_key_id: 'BLAH',
+      aws_secret_access_key: 'DEBLAH'
+    }
+
+    @s3_file = Extractor::S3File.new(
+      bucket: 'the-best-bucket',
+      credentials: credentials,
+      remote_path: "#{prefix}"
+    )
+
+    @s3_file.s3.client.stub_responses(:list_objects, {
       contents: [
         { key: "#{prefix}/file1.csv" },
         { key: "#{prefix}/file2.csv" }
       ]
     })
-
   end
 
   it 'returns all entries' do

data/spec/data_subjects/salesforce_soap_spec.rb

@@ -45,7 +45,7 @@ describe Loader::SalesforceSoap do
       { Id: '1234', Custom__c: 'something', Merge_Id: '5678' }
     ]
 
-    expect(soapforce_client).to receive(:merge) do
+    expect(soapforce_client).to receive(:merge!) do
       [
         :Contact,
         {
@@ -65,7 +65,25 @@ describe Loader::SalesforceSoap do
       { Id: '2', Custom__c: 'something', Merge_Id: '20' }
     ]
 
-    expect(soapforce_client).to receive(:merge).twice
+    expect(soapforce_client).to receive(:merge!).twice
+    loader.load(data)
+  end
+
+  it 'excludes blank data fields from the merge command' do
+    data = [
+      { Id: '1234', Custom__c: '', Merge_Id: '5678' }
+    ]
+
+    expect(soapforce_client).to receive(:merge!) do
+      [
+        :Contact,
+        {
+          Id: '1234'
+        },
+        ['5678']
+      ]
+    end
+
     loader.load(data)
   end
 
@@ -76,5 +94,4 @@ describe Loader::SalesforceSoap do
 
     expect { loader.load(data) }.to raise_error KeyError
   end
-
 end

data/spec/data_subjects/sftp_file_spec.rb

@@ -10,21 +10,25 @@ describe Extractor::SftpFile do
     }
   }
 
-  let(:sftp_file) {
+  def generate_extractor
     Extractor::SftpFile.new(
       credentials: credentials,
       remote_path: remote_path
     )
-  }
+  end
+
+  let(:extractor) { generate_extractor }
 
   let(:remote_filenames) { ['file1.csv', 'file2.csv'] }
-  let(:sftp_session) { instance_double('Net:SFTP::Session') }
 
   before do
-    sftp_dir = instance_double('Net::SFTP::Operations::Dir')
+    allow(extractor).to receive(:begin_connection)
 
-    allow(Net::SFTP).to receive(:start).and_yield sftp_session
-    allow(sftp_session).to receive(:dir).and_return sftp_dir
+    sftp_session = double('sftp_session')
+    allow(extractor).to receive(:sftp_session).and_return(sftp_session)
+
+    sftp_dir = instance_double('Net::SFTP::Operations::Dir')
+    allow(sftp_session).to receive(:dir).and_return(sftp_dir)
 
     allow(sftp_dir).to receive(:entries).and_return(remote_filenames.map { |fname|
       Net::SFTP::Protocol::V04::Name.new(
@@ -36,65 +40,76 @@ describe Extractor::SftpFile do
 
   context '.new' do
     it 'creates an instance with valid parameters' do
-      sftp_file
+      extractor
     end
 
     it 'requires a hostname' do
       credentials.delete(:host)
-      expect { sftp_file }.to raise_error KeyError
+      expect { generate_extractor }.to raise_error KeyError
     end
 
     it 'requires a username' do
       credentials.delete(:username)
-      expect { sftp_file }.to raise_error KeyError
+      expect { generate_extractor }.to raise_error KeyError
     end
 
-    it 'requires a password' do
+    it 'does not require a password' do # If empty, it will use private keys
       credentials.delete(:password)
-      expect { sftp_file }.to raise_error KeyError
+      expect { generate_extractor }.not_to raise_error
     end
 
     it 'defaults to using port 22' do
-      expect(sftp_file.port).to eq '22'
+      expect(extractor.port).to eq '22'
     end
 
     it 'allows the port to be defined in the credentials' do
       credentials[:port] = '1234'
-      expect(sftp_file.port).to eq '1234'
+      expect(generate_extractor.port).to eq '1234'
     end
   end
 
   context '#all_entires' do
     it 'returns all entries' do
-      expect(sftp_file.all_entries.map(&:name)).to eq remote_filenames
+      expect(extractor.all_entries.map(&:name)).to eq remote_filenames
     end
   end
 
   context '#extract' do
     it 'downloads files from the ftp' do
-      expect(sftp_session).to receive(:download!).exactly(remote_filenames.size).times
-      sftp_file.extract
+      expect(extractor.sftp_session).to receive(:download!).exactly(remote_filenames.size).times
+      extractor.extract
     end
 
     it 'creates local files with the right names' do
-      allow(sftp_session).to receive(:download!)
-      expect(sftp_file.extract.map { |f| Pathname.new(f).basename.to_s }).to eq remote_filenames
+      allow(extractor.sftp_session).to receive(:download!)
+      expect(extractor.extract.map { |f| Pathname.new(f).basename.to_s }).to eq remote_filenames
     end
   end
 end
 
 
 describe Loader::SftpFile do
-  let(:loader) { Loader::SftpFile.new(credentials: {}, remote_path: 'some_path') }
+
+  let(:credentials) {
+    {
+      host: 'host',
+      username: 'username',
+      password: 'password'
+    }
+  }
+
+  let(:loader) { Loader::SftpFile.new(credentials: credentials, remote_path: 'some_path') }
   let(:data) { double('some_data') }
-  let(:sftp_session) { instance_double('Net:SFTP::Session') }
 
   before do
-    allow(Net::SFTP).to receive(:start).and_yield sftp_session
+    allow(loader).to receive(:begin_connection)
+
+    sftp_session = double('sftp_session')
+    allow(loader).to receive(:sftp_session).and_return(sftp_session)
   end
 
   it 'loads a csv to a target sftp filesystem' do
-    expect(sftp_session).to receive(:upload!).with(data, 'some_path')
+    expect(loader.sftp_session).to receive(:upload!).with(data, 'some_path')
     loader.load data
   end
 end

data/spec/job/transform_spec.rb

@@ -253,5 +253,89 @@ describe Job do
         my_transform.execute
       end
     end
+
+    describe '#import - edge cases' do
+      before do
+        class MyJob
+          source :job_source do
+            fields({ :id => {}, :name => {} })
+          end
+          target :job_target do
+            fields({ :id => {}, :name => {}, :funny_name => {} })
+          end
+        end
+
+        job.job_source.df = Remi::DataFrame::Daru.new({
+          id:   [1, 2, 3],
+          name: ['one', 'two', 'three']
+        })
+      end
+
+      it 'correctly maps back to a source if the sub transform sorts the data' do
+        sub_transform = Job::Transform.new('arbitrary') do
+          source :st_source, [:id, :name]
+          target :st_target, [:funny_name]
+
+          st_source.df.sort!([:id], ascending: [false])
+
+          Remi::SourceToTargetMap.apply(st_source.df, st_target.df) do
+            map source(:name) .target(:funny_name)
+              .transform(->(v) { "funny-#{v}" })
+          end
+        end
+
+        my_transform = Job::Transform.new(job) do
+          import sub_transform do
+            map_source_fields :job_source, :st_source, {
+              :id   => :id,
+              :name => :name
+            }
+            map_target_fields :st_target, :job_source, {
+              :funny_name => :funny_name
+            }
+          end
+
+          job.job_target.df = job.job_source.df.dup
+        end
+
+        my_transform.execute
+        expect(job.job_target.df[:funny_name].to_a).to eq(
+          job.job_target.df[:name].to_a.map { |v| "funny-#{v}" }
+        )
+      end
+
+      it 'raises an error if the subtransform fucks with index', wip: true do
+        sub_transform = Job::Transform.new('arbitrary') do
+          source :st_source, [:id, :name]
+          target :st_target, [:funny_name]
+
+          duplicated_df = Daru::DataFrame.new({ id: Array(st_source.df[:id][0]) * 3 })
+
+          st_source.df = st_source.df.join(duplicated_df, on: [:id], how: :left)
+
+          Remi::SourceToTargetMap.apply(st_source.df, st_target.df) do
+            map source(:name) .target(:funny_name)
+              .transform(->(v) { "funny-#{v}" })
+          end
+        end
+
+        my_transform = Job::Transform.new(job) do
+          import sub_transform do
+            map_source_fields :job_source, :st_source, {
+              :id   => :id,
+              :name => :name
+            }
+            map_target_fields :st_target, :job_source, {
+              :funny_name => :funny_name
+            }
+          end
+
+          job.job_target.df = job.job_source.df.dup
+        end
+
+        expect { my_transform.execute }.to raise_error Job::Transform::IncompatibleTargetIndexError
+      end
+    end
+
   end
 end

data/spec/source_to_target_map_spec.rb

@@ -298,4 +298,34 @@ describe SourceToTargetMap do
       expect(sttm).to be_a(Remi::DataFrame::Daru)
     end
   end
+
+  describe 'source and target dataframes differ', wip: true do
+    it 'does not fail when the dataframe has been filtered' do
+      some_df = Daru::DataFrame.new(
+        {
+          :id => [1,2,3,4,5],
+          :something => ['x','','x','','x'],
+          :name => ['one', 'two', 'three', 'four', 'five']
+        }
+      )
+
+      filtered_df = some_df.where(some_df[:something].eq('x'))
+      target_df = Remi::DataFrame::Daru.new([])
+
+      Remi::SourceToTargetMap.apply(filtered_df, target_df) do
+        map source(:id) .target(:id)
+        map source(:name) .target(:name)
+      end
+
+      result = target_df[:id, :name].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+      expect(result).to eq({
+        :id => [1, 3, 5],
+        :name => ['one', 'three', 'five']
+      })
+    end
+
+
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-25 00:00:00.000000000 Z
+date: 2017-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -231,6 +231,7 @@ files:
 - features/json.feature
 - features/metadata.feature
 - features/parameters.feature
+- features/s3_file_target_job.feature
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb
@@ -254,6 +255,7 @@ files:
 - jobs/json_job.rb
 - jobs/metadata_job.rb
 - jobs/parameters_job.rb
+- jobs/s3_file_target_job.rb
 - jobs/sample_job.rb
 - jobs/sftp_file_target_job.rb
 - jobs/sub_job_example_job.rb
@@ -372,6 +374,7 @@ test_files:
 - features/json.feature
 - features/metadata.feature
 - features/parameters.feature
+- features/s3_file_target_job.feature
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb