remi 0.0.1 → 0.2.2

data/lib/remi/project/features/support/env.rb

@@ -0,0 +1,10 @@
+# Auto-generated by Remi.
+# Add user-customizations to env_app.rb
+
+require 'bundler/setup'
+require 'remi'
+require 'remi/cucumber'
+
+require_relative 'env_app.rb'
+
+Remi::Settings.log_level = Logger::ERROR

data/lib/remi/project/features/support/env_app.rb

@@ -0,0 +1,3 @@
+require_relative '../../jobs/sample_job'
+require_relative '../../jobs/copy_source_job'
+require_relative '../../jobs/transforms/transform_jobs'

data/lib/remi/project/features/transforms/date_diff.feature

@@ -0,0 +1,50 @@
+Feature: Tests the date_diff transform
+
+  Background:
+    Given the job is 'DateDiff'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the target 'Target Data'
+    And the following example record for 'Source Data':
+      | Date1      | Date2      |
+      | 2015-12-31 | 2016-01-02 |
+
+  Scenario Outline: Calculating date difference in days2.
+    Given the job parameter 'measure' is "days"
+    And the source field 'Date1' has the value "<Date1>"
+    And the source field 'Date2' has the value "<Date2>"
+    Then the target field 'Difference' is set to the value "<Difference>"
+    Examples:
+      | Date1      | Date2      | Difference |
+      | 2015-12-31 | 2016-01-02 | 2          |
+      | 2014-12-31 | 2015-12-31 | 365        |
+      | 2016-01-02 | 2015-12-31 | -2         |
+      | 2015-02-28 | 2015-03-01 | 1          |
+      | 2016-02-28 | 2016-03-01 | 2          | # leap day
+
+
+  Scenario Outline: Calculating date difference in months.
+    Given the job parameter 'measure' is "months"
+    And the source field 'Date1' has the value "<Date1>"
+    And the source field 'Date2' has the value "<Date2>"
+    Then the target field 'Difference' is set to the value "<Difference>"
+    Examples:
+      | Date1      | Date2      | Difference |
+      | 2015-12-31 | 2016-01-02 | 1          |
+      | 2015-12-31 | 2016-02-02 | 2          |
+      | 2015-12-31 | 2017-02-02 | 14         |
+      | 2016-02-02 | 2015-12-31 | -2         |
+
+  Scenario Outline: Calculating date difference in years.
+    Given the job parameter 'measure' is "years"
+    And the source field 'Date1' has the value "<Date1>"
+    And the source field 'Date2' has the value "<Date2>"
+    Then the target field 'Difference' is set to the value "<Difference>"
+    Examples:
+      | Date1      | Date2      | Difference |
+      | 2015-12-31 | 2016-01-02 | 1          |
+      | 2015-01-01 | 2015-12-31 | 0          |
+      | 2015-12-31 | 2017-02-02 | 2          |
+      | 2016-02-02 | 2015-12-31 | -1         |

data/lib/remi/project/features/transforms/parse_date.feature

@@ -0,0 +1,34 @@
+Feature: Tests the parse_date transform
+
+  Background:
+    Given the job is 'ParseDate'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the target 'Target Data'
+    And the following example record for 'Source Data':
+      | Date String |
+      | 2015-12-31  |
+
+  Scenario Outline: Parsing date strings.
+    Given the source field 'Date String' has the value "<Date String>"
+    And the job parameter 'format' is "<Format>"
+    Then the target field 'Parsed Date' is set to the value "<Parsed Date>"
+    Examples:
+      | Date String | Format   | Parsed Date |
+      | 2015-10-21  | %Y-%m-%d | 2015-10-21  |
+      | 10/21/2015  | %m/%d/%Y | 2015-10-21  |
+      | 20151021    | %Y%m%d   | 2015-10-21  |
+      |             | %m/%d/%Y |             |
+
+  Scenario Outline: Parsing date strings for missing values.
+    Given the source field 'Date String' has the value ""
+    And the job parameter 'if_blank' is "<If Blank>"
+
+    Then the target field 'Parsed Date' is set to the value "<Parsed Date>"
+    Examples:
+      | If Blank   | Parsed Date |
+      | low        | 1900-01-01  |
+      | high       | 2999-12-31  |
+      |            |             |

data/lib/remi/project/features/transforms/prefix.feature

@@ -0,0 +1,15 @@
+Feature: Test the prefix transformer.
+
+  Background:
+    Given the job is 'Prefix'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+
+  Scenario: Prefixing a field.
+    Given the source 'Source Data'
+    And the target 'Target Data'
+    Given the following example record for 'Source Data':
+      | Field     |
+      | something |
+    Then the target field 'Field' is set to the value "prefixsomething"

data/lib/remi/project/jobs/all_jobs_shared.rb

@@ -0,0 +1,25 @@
+# This file was auto-generated by Remi but is expected to be overwritten.
+# Put anything in here that should be shared by all jobs.
+
+require 'bundler/setup'
+require 'remi'
+
+module AllJobsShared
+  include Remi::Job
+
+  define_param :sftp, {
+    host: 'example.com',
+    username: 'user',
+    password: '1234567890'
+  }
+
+  define_param :salesforce_credentials, {
+    host: 'login.salesforce.com',
+    username: 'user@example.com',
+    password: 'password',
+    security_token: '4342jn3j4n32n4',
+    client_id: 'dkfjsdkfjoasdjdf',
+    client_secret: '28357245723475',
+    instance_url: 'https://na1.salesforce.com'
+  }
+end

data/lib/remi/project/jobs/copy_source_job.rb

@@ -0,0 +1,12 @@
+require_relative 'all_jobs_shared'
+
+class CopySourceJob
+  include AllJobsShared
+
+  define_source :source_data, Remi::DataSource::DataFrame
+  define_source :target_data, Remi::DataSource::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    target_data.df = source_data.df.monkey_dup
+  end
+end

data/lib/remi/project/jobs/sample_job.rb

@@ -0,0 +1,164 @@
+# This is an example Remi job that was auto-generated by Remi.
+
+require_relative 'all_jobs_shared'
+
+class SampleJob
+  include AllJobsShared
+
+  define_source :existing_contacts, Remi::DataSource::Salesforce,
+    object: :Contact,
+    credentials: params[:salesforce_credentials],
+    api: :bulk,
+    fields: {
+      :Id             => {},
+      :External_ID__c => {}
+    },
+    query: <<-EOQ
+      SELECT
+        Id,
+        External_ID__c
+      FROM
+        Contact
+    EOQ
+
+
+  define_source :sample_file, Remi::DataSource::CsvFile,
+    extractor: Remi::Extractor::SftpFile.new(
+      credentials: params[:sftp],
+      remote_file: /^SampleFile_(\d+)\.txt/,
+      remote_folder: '/',
+      most_recent_only: true
+    ),
+    csv_options: {
+      headers: true,
+      col_sep: ",",
+      encoding: "ISO-8859-1:UTF-8"
+    },
+    fields: {
+      :student_id             => {},
+      :school_id              => {},
+      :school_name            => {},
+      :program                => {},
+      :last_name              => {},
+      :first_name             => {},
+      :current_email          => {},
+      :mailing_address_line_1 => {},
+      :mailing_address_line_2 => {},
+      :mailing_city           => {},
+      :mailing_state          => {},
+      :mailing_postal_code    => {},
+      :birthdate              => { type: :date, format: '%m/%d/%Y'},
+      :applied_date           => { type: :date, format: '%m/%d/%Y'}
+    }
+
+  define_target :all_contacts, Remi::DataTarget::DataFrame
+
+  define_target :contact_updates, Remi::DataTarget::Salesforce,
+    credentials: params[:salesforce_credentials],
+    object: :Contact,
+    operation: :update,
+    api: :bulk
+
+  define_target :contact_creates, Remi::DataTarget::Salesforce,
+    credentials: params[:salesforce_credentials],
+    object: :Contact,
+    operation: :create,
+    api: :bulk
+
+  define_lookup :program_name_lookup, Remi::Lookup::RegexSieve, {
+    /^BIO$/              => "Biology",
+    /^Fake Biology$/     => nil,
+    /(?:B|Microb)iology/ => "Biology",
+    /^CHEM$/             => "Chemistry",
+    /Chemistry/          => "Chemistry",
+    /Physics/            => "Physics"
+  }
+
+  define_transform :map_common_fields, sources: [:sample_file, :existing_contacts], targets: :all_contacts do
+
+    # Exclude all source records with an invalid program name
+    all_contacts.df = sample_file.df.monkey_dup
+    Remi::SourceToTargetMap.apply(all_contacts.df) do
+      map source(:program) .target(:Major__c)
+        .transform(Remi::Transform[:lookup][program_name_lookup])
+    end
+    all_contacts.df = all_contacts.df.where(all_contacts.df[:Major__c].not_eq(nil))
+
+    student_id_to_sf_id = existing_contacts.df.map_rows { |row| [row[:External_ID__c], row[:Id]] }.to_h
+
+    # Map fields that are common to both creates and updates
+    Remi::SourceToTargetMap.apply(all_contacts.df) do
+
+      # Prefixes source id record and then looks up existing salesforce Id
+      map source(:student_id) .target(:External_ID__c, :Id)
+        .transform(Remi::Transform[:prefix]['SAMP'])
+        .transform(->(v) { [v, Remi::Transform[:lookup][student_id_to_sf_id].call(v)] })
+    end
+  end
+
+
+  define_transform :map_creates, sources: :all_contacts, targets: :contact_creates do
+
+    work_contact_creates = all_contacts.df.where(all_contacts.df[:Id].eq(nil))
+    Remi::SourceToTargetMap.apply(work_contact_creates) do
+
+      map source(:school_id)           .target(:School_ID__c)
+      map source(:school_name)         .target(:School_Name__c)
+      map source(:first_name)          .target(:FirstName)
+        .transform(Remi::Transform[:ifblank].('Not Provided'))
+      map source(:last_name)           .target(:LastName)
+        .transform(Remi::Transform[:ifblank].('Not Provided'))
+      map source(:mailing_city)        .target(:MailingCity)
+      map source(:mailing_state)       .target(:MailingState)
+      map source(:mailing_postal_code) .target(:MailingPostalCode)
+
+      map source(:birthdate)                  .target(:Birthdate)
+        .transform(Remi::Transform[:format_date][from_fmt: sample_file.fields[:birthdate][:format]])
+
+      map source(:applied_date)              .target(:Applied_Date__c)
+        .transform(Remi::Transform[:ifblank].(Date.today.strftime(sample_file.fields[:applied_date][:format])))
+        .transform(Remi::Transform[:format_date].(from_fmt: sample_file.fields[:applied_date][:format]))
+
+      map source(:mailing_address_line_1, :mailing_address_line_2) .target(:MailingStreet)
+        .transform(->(line_1, line_2) {
+           Remi::Transform[:ifblank].(nil).call(line_1).nil? ? [] : [line_1, line_2]
+           })
+        .transform(Remi::Transform[:concatenate].(', '))
+
+      map source(:school_id, :school_name) .target(:School__c)
+        .transform(->(id, name) {[
+            Remi::Transform[:ifblank]["Unknown"].call(id),
+            Remi::Transform[:ifblank]["Unknown"].call(name)
+          ]})
+        .transform(Remi::Transform[:concatenate].('-'))
+
+      map source(:current_email)       .target(:Email)
+        .transform(Remi::Transform[:replace].(/,/, '.'))
+        .transform(Remi::Transform[:validate_email].call)
+    end
+
+    contact_creates.df = work_contact_creates[
+      :External_ID__c,
+      :School_ID__c,
+      :School_Name__c,
+      :School__c,
+      :Major__c,
+      :FirstName,
+      :LastName,
+      :Email,
+      :MailingStreet,
+      :MailingCity,
+      :MailingState,
+      :MailingPostalCode,
+      :Birthdate,
+      :Applied_Date__c
+    ]
+  end
+
+  define_transform :map_updates, sources: :all_contacts, targets: :contact_updates do
+    contact_updates.df = all_contacts.df[
+      :Id,
+      :Major__c
+    ].where(all_contacts.df[:Id].not_eq(nil))
+  end
+end

data/lib/remi/project/jobs/transforms/date_diff_job.rb

@@ -0,0 +1,17 @@
+require_relative '../all_jobs_shared'
+
+class DateDiffJob
+  include AllJobsShared
+
+  define_param :measure, :days
+  define_source :source_data, Remi::DataSource::DataFrame
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:date1, :date2) .target(:difference)
+        .transform(->(d1,d2) { [Date.strptime(d1), Date.strptime(d2)] })
+        .transform(Remi::Transform[:date_diff].(params[:measure]))
+    end
+  end
+end

data/lib/remi/project/jobs/transforms/parse_date_job.rb

@@ -0,0 +1,18 @@
+require_relative '../all_jobs_shared'
+
+class ParseDateJob
+  include AllJobsShared
+
+  define_param :format, '%Y-%m-%d'
+  define_param :if_blank, nil
+  define_source :source_data, Remi::DataSource::DataFrame
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:date_string) .target(:parsed_date)
+        .transform(Remi::Transform[:parse_date].(format: params[:format], if_blank: params[:if_blank]))
+    end
+  end
+end

data/lib/remi/project/jobs/transforms/prefix_job.rb

@@ -0,0 +1,16 @@
+require_relative '../all_jobs_shared'
+
+class PrefixJob
+  include AllJobsShared
+
+  define_param :prefix, 'prefix'
+  define_source :source_data, Remi::DataSource::DataFrame
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:field) .target(:field)
+        .transform(Remi::Transform[:prefix].(params[:prefix]))
+    end
+  end
+end

data/lib/remi/project/jobs/transforms/transform_jobs.rb

@@ -0,0 +1,3 @@
+require_relative 'prefix_job'
+require_relative 'parse_date_job'
+require_relative 'date_diff_job'

data/lib/remi/settings.rb

@@ -0,0 +1,39 @@
+module Remi
+  module Settings
+    extend self
+
+    def work_dir
+      @work_dir ||= File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname('',nil))
+    end
+
+    def work_dir=(arg)
+      @work_dir = arg
+    end
+
+    def log_level
+      @log_level ||= Logger::INFO
+    end
+
+    def log_level=(arg)
+      @log_level = arg
+    end
+
+    def logger
+      return @logger.call if @logger.respond_to? :call
+      @logger ||= lambda do
+        l = Logger.new(STDOUT)
+        l.level = log_level
+        l.formatter = proc do |severity, datetime, progname, msg|
+          "#{msg}\n"
+        end
+        l
+      end
+
+      @logger.call
+    end
+
+    def logger=(arg)
+      @logger = arg
+    end
+  end
+end

data/lib/remi/sf_bulk_helper.rb

@@ -0,0 +1,265 @@
+module Remi
+  module SfBulkHelper
+
+    class DupeLookupKeyError < StandardError; end
+    class MaxAttemptError < StandardError; end
+
+    # Public: Class used to execute SF Bulk operations.  This class is not meant to be
+    # used directly.  It is instead meant to be inherited by classes that perform the
+    # specific query, update, create, or upsert operations.
+    #
+    # Examples
+    #
+    #   sf_query = SfBulkQuery.query(client, 'Contact', 'SELECT Id, Name FROM Contact')
+    #   puts sf_query.result
+    #
+    #   mydata = [ { 'Id' => '001G000000ncxb8IAA', 'Name' => 'Happy Feet' } ]
+    #   sf_update = SfBulkUpdate.update(client, 'Contact', mydata)
+    class SfBulkOperation
+
+      # Public: Initializes a SfBulkOperation (does not execute operation).
+      #
+      # restforce_client - An instance of Restforce that is used to authenticate the connection.
+      # object           - The name of the object to operate on (e.g., Contact, Task, etc).
+      # data             - For query operations, this is the SOQL query string.  For other
+      #                    operations, this is an array of hashes, where the hash keys are column names
+      #                    and the values are the data.
+      # batch_size       - Batch size to use to download or upload data (default: 10000)
+      # max_mattempts    - The maximum number of attempts to upload data (default: 2)
+      # logger           - Logger to use (default: Logger.new(STDOUT)
+      def initialize(restforce_client, object, data, batch_size: 5000, max_attempts: 2, logger: Logger.new(STDOUT))
+        @restforce_client = restforce_client
+        @object = object
+        @data = data
+        @batch_size = batch_size
+        @max_attempts = max_attempts
+        @attempts = Hash.new(0)
+        @logger = logger
+      end
+
+      # Public: A symbol representing the operation to be performed (:query, :update, :create, :upsert)
+      def operation
+        :undefined
+      end
+
+      # Public: Returns the instance of SalesforceBulkApi::Api used for bulk operations.
+      def sf_bulk
+        @sf_bulk ||= SalesforceBulkApi::Api.new(@restforce_client).tap { |o| o.connection.set_status_throttle(5) }
+      end
+
+      # Public: Returns the raw result from the SalesforceBulkApi query
+      def raw_result
+        @raw_result || execute
+      end
+
+      # Public: Returns useful metadata about the batch query.
+      def info
+        execute if @attempts[:total] == 0
+
+        return @info if @info and @attempts[:info] == @attempts[:total]
+        @attempts[:info] += 1
+
+        @info = raw_result.reject { |k,v| k == 'batches' }.tap do |h|
+          h['query'] = @data if operation == :query
+        end
+      end
+
+      # Public: Collects the results from all of the batches and aggregates them
+      # into an array of hashes.  Each element of the array represents a record in the
+      # result and the hash gives the column-value.  Note that if multiple retries are
+      # needed, this is just the final result.
+      #
+      # Returns an array of hashes.
+      def result
+        execute if @attempts[:total] == 0
+
+        return @result if @result and @attempts[:result] == @attempts[:total]
+        @attempts[:result] += 1
+
+        @result = []
+        raw_result['batches'].each do |batch|
+          next unless batch['response']
+
+          batch['response'].each do |record|
+            @result << record.inject({}) { |h, (k,v)| h[k] = v.first unless ['xsi:type','type'].include? k; h }
+          end
+
+          # delete raw result at end of processing to free memory
+          batch['response'] = nil
+        end
+
+        @result
+      end
+
+      # Public: Converts the result into a hash that can be used to
+      # lookup the row for a given key (e.g., external id field).
+      #
+      # key        - A string representing the name of the column to be used as the lookup key.
+      # duplicates - Indicates whether duplicate keys are allowed.  If they are allowed,
+      #              only the first row found will be retained.  If duplicates are not allowed,
+      #              an error is raised (default: false).
+      #
+      # Returns a hash.
+      def as_lookup(key:, duplicates: false)
+        execute if @attempts[:total] == 0
+
+        @as_lookup ||= {}
+        @attempts[:as_lookup] = Hash.new(0) if @attempts[:as_lookup] == 0
+
+        return @as_lookup[key] if @as_lookup[key] and @attempts[:as_lookup][key] == @attempts[:total]
+        @attempts[:as_lookup][key] += 1
+
+        @as_lookup[key] = result.inject({}) do |lkp,row|
+          raise DupeLookupKeyError, "Duplicate key: #{row[key]} found in result of query: #{@data}" if lkp.has_key?(row[key]) and not duplicates
+          lkp[row[key]] = row unless lkp.has_key?(row[key])
+          lkp
+        end
+      end
+
+
+      # Public: Returns true if any of the records failed to update.
+      def failed_records?
+        n_failed_records = result.reduce(0) do |count, row|
+          count += 1 if row['success'] != 'true'
+          count
+        end
+
+        n_failed_batches = raw_result['batches'].reduce(0) do |count, batch|
+          count += 1 if batch['state'].first != 'Completed'
+          count
+        end
+
+        n_failed_records > 0 || n_failed_batches > 0
+      end
+
+
+      private
+
+      # Private: Sends the operation to Salesforce using the bulk API.
+      def send_bulk_operation
+        raise "No SF bulk operation defined for #{operation}"
+      end
+
+      # Private: Executes the operation and retries if needed.
+      def execute
+        @attempts[:total] += 1
+        @logger.info "Executing Salesforce Bulk operation: #{operation}"
+
+        @raw_result = send_bulk_operation
+        @logger.info "Bulk operation response: "
+        JSON.pretty_generate(info).split("\n").each { |l| @logger.info l }
+
+        retry_failed if failed_records?
+
+        @logger.info JSON.pretty_generate(info)
+        @raw_result
+      end
+
+      # Private: Drops any data that has already been loaded to salesforce.
+      # Note that this doesn't work for created data since the initial data
+      # wont have a salesforce id.  Sometimes batches can fail completely
+      # and won't give anything in the result set.  Therefore, the only way
+      # to be able to drop data that's already been created would be to
+      # know how the data was split into batches and the gem we use does not
+      # make this simple.  So for now, we live with the defect.
+      def drop_successfully_updated_data
+        lkp_result_by_id = as_lookup(key: 'id', duplicates: true)
+        @data.reject! do |row|
+          sf_bulk_result = lkp_result_by_id[row['Id'] || row[:Id]]
+          sf_bulk_result && (sf_bulk_result['success'] == 'true')
+        end
+
+        nil
+      end
+
+
+      # Private: Selects data needed to be retried and re-executes the operation.
+      def retry_failed
+        raise MaxAttemptError if @attempts[:total] >= @max_attempts
+        @logger.warn "Retrying #{operation} - #{@attempts[:total]} of #{@max_attempts}"
+
+        drop_successfully_updated_data
+
+        execute
+      end
+    end
+
+
+    # Public: Class used to execute SF Bulk Update operations (see SfBulkOperation class for
+    # more details).
+    class SfBulkUpdate < SfBulkOperation
+      def self.update(*args,**kargs)
+        SfBulkUpdate.new(*args,**kargs).tap { |sf| sf.send(:execute) }
+      end
+
+      def operation
+        :update
+      end
+
+      private
+
+      def send_bulk_operation
+        sf_bulk.send(operation, @object, @data, true, false, [], @batch_size)
+      end
+    end
+
+    # Public: Class used to execute SF Bulk Create operations (see SfBulkOperation class for
+    # more details).
+    class SfBulkCreate < SfBulkOperation
+      def self.create(*args,**kargs)
+        SfBulkCreate.new(*args,**kargs).tap { |sf| sf.send(:execute) }
+      end
+
+      def operation
+        :create
+      end
+
+      private
+
+      def send_bulk_operation
+        sf_bulk.send(operation, @object, @data, true, false, @batch_size)
+      end
+    end
+
+    # Public: Class used to execute SF Bulk Upsert operations (see SfBulkOperation class for
+    # more details).
+    class SfBulkUpsert < SfBulkOperation
+      def self.upsert(*args,**kargs)
+        SfBulkUpsert.new(*args,**kargs).tap { |sf| sf.send(:execute) }
+      end
+
+      def operation
+        :upsert
+      end
+
+      private
+
+      def send_bulk_operation
+        # Upsert does not support external id right now
+        sf_bulk.send(operation, @object, @data, 'Id', true, false, [], @batch_size)
+      end
+    end
+
+    # Public: Class used to execute SF Bulk Query operations (see SfBulkOperation class for
+    # more details).
+    class SfBulkQuery < SfBulkOperation
+      def self.query(*args,**kargs)
+        SfBulkQuery.new(*args,**kargs).tap { |sf| sf.send(:execute) }
+      end
+
+      def operation
+        :query
+      end
+
+      def failed_records?
+        false
+      end
+
+      private
+
+      def send_bulk_operation
+        sf_bulk.send(operation, @object, @data, @batch_size)
+      end
+    end
+  end
+end