remi 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea2d2971479e9e0dfcc5de4dd01ac13f5274a6f8
-  data.tar.gz: ea5e3a3280613d00ae29f5265c342740973ea57c
+  metadata.gz: 0de4e8f2de3129e2e4b93c3d22dc5f718a05b56a
+  data.tar.gz: d963548c553f1918b33bd391038bc3481ce4a5d8
 SHA512:
-  metadata.gz: a09f8f926a99891356bcd6363ef4729b3a27bcf9cbfacce2a6bc8a8b60a262cfe83fe40d1e0f1f0c79945693aacb0fb1099b5bad299e022511846730e98642fe
-  data.tar.gz: d160e9840162558b1d9e203825f8e724ce1626e9f8a6922b86f894249b78c5c6680b37fef7e24574d794aac41db025bd887c707df35f6485a3540914552a3293
+  metadata.gz: d01e67e38c2a76784e65a22536d2d9cba7c9f56dc3686e8d0d23ea1e5176cb8495cd06977ba85357d336cbf7fd91641f79795d68df69542ce9e94b39bc85c6ec
+  data.tar.gz: 07cec77fc7c40299207081f5ea7390cdd3cf863ac49295e9de166e4736ff95ae5d57e642de2a10ea74d1d812aeeff2ed4ffdb6a93c232658efc7552332a9f1e9

data/Gemfile

@@ -3,7 +3,8 @@ source 'https://rubygems.org'
 
 gemspec
 gem 'google-api-client', '~> 0.9'
-gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1.4.1-Remi'
+gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1.4.1.2-Remi'
 gem 'restforce', '~> 2.1'
 gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
+gem 'soapforce', '~> 0.5'
 gem 'aws-sdk', '~> 2.3'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: git@github.com:inside-track/daru.git
-  revision: e38e98890c99dee03ceedaca3a5faf40859042ab
-  branch: 0.1.4.1-Remi
+  revision: c8d407ee55b8d5f3143b9a030fe36c38cf3537d1
+  branch: 0.1.4.1.2-Remi
   specs:
     daru (0.1.4.1)
       backports
@@ -18,7 +18,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.3.1)
+    remi (0.3.2)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
@@ -40,6 +40,9 @@ GEM
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.4.0)
+    akami (1.3.1)
+      gyoku (>= 0.4.0)
+      nokogiri
     aws-sdk (2.3.5)
       aws-sdk-resources (= 2.3.5)
     aws-sdk-core (2.3.5)
@@ -87,8 +90,13 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
+    gyoku (1.3.1)
+      builder (>= 2.1.2)
     hashie (3.4.3)
     httpclient (2.8.2.4)
+    httpi (2.4.2)
+      rack
+      socksify
     hurley (0.2)
     i18n (0.7.0)
     iruby (0.2.7)
@@ -110,6 +118,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
     mimemagic (0.3.1)
+    mini_portile2 (2.1.0)
     minitest (5.8.4)
     multi_json (1.11.2)
     multi_test (0.1.2)
@@ -117,8 +126,12 @@ GEM
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (3.1.1)
+    nokogiri (1.7.0.1)
+      mini_portile2 (~> 2.1.0)
+    nori (2.6.0)
     os (0.9.6)
     pg (0.18.4)
+    rack (2.0.1)
     rbczmq (1.7.9)
     redcarpet (3.3.4)
     regex_sieve (0.1.0)
@@ -144,15 +157,29 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
     rspec-support (3.4.1)
+    savon (2.11.1)
+      akami (~> 1.2)
+      builder (>= 2.1.2)
+      gyoku (~> 1.2)
+      httpi (~> 2.3)
+      nokogiri (>= 1.4.0)
+      nori (~> 2.4)
+      wasabi (~> 3.4)
     signet (0.7.3)
       addressable (~> 2.3)
       faraday (~> 0.9)
       jwt (~> 1.5)
       multi_json (~> 1.10)
+    soapforce (0.5.0)
+      savon (>= 2.3.0, < 3.0.0)
+    socksify (1.7.0)
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     uber (0.0.15)
+    wasabi (3.5.0)
+      httpi (~> 2.0)
+      nokogiri (>= 1.4.2)
     xml-simple (1.1.5)
     yard (0.9.0)
 
@@ -169,7 +196,8 @@ DEPENDENCIES
   remi!
   restforce (~> 2.1)
   salesforce_bulk_api!
+  soapforce (~> 0.5)
   yard (~> 0.9)
 
 BUNDLED WITH
-   1.13.5
+   1.14.3

data/features/formulas.feature

@@ -8,6 +8,28 @@ Feature: This tests the creation of example records.
     And the source 'Source Data'
     And the target 'Target Data'
 
+  Scenario: Handling date formulas in the example data with minute units.
+
+    Given the following example record for 'Source Data':
+      | 1MinuteAgo     | 15MinutesAgo     | OneMinuteAgo      | 1MinuteFromNow      |  15MinutesFromNow      |
+      | *1 minute ago* | *15 minutes ago* | *1 minute ago*    | *1 minute from now* | *15 minutes from now*  |
+    Then the target field '1MinuteAgo' is the time 1 minute ago
+    And the target field '2MinutesAgo' is the time 15 minutes from now
+    And the target field 'OneMinuteAgo' is the time 1 minute ago
+    And the target field '1MinuteFromNow' is the time 1 minute from now
+    And the target field '2MinutesFromNow' is the time 15 minutes from now
+
+  Scenario: Handling date formulas in the example data with hour units.
+
+    Given the following example record for 'Source Data':
+      | 1HourAgo     | 2HoursAgo     | OneHourAgo      | 1HourFromNow      |  2HoursFromNow      |
+      | *1 hour ago* | *2 hours ago* | *1 hour ago*    | *1 hour from now* | *2 hours from now*  |
+    Then the target field '1HourAgo' is the time 1 hour ago
+    And the target field '2HoursAgo' is the time 2 hours from now
+    And the target field 'OneHourAgo' is the time 1 hour ago
+    And the target field '1HourFromNow' is the time 1 hour from now
+    And the target field '2HoursFromNow' is the time 2 hours from now
+
   Scenario: Handling date formulas in the example data with day units.
 
     Given the following example record for 'Source Data':

data/features/step_definitions/remi_step.rb

@@ -124,7 +124,6 @@ Given /^the (source|target) file contains all of the following headers in this o
   expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to eq @brt.send(st.to_sym).fields.field_names
 end
 
-
 ### Source
 
 Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
@@ -219,6 +218,20 @@ Given /^the source field is not a valid email address$/ do
   @brt.source.field.value = 'invalid!example.com'
 end
 
+Given /^the source field '([^']+)' is a valid email address$/ do |source_field|
+  step "the source field '#{source_field}'"
+
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+  @brt.sources[source_name].fields[source_field_name].value = 'valid@example.com'
+end
+
+Given /^the source field '([^']+)' is not a valid email address$/ do |source_field|
+  step "the source field '#{source_field}'"
+
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+  @brt.sources[source_name].fields[source_field_name].value = 'invalid!example.com'
+end
+
 ### Target
 
 Given /^the target '([[:alnum:]\s\-_]+)'$/ do |arg|
@@ -242,6 +255,13 @@ Then /^the target field '([^']+)' is copied from the source field '([^']+)'$/ do
   end
 end
 
+Then /^the target field '([^']+)' has the label '([^']+)'$/ do |target_field, label|
+  step "the target field '#{target_field}'"
+  data_field  = @brt.targets.fields.next
+  expect(data_field.metadata[:label]).to eq label
+  expect(data_field.name).to eq target_field
+end
+
 Then /^the target field '([^']+)' is copied from the source field$/ do |target_field|
   @brt.sources.fields.each do |source_field|
     step "the target field '#{target_field}' is copied from the source field '#{source_field.full_name}'"
@@ -483,9 +503,10 @@ Then /^the target field '([^']+)' is populated from the source field '([^']+)' u
 
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
   target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
+  inferred_type = target_format =~ /(%H|%M|%S)/ ? :datetime : :date
 
   source_format = @brt.sources[source_name].fields[source_field_name].metadata[:in_format]
-  source_reformatted = Remi::Transform::FormatDate.new(in_format: source_format, out_format: target_format).to_proc
+  source_reformatted = Remi::Transform::FormatDate.new(in_format: source_format, out_format: target_format, type: inferred_type).to_proc
     .call(@brt.sources[source_name].fields[source_field_name].value)
 
   @brt.run_transforms

data/lib/remi.rb

@@ -74,8 +74,6 @@ require 'remi/data_frame/daru'
 
 require 'remi/transform'
 
-require 'remi/monkeys/daru'
-
 # Remi is Ruby Extract Modify and Integrate, a framework for writing ETL job in Ruby.
 module Remi
 end

data/lib/remi/data_subjects/csv_file.rb

@@ -78,6 +78,12 @@ module Remi
         processed_filename = preprocess(filename)
         csv_df = Daru::DataFrame.from_csv processed_filename, @csv_options
 
+        # Daru 0.1.4 doesn't add vectors if it's a headers-only file
+        if csv_df.vectors.size == 0
+          headers_df = Daru::DataFrame.from_csv processed_filename, @csv_options.merge(return_headers: true)
+          csv_df = Daru::DataFrame.new([], order: headers_df.vectors.to_a)
+        end
+
         csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
         if idx == 0
           result_df = csv_df
@@ -153,15 +159,24 @@ module Remi
     attr_reader :csv_options
 
     # Converts the dataframe to a CSV file stored in the local work directory.
+    # If labels are present write the CSV file with those headers but maintain
+    # the structure of the original dataframe
     #
     # @param dataframe [Remi::DataFrame] The dataframe to be encoded
     # @return [Object] The path to the file
     def encode(dataframe)
       logger.info "Writing CSV file to temporary location #{@working_file}"
+
+      label_columns = self.fields.reduce({}) { |h, (k, v)|
+        if v[:label]
+          h[k] = v[:label].to_sym
+        end
+        h
+      }
+      dataframe.rename_vectors label_columns
       dataframe.write_csv @working_file, @csv_options
       @working_file
     end
-
     private
     def init_csv_file_encoder(*args, work_path: Settings.work_dir, csv_options: {}, **kargs, &block)
       @working_file = File.join(work_path, SecureRandom.uuid)

data/lib/remi/data_subjects/gsheet.rb

@@ -46,8 +46,8 @@ module Remi
       service.list_files(q: "'#{folder_id}' in parents", page_size: 10, order_by: 'createdTime desc', fields: 'nextPageToken, files(id, name, createdTime, mimeType)')
     end
 
-    def get_spreadsheet_vals(service, spreadsheet_id)
-      service.get_spreadsheet_values(spreadsheet_id, 'Sheet1')
+    def get_spreadsheet_vals(service, spreadsheet_id, sheet_name = 'Sheet1')
+      service.get_spreadsheet_values(spreadsheet_id, sheet_name)
     end
 
     def extract
@@ -57,7 +57,8 @@ module Remi
       @data                                   = []
 
       entries.each do |file|
-        response = get_spreadsheet_vals(service, file.raw)
+        logger.info "Extracting Google Sheet data from #{file.pathname}, with sheet name : #{@sheet_name}"
+        response = get_spreadsheet_vals(service, file.raw, @sheet_name)
         data.push(response)
       end
 
@@ -85,8 +86,9 @@ module Remi
 
     private
 
-    def init_gsheet_extractor(*args, credentials:, folder_id:, **kargs)
+    def init_gsheet_extractor(*args, credentials:, folder_id:, sheet_name: 'Sheet1', **kargs)
       @default_folder_id   = folder_id
+      @sheet_name          = sheet_name
       @oob_uri             = 'urn:ietf:wg:oauth:2.0:oob'
       @application_name    = credentials.fetch(:application_name)
 
@@ -111,25 +113,23 @@ module Remi
   class Parser::Gsheet < Parser
 
     def parse(gs_extract)
-      google_vals = gs_extract.data
       return_hash = nil
-      google_vals.each do |google_val|
+      gs_extract.data.each do |gs_data|
 
         if return_hash.nil?
           return_hash = Hash.new
-          google_val.values[0].each do |header|
+          gs_data.values[0].each do |header|
             return_hash[field_symbolizer.call(header)] = []
           end
         end
 
-        keys_temp = return_hash.keys
+        headers = return_hash.keys
+        header_idx = headers.each_with_index.to_h
 
-        google_val.values[1..-1].each do |rows|
-          col_num = 0
-
-          rows.each do |value|
-            return_hash[keys_temp[col_num]] << value
-            col_num +=1
+        gs_data.values[1..-1].each do |row|
+          headers.each do |header|
+            idx = header_idx[header]
+            return_hash[header] << (idx < row.size ? row[idx] : nil)
           end
         end
       end

data/lib/remi/data_subjects/salesforce.rb

@@ -168,7 +168,7 @@ module Remi
     # @option credentials [String] :password Salesforce password
     # @option credentials [String] :security_token Salesforce security token
     # @param object [Symbol] Salesforce object to extract
-    # @param operation [Symbol] Salesforce operation to perform (`:update`, `:create`, `:upsert`)
+    # @param operation [Symbol] Salesforce operation to perform (`:update`, `:create`, `:upsert`, `:delete`)
     # @param batch_size [Integer] Size of batch to use for updates (1-10000)
     # @param external_id [Symbol, String] Field to use as an external id for upsert operations
     # @param api [Symbol] Salesforce API to use (only option supported is `:bulk`)
@@ -188,6 +188,8 @@ module Remi
         Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
       elsif @operation == :upsert
         Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, data, batch_size: @batch_size, external_id: @external_id, logger: logger)
+      elsif @operation == :delete
+        Remi::SfBulkHelper::SfBulkDelete.upsert(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
       else
         raise ArgumentError, "Unknown operation: #{@operation}"
       end

data/lib/remi/data_subjects/salesforce_soap.rb

@@ -0,0 +1,98 @@
+require 'soapforce'
+
+module Remi
+  module DataSubject::SalesforceSoap
+    def soapforce_client
+      @soapforce_client ||= begin
+        client = Soapforce::Client.new(host: @credentials[:host], logger: logger)
+        client.authenticate(
+          username: @credentials[:username],
+          password: "#{@credentials[:password]}#{@credentials[:security_token]}"
+        )
+        client
+      end
+    end
+  end
+
+  # Salesforce SOAP encoder
+  class Encoder::SalesforceSoap < Encoder
+    # Converts the dataframe to an array of hashes, which can be used
+    # by the salesforce soap api.
+    #
+    # @param dataframe [Remi::DataFrame] The dataframe to be encoded
+    # @return [Object] The encoded data to be loaded into the target
+    def encode(dataframe)
+      dataframe.to_a[0]
+    end
+  end
+
+  # Salesforce SOAP loader
+  # The Salesforce SOAP loader can be used to merge salesforce objects (for those
+  # objects that support the merge operation).  To do so, each row of the dataframe must
+  # contain a field called `:Id` that references the master record that survives the
+  # merge operation.  It must also contain a `:Merge_Id` field that specifies the
+  # salesforce Id of the record that is to be merged into the master.  Other fields
+  # may also be specified that will be used to update the master record.
+  #
+  # @example
+  #  class MyJob < Remi::Job
+  #    target :merge_contacts do
+  #      encoder Remi::Encoder::SalesforceSoap.new
+  #      loader Remi::Loader::SalesforceSoap.new(
+  #        credentials: { },
+  #        object: :Contact,
+  #        operation: :merge,
+  #        merge_id_field: :Merge_Id
+  #      )
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.merge_contacts.df = Remi::DataFrame::Daru.new({ Id: ['003g000001IX4HcAAL'], Note__c: ['Cheeseburger in Paradise'], Merge_Id: ['003g000001LE7dXAAT']})
+  #  job.merge_contacts.load
+  #
+  class Loader::SalesforceSoap < Loader
+    include Remi::DataSubject::SalesforceSoap
+
+    # @param credentials [Hash] Used to authenticate with salesforce
+    # @option credentials [String] :host Salesforce host (e.g., login.salesforce.com)
+    # @option credentials [String] :username Salesforce username
+    # @option credentials [String] :password Salesforce password
+    # @option credentials [String] :security_token Salesforce security token
+    # @param object [Symbol] Salesforce object to extract
+    # @param operation [Symbol] Salesforce operation to perform (`:merge`) <- Merge is the only operation currently supported
+    # @param merge_id_field [Symbol] For merge operations, this is the name of the field containing the id of the record to be merged (default: :Merge_Id)
+    def initialize(*args, **kargs, &block)
+      super
+      init_salesforce_loader(*args, **kargs, &block)
+    end
+
+    # @param data [Encoder::Salesforce] Data that has been encoded appropriately to be loaded into the target
+    # @return [true] On success
+    def load(data)
+      logger.info "Performing Salesforce Soap #{@operation} on object #{@sfo}"
+      if @operation == :merge
+        # The Soapforce gem only supports one slow-ass merge at a time :(
+        data.each do |row|
+          unless row.include?(@merge_id_field)
+            raise KeyError, "Merge id field not found: #{@merge_id_field}"
+          end
+
+          merge_id = Array(row.delete(@merge_id_field))
+          soapforce_client.merge(@sfo, row, merge_id)
+        end
+      else
+        raise ArgumentError, "Unknown soap operation: #{@operation}"
+      end
+    end
+
+    private
+
+    def init_salesforce_loader(*args, object:, credentials:, operation:, merge_id_field: :Merge_Id, **kargs, &block)
+      @sfo            = object
+      @credentials    = credentials
+      @operation      = operation
+      @merge_id_field = merge_id_field
+    end
+  end
+end

data/lib/remi/sf_bulk_helper.rb

@@ -81,6 +81,7 @@ module Remi
           next unless batch['response']
 
           batch['response'].each do |record|
+            @logger.error "Salesforce error: #{record}" if record['success'] && record['success'][0] == 'false'
             @result << record.inject({}) { |h, (k,v)| h[k] = v.first unless ['xsi:type','type'].include? k; h }
           end
 
@@ -245,6 +246,24 @@ module Remi
       end
     end
 
+    # Public: Class used to execute SF Bulk Delete operations (see SfBulkOperation class for
+    # more details).
+    class SfBulkDelete < SfBulkOperation
+      def self.delete(*args, **kargs)
+        SfBulkDelete.new(*args, **kargs).tap { |sf| sf.send(:execute) }
+      end
+
+      def operation
+        :delete
+      end
+
+      private
+
+      def send_bulk_operation
+        sf_bulk.send(operation, @object, @data, true, @batch_size)
+      end
+    end
+
     # Public: Class used to execute SF Bulk Query operations (see SfBulkOperation class for
     # more details).
     class SfBulkQuery < SfBulkOperation

data/lib/remi/testing/business_rules.rb

@@ -30,6 +30,7 @@ module Remi::Testing::BusinessRules
     def formulas
       @formulas ||= RegexSieve.new({
         /\*now(|:[^*]+)\*/i => [:time_reference, :match_now],
+        /\*(\d+)\s(hour|hours|minute|minutes) (ago|from now)(|:[^*]+)\*/i => [:time_reference, :match_time],
         /\*(today|yesterday|tomorrow)(|:[^*]+)\*/i => [:date_reference, :match_single_day],
         /\*(this|last|previous|next) (day|month|year|week)(|:[^*]+)\*/i => [:date_reference, :match_single_unit],
         /\*(\d+)\s(day|days|month|months|year|years|week|weeks) (ago|from now)(|:[^*]+)\*/i => [:date_reference, :match_multiple]
@@ -44,12 +45,12 @@ module Remi::Testing::BusinessRules
 
       to_replace = form.match(base_regex)[0]
       replace_with = if form_opt[:value][0] == :date_reference
-        date_reference(form_opt[:value][1], form_opt[:match])
-      elsif form_opt[:value][0] == :time_reference
-        time_reference(form_opt[:value][1], form_opt[:match])
-      else
-        to_replace
-      end
+                       date_reference(form_opt[:value][1], form_opt[:match])
+                     elsif form_opt[:value][0] == :time_reference
+                       time_reference(form_opt[:value][1], form_opt[:match])
+                     else
+                       to_replace
+                     end
 
       form.gsub(to_replace, replace_with)
     end
@@ -62,6 +63,7 @@ module Remi::Testing::BusinessRules
     def date_reference(formula, captured)
       parsed = self.send("date_reference_#{formula}", *captured)
       Date.current.send("#{parsed[:unit]}_#{parsed[:direction]}", parsed[:quantity]).strftime(parsed[:format])
+
     end
 
     def parse_colon_date_format(str)
@@ -80,6 +82,21 @@ module Remi::Testing::BusinessRules
         format: parse_colon_time_format(format)
       }
     end
+    def time_reference_match_time(form, quantity, unit, direction, format=nil)
+      divisor = 1.0
+      if unit.downcase.pluralize =='hours'
+        divisor = 24.0
+      elsif unit.downcase.pluralize == 'minutes'
+        divisor = 24.0*60.0
+      end
+
+      {
+        quantity: quantity.to_i/divisor,
+        unit: 'days',
+        direction: { 'ago' => 'ago', 'from now' => 'since' }[direction.downcase],
+        format: parse_colon_time_format(format)
+      }
+    end
 
     def date_reference_match_single_day(form, direction, format=nil)
       {
@@ -481,10 +498,10 @@ module Remi::Testing::BusinessRules
 
     def value=(arg)
       typed_arg = if metadata[:type] == :json
-        JSON.parse(arg)
-      else
-        arg
-      end
+                    JSON.parse(arg)
+                  else
+                    arg
+                  end
 
       vector.recode! { |_v| typed_arg }
     end
@@ -524,10 +541,10 @@ module Remi::Testing::BusinessRules
     def parse_formula(value)
       parsed_value = ParseFormula.parse(value)
       case parsed_value
-      when '\nil'
-        nil
-      else
-        parsed_value
+        when '\nil'
+          nil
+        else
+          parsed_value
       end
     end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.3.1'
+  VERSION = '0.3.2'
 end

data/spec/data_subjects/csv_file_spec.rb

@@ -65,5 +65,60 @@ describe Parser::CsvFile do
 
     expect(csv.parse(two_files).to_a).to eq expected_df.to_a
   end
+  it 'returns empty vectors if the csv contains headers only' do
+    csv = Parser::CsvFile.new
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: [],
+        column_b: []
+      }
+    )
+
+    expect(csv.parse('spec/fixtures/empty.csv').to_h).to eq expected_df.to_h
+  end
+end
 
+describe Encoder::CsvFile do
+  let(:basic_dataframe) do
+    Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B']
+      }
+    )
+  end
+  it 'creates a csv from a provided dataframe' do
+    encoder = Encoder::CsvFile.new
+    parser = Parser::CsvFile.new
+    provided_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B', nil, nil],
+        column_c: [nil, nil, 'value 1C', 'value 2C']
+      }
+    )
+    expected_contents = "column_a,column_b,column_c\nvalue 1A,value 1B,\nvalue 2A,value 2B,\nvalue 1A,,value 1C\nvalue 2A,,value 2C\n"
+    file_name = encoder.encode(provided_df)
+    expect(File.read(file_name)).to eq expected_contents
+  end
+  it 'uses label headers when provided' do
+    provided_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B', nil, nil],
+        column_c: [nil, nil, 'value 1C', 'value 2C']
+      }
+    )
+    expected_contents = "Column A,Column B,Column C\nvalue 1A,value 1B,\nvalue 2A,value 2B,\nvalue 1A,,value 1C\nvalue 2A,,value 2C\n"
+    column_fields = Remi::Fields.new({
+      :column_a => { label: 'Column A' },
+      :column_b => { label: 'Column B' },
+      :column_c => { label: 'Column C' }
+    })
+    encoder = Encoder::CsvFile.new(fields: column_fields)
+    file_name = encoder.encode(provided_df)
+    expect(File.read(file_name)).to eq expected_contents
+  end
 end
+

data/spec/data_subjects/gsheet_spec.rb

@@ -21,7 +21,8 @@ describe Extractor::Gsheet do
     {
       credentials: credentials,
       folder_id:   'some_google_folder_id',
-      remote_path: remote_path
+      remote_path: remote_path,
+      sheet_name: 'some_google_sheet_name'
     }
   }
 
@@ -109,7 +110,9 @@ describe Parser::Gsheet do
   let(:gs_extract) { double('gs_extract') }
   let(:example_data) do
     [{"headers" => ["header_1", "header_2", "header_3"],
-      "row 1"   => ["value 1", "value 2", "value 3"]
+      "row 1"   => ["value 11", "value 12", "value 13"],
+      "row 2"   => ["value 21", "value 22", "value 23"],
+      "row 3"   => ["value 31", "value 32", "value 33"],
     }]
   end
 
@@ -123,11 +126,21 @@ describe Parser::Gsheet do
 
   it 'converted data into the correct dataframe' do
     expected_df = Daru::DataFrame.new(
-      :header_1 => ['value 1'],
-      :header_2 => ['value 2'],
-      :header_3 => ['value 3'],
+      :header_1 => ['value 11', 'value 21', 'value 31'],
+      :header_2 => ['value 12', 'value 22', 'value 32'],
+      :header_3 => ['value 13', 'value 23', 'value 33']
     )
     expect(parser.parse(gs_extract).to_a).to eq expected_df.to_a
   end
 
+  it 'works when the last column contains blanks' do
+    # Google API only returns an array of dimensions up to the last non-blank column
+    example_data[0]['row 2'].pop
+    expected_df = Daru::DataFrame.new(
+      :header_1 => ['value 11', 'value 21', 'value 31'],
+      :header_2 => ['value 12', 'value 22', 'value 32'],
+      :header_3 => ['value 13', nil,        'value 33']
+    )
+    expect(parser.parse(gs_extract).to_a).to eq expected_df.to_a
+  end
 end

data/spec/data_subjects/salesforce_soap_spec.rb

@@ -0,0 +1,80 @@
+require_relative '../remi_spec'
+require 'remi/data_subjects/salesforce_soap.rb'
+
+
+
+describe Encoder::SalesforceSoap do
+  let(:encoder) { Encoder::SalesforceSoap.new }
+  let(:dataframe) do
+    Daru::DataFrame.new(
+      :Id            => ['003G000001cKYaUIA4', '003G000001cKYbXIA4'],
+      :Student_ID__c => ['FJD385628', nil],
+      :Merge_Id__c   => ['003g000001LE7dXAAT','003g000001IX4HcAAL']
+    )
+  end
+
+  it 'converts the dataframe into an array of hashes' do
+    expected_result = [
+      { :Id => '003G000001cKYaUIA4', :Student_ID__c => 'FJD385628', :Merge_Id__c => '003g000001LE7dXAAT' },
+      { :Id => '003G000001cKYbXIA4', :Student_ID__c => nil,         :Merge_Id__c => '003g000001IX4HcAAL' },
+    ]
+    expect(encoder.encode dataframe).to eq expected_result
+  end
+end
+
+
+describe Loader::SalesforceSoap do
+  let(:loader) { Loader::SalesforceSoap.new(object: :Contact, credentials: {}, operation: :merge) }
+  let(:soapforce_client) { double('soapforce_client') }
+
+  before do
+    allow(loader).to receive(:soapforce_client) { soapforce_client }
+  end
+
+  it 'raises an error if an unknown operation is requested' do
+    data = [
+      { Id: '1234', Custom__c: 'something', Merge_Id: '5678' }
+    ]
+
+    loader = Loader::SalesforceSoap.new(object: :Contact, credentials: {}, operation: :not_defined)
+    expect { loader.load(data) }.to raise_error ArgumentError
+  end
+
+  it 'submits the right merge command' do
+    data = [
+      { Id: '1234', Custom__c: 'something', Merge_Id: '5678' }
+    ]
+
+    expect(soapforce_client).to receive(:merge) do
+      [
+        :Contact,
+        {
+          Id: '1234',
+          Custom__c: 'something'
+        },
+        ['5678']
+      ]
+    end
+
+    loader.load(data)
+  end
+
+  it 'submits a merge command for each row of data' do
+    data = [
+      { Id: '1', Custom__c: 'something', Merge_Id: '10' },
+      { Id: '2', Custom__c: 'something', Merge_Id: '20' }
+    ]
+
+    expect(soapforce_client).to receive(:merge).twice
+    loader.load(data)
+  end
+
+  it 'raises an error if the merge id field is not found' do
+    data = [
+      { Id: '1234', Custom__c: 'something', Alt_Merge_Id: '5678' }
+    ]
+
+    expect { loader.load(data) }.to raise_error KeyError
+  end
+
+end

data/spec/fixtures/empty.csv

@@ -0,0 +1 @@
+column A,column B

data/spec/fixtures/sf_bulk_helper_stubs.rb

@@ -17,6 +17,123 @@ module Remi::SfBulkHelperStubs
     EOT
   end
 
+  def delete_raw_result
+    {
+      "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
+      "id" => [
+        "750g0000004iys2AAA"
+      ],
+      "operation" => [
+        "delete"
+      ],
+      "object" => [
+        "Contact"
+      ],
+      "createdById" => [
+        "005A0000000eJ57IAE"
+      ],
+      "createdDate" => [
+        "2017-01-25T20:06:30.000Z"
+      ],
+      "systemModstamp" => [
+        "2017-01-25T20:06:30.000Z"
+      ],
+      "state" => [
+        "Closed"
+      ],
+      "concurrencyMode" => [
+        "Parallel"
+      ],
+      "contentType" => [
+        "XML"
+      ],
+      "numberBatchesQueued" => [
+        "1"
+      ],
+      "numberBatchesInProgress" => [
+        "0"
+      ],
+      "numberBatchesCompleted" => [
+        "0"
+      ],
+      "numberBatchesFailed" => [
+        "0"
+      ],
+      "numberBatchesTotal" => [
+        "1"
+      ],
+      "numberRecordsProcessed" => [
+        "0"
+      ],
+      "numberRetries" => [
+        "0"
+      ],
+      "apiVersion" => [
+        "32.0"
+      ],
+      "numberRecordsFailed" => [
+        "0"
+      ],
+      "totalProcessingTime" => [
+        "0"
+      ],
+      "apiActiveProcessingTime" => [
+        "0"
+      ],
+      "apexProcessingTime" => [
+        "0"
+      ],
+      "batches" => [
+        {
+          "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",
+          "id" => [
+            "751g0000002ozU5AAI"
+          ],
+          "jobId" => [
+            "750g0000004iys2AAA"
+          ],
+          "state" => [
+            "Completed"
+          ],
+          "createdDate" => [
+            "2017-01-25T20:06:31.000Z"
+          ],
+          "systemModstamp" => [
+            "2017-01-25T20:07:19.000Z"
+          ],
+          "numberRecordsProcessed" => [
+            "1"
+          ],
+          "numberRecordsFailed" => [
+            "0"
+          ],
+          "totalProcessingTime" => [
+            "684"
+          ],
+          "apiActiveProcessingTime" => [
+            "459"
+          ],
+          "apexProcessingTime" => [
+            "74"
+          ],
+          "response" => [
+            {
+              "id" => [
+                "003g000001LVMx3AAH"
+              ],
+              "success" => [
+                "true"
+              ],
+              "created" => [
+                "false"
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  end
+
   def contact_query_raw_result
     {
       "xmlns" => "http://www.force.com/2009/06/asyncapi/dataload",

data/spec/sf_bulk_helper_spec.rb

@@ -114,4 +114,19 @@ describe Remi::SfBulkHelper do
       end
     end
   end
+
+  describe SfBulkHelper::SfBulkDelete do
+    let(:sf_delete) { SfBulkHelper::SfBulkDelete.new({}, 'Contact', [{ 'Id' => '1234' }]) }
+    let(:sf_bulk) { double('sf_bulk') }
+
+    before do
+      allow(sf_delete).to receive(:sf_bulk) { sf_bulk }
+      allow(sf_bulk).to receive(:delete) { SfBulkHelperStubs.delete_raw_result }
+    end
+
+    it 'sends a delete request to the salesforce bulk api' do
+      expect(sf_bulk).to receive(:delete) { SfBulkHelperStubs.delete_raw_result }
+      sf_delete.send(:execute)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-25 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -283,6 +283,7 @@ files:
 - lib/remi/data_subjects/postgres.rb
 - lib/remi/data_subjects/s3_file.rb
 - lib/remi/data_subjects/salesforce.rb
+- lib/remi/data_subjects/salesforce_soap.rb
 - lib/remi/data_subjects/sftp_file.rb
 - lib/remi/data_subjects/sub_job.rb
 - lib/remi/dsl.rb
@@ -295,7 +296,6 @@ files:
 - lib/remi/job/sub_job.rb
 - lib/remi/job/transform.rb
 - lib/remi/loader.rb
-- lib/remi/monkeys/daru.rb
 - lib/remi/parser.rb
 - lib/remi/refinements/symbolizer.rb
 - lib/remi/settings.rb
@@ -317,6 +317,7 @@ files:
 - spec/data_subjects/none_spec.rb
 - spec/data_subjects/postgres_spec.rb
 - spec/data_subjects/s3_file_spec.rb
+- spec/data_subjects/salesforce_soap_spec.rb
 - spec/data_subjects/salesforce_spec.rb
 - spec/data_subjects/sftp_file_spec.rb
 - spec/data_subjects/sub_job_spec.rb
@@ -325,6 +326,7 @@ files:
 - spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
+- spec/fixtures/empty.csv
 - spec/fixtures/sf_bulk_helper_stubs.rb
 - spec/fixtures/unsupported_escape.csv
 - spec/job/transform_spec.rb
@@ -396,6 +398,7 @@ test_files:
 - spec/data_subjects/none_spec.rb
 - spec/data_subjects/postgres_spec.rb
 - spec/data_subjects/s3_file_spec.rb
+- spec/data_subjects/salesforce_soap_spec.rb
 - spec/data_subjects/salesforce_spec.rb
 - spec/data_subjects/sftp_file_spec.rb
 - spec/data_subjects/sub_job_spec.rb
@@ -404,6 +407,7 @@ test_files:
 - spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
+- spec/fixtures/empty.csv
 - spec/fixtures/sf_bulk_helper_stubs.rb
 - spec/fixtures/unsupported_escape.csv
 - spec/job/transform_spec.rb

data/lib/remi/monkeys/daru.rb

@@ -1,4 +0,0 @@
-# Needed to fix issue in Daru 0.1.4.1
-class Daru::DataFrame
-  remove_method :to_hash
-end