remi 0.2.27 → 0.2.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 699ffcfaee3db2ae6e0fc56d79458c55d2241405
-  data.tar.gz: 70b17f181c0090eefb842da7f71a66882376475a
+  metadata.gz: dc30c0a301e3be2a32b11febab47f55037a2d621
+  data.tar.gz: 2474212de553dafd2fb51ab30643ba65d30ec908
 SHA512:
-  metadata.gz: eea0b513471b2767dc4ac7053d8cb574bd747d5120b127f42d254952a166bf35e40ef52fa3861d8e4351ff95e1b967094f90f0860277174d0e52e235a819e606
-  data.tar.gz: c9b8b111b7f3fddf7ff4fcd1310ca6102dd027acaaa1a90d15cfd80e65c7784c21396fe8c4bcfbe1b2177080479d445f81ee6ecf2ddec24aeeb5e3df7602b0c9
+  metadata.gz: 50dea06798dd136475c4a0401be65d8b19d1f0b8c82b2841e9551f95de6fc31897c55e4be5b58065e8cd792d2bfd9e37bd24a2001eddfbea8da44428bb32b1f2
+  data.tar.gz: ba9d970c2eecd2aba4bbd5467b25ae60bbc6827fcd36b1b89bbaacc301125d7e09af583874a0df0e9172dce1228934a7eff70aab641e53036909a06e38ee5e70

data/Gemfile

@@ -2,3 +2,6 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'daru', '0.1.2', git: 'git@github.com:inside-track/daru.git', branch: 'itk-master'
+gem 'restforce', '~> 2.1'
+gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'

data/Gemfile.lock

@@ -1,11 +1,28 @@
+GIT
+  remote: git@github.com:inside-track/daru.git
+  revision: ad71b644c8e26789e3ab2258f046d5bc68b8df71
+  branch: itk-master
+  specs:
+    daru (0.1.2)
+      reportbuilder (~> 1.4)
+      spreadsheet (~> 1.1.1)
+
+GIT
+  remote: git@github.com:inside-track/salesforce_bulk_api.git
+  revision: f597f06e5aa0cec3f9072b23fa09769676ec73da
+  branch: master
+  specs:
+    salesforce_bulk_api (0.0.12)
+      json
+      xml-simple
+
 PATH
   remote: .
   specs:
-    remi (0.2.27)
+    remi (0.2.28)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
-      daru (= 0.1.2)
       docile (~> 1.1)
       faker (~> 1.6)
       net-sftp (~> 2.1)
@@ -37,14 +54,16 @@ GEM
     cucumber-core (1.4.0)
       gherkin (~> 3.2.0)
     cucumber-wire (0.0.1)
-    daru (0.1.2)
-      reportbuilder (~> 1.4)
-      spreadsheet (~> 1.1.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
     faker (1.6.3)
       i18n (~> 0.5)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
     gherkin (3.2.0)
+    hashie (3.4.3)
     i18n (0.7.0)
     iruby (0.2.7)
       bond (~> 0.5)
@@ -56,6 +75,7 @@ GEM
     minitest (5.8.4)
     multi_json (1.11.2)
     multi_test (0.1.2)
+    multipart-post (2.0.0)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
     net-ssh (3.1.1)
@@ -77,6 +97,11 @@ GEM
       prawn (~> 0.8.4)
       prawn-svg (~> 0.9.1)
       text-table (~> 1.2)
+    restforce (2.2.0)
+      faraday (~> 0.9.0)
+      faraday_middleware (>= 0.8.8)
+      hashie (>= 1.2.0, < 4.0)
+      json (>= 1.7.5, < 1.9.0)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
@@ -97,13 +122,17 @@ GEM
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
+    xml-simple (1.1.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  daru (= 0.1.2)!
   iruby (= 0.2.7)
   remi!
+  restforce (~> 2.1)
+  salesforce_bulk_api!
 
 BUNDLED WITH
    1.11.2

data/features/metadata.feature

@@ -0,0 +1,17 @@
+Feature: This tests the application of metadata.
+  We need some additional tests to check for errors in parsing.
+
+
+  Background:
+    Given the job is 'Metadata'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+    And the following example record for 'Source Data':
+      | activity_id | student_id | student_dob | activity_type | activity_counter | activity_score | activity_cost | activity_date     | source_filename |
+      |           1 |          1 |    3/3/1998 |             A |                1 |            3.8 |         12.23 | 1/3/2016 03:22:36 |         one.csv |
+
+  Scenario: Metadata is used to parse date fields
+    Then the target field 'student_dob' is set to the value "1998-03-03"

data/features/step_definitions/remi_step.rb

@@ -110,7 +110,7 @@ Given /^the (source|target) file contains at least the following headers in no p
     field = row.first
     step "the #{st} field '#{field}'"
   end
-  expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.field_names)
+  expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to include(*@brt.send(st.to_sym).fields.field_names)
 end
 
 Given /^the (source|target) file contains all of the following headers in this order:$/ do |st, table|
@@ -120,7 +120,7 @@ Given /^the (source|target) file contains all of the following headers in this o
   end
 
   @brt.run_transforms if st == 'target'
-  expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.field_names
+  expect(@brt.send(st.to_sym).data_subject.df.vectors.to_a).to eq @brt.send(st.to_sym).fields.field_names
 end
 
 
@@ -188,7 +188,7 @@ Given /^the source field '([^']+)' is parsed with the date format "([^"]*)"$/ do
   step "the source field '#{source_field}'"
 
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
-  expect(@brt.sources[source_name].fields[source_field_name].metadata[:format]).to eq date_format
+  expect(@brt.sources[source_name].fields[source_field_name].metadata[:in_format]).to eq date_format
 end
 
 Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
@@ -452,8 +452,8 @@ Then /^the target field '([^']+)' is populated from the source field '([^']+)' u
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
   target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
 
-  source_format = @brt.sources[source_name].fields[source_field_name].metadata[:format]
-  source_reformatted = Remi::Transform[:format_date].(from_fmt: source_format, to_fmt: target_format)
+  source_format = @brt.sources[source_name].fields[source_field_name].metadata[:in_format]
+  source_reformatted = Remi::Transform::FormatDate.new(in_format: source_format, out_format: target_format).to_proc
     .call(@brt.sources[source_name].fields[source_field_name].value)
 
   @brt.run_transforms
@@ -592,7 +592,7 @@ Then /^only the following fields should be present on the target:$/ do |table|
   end
 
   @brt.run_transforms
-  expect(@brt.target.data_obj.fields.keys).to match_array @brt.target.fields.field_names
+  expect(@brt.target.data_subject.df.vectors.to_a).to match_array @brt.target.fields.field_names
 end
 
 ### Record-level expectations

data/features/transforms/date_diff.feature

@@ -8,6 +8,7 @@ Feature: Tests the date_diff transform
     And the source 'Source Data'
     And the target 'Target Data'
 
+
   Scenario Outline: Calculating date difference in days2.
     Given the job parameter 'measure' is "days"
     And the source field 'Date1' has the value "<Date1>"

data/jobs/aggregate_job.rb

@@ -2,7 +2,6 @@ require_relative 'all_jobs_shared'
 
 class AggregateJob
   include AllJobsShared
-  using Remi::Refinements::Daru
 
   define_source :source_data, Remi::DataSource::DataFrame
   define_target :target_data, Remi::DataTarget::DataFrame

data/jobs/all_jobs_shared.rb

@@ -6,8 +6,6 @@ require 'remi'
 
 module AllJobsShared
   include Remi::Job
-  include Remi::Refinements::Daru
-  using Remi::Refinements::Daru
 
   define_param :sftp, {
     host: 'example.com',

data/jobs/copy_source_job.rb

@@ -2,7 +2,6 @@ require_relative 'all_jobs_shared'
 
 class CopySourceJob
   include AllJobsShared
-  using Remi::Refinements::Daru
 
   define_source :source_data, Remi::DataSource::DataFrame,
     fields: {

data/jobs/csv_file_target_job.rb

@@ -2,7 +2,6 @@ require_relative 'all_jobs_shared'
 
 class CsvFileTargetJob
   include AllJobsShared
-  using Remi::Refinements::Daru
 
   define_target :some_csv_file, Remi::DataTarget::CsvFile,
     path: "#{Remi::Settings.work_dir}/some_file.csv",

data/jobs/metadata_job.rb

@@ -0,0 +1,60 @@
+require_relative 'all_jobs_shared'
+
+class MetadataJob
+  include AllJobsShared
+
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :activity_id      => { from: 'in', in: true, cdc_type: 2 },
+      :student_id       => { from: 'in', in: true, type: :string, cdc_type: 2 },
+      :student_dob      => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
+      :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
+      :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
+      :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
+      :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 16, scale: 2, cdc_type: 2 },
+      :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
+      :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
+    }
+
+  define_target :target_data, Remi::DataTarget::CsvFile,
+    path: "#{Remi::Settings.work_dir}/target_data.csv",
+    fields: {
+      :activity_id      => { from: 'out', out: true },
+      :student_id       => { from: 'out', out: true, type: :string },
+      :student_dob      => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
+      :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
+      :activity_counter => { from: 'out', out: true, type: :integer },
+      :activity_score   => { from: 'out', out: true, type: :float },
+      :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 16, scale: 2 },
+      :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
+      :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
+    }
+
+  define_transform :main do
+
+=begin
+    source_data.df = Remi::DataFrame.daru([
+      ['1','1','3/3/1998','A','1','3.8','12.23','1/3/2016 03:22:36','one.csv'],
+      ['2','1','3/3/1998','B','3','4.2','10.53','1/3/2016 03:58:22','one.csv'],
+      ['2','1','','B','2','4.23','10.539','1/3/2016 03:58:22','one.csv']
+    ].transpose, order: [
+      :activity_id,
+      :student_id,
+      :student_dob,
+      :activity_type,
+      :activity_counter,
+      :activity_score,
+      :activity_cost,
+      :activity_date,
+      :source_filename
+    ])
+=end
+
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields) do
+      target_data.fields.keys.each do |field|
+        map source(field) .target(field)
+          .transform(Remi::Transform::EnforceType.new)
+      end
+    end
+  end
+end

data/jobs/parameters_job.rb

@@ -14,7 +14,7 @@ class ParametersJob
   define_transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(nil) .target(:myparam)
-        .transform(Remi::Transform[:constant].(params[:myparam]))
+        .transform(Remi::Transform::Constant.new(params[:myparam]))
       map source(:parameter_name) .target(:parameter_name)
         .transform(->(v) { params[v.to_sym] })
     end

data/jobs/sample_job.rb

@@ -1,10 +1,9 @@
 # This is an example Remi job that was auto-generated by Remi.
-
 require_relative 'all_jobs_shared'
+require 'remi/data_subject/salesforce'
 
 class SampleJob
   include AllJobsShared
-  using Remi::Refinements::Daru
 
   define_source :existing_contacts, Remi::DataSource::Salesforce,
     object: :Contact,
@@ -14,7 +13,7 @@ class SampleJob
       :Id             => {},
       :External_ID__c => {},
       :IsActive       => { type: :boolean },
-      :CreatedDate    => { type: :date, format: '%Y-%m-%d %H:%M:%S' }
+      :CreatedDate    => { type: :date, in_format: '%Y-%m-%d %H:%M:%S' }
     },
     query: <<-EOQ
       SELECT
@@ -50,8 +49,8 @@ class SampleJob
       :mailing_city           => {},
       :mailing_state          => {},
       :mailing_postal_code    => {},
-      :birthdate              => { type: :date, format: '%m/%d/%Y'},
-      :applied_date           => { type: :date, format: '%m/%d/%Y'}
+      :birthdate              => { type: :date, in_format: '%m/%d/%Y'},
+      :applied_date           => { type: :date, in_format: '%m/%d/%Y'}
     }
 
   define_target :all_contacts, Remi::DataTarget::DataFrame
@@ -83,7 +82,7 @@ class SampleJob
     all_contacts.df = sample_file.df.dup
     Remi::SourceToTargetMap.apply(all_contacts.df) do
       map source(:program) .target(:Major__c)
-        .transform(Remi::Transform[:lookup][params[:program_name_lookup]])
+        .transform(Remi::Transform::Lookup.new(params[:program_name_lookup]))
     end
     all_contacts.df = all_contacts.df.where(all_contacts.df[:Major__c].not_eq(nil))
 
@@ -94,8 +93,8 @@ class SampleJob
 
       # Prefixes source id record and then looks up existing salesforce Id
       map source(:student_id) .target(:External_ID__c, :Id)
-        .transform(Remi::Transform[:prefix]['SAMP'])
-        .transform(->(v) { [v, Remi::Transform[:lookup][student_id_to_sf_id].call(v)] })
+        .transform(Remi::Transform::Prefix.new('SAMP'))
+        .transform(->(v) { [v, Remi::Transform::Lookup.new(student_id_to_sf_id).call(v)] })
     end
   end
 
@@ -108,36 +107,36 @@ class SampleJob
       map source(:school_id)           .target(:School_ID__c)
       map source(:school_name)         .target(:School_Name__c)
       map source(:first_name)          .target(:FirstName)
-        .transform(Remi::Transform[:ifblank].('Not Provided'))
+        .transform(Remi::Transform::IfBlank.new('Not Provided'))
       map source(:last_name)           .target(:LastName)
-        .transform(Remi::Transform[:ifblank].('Not Provided'))
+        .transform(Remi::Transform::IfBlank.new('Not Provided'))
       map source(:mailing_city)        .target(:MailingCity)
       map source(:mailing_state)       .target(:MailingState)
       map source(:mailing_postal_code) .target(:MailingPostalCode)
 
       map source(:birthdate)                  .target(:Birthdate)
-        .transform(Remi::Transform[:format_date][from_fmt: sample_file.fields[:birthdate][:format]])
+        .transform(Remi::Transform::FormatDate.new(in_format: sample_file.fields[:birthdate][:in_format]))
 
       map source(:applied_date)              .target(:Applied_Date__c)
-        .transform(Remi::Transform[:ifblank].(Date.today.strftime(sample_file.fields[:applied_date][:format])))
-        .transform(Remi::Transform[:format_date].(from_fmt: sample_file.fields[:applied_date][:format]))
+        .transform(Remi::Transform::IfBlank.new(Date.today.strftime(sample_file.fields[:applied_date][:in_format])))
+        .transform(Remi::Transform::FormatDate.new(in_format: sample_file.fields[:applied_date][:in_format]))
 
       map source(:mailing_address_line_1, :mailing_address_line_2) .target(:MailingStreet)
         .transform(->(line_1, line_2) {
-           Remi::Transform[:ifblank].(nil).call(line_1).nil? ? [] : [line_1, line_2]
+           Remi::Transform::IfBlank.new(nil).call(line_1).nil? ? [] : [line_1, line_2]
            })
-        .transform(Remi::Transform[:concatenate].(', '))
+        .transform(Remi::Transform::Concatenate.new(', '))
 
       map source(:school_id, :school_name) .target(:School__c)
         .transform(->(id, name) {[
-            Remi::Transform[:ifblank]["Unknown"].call(id),
-            Remi::Transform[:ifblank]["Unknown"].call(name)
+            Remi::Transform::IfBlank.new("Unknown").call(id),
+            Remi::Transform::IfBlank.new("Unknown").call(name)
           ]})
-        .transform(Remi::Transform[:concatenate].('-'))
+        .transform(Remi::Transform::Concatenate.new('-'))
 
       map source(:current_email)       .target(:Email)
-        .transform(Remi::Transform[:replace].(/,/, '.'))
-        .transform(Remi::Transform[:validate_email].call)
+        .transform(Remi::Transform::Replace.new(/,/, '.'))
+        .transform(Remi::Transform::ValidateEmail.new)
     end
 
     contact_creates.df = work_contact_creates[

data/jobs/sftp_file_target_job.rb

@@ -2,7 +2,6 @@ require_relative 'all_jobs_shared'
 
 class SftpFileTargetJob
   include AllJobsShared
-  using Remi::Refinements::Daru
 
 
   define_target :some_file, Remi::DataTarget::SftpFile,

data/jobs/transforms/date_diff_job.rb

@@ -15,7 +15,7 @@ class DateDiffJob
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:date1, :date2) .target(:difference)
         .transform(->(d1,d2) { [Date.strptime(d1), Date.strptime(d2)] })
-        .transform(Remi::Transform[:date_diff].(params[:measure]))
+        .transform(Remi::Transform::DateDiff.new(params[:measure]))
     end
   end
 end

data/jobs/transforms/nvl_job.rb

@@ -15,7 +15,7 @@ class NvlJob
   define_transform :main, sources: :source_data, targets: :target_data do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:field1, :field2, :field3) .target(:result_field)
-        .transform(Remi::Transform[:nvl].(params[:default]))
+        .transform(Remi::Transform::Nvl.new(params[:default]))
     end
   end
 end

data/jobs/transforms/parse_date_job.rb

@@ -7,19 +7,22 @@ class ParseDateJob
   define_param :if_blank, nil
   define_source :source_data, Remi::DataSource::DataFrame,
     fields: {
-      :date_string  => { type: :date, format: params[:format] },
-      :stubbed_date => { type: :date, format: params[:format] }
+      :date_string  => { type: :date, in_format: params[:format] },
+      :stubbed_date => { type: :date, in_format: params[:format] }
     }
   define_target :target_data, Remi::DataTarget::DataFrame
 
   define_transform :main, sources: :source_data, targets: :target_data do
 
+    # Only needed for testing, would be nice to make it testable without this
+    params[:if_blank] = ['high', 'low'].include?(params[:if_blank]) ? params[:if_blank].to_sym : params[:if_blank]
+
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:date_string) .target(:parsed_date)
-        .transform(Remi::Transform[:parse_date].(format: params[:format], if_blank: params[:if_blank]))
+        .transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
 
       map source(:stubbed_date) .target(:parsed_stubbed_date)
-        .transform(Remi::Transform[:parse_date].(format: params[:format], if_blank: params[:if_blank]))
+        .transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
     end
   end
 end

data/jobs/transforms/prefix_job.rb

@@ -13,7 +13,7 @@ class PrefixJob
   define_transform :main, sources: :source_data, targets: :target_data do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:my_field) .target(:prefixed_field)
-        .transform(Remi::Transform[:prefix].(params[:prefix]))
+        .transform(Remi::Transform::Prefix.new(params[:prefix]))
     end
   end
 end