remi 0.2.42 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 078d9d338ea606fee80194b60e6bd52eeadd1211
-  data.tar.gz: 05ca9a6b08896ad3266fbfcd61932a71e08c5318
+  metadata.gz: 71164673ece850e218a1ef18a72aa02f4ca5d065
+  data.tar.gz: b52e6e86bc4995f2caf79dad79a7f0a195cdb9f9
 SHA512:
-  metadata.gz: 454633037cc429d0ad056e150ec8f07aa2f064f62bf9ee08e3c85db52f31fd92de59c3dd1d3e3dae27d19163c3146619dd340b751ee70af3e0b98176a3d37611
-  data.tar.gz: 82a6aef7e5bcece2f2012b2b76f120074adfd3387173d18d1bf4a11aa8d4d6e6994e292da1287a90aab91c01367f7e7dac674111b20c6aaa7b75e4a672e0290c
+  metadata.gz: 708b6fdf566acb525caa85bc038685bad46c3999f65045b5281c9393d731f5a3cdb49ac30a7ecde12f16abab5a27364449a83fc840adfa531e2a04a783fcd69a
+  data.tar.gz: 48730dd300bb84dadb93d911e37baa274958d0d1deaddcbc769aa43a7f7c1887ffbcf025ce43d4188e09630de31868acc6315ae2559dbd80853febc42e13b5d7

data/.yardopts

@@ -0,0 +1,7 @@
+--title 'Remi Documentation'
+
+--no-private
+--main=README.md
+
+--markup-provider=redcarpet
+--markup=markdown

data/Gemfile

@@ -2,7 +2,7 @@
 source 'https://rubygems.org'
 
 gemspec
-gem 'daru', '0.1.2', git: 'git@github.com:inside-track/daru.git', branch: 'itk-master'
+gem 'daru', '0.1.4.1', git: 'git@github.com:inside-track/daru.git', branch: '0.1.4.1-Remi'
 gem 'restforce', '~> 2.1'
 gem 'salesforce_bulk_api', git: 'git@github.com:inside-track/salesforce_bulk_api.git', branch: 'master'
 gem 'aws-sdk', '~> 2.3'

data/Gemfile.lock

@@ -1,11 +1,10 @@
 GIT
   remote: git@github.com:inside-track/daru.git
-  revision: 80b8024102bdb31ef402cb8d6c5cef86dd31cd41
-  branch: itk-master
+  revision: e38e98890c99dee03ceedaca3a5faf40859042ab
+  branch: 0.1.4.1-Remi
   specs:
-    daru (0.1.2)
-      reportbuilder (~> 1.4)
-      spreadsheet (~> 1.1.1)
+    daru (0.1.4.1)
+      backports
 
 GIT
   remote: git@github.com:inside-track/salesforce_bulk_api.git
@@ -19,7 +18,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.42)
+    remi (0.3.0)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
@@ -46,9 +45,9 @@ GEM
       jmespath (~> 1.0)
     aws-sdk-resources (2.3.5)
       aws-sdk-core (= 2.3.5)
+    backports (3.6.8)
     bond (0.5.1)
     builder (3.2.2)
-    clbustos-rtf (0.4.2)
     cucumber (2.3.3)
       builder (>= 2.1.2)
       cucumber-core (~> 1.4.0)
@@ -69,6 +68,7 @@ GEM
     faraday_middleware (0.10.0)
       faraday (>= 0.7.4, < 0.10)
     gherkin (3.2.0)
+    github-markup (1.4.0)
     hashie (3.4.3)
     i18n (0.7.0)
     iruby (0.2.7)
@@ -89,23 +89,10 @@ GEM
       net-ssh (>= 2.6.5)
     net-ssh (3.1.1)
     pg (0.18.4)
-    prawn (0.8.4)
-      prawn-core (>= 0.8.4, < 0.9)
-      prawn-layout (>= 0.8.4, < 0.9)
-      prawn-security (>= 0.8.4, < 0.9)
-    prawn-core (0.8.4)
-    prawn-layout (0.8.4)
-    prawn-security (0.8.4)
-    prawn-svg (0.9.1.11)
-      prawn (>= 0.8.4)
     rbczmq (1.7.9)
+    redcarpet (3.3.4)
     regex_sieve (0.1.0)
     regexp-examples (1.2.0)
-    reportbuilder (1.4.2)
-      clbustos-rtf (~> 0.4.0)
-      prawn (~> 0.8.4)
-      prawn-svg (~> 0.9.1)
-      text-table (~> 1.2)
     restforce (2.2.0)
       faraday (~> 0.9.0)
       faraday_middleware (>= 0.8.8)
@@ -124,25 +111,25 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
     rspec-support (3.4.1)
-    ruby-ole (1.2.12)
-    spreadsheet (1.1.2)
-      ruby-ole (>= 1.0)
-    text-table (1.2.4)
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     xml-simple (1.1.5)
+    yard (0.9.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   aws-sdk (~> 2.3)
-  daru (= 0.1.2)!
+  daru (= 0.1.4.1)!
+  github-markup
   iruby (= 0.2.7)
+  redcarpet
   remi!
   restforce (~> 2.1)
   salesforce_bulk_api!
+  yard
 
 BUNDLED WITH
    1.11.2

data/README.md

@@ -110,7 +110,7 @@ Remi::SourceToTargetMap.apply(df) do
   map source(:a, :b) .target(:aprime, :ab)
     .transform(->(row) {
       row[:aprime] = row[:a]
-      row[:ab] = "#{row[:a]}#{row[:b]}" })
+      row[:ab] = "#{row[:a]}#{row[:b]}"
     })
 end
 

data/features/step_definitions/remi_step.rb

@@ -44,57 +44,65 @@ end
 ### Source file processing
 
 Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
-  expect(@brt.source.data_subject.extractor.pattern).to eq Regexp.new(pattern)
+  expect(@brt.source.data_subject.extractors.map(&:pattern)).to include Regexp.new(pattern)
 end
 
 Given /^download groups defined by the pattern \/(.*)\/$/ do |pattern|
-  expect(@brt.source.data_subject.extractor.group_by).to eq Regexp.new(pattern)
+  expect(@brt.source.data_subject.extractors.map(&:group_by)).to include Regexp.new(pattern)
 end
 
 Then /^the file with the latest date stamp will be downloaded for processing$/ do
-  expect(@brt.source.data_subject.extractor.most_recent_by).to eq :create_time
+  expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :create_time
 end
 
 Then /^all files matching the pattern will be downloaded for processing$/ do
-  expect(@brt.source.data_subject.extractor.most_recent_only).to eq false
+  expect(@brt.source.data_subject.extractors.map(&:most_recent_only)).not_to include eq true
 end
 
 Then /^the file that comes last in an alphanumeric sort by group will be downloaded for processing$/ do
-  expect(@brt.source.data_subject.extractor.most_recent_by).to eq :name
+  expect(@brt.source.data_subject.extractors.map(&:most_recent_by)).to include :name
 end
 
 Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
-  expect(@brt.target.get_attrib(:remote_path)).to eq Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
+  expected_path = Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
+  expect(@brt.target.data_subject.loaders.map(&:remote_path)).to include expected_path
 end
 
 ## CSV Options
 
 Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
-  expect(@brt.send(st.to_sym).csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
 end
 
 Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, encoding|
-  expect(@brt.send(st.to_sym).csv_options[:encoding].split(':').first).to eq encoding
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:encoding].split(':').first).to eq encoding
 end
 
 Given /^the (source|target) file uses a ([\w ]+) to quote embedded delimiters$/ do |st, quote_char|
-  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
 end
 
 Given /^the (source|target) file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |st, escape_char|
-  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
 end
 
 Given /^the (source|target) file uses ([\w ]+) line endings$/ do |st, line_endings|
-  expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
 end
 
 Given /^the (source|target) file uses "([^"]+)" as a record separator$/ do |st, line_endings|
-  expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq line_endings.gsub(/\\n/, "\n").gsub(/\\r/, "\r")
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:row_sep]).to eq line_endings.gsub(/\\n/, "\n").gsub(/\\r/, "\r")
 end
 
 Given /^the (source|target) file (contains|does not contain) a header row$/ do |st, header|
-  expect(@brt.send(st.to_sym).csv_options[:headers]).to eq (header == 'contains')
+  csv_subject = @brt.send(st.to_sym).data_subject.send(st == 'source' ? :parser : :encoder)
+  expect(csv_subject.csv_options[:headers]).to eq (header == 'contains')
 end
 
 Given /^the (source|target) file contains at least the following headers in no particular order:$/ do |st, table|
@@ -301,6 +309,12 @@ Then /^the target '(.+)' should match the example '([[:alnum:]\-\s]+)'$/ do |tar
   target_hash = @brt.targets[target_name].column_hash
   example_hash = @brt.examples[example_name].column_hash
   common_keys = target_hash.keys & example_hash.keys
+  expect(common_keys).to match_array(example_hash.keys), <<-EOT
+    Fields in example not found in target
+    Example fields: #{example_hash.keys}
+    Target fields: #{target_hash.keys}
+    Missing fields: #{example_hash.keys - target_hash.keys}
+  EOT
 
   target_to_compare = target_hash.select { |k,v| common_keys.include? k }
   target_to_compare.each do |k, v|
@@ -317,6 +331,12 @@ Then /^the target should match the example '([[:alnum:]\-\s]+)'$/ do |example_na
   step "the target '#{target_name}' should match the example '#{example_name}'"
 end
 
+Then /^the target '(.+)' should match the example:/ do |target_name, example_table|
+  example_name = SecureRandom.uuid
+  @brt.add_example example_name, example_table
+  step "the target '#{target_name}' should match the example '#{example_name}'"
+end
+
 Then /^the target should match the example:/ do |example_table|
   example_name = SecureRandom.uuid
   @brt.add_example example_name, example_table

data/features/sub_job_example.feature

@@ -0,0 +1,24 @@
+Feature: This is a job that demonstrates how to use Remi subjobs
+
+  Background:
+    Given the job is 'Sub Job Example'
+    And the job target 'Just Beers'
+    And the job target 'Zombified Beers'
+
+  Scenario: Sub jobs can be used to extract data
+
+    Given the target 'Just Beers'
+
+    Then the target should match the example:
+      | Brewer  | Style |
+      | Baerlic | IPA   |
+      | Ex Novo | Red   |
+
+  Scenario: Sub jobs can be used to transform source data
+
+    Given the target 'Zombified Beers'
+
+    Then the target should match the example:
+      | Brewer         | Style      |
+      | Zombie Baerlic | Zombie IPA |
+      | Zombie Ex Novo | Zombie Red |

data/features/sub_transform_example.feature

@@ -0,0 +1,35 @@
+Feature: This is a job that demonstrates how to use Remi sub transforms
+
+  Background:
+    Given the job is 'Sub Transform Example'
+    And the job source 'My Source'
+    And the job target 'My Target'
+
+  Scenario: It uses the subtransform to prefix a field with the default value
+
+    Given the source 'My Source'
+    And the target 'My Target'
+
+    And the following example for 'My Source':
+      | id |
+      | 1  |
+
+    Then the target should match the example:
+      | id | default_id |
+      | 1  | DEFAULT1   |
+
+
+  Scenario: It uses parameters to customize subtansforms
+
+    Given the source 'My Source'
+    And the target 'My Target'
+
+    And the job parameter 'job_prefix' is "UC"
+
+    And the following example for 'My Source':
+      | id |
+      | 1  |
+
+    Then the target should match the example:
+      | id | default_id |
+      | 1  | UC1        |

data/features/sub_transform_many_to_many.feature

@@ -0,0 +1,49 @@
+Feature: This is a job that demonstrates how to use Remi sub transforms
+
+  Background:
+    Given the job is 'Sub Transform Many To Many'
+    And the job source 'Beer Fact'
+    And the job source 'Beer Dim'
+    And the job target 'Beer Count'
+    And the job target 'Style Count'
+
+
+  Scenario: It runs the subtransform as expected
+
+    Given the source 'Beer Fact'
+    And the source 'Beer Dim'
+    And the target 'Beer Count'
+    And the target 'Style Count'
+
+    And the following example for 'Beer Fact':
+      | fact_sk | beer_sk |
+      | 1       | 1       |
+      | 2       | 1       |
+      | 3       | 1       |
+      | 4       | 2       |
+      | 5       | 2       |
+      | 6       | 2       |
+      | 7       | 3       |
+      | 8       | 3       |
+      | 9       | 3       |
+      | 10      | 4       |
+
+    And the following example for 'Beer Dim':
+      | beer_sk | name            | style     |
+      | 1       | Invincible      | IPA       |
+      | 2       | Pipewrench      | IPA       |
+      | 3       | Altera          | Red       |
+      | 4       | Urban Farmhouse | Farmhouse |
+
+    Then the target 'Beer Count' should match the example:
+      | name            | count |
+      | Altera          | 3     |
+      | Invincible      | 3     |
+      | Pipewrench      | 3     |
+      | Urban Farmhouse | 1     |
+
+    And the target 'Style Count' should match the example:
+      | style | count |
+      | Farmhouse | 1 |
+      | IPA       | 6 |
+      | Red       | 3 |

data/features/support/env_app.rb

@@ -1 +1 @@
-Dir["#{File.dirname(__FILE__)}/../../jobs/**/*_job.rb"].each { |f| require(f) }
+# Put stuff here that is specific to your testing environment

data/jobs/all_jobs_shared.rb

@@ -4,22 +4,25 @@
 require 'bundler/setup'
 require 'remi'
 
-module AllJobsShared
-  include Remi::Job
+class AllJobsShared < Remi::Job
 
-  define_param :sftp, {
-    host: 'example.com',
-    username: 'user',
-    password: '1234567890'
-  }
+  param :sftp do
+    {
+      host: 'example.com',
+      username: 'user',
+      password: '1234567890'
+    }
+  end
 
-  define_param :salesforce_credentials, {
-    host: 'login.salesforce.com',
-    username: 'user@example.com',
-    password: 'password',
-    security_token: '4342jn3j4n32n4',
-    client_id: 'dkfjsdkfjoasdjdf',
-    client_secret: '28357245723475',
-    instance_url: 'https://na1.salesforce.com'
-  }
+  param :salesforce_credentials do
+    {
+      host: 'login.salesforce.com',
+      username: 'user@example.com',
+      password: 'password',
+      security_token: '4342jn3j4n32n4',
+      client_id: 'dkfjsdkfjoasdjdf',
+      client_secret: '28357245723475',
+      instance_url: 'https://na1.salesforce.com'
+    }
+  end
 end

data/jobs/copy_source_job.rb

@@ -1,16 +1,18 @@
 require_relative 'all_jobs_shared'
 
-class CopySourceJob
-  include AllJobsShared
+class CopySourceJob < Remi::Job
+  source :source_data do
+    fields(
+      {
+        :some_field => {},
+        :some_date => { type: :date, format: '%Y-%m-%d' }
+      }
+    )
+  end
 
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :some_field => {},
-      :some_date => { type: :date, format: '%Y-%m-%d' }
-    }
-  define_source :target_data, Remi::DataSource::DataFrame
+  target :target_data
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  transform :main do
     target_data.df = source_data.df.dup
   end
 end

data/jobs/csv_file_target_job.rb

@@ -1,15 +1,16 @@
 require_relative 'all_jobs_shared'
 
-class CsvFileTargetJob
-  include AllJobsShared
-
-  define_target :some_csv_file, Remi::DataTarget::CsvFile,
-    path: "#{Remi::Settings.work_dir}/some_file.csv",
-    csv_options: {
-      col_sep: '|'
-    }
+class CsvFileTargetJob < Remi::Job
+  target :some_csv_file do
+    encoder Remi::Encoder::CsvFile.new(
+      path: "#{Remi::Settings.work_dir}/some_file.csv",
+      csv_options: {
+        col_sep: '|'
+      }
+    )
+  end
 
-  define_transform :main do
+  transform :main do
     some_csv_file.df = Daru::DataFrame.new({
       col3: Faker::Hipster.words(10),
       col1: Faker::Hipster.words(10),