remi 0.2.42 → 0.3.0

data/spec/loader_spec.rb

@@ -0,0 +1,11 @@
+require_relative 'remi_spec'
+
+describe Remi::Loader do
+  let(:loader) { Loader.new }
+
+  context '#load' do
+    it 'has a load method' do
+      expect(loader).respond_to? :load
+    end
+  end
+end

data/spec/parser_spec.rb

@@ -0,0 +1,38 @@
+require_relative 'remi_spec'
+
+describe Remi::Parser do
+  let(:field_symbolizer) { double('field_symbolizer') }
+  let(:context) { double('context') }
+  let(:fields) { double('fields') }
+  let(:parser) { Parser.new(context: context, fields: fields, field_symbolizer: field_symbolizer) }
+
+  context '#parse' do
+    it 'has a parse method' do
+      expect(parser).respond_to? :parse
+    end
+  end
+
+  context '#field_symbolizer' do
+    it 'can be set in the constructor' do
+      expect(parser.field_symbolizer).to eq field_symbolizer
+    end
+
+    it 'the field_symbolizer defined in the context takes priority' do
+      symbolizer_from_context = double('symbolizer_from_context')
+      allow(context).to receive(:field_symbolizer) { symbolizer_from_context }
+      expect(parser.field_symbolizer).to eq symbolizer_from_context
+    end
+  end
+
+  context '#fields' do
+    it 'can be set in the constructor' do
+      expect(parser.fields).to eq fields
+    end
+
+    it 'the field_symbolizer defined in the context takes priority' do
+      fields_from_context = double('fields_from_context')
+      allow(context).to receive(:fields) { fields_from_context }
+      expect(parser.fields).to eq fields_from_context
+    end
+  end
+end

data/spec/sf_bulk_helper_spec.rb

@@ -0,0 +1,117 @@
+require_relative 'remi_spec'
+require 'remi/data_subjects/salesforce'
+require_relative 'fixtures/sf_bulk_helper_stubs'
+
+describe Remi::SfBulkHelper do
+
+  describe SfBulkHelper::SfBulkQuery do
+    before do
+      @sf_query = SfBulkHelper::SfBulkQuery.new({}, 'Contact', SfBulkHelperStubs.contact_query)
+      allow(@sf_query).to receive(:send_bulk_operation) { SfBulkHelperStubs.contact_query_raw_result }
+    end
+
+
+    describe 'info' do
+      it 'contains the bulk job id' do
+        expect(@sf_query.info['id']).to eq SfBulkHelperStubs.contact_query_raw_result['id']
+      end
+
+      it 'contains the query submitted' do
+        expect(@sf_query.info['query']).to eq SfBulkHelperStubs.contact_query
+      end
+    end
+
+    describe 'collected result' do
+      it 'returns the number of records specified in info' do
+        expect(@sf_query.result.size).to eq @sf_query.info['numberRecordsProcessed'].first.to_i
+      end
+
+      it 'has elements composed of hashes' do
+        expect(@sf_query.result.first).to be_a Hash
+      end
+
+      it 'has hash values that are not arrays' do
+        expect(@sf_query.result.first['Id']).to be_a String
+      end
+    end
+
+    describe 'as_lookup' do
+      it 'creates a row lookup based on a key' do
+        some_row = @sf_query.result[2]
+        expect(@sf_query.as_lookup(key: 'Id')[some_row['Id']]).to eq some_row
+      end
+
+      context 'with duplicates' do
+        before do
+          allow(@sf_query).to receive(:result) { SfBulkHelperStubs.contact_query_duplicated_result }
+        end
+
+        it 'returns the first record when duplicates are allowed' do
+          first_duplicated_row = @sf_query.result[0]
+          expect(@sf_query.as_lookup(key: 'Student_ID__c', duplicates: true)[first_duplicated_row['Student_ID__c']]).to eq first_duplicated_row
+        end
+
+        it 'raises an error when duplicates are not allowed' do
+          expect { @sf_query.as_lookup(key: 'Student_ID__c') }.to raise_error SfBulkHelper::DupeLookupKeyError
+        end
+      end
+    end
+
+    it 'can query for realz', skip: 'Skip live test' do
+      sf_client = SfConnection.new
+      q = SfBulkHelper::SfBulkQuery.new(sf_client.client, 'Contact', SfBulkHelperStubs.contact_query)
+      puts JSON.pretty_generate(q.raw_result)
+    end
+  end
+
+
+
+  describe SfBulkHelper::SfBulkUpdate do
+    before do
+      @sf_update = SfBulkHelper::SfBulkUpdate.new({}, 'Contact', SfBulkHelperStubs.contact_update_data, max_attempts: 3)
+      allow(@sf_update).to receive(:send_bulk_operation) { SfBulkHelperStubs.contact_update_raw_result }
+    end
+
+    describe 'collected result' do
+      it 'indicates whether each record was successful' do
+        expect(@sf_update.result.map { |r| r.has_key? 'success' }.uniq).to match_array([true])
+      end
+
+      it 'indicates whether each record was created' do
+        expect(@sf_update.result.map { |r| r.has_key? 'created' }.uniq).to match_array([true])
+      end
+    end
+
+    describe 'retrying failed records' do
+      context 'when the retry fails' do
+        before do
+          allow(@sf_update).to receive(:send_bulk_operation) { SfBulkHelperStubs.contact_update_with_fail_raw_result }
+        end
+
+        it 'raises an error' do
+          expect { @sf_update.result }.to raise_error(SfBulkHelper::MaxAttemptError)
+        end
+
+        it 'includes only the failed records in the retry attempt' do
+          @sf_update.result rescue SfBulkHelper::MaxAttemptError
+          retry_data = @sf_update.instance_eval('@data')
+          expect(retry_data.size).to eq SfBulkHelperStubs.contact_update_with_fail_raw_result['numberRecordsFailed'].first.to_i
+        end
+      end
+
+      context 'when the retry is successful' do
+        before do
+          allow(@sf_update).to receive(:send_bulk_operation).and_return(
+            SfBulkHelperStubs.contact_update_with_fail_raw_result,
+            SfBulkHelperStubs.contact_update_subsequent_success_raw_result
+          )
+        end
+
+        it 'stops retrying after success' do
+          @sf_update.result
+          expect(@sf_update.instance_eval('@attempts[:total]')).to eq 2
+        end
+      end
+    end
+  end
+end

data/spec/testing/data_stub_spec.rb

@@ -137,11 +137,13 @@ describe Testing::DataStub do
 
 
   context 'stubbed dataframe data' do
-    let(:stub_tester) do
-      StubTester.new(fields: {
+    let(:stub_tester) { StubTester.new }
+
+    before do
+      stub_tester.fields = {
         my_date: { type: :date, in_format: '%m/%d/%Y' },
         my_str: {}
-      })
+      }
     end
 
     context '#empty_stub_df' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.42
+  version: 0.3.0
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-08 00:00:00.000000000 Z
+date: 2016-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -164,6 +164,48 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: github-markup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Data manipulation and ETL in Ruby
 email:
 - sterling.paramore@insidetrack.com
@@ -176,13 +218,13 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"
+- ".yardopts"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - bin/remi
 - doc/install-rbenv-os_x.md
-- features/aggregate.feature
 - features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
@@ -192,6 +234,9 @@ files:
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb
+- features/sub_job_example.feature
+- features/sub_transform_example.feature
+- features/sub_transform_many_to_many.feature
 - features/support/env.rb
 - features/support/env_app.rb
 - features/transforms/concatenate.feature
@@ -203,7 +248,6 @@ files:
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
 - features/transforms/truthy.feature
-- jobs/aggregate_job.rb
 - jobs/all_jobs_shared.rb
 - jobs/copy_source_job.rb
 - jobs/csv_file_target_job.rb
@@ -212,6 +256,9 @@ files:
 - jobs/parameters_job.rb
 - jobs/sample_job.rb
 - jobs/sftp_file_target_job.rb
+- jobs/sub_job_example_job.rb
+- jobs/sub_transform_example_job.rb
+- jobs/sub_transform_many_to_many_job.rb
 - jobs/transforms/concatenate_job.rb
 - jobs/transforms/data_frame_sieve_job.rb
 - jobs/transforms/date_diff_job.rb
@@ -219,7 +266,6 @@ files:
 - jobs/transforms/parse_date_job.rb
 - jobs/transforms/partitioner_job.rb
 - jobs/transforms/prefix_job.rb
-- jobs/transforms/transform_jobs.rb
 - jobs/transforms/truncate_job.rb
 - jobs/transforms/truthy_job.rb
 - lib/remi.rb
@@ -228,18 +274,28 @@ files:
 - lib/remi/data_frame.rb
 - lib/remi/data_frame/daru.rb
 - lib/remi/data_subject.rb
-- lib/remi/data_subject/csv_file.rb
-- lib/remi/data_subject/data_frame.rb
-- lib/remi/data_subject/postgres.rb
-- lib/remi/data_subject/salesforce.rb
-- lib/remi/data_subject/sftp_file.rb
-- lib/remi/extractor/file_system.rb
-- lib/remi/extractor/local_file.rb
-- lib/remi/extractor/s3_file.rb
-- lib/remi/extractor/sftp_file.rb
+- lib/remi/data_subjects/csv_file.rb
+- lib/remi/data_subjects/data_frame.rb
+- lib/remi/data_subjects/file_system.rb
+- lib/remi/data_subjects/local_file.rb
+- lib/remi/data_subjects/none.rb
+- lib/remi/data_subjects/postgres.rb
+- lib/remi/data_subjects/s3_file.rb
+- lib/remi/data_subjects/salesforce.rb
+- lib/remi/data_subjects/sftp_file.rb
+- lib/remi/data_subjects/sub_job.rb
+- lib/remi/dsl.rb
+- lib/remi/encoder.rb
+- lib/remi/extractor.rb
 - lib/remi/field_symbolizers.rb
 - lib/remi/fields.rb
 - lib/remi/job.rb
+- lib/remi/job/parameters.rb
+- lib/remi/job/sub_job.rb
+- lib/remi/job/transform.rb
+- lib/remi/loader.rb
+- lib/remi/monkeys/daru.rb
+- lib/remi/parser.rb
 - lib/remi/refinements/symbolizer.rb
 - lib/remi/settings.rb
 - lib/remi/sf_bulk_helper.rb
@@ -251,19 +307,31 @@ files:
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
-- spec/data_subject/csv_file_spec.rb
-- spec/data_subject/data_frame.rb
 - spec/data_subject_spec.rb
-- spec/extractor/file_system_spec.rb
-- spec/extractor/local_file_spec.rb
-- spec/extractor/s3_file_spec.rb
-- spec/extractor/sftp_file_spec.rb
+- spec/data_subjects/csv_file_spec.rb
+- spec/data_subjects/data_frame_spec.rb
+- spec/data_subjects/file_system_spec.rb
+- spec/data_subjects/local_file_spec.rb
+- spec/data_subjects/none_spec.rb
+- spec/data_subjects/postgres_spec.rb
+- spec/data_subjects/s3_file_spec.rb
+- spec/data_subjects/salesforce_spec.rb
+- spec/data_subjects/sftp_file_spec.rb
+- spec/data_subjects/sub_job_spec.rb
+- spec/encoder_spec.rb
+- spec/extractor_spec.rb
 - spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
+- spec/fixtures/sf_bulk_helper_stubs.rb
 - spec/fixtures/unsupported_escape.csv
+- spec/job/transform_spec.rb
+- spec/job_spec.rb
+- spec/loader_spec.rb
 - spec/metadata_spec.rb
+- spec/parser_spec.rb
 - spec/remi_spec.rb
+- spec/sf_bulk_helper_spec.rb
 - spec/source_to_target_map_spec.rb
 - spec/testing/data_stub_spec.rb
 - spec/transform_spec.rb
@@ -294,7 +362,6 @@ signing_key:
 specification_version: 4
 summary: Remi (Ruby Extract Map Integrate)
 test_files:
-- features/aggregate.feature
 - features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
@@ -304,6 +371,9 @@ test_files:
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb
+- features/sub_job_example.feature
+- features/sub_transform_example.feature
+- features/sub_transform_many_to_many.feature
 - features/support/env.rb
 - features/support/env_app.rb
 - features/transforms/concatenate.feature
@@ -315,19 +385,31 @@ test_files:
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
 - features/transforms/truthy.feature
-- spec/data_subject/csv_file_spec.rb
-- spec/data_subject/data_frame.rb
 - spec/data_subject_spec.rb
-- spec/extractor/file_system_spec.rb
-- spec/extractor/local_file_spec.rb
-- spec/extractor/s3_file_spec.rb
-- spec/extractor/sftp_file_spec.rb
+- spec/data_subjects/csv_file_spec.rb
+- spec/data_subjects/data_frame_spec.rb
+- spec/data_subjects/file_system_spec.rb
+- spec/data_subjects/local_file_spec.rb
+- spec/data_subjects/none_spec.rb
+- spec/data_subjects/postgres_spec.rb
+- spec/data_subjects/s3_file_spec.rb
+- spec/data_subjects/salesforce_spec.rb
+- spec/data_subjects/sftp_file_spec.rb
+- spec/data_subjects/sub_job_spec.rb
+- spec/encoder_spec.rb
+- spec/extractor_spec.rb
 - spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
+- spec/fixtures/sf_bulk_helper_stubs.rb
 - spec/fixtures/unsupported_escape.csv
+- spec/job/transform_spec.rb
+- spec/job_spec.rb
+- spec/loader_spec.rb
 - spec/metadata_spec.rb
+- spec/parser_spec.rb
 - spec/remi_spec.rb
+- spec/sf_bulk_helper_spec.rb
 - spec/source_to_target_map_spec.rb
 - spec/testing/data_stub_spec.rb
 - spec/transform_spec.rb

data/features/aggregate.feature

@@ -1,42 +0,0 @@
-Feature: Tests the aggregate refinement to the Daru library
-
-  Background:
-    Given the job is 'Aggregate'
-    And the job source 'Source Data'
-    And the job target 'Target Data'
-    And the job target 'Multigroup Target Data'
-
-    And the source 'Source Data'
-
-
-  Scenario: The aggregator should find the minimum year for each 'Alpha'
-    Given the target 'Target Data'
-    And the following example record for 'Source Data':
-      | Alpha | Beta | Year |
-      | a     | aa   | 2016 |
-      | a     | aa   | 2018 |
-      | b     | bb   | 2016 |
-      | b     | bb   | 2010 |
-      | a     | ab   | 2017 |
-    And the following example record called 'expected result':
-      | Alpha | Year |
-      | a     | Group a has a minimum value of 2016 |
-      | b     | Group b has a minimum value of 2010 |
-    Then the target should match the example 'expected result'
-
-
-  Scenario: The aggregator should find the minimum year for each 'Alpha'
-    Given the target 'Multigroup Target Data'
-    And the following example record for 'Source Data':
-      | Alpha | Beta | Year |
-      | a     | aa   | 2016 |
-      | a     | aa   | 2018 |
-      | b     | bb   | 2016 |
-      | b     | bb   | 2010 |
-      | a     | ab   | 2017 |
-    And the following example record called 'expected result':
-      | Alpha | Beta | Year |
-      | a     | aa   | Group ["a", "aa"] has a minimum value of 2016 |
-      | a     | ab   | Group ["a", "ab"] has a minimum value of 2017 |
-      | b     | bb   | Group ["b", "bb"] has a minimum value of 2010 |
-    Then the target should match the example 'expected result'