remi 0.2.14 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6434dd8f6eadd170a3e6b64a1c35ffd450277483
-  data.tar.gz: fb4edbdad9f5695ca24c5bae9c2fdab484788d4e
+  metadata.gz: 89aa19a0e9852eb3fcbf63cd3890102501fc27ff
+  data.tar.gz: ef447ea96c09edd89953aabef5b9e18188d06c50
 SHA512:
-  metadata.gz: 2584bf727adb2b174c9296cd4b3bc7b7967b94c21a946097e2ed2bcd0946edffd61ddac2a2946c21da734d8e3f61c08263d9069195850a91fd39b53c6f7ebef9
-  data.tar.gz: 50a41704c02a33072c705c48adbc0ba4b0b9e2919331eb188f0861f3be04de7dbdf315a0a2a14759bbaaea8fb780e423460974d2d83458cb149e4ce4446cd5bf
+  metadata.gz: 11db98f16aff0c37c83a02dd44dec3ec6e1f908c61e85aaeb277b789acc58b2eaf0706b10274591004da92e0ea66321b1ee2fe5318422772c251519150d41d80
+  data.tar.gz: 7639ef4a4b9b3aa03385e270b5aa3ea10796dd9e9250819e706be0fa366d95dcb4c886e6ccab54e5546f0aa7a38ea6adbb61909fa6919be3350aa68dc8d6b0c0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    remi (0.2.14)
+    remi (0.2.15)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/examples.feature

@@ -47,3 +47,17 @@ Feature: This tests the creation of example records.
     And the target has 3 records where 'Quantity' is greater than 0.9
     And the target has 3 records where 'Quantity' is between 0 and 1
     And the target has 2 records where 'Quantity' is between 0.6 and 2
+
+
+
+  Scenario: Adding data to a source that already has example data.
+    Given the following example for 'Source Data':
+      | Id | Name  |
+      | 1  | Alpha |
+      | 2  | Beta  |
+      | 3  | Gamma |
+    And the following records are appended to 'Source Data':
+      | Id | Name    |
+      | 4  | Delta   |
+      | 5  | Epsilon |
+    Then the target has 5 records

data/features/parameters.feature

@@ -0,0 +1,24 @@
+Feature: Tests setting parameters
+
+  Background:
+    Given the job is 'Parameters'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+    And the source field 'parameter_name' is set to the value "test_parameter"
+
+  Scenario: Setting a parameter explicitly.
+    Given the job parameter 'myparam' is "happypants"
+    Then the target field 'myparam' is set to the value "happypants"
+
+  Scenario: Dynamically retreiving a parameter that exists.
+    Given the source field 'parameter_name' is set to the value "test_parameter"
+    Then the target field 'parameter_name' is set to the value "my test parameter value"
+
+  @fails
+  Scenario: Dynamically retreiving a parameter that does not exists.
+    Given the source field 'parameter_name' is set to the value "nonexisting_parameter"
+    Then the target field 'parameter_name' is set to the value ""

data/features/step_definitions/remi_step.rb

@@ -15,16 +15,16 @@ Given /^the job target '([[:alnum:]\s\-_]+)'$/ do |arg|
   @brt.add_job_target arg
 end
 
-Given /^the following example(?: record| records|) called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
-  @brt.add_example arg, example_table
-end
-
 Given /^the job parameter '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |param, value|
   @brt.set_job_parameter(param, value)
 end
 
 ### Setting up example data
 
+Given /^the following example(?: record| records|) called '([[:alnum:]\s\-_]+)':$/ do |arg, example_table|
+  @brt.add_example arg, example_table
+end
+
 Given /^the following example(?: record| records|) for '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
   example_name = source_name
   @brt.add_example example_name, example_table
@@ -35,6 +35,11 @@ Given /^the example '([[:alnum:]\s\-_]+)' for '([[:alnum:]\s\-_]+)'$/ do |exampl
   @brt.job_sources[source_name].stub_data_with(@brt.examples[example_name])
 end
 
+Given /^the following (?:record is|records are) appended to '([[:alnum:]\s\-_]+)':$/ do |source_name, example_table|
+  example_name = SecureRandom.uuid
+  @brt.add_example example_name, example_table
+  @brt.job_sources[source_name].append_data_with(@brt.examples[example_name])
+end
 
 ### Source file processing
 
@@ -106,8 +111,8 @@ Given /^the source '([[:alnum:]\s\-_]+)'$/ do |arg|
   @brt.add_source(arg)
 end
 
-Given /^the source field '(.+)'$/ do |arg|
-  @brt.sources.add_field(arg)
+Given /^the source field '([^']+)'$/ do |source_field_name|
+  @brt.sources.add_field(source_field_name)
 end
 
 Given /^the source field has the value "([^"]*)"$/ do |arg|
@@ -120,7 +125,8 @@ end
 
 When /^the source field '(.+)' (?:has an empty value|is blank)$/ do |source_field|
   step "the source field '#{source_field}'"
-  @brt.source.fields[source_field].value = ''
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+  @brt.sources[source_name].fields[source_field_name].value = ''
 end
 
 Given /^the source field '([^:]+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
@@ -130,20 +136,33 @@ end
 
 Given /^the source field '(.+:.+)' (?:has|is set to) the value "([^"]*)"$/ do |source_field, value|
   step "the source field '#{source_field}'"
-  source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
+  source_name, field_name = @brt.sources.parse_full_field(source_field)
   @brt.sources[source_name].fields[field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
 end
 
+Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)'$/ do |source_field, other_source_field|
+  step "the source field '#{source_field}'"
+  step "the source field '#{other_source_field}'"
+  source_name, field_name = @brt.sources.parse_full_field(source_field)
+  other_source_name, other_field_name = @brt.sources.parse_full_field(other_source_field)
+
+  @brt.sources[source_name].fields[field_name].value = @brt.sources[other_source_name].fields[other_field_name].value
+end
+
 Given /^the source field '(.+:.+)' (?:has|is set to) the value in the source field '(.+:.+)', prefixed with "([^"]*)"$/ do |source_field, other_source_field, prefix|
   step "the source field '#{source_field}'"
   step "the source field '#{other_source_field}'"
-  source_name, field_name = *Remi::BusinessRules.parse_full_field(source_field)
-  other_source_name, other_field_name = *Remi::BusinessRules.parse_full_field(other_source_field)
+  source_name, field_name = @brt.sources.parse_full_field(source_field)
+  other_source_name, other_field_name = @brt.sources.parse_full_field(other_source_field)
 
   prefixed = "#{prefix}#{@brt.sources[other_source_name].fields[other_field_name].value}"
   @brt.sources[source_name].fields[field_name].value = prefixed
 end
 
+Given /^the source data are tied through the fields '(.+:.+)' and '(.+:.+)'$/ do |source_field, other_source_field|
+  step "the source field '#{other_source_field}' is set to the value in the source field '#{source_field}'"
+end
+
 Given /^the source field is parsed with the date format "([^"]*)"$/ do |date_format|
   expect(@brt.source.field.metadata[:format]).to eq date_format
 end
@@ -180,10 +199,10 @@ Then /^the target field '(.+)' is copied from the source field '(.+:.+)'$/ do |t
   step "the target field '#{target_field}'"
   step "the source field '#{source_field}'"
 
-  source_name, source_field_name = *Remi::BusinessRules.parse_full_field(source_field)
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
 
   @brt.run_transforms
-  expect(@brt.target.field.value).to eq (@brt.sources[source_name].fields[source_field_name].value)
+  expect(@brt.target.fields[target_field].value).to eq (@brt.sources[source_name].fields[source_field_name].value)
 end
 
 Then /^the target field '(.+)' is copied from the source field '([^:]+)'$/ do |target_field, source_field|
@@ -200,9 +219,13 @@ Then /^the target field is (?:set to the value|populated with) "([^"]*)"$/ do |v
 end
 
 Then /^the target field '(.+)' is (?:set to the value|populated with) "([^"]*)"$/ do |target_field, value|
+  expect_cucumber {
+    expect {
   @brt.targets.add_field(target_field)
   @brt.run_transforms
+    }.not_to raise_error
   expect(@brt.targets.fields[target_field].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
+  }
 end
 
 Then /^the target field '(.+)' is in the list "([^"]*)"$/ do |target_field, list|
@@ -234,6 +257,15 @@ Then /^the target should match the example '([[:alnum:]\s]+)'$/ do |example_name
   step "the target '#{target_name}' should match the example '#{example_name}'"
 end
 
+Then /^the target field '(.+)' is populated from the source field '(.+)' after translating it according to:$/ do |target_field, source_field, translation_table|
+  step "the target field '#{target_field}'"
+
+  translation_table.rows.each do |translation_row|
+    step "the source field '#{source_field}' is set to the value \"#{translation_row[0]}\""
+    @brt.run_transforms
+    expect(@brt.target.fields[target_field].value).to eq translation_row[1]
+  end
+end
 
 ### Transforms
 
@@ -292,23 +324,26 @@ Then /^the target field '(.+)' is populated with "([^"]*)" using the format "([^
 end
 
 Then /^the target field '(.+)' is the first non-blank value from source fields '(.+)'$/ do |target_field_name, source_field_list|
-  source_field_names = "'#{source_field_list}'".split(',').map do |field_with_quotes|
-    field_with_quotes.match(/'(.+)'/)[1]
+  source_fields = "'#{source_field_list}'".split(',').map do |field_with_quotes|
+    full_field_name = field_with_quotes.match(/'(.+)'/)[1]
+
+    source_name, field_name = @brt.sources.parse_full_field(full_field_name)
+    { full_field_name: full_field_name, source: source_name, field: field_name }
   end
 
-  source_field_names.each do |name|
-    step "the source field '#{name}'"
+  source_fields.each do |source_field|
+    step "the source field '#{source_field[:full_field_name]}'"
   end
   step "the target field '#{target_field_name}'"
 
-  source_field_names.each do |source_field_name|
+  source_fields.each do |source_field|
     @brt.run_transforms
 
-    source_values = source_field_names.map { |name| @brt.source.fields[name].value }
+    source_values = source_fields.map { |v_source_field| @brt.sources[v_source_field[:source]].fields[v_source_field[:field]].value }
     source_values_nvl = source_values.find { |arg| !arg.blank? }
 
     expect_cucumber { expect(@brt.target.fields[target_field_name].value).to eq source_values_nvl }
-    @brt.source.fields[source_field_name].value = ''
+    @brt.sources[source_field[:source]].fields[source_field[:field]].value = ''
   end
 
 end
@@ -384,7 +419,7 @@ end
 
 Then /^the target '([[:alnum:]\s\-_]+)' has (\d+) record(?:s|)$/ do |target_name, nrecords|
   @brt.run_transforms
-  expect(@brt.targets[target_name].size).to be nrecords.to_i
+  expect(@brt.targets[target_name].size).to eq nrecords.to_i
 end
 
 Then /^the target has (\d+) record(?:s|) where '([[:alnum:]\s\-_]+)' is "([^"]*)"$/ do |nrecords, field_name, value|

data/features/support/env_app.rb

@@ -2,3 +2,4 @@ require_relative '../../jobs/sample_job'
 require_relative '../../jobs/copy_source_job'
 require_relative '../../jobs/transforms/transform_jobs'
 require_relative '../../jobs/aggregate_job'
+require_relative '../../jobs/parameters_job'

data/jobs/parameters_job.rb

@@ -0,0 +1,23 @@
+require_relative 'all_jobs_shared'
+
+class ParametersJob
+  include AllJobsShared
+
+  define_param :test_parameter, "my test parameter value"
+
+  define_target :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :parameter_name => {}
+    }
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(nil) .target(:myparam)
+        .transform(Remi::Transform[:constant].(params[:myparam]))
+      map source(:parameter_name) .target(:parameter_name)
+        .transform(->(v) { params[v.to_sym] })
+    end
+  end
+
+end

data/lib/remi/cucumber/business_rules.rb

@@ -1,10 +1,6 @@
 module Remi::BusinessRules
   using Remi::Refinements::Symbolizer
 
-  def self.parse_full_field(full_field_name)
-    full_field_name.split(':').map(&:strip)
-  end
-
   def self.csv_opt_map
     {
       'tab'             => "\t",
@@ -173,18 +169,25 @@ module Remi::BusinessRules
       @subjects.keys
     end
 
+    def parse_full_field(full_field_name, multi: false)
+      if full_field_name.include? ':'
+        full_field_name.split(':').map(&:strip)
+      elsif multi
+        [@subjects.keys, full_field_name]
+      else
+        raise "Multiple subjects defined: #{keys}" unless @subjects.size == 1
+        [@subjects.keys.first, full_field_name]
+      end
+    end
+
     def add_subject(subject_name, subject)
       @subjects[subject_name] ||= DataSubject.new(subject)
     end
 
     def add_field(full_field_name)
-      if full_field_name.include? ':'
-        subject_name, field_name = *Remi::BusinessRules.parse_full_field(full_field_name)
+      subject_names, field_name = parse_full_field(full_field_name, multi: true)
+      Array(subject_names).each do |subject_name|
         @subjects[subject_name].add_field(field_name)
-      else
-        @subjects.each do |subject_name, subject|
-          subject.add_field(full_field_name)
-        end
       end
     end
 
@@ -285,9 +288,17 @@ module Remi::BusinessRules
       @data_obj.stub_df if @data_obj.respond_to? :stub_df
     end
 
+    def example_to_df(example)
+      example.to_df(@data_obj.df.row[0].to_hash, field_symbolizer: @data_obj.field_symbolizer)
+    end
+
     def stub_data_with(example)
       stub_data
-      @data_obj.df = example.to_df(@data_obj.df.row[0].to_hash, field_symbolizer: @data_obj.field_symbolizer)
+      @data_obj.df = example_to_df(example)
+    end
+
+    def append_data_with(example)
+      @data_obj.df = @data_obj.df.concat example_to_df(example)
     end
 
 

data/lib/remi/cucumber/data_source.rb

@@ -23,7 +23,7 @@ module Remi
           float:    ->() { Faker::Number.decimal(2,2) },
           integer:  ->() { Faker::Number.number(4) },
           date:     ->() { Faker::Date.backward(3650) },
-          datetime: ->() { Faker::Time.backward(3650) },
+          datetime: ->() { Faker::Time.backward(3650).to_datetime },
           boolean:  ->() { ['T','F'].shuffle.first }
         })
       end

data/lib/remi/job.rb

@@ -7,7 +7,7 @@ module Remi
       attr_accessor :transforms
 
       def define_param(key, value)
-        @params ||= {}
+        @params ||= Hash.new { |h, key| raise "Parameter #{key} is not defined" }
         @params[key] = value
       end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.14'
+  VERSION = '0.2.15'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.15
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-12 00:00:00.000000000 Z
+date: 2016-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
@@ -227,6 +227,7 @@ files:
 - features/aggregate.feature
 - features/examples.feature
 - features/formulas.feature
+- features/parameters.feature
 - features/sample_job.feature
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
@@ -238,6 +239,7 @@ files:
 - jobs/aggregate_job.rb
 - jobs/all_jobs_shared.rb
 - jobs/copy_source_job.rb
+- jobs/parameters_job.rb
 - jobs/sample_job.rb
 - jobs/transforms/date_diff_job.rb
 - jobs/transforms/nvl_job.rb
@@ -300,6 +302,7 @@ test_files:
 - features/aggregate.feature
 - features/examples.feature
 - features/formulas.feature
+- features/parameters.feature
 - features/sample_job.feature
 - features/step_definitions/remi_step.rb
 - features/support/env.rb