remi 0.2.11 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 521283fb3e00330bcbd4bc47ecc65dec7dcfe32d
-  data.tar.gz: ff6cb6d01002b10b0e68cd50906de59c7b2af16d
+  metadata.gz: 2ee4f5ef99f18b36083cf0fa0a31e3c5e9dbab17
+  data.tar.gz: f78cdac5e10cc84cde78e7b54ef1955a591eb115
 SHA512:
-  metadata.gz: 43b033f8d7f85874e1aab2e726c850580c557ce1a45eafe6481039a3d54fd5e3bb29c04d14d8adb5e007ce5bfdc4e905f2f46c16a5be578f1911daaa281b8318
-  data.tar.gz: 9513915698ed36bb92949c69fca2bc87ed787fb177435059523342626d7bb9c26624d20c3dbe2710844bd3b2e5c3dd19df2e0d2ef5f62ebf9911e102c14af487
+  metadata.gz: 35b23fcdf2de08e344a12b8326159bb6ddb92dd322f732c0f5e3f5693ef0ffdf1731b1293b652581a2353aeb6b2daa732b93684813b8096cdeb0d4eddc9abbbf
+  data.tar.gz: f1953467dbc6f9b91cd804014bfb9b427308b974508272891d399f09c3142408f9426ba2af5283c05751b8270a347a6a25570e19d337f575611dcf322cf855ed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    remi (0.2.10)
+    remi (0.2.12)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
@@ -9,6 +9,7 @@ PATH
       docile (~> 1.1)
       net-sftp (~> 2.1)
       pg (~> 0.18)
+      regex_sieve (~> 0.1)
       regexp-examples (~> 1.1)
       restforce (~> 2.1)
       rspec (~> 3.3)
@@ -17,7 +18,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.5)
+    activesupport (4.2.5.1)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -26,12 +27,11 @@ GEM
     bond (0.5.1)
     builder (3.2.2)
     clbustos-rtf (0.4.2)
-    cucumber (2.3.0)
+    cucumber (2.3.2)
       builder (>= 2.1.2)
       cucumber-core (~> 1.4.0)
       cucumber-wire (~> 0.0.1)
       diff-lcs (>= 1.1.3)
-      event-bus (~> 0.1.0)
       gherkin (~> 3.2.0)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.1.2)
@@ -43,7 +43,6 @@ GEM
       spreadsheet (~> 1.0.3)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    event-bus (0.1.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.10.0)
@@ -58,7 +57,7 @@ GEM
       rbczmq (~> 1.7)
     json (1.8.3)
     mimemagic (0.3.1)
-    minitest (5.8.3)
+    minitest (5.8.4)
     multi_json (1.11.2)
     multi_test (0.1.2)
     multipart-post (2.0.0)
@@ -76,7 +75,8 @@ GEM
     prawn-svg (0.9.1.11)
       prawn (>= 0.8.4)
     rbczmq (1.7.9)
-    regexp-examples (1.1.4)
+    regex_sieve (0.1.0)
+    regexp-examples (1.2.0)
     reportbuilder (1.4.2)
       clbustos-rtf (~> 0.4.0)
       prawn (~> 0.8.4)
@@ -91,7 +91,7 @@ GEM
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
       rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.1)
+    rspec-core (3.4.2)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
       diff-lcs (>= 1.2.0, < 2.0)

data/features/step_definitions/remi_step.rb

@@ -283,6 +283,27 @@ Then /^the target field '(.+)' is populated with "([^"]*)" using the format "([^
   expect(@brt.target.field.value).to eq target_reformatted
 end
 
+Then /^the target field '(.+)' is the first non-blank value from source fields '(.+)'$/ do |target_field_name, source_field_list|
+  source_field_names = "'#{source_field_list}'".split(',').map do |field_with_quotes|
+    field_with_quotes.match(/'(.+)'/)[1]
+  end
+
+  source_field_names.each do |name|
+    step "the source field '#{name}'"
+  end
+  step "the target field '#{target_field_name}'"
+
+  source_field_names.each do |source_field_name|
+    @brt.run_transforms
+
+    source_values = source_field_names.map { |name| @brt.source.fields[name].value }
+    source_values_nvl = source_values.find { |arg| !arg.blank? }
+
+    expect_cucumber { expect(@brt.target.fields[target_field_name].value).to eq source_values_nvl }
+    @brt.source.fields[source_field_name].value = ''
+  end
+
+end
 
 When /^in the source field, periods have been used in place of commas$/ do
   @brt.source.field.value = @brt.source.field.value.gsub(/\./, ',')

data/features/support/env.rb

@@ -8,3 +8,19 @@ require 'remi/cucumber'
 require_relative 'env_app.rb'
 
 Remi::Settings.log_level = Logger::ERROR
+
+Before '~@fails' do
+  def expect_cucumber(&block)
+    block.call
+  end
+end
+
+Before '@fails' do
+  def expect_cucumber(&block)
+    begin
+      block.call
+    rescue RSpec::Expectations::ExpectationNotMetError => err
+      puts "Expected error: #{err}"
+    end
+  end
+end

data/features/transforms/nvl.feature

@@ -0,0 +1,38 @@
+Feature: Test the NVL (Next Value Lookup) transformer.
+
+  Background:
+    Given the job is 'Nvl'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+  Scenario Outline: Performing an NVL
+    Given the source 'Source Data'
+    And the target 'Target Data'
+
+    And the source field 'Field1' is set to the value "<Field1>"
+    And the source field 'Field2' is set to the value "<Field2>"
+    And the source field 'Field3' is set to the value "<Field3>"
+    And the job parameter 'default' is "<Default>"
+    Then the target field 'Result Field' is set to the value "<Expected>"
+
+    Examples:
+      | Field1 | Field2 | Field3 | Default | Expected |
+      | A      | B      | C      |         | A        |
+      |        | B      | C      |         | B        |
+      |        |        | C      |         | C        |
+      |        |        |        |         |          |
+      |        |        |        | UNK     | UNK      |
+
+
+  Scenario: Testing an NVL with the short form version
+    Given the source 'Source Data'
+    And the target 'Target Data'
+
+    Then the target field 'Result Field' is the first non-blank value from source fields 'Field1', 'Field2', 'Field3'
+
+  @fails
+  Scenario: Testing that the NVL with the short form version fails appropriately
+    Given the source 'Source Data'
+    And the target 'Target Data'
+
+    Then the target field 'Field2 Copy' is the first non-blank value from source fields 'Field1', 'Field2', 'Field3'

data/jobs/sample_job.rb

@@ -66,14 +66,14 @@ class SampleJob
     operation: :create,
     api: :bulk
 
-  define_lookup :program_name_lookup, Remi::Lookup::RegexSieve, {
+  define_param :program_name_lookup, RegexSieve.new({
     /^BIO$/              => "Biology",
     /^Fake Biology$/     => nil,
     /(?:B|Microb)iology/ => "Biology",
     /^CHEM$/             => "Chemistry",
     /Chemistry/          => "Chemistry",
     /Physics/            => "Physics"
-  }
+  })
 
   define_transform :map_common_fields, sources: [:sample_file, :existing_contacts], targets: :all_contacts do
 
@@ -81,7 +81,7 @@ class SampleJob
     all_contacts.df = sample_file.df.dup
     Remi::SourceToTargetMap.apply(all_contacts.df) do
       map source(:program) .target(:Major__c)
-        .transform(Remi::Transform[:lookup][program_name_lookup])
+        .transform(Remi::Transform[:lookup][params[:program_name_lookup]])
     end
     all_contacts.df = all_contacts.df.where(all_contacts.df[:Major__c].not_eq(nil))
 

data/jobs/transforms/nvl_job.rb

@@ -0,0 +1,23 @@
+require_relative '../all_jobs_shared'
+
+class NvlJob
+  include AllJobsShared
+
+  define_param :default, ''
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :field1 => {},
+      :field2 => {},
+      :field3 => {}
+    }
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:field1, :field2, :field3) .target(:result_field)
+        .transform(Remi::Transform[:nvl].(params[:default]))
+
+      map source(:field2) .target(:field2_copy)
+    end
+  end
+end

data/jobs/transforms/transform_jobs.rb

@@ -1,3 +1,4 @@
 require_relative 'prefix_job'
 require_relative 'parse_date_job'
 require_relative 'date_diff_job'
+require_relative 'nvl_job'

data/lib/remi.rb

@@ -10,6 +10,7 @@ require 'daru'
 require 'docile'
 require 'net/sftp'
 require 'pg'
+require 'regex_sieve'
 
 # ActiveSupport extensions
 require 'active_support'
@@ -56,5 +57,4 @@ require 'remi/data_target/data_frame'
 require 'remi/data_target/salesforce'
 require 'remi/data_target/csv_file'
 
-require 'remi/lookup/regex_sieve'
 require 'remi/transform'

data/lib/remi/cucumber/business_rules.rb

@@ -15,6 +15,7 @@ module Remi::BusinessRules
       'windows'         => "\r\n",
       'unix'            => "\n",
       'windows or unix' => :auto,
+      'null character'  => 0.chr,
     }
   end
 
@@ -31,7 +32,7 @@ module Remi::BusinessRules
     end
 
     def formulas
-      @formulas ||= Remi::Lookup::RegexSieve.new({
+      @formulas ||= RegexSieve.new({
         /(today|yesterday|tomorrow)/i => [:date_reference, :match_single_day],
         /(this|last|previous|next) (day|month|year|week)/i => [:date_reference, :match_single_unit],
         /(\d+)\s(day|days|month|months|year|years|week|weeks) (ago|from now)/i => [:date_reference, :match_multiple]

data/lib/remi/job.rb

@@ -2,7 +2,6 @@ module Remi
   module Job
     module JobClassMethods
       attr_accessor :params
-      attr_accessor :lookups
       attr_accessor :sources
       attr_accessor :targets
       attr_accessor :transforms
@@ -12,23 +11,6 @@ module Remi
         @params[key] = value
       end
 
-      def define_lookup(name, type_class, options)
-        @lookups ||= []
-        @lookups << name
-
-        define_method(name) do
-          iv_name = instance_variable_get("@#{name}")
-          return iv_name if iv_name
-
-          if type_class == Hash
-            lookup = options
-          else
-            lookup = type_class.new(options)
-          end
-          instance_variable_set("@#{name}", lookup)
-        end
-      end
-
       def define_source(name, type_class, **options)
         @sources ||= []
         @sources << name
@@ -69,10 +51,6 @@ module Remi
         @params || {}
       end
 
-      def lookups
-        @lookups || []
-      end
-
       def sources
         @sources || []
       end
@@ -96,7 +74,6 @@ module Remi
       def included(receiver)
         receiver.extend(JobClassMethods)
         receiver.params     = self.params.merge(receiver.params)
-        receiver.lookups    = self.lookups + receiver.lookups
         receiver.sources    = self.sources + receiver.sources
         receiver.targets    = self.targets + receiver.targets
         receiver.transforms = self.transforms.merge(receiver.transforms)
@@ -112,10 +89,6 @@ module Remi
       self.class.params
     end
 
-    def lookups
-      self.class.lookups
-    end
-
     def sources
       self.class.sources
     end

data/lib/remi/transform.rb

@@ -50,9 +50,9 @@ module Remi
       end
     end
 
-    def nvl
-      memoize_as_lambda(__method__) do |*largs|
-        Array(largs).find('') { |arg| !arg.blank? }
+    def nvl(default='')
+      memoize_as_lambda(__method__, default) do |(mdefault), *largs|
+        Array(largs).find(->() { mdefault }) { |arg| !arg.blank? }
       end
     end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.11'
+  VERSION = '0.2.12'
 end

data/remi.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'docile', ['~> 1.1']
   s.add_runtime_dependency 'net-sftp', ['~> 2.1']
   s.add_runtime_dependency 'pg', ['~> 0.18']
+  s.add_runtime_dependency 'regex_sieve', ['~> 0.1']
 
   s.add_runtime_dependency "cucumber", ["~> 2.1"]
   s.add_runtime_dependency "rspec", ["~> 3.3"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-10 00:00:00.000000000 Z
+date: 2016-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: regex_sieve
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: cucumber
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
 - features/support/env.rb
 - features/support/env_app.rb
 - features/transforms/date_diff.feature
+- features/transforms/nvl.feature
 - features/transforms/parse_date.feature
 - features/transforms/prefix.feature
 - jobs/aggregate_job.rb
@@ -211,6 +226,7 @@ files:
 - jobs/copy_source_job.rb
 - jobs/sample_job.rb
 - jobs/transforms/date_diff_job.rb
+- jobs/transforms/nvl_job.rb
 - jobs/transforms/parse_date_job.rb
 - jobs/transforms/prefix_job.rb
 - jobs/transforms/transform_jobs.rb
@@ -232,7 +248,6 @@ files:
 - lib/remi/extractor/sftp_file.rb
 - lib/remi/field_symbolizers.rb
 - lib/remi/job.rb
-- lib/remi/lookup/regex_sieve.rb
 - lib/remi/refinements/daru.rb
 - lib/remi/refinements/symbolizer.rb
 - lib/remi/settings.rb
@@ -276,5 +291,6 @@ test_files:
 - features/support/env.rb
 - features/support/env_app.rb
 - features/transforms/date_diff.feature
+- features/transforms/nvl.feature
 - features/transforms/parse_date.feature
 - features/transforms/prefix.feature

data/lib/remi/lookup/regex_sieve.rb

@@ -1,55 +0,0 @@
-module Remi
-  module Lookup
-
-    # Public: RegexSieve class.  The RegexSieve functions in a manner similar
-    # a hash.  The regex sieve is initialized with a hash where the keys are
-    # regular expressions and the values can be any valid Ruby object.  The order
-    # of the keys matters.  When the regex sieve is accessed using the array
-    # accessor [], it returns the first matching record.  By default, only
-    # the values are returned, but the key and all matching capture groups
-    # can optionally be returned.
-    #
-    # Examples:
-    #
-    #   r = RegexSieve.new({
-    #     /something/ => 'Something',
-    #     /something else/ => 'This will never get matched because the one above will match first',
-    #     /cool$/ => 'Cool',
-    #     /cool beans/ => 'Really Cool'
-    #   })
-    #
-    #   r['something else'] # => 'Something'
-    #   r['cool beans'] # => 'Really Cool'
-    class RegexSieve
-      def initialize(sieve)
-        @sieve = sieve
-      end
-
-      # Public: Array accessor for Regex Sieve.
-      #
-      # key - The string that will be matched to the keys in the sieve.
-      # opt - By default, only the values in the hash used to initialize the sieve
-      #       will be returned.  However, if you want to return the keys or the
-      #       capture groups then use :regex, :match, or both, respectively.
-      #
-      # Example:
-      #   r['something'] # => 'Something
-      #   r['something', :regex] # => { value: 'Something', regex: /something/ }
-      #   r['sometinng', :match, :regex] # => { value: 'Something', regex: /something/, match: #<MatchData "something"> }
-      def [](key, *opt)
-        opt = opt | [:value]
-
-        regex_match = nil
-        found = @sieve.find do |regex, v|
-          regex_match = regex.match(key)
-        end
-
-        return nil if found.nil?
-        full_result = { value: found[1], regex: found[0], match: regex_match }
-
-        full_result.select! { |k, v| opt.include?(k) }
-        full_result.size > 1 ? full_result : full_result.values.first
-      end
-    end
-  end
-end