remi 0.2.30 → 0.2.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f58071ddcbc5d59548e0e174b770871556ebd88
-  data.tar.gz: 86eb7d201d268190c4546ce67ae816b97b87291b
+  metadata.gz: 7c7c81a972d6fc604a4761b7fff19a26ec187065
+  data.tar.gz: f07165f263dee57adb1802182cf72551d3326ab4
 SHA512:
-  metadata.gz: 2219591db9125214867d5a42be832808b1a156ecb2a334f84ca4006bf2e8ed1ca3e41ef342d631a6069717be22e7deca67f4556cbc2cf5a1db57e057d22edc17
-  data.tar.gz: 74b06952815620244b94f413bc22eddaccbab1592012d7407477ddec618253dfefdc9d39454492121a8093469ee2e249d5d60c6def072b8f4cc22d08f79918d1
+  metadata.gz: d4ea4cde266110c374b75ef527f0ec71a674bba865c3da46107687fddb4871b3fda5aa601c44cd0fad5857038d8dff1c4dce297cf1d710ca335e2afafaa882dd
+  data.tar.gz: 7a25d3e85b4ae727998ff196b79395db40bc0d584c04c061069c4d0a457bd79530f7a66b4ce3480df739ebfcef6dda7ef011c5e8171f76bac2326ee392e1d01a

data/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: git@github.com:inside-track/daru.git
-  revision: ad71b644c8e26789e3ab2258f046d5bc68b8df71
+  revision: 80b8024102bdb31ef402cb8d6c5cef86dd31cd41
   branch: itk-master
   specs:
     daru (0.1.2)
@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.30)
+    remi (0.2.31)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/step_definitions/remi_step.rb

@@ -65,6 +65,9 @@ Then /^the file with the latest date stamp will be downloaded for processing$/ d
   expect(@brt.source.extract).to match_array Array(@brt.filestore.latest)
 end
 
+Then /^files will be downloaded for processing$/ do
+end
+
 Then /^no files will be downloaded for processing$/ do
   @brt.filestore.generate
   @brt.source.mock_extractor(@brt.filestore)
@@ -415,13 +418,16 @@ Then /^the source field '([^']+)' is prefixed with "([^"]*)" and loaded into the
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
   target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
 
-  prefixed_source = "#{prefix}#{@brt.sources[source_name].fields[source_field_name].value}"
+  prefixed_source = @brt.sources[source_name].fields[source_field_name].values.map do |value|
+    "#{prefix}#{value}"
+  end.uniq.sort
 
   @brt.run_transforms
-  Array(target_names).each do |target_name|
-    expect(@brt.targets[target_name].fields[target_field_name].value).to eq prefixed_source
-  end
+  results = Array(target_names).map do |target_name|
+    @brt.targets[target_name].fields[target_field_name].values.uniq
+  end.flatten.uniq.sort
 
+  expect(results).to eq prefixed_source
 end
 
 Then /^the source field is prefixed with "([^"]*)" and loaded into the target field '([^']+)'$/ do |prefix, target_field|
@@ -633,6 +639,15 @@ Then /^a target record is not created$/ do
 end
 
 
+### Setting up data for multiple records
+
+Given /^the source field '([^']+)' is a unique integer$/ do |source_field|
+  step "the source field '#{source_field}'"
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+
+  @brt.sources[source_name].unique_integer_field(source_field_name)
+end
+
 ### Record counting
 
 Then /^the target has (\d+) record(?:s|)$/ do |nrecords|

data/features/transforms/data_frame_sieve.feature

@@ -0,0 +1,38 @@
+Feature: Tests the DataFrameSieve transform
+
+  Background:
+    Given the job is 'Data Frame Sieve'
+    And the job source 'Source Data'
+    And the job source 'Sieve'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the source 'Sieve'
+    And the target 'Target Data'
+
+
+  Scenario: A slightly complicated sieve.
+
+    Given the following example for 'Sieve':
+      | level       | program | contact | group     |
+      | Undergrad   | NURS    | \\nil   | intensive |
+      | Undergrad   | \\nil   | true    | intensive |
+      | Undergrad   | \\nil   | false   | base      |
+      | Grad        | ENG     | true    | intensive |
+      | \\nil       | \\nil   | \\nil   | base      |
+
+    And the following example for 'Source Data':
+      | id | level       | program | contact |
+      | 1  | Undergrad   | CHEM    | false   |
+      | 2  | Undergrad   | CHEM    | true    |
+      | 3  | Grad        | CHEM    | true    |
+      | 4  | Undergrad   | NURS    | false   |
+      | 5  | Unknown     | CHEM    | true    |
+
+    Then the target should match the example:
+      | id | level       | program | contact | group     |
+      | 1  | Undergrad   | CHEM    | false   | base      |
+      | 2  | Undergrad   | CHEM    | true    | intensive |
+      | 3  | Grad        | CHEM    | true    | base      |
+      | 4  | Undergrad   | NURS    | false   | intensive |
+      | 5  | Unknown     | CHEM    | true    | base      |

data/features/transforms/partitioner.feature

@@ -0,0 +1,72 @@
+Feature: Tests the Partitioner transform
+  The partitioner keeps track of which groups it has assigned records to
+  in order to keep the distribution of records into groups as precise as
+  possible.
+
+
+  Background:
+    Given the job is 'Partitioner'
+    And the job source 'Source Data'
+    And the job source 'Current Population'
+    And the job source 'Distribution'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the source 'Current Population'
+    And the source 'Distribution'
+    And the target 'Target Data'
+
+
+
+  Scenario: Partitioning records into groups with no prior population
+
+    Given the following example for 'Source Data':
+      | id |
+      |  1 |
+      |  2 |
+      |  3 |
+      |  4 |
+
+    And the following example for 'Distribution':
+      | group | weight |
+      | A     | 0.5    |
+      | B     | 0.5    |
+      | C     | 1      |
+
+    Then the target has 1 records where 'group' is "A"
+    Then the target has 1 records where 'group' is "B"
+    Then the target has 2 records where 'group' is "C"
+
+
+  Scenario: Partitioning records into groups with a prior population
+
+    Given the following example for 'Source Data':
+      | id |
+      |  1 |
+      |  2 |
+      |  3 |
+      |  4 |
+      |  5 |
+
+    And the following example for 'Distribution':
+      | group | weight |
+      | A     | 0.5    |
+      | B     | 0.5    |
+      | C     | 1      |
+
+    And the following example for 'Current Population':
+      | group | count |
+      | A     | 2     |
+      | B     | 1     |
+
+
+
+    Then the target has 0 records where 'group' is "A"
+    Then the target has 1 records where 'group' is "B"
+    Then the target has 4 records where 'group' is "C"
+
+
+#  Scenario: Remainders
+#    When the target population is matched exactly, the next
+#    assignment is random (and weighted by the given weights).
+#    I don't know how to test this.

data/features/transforms/truthy.feature

@@ -0,0 +1,52 @@
+Feature: Tests the Truthy transform
+
+  Background:
+    Given the job is 'Truthy'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+  Scenario Outline: Truthy without allowing nils
+
+    Given the source field 'truthy'
+    And the target field 'no_nils'
+
+    When the source field has the value "<source>"
+    Then the target field is set to the value "<target>"
+
+  Examples:
+    | source   | target |
+    | True     | true   |
+    | t        | true   |
+    | yEs      | true   |
+    | Y        | true   |
+    | 1        | true   |
+    | Yessir   | false  |
+    | anything | false  |
+
+
+
+  Scenario Outline: Truthy allowing nils
+
+    Given the source field 'truthy'
+    And the target field 'allow_nils'
+
+    When the source field has the value "<source>"
+    Then the target field is set to the value "<target>"
+
+  Examples:
+    | source   | target |
+    | True     | true   |
+    | t        | true   |
+    | yEs      | true   |
+    | Y        | true   |
+    | 1        | true   |
+    | Yessir   |        |
+    | anything |        |
+    | FALSE    | false  |
+    | f        | false  |
+    | no       | false  |
+    | N        | false  |
+    | 0        | false  |

data/jobs/transforms/data_frame_sieve_job.rb

@@ -0,0 +1,30 @@
+require_relative '../all_jobs_shared'
+
+class DataFrameSieveJob
+  include AllJobsShared
+
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :id      => {},
+      :level   => {},
+      :program => {},
+      :contact => {}
+    }
+
+  define_source :sieve, Remi::DataSource::DataFrame,
+    fields: {
+      :level   => {},
+      :program => {},
+      :contact => {},
+      :group   => {}
+    }
+
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:level, :program, :contact) .target(:group)
+        .transform(Remi::Transform::DataFrameSieve.new(sieve.df))
+    end
+  end
+end

data/jobs/transforms/partitioner_job.rb

@@ -0,0 +1,35 @@
+require_relative '../all_jobs_shared'
+
+class PartitionerJob
+  include AllJobsShared
+
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :id      => {}
+    }
+
+  define_source :distribution, Remi::DataSource::DataFrame,
+    fields: {
+      :group  => {},
+      :weight => {}
+    }
+
+  define_source :current_population, Remi::DataSource::DataFrame,
+    fields: {
+      :group => {},
+      :count => {}
+    }
+
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+
+    distribution_hash = distribution.df.map(:row) { |row| [row[:group], row[:weight].to_f] }.to_h
+    current_population_hash = current_population.df.map(:row) { |row| [row[:group], row[:count].to_i] }.to_h
+
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(nil) .target(:group)
+        .transform(Remi::Transform::Partitioner.new(buckets: distribution_hash, initial_population: current_population_hash))
+    end
+  end
+end

data/jobs/transforms/truthy_job.rb

@@ -0,0 +1,21 @@
+require_relative '../all_jobs_shared'
+
+class TruthyJob
+  include AllJobsShared
+
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :truthy => {}
+    }
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:truthy) .target(:allow_nils)
+        .transform(Remi::Transform::Truthy.new(allow_nils: true))
+
+      map source(:truthy) .target(:no_nils)
+        .transform(Remi::Transform::Truthy.new)
+    end
+  end
+end

data/lib/remi/cucumber/business_rules.rb

@@ -390,6 +390,12 @@ module Remi::BusinessRules
       @data_subject.df.group_by(field_names).size * 1.0 / @data_subject.df.size
     end
 
+    def unique_integer_field(field_name)
+      vector_name = fields[field_name].field_name
+      i = 0
+      @data_subject.df[vector_name].recode! { |v| i += 1 }
+    end
+
     def mock_extractor(filestore)
       extractor = class << @data_subject.extractor; self; end
 
@@ -526,7 +532,14 @@ module Remi::BusinessRules
       df = Daru::DataFrame.new([], order: seed_hash.keys | table_headers)
       @table.hashes.each do |example_row|
         example_row_sym = example_row.reduce({}) do |h, (k,v)|
-          h[k.symbolize(field_symbolizer)] = ParseFormula.parse(v)
+          formula_value = ParseFormula.parse(v)
+          value = case formula_value
+            when '\nil'
+              nil
+            else
+              formula_value
+            end
+          h[k.symbolize(field_symbolizer)] = value
           h
         end
         df.add_row(seed_hash.merge(example_row_sym))

data/lib/remi/source_to_target_map.rb

@@ -84,7 +84,7 @@ module Remi
 
     def do_map_single_source_and_target_vector
       @target_df[@target_vectors.first] = @source_df[@source_vectors.first].recode do |vector_value|
-        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value || [nil])) }
+        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value.nil? ? [nil] : value)) }
       end
     end
 
@@ -100,7 +100,7 @@ module Remi
       end
 
       work_vector.recode! do |vector_value|
-        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value || [nil])) }
+        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value.nil? ? [nil] : value)) }
       end
 
       @target_vectors.each_with_index do |target_vector, vector_idx|

data/lib/remi/transform.rb

@@ -483,7 +483,7 @@ module Remi
           when :decimal
             Float("%.#{scale}f" % Float(value))
           when :date
-            Date.strptime(value, in_format)
+            value.is_a?(Date) ? value : Date.strptime(value, in_format) # value.is_a?(Date) is only needed becuase we stub date types with actual dates, rather than strings like we probably should
           when :datetime
             Time.strptime(value, in_format)
           else
@@ -493,5 +493,209 @@ module Remi
       end
     end
 
+
+
+
+
+    # Public: Converts strings into booleans.
+    # Uses a regex to convert strings representing booleans to actual booleans.
+    # The truthy regex is /^(t|true|y|yes|1)$/i and the falsey regex is /^(f|false|n|no|0)$/i
+    #
+    # allow_nils - Specifies whether to allow the result to include nils.  If this is set
+    #              to false, then the value is only checked against the truthy regex and
+    #              the returned value is false if it doesn't match.  If allow_nils
+    #              is set to true, the both the truthy and the falsey regex are checked.
+    #              If neither match, then the result is nil.  (Default: false).
+    #
+    # Examples:
+    #
+    # Truthy.new.to_proc.call('True')                         # => true
+    # Truthy.new.to_proc.call('Yes')                          # => true
+    # Truthy.new.to_proc.call('y')                            # => true
+    # Truthy.new.to_proc.call('Yessire')                      # => false
+    # Truthy.new.to_proc.call('0')                            # => false
+    # Truthy.new.to_proc.call('Pineapple')                    # => false
+    # Truthy.new(allow_nils: false).to_proc.call('Pineapple') # => nil
+    class Truthy < Transform
+      def initialize(*args, allow_nils: false, **kargs, &block)
+        super
+        @allow_nils = allow_nils
+
+        @true_regex = /^(t|true|y|yes|1)$/i
+        @false_regex = /^(f|false|n|no|0)$/i
+      end
+
+      def match_true(value)
+        !!value.match(@true_regex)
+      end
+
+      def match_false(value)
+        !!value.match(@false_regex)
+      end
+
+      def transform(value)
+        value = value.to_s
+
+        if @allow_nils
+          if match_true(value)
+            true
+          elsif match_false(value)
+            false
+          else
+            nil
+          end
+        else
+          match_true(value)
+        end
+      end
+    end
+
+
+    # Public: Applies a DataFrame grouping sieve.
+    #
+    # The DataFrame sieve can be used to simplify very complex nested
+    # if-then logic to group data into buckets.  Given a DataFrame
+    # with N columns, the first N-1 columns represent the variables
+    # needed to group data into buckets.  The last column is the
+    # desired group.  The sieve then progresses down the rows of the
+    # DataFrame and checks to see if the input data matches the values
+    # in the columns of the sieve.  Nils in the sieve are treated as
+    # wildcards and match anything.  The first row that matches wins
+    # and the sieve progression stops.
+    #
+    # sieve_df - The sieve, defined as a dataframe.  The arguments
+    #            to the transform must appear in the same order as the
+    #            first N-1 columns of the sieve.
+    #
+    #
+    # Examples:
+    #
+    #   # This sieve captures the following business logic
+    #   # 1 - All Non-Graduate Nursing, regardless of contact, gets assigned to the :intensive group.
+    #   # 2 - All Undergraduate programs with contact get assigned to the :intensive group.
+    #   # 3 - All Undergraduate programs without a contact get assigned to the :base group.
+    #   # 4 - All Graduate engineering programs with a contact get assigned to the :intensive group.
+    #   # 5 - All other programs get assigned to the :base group
+    #   sieve_df = Daru::DataFrame.new([
+    #     [ 'Undergrad' , 'NURS' , nil   , :intensive ],
+    #     [ 'Undergrad' , nil    , true  , :intensive ],
+    #     [ 'Undergrad' , nil    , false , :base ],
+    #     [ 'Grad'      , 'ENG'  , true  , :intensive ],
+    #     [ nil         , nil    , nil   , :base ],
+    #     ].transpose,
+    #     order: [:level, :program, :contact, :group]
+    #     )
+    #
+    #   test_df = Daru::DataFrame.new([
+    #     ['Undergrad' , 'CHEM' , false],
+    #     ['Undergrad' , 'CHEM' , true],
+    #     ['Grad'      , 'CHEM' , true],
+    #     ['Undergrad' , 'NURS' , false],
+    #     ['Unknown'   , 'CHEM' , true],
+    #     ].transpose,
+    #     order: [:level, :program, :contact]
+    #   )
+    #
+    #   Remi::SourceToTargetMap.apply(test_df) do
+    #     map source(:level, :program, :contact,) .target(:group)
+    #     .transform(Remi::Transform::DataFrameSieve.new(sieve_df))
+    #   end
+    #
+    #   test_df
+    #   # =>  #<Daru::DataFrame:70099624408400 @name = d30888fd-6ca8-48dd-9be3-558f81ae1015 @size = 5>
+    #             level    program    contact      group
+    #      0  Undergrad       CHEM        nil       base
+    #      1  Undergrad       CHEM       true  intensive
+    #      2       Grad       CHEM       true       base
+    #      3  Undergrad       NURS        nil  intensive
+    #      4    Unknown       CHEM       true       base
+    class DataFrameSieve < Transform
+      def initialize(sieve_df, *args, **kargs, &block)
+        super
+        @sieve_df = sieve_df.transpose.to_h.values
+      end
+
+      def transform(*values)
+        sieve_keys = @sieve_df.first.index.to_a
+        sieve_result_key = sieve_keys.pop
+
+        @sieve_df.each.find do |sieve_row|
+          match_row = true
+          sieve_keys.each_with_index do |key,idx|
+            match_row &&= sieve_row[key].nil? || sieve_row[key] == values[idx]
+          end
+          match_row
+        end[sieve_result_key]
+      end
+    end
+
+
+    # Public: Used to partition elements into groups (buckets).
+    #
+    # buckets            - A hash where the keys are groups and the values are weights or percentages.
+    # current_population - A hashable object holding a count of the current number of
+    #                      elements in each bucket.
+    #
+    # Example:
+    #
+    #   # The current population has 2 record in the A bucket and 3 in B
+    #   current_pop = Daru::Vector.new([2,3], index: ['A', 'B'])
+    #
+    #   # We want to generate 7 new records that will evenly populate the A, B, and C buckets, given the current populations.
+    #   part = Remi::Transform::Partitioner.new(buckets: { 'A' => 1, 'B' => 1,'C' => 1 }, initial_population: current_pop)
+    #
+    #   1.upt(7).map { |iter| part.call } # => ["C", "C", "A", "C", "C", "B", "A"]
+    class Partitioner < Transform
+      def initialize(buckets:, initial_population: {}, **kargs, &block)
+        super
+        @buckets = buckets
+        @current_population = sanitize_initial_population(buckets, initial_population)
+      end
+
+      attr_reader :buckets
+      attr_reader :current_population
+
+      def transform(*values)
+        get_next_value
+      end
+
+      def size
+        @size ||= @current_population.reduce(0) { |sum, (group, n)| sum += n }
+      end
+
+      def total_weight
+        @total_weight ||= @buckets.reduce(0) { |sum, (bucket, weight)| sum += 1.0 * weight }
+      end
+
+      def get_next_value
+        assigned = @buckets.max_by do |(group, weight)|
+          expected = @buckets[group] / total_weight * size
+          actual = @current_population[group]
+
+          diff = expected - actual
+          if diff > 0
+            rand**(1.0 / diff)
+          else
+            -rand**(- 1.0 / @buckets[group])
+          end
+        end.first
+
+        @current_population[assigned] += 1
+        @size += 1
+
+        assigned
+      end
+
+      private
+
+      def sanitize_initial_population(buckets, dist)
+        dist = dist.to_h
+
+        zero_distribution = buckets.keys.reduce({}) { |h, group| h[group] = 0; h }
+        zero_distribution.merge(dist.select { |k,v| buckets.keys.include? k })
+      end
+    end
+
+
   end
 end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.30'
+  VERSION = '0.2.31'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.30
+  version: 0.2.31
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-29 00:00:00.000000000 Z
+date: 2016-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -193,11 +193,14 @@ files:
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb
+- features/transforms/data_frame_sieve.feature
 - features/transforms/date_diff.feature
 - features/transforms/nvl.feature
 - features/transforms/parse_date.feature
+- features/transforms/partitioner.feature
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
+- features/transforms/truthy.feature
 - jobs/aggregate_job.rb
 - jobs/all_jobs_shared.rb
 - jobs/copy_source_job.rb
@@ -206,12 +209,15 @@ files:
 - jobs/parameters_job.rb
 - jobs/sample_job.rb
 - jobs/sftp_file_target_job.rb
+- jobs/transforms/data_frame_sieve_job.rb
 - jobs/transforms/date_diff_job.rb
 - jobs/transforms/nvl_job.rb
 - jobs/transforms/parse_date_job.rb
+- jobs/transforms/partitioner_job.rb
 - jobs/transforms/prefix_job.rb
 - jobs/transforms/transform_jobs.rb
 - jobs/transforms/truncate_job.rb
+- jobs/transforms/truthy_job.rb
 - lib/remi.rb
 - lib/remi/cli.rb
 - lib/remi/cucumber.rb
@@ -277,11 +283,14 @@ test_files:
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb
+- features/transforms/data_frame_sieve.feature
 - features/transforms/date_diff.feature
 - features/transforms/nvl.feature
 - features/transforms/parse_date.feature
+- features/transforms/partitioner.feature
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
+- features/transforms/truthy.feature
 - spec/extractor/sftp_file_spec.rb
 - spec/metadata_spec.rb
 - spec/remi_spec.rb