remi 0.2.37 → 0.2.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4053acf8d3794062479455dd42afc7d3820832b5
-  data.tar.gz: 363985988b27cda161515842f3469d8b34487904
+  metadata.gz: a59538393438e759c02554c7dac61c914841e468
+  data.tar.gz: 181df9c16e528b0d1315e992fb25a97cc711c678
 SHA512:
-  metadata.gz: 5882b3ea3e2ee615e7c280108138a5e3cb8f7d30321655191c2973a083e722f974a4250ed40f1aa5f52ee610529b0078051355c160b914feb821e7f652117dff
-  data.tar.gz: 6cfff4363a00d7b030a700340da60e1b590432f67ff74e6adc96783b6e14be3078c244f890f8f4a77fddfabc1567d18cb64657375b0a882f51cc8c59a25b574c
+  metadata.gz: 92520cd0b2dc002879bfef7cfaf78e8e6f4a3609b121d23c9a04de054fcd86e9dfe8fd1d06b0cef3870a486c948a29994ed58eac6c8caefdd0fbab4d7b06fc8a
+  data.tar.gz: f50d012217b786c3fdebd97caa1a183545a6e99689f82fce9e888a933a33cd96247b31f63ab0265a80cb443a9cd9258237884a464747ebb75180e481b64d49c4

data/Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.37)
+    remi (0.2.38)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/README.md

@@ -48,6 +48,84 @@ Examples setting up a job class with
 * parameters
 * maps
 
+
+### Transform cardinality
+
+Within a source-to-target map block, there are a few different
+possible transform cardinalities: one-to-one, many-to-one, one-to-many,
+many-to-many, zero-to-one, and zero-to-many.  The lambda functions that
+are supplied to `#transfrom` method must satisfy different conditions based
+on cardinality.
+
+For all of the following examples, we'll assume that a dataframe exists defined by
+````ruby
+  df = Remi::DataFrame::Daru.new(
+    [
+      ['a1','b1','c1', ['d',1]],
+      ['a2','b2','c2', ['d',2]],
+      ['a3','b3','c3', ['d',3]],
+    ].transpose,
+    order: [:a, :b, :c, :d]
+  )
+````
+
+**one-to-one** - These maps expect a lambda that accepts the value of a
+field as an argument and returns the result of some operation, which
+is used to populate the target.
+
+````ruby
+Remi::SourceToTargetMap.apply(df) do
+  map source(:a) .target(:aprime)
+    .transform(->(v) { "#{v}prime" })
+end
+
+df[:aprime].to_a #=> ['a1prime', 'a2prime', 'a3prime']
+````
+
+**many-to-one** - These maps expect that the lambda accepts a row object as an argument
+and returns the result of the operation, which is used to populate the target.
+
+````ruby
+Remi::SourceToTargetMap.apply(df) do
+  map source(:a, :b) .target(:ab)
+    .transform(->(row) { "#{row[:a]}#{row[:b]}" })
+end
+
+df[:ab].to_a #=> ['a1b1', 'a2b2', 'a3b3']
+````
+
+**zero-to-many/one-to-many/many-to-many** - These maps expect that the
+lambda accepts a row object as an argument.  The row object is then
+modified in place, which is used to populate the targets.  The return
+value of the lambda is ignored.
+
+````ruby
+Remi::SourceToTargetMap.apply(df) do
+  map source(:a, :b) .target(:aprime, :ab)
+    .transform(->(row) {
+      row[:aprime] = row[:a]
+      row[:ab] = "#{row[:a]}#{row[:b]}" })
+    })
+end
+
+df[:aprime].to_a #=> ['a1prime', 'a2prime', 'a3prime']
+df[:ab].to_a #=> ['a1b1', 'a2b2', 'a3b3']
+````
+
+**zero-to-one** - These maps expect that the lambda accepts no arguments and returns the
+result of some operation, which is used to populate the target.
+
+````ruby
+Remi::SourceToTargetMap.apply(df) do
+  counter = 1.upto(3).to_a
+  map target(:counter)
+    .transform(->() { counter.pop })
+end
+
+df[:counter].to_a #=> [1, 2, 3]
+````
+
+
 ## Business Rules
 
 TODO: Description of writing Business Rules.

data/features/json.feature

@@ -7,7 +7,6 @@ Feature: This tests using json data in tests.
     And the source 'Source Data'
     And the target 'Target Data'
 
-
   Scenario: Using JSON data in an example record.
 
     Given the following example record for 'Source Data':

data/features/transforms/concatenate.feature

@@ -0,0 +1,30 @@
+Feature: Test the concatenate transformer.
+
+  Background:
+    Given the job is 'Concatenate'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+  Scenario Outline: Performing a concatenation
+    Given the source 'Source Data'
+    And the target 'Target Data'
+
+    And the source field 'Field1' is set to the value "<Field1>"
+    And the source field 'Field2' is set to the value "<Field2>"
+    And the source field 'Field3' is set to the value "<Field3>"
+    And the job parameter 'delimiter' is "<Delimiter>"
+    Then the target field 'Result Field' is set to the value "<Expected>"
+
+    Examples:
+      | Field1 | Field2 | Field3 | Delimiter | Expected |
+      | A      | B      | C      | ,         | A,B,C    |
+      |        | B      | C      | -         | B-C      |
+      |        |        | C      | ,         | C        |
+      |        |        |        | ,         |          |
+
+
+  Scenario: Testing a concatenation with the short form version
+    Given the source 'Source Data'
+    And the target 'Target Data'
+
+    Then the target field 'Result Field' is a concatenation of the source fields 'Field1', 'Field2', 'Field3', delimited by ","

data/features/transforms/date_diff.feature

@@ -8,8 +8,7 @@ Feature: Tests the date_diff transform
     And the source 'Source Data'
     And the target 'Target Data'
 
-
-  Scenario Outline: Calculating date difference in days2.
+  Scenario Outline: Calculating date difference in days.
     Given the job parameter 'measure' is "days"
     And the source field 'Date1' has the value "<Date1>"
     And the source field 'Date2' has the value "<Date2>"

data/jobs/json_job.rb

@@ -18,13 +18,9 @@ class JsonJob
   define_transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields) do
       map source(:json_array) .target(:second_element)
-        .transform(->(*values) { values[1] })
-      # This is NOT the way I would like it to work, but we need to do some work on STTM first
+        .transform(->(values) { values[1] })
       map source(:json_hash) .target(:name_field)
-        .transform(->(*json_hash) { json_hash.to_h['name'] })
-      # preferred
-#      map source(:json_hash) .target(:name_field)
-#        .transform(->(json_hash) { json_hash['name'] })
+        .transform(->(json_hash) { json_hash['name'] })
     end
   end
 end

data/jobs/parameters_job.rb

@@ -13,7 +13,7 @@ class ParametersJob
 
   define_transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
-      map source(nil) .target(:myparam)
+      map target(:myparam)
         .transform(Remi::Transform::Constant.new(params[:myparam]))
       map source(:parameter_name) .target(:parameter_name)
         .transform(->(v) { params[v.to_sym] })

data/jobs/sample_job.rb

@@ -92,9 +92,12 @@ class SampleJob
     Remi::SourceToTargetMap.apply(all_contacts.df) do
 
       # Prefixes source id record and then looks up existing salesforce Id
+      prefixer = Remi::Transform::Prefix.new('SAMP')
       map source(:student_id) .target(:External_ID__c, :Id)
-        .transform(Remi::Transform::Prefix.new('SAMP'))
-        .transform(->(v) { [v, Remi::Transform::Lookup.new(student_id_to_sf_id).call(v)] })
+        .transform(->(row) {
+          row[:External_ID__c] = prefixer.call(row[:student_id])
+          row[:Id] = student_id_to_sf_id[row[:External_ID__c]]
+        })
     end
   end
 
@@ -102,9 +105,11 @@ class SampleJob
   define_transform :map_creates, sources: :all_contacts, targets: :contact_creates do
 
     work_contact_creates = all_contacts.df.where(all_contacts.df[:Id].eq(nil))
+
     Remi::SourceToTargetMap.apply(work_contact_creates) do
 
       map source(:school_id)           .target(:School_ID__c)
+
       map source(:school_name)         .target(:School_Name__c)
       map source(:first_name)          .target(:FirstName)
         .transform(Remi::Transform::IfBlank.new('Not Provided'))
@@ -122,16 +127,20 @@ class SampleJob
         .transform(Remi::Transform::FormatDate.new(in_format: sample_file.fields[:applied_date][:in_format]))
 
       map source(:mailing_address_line_1, :mailing_address_line_2) .target(:MailingStreet)
-        .transform(->(line_1, line_2) {
-           Remi::Transform::IfBlank.new(nil).call(line_1).nil? ? [] : [line_1, line_2]
-           })
-        .transform(Remi::Transform::Concatenate.new(', '))
-
+        .transform(->(row) {
+          if row[:mailing_address_line_1].blank?
+            ''
+          else
+            [row[:mailing_address_line_1], row[:mailing_address_line_2]].join(', ')
+          end
+        })
+
+      if_blank_unknown = Remi::Transform::IfBlank.new("Unknown")
       map source(:school_id, :school_name) .target(:School__c)
-        .transform(->(id, name) {[
-            Remi::Transform::IfBlank.new("Unknown").call(id),
-            Remi::Transform::IfBlank.new("Unknown").call(name)
-          ]})
+        .transform(->(row) {
+          row[:school_id] = if_blank_unknown.call(row[:school_id])
+          row[:school_name] = if_blank_unknown.call(row[:school_name])
+        })
         .transform(Remi::Transform::Concatenate.new('-'))
 
       map source(:current_email)       .target(:Email)

data/jobs/transforms/concatenate_job.rb

@@ -0,0 +1,21 @@
+require_relative '../all_jobs_shared'
+
+class ConcatenateJob
+  include AllJobsShared
+
+  define_param :delimiter, ','
+  define_source :source_data, Remi::DataSource::DataFrame,
+    fields: {
+      :field1 => {},
+      :field2 => {},
+      :field3 => {}
+    }
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
+      map source(:field1, :field2, :field3) .target(:result_field)
+        .transform(Remi::Transform::Concatenate.new(params[:delimiter]))
+    end
+  end
+end

data/jobs/transforms/date_diff_job.rb

@@ -14,7 +14,10 @@ class DateDiffJob
   define_transform :main, sources: :source_data, targets: :target_data do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:date1, :date2) .target(:difference)
-        .transform(->(d1,d2) { [Date.strptime(d1), Date.strptime(d2)] })
+        .transform(->(row) {
+          row[:date1] = Date.strptime(row[:date1])
+          row[:date2] = Date.strptime(row[:date2])
+        })
         .transform(Remi::Transform::DateDiff.new(params[:measure]))
     end
   end

data/jobs/transforms/partitioner_job.rb

@@ -28,7 +28,7 @@ class PartitionerJob
     current_population_hash = current_population.df.map(:row) { |row| [row[:group], row[:count].to_i] }.to_h
 
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
-      map source(nil) .target(:group)
+      map target(:group)
         .transform(Remi::Transform::Partitioner.new(buckets: distribution_hash, initial_population: current_population_hash))
     end
   end

data/lib/remi/source_to_target_map/map.rb

@@ -0,0 +1,209 @@
+module Remi
+  class SourceToTargetMap
+
+    # Public: Class used to perform source to target mappings.
+    #
+    # Examples
+    #
+    #   # One-to-one map
+    #   map = Map.new(source_df, target_df)
+    #   map.source(:a).target(:aprime)
+    #     .transform(->(v) { "#{v}prime" })
+    #   # see tests for more
+    class Map
+
+      # Public: Initializes a map
+      #
+      # source_df       - The source dataframe.
+      # target_df       - The target dataframe (default: source_df).
+      # source_metadata - Metadata (Remi::Fields) for the source fields.
+      # target_metadata - Metadata (Remi::Fields) for the target fields.
+      def initialize(source_df, target_df, source_metadata: Remi::Fields.new, target_metadata: Remi::Fields.new)
+        @source_df = source_df
+        @target_df = target_df
+
+        @source_metadata = source_metadata
+        @target_metadata = target_metadata
+
+        @source_vectors  = []
+        @target_vectors  = []
+        @transforms      = []
+        @transform_procs = []
+      end
+
+      # Public: Returns the map's source dataframe
+      attr_reader :source_df
+
+      # Public: Returns the map's target dataframe
+      attr_reader :target_df
+
+      # Public: Returns all of the map's source vectors
+      attr_reader :source_vectors
+
+      # Public: Returns all of the map's target vectors
+      attr_reader :target_vectors
+
+      # Public: Returns all of the map's defined transforms
+      attr_reader :transforms
+
+
+      # Public: Adds a list of source vectors to a map
+      #
+      # source_vectors - A list of source vectors.
+      #
+      # Returns self
+      def source(*source_vectors)
+        @source_vectors += Array(source_vectors)
+        self
+      end
+
+      # Public: Adds a list of target vectors to a map
+      #
+      # target_vectors - A list of target vectors.
+      #
+      # Returns self
+      def target(*target_vectors)
+        @target_vectors += Array(target_vectors)
+        self
+      end
+
+      # Public: Adds a transform to the map
+      # A transform is an object that behaves like a proc and responds
+      # to #call and #to_proc.  This method returns self, so transforms
+      # may be chained.  They will be executed in the order that they are
+      # applied to the map.
+      #
+      # tform - The transform to add
+      #
+      # Returns self
+      def transform(tform)
+        @transforms << tform
+        @transform_procs << tform.to_proc
+        self
+      end
+
+      # Public: Executes the map defined by the source vectors, target vectors, and transforms.
+      #
+      # Returns the target dataframe.
+      def execute
+        inject_transforms_with_metadata
+        set_default_transform
+        map_to_target_df
+      end
+
+      # Public: Returns the number of source vectors defined
+      def source_cardinality
+        @source_vectors.size
+      end
+
+      # Public: Returns the number of target vectors defined
+      def target_cardinality
+        @target_vectors.size
+      end
+
+
+
+
+      private
+
+      def inject_transforms_with_metadata
+        @transforms.each do |tform|
+          if tform.respond_to? :source_metadata
+            meta = @source_vectors.map { |v| @source_metadata[v] || {} }
+            tform.source_metadata = meta.size > 1 ? meta : meta.first
+          end
+          if tform.respond_to? :target_metadata
+            meta = @target_vectors.map { |v| @target_metadata[v] || {} }
+            tform.target_metadata = meta.size > 1 ? meta : meta.first
+          end
+        end
+      end
+
+      # Private: If no transforms are defined, assume it's a simple copy
+      def set_default_transform
+        if @transforms.size == 0
+          transform(->(v) { v })
+        end
+      end
+
+      # Private: Converts the transformed data into vectors in the target dataframe.
+      def map_to_target_df
+        result_hash_of_arrays.each do |vector, values|
+          @target_df[vector] = Daru::Vector.new(values, index: @source_df.index)
+        end
+
+        @target_df
+      end
+
+      # Private: Splits the transformed rows into separate arrays, indexed by vector name
+      def result_hash_of_arrays
+        result = @target_vectors.each_with_object({}) { |v,h| h[v] = [] }
+
+        transformed_rows.each do |result_row|
+          result.keys.each do |vector|
+            result[vector] << result_row[vector]
+          end
+        end
+
+        result
+      end
+
+      # Private: Applies all of the transforms to each row.
+      def transformed_rows
+        work_rows.map do |row|
+          @transform_procs.each do |tform|
+            result = call_transform(tform, row)
+            row[*@target_vectors] = result if target_cardinality == 1
+            row[*@source_vectors] = result if source_cardinality == 1 && target_cardinality == 1
+          end
+
+          row
+        end
+      end
+
+      # Private: Applies the given transform to the given row.
+      #
+      # tform - The transform (proc).
+      # row   - The row.
+      #
+      # Returns the return value of the transform.
+      def call_transform(tform, row)
+        if source_cardinality == 0 && target_cardinality == 1
+          tform.call
+        elsif source_cardinality == 1 && target_cardinality == 1
+          tform.call(row[*@source_vectors])
+        else
+          tform.call(row)
+        end
+      end
+
+      # Private: Returns a unique list of all vectors (source and target) invovled in the map.
+      def all_vectors
+        @all_vectors ||= (@source_vectors + @target_vectors).uniq
+      end
+
+      # Private: Returns a hash that maps vector names to an index
+      # The index is the position of the vector value for a row in #work_rows
+      def rows_index
+        @rows_index ||= all_vectors.each_with_index.to_h
+      end
+
+      # Private: Converts all of vectors involved in the map into an array of row objects.
+      def work_rows
+        all_vectors.map do |vector|
+          is_source_vector = @source_vectors.include? vector
+
+          if is_source_vector && @source_df.vectors.include?(vector)
+            @source_df[vector].to_a
+          elsif is_source_vector && @target_df.vectors.include?(vector)
+            @target_df[vector].to_a
+          else
+            Array.new(@source_df.size)
+          end
+        end.transpose.map do |row_as_array|
+          Row.new(rows_index, row_as_array, source_keys: @source_vectors)
+        end
+      end
+    end
+  end
+end

data/lib/remi/source_to_target_map/row.rb

@@ -0,0 +1,99 @@
+module Remi
+  class SourceToTargetMap
+
+    # Public: A row is composed of an array and an index hash.
+    # The index hash converts a key into a number representing the position in the array.
+    # Functionally, it's very similar to how a hash works.  However,
+    # we need to create a lot of Row objects that all have the same
+    # index hash.  All of those row objects can reference the same
+    # index hash object and thus dramatically reduce the amount of memory
+    # needed store a lot of rows.
+    #
+    # Examples
+    #
+    #  row = Row.new({ a: 1, b: 2}, ['alpha', 'beta'])
+    #  row[:a] #=> 'alpha'
+    #  row[:b] #=> 'beta'
+    class Row
+
+      # Public: Converts hash-like objects into rows, array-like objects into rows,
+      # or just returns a row if one is provied.
+      #
+      # arg - A Row, array-like object, or hash-like object.
+      #
+      # Examples:
+      #
+      #   Row[{ a: 'one', b: 'two' }] #=> #<Row @index={:a=>0, :b=>1} @values=["one", "two"]>
+      # Returns a Row
+      def self.[](arg)
+        return arg if arg.is_a? Row
+
+        if arg.respond_to? :keys
+          Row.new(arg.keys.each_with_index.to_h, arg.values)
+        else
+          Row.new(0.upto(arg.size).each_with_index.to_h, arg)
+        end
+      end
+
+
+      # Public: Initializes a row object.
+      #
+      # index       - A hash containing keys that are usually symbols and values that
+      #               represent a position in the values array.
+      # values      - An array of values.
+      # source_keys - Array of keys that should be treated as data
+      #               sources for a row transformation
+      def initialize(index, values, source_keys: nil)
+        @index = index
+        @inverted_index = index.invert
+        @values = values
+        @source_keys = source_keys || index.keys
+      end
+
+      # Public: Returns the value of the row array for the given key
+      def [](key)
+        @values[@index[key]]
+      end
+
+      # Public: Sets the value of the row array for the given key
+      def []=(key, value)
+        @values[@index[key]] = value
+      end
+
+      # Public: Makes Row enumerable, and acts like a hash.
+      def each &block
+        @values.each_with_index { |value, idx| block.call([@inverted_index[idx], value]) }
+      end
+
+      def each_source &block
+        Enumerator.new do |y|
+          source_keys.each { |key| y << [key, self[key]] }
+        end
+      end
+
+      def each_target &block
+        Enumerator.new do |y|
+          target_keys.each { |key| y << [key, self[key]] }
+        end
+      end
+
+      # Public: Returns the values stored in the row.
+      def to_a
+        @values
+      end
+
+      # Public: Returns the keys of the index.
+      def keys
+        @index.keys
+      end
+
+      def source_keys
+        @source_keys
+      end
+
+      def target_keys
+        @target_keys ||= keys - source_keys
+      end
+    end
+  end
+end

data/lib/remi/source_to_target_map.rb

@@ -1,117 +1,82 @@
 module Remi
+
+  # Public: Class used to define a DSL for source to target maps.
+  #
+  # Examples
+  #
+  #   SourceToTargetMap.apply(df) do
+  #     map source(:a) .target(:aprime)
+  #       .transform(->(v) { "#{v}prime" })
+  #     map source(:a) .target(:aup)
+  #       .transform(->(v) { "#{v.upcase}" })
+  #   end
+  #   #=> <Daru::DataFrame:70291322684920 @name = 8c546a52-c1a7-495a-996a-7f352b0087b7 @size = 3>
+  #                         a     aprime       aup
+  #              0         a1    a1prime        A1
+  #              1         a2    a2prime        A2
+  #              2         a3    a3prime        A3
   class SourceToTargetMap
+
+    # Public: Initializes the SourceToTargetMap DSL
+    #
+    # source_df       - The source dataframe.
+    # target_df       - The target dataframe (default: source_df).
+    # source_metadata - Metadata (Remi::Fields) for the source fields.
+    # target_metadata - Metadata (Remi::Fields) for the target fields.
     def initialize(source_df, target_df=nil, source_metadata: Remi::Fields.new, target_metadata: Remi::Fields.new)
       @source_df = source_df
       @source_metadata = source_metadata
 
-      if target_df
-        @target_df = target_df
-        @target_metadata = target_metadata
-      else
-        @target_df = @source_df
-        @target_metadata = @source_metadata
-      end
-
-      reset_map
+      @target_df = target_df || source_df
+      @target_metadata = target_metadata || source_metadata
     end
 
+    attr_reader :source_df, :target_df
+
+    # Public: Expects a block in which the DSL will be applied.
+    #
+    # Same arguments as the constructor.
+    #
+    # Returns the target dataframe.
     def self.apply(source_df, target_df=nil, source_metadata: Remi::Fields.new, target_metadata: Remi::Fields.new, &block)
       sttm = SourceToTargetMap.new(source_df, target_df, source_metadata: source_metadata, target_metadata: target_metadata)
       Docile.dsl_eval(sttm, &block)
+      target_df || source_df
     end
 
+    # Public: Adds a list of source vectors to a new mapping.
+    #
+    # source_vectors - A list of vector names.
+    #
+    # Returns a SourceToTargetMap::Map with the defined source vectors.
     def source(*source_vectors)
-      @source_vectors = Array(source_vectors)
-      self
-    end
-
-    def transform(*transforms)
-      @transforms += Array(transforms)
-      @transform_procs += Array(transforms).map { |t| t.to_proc }
-      self
+      new_map.source(*source_vectors)
     end
 
+    # Public: Adds a list of targets vectors to a new mapping.
+    #
+    # target_vectors - A list of target names.
+    #
+    # Returns a SourceToTargetMap::Map with the defined target vectors.
     def target(*target_vectors)
-      @target_vectors = Array(target_vectors)
-      self
-    end
-
-    def reset_map
-      @source_vectors = []
-      @target_vectors = []
-      @transforms = []
-      @transform_procs = []
+      new_map.target(*target_vectors)
     end
 
-    def map(*args)
-      inject_transform_with_metadata
-
-      case
-      when @source_vectors.include?(nil)
-        do_map_generic
-      when @source_vectors.size == 1 && @transforms.size == 0
-        do_map_direct_copy
-      when @source_vectors.size == 1 && @target_vectors.size == 1
-        do_map_single_source_and_target_vector
-      else
-        do_map_generic
-      end
-      reset_map
+    # Public: Executes a mapping.
+    #
+    # defined_map - The SourceToTargetMap::Map object to execute
+    #
+    # Returns the target dataframe.
+    def map(defined_map)
+      defined_map.execute
     end
 
 
-
     private
 
-    def inject_transform_with_metadata
-      @transforms.each do |tform|
-        if tform.respond_to? :source_metadata
-          meta = @source_vectors.map { |v| @source_metadata[v] || {} }
-          tform.source_metadata = meta.size > 1 ? meta : meta.first
-        end
-        if tform.respond_to? :target_metadata
-          meta = @target_vectors.map { |v| @target_metadata[v] || {} }
-          tform.target_metadata = meta.size > 1 ? meta : meta.first
-        end
-      end
-    end
-
-    def do_map_direct_copy
-      @target_vectors.each do |target_vector|
-        @target_df[target_vector] = @source_df[@source_vectors.first].dup
-      end
-    end
-
-    def do_map_single_source_and_target_vector
-      @target_df[@target_vectors.first] = @source_df[@source_vectors.first].recode do |vector_value|
-        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value.nil? ? [nil] : value)) }
-      end
-    end
-
-    def do_map_generic
-      work_vector = if @source_vectors.size == 1 && @source_vectors.first != nil
-        @source_df[@source_vectors.first].dup
-      elsif @source_vectors.size > 1
-        # It's faster to zip together several vectors and recode those than it is to
-        # recode a dataframe row by row!
-        Daru::Vector.new(@source_df[@source_vectors.first].zip(*@source_vectors[1..-1].map { |name| @source_df[name] }), index: @source_df.index)
-      else
-        Daru::Vector.new([], index: @source_df.index)
-      end
-
-      work_vector.recode! do |vector_value|
-        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value.nil? ? [nil] : value)) }
-      end
-
-      @target_vectors.each_with_index do |target_vector, vector_idx|
-        @target_df[target_vector] = work_vector.recode do |vector_value|
-          if vector_value.is_a?(Array) then
-            vector_value[vector_idx]
-          else
-            vector_value
-          end
-        end
-      end
+    # Public: Returns a new SourceToTargetMap::Map
+    def new_map
+      Map.new(@source_df, @target_df, source_metadata: @source_metadata, target_metadata: @target_metadata)
     end
   end
 end

data/lib/remi/transform.rb

@@ -35,11 +35,11 @@ module Remi
     # values - The values to be transformed.
     #
     # Returns the transformed value.
-    def call(*values)
-      if @multi_args
-        to_proc.call(*values)
+    def call(*args)
+      if to_proc.arity == 0
+        to_proc.call
       else
-        to_proc.call(Array(values).first)
+        to_proc.call(*args)
       end
     end
 
@@ -135,8 +135,9 @@ module Remi
         @delimiter = delimiter
       end
 
-      def transform(*values)
-        Array(values).join(@delimiter)
+      def transform(row)
+        row = SourceToTargetMap::Row[row]
+        row.each_source.map { |key, value| value.blank? ? nil : value }.compact.join(@delimiter)
       end
     end
 
@@ -188,8 +189,9 @@ module Remi
         @default = default
       end
 
-      def transform(*values)
-        Array(values).find(->() { @default }) { |arg| !arg.blank? }
+      def transform(row)
+        row = SourceToTargetMap::Row[row]
+        row.each_source.find(->() { [nil, @default] }) { |key, value| !value.blank? }[1]
       end
     end
 
@@ -338,7 +340,10 @@ module Remi
         @measure = measure
       end
 
-      def transform(from_date, to_date)
+      def transform(row)
+        row = SourceToTargetMap::Row[row]
+        from_date = row[row.keys[0]]
+        to_date = row[row.keys[1]]
 
         case @measure.to_sym
         when :days
@@ -366,7 +371,7 @@ module Remi
         @constant = constant
       end
 
-      def transform(values)
+      def transform
         @constant
       end
     end
@@ -563,9 +568,10 @@ module Remi
     # wildcards and match anything.  The first row that matches wins
     # and the sieve progression stops.
     #
-    # sieve_df - The sieve, defined as a dataframe.  The arguments
-    #            to the transform must appear in the same order as the
-    #            first N-1 columns of the sieve.
+    # sieve_df - The sieve, defined as a dataframe.  The names of the
+    #            sieve vectors must correspond to the names of the
+    #            vectors in the dataframe source to target map.  The
+    #            last vector in the sieve_df is used as the result of the sieve.
     #
     #
     # Examples:
@@ -612,23 +618,26 @@ module Remi
     class DataFrameSieve < Transform
       def initialize(sieve_df, *args, **kargs, &block)
         super
-        @sieve_df = sieve_df.transpose.to_h.values
+        @sieve_table = sieve_df.transpose.to_h.values
       end
 
-      def transform(*values)
-        sieve_keys = @sieve_df.first.index.to_a
+
+      def transform(row)
+        sieve_keys = @sieve_table.first.index.to_a
         sieve_result_key = sieve_keys.pop
 
-        @sieve_df.each.find do |sieve_row|
+        raise ArgumentError, "#{sieve_keys - row.source_keys} not found in row" unless (sieve_keys - row.source_keys).size == 0
+
+        @sieve_table.each.find do |sieve_row|
           match_row = true
-          sieve_keys.each_with_index do |key,idx|
-            match_value = if sieve_row[key].is_a?(Regexp)
-              !!sieve_row[key].match(values[idx])
-            else
-              sieve_row[key] == values[idx]
-            end
-
-            match_row &&= sieve_row[key].nil? || match_value
+          sieve_keys.each do |sieve_key|
+            match_value = if sieve_row[sieve_key].is_a?(Regexp)
+                            !!sieve_row[sieve_key].match(row[sieve_key])
+                          else
+                            sieve_row[sieve_key] == row[sieve_key]
+                          end
+
+            match_row &&= sieve_row[sieve_key].nil? || match_value
           end
           match_row
         end[sieve_result_key]
@@ -661,7 +670,7 @@ module Remi
       attr_reader :buckets
       attr_reader :current_population
 
-      def transform(*values)
+      def transform
         get_next_value
       end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.37'
+  VERSION = '0.2.38'
 end

data/lib/remi.rb

@@ -38,6 +38,8 @@ require 'remi/version.rb'
 require 'remi/settings'
 require 'remi/job'
 require 'remi/source_to_target_map'
+require 'remi/source_to_target_map/map'
+require 'remi/source_to_target_map/row'
 require 'remi/field_symbolizers'
 
 require 'remi/refinements/symbolizer'

data/spec/source_to_target_map_spec.rb

@@ -0,0 +1,301 @@
+require_relative 'remi_spec'
+
+describe SourceToTargetMap do
+  let(:df) do
+    Remi::DataFrame::Daru.new(
+      [
+        ['a1','b1','c1', ['d',1]],
+        ['a2','b2','c2', ['d',2]],
+        ['a3','b3','c3', ['d',3]],
+      ].transpose,
+      order: [:a, :b, :c, :d]
+    )
+  end
+
+
+  let(:map) { SourceToTargetMap::Map.new(df, df) }
+
+  describe 'one-to-one maps' do
+    shared_examples_for 'one-to-one map' do
+      it 'provides a value to the transform, and expects a return value' do
+        expect(result).to eq ['a1prime', 'a2prime', 'a3prime']
+      end
+
+      it 'accepts chained transformations with the same source/target cardinality' do
+        map.transform(->(v) { "#{v}-prime" })
+        expect(result).to eq ['a1prime-prime', 'a2prime-prime', 'a3prime-prime']
+      end
+    end
+
+    context 'standard use' do
+      before { map.source(:a) .target(:aprime) .transform(->(v) { "#{v}prime" }) }
+
+      let(:result) do
+        map.execute
+        df[:aprime].to_a
+      end
+
+      it_behaves_like 'one-to-one map'
+    end
+
+    context 'the source and target have the same name' do
+      before { map.source(:a) .target(:a) .transform(->(v) { "#{v}prime" }) }
+
+      let(:result) do
+        map.execute
+        df[:a].to_a
+      end
+
+      it_behaves_like 'one-to-one map'
+    end
+
+    context 'without any transforms', wip: true do
+      before { map.source(:a) .target(:aprime) }
+
+      let(:result) do
+        map.execute
+        df[:aprime].to_a
+      end
+
+      it 'copies data from source to target' do
+        expect(result).to eq ['a1', 'a2', 'a3']
+      end
+
+    end
+
+    context 'source and target dataframe are different' do
+      let(:map) { SourceToTargetMap::Map.new(df, df_target) }
+
+      context 'vectors referenced in the source only exist on the target' do
+        let(:df_target) do
+          Remi::DataFrame::Daru.new({ a_in_target: [ 'a1target', 'a2target', 'a3target' ] }, index: df.index)
+        end
+
+        before { map.source(:a_in_target) .target(:aprime) .transform(->(v) { "#{v}prime" }) }
+
+        let(:result) do
+          map.execute
+          df_target[:aprime].to_a
+        end
+
+        it 'uses the target values' do
+          expect(result).to eq ['a1targetprime', 'a2targetprime', 'a3targetprime']
+        end
+      end
+
+      context 'vectors referenced in the source exist on both source and target' do
+        let(:df_target) do
+          Remi::DataFrame::Daru.new({ a: [ 'a1target', 'a2target', 'a3target' ] }, index: df.index)
+        end
+
+        before { map.source(:a) .target(:aprime) .transform(->(v) { "#{v}prime" }) }
+
+        let(:result) do
+          map.execute
+          df_target[:aprime].to_a
+        end
+
+        it 'uses the source values' do
+          expect(result).to eq ['a1prime', 'a2prime', 'a3prime']
+        end
+      end
+    end
+
+  end
+
+  describe 'one-to-one maps where the source and target have the same name' do
+    before { map.source(:a) .target(:a) .transform(->(v) { "#{v}prime" }) }
+
+    let(:result) do
+      map.execute
+      df[:a].to_a
+    end
+
+    it 'provides a value to the transform, and expects a return value' do
+      expect(result).to eq ['a1prime', 'a2prime', 'a3prime']
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->(v) { "#{v}-prime" })
+      expect(result).to eq ['a1prime-prime', 'a2prime-prime', 'a3prime-prime']
+    end
+  end
+
+  describe 'many-to-one maps' do
+    before { map.source(:a,:b) .target(:ab) .transform(->(row) { row[:a] + row[:b] }) }
+
+    let(:result) do
+      map.execute
+      df[:ab].to_a
+    end
+
+    it 'provides a row to the transform, and expects a return value' do
+      expect(result).to eq ['a1b1', 'a2b2', 'a3b3']
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->(row) { "-#{row[:ab]}-" })
+      expect(result).to eq ['-a1b1-', '-a2b2-', '-a3b3-']
+    end
+  end
+
+  describe 'one-to-many maps' do
+    before do
+      map.source(:a) .target(:a_col, :a_row)
+        .transform(->(row) {
+          row[:a_col] = row[:a][0]
+          row[:a_row] = row[:a][1]
+        })
+    end
+
+    let(:result) do
+      map.execute
+      df[:a_col, :a_row].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+    end
+
+    it 'provides a row to the transform and expects the row to be populated' do
+      expect(result).to eq({ :a_col => ['a', 'a', 'a'], :a_row => ['1', '2', '3'] })
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->(row) {
+          row[:a_col] = "COL#{row[:a_col]}"
+          row[:a_row] = "ROW#{row[:a_row]}"
+        })
+
+      expect(result).to eq({ :a_col => ['COLa', 'COLa', 'COLa'], :a_row => ['ROW1', 'ROW2', 'ROW3'] })
+    end
+  end
+
+  describe 'many-to-many maps' do
+    before do
+      map.source(:b, :c) .target(:b_is_c, :c_is_b)
+        .transform(->(row) {
+          row[:b], row[:c] = row[:c], row[:b]
+          row[:b_is_c] = row[:b]
+          row[:c_is_b] = row[:c]
+        })
+    end
+
+    let(:result) do
+      map.execute
+      df[:b_is_c, :c_is_b].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+    end
+
+    it 'provides a row to the transform and expects the row to be populated' do
+      expect(result).to eq({ :b_is_c => ['c1', 'c2', 'c3'], :c_is_b => ['b1', 'b2', 'b3'] })
+    end
+
+    it 'does not modify source vectors' do
+      map.execute
+      source_vectors = df[:b, :c].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+      expect(source_vectors).to eq({ :b => ['b1', 'b2', 'b3'], :c => ['c1', 'c2', 'c3'] })
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->(row) {
+          row[:b_is_c] = row[:b_is_c].reverse
+          row[:c_is_b] = row[:c_is_b].reverse
+        })
+
+      expect(result).to eq({ :b_is_c => ['1c', '2c', '3c'], :c_is_b => ['1b', '2b', '3b'] })
+    end
+  end
+
+  describe 'zero-to-one maps' do
+    before do
+      values = ['x1', 'x2', 'x3']
+      map.target(:x) .transform(->() { values.shift })
+    end
+
+    let(:result) do
+      map.execute
+      df[:x].to_a
+    end
+
+    it 'expects no argument and expects a return value' do
+      expect(result).to eq ['x1', 'x2', 'x3']
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->() { 'useless' })
+      expect(result).to eq ['useless']*3
+    end
+  end
+
+  describe 'zero-to-many maps' do
+    before do
+      values = ['x1', 'x2', 'x3']
+      map.target(:x_col, :x_row)
+        .transform(->(row) {
+          x = values.shift
+          row[:x_col] = x[0]
+          row[:x_row] = x[1]
+        })
+    end
+
+    let(:result) do
+      map.execute
+      df[:x_col, :x_row].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+    end
+
+    it 'provides a row to the transform and expects the row to be populated' do
+      expect(result).to eq({ :x_col => ['x', 'x', 'x'], :x_row => ['1', '2', '3'] })
+    end
+
+    it 'accepts chained transformations with the same source/target cardinality' do
+      map.transform(->(row) { row[:x_row] = "ROW#{row[:x_row]}" })
+      expect(result).to eq({ :x_col => ['x', 'x', 'x'], :x_row => ['ROW1', 'ROW2', 'ROW3'] })
+    end
+  end
+
+  describe 'vectors containing arrays' do
+    it 'provides the array as a value the transform with one-to-one maps' do
+      map.source(:d) .target(:dprime)
+        .transform(->(v) { v.join('-') })
+      map.execute
+
+      expect(df[:dprime].to_a).to eq ['d-1', 'd-2', 'd-3']
+    end
+
+    it 'provides the array in the row with one-to-many maps' do
+      map.source(:d) .target(:d_col, :d_row)
+        .transform(->(row) {
+          row[:d_col] = row[:d].first
+          row[:d_row] = row[:d].last
+        })
+      map.execute
+
+      result = df[:d_col, :d_row].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+      expect(result).to eq({ :d_col => ['d', 'd', 'd'], :d_row => [1, 2, 3] })
+    end
+  end
+
+  describe 'using the DSL' do
+    let(:sttm) do
+      SourceToTargetMap.apply(df) do
+        map source(:a) .target(:aprime)
+          .transform(->(v) { "#{v}prime" })
+        map source(:a) .target(:aprimeprime)
+          .transform(->(v) { "#{v}prime" })
+          .transform(->(v) { "#{v}-prime" })
+        map source(:a, :d) .target(:ad)
+          .transform(->(row) { "#{row[:a][0]}-#{row[:d].first}-#{row[:d].last}" })
+      end
+    end
+
+    it 'allows one to specify multiple source-to-target maps in one block' do
+      sttm
+      result = df[:aprime, :aprimeprime, :ad].to_h.each_with_object({}) { |(k,v), h| h[k] = v.to_a }
+      expect(result).to eq({
+        :aprime => ['a1prime', 'a2prime', 'a3prime'],
+        :aprimeprime => ['a1prime-prime', 'a2prime-prime', 'a3prime-prime'],
+        :ad => ['a-d-1', 'a-d-2', 'a-d-3']
+      })
+    end
+
+    it 'returns a dataframe' do
+      expect(sttm).to be_a(Remi::DataFrame::Daru)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.37
+  version: 0.2.38
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-21 00:00:00.000000000 Z
+date: 2016-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -194,6 +194,7 @@ files:
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb
+- features/transforms/concatenate.feature
 - features/transforms/data_frame_sieve.feature
 - features/transforms/date_diff.feature
 - features/transforms/nvl.feature
@@ -211,6 +212,7 @@ files:
 - jobs/parameters_job.rb
 - jobs/sample_job.rb
 - jobs/sftp_file_target_job.rb
+- jobs/transforms/concatenate_job.rb
 - jobs/transforms/data_frame_sieve_job.rb
 - jobs/transforms/date_diff_job.rb
 - jobs/transforms/nvl_job.rb
@@ -244,6 +246,8 @@ files:
 - lib/remi/settings.rb
 - lib/remi/sf_bulk_helper.rb
 - lib/remi/source_to_target_map.rb
+- lib/remi/source_to_target_map/map.rb
+- lib/remi/source_to_target_map/row.rb
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
@@ -259,6 +263,7 @@ files:
 - spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
+- spec/source_to_target_map_spec.rb
 - spec/transform_spec.rb
 - workbooks/sample_workbook.ipynb
 - workbooks/workbook_helper.rb
@@ -299,6 +304,7 @@ test_files:
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb
+- features/transforms/concatenate.feature
 - features/transforms/data_frame_sieve.feature
 - features/transforms/date_diff.feature
 - features/transforms/nvl.feature
@@ -319,4 +325,5 @@ test_files:
 - spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
+- spec/source_to_target_map_spec.rb
 - spec/transform_spec.rb