remi 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e96cbd80b9948b85124da0bb1a7b5618c0692ec
-  data.tar.gz: 8eee478cbfdb78fc3c670c5fb2978f076b9d4673
+  metadata.gz: 92b5bf2de20c5287b1879fb4602e39e71da6f6a7
+  data.tar.gz: 05071d5457f84bbc42cf14fc4ee566c658492962
 SHA512:
-  metadata.gz: c355c76c0abbbd1bb9f37ff7d4b384b8c54772bbdfad3d1eff0a4531c3da063e9bb777aca0e69936cd5047c7f059b1f393da889b9ac047cbe0d77411d9e9fc18
-  data.tar.gz: 87df4f46a2f4f1127ccd885baf89e2466829dd56db840ce2d227c2ae8a7a96ec145873aac2ab9220a15b78158b8ff0afde56f314a6ecfa3a8c5a5523df89a154
+  metadata.gz: d41fdb4d52f15b3632538fffc3d987c35b204ddb3a68f06db71ab44073884250ec7be7495531fa2965365a4d41a9508bb346d31dd91c203c877b4ed5821c83d3
+  data.tar.gz: 92be48c0d23b22dbcfbaca25a43e94edda2c89947abf6505425a6439a665b99f39e317daaf6190bf4f0e3c7086ae73178c2dd688c400aeefefd8c736963c7264

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    remi (0.2.1)
+    remi (0.2.3)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/lib/remi.rb

@@ -18,6 +18,8 @@ require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/try'
 require 'active_support/core_ext/object/inclusion'
 require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/string/strip'
+require 'active_support/core_ext/string/filters'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/numeric/conversions'
 require 'active_support/core_ext/date/calculations'
@@ -30,7 +32,6 @@ require 'salesforce_bulk_api'
 
 # Remi
 require 'remi/version.rb'
-require 'remi/core/string.rb'
 
 require 'remi/settings'
 require 'remi/job'
@@ -39,8 +40,8 @@ require 'remi/field_symbolizers'
 require 'remi/data_subject'
 require 'remi/sf_bulk_helper' # separate into SF support package
 
-require 'remi/core/daru'
-require 'remi/core/refinements'
+require 'remi/refinements/symbolizer'
+require 'remi/refinements/daru'
 
 require 'remi/extractor/sftp_file'
 

data/lib/remi/cli.rb

@@ -15,7 +15,7 @@ module Remi
       options = {}
 
       opt_parser = OptionParser.new do |opts|
-        opts.banner = <<-EOT.unindent
+        opts.banner = <<-EOT.strip_heredoc
           Usage: Command line helpers for Remi.
         EOT
 

data/lib/remi/cucumber/business_rules.rb

@@ -1,5 +1,5 @@
 module Remi::BusinessRules
-  using Remi::Core::Refinements
+  using Remi::Refinements::Symbolizer
 
   def self.parse_full_field(full_field_name)
     full_field_name.split(':').map(&:strip)
@@ -230,6 +230,15 @@ module Remi::BusinessRules
       @data_obj.df.size
     end
 
+    # Public: Converts the data subject to a hash where the keys are the table
+    # columns and the values are an array for the value of column for each row.
+    def column_hash
+      @data_obj.df.to_hash.reduce({}) do |h, (k,v)|
+        h[k.symbolize] = v.to_a
+        h
+      end
+    end
+
     # For debugging only
     def _df
       @data_obj.df
@@ -429,6 +438,17 @@ module Remi::BusinessRules
       end
       df
     end
+
+    # Public: Converts a Cucumber::Ast::Table to a hash where the keys are the table
+    # columns and the values are an array for the value of column for each row.
+    def column_hash
+      @table.hashes.reduce({}) do |h, row|
+        row.each do |k,v|
+          (h[k.symbolize] ||= []) << v
+        end
+        h
+      end
+    end
   end
 
 

data/lib/remi/project/features/aggregate.feature

@@ -0,0 +1,23 @@
+Feature: Tests the aggregate refinement to the Daru library
+
+  Background:
+    Given the job is 'Aggregate'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+  Scenario: The aggregator should find the minimum year for each 'Alpha'
+    Given the following example record for 'Source Data':
+      | Alpha | Year | something |
+      | a     | 2016 | 1 |
+      | a     | 2018 | 1 |
+      | b     | 2016 | 2 |
+      | b     | 2010 | 3 |
+      | a     | 2017 | 4 |
+    And the following example record called 'expected result':
+      | Alpha | Year |
+      | a     | 2016 |
+      | b     | 2010 |
+    Then the target should match the example 'expected result'

data/lib/remi/project/features/step_definitions/remi_step.rb

@@ -201,6 +201,22 @@ Then /^the target field '(.+)' is the date (.+)$/ do |target_field, date_referen
   step "the target field '#{target_field}' is set to the value \"*#{date_reference}*\""
 end
 
+Then /^the target '(.+)' should match the example '([[:alnum:]\s]+)'$/ do |target_name, example_name|
+  @brt.run_transforms
+
+  target_hash = @brt.targets[target_name].column_hash
+  example_hash = @brt.examples[example_name].column_hash
+  common_keys = target_hash.keys & example_hash.keys
+
+  expect(target_hash.select { |k,v| common_keys.include? k })
+    .to eq example_hash.select { |k,v| common_keys.include? k }
+end
+
+Then /^the target should match the example '([[:alnum:]\s]+)'$/ do |example_name|
+  target_name = @brt.targets.keys.first
+  step "the target '#{target_name}' should match the example '#{example_name}'"
+end
+
 
 ### Transforms
 

data/lib/remi/project/features/support/env_app.rb

@@ -1,3 +1,4 @@
 require_relative '../../jobs/sample_job'
 require_relative '../../jobs/copy_source_job'
 require_relative '../../jobs/transforms/transform_jobs'
+require_relative '../../jobs/aggregate_job'

data/lib/remi/project/jobs/aggregate_job.rb

@@ -0,0 +1,20 @@
+require_relative 'all_jobs_shared'
+
+class AggregateJob
+  include AllJobsShared
+  using Remi::Refinements::Daru
+
+  define_source :source_data, Remi::DataSource::DataFrame
+  define_target :target_data, Remi::DataTarget::DataFrame
+
+  define_transform :main, sources: :source_data, targets: :target_data do
+
+    mymin = lambda do |field, df, indicies|
+      values = indicies.map { |idx| df.row[idx][field] }
+      values.min
+    end
+
+    target_data.df = source_data.df.aggregate(by: :alpha, func: mymin.curry.(:year)).detach_index
+    target_data.df.vectors = Daru::Index.new([:alpha, :year])
+  end
+end

data/lib/remi/project/jobs/copy_source_job.rb

@@ -2,11 +2,12 @@ require_relative 'all_jobs_shared'
 
 class CopySourceJob
   include AllJobsShared
+  using Remi::Refinements::Daru
 
   define_source :source_data, Remi::DataSource::DataFrame
   define_source :target_data, Remi::DataSource::DataFrame
 
   define_transform :main, sources: :source_data, targets: :target_data do
-    target_data.df = source_data.df.monkey_dup
+    target_data.df = source_data.df.dup
   end
 end

data/lib/remi/project/jobs/sample_job.rb

@@ -4,6 +4,7 @@ require_relative 'all_jobs_shared'
 
 class SampleJob
   include AllJobsShared
+  using Remi::Refinements::Daru
 
   define_source :existing_contacts, Remi::DataSource::Salesforce,
     object: :Contact,
@@ -77,7 +78,7 @@ class SampleJob
   define_transform :map_common_fields, sources: [:sample_file, :existing_contacts], targets: :all_contacts do
 
     # Exclude all source records with an invalid program name
-    all_contacts.df = sample_file.df.monkey_dup
+    all_contacts.df = sample_file.df.dup
     Remi::SourceToTargetMap.apply(all_contacts.df) do
       map source(:program) .target(:Major__c)
         .transform(Remi::Transform[:lookup][program_name_lookup])

data/lib/remi/refinements/daru.rb

@@ -0,0 +1,58 @@
+module Remi
+  module Refinements
+    module Daru
+      refine ::Daru::DataFrame do
+
+        # Public: Fixes dup issues in the Daru library (vectors not being duped).
+        def dup
+          dupdf = ::Daru::DataFrame.new([], index: self.index)
+          self.vectors.each do |v|
+            dupdf[v] = self[v]
+          end
+
+          dupdf
+        end
+
+        # Public: Saves a Dataframe to a file.
+        def hash_dump(filename)
+          File.write(filename, Marshal.dump(self.to_hash))
+        end
+
+        # Public: Creates a DataFrame by reading the dumped version from a file.
+        def self.from_hash_dump(filename)
+          ::Daru::DataFrame.new(Marshal.load(File.read(filename)))
+        end
+
+        # Public: Allows the user to define an arbitrary aggregation function.
+        #
+        # by   - The name of the DataFrame vector to use to group records.
+        # func - A lambda function that accepts two arguments - the first argument
+        #        is the DataFrame and the second is the index of the elements belonging
+        #        to a group.
+        #
+        # Example:
+        #   df = Daru::DataFrame.new( { a: ['a','a','a','b','b'], year: ['2018','2015','2019', '2014', '2013'] })
+        #
+        #   mymin = lambda do |field, df, indicies|
+        #     values = indicies.map { |idx| df.row[idx][field] }
+        #     values.min
+        #   end
+        #
+        #   df.aggregate(by: :a, func: mymin.curry.(:year))
+        #
+        #
+        # Returns a Daru::Vector.
+        def aggregate(by:, func:)
+          grouped = self.group_by(by)
+          ::Daru::Vector.new(
+            grouped.groups.reduce({}) do |h, (key, indicies)|
+              h[key.size == 1 ? key.first : key] = func.(self, indicies)
+              h
+            end
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/remi/{core/refinements.rb → refinements/symbolizer.rb}

@@ -1,6 +1,6 @@
 module Remi
-  module Core
-    module Refinements
+  module Refinements
+    module Symbolizer
       refine String do
         def symbolize(symbolizer=nil)
           if symbolizer

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-21 00:00:00.000000000 Z
+date: 2016-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
@@ -198,9 +198,6 @@ files:
 - doc/install-rbenv-os_x.md
 - lib/remi.rb
 - lib/remi/cli.rb
-- lib/remi/core/daru.rb
-- lib/remi/core/refinements.rb
-- lib/remi/core/string.rb
 - lib/remi/cucumber.rb
 - lib/remi/cucumber/business_rules.rb
 - lib/remi/cucumber/data_source.rb
@@ -218,6 +215,7 @@ files:
 - lib/remi/field_symbolizers.rb
 - lib/remi/job.rb
 - lib/remi/lookup/regex_sieve.rb
+- lib/remi/project/features/aggregate.feature
 - lib/remi/project/features/examples.feature
 - lib/remi/project/features/formulas.feature
 - lib/remi/project/features/sample_job.feature
@@ -227,6 +225,7 @@ files:
 - lib/remi/project/features/transforms/date_diff.feature
 - lib/remi/project/features/transforms/parse_date.feature
 - lib/remi/project/features/transforms/prefix.feature
+- lib/remi/project/jobs/aggregate_job.rb
 - lib/remi/project/jobs/all_jobs_shared.rb
 - lib/remi/project/jobs/copy_source_job.rb
 - lib/remi/project/jobs/sample_job.rb
@@ -234,6 +233,8 @@ files:
 - lib/remi/project/jobs/transforms/parse_date_job.rb
 - lib/remi/project/jobs/transforms/prefix_job.rb
 - lib/remi/project/jobs/transforms/transform_jobs.rb
+- lib/remi/refinements/daru.rb
+- lib/remi/refinements/symbolizer.rb
 - lib/remi/settings.rb
 - lib/remi/sf_bulk_helper.rb
 - lib/remi/source_to_target_map.rb

data/lib/remi/core/daru.rb

@@ -1,28 +0,0 @@
-module Daru
-  class DataFrame
-    def monkey_dup
-      dupdf = Daru::DataFrame.new([], index: self.index)
-      self.vectors.each do |v|
-        dupdf[v] = self[v]
-      end
-
-      dupdf
-    end
-
-    def monkey_merge(other)
-      other.vectors.each do |v|
-        self[v] = other[v]
-      end
-
-      self
-    end
-
-    def hash_dump(filename)
-      File.write(filename, Marshal.dump(self.to_hash))
-    end
-
-    def self.from_hash_dump(filename)
-      Daru::DataFrame.new(Marshal.load(File.read(filename)))
-    end
-  end
-end

data/lib/remi/core/string.rb

@@ -1,8 +0,0 @@
-class String
-  # Strip leading whitespace from each line that is the same as the
-  # amount of whitespace on the first line of the string.
-  # Leaves _additional_ indentation on later lines intact.
-  def unindent
-    gsub /^#{self[/\A\s*/]}/, ''
-  end
-end