remi 0.2.39 → 0.2.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fc73b233d7181b3e40c96c5da2d52bf8d81d99a8
-  data.tar.gz: 536421bb23bf564eeb4394f3046467fe1dd9adf4
+  metadata.gz: d9f919d918cc6c2b83a8f6849a30a111ddf07e87
+  data.tar.gz: 7eeaee62f683ee9fd0851e8c61cd47fab8a7f1bd
 SHA512:
-  metadata.gz: 08307536b89a25d60ddfdaad8595073b90e67e9094f22c709b3005abca6fec4bed5a838270579b4f81df1e803b8fba31c75f5ddb4bb1d263a3d5157006396d90
-  data.tar.gz: 344321318f9b76b95af19b8ff0c58d0c528da3bdeed6e15f6f60276513a2c24677cca360cc930d26087394a66082d5047b40706e0a87c8a4b9d5f20788e1ee64
+  metadata.gz: eecc73e562cf266445cf85f7c144424a730f71d95ea588b9ad7c75fc2aba0a09c0aba7440c78639c7433340e3e89888c329aec17660471038996763593e50e45
+  data.tar.gz: 96852fba98c17a79ef1763ece9f5b5cb9893cca9acf5c21da4ac09e352a1fcc2cb8e8f760836f3111493ef3f4b6258f01688b10b8c14952a7978e51623f70653

data/Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.39)
+    remi (0.2.40)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/metadata.feature

@@ -9,9 +9,35 @@ Feature: This tests the application of metadata.
     And the source 'Source Data'
     And the target 'Target Data'
 
-    And the following example record for 'Source Data':
-      | activity_id | student_id | student_dob | activity_type | activity_counter | activity_score | activity_cost | activity_date     | source_filename |
-      |           1 |          1 |    3/3/1998 |             A |                1 |            3.8 |         12.23 | 1/3/2016 03:22:36 |         one.csv |
 
   Scenario: Metadata is used to parse date fields
-    Then the target field 'student_dob' is set to the value "1998-03-03"
+
+    Given the following example record for 'Source Data':
+      | activity_id | student_id | student_dob | activity_type | activity_counter | activity_score | activity_cost | activity_date             | source_filename |
+      |           1 |          1 |    3/3/1998 |             A |                1 |            3.8 |         12.23 | 1/3/2016 03:22:36         |         one.csv |
+
+    Then the target should match the example:
+      | activity_id | student_id | student_dob | activity_type | activity_counter | activity_score | activity_cost | activity_date             | source_filename |
+      |           1 |          1 |  1998-03-03 |             A |                1 |            3.8 |         12.23 | 2016-01-03 03:22:36 +0000 |         one.csv |
+
+  Scenario Outline: Metadata is used to stub records with values that conform to the metadata
+
+    Then the target field '<Field>' is set to the value "<Class>"
+
+    Examples:
+      | Field                  | Class   |
+      | activity_id_class      | String  |
+      | student_id_class       | String  |
+      | student_dob_class      | Date    |
+      | activity_type_class    | String  |
+      | activity_counter_class | Fixnum  |
+      | activity_score_class   | Float   |
+      | activity_cost_class    | Float   |
+      | activity_date_class    | Time    |
+      | source_filename_class  | String  |
+
+
+  Scenario: Metadata for decimals is stubbed
+
+    Then the target field 'activity_cost_precision' is populated with "8"
+    And the target field 'activity_cost_scale' is populated with "2"

data/features/step_definitions/remi_step.rb

@@ -4,7 +4,7 @@
 ### Job and background setup
 
 Given /^the job is '([[:alnum:]\s]+)'$/ do |arg|
-  @brt = Remi::BusinessRules::Tester.new(arg)
+  @brt = Remi::Testing::BusinessRules::Tester.new(arg)
 end
 
 Given /^the job source '([[:alnum:]\s\-_]+)'$/ do |arg|
@@ -64,13 +64,13 @@ Then /^the file that comes last in an alphanumeric sort by group will be downloa
 end
 
 Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
-  expect(@brt.target.get_attrib(:remote_path)).to eq Remi::BusinessRules::ParseFormula.parse(remote_path)
+  expect(@brt.target.get_attrib(:remote_path)).to eq Remi::Testing::BusinessRules::ParseFormula.parse(remote_path)
 end
 
 ## CSV Options
 
 Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
-  expect(@brt.send(st.to_sym).csv_options[:col_sep]).to eq Remi::BusinessRules.csv_opt_map[delimiter]
+  expect(@brt.send(st.to_sym).csv_options[:col_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[delimiter]
 end
 
 Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, encoding|
@@ -78,15 +78,15 @@ Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, enc
 end
 
 Given /^the (source|target) file uses a ([\w ]+) to quote embedded delimiters$/ do |st, quote_char|
-  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[quote_char]
+  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[quote_char]
 end
 
 Given /^the (source|target) file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |st, escape_char|
-  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[escape_char]
+  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::Testing::BusinessRules.csv_opt_map[escape_char]
 end
 
 Given /^the (source|target) file uses ([\w ]+) line endings$/ do |st, line_endings|
-  expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq Remi::BusinessRules.csv_opt_map[line_endings]
+  expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq Remi::Testing::BusinessRules.csv_opt_map[line_endings]
 end
 
 Given /^the (source|target) file uses "([^"]+)" as a record separator$/ do |st, line_endings|
@@ -130,7 +130,7 @@ Given /^the source field '([^']+)' (?:has|is set to) the value "([^"]*)"$/ do |s
   step "the source field '#{source_field}'"
 
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
-  @brt.sources[source_name].fields[source_field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
+  @brt.sources[source_name].fields[source_field_name].value = Remi::Testing::BusinessRules::ParseFormula.parse(value)
 end
 
 Given /^the source field (?:has|is set to) the value "([^"]*)"$/ do |value|
@@ -143,7 +143,7 @@ Given /^the source field '([^']+)' (?:has|is set to) the multiline value$/ do |s
   step "the source field '#{source_field}'"
 
   source_name, source_field_name = @brt.sources.parse_full_field(source_field)
-  @brt.sources[source_name].fields[source_field_name].value = Remi::BusinessRules::ParseFormula.parse(value)
+  @brt.sources[source_name].fields[source_field_name].value = Remi::Testing::BusinessRules::ParseFormula.parse(value)
 end
 
 Given /^the source field (?:has|is set to) the multiline value$/ do |value|
@@ -266,7 +266,7 @@ Then /^the target field '([^']+)' is (?:set to the value|populated with) "([^"]*
       @brt.run_transforms
     }.not_to raise_error
     Array(target_names).each do |target_name|
-      expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq [Remi::BusinessRules::ParseFormula.parse(value)]
+      expect(@brt.targets[target_name].fields[target_field_name].values.uniq).to eq [Remi::Testing::BusinessRules::ParseFormula.parse(value)]
     end
   }
 end

data/features/transforms/parse_date.feature

@@ -12,6 +12,7 @@ Feature: Tests the parse_date transform
     Given the source field 'Date String' has the value "<Date String>"
     And the job parameter 'format' is "<Format>"
     Then the target field 'Parsed Date' is set to the value "<Parsed Date>"
+
     Examples:
       | Date String | Format   | Parsed Date |
       | 2015-10-21  | %Y-%m-%d | 2015-10-21  |

data/jobs/metadata_job.rb

@@ -1,4 +1,5 @@
 require_relative 'all_jobs_shared'
+ENV['TZ'] = 'UTC'
 
 class MetadataJob
   include AllJobsShared
@@ -11,7 +12,7 @@ class MetadataJob
       :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
       :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
       :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
-      :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 16, scale: 2, cdc_type: 2 },
+      :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 8, scale: 2, cdc_type: 2 },
       :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
       :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
     }
@@ -25,36 +26,28 @@ class MetadataJob
       :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
       :activity_counter => { from: 'out', out: true, type: :integer },
       :activity_score   => { from: 'out', out: true, type: :float },
-      :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 16, scale: 2 },
+      :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 8, scale: 2 },
       :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
       :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
     }
 
   define_transform :main do
+    source_data.enforce_types
 
-=begin
-    source_data.df = Remi::DataFrame.daru([
-      ['1','1','3/3/1998','A','1','3.8','12.23','1/3/2016 03:22:36','one.csv'],
-      ['2','1','3/3/1998','B','3','4.2','10.53','1/3/2016 03:58:22','one.csv'],
-      ['2','1','','B','2','4.23','10.539','1/3/2016 03:58:22','one.csv']
-    ].transpose, order: [
-      :activity_id,
-      :student_id,
-      :student_dob,
-      :activity_type,
-      :activity_counter,
-      :activity_score,
-      :activity_cost,
-      :activity_date,
-      :source_filename
-    ])
-=end
-
-    Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields) do
+    Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields, target_metadata: target_data.fields) do
       target_data.fields.keys.each do |field|
         map source(field) .target(field)
-          .transform(Remi::Transform::EnforceType.new)
+
+        map source(field) .target("#{field}_class".to_sym)
+          .transform(->(v) { v.class })
       end
+
+      map source(:activity_cost) .target(:activity_cost_precision, :activity_cost_scale)
+        .transform(->(row) {
+          components = row[:activity_cost].to_s.split('.')
+          row[:activity_cost_precision] = components.first.size
+          row[:activity_cost_scale] = components.last.size
+        })
     end
   end
 end

data/jobs/transforms/parse_date_job.rb

@@ -22,7 +22,7 @@ class ParseDateJob
         .transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
 
       map source(:stubbed_date) .target(:parsed_stubbed_date)
-        .transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
+        .transform(Remi::Transform::ParseDate.new(in_format: source_data.fields[:stubbed_date][:in_format], if_blank: params[:if_blank]))
     end
   end
 end

data/lib/remi/cucumber.rb

@@ -3,5 +3,9 @@ require 'cucumber/rspec/doubles'
 
 require 'regexp-examples'
 
-require_relative 'cucumber/data_source'
-require_relative 'cucumber/business_rules'
+require_relative 'testing/data_stub'
+require_relative 'testing/business_rules'
+
+class Remi::DataSource
+  include Remi::Testing::DataStub
+end

data/lib/remi/data_subject.rb

@@ -1,9 +1,4 @@
 module Remi
-
-  # Namespaces for specific sources/targets
-  module DataSource; end
-  module DataTarget; end
-
   class DataSubject
     def initialize(*args, fields: Remi::Fields.new, remi_df_type: :daru, logger: Remi::Settings.logger, **kargs, &block)
       @fields = fields
@@ -11,16 +6,24 @@ module Remi
       @logger = logger
     end
 
+    # Public: Fields defined for this data subject
     attr_accessor :fields
 
+    # Public: The default method for symbolizing field names
     def field_symbolizer
       Remi::FieldSymbolizers[:standard]
     end
 
+    # Public: Access the dataframe from a DataSource
+    #
+    # Returns a Remi::DataFrame
     def df
       @dataframe ||= Remi::DataFrame.create(@remi_df_type, [], order: @fields.keys)
     end
 
+    # Public: Reassigns the dataframe associated with this subject
+    #
+    # Returns the assigned dataframe
     def df=(new_dataframe)
       if new_dataframe.respond_to? :remi_df_type
         @dataframe = new_dataframe
@@ -29,58 +32,78 @@ module Remi
       end
     end
 
-    module DataSource
-
-      # Public: Access the dataframe from a DataSource
-      #
-      # Returns a Remi::DataFrame
-      def df
-        @dataframe ||= to_dataframe
+    # Public: Enforces types defined in the field metadata.
+    # For example, if a field has metadata with type: :date, then the
+    # type enforcer will convert data in that field into a date, and will
+    # throw an error if it is unable to parse any of the values.
+    #
+    # types - If set, restricts the data types that are enforced to just those listed.
+    #
+    # Returns nothing.
+    def enforce_types(*types)
+      sttm = SourceToTargetMap.new(df, source_metadata: fields)
+      fields.keys.each do |field|
+        next unless (types.size == 0 || types.include?(fields[field][:type])) && df.vectors.include?(field)
+        sttm.source(field).target(field).transform(Remi::Transform::EnforceType.new).execute
       end
 
-      # Public: Memoized version of extract!
-      def extract
-        @extract ||= extract!
-      end
+      nil
+    end
+  end
 
-      # Public: Called to extract data from the source.
-      #
-      # Returns data in a format that can be used to create a dataframe.
-      def extract!
-        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
-        @extract
-      end
 
-      # Public: Converts extracted data to a dataframe
-      #
-      # Returns a Remi::DataFrame
-      def to_dataframe
-        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
-      end
+  class DataSource < DataSubject
+
+    # Public: Access the dataframe from a DataSource
+    #
+    # Returns a Remi::DataFrame
+    def df
+      @dataframe ||= to_dataframe
     end
 
-    module DataTarget
+    # Public: Memoized version of extract!
+    def extract
+      @extract ||= extract!
+    end
 
-      # Public: Loads data to the target.  This is automatically called
-      # after all transforms have executed, but could also get called manually.
-      # The actual load operation is only executed if hasn't already.
-      #
-      # Returns true if the load operation was successful.
-      def load
-        return true if @loaded || df.size == 0
+    # Public: Called to extract data from the source.
+    #
+    # Returns data in a format that can be used to create a dataframe.
+    def extract!
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+      @extract
+    end
 
-        @loaded = load!
-      end
+    # Public: Converts extracted data to a dataframe
+    #
+    # Returns a Remi::DataFrame
+    def to_dataframe
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+    end
+  end
 
-      # Public: Performs the load operation, regardless of whether it has
-      # already executed.
-      #
-      # Returns true if the load operation was successful
-      def load!
-        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
 
-        false
-      end
+  class DataTarget < DataSubject
+
+    # Public: Loads data to the target.  This is automatically called
+    # after all transforms have executed, but could also get called manually.
+    # The actual load operation is only executed if hasn't already.
+    #
+    # Returns true if the load operation was successful.
+    def load
+      return true if @loaded || df.size == 0
+
+      @loaded = load!
+    end
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+
+      false
     end
   end
 end

data/lib/remi/data_subject/csv_file.rb

@@ -26,8 +26,7 @@ module Remi
 
 
 
-  class DataSource::CsvFile < Remi::DataSubject
-    include Remi::DataSubject::DataSource
+  class DataSource::CsvFile < DataSource
     include Remi::DataSubject::CsvFile
 
     def initialize(*args, **kargs, &block)
@@ -130,8 +129,7 @@ module Remi
 
 
 
-  class DataTarget::CsvFile < Remi::DataSubject
-    include ::Remi::DataSubject::DataTarget
+  class DataTarget::CsvFile < DataTarget
     include ::Remi::DataSubject::CsvFile
 
     default_csv_options[:row_sep] = "\n"

data/lib/remi/data_subject/data_frame.rb

@@ -1,7 +1,6 @@
 module Remi
 
-  class DataSource::DataFrame < Remi::DataSubject
-    include Remi::DataSubject::DataSource
+  class DataSource::DataFrame < DataSource
 
     def initialize(*args, **kargs, &block)
       super
@@ -30,8 +29,7 @@ module Remi
   end
 
 
-  class DataTarget::DataFrame < Remi::DataSubject
-    include Remi::DataSubject::DataTarget
+  class DataTarget::DataFrame < DataTarget
 
     def initialize(*args, **kargs, &block)
       super

data/lib/remi/data_subject/postgres.rb

@@ -13,8 +13,7 @@ module Remi
   end
 
 
-  class DataSource::Postgres < Remi::DataSubject
-    include Remi::DataSubject::DataSource
+  class DataSource::Postgres < DataSource
     include Remi::DataSubject::Postgres
 
 
@@ -65,8 +64,7 @@ module Remi
 
   # VERY PRELIMINARY IMPLEMENTAtION - ONLY LOADS TO TEMP TABLES
   # IT IS THEN UP TO THE USER TO DO ELT TO LOAD THE FINAL TABLE
-  class DataTarget::Postgres < Remi::DataSubject
-    include Remi::DataSubject::DataTarget
+  class DataTarget::Postgres < DataTarget
     include Remi::DataSubject::Postgres
 
     def initialize(*args, **kargs, &block)

data/lib/remi/data_subject/salesforce.rb

@@ -21,8 +21,7 @@ module Remi
   end
 
 
-  class DataSource::Salesforce < Remi::DataSubject
-    include Remi::DataSubject::DataSource
+  class DataSource::Salesforce < DataSource
     include Remi::DataSubject::Salesforce
 
     def initialize(*args, **kargs, &block)
@@ -92,8 +91,7 @@ module Remi
   end
 
 
-  class DataTarget::Salesforce < Remi::DataSubject
-    include Remi::DataSubject::DataTarget
+  class DataTarget::Salesforce < DataTarget
     include Remi::DataSubject::Salesforce
 
     def initialize(*args, **kargs, &block)

data/lib/remi/data_subject/sftp_file.rb

@@ -1,7 +1,6 @@
 module Remi
 
-  class DataTarget::SftpFile < Remi::DataSubject
-    include Remi::DataSubject::DataTarget
+  class DataTarget::SftpFile < DataTarget
 
     def initialize(*args, **kargs, &block)
       super

data/lib/remi/{cucumber → testing}/business_rules.rb

@@ -1,4 +1,4 @@
-module Remi::BusinessRules
+module Remi::Testing::BusinessRules
   using Remi::Refinements::Symbolizer
 
   def self.csv_opt_map

data/lib/remi/testing/data_stub.rb

@@ -0,0 +1,72 @@
+module Remi
+  module Testing
+    module DataStub
+      def stub_row_array
+        @fields.values.map do |attribs|
+          stub_values(attribs)
+        end
+      end
+
+      def empty_stub_df
+        self.df = Daru::DataFrame.new([], order: @fields.keys)
+      end
+
+      def stub_df
+        empty_stub_df
+        self.df.add_row(stub_row_array)
+      end
+
+      def stub_values(**attribs)
+        stub_type = "stub_#{attribs[:type]}".to_sym
+        if respond_to?(stub_type)
+          send(stub_type, attribs)
+        else
+          stub_string(attribs)
+        end
+      end
+
+      def stub_string(**attribs)
+        Faker::Hipster.word
+      end
+
+      def stub_float(**attribs)
+        Faker::Number.decimal(2,3)
+      end
+
+      def stub_decimal(**attribs)
+        Faker::Number.decimal(attribs[:precision],attribs[:scale])
+      end
+
+      def stub_integer(**attribs)
+        Faker::Number.number(4).to_s
+      end
+
+      def stub_date(**attribs)
+        in_format = attribs[:in_format]
+        result = Faker::Date.backward(3650)
+        result = result.strftime(in_format) if in_format
+        result
+      end
+
+      def stub_datetime(**attribs)
+        in_format = attribs[:in_format]
+        result = Faker::Time.backward(3650)
+        result = result.strftime(in_format) if in_format
+        result
+      end
+
+      def stub_boolean(**attribs)
+        ['T','F'].shuffle.first
+      end
+
+      def stub_json(**attribs)
+        if attribs[:json_array]
+          [ stub_string ]
+        else
+          { Faker::Hipster.words(1, true, true) => stub_string }
+        end.to_json
+      end
+
+    end
+  end
+end

data/lib/remi/transform.rb

@@ -218,6 +218,7 @@ module Remi
     # This transform is metadata aware and will use :in_format metadata
     # from the source
     #
+    # type      - Specify either :date, or :datetime type (default: date)
     # in_format - The date format to use to convert the string (default: uses :in_format
     #             from the source metadata.  If that is not defined, use '%Y-%m-%d').
     # if_blank  - Value to use if the the incoming value is blank (default: uses :if_blank
@@ -232,20 +233,33 @@ module Remi
     #  tform.source_metadata = { in_format: '%m/%d/%Y' }
     #  tform.to_proc.call('02/22/2013') # => Date.new(2013,2,22)
     class ParseDate < Transform
-      def initialize(*args, in_format: nil, if_blank: nil, **kargs, &block)
+      def initialize(*args, type: nil, in_format: nil, if_blank: nil, **kargs, &block)
         super
+        @type      = type
         @in_format = in_format
         @if_blank  = if_blank
       end
 
+      def type
+        @type ||= @source_metadata.fetch(:type, :date)
+      end
+
       def in_format
-        @in_format ||= @source_metadata.fetch(:in_format, '%Y-%m-%d')
+        @in_format ||= @source_metadata.fetch(:in_format, default_date_format)
       end
 
       def if_blank
         @if_blank ||= @source_metadata.fetch(:if_blank, nil)
       end
 
+      def default_date_format
+        if type == :datetime
+          '%Y-%m-%d %H:%M:%S'
+        else
+          '%Y-%m-%d'
+        end
+      end
+
       def transform(value)
         begin
           if value.respond_to?(:strftime)
@@ -260,15 +274,19 @@ module Remi
         end
       end
 
+      def class_type
+        @class_type ||= type == :datetime ? Time : Date
+      end
+
       def string_to_date(value)
-        Date.strptime(value, in_format)
+        class_type.strptime(value, in_format)
       end
 
       def blank_handler(value)
         if if_blank == :low
-          Date.new(1900,01,01)
+          class_type.new(1900,01,01)
         elsif if_blank == :high
-          Date.new(2999,12,31)
+          class_type.new(2999,12,31)
         elsif if_blank.respond_to? :call
           if_blank.call(value)
         else
@@ -282,6 +300,7 @@ module Remi
     # This transform is metadata aware and will use :in_format/:out_format metadata
     # from the source.
     #
+    # type       - Specify either :date, or :datetime type (default: date)
     # in_format  - The date format to used to parse the input value.  If the input value
     #              is a date, then then parameter is ignored.  (default: uses :in_format
     #              from the source metadata.  If that is not defined, use '%Y-%m-%d')
@@ -297,18 +316,35 @@ module Remi
     #  tform.source_metadata = { in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' }
     #  tform.to_proc.call('02/22/2013') # => "2013-02-22"
     class FormatDate < Transform
-      def initialize(*args, in_format: nil, out_format: nil, **kargs, &block)
+      def initialize(*args, type: nil, in_format: nil, out_format: nil, **kargs, &block)
         super
+        @type       = type
         @in_format  = in_format
         @out_format = out_format
       end
 
+      def type
+        @type ||= @source_metadata.fetch(:type, :date)
+      end
+
       def in_format
-        @in_format ||= @source_metadata.fetch(:in_format, '%Y-%m-%d')
+        @in_format ||= @source_metadata.fetch(:in_format, default_date_format)
       end
 
       def out_format
-        @out_format ||= @source_metadata.fetch(:out_format, '%Y-%m-%d')
+        @out_format ||= @source_metadata.fetch(:out_format, default_date_format)
+      end
+
+      def default_date_format
+        if type == :datetime
+          '%Y-%m-%d %H:%M:%S'
+        else
+          '%Y-%m-%d'
+        end
+      end
+
+      def class_type
+        @class_type ||= type == :datetime ? Time : Date
       end
 
       def transform(value)
@@ -318,7 +354,7 @@ module Remi
           elsif value.respond_to? :strftime
             value.strftime(out_format)
           else
-            Date.strptime(value, in_format).strftime(out_format)
+            class_type.strptime(value, in_format).strftime(out_format)
           end
         rescue ArgumentError => err
           raise err, "Error parsing date (#{value.class}): '#{value}' using the format #{in_format} => #{out_format}"
@@ -461,7 +497,7 @@ module Remi
       def if_blank
         return @if_blank if @if_blank_set
         @if_blank_set = true
-        @if_blank = @source_metadata.fetch(:if_blank, nil)
+        @if_blank = @source_metadata.fetch(:if_blank, default_if_blank)
       end
 
       def blank_handler(value)
@@ -474,8 +510,12 @@ module Remi
         end
       end
 
+      def default_if_blank
+        type == :string ? '' : nil
+      end
+
       def transform(value)
-        if value.blank?
+        if value.blank? && type != :json
           blank_handler(value)
         else
           case type
@@ -488,9 +528,15 @@ module Remi
           when :decimal
             Float("%.#{scale}f" % Float(value))
           when :date
-            value.is_a?(Date) ? value : Date.strptime(value, in_format) # value.is_a?(Date) is only needed becuase we stub date types with actual dates, rather than strings like we probably should
+            value.is_a?(Date) ? value : Date.strptime(value, in_format)
           when :datetime
-            Time.strptime(value, in_format)
+            value.is_a?(Time) ? value : Time.strptime(value, in_format)
+          when :json
+            if value.blank? && value != [] && value != {}
+              blank_handler(value)
+            else
+              value.is_a?(Hash) || value.is_a?(Array) ? value : JSON.parse(value)
+            end
           else
             raise ArgumentError, "Unknown type enforcement: #{type}"
           end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.39'
+  VERSION = '0.2.40'
 end

data/spec/data_subject_spec.rb

@@ -0,0 +1,44 @@
+require_relative 'remi_spec'
+
+# VERY SPARSE TESTING!  DO MORE!
+
+describe DataSubject do
+
+  describe 'enforcing types' do
+    let(:dataframe) do
+      Remi::DataFrame::Daru.new({ my_date: ['10/21/2015'] })
+    end
+
+    let(:data_subject) do
+      DataSubject.new(fields: fields).tap { |ds| ds.df = dataframe }
+    end
+
+    let(:fields) do
+      Fields.new({
+        my_date:     { type: :date, in_format: '%m/%d/%Y' },
+        other_date: { type: :date, in_format: '%m/%d/%Y' }
+      })
+    end
+
+    it 'converts a date string to a date using an in_format' do
+      data_subject.enforce_types
+      expect(data_subject.df[:my_date].to_a).to eq [Date.new(2015, 10, 21)]
+    end
+
+    it 'does not do any conversion if the type is not specified' do
+      fields[:my_date].delete(:type)
+      data_subject.enforce_types
+      expect(data_subject.df[:my_date].to_a).to eq ['10/21/2015']
+    end
+
+    it 'throws an error if the data does not conform to its type' do
+      dataframe[:my_date].recode! { |v| '2015-10-21' }
+      expect { data_subject.enforce_types }.to raise_error ArgumentError
+    end
+
+    it 'does not create new vectors during enforcement' do
+      data_subject.enforce_types
+      expect(dataframe.vectors.to_a).to eq [:my_date]
+    end
+  end
+end

data/spec/source_to_target_map_spec.rb

@@ -49,7 +49,7 @@ describe SourceToTargetMap do
       it_behaves_like 'one-to-one map'
     end
 
-    context 'without any transforms', wip: true do
+    context 'without any transforms' do
       before { map.source(:a) .target(:aprime) }
 
       let(:result) do

data/spec/testing/data_stub_spec.rb

@@ -0,0 +1,171 @@
+require_relative '../remi_spec'
+require 'remi/testing/data_stub'
+
+describe Testing::DataStub do
+  class StubTester < DataSubject
+    include Testing::DataStub
+  end
+
+  context 'data type stubs' do
+    let(:stub_tester) { StubTester.new }
+
+    context '#stub_string' do
+      let(:stub) { stub_tester.stub_string }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+    end
+
+    context '#stub_float' do
+      let(:stub) { stub_tester.stub_float }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+
+      it 'represents a floating point number' do
+        expect(Float(stub) % 1).not_to eq 0.0
+      end
+    end
+
+    context '#stub_decimal' do
+      let(:stub) { stub_tester.stub_decimal(precision: 8, scale: 2) }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+
+      it 'represents a floating point number' do
+        expect(Float(stub) % 1).not_to eq 0.0
+      end
+
+      it 'comes with the specified precision' do
+        expect(Float(stub).to_s.split('.').first.size).to eq 8
+      end
+
+      it 'comes with the specified scale' do
+        expect(Float(stub).to_s.split('.').last.size).to eq 2
+      end
+    end
+
+    context '#stub_integer' do
+      let(:stub) { stub_tester.stub_integer }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+
+      it 'represents an integer' do
+        expect(Float(stub) % 1).to eq 0.0
+      end
+
+      it 'converts to an integer' do
+        expect { Integer(stub) }.not_to raise_error
+      end
+    end
+
+
+    context '#stub_date' do
+      context 'without an in_format' do
+        let(:stub) { stub_tester.stub_date }
+
+        it 'stubs as a date' do
+          expect(stub).to be_a Date
+        end
+      end
+
+      context 'with an in_format' do
+        let(:stub) { stub_tester.stub_date(in_format: '%m/%d/%Y') }
+
+        it 'stubs as strings' do
+          expect(stub).to be_a String
+        end
+
+        it 'can parsed as a date using the specified in_format' do
+          expect { Date.strptime(stub, '%m/%d/%Y') }.not_to raise_error
+        end
+      end
+    end
+
+    context '#stub_datetime' do
+      context 'without an in_format' do
+        let(:stub) { stub_tester.stub_datetime }
+
+        it 'stubs as a time' do
+          expect(stub).to be_a Time
+        end
+      end
+
+      context 'with an in_format' do
+        let(:stub) { stub_tester.stub_datetime(in_format: '%m/%d/%Y %H:%M:%S') }
+
+        it 'stubs as strings' do
+          expect(stub).to be_a String
+        end
+
+        it 'can parsed as a time using the specified in_format' do
+          expect { Time.strptime(stub, '%m/%d/%Y %H:%M:%S') }.not_to raise_error
+        end
+      end
+    end
+
+    context '#stub_boolean' do
+      let(:stub) { stub_tester.stub_boolean }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+
+      it 'is either T or F' do
+        expect(stub).to eq('T').or eq('F')
+      end
+    end
+
+    context '#stub_json' do
+      let(:stub) { stub_tester.stub_json }
+
+      it 'stubs as strings' do
+        expect(stub).to be_a String
+      end
+
+      it 'can be parsed as JSON' do
+        expect { JSON.parse(stub) }.not_to raise_error
+      end
+    end
+  end
+
+
+  context 'stubbed dataframe data' do
+    let(:stub_tester) do
+      StubTester.new(fields: {
+        my_date: { type: :date, in_format: '%m/%d/%Y' },
+        my_str: {}
+      })
+    end
+
+    context '#empty_stub_df' do
+      before { stub_tester.empty_stub_df }
+
+      it 'creates a dataframe with no data' do
+        expect(stub_tester.df.size).to eq 0
+      end
+
+      it 'creates a dataframe with the right number of vectors' do
+        expect(stub_tester.df.vectors.size).to eq 2
+      end
+    end
+
+    context '#stub_df' do
+      before { stub_tester.stub_df }
+
+      it 'creates a row of data' do
+        expect(stub_tester.df.size).to eq 1
+      end
+
+      it 'creates data according to the supplied metadata' do
+        expect { Date.strptime(stub_tester.df[:my_date].first, '%m/%d/%Y') }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/transform_spec.rb

@@ -50,4 +50,79 @@ describe Transform do
       expect(transform.call(1, 2)).to eq [1, 2]
     end
   end
+
+  describe Transform::ParseDate do
+    it 'parses a date using the specified in_format' do
+      parser = Transform::ParseDate.new(in_format: '%m/%d/%Y')
+      expect(parser.call('03/22/2011')).to eq Date.new(2011,3,22)
+    end
+
+    it 'returns a date if it has already been parsed' do
+      parser = Transform::ParseDate.new(in_format: '%m/%d/%Y')
+      expect(Date.new(2011,3,22)).to eq Date.new(2011,3,22)
+    end
+
+    it 'uses ISO 8601 as the default date parser' do
+      parser = Transform::ParseDate.new
+      expect(parser.call('2011-03-22')).to eq Date.new(2011,3,22)
+    end
+
+    it 'fails when an unparseable date is provided' do
+      parser = Transform::ParseDate.new
+      expect { parser.call('03/22/2011') }.to raise_error ArgumentError
+    end
+
+    it 'parses datetimes when the type is specified' do
+      parser = Transform::ParseDate.new(type: :datetime, in_format: '%m/%d/%Y %H:%M:%S')
+      expect(parser.call('03/22/2011 04:22:00')).to eq Time.new(2011,3,22,4,22,0)
+    end
+
+    it 'uses ISO 8601 as the default datetime parser' do
+      parser = Transform::ParseDate.new(type: :datetime)
+      expect(parser.call('2011-03-22 04:22:00')).to eq Time.new(2011,3,22,4,22,0)
+    end
+  end
+
+  context Transform::FormatDate do
+    it 'formats a date using the specified out_format' do
+      formatter = Transform::FormatDate.new(out_format: '%m/%d/%Y')
+      expect(formatter.call(Date.new(2011,3,22))).to eq '03/22/2011'
+    end
+
+    it 'formats a datetime using the specified out_format' do
+      formatter = Transform::FormatDate.new(type: :datetime, out_format: '%m/%d/%Y %H:%M:%S')
+      expect(formatter.call(Time.new(2011,3,22,4,22,0))).to eq '03/22/2011 04:22:00'
+    end
+
+    it 'uses the in_format to parse strings when the source is not already a date' do
+      formatter = Transform::FormatDate.new(in_format: '%d/%m/%Y', out_format: '%m/%d/%Y')
+      expect(formatter.call('22/03/2011')).to eq '03/22/2011'
+    end
+
+    it 'fails when an unparseable date is provided' do
+      formatter = Transform::FormatDate.new(in_format: '%d/%m/%Y', out_format: '%m/%d/%Y')
+      expect { formatter.call('22/22/2011') }.to raise_error ArgumentError
+    end
+
+    it 'uses ISO 8601 as the default date parser' do
+      formatter = Transform::FormatDate.new(out_format: '%m/%d/%Y')
+      expect(formatter.call('2011-03-22')).to eq '03/22/2011'
+    end
+
+    it 'uses ISO 8601 as the default date formatter' do
+      formatter = Transform::FormatDate.new(in_format: '%m/%d/%Y')
+      expect(formatter.call('03/22/2011')).to eq '2011-03-22'
+    end
+
+    it 'uses ISO 8601 as the default datetime parser' do
+      formatter = Transform::FormatDate.new(type: :datetime, out_format: '%m/%d/%Y %H:%M:%S')
+      expect(formatter.call('2011-03-22 04:22:00')).to eq '03/22/2011 04:22:00'
+    end
+
+    it 'uses ISO 8601 as the default datetime formatter' do
+      formatter = Transform::FormatDate.new(type: :datetime, in_format: '%m/%d/%Y %H:%M:%S')
+      expect(formatter.call('03/22/2011 04:22:00')).to eq '2011-03-22 04:22:00'
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.39
+  version: 0.2.40
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-30 00:00:00.000000000 Z
+date: 2016-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -225,8 +225,6 @@ files:
 - lib/remi.rb
 - lib/remi/cli.rb
 - lib/remi/cucumber.rb
-- lib/remi/cucumber/business_rules.rb
-- lib/remi/cucumber/data_source.rb
 - lib/remi/data_frame.rb
 - lib/remi/data_frame/daru.rb
 - lib/remi/data_subject.rb
@@ -248,11 +246,14 @@ files:
 - lib/remi/source_to_target_map.rb
 - lib/remi/source_to_target_map/map.rb
 - lib/remi/source_to_target_map/row.rb
+- lib/remi/testing/business_rules.rb
+- lib/remi/testing/data_stub.rb
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
 - spec/data_subject/csv_file_spec.rb
 - spec/data_subject/data_frame.rb
+- spec/data_subject_spec.rb
 - spec/extractor/file_system_spec.rb
 - spec/extractor/local_file_spec.rb
 - spec/extractor/s3_file_spec.rb
@@ -264,6 +265,7 @@ files:
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
 - spec/source_to_target_map_spec.rb
+- spec/testing/data_stub_spec.rb
 - spec/transform_spec.rb
 - workbooks/sample_workbook.ipynb
 - workbooks/workbook_helper.rb
@@ -315,6 +317,7 @@ test_files:
 - features/transforms/truthy.feature
 - spec/data_subject/csv_file_spec.rb
 - spec/data_subject/data_frame.rb
+- spec/data_subject_spec.rb
 - spec/extractor/file_system_spec.rb
 - spec/extractor/local_file_spec.rb
 - spec/extractor/s3_file_spec.rb
@@ -326,4 +329,5 @@ test_files:
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
 - spec/source_to_target_map_spec.rb
+- spec/testing/data_stub_spec.rb
 - spec/transform_spec.rb

data/lib/remi/cucumber/data_source.rb

@@ -1,70 +0,0 @@
-module Remi
-  module DataSource
-    module DataStub
-      def stub_row_array
-        @fields.values.map do |attrib|
-          stub_values[attrib[:type]].call
-        end
-      end
-
-      def empty_stub_df
-        self.df = Daru::DataFrame.new([], order: @fields.keys)
-      end
-
-      def stub_df
-        empty_stub_df
-        self.df.add_row(stub_row_array)
-      end
-
-      def stub_values
-        @stub_values ||= Hash.new(->() { Faker::Hipster.word }).merge({
-          string:   ->() { Faker::Hipster.word },
-          number:   ->() { Faker::Number.decimal(4,4) },
-          float:    ->() { Faker::Number.decimal(2,2) },
-          integer:  ->() { Faker::Number.number(4) },
-          date:     ->() { Faker::Date.backward(3650) },
-          datetime: ->() { Faker::Time.backward(3650).to_datetime },
-          boolean:  ->() { ['T','F'].shuffle.first }
-        })
-      end
-    end
-
-
-    class CsvFile
-      include DataStub
-      def stub_tmp_file
-        @stub_tmp_file ||= Tempfile.new('stub_tmp_file.csv').path
-      end
-
-      def write_stub_tmp_file
-        File.open(stub_tmp_file, "wb") do |file|
-          file.puts stub_header
-          file.puts stub_row_csv
-        end
-
-        stub_tmp_file
-      end
-
-      def stub_header
-        @fields.keys.join(@csv_options[:col_sep])
-      end
-
-      def stub_row_csv
-        stub_row_array.join(@csv_options[:col_sep])
-      end
-    end
-
-    # Hmmm.... this gets called first because I'm trying to split SF off as a "plugin"
-    class Salesforce < Remi::DataSubject
-      include DataStub
-    end
-
-    class DataFrame
-      include DataStub
-    end
-
-    class Postgres
-      include DataStub
-    end
-  end
-end