remi 0.2.42 → 0.3.0

data/lib/remi/job/parameters.rb

@@ -0,0 +1,90 @@
+module Remi
+  class Job
+    # A job parameter adds flexiblity to defining job templates.  An
+    # instance of Parameters contains a collection of parameters that
+    # are evaluatin in the context of a job.  It functions very
+    # similarly to Rspec's #let, in that in can be defined using a
+    # block of code that is only evaluated the first time it is used,
+    # and cached for later use.
+    #
+    # Parameters should only be used in the context of a job.
+    # @example
+    #   class MyJob < Remi::Job
+    #     param(:my_param) { 'some parameter' }
+    #     param :my_calculated_param do
+    #       1.upto(1000).size
+    #     end
+    #
+    #     transform :something do
+    #       puts "my_param is #{job.params[:my_param]}"
+    #       puts "my_calculated_param is #{job.params[:my_calculated_param]}"
+    #     end
+    #   end
+    #
+    #   job1 = MyJob.new
+    #   job1.execute
+    #   #=> my_param is some parameter
+    #   #=> my_calculated_param is 1000
+    #
+    #   job2 = MyJob.new
+    #   job2.params[:my_param] = 'override'
+    #   job2.execute
+    #   #=> my_param is override
+    #   #=> my_calculated_param is 1000
+    #
+    #   job3 = MyJob.new(my_param: 'constructor override', my_calculated_param: 322)
+    #   job3.execute
+    #   #=> my_param is constructor override
+    #   #=> my_calculated_param is 322
+    class Parameters
+      def initialize(context=nil)
+        @context = context
+        @params = {}
+      end
+
+      # @return [Object] The context in which parameter blocks will be evaluated
+      attr_accessor :context
+
+      # Get the value of a parameter
+      #
+      # @param name [Symbol] The name of the parameter
+      #
+      # @return [Object] The value of the parameter
+      def [](name)
+        return send(name) if respond_to?(name)
+        raise ArgumentError, "Job parameter #{name} is not defined"
+      end
+
+
+      # Set the value of a parameter
+      #
+      # @param name [Symbol] The name of the parameter
+      # @param value [Object] The new value of the parameter
+      #
+      # @return [Object] The new value of the parameter
+      def []=(name, value)
+        __define__(name) { value } unless respond_to? name
+        @params[name] = value
+      end
+
+      # @return [Hash] The parameters as a hash
+      def to_h
+        @params
+      end
+
+      # @return [Job::Parameters] A clone of this parameter set
+      def clone
+        the_clone = super
+        the_clone.instance_variable_set(:@params, @params.dup)
+        the_clone
+      end
+
+      def __define__(name, &block)
+        @params[name] = nil
+        define_singleton_method name do
+          @params[name] ||= Remi::Dsl.dsl_return(self, @context, &block)
+        end
+      end
+    end
+  end
+end

data/lib/remi/job/sub_job.rb

@@ -0,0 +1,35 @@
+module Remi
+  class Job
+    class SubJob
+      def initialize(context=nil, name: 'UNDEFINED SubJob', **kargs, &block)
+        @context = context
+        @name = name
+        @block = block
+      end
+
+      attr_accessor :context, :name
+
+      def dsl_return
+        sub_job = Dsl.dsl_return(self, @context, &@block)
+        raise ArgumentError, "SubJob DSL must return a Remi::Job" unless sub_job.is_a? Job
+        sub_job
+      end
+
+      def job
+        @job ||= dsl_return
+      end
+
+      def fields(data_subject)
+        job.send(data_subject).dsl_eval.fields
+      end
+
+      def execute
+        job.execute
+      end
+
+      def execute_transforms
+        job.execute(:transforms)
+      end
+    end
+  end
+end

data/lib/remi/job/transform.rb

@@ -0,0 +1,165 @@
+module Remi
+  class Job
+    # A Transform contains a block of code that is executed in a context.
+    # Transforms are usually defined in a Job, according to the Job DSL.
+    #
+    # Transforms may optionally have a mapping defined that links a
+    # local definition of a data frame to a definition of the data
+    # frame in the associated context.
+    # @example
+    #
+    #   # Transforms should typically be defined using the Job DSL
+    #   job = MyJob.new
+    #   tform = Job::Transform.new(job) do
+    #     # ... stuff to do in the context of the job
+    #   end
+    #   tform.execute
+    class Transform
+
+      FieldMap = Struct.new(:from_subject, :to_subject, :field_from_to)
+
+      # Initializes a transform
+      #
+      # @param context [Object, Job] sets the context in which the block will be executed
+      # @param name [String, Symbol] optionally gives the transform a name
+      # @param kargs [Hash] any keyword arguments are accessable within the block as `#params` (e.g., `params[:my_custom_param]`)
+      # @param block [Proc] a block of code to execute in the context
+      def initialize(context, name: 'NOT DEFINED', **kargs, &block)
+        @context = context
+        @name = name
+        @block = block
+        params.merge! kargs
+
+        @sources = []
+        @targets = []
+
+        @field_maps = { sources: {}, targets: {} }
+      end
+
+      attr_accessor :context, :name, :sources, :targets, :field_maps
+
+      # Executes the transform block
+      # @return [Object] the context of the transform after executing
+      def execute
+        context.logger.info "Running transformation #{@name}"
+        Dsl.dsl_eval(self, @context, &@block)
+      end
+
+      # @return [Hash] the parameters defined during initialization of the transform
+      def params
+        @params ||= Hash.new { |_, key| raise ArgumentError, "Transform parameter #{key} is not defined" }
+      end
+
+      # Validates that a data source used in the transform has been defined
+      # @param name [Symbol] the name of a data source used in the transform
+      # @param fields [Array<Symbol>] a list of fields used by the transform for this data source
+      # @raise [ArgumentError] if the transform source is not defined
+      def source(name, fields)
+        raise NoMethodError, "Need to define a source mapping for #{name}" unless sources.include? name
+        raise ArgumentError, "Need to map fields to source #{name} (#{fields})" unless (fields - field_maps[:sources][name].field_from_to.values).empty?
+      end
+
+      # Validates that a data target used in the transform has been defined
+      # @param name [Symbol] the name of a data target used in the transform
+      # @param fields [Array<Symbol>] a list of fields used by the transform for this data target
+      # @raise [ArgumentError] if the transform target is not defined
+      def target(name, fields)
+        raise NoMethodError, "Need to define a target mapping for #{name}" unless targets.include? name
+        raise ArgumentError, "Need to map fields to target #{name} (#{fields})" unless (fields - field_maps[:targets][name].field_from_to.keys).empty?
+      end
+
+      # Maps data sources and fields from the transform context to the local transform
+      # @param from_source [Symbol] name of the source data in the context
+      # @param to_source [Symbol] name of the source data local to the transform
+      # @param field_map [Hash] mapping of the key names from the context source to the local source
+      def map_source_fields(from_source, to_source, field_map)
+        sources << to_source unless sources.include? to_source
+
+        job_ds = context.send(from_source)
+        sub_trans_ds = Remi::DataSubject.new(name: to_source)
+        define_singleton_method(to_source) { sub_trans_ds }
+
+        field_maps[:sources][to_source] = FieldMap.new(job_ds, send(to_source), field_map)
+      end
+
+      # Maps data targets and fields from the local tarnsform to the transform context
+      # @param from_target [Symbol] name of the target data local to the transform
+      # @param to_target [Symbol] name of the target data in the context
+      # @param field_map [Hash] mapping of the key names from the local transform target to the context target
+      def map_target_fields(from_target, to_target, field_map)
+        targets << from_target unless targets.include? from_target
+
+        job_ds = context.send(to_target)
+        sub_trans_ds = Remi::DataSubject.new
+        define_singleton_method(from_target) { sub_trans_ds }
+
+        field_maps[:targets][from_target] = FieldMap.new(send(from_target), job_ds, field_map)
+      end
+
+      # Imports another transform to be executed as part of this transform.  The block
+      # is used to perform any source/target field mapping.
+      #
+      # @param sub_transform [Job::Transform] the transform to import into this one
+      # @param block [Proc] a block of code to be executed prior to the execution of the
+      #                     imported transform.  This is where field mapping would be defined.
+      # @example
+      #
+      #   sub_transform = Job::Transform.new('arbitrary') do
+      #     source :sub_transform_source, [] # validate that this source has been defined
+      #     # do stuff to sub_transform_source here
+      #   end
+      #
+      #   job = MyJob.new
+      #   my_transform = Job::Transform.new(job) do
+      #     import sub_transform do
+      #       map_source_fields :some_method_in_my_job, :sub_sub_transform_source, { :job_id => :sub_transform_id }
+      #     end
+      #   end
+      def import(sub_transform, **kargs, &block)
+        sub_transform.context = context
+        sub_transform.params.merge! kargs
+        Dsl.dsl_eval(sub_transform, context, &block)
+
+        sub_transform.map_inputs
+        sub_transform.execute
+        sub_transform.map_outputs
+      end
+
+
+
+      protected
+
+      def map_inputs
+        sources.each do |source_input|
+          field_map = field_maps[:sources][source_input]
+          job_ds = field_map.from_subject
+          sub_trans_ds = field_map.to_subject
+          fields_to_map = field_map.field_from_to.keys
+
+          fields_to_map.each do |job_field|
+            sub_trans_field = field_map.field_from_to[job_field]
+            sub_trans_ds.fields[sub_trans_field] = job_ds.fields[job_field]
+
+            sub_trans_ds.df[sub_trans_field] = job_ds.df[job_field]
+          end
+        end
+      end
+
+      def map_outputs
+        targets.each do |target_output|
+          field_map = field_maps[:targets][target_output]
+          job_ds = field_map.to_subject
+          sub_trans_ds = field_map.from_subject
+          fields_to_map = field_map.field_from_to.keys
+
+          fields_to_map.each do |sub_trans_field|
+            job_field = field_map.field_from_to[sub_trans_field]
+            job_ds.fields[job_field].merge! sub_trans_ds.fields[sub_trans_field]
+            job_ds.df[job_field] = sub_trans_ds.df[sub_trans_field]
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/remi/loader.rb

@@ -0,0 +1,22 @@
+module Remi
+  # A loader is an object meant to load data into a some external system.
+  # This is a parent class meant to be inherited by child classes that
+  # define specific ways to load data.
+  class Loader
+
+    def initialize(*args, logger: Remi::Settings.logger, **kargs, &block)
+      @logger = logger
+    end
+
+    attr_accessor :logger
+
+    # Any child classes need to define a load method that loads data from
+    # the given dataframe into the target system.
+    # @param data [Remi::DataFrame] Data that has been encoded appropriately to be loaded into the target
+    # @return [true] On success
+    def load(data)
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+    end
+
+  end
+end

data/lib/remi/monkeys/daru.rb

@@ -0,0 +1,4 @@
+# Needed to fix issue in Daru 0.1.4.1
+class Daru::DataFrame
+  remove_method :to_hash
+end

data/lib/remi/parser.rb

@@ -0,0 +1,44 @@
+module Remi
+  # A parser is an object that converts data returned from an
+  # Remi::Extractor into a dataframe.  This is a parent class meant to be
+  # inherited by child classes that define specific ways to parse
+  # data.
+  class Parser
+
+    # @param context [Object] The context (e.g., DataSource) for the parser (default: `nil`)
+    # @param field_symbolizer [Proc] The field symbolizer to use for this parser
+    # @param fields [Remi::Fields] A hash of field metadata to be used by the parser
+    def initialize(*args, context: nil, field_symbolizer: Remi::FieldSymbolizers[:standard], fields: Remi::Fields.new({}), logger: Remi::Settings.logger, **kargs, &block)
+      @context = context
+      @field_symbolizer = field_symbolizer
+
+      @fields = fields
+      @logger = logger
+    end
+
+    attr_accessor :context
+    attr_accessor :logger
+    attr_writer :field_symbolizer
+    attr_writer :fields
+
+    # Any child classes need to define a parse method that converts extracted data
+    # into a dataframe.
+    # @param data [Object] Extracted data that needs to be parsed
+    # @return [Remi::DataFrame] The data converted into a dataframe
+    def parse(data)
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+    end
+
+    # @return [Proc] The field symbolizer (uses the context field symbolizer if defined)
+    def field_symbolizer
+      return context.field_symbolizer if context.respond_to? :field_symbolizer
+      @field_symbolizer
+    end
+
+    # @return [Remi::Fields] The fields (uses the context fields if defined)
+    def fields
+      return context.fields if context if context.respond_to? :fields
+      @fields
+    end
+  end
+end

data/lib/remi/testing/business_rules.rb

@@ -175,7 +175,7 @@ module Remi::Testing::BusinessRules
     end
 
     def run_transforms
-      @job.run_all_transforms
+      @job.execute(:transforms)
     end
   end
 
@@ -262,7 +262,6 @@ module Remi::Testing::BusinessRules
     end
 
     attr_reader :name
-    attr_reader :data_subject
 
     def add_field(field_name)
       @fields.add_field(self, field_name)
@@ -277,17 +276,17 @@ module Remi::Testing::BusinessRules
     end
 
     def size
-      @data_subject.df.size
+      data_subject.df.size
     end
 
-    def get_attrib(name)
-      @data_subject.send(name)
+    def data_subject
+      @data_subject.dsl_eval
     end
 
     # Public: Converts the data subject to a hash where the keys are the table
     # columns and the values are an array for the value of column for each row.
     def column_hash
-      @data_subject.df.to_h.reduce({}) do |h, (k,v)|
+      data_subject.df.to_h.reduce({}) do |h, (k,v)|
         h[k.symbolize] = v.to_a
         h
       end
@@ -297,7 +296,7 @@ module Remi::Testing::BusinessRules
     # Need more robust duping to make that feasible.
     # Don't use results for anything more than size.
     def where(field_name, operation)
-      @data_subject.df.where(@data_subject.df[field_name.symbolize(@data_subject.field_symbolizer)].recode { |v| operation.call(v) })
+      data_subject.df.where(data_subject.df[field_name.symbolize(data_subject.field_symbolizer)].recode { |v| operation.call(v) })
     end
 
     def where_is(field_name, value)
@@ -323,11 +322,11 @@ module Remi::Testing::BusinessRules
 
 
     def stub_data
-      @data_subject.stub_df if @data_subject.respond_to? :stub_df
+      data_subject.stub_df if data_subject.respond_to? :stub_df
     end
 
     def example_to_df(example)
-      df = example.to_df(@data_subject.df.row[0].to_h, field_symbolizer: @data_subject.field_symbolizer)
+      df = example.to_df(data_subject.df.row[0].to_h, field_symbolizer: data_subject.field_symbolizer)
       data_subject.fields.each do |vector, metadata|
         if metadata[:type] == :json
           df[vector].recode! { |v| JSON.parse(v) rescue v }
@@ -338,20 +337,20 @@ module Remi::Testing::BusinessRules
 
     def stub_data_with(example)
       stub_data
-      @data_subject.df = example_to_df(example)
+      data_subject.df = example_to_df(example)
     end
 
     def append_data_with(example)
-      @data_subject.df = @data_subject.df.concat example_to_df(example)
+      data_subject.df = data_subject.df.concat example_to_df(example)
     end
 
 
     def replicate_rows(n_rows)
-      replicated_df = Daru::DataFrame.new([], order: @data_subject.df.vectors.to_a)
-      @data_subject.df.each do |vector|
+      replicated_df = Daru::DataFrame.new([], order: data_subject.df.vectors.to_a)
+      data_subject.df.each do |vector|
         replicated_df[vector.name] = vector.to_a * n_rows
       end
-      @data_subject.df = replicated_df
+      data_subject.df = replicated_df
     end
 
     def cumulative_dist_from_freq_table(table, freq_field: 'frequency')
@@ -383,28 +382,23 @@ module Remi::Testing::BusinessRules
 
     def distribute_values(table)
       cumulative_dist = cumulative_dist_from_freq_table(table)
-      generated_data = generate_values_from_cumulative_dist(@data_subject.df.size, cumulative_dist)
+      generated_data = generate_values_from_cumulative_dist(data_subject.df.size, cumulative_dist)
 
       generated_data.each do |field_name, data_array|
         vector_name = fields[field_name].field_name
-        @data_subject.df[vector_name] = Daru::Vector.new(data_array, index: @data_subject.df.index)
+        data_subject.df[vector_name] = Daru::Vector.new(data_array, index: data_subject.df.index)
       end
     end
 
     def freq_by(*field_names)
-      @data_subject.df.group_by(field_names).size * 1.0 / @data_subject.df.size
+      data_subject.df.group_by(field_names).size * 1.0 / data_subject.df.size
     end
 
     def unique_integer_field(field_name)
       vector_name = fields[field_name].field_name
       i = 0
-      @data_subject.df[vector_name].recode! { |v| i += 1 }
+      data_subject.df[vector_name].recode! { |v| i += 1 }
     end
-
-    def csv_options
-      @data_subject.csv_options
-    end
-
   end