remi 0.2.42 → 0.3.0

data/lib/remi.rb

@@ -36,7 +36,11 @@ require 'active_support/core_ext/time/calculations'
 require 'remi/version.rb'
 
 require 'remi/settings'
+require 'remi/dsl'
 require 'remi/job'
+require 'remi/job/parameters'
+require 'remi/job/sub_job'
+require 'remi/job/transform'
 require 'remi/source_to_target_map'
 require 'remi/source_to_target_map/map'
 require 'remi/source_to_target_map/row'
@@ -44,21 +48,31 @@ require 'remi/field_symbolizers'
 
 require 'remi/refinements/symbolizer'
 
-require 'remi/extractor/file_system'
-require 'remi/extractor/local_file'
-require 'remi/extractor/sftp_file'
-require 'remi/extractor/s3_file'
+require 'remi/extractor'
+require 'remi/parser'
+require 'remi/encoder'
+require 'remi/loader'
 
+require 'remi/data_subject'
+require 'remi/data_subjects/file_system'
+require 'remi/data_subjects/local_file'
+require 'remi/data_subjects/sftp_file'
+require 'remi/data_subjects/s3_file'
+require 'remi/data_subjects/csv_file'
+#require 'remi/data_subjects/salesforce' # intentionally not included by default
+require 'remi/data_subjects/postgres'
+require 'remi/data_subjects/data_frame'
+require 'remi/data_subjects/none'
+require 'remi/data_subjects/sub_job'
 
 require 'remi/fields'
 require 'remi/data_frame'
 require 'remi/data_frame/daru'
 
-require 'remi/data_subject'
-require 'remi/data_subject/csv_file'
-#require 'remi/data_subject/salesforce' # intentionally not included by default
-require 'remi/data_subject/postgres'
-require 'remi/data_subject/sftp_file'
-require 'remi/data_subject/data_frame'
-
 require 'remi/transform'
+
+require 'remi/monkeys/daru'
+
+# Remi is Ruby Extract Modify and Integrate, a framework for writing ETL job in Ruby.
+module Remi
+end

data/lib/remi/data_frame.rb

@@ -1,12 +1,12 @@
 module Remi
   module DataFrame
     class << self
-      def create(remi_df_type = :daru, *args, **kargs, &block)
-        dataframe = case remi_df_type
+      def create(df_type = :daru, *args, **kargs, &block)
+        dataframe = case df_type
           when :daru
             Remi::DataFrame::Daru.new(*args, **kargs, &block)
           else
-            raise TypeError, "Unknown frame type: #{remi_df_type}"
+            raise TypeError, "Unknown frame type: #{df_type}"
           end
       end
 
@@ -29,7 +29,7 @@ module Remi
     end
 
     # Public: Returns the type of DataFrame
-    def remi_df_type
+    def df_type
       raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
     end
   end

data/lib/remi/data_frame/daru.rb

@@ -13,7 +13,7 @@ module Remi
 
 
       # Public: Returns the type of DataFrame
-      def remi_df_type
+      def df_type
         :daru
       end
 
@@ -26,42 +26,6 @@ module Remi
       def self.from_hash_dump(filename)
         Marshal.load(File.binread(filename))
       end
-
-      # Public: Allows the user to define an arbitrary aggregation function.
-      #
-      # by   - The name of the DataFrame vector to use to group records.
-      # func - A lambda function that accepts three arguments - the
-      #        first argument is the DataFrame, the second is the
-      #        key to the current group, and the third is the index
-      #        of the elements belonging to a group.
-      #
-      # Example:
-      #   df = Remi::DataFrame::Daru.new( { a: ['a','a','a','b','b'], year: ['2018','2015','2019', '2014', '2013'] })
-      #
-      #   mymin = lambda do |vector, df, group_key, indices|
-      #     values = indices.map { |idx| df.row[idx][vector] }
-      #     "Group #{group_key} has a minimum value of #{values.min}"
-      #   end
-      #
-      #   df.aggregate(by: :a, func: mymin.curry.(:year))
-      #
-      #
-      # Returns a Daru::Vector.
-      def aggregate(by:, func:)
-        grouped = self.group_by(by)
-        df_indices = self.index.to_a
-        ::Daru::Vector.new(
-          grouped.groups.reduce({}) do |h, (key, indices)|
-            # Daru groups don't use the index of the dataframe when returning groups (WTF?).
-            # Instead they return the position of the record in the dataframe.  Here, we
-            group_df_indices = indices.map { |v| df_indices[v] }
-            group_key = key.size == 1 ? key.first : key
-            h[group_key] = func.(self, group_key, group_df_indices)
-            h
-          end
-        )
-      end
-
     end
   end
 end

data/lib/remi/data_subject.rb

@@ -1,45 +1,92 @@
 module Remi
+
+  # The DataSubject is the parent class for DataSource and DataTarget.  It is not intended
+  # to be used as a standalone class.
+  #
+  # A DataSubject is either a source or a target.  It is largely used to associate
+  # a dataframe with a set of "fields" containing metadata describing how the vectors
+  # of the dataframe are meant to be interpreted.  For example, one of the fields
+  # might represent a date with MM-DD-YYYY format.
+  #
+  # DataSubjects can be defined either using the standard `DataSubject.new(<args>)`
+  # convention, or through a DSL, which is convenient for data subjects defined
+  # in as part of job class definition.
   class DataSubject
-    def initialize(*args, fields: Remi::Fields.new, remi_df_type: :daru, logger: Remi::Settings.logger, **kargs, &block)
-      @fields = fields
-      @remi_df_type = remi_df_type
-      @logger = logger
+
+    # @param context [Object] the context in which the DSL is evaluated
+    # @param name [Symbol,String] the name of the data subject
+    # @param block [Proc] a block of code to be executed to define the data subject
+    def initialize(context=nil, name: 'NOT DEFINED', **kargs, &block)
+      @context = context
+      @name = name
+      @block = block
+      @df_type = :daru
+      @fields = Remi::Fields.new
+      @field_symbolizer = Remi::FieldSymbolizers[:standard]
+    end
+
+    attr_accessor :context, :name
+
+
+    # @param arg [Symbol] sets the type of dataframe to use for this subject
+    # @return [Symbol] the type of dataframe (defaults to `:daru` if not explicitly set)
+    def df_type(arg = nil)
+      return get_df_type unless arg
+      set_df_type arg
     end
 
-    # Public: Fields defined for this data subject
-    attr_accessor :fields
+    # @param arg [Hash, Remi::Fields] set the field metadata for this data subject
+    # @return [Remi::Fields] the field metadata for this data subject
+    def fields(arg = nil)
+      return get_fields unless arg
+      set_fields arg
+    end
 
-    # Public: The default method for symbolizing field names
-    def field_symbolizer
-      Remi::FieldSymbolizers[:standard]
+    # @param arg [Hash, Remi::Fields] set the field metadata for this data subject
+    # @return [Remi::Fields] the field metadata for this data subject
+    def fields=(arg)
+      @fields = Remi::Fields.new(arg)
     end
 
-    # Public: Access the dataframe from a DataSource
+    # Field symbolizer used to convert field names into symbols.  This method sets
+    # the symbolizer for the data subject and also sets the symbolizers for
+    # any associated parser and encoders.
     #
-    # Returns a Remi::DataFrame
+    # @return [Proc] the method for symbolizing field names
+    def field_symbolizer(arg = nil)
+      return @field_symbolizer unless arg
+      @field_symbolizer = if arg.is_a? Symbol
+                            Remi::FieldSymbolizers[arg]
+                          else
+                            arg
+                          end
+    end
+
+    # @return [Remi::DataFrame] the dataframe associated with this DataSubject
     def df
-      @dataframe ||= Remi::DataFrame.create(@remi_df_type, [], order: @fields.keys)
+      @dataframe ||= Remi::DataFrame.create(df_type, [], order: fields.keys)
     end
 
-    # Public: Reassigns the dataframe associated with this subject
-    #
-    # Returns the assigned dataframe
+    # Reassigns the dataframe associated with this DataSubject.
+    # @param new_dataframe [Object] The new dataframe object to be associated.
+    # @return [Remi::DataFrame] the associated dataframe
     def df=(new_dataframe)
-      if new_dataframe.respond_to? :remi_df_type
+      if new_dataframe.respond_to? :df_type
         @dataframe = new_dataframe
       else
-        @dataframe = Remi::DataFrame.create(@remi_df_type, new_dataframe)
+        @dataframe = Remi::DataFrame.create(df_type, new_dataframe)
       end
     end
 
-    # Public: Enforces types defined in the field metadata.
-    # For example, if a field has metadata with type: :date, then the
-    # type enforcer will convert data in that field into a date, and will
+    # Enforces the types defined in the field metadata.  Throws an
+    # error if a data element does not conform to the type.  For
+    # example, if a field has metadata with type: :date, then the type
+    # enforcer will convert data in that field into a date, and will
     # throw an error if it is unable to parse any of the values.
     #
-    # types - If set, restricts the data types that are enforced to just those listed.
-    #
-    # Returns nothing.
+    # @param types [Array<Symbol>] a list of metadata types to use to enforce.  If none are given,
+    #   all types are enforced.
+    # @return [self]
     def enforce_types(*types)
       sttm = SourceToTargetMap.new(df, source_metadata: fields)
       fields.keys.each do |field|
@@ -47,63 +94,202 @@ module Remi
         sttm.source(field).target(field).transform(Remi::Transform::EnforceType.new).execute
       end
 
-      nil
+      self
+    end
+
+    # Defines the subject using the DSL in the block provided
+    #
+    # @return [self]
+    def dsl_eval
+      dsl_eval! unless @dsl_evaluated
+      @dsl_evaluated = true
+      self
+    end
+
+    def dsl_eval!
+      return self unless @block
+      Dsl.dsl_eval(self, @context, &@block)
+    end
+
+    private
+
+    def set_fields(arg)
+      self.fields = arg
+    end
+
+    def get_fields
+      dsl_eval
+      @fields
+    end
+
+    def set_df_type(arg)
+      @df_type = arg
+    end
+
+    def get_df_type
+      dsl_eval
+      @df_type
     end
   end
 
 
+
+  # The DataSource is a DataSubject meant to extract data from an external source
+  # and convert (parse) it into a dataframe.
+  #
+  # @example
+  #
+  #   my_data_source = DataSource.new do
+  #     extractor some_extractor
+  #     parser some_parser
+  #   end
+  #
+  #   my_data_source.df #=> Returns a dataframe that is created by extracting data
+  #                     #   from some_extractor and parsing it using some_parser.
   class DataSource < DataSubject
 
-    # Public: Access the dataframe from a DataSource
+    def initialize(*args, **kargs, &block)
+      @parser = Parser::None.new
+      @parser.context = self
+      super
+    end
+
+    # @return [Array] the list of extractors that are defined for this data source
+    def extractors
+      @extractors ||= []
+    end
+
+    # @param obj [Object] adds an extractor object to the list of extractors
+    # @return [Array] the full list of extractors
+    def extractor(obj)
+      extractors << obj unless extractors.include? obj
+    end
+
+    # @param obj [Object] sets the parser for this data source
+    # @return [Object] the parser set for this data source
+    def parser(obj = nil)
+      return @parser unless obj
+      obj.context = self
+
+      @parser = obj
+    end
+
+    # Extracts data from all of the extractors.
+    # @return [Array] the result of each extractor
+    def extract!
+      extractors.map { |e| e.extract }
+    end
+
+    # Converts all of the extracted data to a dataframe
+    # @return [Remi::DataFrame]
+    def parse
+      parser.parse *extract
+    end
+
+    # The dataframe will only be extracted and parsed once, and only if it
+    # has not already been set (e.g., using #df=).
     #
-    # Returns a Remi::DataFrame
+    # @return [Remi::DataFrame] the dataframe associated with this DataSubject
     def df
-      @dataframe ||= to_dataframe
+      @dataframe ||= parsed_as_dataframe
     end
 
-    # Public: Memoized version of extract!
+    # This clears any previously extracted and parsed results.
+    # A subsequent call to #df will redo the extract and parse.
+    #
+    # @return [Remi::DataFrame] the dataframe associated with this DataSubject
+    def reset
+      @block = nil
+      @dataframe = nil
+      @extract = nil
+    end
+
+    # @return [Array<Object>] all of the data extracted from the extractors (memoized).
     def extract
       @extract ||= extract!
     end
 
-    # Public: Called to extract data from the source.
-    #
-    # Returns data in a format that can be used to create a dataframe.
-    def extract!
-      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
-      @extract
-    end
 
-    # Public: Converts extracted data to a dataframe
-    #
-    # Returns a Remi::DataFrame
-    def to_dataframe
-      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+    private
+
+    # Runs the DSL definitions and all extracts, parses, and enforced types
+    # @return [Remi::DataFrame] the source extracted and parsed as a dataframe
+    def parsed_as_dataframe
+      dsl_eval if @block
+      dataframe = parse
+      dataframe
     end
   end
 
 
+  # The DataTarget is a DataSubject meant to load data from an associated dataframe
+  # into one or more target systems.
+  #
+  # @example
+  #
+  #   my_data_target = DataTarget.new do
+  #     encoder some_encoder
+  #     loader some_loader
+  #   end
+  #
+  #   my_data_target.df = some_great_dataframe
+  #   my_data_target.load #=> loads data from the dataframe into some target defined by some_loader
   class DataTarget < DataSubject
 
-    # Public: Loads data to the target.  This is automatically called
+    def initialize(*args, **kargs, &block)
+      @encoder = Encoder::None.new
+      @encoder.context = self
+      super
+    end
+
+    # @param obj [Object] sets the encoder for this data target
+    # @return [Object] the encoder set for this data source
+    def encoder(obj = nil)
+      return @encoder unless obj
+      obj.context = self
+
+      @encoder = obj
+    end
+
+    # @return [Array] the list of loaders associated with the this data target
+    def loaders
+      @loaders ||= []
+    end
+
+    # @param obj [Object] adds a loader object to the list of loaders
+    # @return [Array] the full list of loaders
+    def loader(obj)
+      loaders << obj unless loaders.include? obj
+    end
+
+    # Loads data to all targets.  This is automatically called
     # after all transforms have executed, but could also get called manually.
     # The actual load operation is only executed if hasn't already.
     #
-    # Returns true if the load operation was successful.
+    # @return [true] if successful
     def load
-      return true if @loaded || df.size == 0
+      return nil if @loaded || df.size == 0
+      dsl_eval if @block
 
-      @loaded = load!
+      load!
+      @loaded = true
     end
 
-    # Public: Performs the load operation, regardless of whether it has
+    # Performs the load operation, regardless of whether it has
     # already executed.
     #
-    # Returns true if the load operation was successful
+    # @return [nil] nothing
     def load!
-      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+      loaders.each { |l| l.load encoded_dataframe }
+      true
+    end
+
+    private
 
-      false
+    # @return [Object] the encoded data suitable for the loaders
+    def encoded_dataframe
+      @encoded_dataframe ||= encoder.encode df
     end
+
   end
 end