remi 0.2.27 → 0.2.28

data/jobs/transforms/truncate_job.rb

@@ -13,7 +13,7 @@ class TruncateJob
   define_transform :main, sources: :source_data, targets: :target_data do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:my_field) .target(:truncated_field)
-        .transform(Remi::Transform[:truncate].(params[:truncate_len].to_i))
+        .transform(Remi::Transform::Truncate.new(params[:truncate_len].to_i))
     end
   end
 end

data/lib/remi.rb

@@ -39,25 +39,20 @@ require 'remi/settings'
 require 'remi/job'
 require 'remi/source_to_target_map'
 require 'remi/field_symbolizers'
-require 'remi/data_subject'
-require 'remi/sf_bulk_helper' # separate into SF support package
 
 require 'remi/refinements/symbolizer'
-require 'remi/refinements/daru'
 
 require 'remi/extractor/sftp_file'
 
-require 'remi/data_source.rb'
-require 'remi/data_source/data_frame'
-require 'remi/data_source/csv_file'
-require 'remi/data_source/salesforce'
-require 'remi/data_source/postgres'
-
-require 'remi/data_target.rb'
-require 'remi/data_target/data_frame'
-require 'remi/data_target/salesforce'
-require 'remi/data_target/csv_file'
-require 'remi/data_target/sftp_file'
-require 'remi/data_target/postgres'
+require 'remi/fields'
+require 'remi/data_frame'
+require 'remi/data_frame/daru'
+
+require 'remi/data_subject'
+require 'remi/data_subject/csv_file'
+#require 'remi/data_subject/salesforce' # intentionally not included by default
+require 'remi/data_subject/postgres'
+require 'remi/data_subject/sftp_file'
+require 'remi/data_subject/data_frame'
 
 require 'remi/transform'

data/lib/remi/cucumber/business_rules.rb

@@ -250,14 +250,14 @@ module Remi::BusinessRules
   class DataSubject
     def initialize(name, subject)
       @name = name
-      @data_obj = subject
+      @data_subject = subject
       @fields = DataFieldCollection.new
 
       stub_data
     end
 
     attr_reader :name
-    attr_reader :data_obj
+    attr_reader :data_subject
 
     def add_field(field_name)
       @fields.add_field(self, field_name)
@@ -272,17 +272,17 @@ module Remi::BusinessRules
     end
 
     def size
-      @data_obj.df.size
+      @data_subject.df.size
     end
 
     def get_attrib(name)
-      @data_obj.send(name)
+      @data_subject.send(name)
     end
 
     # Public: Converts the data subject to a hash where the keys are the table
     # columns and the values are an array for the value of column for each row.
     def column_hash
-      @data_obj.df.to_hash.reduce({}) do |h, (k,v)|
+      @data_subject.df.to_h.reduce({}) do |h, (k,v)|
         h[k.symbolize] = v.to_a
         h
       end
@@ -290,7 +290,7 @@ module Remi::BusinessRules
 
     # For debugging only
     def _df
-      @data_obj.df
+      @data_subject.df
     end
 
 
@@ -298,7 +298,7 @@ module Remi::BusinessRules
     # Need more robust duping to make that feasible.
     # Don't use results for anything more than size.
     def where(field_name, operation)
-      @data_obj.df.where(@data_obj.df[field_name.symbolize(@data_obj.field_symbolizer)].recode { |v| operation.call(v) })
+      @data_subject.df.where(@data_subject.df[field_name.symbolize(@data_subject.field_symbolizer)].recode { |v| operation.call(v) })
     end
 
     def where_is(field_name, value)
@@ -324,29 +324,29 @@ module Remi::BusinessRules
 
 
     def stub_data
-      @data_obj.stub_df if @data_obj.respond_to? :stub_df
+      @data_subject.stub_df if @data_subject.respond_to? :stub_df
     end
 
     def example_to_df(example)
-      example.to_df(@data_obj.df.row[0].to_hash, field_symbolizer: @data_obj.field_symbolizer)
+      example.to_df(@data_subject.df.row[0].to_h, field_symbolizer: @data_subject.field_symbolizer)
     end
 
     def stub_data_with(example)
       stub_data
-      @data_obj.df = example_to_df(example)
+      @data_subject.df = example_to_df(example)
     end
 
     def append_data_with(example)
-      @data_obj.df = @data_obj.df.concat example_to_df(example)
+      @data_subject.df = @data_subject.df.concat example_to_df(example)
     end
 
 
     def replicate_rows(n_rows)
-      replicated_df = Daru::DataFrame.new([], order: @data_obj.df.vectors.to_a)
-      @data_obj.df.each do |vector|
+      replicated_df = Daru::DataFrame.new([], order: @data_subject.df.vectors.to_a)
+      @data_subject.df.each do |vector|
         replicated_df[vector.name] = vector.to_a * n_rows
       end
-      @data_obj.df = replicated_df
+      @data_subject.df = replicated_df
     end
 
     def cumulative_dist_from_freq_table(table, freq_field: 'frequency')
@@ -378,31 +378,31 @@ module Remi::BusinessRules
 
     def distribute_values(table)
       cumulative_dist = cumulative_dist_from_freq_table(table)
-      generated_data = generate_values_from_cumulative_dist(@data_obj.df.size, cumulative_dist)
+      generated_data = generate_values_from_cumulative_dist(@data_subject.df.size, cumulative_dist)
 
       generated_data.each do |field_name, data_array|
         vector_name = fields[field_name].field_name
-        @data_obj.df[vector_name] = Daru::Vector.new(data_array, index: @data_obj.df.index)
+        @data_subject.df[vector_name] = Daru::Vector.new(data_array, index: @data_subject.df.index)
       end
     end
 
     def freq_by(*field_names)
-      @data_obj.df.group_by(field_names).size * 1.0 / @data_obj.df.size
+      @data_subject.df.group_by(field_names).size * 1.0 / @data_subject.df.size
     end
 
     def mock_extractor(filestore)
-      extractor = class << @data_obj.extractor; self; end
+      extractor = class << @data_subject.extractor; self; end
 
       extractor.send(:define_method, :all_entries, ->() { filestore.sftp_entries })
       extractor.send(:define_method, :download, ->(to_download) { to_download.map { |e| e.name } })
     end
 
     def extract
-      @data_obj.extractor.extract
+      @data_subject.extractor.extract
     end
 
     def csv_options
-      @data_obj.csv_options
+      @data_subject.csv_options
     end
 
   end
@@ -456,7 +456,7 @@ module Remi::BusinessRules
     def initialize(subject, name)
       @subject = subject
       @name = name
-      @field_name = name.symbolize(subject.data_obj.field_symbolizer)
+      @field_name = name.symbolize(subject.data_subject.field_symbolizer)
     end
 
     attr_reader :name
@@ -468,11 +468,11 @@ module Remi::BusinessRules
     end
 
     def metadata
-      @subject.data_obj.fields[@field_name]
+      @subject.data_subject.fields[@field_name]
     end
 
     def vector
-      @subject.data_obj.df[@field_name]
+      @subject.data_subject.df[@field_name]
     end
 
     def value

data/lib/remi/cucumber/data_source.rb

@@ -54,7 +54,8 @@ module Remi
       end
     end
 
-    class Salesforce
+    # Hmmm.... this gets called first because I'm trying to split SF off as a "plugin"
+    class Salesforce < Remi::DataSubject
       include DataStub
     end
 

data/lib/remi/data_frame.rb

@@ -0,0 +1,36 @@
+module Remi
+  module DataFrame
+    class << self
+      def create(remi_df_type = :daru, *args, **kargs, &block)
+        dataframe = case remi_df_type
+          when :daru
+            Remi::DataFrame::Daru.new(*args, **kargs, &block)
+          else
+            raise TypeError, "Unknown frame type: #{remi_df_type}"
+          end
+      end
+
+      def daru(*args, **kargs, &block)
+        self.create(:daru, *args, **kargs, &block)
+      end
+    end
+
+
+    def [](*args)
+      super
+    end
+
+    def size
+      super
+    end
+
+    def write_csv(*args, **kargs, &block)
+      super
+    end
+
+    # Public: Returns the type of DataFrame
+    def remi_df_type
+      raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+    end
+  end
+end

data/lib/remi/data_frame/daru.rb

@@ -0,0 +1,67 @@
+module Remi
+  module DataFrame
+    class Daru < SimpleDelegator
+      include Remi::DataFrame
+
+      def initialize(*args, **kargs, &block)
+        if args[0].is_a? ::Daru::DataFrame
+          super(args[0])
+        else
+          super(::Daru::DataFrame.new(*args, **kargs, &block))
+        end
+      end
+
+
+      # Public: Returns the type of DataFrame
+      def remi_df_type
+        :daru
+      end
+
+      # Public: Saves a Dataframe to a file.
+      def hash_dump(filename)
+        File.binwrite(filename, Marshal.dump(self))
+      end
+
+      # Public: Creates a DataFrame by reading the dumped version from a file.
+      def self.from_hash_dump(filename)
+        Marshal.load(File.binread(filename))
+      end
+
+      # Public: Allows the user to define an arbitrary aggregation function.
+      #
+      # by   - The name of the DataFrame vector to use to group records.
+      # func - A lambda function that accepts three arguments - the
+      #        first argument is the DataFrame, the second is the
+      #        key to the current group, and the third is the index
+      #        of the elements belonging to a group.
+      #
+      # Example:
+      #   df = Remi::DataFrame::Daru.new( { a: ['a','a','a','b','b'], year: ['2018','2015','2019', '2014', '2013'] })
+      #
+      #   mymin = lambda do |vector, df, group_key, indices|
+      #     values = indices.map { |idx| df.row[idx][vector] }
+      #     "Group #{group_key} has a minimum value of #{values.min}"
+      #   end
+      #
+      #   df.aggregate(by: :a, func: mymin.curry.(:year))
+      #
+      #
+      # Returns a Daru::Vector.
+      def aggregate(by:, func:)
+        grouped = self.group_by(by)
+        df_indices = self.index.to_a
+        ::Daru::Vector.new(
+          grouped.groups.reduce({}) do |h, (key, indices)|
+            # Daru groups don't use the index of the dataframe when returning groups (WTF?).
+            # Instead they return the position of the record in the dataframe.  Here, we
+            group_df_indices = indices.map { |v| df_indices[v] }
+            group_key = key.size == 1 ? key.first : key
+            h[group_key] = func.(self, group_key, group_df_indices)
+            h
+          end
+        )
+      end
+
+    end
+  end
+end

data/lib/remi/data_subject.rb

@@ -1,24 +1,85 @@
 module Remi
-  module DataSubject
+
+  # Namespaces for specific sources/targets
+  module DataSource; end
+  module DataTarget; end
+
+  class DataSubject
+    def initialize(*args, fields: Remi::Fields.new, remi_df_type: :daru, logger: Remi::Settings.logger, **kargs, &block)
+      @fields = fields
+      @remi_df_type = remi_df_type
+      @logger = logger
+    end
+
+    attr_accessor :fields
+
     def field_symbolizer
       Remi::FieldSymbolizers[:standard]
     end
 
     def df
-      @dataframe ||= Daru::DataFrame.new([])
+      @dataframe ||= Remi::DataFrame.create(@remi_df_type, [], order: @fields.keys)
     end
 
     def df=(new_dataframe)
-      @dataframe = new_dataframe
+      if new_dataframe.respond_to? :remi_df_type
+        @dataframe = new_dataframe
+      else
+        @dataframe = Remi::DataFrame.create(@remi_df_type, new_dataframe)
+      end
     end
 
-    # Fields is a hash where the keys are the data field names and the values
-    # are a hash of metadata.  DataFrames do not currently support metadata,
-    # so the metdata will be empty unless overridden by the specific target.
-    def fields
-      df.vectors.to_a.reduce({}) do |h, v|
-        h[v] = {}
-        h
+    module DataSource
+
+      # Public: Access the dataframe from a DataSource
+      #
+      # Returns a Remi::DataFrame
+      def df
+        @dataframe ||= to_dataframe
+      end
+
+      # Public: Memoized version of extract!
+      def extract
+        @extract ||= extract!
+      end
+
+      # Public: Called to extract data from the source.
+      #
+      # Returns data in a format that can be used to create a dataframe.
+      def extract!
+        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+        @extract
+      end
+
+      # Public: Converts extracted data to a dataframe
+      #
+      # Returns a Remi::DataFrame
+      def to_dataframe
+        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+      end
+    end
+
+    module DataTarget
+
+      # Public: Loads data to the target.  This is automatically called
+      # after all transforms have executed, but could also get called manually.
+      # The actual load operation is only executed if hasn't already.
+      #
+      # Returns true if the load operation was successful.
+      def load
+        return true if @loaded || df.size == 0
+
+        @loaded = load!
+      end
+
+      # Public: Performs the load operation, regardless of whether it has
+      # already executed.
+      #
+      # Returns true if the load operation was successful
+      def load!
+        raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
+
+        false
       end
     end
   end

data/lib/remi/data_subject/csv_file.rb

@@ -0,0 +1,151 @@
+module Remi
+  module DataSubject::CsvFile
+    def self.included(base)
+      base.extend(CsvFileClassMethods)
+    end
+
+    def field_symbolizer
+      self.class.default_csv_options[:header_converters]
+    end
+
+    module CsvFileClassMethods
+      def default_csv_options
+        @default_csv_options ||= CSV::DEFAULT_OPTIONS.merge({
+          headers: true,
+          header_converters: Remi::FieldSymbolizers[:standard],
+          converters: [],
+          col_sep: ',',
+          encoding: 'UTF-8',
+          quote_char: '"'
+        })
+      end
+    end
+  end
+
+
+
+
+
+  class DataSource::CsvFile < Remi::DataSubject
+    include Remi::DataSubject::DataSource
+    include Remi::DataSubject::CsvFile
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_csv_file(*args, **kargs, &block)
+    end
+
+    attr_reader   :extractor
+    attr_reader   :csv_options
+
+    # Public: Called to extract data from the source.
+    #
+    # Returns data in a format that can be used to create a dataframe.
+    def extract!
+      @extract = Array(@extractor.extract)
+    end
+
+    # Public: Converts extracted data to a dataframe.
+    # Currently only supports Daru DataFrames.
+    #
+    # Returns a Remi::DataFrame
+    def to_dataframe
+      # Assumes that each file has exactly the same structure
+      result_df = nil
+      extract.each_with_index do |filename, idx|
+        @logger.info "Converting #{filename} to a dataframe"
+        csv_df = Daru::DataFrame.from_csv filename, @csv_options
+
+        csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
+        if idx == 0
+          result_df = csv_df
+        else
+          result_df = result_df.concat csv_df
+        end
+      end
+
+      Remi::DataFrame.create(:daru, result_df)
+    end
+
+
+
+    def extractor=(arg)
+      case arg
+      when Extractor::SftpFile, Extractor::LocalFile
+        @extractor = arg
+      when String
+        @extractor = Extractor::LocalFile.new(path: arg)
+      when Regexp
+        raise "Adding regex matching to local files would be easy, not done yet"
+      else
+        raise "Unknown extractor of type #{arg.class}: #{arg}"
+      end
+    end
+
+    # Only going to support single file for now
+    def source_filename
+      raise "Multiple source files detected" if extract.size > 1
+      @source_filename ||= extract.first
+    end
+
+    def first_line
+      # Readline assumes \n line endings.  Strip out \r if it is a DOS file.
+      @first_line ||= File.open(source_filename) do |f|
+        f.readline.gsub(/\r/,'')
+      end
+    end
+
+    def headers
+      @headers ||= CSV.open(source_filename, 'r', source_csv_options) { |csv| csv.first }.headers
+    end
+
+    def valid_headers?
+      (fields.keys - headers).empty?
+    end
+
+
+    private
+
+    def init_csv_file(*args, extractor:, csv_options: {}, filename_field: nil, **kargs, &block)
+      self.extractor = extractor
+      @csv_options = self.class.default_csv_options.merge(csv_options)
+      @filename_field = filename_field
+    end
+  end
+
+
+
+
+
+  class DataTarget::CsvFile < Remi::DataSubject
+    include ::Remi::DataSubject::DataTarget
+    include ::Remi::DataSubject::CsvFile
+
+    default_csv_options[:row_sep] = "\n"
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_csv_file(*args, **kargs, &block)
+    end
+
+    attr_reader   :csv_options
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      @logger.info "Writing CSV file #{@path}"
+      df.write_csv @path, @csv_options
+      true
+    end
+
+
+    private
+
+    def init_csv_file(*args, path:, csv_options: {}, **kargs, &block)
+      @path = path
+      @csv_options = self.class.default_csv_options.merge(csv_options)
+    end
+  end
+end