remi 0.2.42 → 0.3.0

data/jobs/aggregate_job.rb

@@ -1,31 +0,0 @@
-require_relative 'all_jobs_shared'
-
-class AggregateJob
-  include AllJobsShared
-
-  define_source :source_data, Remi::DataSource::DataFrame
-  define_target :target_data, Remi::DataTarget::DataFrame
-  define_target :multigroup_target_data, Remi::DataTarget::DataFrame
-
-  define_transform :main, sources: :source_data, targets: :target_data do
-    mymin = lambda do |field, df, group_key, indicies|
-      values = indicies.map { |idx| df.row[idx][field] }
-      "Group #{group_key} has a minimum value of #{values.min}"
-    end
-
-    # Daru groups don't use the index of the dataframe when returning groups (WTF?).
-    # Instead they return the position of the record in the dataframe.  Here, we
-    # shift the indexes which causes a failure if this artifact is not handled
-    # properly in the aggregate function
-    source_data.df.index = Daru::Index.new(1.upto(source_data.df.size).to_a)
-
-    target_data.df = source_data.df.aggregate(by: :alpha, func: mymin.curry.(:year)).detach_index
-    target_data.df.vectors = Daru::Index.new([:alpha, :year])
-
-    multigroup_target_data.df = source_data.df.aggregate(by: [:alpha,:beta], func: mymin.curry.(:year)).detach_index
-    multigroup_target_data.df.vectors = Daru::Index.new([:alpha_beta, :year])
-
-
-
-  end
-end

data/jobs/transforms/transform_jobs.rb

@@ -1,4 +0,0 @@
-require_relative 'prefix_job'
-require_relative 'parse_date_job'
-require_relative 'date_diff_job'
-require_relative 'nvl_job'

data/lib/remi/data_subject/csv_file.rb

@@ -1,162 +0,0 @@
-module Remi
-  module DataSubject::CsvFile
-    def self.included(base)
-      base.extend(CsvFileClassMethods)
-    end
-
-    def field_symbolizer
-      self.class.default_csv_options[:header_converters]
-    end
-
-    module CsvFileClassMethods
-      def default_csv_options
-        @default_csv_options ||= CSV::DEFAULT_OPTIONS.merge({
-          headers: true,
-          header_converters: Remi::FieldSymbolizers[:standard],
-          converters: [],
-          col_sep: ',',
-          encoding: 'UTF-8',
-          quote_char: '"'
-        })
-      end
-    end
-  end
-
-
-
-
-
-  class DataSource::CsvFile < DataSource
-    include Remi::DataSubject::CsvFile
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_csv_file(*args, **kargs, &block)
-    end
-
-    attr_reader   :extractor
-    attr_reader   :csv_options
-
-    # Public: Called to extract data from the source.
-    #
-    # Returns data in a format that can be used to create a dataframe.
-    def extract!
-      @extract = Array(@extractor.extract)
-    end
-
-    # Public: Converts extracted data to a dataframe.
-    # Currently only supports Daru DataFrames.
-    #
-    # Returns a Remi::DataFrame
-    def to_dataframe
-      # Assumes that each file has exactly the same structure
-      result_df = nil
-      extract.each_with_index do |filename, idx|
-        filename = filename.to_s
-
-        @logger.info "Converting #{filename} to a dataframe"
-        processed_filename = preprocess(filename)
-        csv_df = Daru::DataFrame.from_csv processed_filename, @csv_options
-
-        csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
-        if idx == 0
-          result_df = csv_df
-        else
-          result_df = result_df.concat csv_df
-        end
-      end
-
-      Remi::DataFrame.create(:daru, result_df)
-    end
-
-
-
-    def extractor=(arg)
-      @extractor = arg.respond_to?(:extract) ? arg : Extractor::LocalFile.new(remote_path: arg.to_s)
-    end
-
-    # Only going to support single file for now
-    def source_filename
-      raise "Multiple source files detected" if extract.size > 1
-      @source_filename ||= extract.first
-    end
-
-    def first_line
-      # Readline assumes \n line endings.  Strip out \r if it is a DOS file.
-      @first_line ||= File.open(source_filename) do |f|
-        f.readline.gsub(/\r/,'')
-      end
-    end
-
-    def headers
-      @headers ||= CSV.open(source_filename, 'r', source_csv_options) { |csv| csv.first }.headers
-    end
-
-    def valid_headers?
-      (fields.keys - headers).empty?
-    end
-
-
-
-    private
-
-    def preprocess(filename)
-      return filename unless @preprocessor
-      @logger.info "Preprocessing #{filename}"
-      tmp_filename = File.join(Remi::Settings.work_dir, "#{Pathname.new(filename).basename}-#{SecureRandom.uuid}")
-
-      dirname = Pathname.new(tmp_filename).dirname
-      FileUtils.mkdir_p(dirname) unless File.directory? dirname
-
-      File.open(tmp_filename, 'w') do |outfile|
-        File.foreach(filename) do |in_line|
-          outfile.write @preprocessor.call(in_line)
-        end
-      end
-
-      tmp_filename
-    end
-
-    def init_csv_file(*args, extractor:, csv_options: {}, filename_field: nil, preprocessor: nil, **kargs, &block)
-      self.extractor = extractor
-      @csv_options = self.class.default_csv_options.merge(csv_options)
-      @filename_field = filename_field
-      @preprocessor = preprocessor
-    end
-  end
-
-
-
-
-
-  class DataTarget::CsvFile < DataTarget
-    include ::Remi::DataSubject::CsvFile
-
-    default_csv_options[:row_sep] = "\n"
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_csv_file(*args, **kargs, &block)
-    end
-
-    attr_reader   :csv_options
-
-    # Public: Performs the load operation, regardless of whether it has
-    # already executed.
-    #
-    # Returns true if the load operation was successful
-    def load!
-      @logger.info "Writing CSV file #{@path}"
-      df.write_csv @path, @csv_options
-      true
-    end
-
-
-    private
-
-    def init_csv_file(*args, path:, csv_options: {}, **kargs, &block)
-      @path = path
-      @csv_options = self.class.default_csv_options.merge(csv_options)
-    end
-  end
-end

data/lib/remi/data_subject/data_frame.rb

@@ -1,52 +0,0 @@
-module Remi
-
-  class DataSource::DataFrame < DataSource
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_df(*args, **kargs, &block)
-    end
-
-    # Public: Called to extract data from the source.
-    #
-    # Returns data in a format that can be used to create a dataframe.
-    def extract!
-      @extract = @data.transpose
-    end
-
-    # Public: Converts extracted data to a dataframe
-    #
-    # Returns a Remi::DataFrame
-    def to_dataframe
-      DataFrame.create(@remi_df_type, extract, order: @fields.keys)
-    end
-
-    private
-
-    def init_df(*args, data: [], **kargs, &block)
-      @data = data
-    end
-  end
-
-
-  class DataTarget::DataFrame < DataTarget
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_df(*args, **kargs, &block)
-    end
-
-    # Public: Performs the load operation, regardless of whether it has
-    # already executed.
-    #
-    # Returns true if the load operation was successful
-    def load!
-      true
-    end
-
-    private
-
-    def init_df(*args, **kargs, &block)
-    end
-  end
-end

data/lib/remi/data_subject/postgres.rb

@@ -1,134 +0,0 @@
-module Remi
-  module DataSubject::Postgres
-    def connection
-      @connection ||= PG.connect(
-        host:     @credentials[:host] || 'localhost',
-        port:     @credentials[:port] || 5432,
-        dbname:   @credentials[:dbname],
-        user:     @credentials[:user] || `whoami`.chomp,
-        password: @credentials[:password],
-        sslmode:  @credentials[:sslmode] || 'allow'
-      )
-    end
-  end
-
-
-  class DataSource::Postgres < DataSource
-    include Remi::DataSubject::Postgres
-
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_postgres(*args, **kargs, &block)
-    end
-
-    # Public: Called to extract data from the source.
-    #
-    # Returns data in a format that can be used to create a dataframe.
-    def extract!
-      @logger.info "Executing query #{@query}"
-      @extract = connection.exec @query
-    end
-
-    # Public: Converts extracted data to a dataframe.
-    # Currently only supports Daru DataFrames.
-    #
-    # Returns a Remi::DataFrame
-    def to_dataframe
-      # Performance for larger sets could be improved by using bulk query (via COPY)
-      @logger.info "Converting query to a dataframe"
-
-      hash_array = {}
-      extract.each do |row|
-        row.each do |field, value|
-          (hash_array[field_symbolizer.call(field)] ||= []) << value
-        end
-      end
-
-      # After converting to DF, clear the PG results to save memory.
-      extract.clear
-
-      Remi::DataFrame.create(@remi_df_type, hash_array, order: hash_array.keys)
-    end
-
-
-    private
-
-    def init_postgres(*args, credentials:, query:, **kargs, &block)
-      @credentials = credentials
-      @query = query
-    end
-  end
-
-
-
-  # VERY PRELIMINARY IMPLEMENTAtION - ONLY LOADS TO TEMP TABLES
-  # IT IS THEN UP TO THE USER TO DO ELT TO LOAD THE FINAL TABLE
-  class DataTarget::Postgres < DataTarget
-    include Remi::DataSubject::Postgres
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_postgres(*args, **kargs, &block)
-    end
-
-    # Public: Performs the load operation, regardless of whether it has
-    # already executed.
-    #
-    # Returns true if the load operation was successful
-    def load!
-      @logger.info "Performing postgres load to table #{@table_name}"
-      create_target_table
-      load_target_table
-
-      true
-    end
-
-
-    private
-
-    def init_postgres(*args, credentials:, table_name:, **kargs, &block)
-      @credentials = credentials
-      @table_name = table_name
-    end
-
-    def fields_with_type_ddl
-      @fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
-    end
-
-    def create_target_table
-      create_table_sql = <<-EOT
-        CREATE TEMPORARY TABLE #{@table_name} (
-          #{fields_with_type_ddl}
-        )
-      EOT
-
-      @logger.info create_table_sql
-      connection.exec create_table_sql
-    end
-
-    def load_target_table
-      connection.copy_data "COPY #{@table_name} (#{@fields.keys.join(', ')}) FROM STDIN" do
-        df.each(:row) do |row|
-          row_str = @fields.keys.map do |field|
-            field = row[field]
-            case
-            when field.respond_to?(:strftime)
-              field.strftime('%Y-%m-%d %H:%M:%S')
-            when field.respond_to?(:map)
-              field.to_json.gsub("\t", '\t')
-            when field.blank? && !field.nil?
-              ''
-            when field.nil?
-              '\N'
-            else
-              field.to_s.gsub(/[\t\n\r]/, "\t" => '\t', "\n" => '\n', "\r" => '\r')
-            end
-          end.join("\t")
-
-          connection.put_copy_data row_str + "\n"
-        end
-      end
-    end
-  end
-end

data/lib/remi/data_subject/salesforce.rb

@@ -1,136 +0,0 @@
-require 'restforce'
-require 'salesforce_bulk_api'
-require 'remi/sf_bulk_helper'
-
-module Remi
-  module DataSubject::Salesforce
-    def field_symbolizer
-      Remi::FieldSymbolizers[:salesforce]
-    end
-
-    def restforce_client
-      @restforce_client ||= begin
-        client = Restforce.new(@credentials)
-
-        #run a dummy query to initiate a connection. Workaround for Bulk API problem
-        # https://github.com/yatish27/salesforce_bulk_api/issues/33
-        client.query('SELECT Id FROM Contact LIMIT 1')
-        client
-      end
-    end
-  end
-
-
-  class DataSource::Salesforce < DataSource
-    include Remi::DataSubject::Salesforce
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_salesforce(*args, **kargs, &block)
-    end
-
-    # Public: Called to extract data from the source.
-    #
-    # Returns data in a format that can be used to create a dataframe.
-    def extract!
-      @extract = sf_bulk.query(@sfo, @query, 10000)
-
-      check_for_errors(@extract)
-      @extract
-    end
-
-    def sf_bulk
-      @sf_bulk ||= SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
-    end
-
-    # Public: Converts extracted data to a dataframe.
-    # Currently only supports Daru DataFrames.
-    #
-    # Returns a Remi::DataFrame
-    def to_dataframe
-      @logger.info "Converting salesforce query results to a dataframe"
-
-      hash_array = {}
-      extract['batches'].each do |batch|
-        next unless batch['response']
-
-        batch['response'].each do |record|
-          record.each do |field, value|
-            next if ['xsi:type','type'].include? field
-            (hash_array[field.to_sym] ||= []) << case value.first
-              when Hash
-                value.first["xsi:nil"] == "true" ? nil : value.first
-              else
-                value.first
-              end
-          end
-        end
-
-        # delete raw result at end of processing to free memory
-        batch['response'] = nil
-      end
-
-      Remi::DataFrame.create(@remi_df_type, hash_array, order: hash_array.keys)
-    end
-
-
-    private
-
-    def init_salesforce(*args, object:, query:, credentials:, api: :bulk, **kargs, &block)
-      @sfo = object
-      @query = query
-      @credentials = credentials
-      @api = api
-    end
-
-    def check_for_errors(sf_result)
-      sf_result['batches'].each do |batch|
-        raise "Error with batch #{batch['id']} - #{batch['state']}: #{batch['stateMessage']}" unless batch['state'].first == 'Completed'
-      end
-    end
-  end
-
-
-  class DataTarget::Salesforce < DataTarget
-    include Remi::DataSubject::Salesforce
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_salesforce(*args, **kargs, &block)
-    end
-
-    # Public: Performs the load operation, regardless of whether it has
-    # already executed.
-    #
-    # Returns true if the load operation was successful
-    def load!
-      @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
-
-      df_as_array_of_hashes = df.to_a[0] # This probably wouldn't work with a non-Daru df
-      if @operation == :update
-        Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, logger: @logger)
-      elsif @operation == :create
-        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, logger: @logger)
-      elsif @operation == :upsert
-        Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, df_as_array_of_hashes, batch_size: @batch_size, external_id: @external_id, logger: @logger)
-      else
-        raise ArgumentError, "Unknown operation: #{@operation}"
-      end
-
-      true
-    end
-
-    private
-
-    def init_salesforce(*args, object:, operation:, credentials:, batch_size: 5000, external_id: 'Id', api: :bulk, **kargs, &block)
-      @sfo = object
-      @operation = operation
-      @batch_size = batch_size
-      @external_id = external_id
-      @credentials = credentials
-      @api = api
-    end
-  end
-
-
-end