remi 0.2.27 → 0.2.28

data/lib/remi/data_subject/data_frame.rb

@@ -0,0 +1,53 @@
+module Remi
+
+  class DataSource::DataFrame < Remi::DataSubject
+    include Remi::DataSubject::DataSource
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_df(*args, **kargs, &block)
+    end
+
+    # Public: Called to extract data from the source.
+    #
+    # Returns data in a format that can be used to create a dataframe.
+    def extract!
+      @extract = []
+    end
+
+    # Public: Converts extracted data to a dataframe
+    #
+    # Returns a Remi::DataFrame
+    def to_dataframe
+      DataFrame.create(@remi_df_type, extract, order: @fields.keys)
+    end
+
+    private
+
+    def init_df(*args, **kargs, &block)
+    end
+  end
+
+
+  class DataTarget::DataFrame < Remi::DataSubject
+    include Remi::DataSubject::DataTarget
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_df(*args, **kargs, &block)
+    end
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      true
+    end
+
+    private
+
+    def init_df(*args, **kargs, &block)
+    end
+  end
+end

data/lib/remi/data_subject/postgres.rb

@@ -0,0 +1,136 @@
+module Remi
+  module DataSubject::Postgres
+    def connection
+      @connection ||= PG.connect(
+        host:     @credentials[:host] || 'localhost',
+        port:     @credentials[:port] || 5432,
+        dbname:   @credentials[:dbname],
+        user:     @credentials[:user] || `whoami`.chomp,
+        password: @credentials[:password],
+        sslmode:  @credentials[:sslmode] || 'allow'
+      )
+    end
+  end
+
+
+  class DataSource::Postgres < Remi::DataSubject
+    include Remi::DataSubject::DataSource
+    include Remi::DataSubject::Postgres
+
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_postgres(*args, **kargs, &block)
+    end
+
+    # Public: Called to extract data from the source.
+    #
+    # Returns data in a format that can be used to create a dataframe.
+    def extract!
+      @logger.info "Executing query #{@query}"
+      @extract = connection.exec @query
+    end
+
+    # Public: Converts extracted data to a dataframe.
+    # Currently only supports Daru DataFrames.
+    #
+    # Returns a Remi::DataFrame
+    def to_dataframe
+      # Performance for larger sets could be improved by using bulk query (via COPY)
+      @logger.info "Converting query to a dataframe"
+
+      hash_array = {}
+      extract.each do |row|
+        row.each do |field, value|
+          (hash_array[field_symbolizer.call(field)] ||= []) << value
+        end
+      end
+
+      # After converting to DF, clear the PG results to save memory.
+      extract.clear
+
+      Remi::DataFrame.create(@remi_df_type, hash_array, order: hash_array.keys)
+    end
+
+
+    private
+
+    def init_postgres(*args, credentials:, query:, **kargs, &block)
+      @credentials = credentials
+      @query = query
+    end
+  end
+
+
+
+  # VERY PRELIMINARY IMPLEMENTAtION - ONLY LOADS TO TEMP TABLES
+  # IT IS THEN UP TO THE USER TO DO ELT TO LOAD THE FINAL TABLE
+  class DataTarget::Postgres < Remi::DataSubject
+    include Remi::DataSubject::DataTarget
+    include Remi::DataSubject::Postgres
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_postgres(*args, **kargs, &block)
+    end
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      @logger.info "Performing postgres load to table #{@table_name}"
+      create_target_table
+      load_target_table
+
+      true
+    end
+
+
+    private
+
+    def init_postgres(*args, credentials:, table_name:, **kargs, &block)
+      @credentials = credentials
+      @table_name = table_name
+    end
+
+    def fields_with_type_ddl
+      @fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
+    end
+
+    def create_target_table
+      create_table_sql = <<-EOT
+        CREATE TEMPORARY TABLE #{@table_name} (
+          #{fields_with_type_ddl}
+        )
+      EOT
+
+      @logger.info create_table_sql
+      connection.exec create_table_sql
+    end
+
+    def load_target_table
+      connection.copy_data "COPY #{@table_name} (#{@fields.keys.join(', ')}) FROM STDIN" do
+        df.each(:row) do |row|
+          row_str = @fields.keys.map do |field|
+            field = row[field]
+            case
+            when field.respond_to?(:strftime)
+              field.strftime('%Y-%m-%d %H:%M:%S')
+            when field.respond_to?(:map)
+              field.to_json.gsub("\t", '\t')
+            when field.blank? && !field.nil?
+              ''
+            when field.nil?
+              '\N'
+            else
+              field.to_s.gsub(/[\t\n\r]/, "\t" => '\t', "\n" => '\n', "\r" => '\r')
+            end
+          end.join("\t")
+
+          connection.put_copy_data row_str + "\n"
+        end
+      end
+    end
+  end
+end

data/lib/remi/data_subject/salesforce.rb

@@ -0,0 +1,136 @@
+require 'restforce'
+require 'salesforce_bulk_api'
+require 'remi/sf_bulk_helper'
+
+module Remi
+  module DataSubject::Salesforce
+    def field_symbolizer
+      Remi::FieldSymbolizers[:salesforce]
+    end
+
+    def restforce_client
+      @restforce_client ||= begin
+        client = Restforce.new(@credentials)
+
+        #run a dummy query to initiate a connection. Workaround for Bulk API problem
+        # https://github.com/yatish27/salesforce_bulk_api/issues/33
+        client.query('SELECT Id FROM Contact LIMIT 1')
+        client
+      end
+    end
+  end
+
+
+  class DataSource::Salesforce < Remi::DataSubject
+    include Remi::DataSubject::DataSource
+    include Remi::DataSubject::Salesforce
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_salesforce(*args, **kargs, &block)
+    end
+
+    # Public: Called to extract data from the source.
+    #
+    # Returns data in a format that can be used to create a dataframe.
+    def extract!
+      @extract = sf_bulk.query(@sfo, @query, 10000)
+
+      check_for_errors(@extract)
+      @extract
+    end
+
+    def sf_bulk
+      @sf_bulk ||= SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
+    end
+
+    # Public: Converts extracted data to a dataframe.
+    # Currently only supports Daru DataFrames.
+    #
+    # Returns a Remi::DataFrame
+    def to_dataframe
+      @logger.info "Converting salesforce query results to a dataframe"
+
+      hash_array = {}
+      extract['batches'].each do |batch|
+        next unless batch['response']
+
+        batch['response'].each do |record|
+          record.each do |field, value|
+            next if ['xsi:type','type'].include? field
+            (hash_array[field.to_sym] ||= []) << case value.first
+              when Hash
+                value.first["xsi:nil"] == "true" ? nil : value.first
+              else
+                value.first
+              end
+          end
+        end
+
+        # delete raw result at end of processing to free memory
+        batch['response'] = nil
+      end
+
+      Remi::DataFrame.create(@remi_df_type, hash_array, order: hash_array.keys)
+    end
+
+
+    private
+
+    def init_salesforce(*args, object:, query:, credentials:, api: :bulk, **kargs, &block)
+      @sfo = object
+      @query = query
+      @credentials = credentials
+      @api = api
+    end
+
+    def check_for_errors(sf_result)
+      sf_result['batches'].each do |batch|
+        raise "Error with batch #{batch['id']} - #{batch['state']}: #{batch['stateMessage']}" unless batch['state'].first == 'Completed'
+      end
+    end
+  end
+
+
+  class DataTarget::Salesforce < Remi::DataSubject
+    include Remi::DataSubject::DataTarget
+    include Remi::DataSubject::Salesforce
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_salesforce(*args, **kargs, &block)
+    end
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
+
+      if @operation == :update
+        Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+      elsif @operation == :create
+        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+      elsif @operation == :upsert
+        Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, df_as_array_of_hashes, external_id: @external_id, logger: @logger)
+      else
+        raise ArgumentError, "Unknown operation: #{@operation}"
+      end
+
+      true
+    end
+
+    private
+
+    def init_salesforce(*args, object:, operation:, credentials:, external_id: 'Id', api: :bulk, **kargs, &block)
+      @sfo = object
+      @operation = operation
+      @external_id = external_id
+      @credentials = credentials
+      @api = api
+    end
+  end
+
+
+end

data/lib/remi/data_subject/sftp_file.rb

@@ -0,0 +1,66 @@
+module Remi
+
+  class DataTarget::SftpFile < Remi::DataSubject
+    include Remi::DataSubject::DataTarget
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_sftp_file(*args, **kargs, &block)
+    end
+
+    attr_reader :local_path
+    attr_reader :remote_path
+
+    # Public: Performs the load operation, regardless of whether it has
+    # already executed.
+    #
+    # Returns true if the load operation was successful
+    def load!
+      @logger.info "Uploading #{@local_path} to #{@credentials[:username]}@#{@credentials[:host]}: #{@remote_path}"
+      connection do |sftp|
+        retry_upload { sftp.upload! @local_path, @remote_path }
+      end
+
+      true
+    end
+
+
+    private
+
+    def init_sftp_file(*args, credentials:, local_path:, remote_path: File.basename(local_path), **kargs, &block)
+      @credentials = credentials
+      @local_path = local_path
+      @remote_path = remote_path
+      init_df
+    end
+
+    def init_df
+      parameter_df = Daru::DataFrame.new(
+        local_path: Array(@local_path),
+        remote_path: Array(@remote_path)
+      )
+      self.df = parameter_df
+    end
+
+    def connection(&block)
+      result = nil
+      Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @credentials[:port] || '22') do |sftp|
+        result = yield sftp
+      end
+      result
+    end
+
+    def retry_upload(ntry=2, &block)
+      1.upto(ntry).each do |itry|
+        begin
+          block.call
+        rescue RuntimeError => err
+          raise err unless itry < ntry
+          @logger.error "Upload failed with error: #{err.message}"
+          @logger.error "Retry attempt #{itry}/#{ntry-1}"
+          sleep(1)
+        end
+      end
+    end
+  end
+end

data/lib/remi/fields.rb

@@ -0,0 +1,8 @@
+module Remi
+  class Fields < SimpleDelegator
+    def initialize(fields=Hash.new({}))
+      @fields = Hash.new({}).merge fields
+      super(@fields)
+    end
+  end
+end

data/lib/remi/source_to_target_map.rb

@@ -1,50 +1,61 @@
 module Remi
   class SourceToTargetMap
-    def initialize(source_df, target_df=nil)
+    def initialize(source_df, target_df=nil, source_metadata: Remi::Fields.new, target_metadata: Remi::Fields.new)
       @source_df = source_df
-      @target_df = target_df || @source_df
+      @source_metadata = source_metadata
+
+      if target_df
+        @target_df = target_df
+        @target_metadata = target_metadata
+      else
+        @target_df = @source_df
+        @target_metadata = @source_metadata
+      end
 
       reset_map
     end
 
-    def self.apply(source_df, target_df=nil, &block)
-      target_df ||= source_df
-      Docile.dsl_eval(SourceToTargetMap.new(source_df, target_df), &block)
+    def self.apply(source_df, target_df=nil, source_metadata: Remi::Fields.new, target_metadata: Remi::Fields.new, &block)
+      sttm = SourceToTargetMap.new(source_df, target_df, source_metadata: source_metadata, target_metadata: target_metadata)
+      Docile.dsl_eval(sttm, &block)
     end
 
-    def source(*source_fields)
-      @source_fields = Array(source_fields)
+    def source(*source_vectors)
+      @source_vectors = Array(source_vectors)
       self
     end
 
     def transform(*transforms)
       @transforms += Array(transforms)
+      @transform_procs += Array(transforms).map { |t| t.to_proc }
       self
     end
 
-    def target(*target_fields)
-      @target_fields = Array(target_fields)
+    def target(*target_vectors)
+      @target_vectors = Array(target_vectors)
       self
     end
 
     def reset_map
-      @source_fields = []
-      @target_fields = []
+      @source_vectors = []
+      @target_vectors = []
       @transforms = []
+      @transform_procs = []
     end
 
     def map(*args)
+      inject_transform_with_metadata
+
       case
-      when @source_fields.include?(nil)
+      when @source_vectors.include?(nil)
         do_map_generic
-      when @source_fields.size == 1 && @transforms.size == 0
+      when @source_vectors.size == 1 && @transforms.size == 0
         do_map_direct_copy
-      when @source_fields.size == 1 && @target_fields.size == 1
-        do_map_single_source_and_target_field
+      when @source_vectors.size == 1 && @target_vectors.size == 1
+        do_map_single_source_and_target_vector
       else
         do_map_generic
       end
-
       reset_map
     end
 
@@ -52,39 +63,52 @@ module Remi
 
     private
 
+    def inject_transform_with_metadata
+      @transforms.each do |tform|
+        if tform.respond_to? :source_metadata
+          meta = @source_vectors.map { |v| @source_metadata[v] || {} }
+          tform.source_metadata = meta.size > 1 ? meta : meta.first
+        end
+        if tform.respond_to? :target_metadata
+          meta = @target_vectors.map { |v| @target_metadata[v] || {} }
+          tform.target_metadata = meta.size > 1 ? meta : meta.first
+        end
+      end
+    end
+
     def do_map_direct_copy
-      @target_fields.each do |target_field|
-        @target_df[target_field] = @source_df[@source_fields.first].dup
+      @target_vectors.each do |target_vector|
+        @target_df[target_vector] = @source_df[@source_vectors.first].dup
       end
     end
 
-    def do_map_single_source_and_target_field
-      @target_df[@target_fields.first] = @source_df[@source_fields.first].recode do |field_value|
-        @transforms.reduce(field_value) { |value, tform| tform.call(*(value || [nil])) }
+    def do_map_single_source_and_target_vector
+      @target_df[@target_vectors.first] = @source_df[@source_vectors.first].recode do |vector_value|
+        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value || [nil])) }
       end
     end
 
     def do_map_generic
-      work_vector = if @source_fields.size == 1 && @source_fields.first != nil
-        @source_df[@source_fields.first].dup
-      elsif @source_fields.size > 1
+      work_vector = if @source_vectors.size == 1 && @source_vectors.first != nil
+        @source_df[@source_vectors.first].dup
+      elsif @source_vectors.size > 1
         # It's faster to zip together several vectors and recode those than it is to
         # recode a dataframe row by row!
-        Daru::Vector.new(@source_df[@source_fields.first].zip(*@source_fields[1..-1].map { |name| @source_df[name] }), index: @source_df.index)
+        Daru::Vector.new(@source_df[@source_vectors.first].zip(*@source_vectors[1..-1].map { |name| @source_df[name] }), index: @source_df.index)
       else
         Daru::Vector.new([], index: @source_df.index)
       end
 
-      work_vector.recode! do |field_value|
-        @transforms.reduce(field_value) { |value, tform| tform.call(*(value || [nil])) }
+      work_vector.recode! do |vector_value|
+        @transform_procs.reduce(vector_value) { |value, tform| tform.call(*(value || [nil])) }
       end
 
-      @target_fields.each_with_index do |target_field, field_idx|
-        @target_df[target_field] = work_vector.recode do |field_value|
-          if field_value.is_a?(Array) then
-            field_value[field_idx]
+      @target_vectors.each_with_index do |target_vector, vector_idx|
+        @target_df[target_vector] = work_vector.recode do |vector_value|
+          if vector_value.is_a?(Array) then
+            vector_value[vector_idx]
           else
-            field_value
+            vector_value
           end
         end
       end