remi 0.2.42 → 0.3.0

data/lib/remi/data_subjects/none.rb

@@ -0,0 +1,31 @@
+module Remi
+
+  # The None extractor doesn't do anything.
+  class Extractor::None < Extractor
+    def extract
+      nil
+    end
+  end
+
+
+  # The None Parser just returns what it is given.
+  class Parser::None < Parser
+    def parse(data)
+      data
+    end
+  end
+
+  # The None Encoder just returns what it is given.
+  class Encoder::None < Encoder
+    def encode(data_frame)
+      data_frame
+    end
+  end
+
+  # The None loader doesn't do anything.
+  class Loader::None < Loader
+    def load(data)
+      true
+    end
+  end
+end

data/lib/remi/data_subjects/postgres.rb

@@ -0,0 +1,186 @@
+module Remi
+  # Contains methods shared between Postgres Extractor/Parser/Encoder/Loader
+  module DataSubject::Postgres
+
+    # @return [PG::Connection] An authenticated postgres connection
+    def connection
+      @connection ||= PG.connect(
+        host:     @credentials[:host] || 'localhost',
+        port:     @credentials[:port] || 5432,
+        dbname:   @credentials[:dbname],
+        user:     @credentials[:user] || `whoami`.chomp,
+        password: @credentials[:password],
+        sslmode:  @credentials[:sslmode] || 'allow'
+      )
+    end
+  end
+
+  # Postgres extractor
+  #
+  # @example
+  #  class MyJob < Remi::Job
+  #    source :some_table do
+  #      extractor Remi::Extractor::Postgres.new(
+  #        credentials: {
+  #          dbname: 'my_local_db'
+  #        },
+  #        query: 'SELECT * FROM job_specs'
+  #      )
+  #      parser Remi::Parser::Postgres.new
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.some_table.df[:id, :name]
+  #  # =>#<Daru::DataFrame:70153144824760 @name = 53c8e878-55e7-4859-bc34-ec29309c11fd @size = 3>
+  #  #                    id       name
+  #  #          0         24 albert
+  #  #          1         26 betsy
+  #  #          2         25 camu
+
+  class Extractor::Postgres < Extractor
+    include DataSubject::Postgres
+
+    # @param credentials [Hash] Used to authenticate with the postgres db
+    # @option credentials [String] :host Postgres host (default: localhost)
+    # @option credentials [Integer] :port Postgres host (default: 5432)
+    # @option credentials [String] :dbname Database name
+    # @option credentials [String] :user Postgres login username (default: `whoami`)
+    # @option credentials [String] :password Postgres login password
+    # @option credentials [String] :sslmode Postgres SSL mode (default: allow)
+    # @param query [String] Query to use to extract data
+    def initialize(*args, **kargs, &block)
+      super
+      init_postgres_extractor(*args, **kargs, &block)
+    end
+
+    # @return [Object] Data extracted from Postgres system
+    attr_reader :data
+
+    # @return [Object] self after querying Postgres data
+    def extract
+      logger.info "Executing Postgres query #{@query}"
+      @data = execute_query
+      self
+    end
+
+    private
+
+    def execute_query
+      connection.exec @query
+    end
+
+    def init_postgres_extractor(*args, credentials:, query:, **kargs, &block)
+      @credentials = credentials
+      @query = query
+    end
+  end
+
+
+  # Postgres parser
+  # Used to parse results from a postgres extractor (see Extractor::Postgres).
+  class Parser::Postgres < Parser
+
+    # @param postgres_extract [Extractor::Postgres] An object containing data extracted from Postgres
+    # @return [Remi::DataFrame] The data converted into a dataframe
+    def parse(postgres_extract)
+      # Performance for larger sets could be improved by using bulk query (via COPY)
+      logger.info "Converting Postgres query to a dataframe"
+
+      hash_array = {}
+      postgres_extract.data.each do |row|
+        row.each do |field, value|
+          (hash_array[field_symbolizer.call(field)] ||= []) << value
+        end
+      end
+
+      # After converting to DF, clear the PG results to save memory.
+      postgres_extract.data.clear
+
+      Remi::DataFrame.create(:daru, hash_array, order: hash_array.keys)
+    end
+  end
+
+
+  # Postgres encoder
+  class Encoder::Postgres < Encoder
+
+    # @return [Array<String>] All records of the dataframe encoded as strings to be used by Postgres Bulk updater
+    attr_accessor :values
+
+    # Converts the dataframe to an array of hashes, which can be used
+    # by the postgres loader.
+    #
+    # @param dataframe [Remi::DataFrame] The dataframe to be encoded
+    # @return [Object] The encoded data to be loaded into the target
+    def encode(dataframe)
+      @values = encode_data(dataframe)
+      self
+    end
+
+    # @return [String] Field definitions to be used in the DDL
+    def ddl_fields
+      fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
+    end
+
+    private
+
+    def encode_data(dataframe)
+      dataframe.map(:row) do |row|
+        fields.keys.map do |field|
+          field = row[field]
+          case
+          when field.respond_to?(:strftime)
+            field.strftime('%Y-%m-%d %H:%M:%S')
+          when field.respond_to?(:map)
+            field.to_json.gsub("\t", '\t')
+          when field.blank? && !field.nil?
+            ''
+          when field.nil?
+            '\N'
+          else
+            field.to_s.gsub(/[\t\n\r]/, "\t" => '\t', "\n" => '\n', "\r" => '\r')
+          end
+        end.join("\t")
+      end
+    end
+  end
+
+
+  # Postgres Loader
+  # VERY PRELIMINARY IMPLEMENTAtION - ONLY LOADS TO TEMP TABLES
+  # IT IS THEN UP TO THE USER TO DO ELT TO LOAD THE FINAL TABLE
+  class Loader::Postgres < Loader
+    include DataSubject::Postgres
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_postgres_loader(*args, **kargs, &block)
+    end
+
+    # @param data [Encoder::Postgres] Data that has been encoded appropriately to be loaded into the target
+    # @return [true] On success
+    def load(data)
+      logger.info "Performing postgres load to table #{@table_name}"
+      create_table_sql = "CREATE TEMPORARY TABLE #{@table_name} (#{data.ddl_fields})"
+      logger.info create_table_sql
+      connection.exec create_table_sql
+
+      connection.copy_data "COPY #{@table_name} (#{data.fields.keys.join(', ')}) FROM STDIN" do
+        data.values.each do |row|
+          connection.put_copy_data "#{row}\n"
+        end
+      end
+
+      true
+    end
+
+
+    private
+
+    def init_postgres_loader(*args, credentials:, table_name:, **kargs, &block)
+      @credentials = credentials
+      @table_name = table_name
+    end
+  end
+end

data/lib/remi/data_subjects/s3_file.rb

@@ -0,0 +1,84 @@
+module Remi
+
+  # S3 File extractor
+  # Used to extract files from Amazon S3
+  #
+  # @example
+  #
+  #  class MyJob < Remi::Job
+  #    source :some_file do
+  #      extractor Remi::Extractor::S3File.new(
+  #        bucket: 'my-awesome-bucket',
+  #        remote_path: 'some_file-',
+  #        most_recent_only: true
+  #      )
+  #      parser Remi::Parser::CsvFile.new(
+  #        csv_options: {
+  #          headers: true,
+  #          col_sep: '|'
+  #        }
+  #      )
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.some_file.df
+  #  # =>#<Daru::DataFrame:70153153438500 @name = 4c59cfdd-7de7-4264-8666-83153f46a9e4 @size = 3>
+  #  #                    id       name
+  #  #          0          1     Albert
+  #  #          1          2      Betsy
+  #  #          2          3       Camu
+  class Extractor::S3File < Extractor::FileSystem
+
+    # @param bucket_name [String] S3 bucket containing the files
+    def initialize(*args, **kargs, &block)
+      super
+      init_s3_file(*args, **kargs, &block)
+    end
+
+    # Called to extract files from the source filesystem.
+    # @return [Array<String>] An array of paths to a local copy of the files extacted
+    def extract
+      entries.map do |entry|
+        local_file = File.join(@local_path, entry.name)
+        logger.info "Downloading #{entry.pathname} from S3 to #{local_file}"
+        File.open(local_file, 'wb') { |file| entry.raw.get(response_target: file) }
+        local_file
+      end
+    end
+
+    # @return [Array<Extractor::FileSystemEntry>] (Memoized) list of objects in the bucket/prefix
+    def all_entries
+      @all_entries ||= all_entries!
+    end
+
+    # @return [Array<Extractor::FileSystemEntry>] List of objects in the bucket/prefix
+    def all_entries!
+      # S3 does not track anything like a create time, so use last modified for both
+      bucket.objects(prefix: @remote_path.to_s).map do |entry|
+        Extractor::FileSystemEntry.new(
+          pathname: entry.key,
+          create_time: entry.last_modified,
+          modified_time: entry.last_modified,
+          raw: entry
+        )
+      end
+    end
+
+    # @return [Aws::S3::Client] The S3 client used
+    def s3_client
+      @s3_client ||= Aws::S3::Client.new
+    end
+
+    private
+
+    def init_s3_file(*args, bucket:, **kargs)
+      @bucket_name = bucket
+    end
+
+    def bucket
+      @bucket ||= Aws::S3::Bucket.new(@bucket_name, client: s3_client)
+    end
+
+  end
+end

data/lib/remi/data_subjects/salesforce.rb

@@ -0,0 +1,211 @@
+require 'restforce'
+require 'salesforce_bulk_api'
+require 'remi/sf_bulk_helper'
+
+module Remi
+
+  # Contains methods shared between Salesforce Extractor/Parser/Encoder/Loader
+  module DataSubject::Salesforce
+
+    # @return [Restforce] An authenticated restforce client
+    def restforce_client
+      @restforce_client ||= begin
+        client = Restforce.new(@credentials)
+
+        #run a dummy query to initiate a connection. Workaround for Bulk API problem
+        # https://github.com/yatish27/salesforce_bulk_api/issues/33
+        client.query('SELECT Id FROM Contact LIMIT 1')
+        client
+      end
+    end
+  end
+
+
+  # Salesforce extractor
+  #
+  # @example
+  #
+  #  class MyJob < Remi::Job
+  #    source :contacts do
+  #      extractor Remi::Extractor::Salesforce.new(
+  #        credentials: { },
+  #        object: :Contact,
+  #        api: :bulk,
+  #        query: 'SELECT Id, Student_ID__c, Name FROM Contact LIMIT 1000'
+  #      )
+  #      parser Remi::Parser::Salesforce.new
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.contacts.df
+  #  # #<Daru::DataFrame:70134211545860 @name = 7cddb460-6bfc-4737-a72c-60ed2c1a97d5 @size = 1>
+  #  #                    Id Student_ID       Name
+  #  #          0 0031600002   test1111  Run Logan
+  class Extractor::Salesforce < Extractor
+    include Remi::DataSubject::Salesforce
+
+    class ExtractError < StandardError; end
+
+    # @param credentials [Hash] Used to authenticate with salesforce
+    # @option credentials [String] :host Salesforce host (e.g., login.salesforce.com)
+    # @option credentials [String] :client_id Salesforce Rest client id
+    # @option credentials [String] :client_secret Salesforce Rest client secret
+    # @option credentials [String] :instance_url Salesforce instance URL (e.g., https://na1.salesforce.com)
+    # @option credentials [String] :username Salesforce username
+    # @option credentials [String] :password Salesforce password
+    # @option credentials [String] :security_token Salesforce security token
+    # @param object [Symbol] Salesforce object to extract
+    # @param query [String] The SOQL query to execute to extract data
+    # @param api [Symbol] Salesforce API to use (only option supported is `:bulk`)
+    def initialize(*args, **kargs, &block)
+      super
+      init_salesforce_extractor(*args, **kargs, &block)
+    end
+
+    attr_reader :data
+
+    # @return [Object] self after querying salesforce data
+    def extract
+      logger.info "Executing salesforce query #{@query}"
+      @data = sf_bulk.query(@sfo, @query, 10000)
+      check_for_errors(@data)
+      self
+    end
+
+    # @return [SalesforceBulkApi::Api] The bulk API salesforce client
+    def sf_bulk
+      SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
+    end
+
+    private
+
+    def init_salesforce_extractor(*args, object:, query:, credentials:, api: :bulk, **kargs, &block)
+      @sfo         = object
+      @query       = query
+      @credentials = credentials
+      @api         = api
+    end
+
+    def check_for_errors(sf_result)
+      sf_result['batches'].each do |batch|
+        raise ExtractError, "Error with batch #{batch['id']} - #{batch['state']}: #{batch['stateMessage']}" unless batch['state'].first == 'Completed'
+      end
+    end
+  end
+
+  # Salesforce parser
+  class Parser::Salesforce < Parser
+
+    # @param sf_extract [Extractor::Salesforce] An object containing data extracted from salesforce
+    # @return [Remi::DataFrame] The data converted into a dataframe
+    def parse(sf_extract)
+      logger.info 'Parsing salesforce query results'
+
+      hash_array = {}
+      sf_extract.data['batches'].each do |batch|
+        next unless batch['response']
+
+        batch['response'].each do |record|
+          record.each do |field, value|
+            next if ['xsi:type','type'].include? field
+            (hash_array[field.to_sym] ||= []) << case value.first
+              when Hash
+                value.first["xsi:nil"] == "true" ? nil : value.first
+              else
+                value.first
+              end
+          end
+        end
+
+        # delete raw result at end of processing to free memory
+        batch['response'] = nil
+      end
+
+      Remi::DataFrame.create(:daru, hash_array, order: hash_array.keys)
+    end
+  end
+
+  # Salesforce encoder
+  class Encoder::Salesforce < Encoder
+    # Converts the dataframe to an array of hashes, which can be used
+    # by the salesforce bulk api.
+    #
+    # @param dataframe [Remi::DataFrame] The dataframe to be encoded
+    # @return [Object] The encoded data to be loaded into the target
+    def encode(dataframe)
+      dataframe.to_a[0]
+    end
+  end
+
+  # Salesforce loader
+  #
+  # @example
+  #  class MyJob < Remi::Job
+  #    target :contacts do
+  #      encoder Remi::Encoder::Salesforce.new
+  #      loader Remi::Loader::Salesforce.new(
+  #        credentials: { },
+  #        object: :Contact,
+  #        api: :bulk,
+  #        operation: :update
+  #      )
+  #    end
+  #  end
+  #
+  #  job = MyJob.new
+  #  job.contacts.df = Daru::DataFrame.new({ :Id => ['0031600002Pm7'], :Student_ID__c => ['test1111']})
+  #  job.contacts.load
+  class Loader::Salesforce < Loader
+    include Remi::DataSubject::Salesforce
+
+    # @param credentials [Hash] Used to authenticate with salesforce
+    # @option credentials [String] :host Salesforce host (e.g., login.salesforce.com)
+    # @option credentials [String] :client_id Salesforce Rest client id
+    # @option credentials [String] :client_secret Salesforce Rest client secret
+    # @option credentials [String] :instance_url Salesforce instance URL (e.g., https://na1.salesforce.com)
+    # @option credentials [String] :username Salesforce username
+    # @option credentials [String] :password Salesforce password
+    # @option credentials [String] :security_token Salesforce security token
+    # @param object [Symbol] Salesforce object to extract
+    # @param operation [Symbol] Salesforce operation to perform (`:update`, `:create`, `:upsert`)
+    # @param batch_size [Integer] Size of batch to use for updates (1-10000)
+    # @param external_id [Symbol, String] Field to use as an external id for upsert operations
+    # @param api [Symbol] Salesforce API to use (only option supported is `:bulk`)
+    def initialize(*args, **kargs, &block)
+      super
+      init_salesforce_loader(*args, **kargs, &block)
+    end
+
+    # @param data [Encoder::Salesforce] Data that has been encoded appropriately to be loaded into the target
+    # @return [true] On success
+    def load(data)
+      logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
+
+      if @operation == :update
+        Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
+      elsif @operation == :create
+        Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, data, batch_size: @batch_size, logger: logger)
+      elsif @operation == :upsert
+        Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, data, batch_size: @batch_size, external_id: @external_id, logger: logger)
+      else
+        raise ArgumentError, "Unknown operation: #{@operation}"
+      end
+
+      true
+    end
+
+    private
+
+    def init_salesforce_loader(*args, object:, operation:, credentials:, batch_size: 5000, external_id: 'Id', api: :bulk, **kargs, &block)
+      @sfo         = object
+      @operation   = operation
+      @batch_size  = batch_size
+      @external_id = external_id
+      @credentials = credentials
+      @api         = api
+    end
+
+  end
+
+end