remi 0.0.1 → 0.2.2

data/lib/remi/cucumber/data_source.rb

@@ -0,0 +1,63 @@
+module Remi
+  module DataSource
+    module DataStub
+      def stub_row_array
+        @fields.values.map do |attrib|
+          case attrib[:type]
+          when :date
+            stub_values[:date].strftime(attrib[:format])
+          when nil
+            stub_values[:string]
+          else
+            stub_values[attrib[:type]]
+          end
+        end
+      end
+
+      def stub_df
+        wdf = Daru::DataFrame.new([], order: @fields.keys)
+        wdf.add_row(stub_row_array)
+        self.df = wdf
+      end
+
+      def stub_values
+        @stub_values ||= {
+          string: "Some String",
+          number: 133,
+          float: 3.14159,
+          integer: 38,
+          date: Date.parse('2015-10-21')
+        }
+      end
+    end
+
+
+    class CsvFile
+      include DataStub
+      def stub_tmp_file
+        @stub_tmp_file ||= Tempfile.new('stub_tmp_file.csv').path
+      end
+
+      def write_stub_tmp_file
+        File.open(stub_tmp_file, "wb") do |file|
+          file.puts stub_header
+          file.puts stub_row_csv
+        end
+
+        stub_tmp_file
+      end
+
+      def stub_header
+        @fields.keys.join(@csv_options[:col_sep])
+      end
+
+      def stub_row_csv
+        stub_row_array.join(@csv_options[:col_sep])
+      end
+    end
+
+    class Salesforce
+      include DataStub
+    end
+  end
+end

data/lib/remi/data_source.rb

@@ -0,0 +1,13 @@
+module Remi
+  module DataSource
+    include DataSubject
+
+    def extract
+      raise "Extract function undefined for #{self.class.name}"
+    end
+
+    def feild_symbolizer
+      Remi::FieldSymbolizers[:standard]
+    end
+  end
+end

data/lib/remi/data_source/csv_file.rb

@@ -0,0 +1,79 @@
+module Remi
+  module DataSource
+    class CsvFile
+      include DataSource
+
+      def self.default_csv_options
+        CSV::DEFAULT_OPTIONS.merge({
+          headers: true,
+          header_converters: Remi::FieldSymbolizers[:standard],
+          col_sep: ',',
+          encoding: 'UTF-8',
+          quote_char: '"'
+        })
+      end
+
+
+      def initialize(fields: {}, extractor:, csv_options: {}, logger: Remi::Settings.logger)
+        @fields = fields
+        self.extractor = extractor
+        @csv_options = self.class.default_csv_options.merge(csv_options)
+        @logger = logger
+      end
+
+      attr_accessor :fields
+      attr_reader   :extractor
+      attr_reader   :csv_options
+
+      def field_symbolizer
+        self.class.default_csv_options[:header_converters]
+      end
+
+      def extract
+        Array(@extractor.extract).tap { |x| raise "Multiple files not supported" if x.size > 1 }
+      end
+
+      def extractor=(arg)
+        case arg
+        when Extractor::SftpFile, Extractor::LocalFile
+          @extractor = arg
+        when String
+          @extractor = Extractor::LocalFile.new(arg)
+        when Regexp
+          raise "Adding regex matching to local files would be easy, not done yet"
+        else
+          raise "Unknown extractor of type #{arg.class}: #{arg}"
+        end
+      end
+
+      # Only going to support single file for now
+      def source_filename
+        @source_filename ||= extract.first
+      end
+
+      def first_line
+        # Readline assumes \n line endings.  Strip out \r if it is a DOS file.
+        @first_line ||= File.open(source_filename) do |f|
+          f.readline.gsub(/\r/,'')
+        end
+      end
+
+      def headers
+        @headers ||= CSV.open(source_filename, 'r', source_csv_options) { |csv| csv.first }.headers
+      end
+
+      def valid_headers?
+        (fields.keys - headers).empty?
+      end
+
+      def to_dataframe
+        @logger.info "Converting #{source_filename} to a dataframe"
+        Daru::DataFrame.from_csv source_filename, @csv_options
+      end
+
+      def df
+        @dataframe ||= to_dataframe
+      end
+    end
+  end
+end

data/lib/remi/data_source/data_frame.rb

@@ -0,0 +1,10 @@
+module Remi
+  module DataSource
+    class DataFrame
+      include DataSubject
+
+      def initialize(**args)
+      end
+    end
+  end
+end

data/lib/remi/data_source/postgres.rb

@@ -0,0 +1,58 @@
+module Remi
+  module DataSource
+    class Postgres
+      include DataSource
+
+      def initialize(fields: {}, credentials:, query:, logger: Remi::Settings.logger)
+        @fields = fields
+        @credentials = credentials
+        @query = query
+        @logger = logger
+      end
+
+      attr_accessor :fields
+
+      def extract
+        @logger.info "Executing query #{@query}"
+        @raw_result = pg_conn.exec @query
+      end
+
+      def raw_result
+        @raw_result ||= extract
+      end
+
+      def pg_conn
+        @pg_conn ||= PG.connect(
+          host:     @credentials[:host] || 'localhost',
+          port:     @credentials[:port] || 5432,
+          dbname:   @credentials[:dbname],
+          user:     @credentials[:user] || `whoami`.chomp,
+          password: @credentials[:password],
+          sslmode:  @credentials[:sslmode] || 'require'
+        )
+      end
+
+
+      def to_dataframe
+        # Performance for larger sets could be improved by using bulk query (via COPY)
+        @logger.info "Converting query to a dataframe"
+
+        hash_array = {}
+        raw_result.each do |row|
+          row.each do |field, value|
+            (hash_array[field_symbolizer.call(field)] ||= []) << value
+          end
+        end
+
+        # After converting to DF, clear the PG results to save memory.
+        raw_result.clear
+
+        Daru::DataFrame.new hash_array, order: hash_array.keys
+      end
+
+      def df
+        @dataframe ||= to_dataframe
+      end
+    end
+  end
+end

data/lib/remi/data_source/salesforce.rb

@@ -0,0 +1,78 @@
+module Remi
+  module DataSource
+    class Salesforce
+      include DataSource
+
+      def initialize(fields: {}, object:, query:, credentials:, api: :bulk, logger: Remi::Settings.logger)
+        @fields = fields
+        @sfo = object
+        @query = query
+        @credentials = credentials
+        @api = api
+        @logger = logger
+      end
+
+      attr_accessor :fields
+      attr_accessor :raw_result
+
+      def field_symbolizer
+        Remi::FieldSymbolizers[:salesforce]
+      end
+
+      def extract
+        @raw_result = sf_bulk.query(@sfo, @query, 10000)
+      end
+
+      def raw_result
+        @raw_result ||= extract
+      end
+
+
+
+      def restforce_client
+        @restforce_client ||= begin
+          client = Restforce.new(@credentials)
+
+          #run a dummy query to initiate a connection. Workaround for Bulk API problem
+          # https://github.com/yatish27/salesforce_bulk_api/issues/33
+          client.query('SELECT Id FROM Contact LIMIT 1')
+          client
+        end
+      end
+
+      def sf_bulk
+        @sf_bulk ||= SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
+      end
+
+      def to_dataframe
+        @logger.info "Converting salesforce query results to a dataframe"
+
+        hash_array = {}
+        raw_result['batches'].each do |batch|
+          next unless batch['response']
+
+          batch['response'].each do |record|
+            record.each do |field, value|
+              next if ['xsi:type','type'].include? field
+              (hash_array[field.to_sym] ||= []) << case value.first
+                when Hash
+                  value.first["xsi:nil"] == "true" ? nil : value.first
+                else
+                  value.first
+                end
+            end
+          end
+
+          # delete raw result at end of processing to free memory
+          batch['response'] = nil
+        end
+
+        Daru::DataFrame.new hash_array, order: hash_array.keys
+      end
+
+      def df
+        @dataframe ||= to_dataframe
+      end
+    end
+  end
+end

data/lib/remi/data_subject.rb

@@ -0,0 +1,25 @@
+module Remi
+  module DataSubject
+    def field_symbolizer
+      Remi::FieldSymbolizers[:standard]
+    end
+
+    def df
+      @dataframe ||= Daru::DataFrame.new([])
+    end
+
+    def df=(new_dataframe)
+      @dataframe = new_dataframe
+    end
+
+    # Fields is a hash where the keys are the data field names and the values
+    # are a hash of metadata.  DataFrames do not currently support metadata,
+    # so the metdata will be empty unless overridden by the specific target.
+    def fields
+      df.vectors.to_a.reduce({}) do |h, v|
+        h[v] = {}
+        h
+      end
+    end
+  end
+end

data/lib/remi/data_target.rb

@@ -0,0 +1,15 @@
+module Remi
+  module DataTarget
+    include DataSubject
+
+    # Gets called automatically at the end of a job, but could
+    # also get manually called at the end of a transform so make
+    # sure it doesn't do it twice.
+    def load
+      @logger.info "Loading target"
+      return true if @loaded
+      @loaded = true
+      raise "Load function undefined for #{self.class.name}"
+    end
+  end
+end

data/lib/remi/data_target/csv_file.rb

@@ -0,0 +1,49 @@
+module Remi
+  module DataTarget
+    class Salesforce
+      include DataTarget
+
+      def initialize(object:, operation:, credentials:, api: :bulk, logger: Remi::Settings.logger)
+        @sfo = object
+        @operation = operation
+        @credentials = credentials
+        @api = api
+        @logger = logger
+      end
+
+      def field_symbolizer
+        Remi::FieldSymbolizers[:salesforce]
+      end
+
+      def load
+        return true if @loaded || df.size == 0
+
+        @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
+
+        if @operation == :update
+          Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        elsif @operation == :create
+          Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        end
+
+        @loaded = true
+      end
+
+      def restforce_client
+        @restforce_client ||= begin
+          client = Restforce.new(@credentials)
+
+          #run a dummy query to initiate a connection. Workaround for Bulk API problem
+          # https://github.com/yatish27/salesforce_bulk_api/issues/33
+          client.query('SELECT Id FROM Contact LIMIT 1')
+          client
+        end
+      end
+
+      def df_as_array_of_hashes
+        df.to_a[0]
+      end
+
+    end
+  end
+end

data/lib/remi/data_target/data_frame.rb

@@ -0,0 +1,14 @@
+module Remi
+  module DataTarget
+    class DataFrame
+      include DataSubject
+
+      def initialize(**args)
+      end
+
+      def load
+        true
+      end
+    end
+  end
+end

data/lib/remi/data_target/salesforce.rb

@@ -0,0 +1,49 @@
+module Remi
+  module DataTarget
+    class Salesforce
+      include DataTarget
+
+      def initialize(object:, operation:, credentials:, api: :bulk, logger: Remi::Settings.logger)
+        @sfo = object
+        @operation = operation
+        @credentials = credentials
+        @api = api
+        @logger = logger
+      end
+
+      def field_symbolizer
+        Remi::FieldSymbolizers[:salesforce]
+      end
+
+      def load
+        return true if @loaded || df.size == 0
+
+        @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
+
+        if @operation == :update
+          Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        elsif @operation == :create
+          Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
+        end
+
+        @loaded = true
+      end
+
+      def restforce_client
+        @restforce_client ||= begin
+          client = Restforce.new(@credentials)
+
+          #run a dummy query to initiate a connection. Workaround for Bulk API problem
+          # https://github.com/yatish27/salesforce_bulk_api/issues/33
+          client.query('SELECT Id FROM Contact LIMIT 1')
+          client
+        end
+      end
+
+      def df_as_array_of_hashes
+        df.to_a[0]
+      end
+
+    end
+  end
+end