remi 0.2.27 → 0.2.28

data/lib/remi/data_target.rb

@@ -1,15 +0,0 @@
-module Remi
-  module DataTarget
-    include DataSubject
-
-    # Gets called automatically at the end of a job, but could
-    # also get manually called at the end of a transform so make
-    # sure it doesn't do it twice.
-    def load
-      @logger.info "Loading target"
-      return true if @loaded
-      @loaded = true
-      raise "Load function undefined for #{self.class.name}"
-    end
-  end
-end

data/lib/remi/data_target/csv_file.rb

@@ -1,42 +0,0 @@
-module Remi
-  module DataTarget
-    class CsvFile
-      include DataTarget
-
-      def self.default_csv_options
-        CSV::DEFAULT_OPTIONS.merge({
-          headers: true,
-          header_converters: Remi::FieldSymbolizers[:standard],
-          col_sep: ',',
-          encoding: 'UTF-8',
-          quote_char: '"',
-          row_sep: "\n"
-        })
-      end
-
-      def initialize(path:, csv_options: {}, logger: Remi::Settings.logger)
-        @path = path
-        @csv_options = self.class.default_csv_options.merge(csv_options)
-        @logger = logger
-      end
-
-      attr_reader   :path
-      attr_reader   :csv_options
-
-      def field_symbolizer
-        self.class.default_csv_options[:header_converters]
-      end
-
-      def load
-        return true if @loaded || df.size == 0
-
-        @logger.info "Writing CSV file #{@path}"
-
-        df.write_csv @path, @csv_options
-
-        @loaded = true
-      end
-
-    end
-  end
-end

data/lib/remi/data_target/data_frame.rb

@@ -1,14 +0,0 @@
-module Remi
-  module DataTarget
-    class DataFrame
-      include DataSubject
-
-      def initialize(**args)
-      end
-
-      def load
-        true
-      end
-    end
-  end
-end

data/lib/remi/data_target/postgres.rb

@@ -1,74 +0,0 @@
-module Remi
-  module DataTarget
-    class Postgres
-      include DataTarget
-
-      def initialize(credentials:, table_name:, fields:, logger: Remi::Settings.logger)
-        @credentials = credentials
-        @table_name = table_name
-        @fields = fields
-        @logger = logger
-      end
-
-      def load
-        return true if @loaded || df.size == 0
-
-        @logger.info "Performing postgres load to table #{@table_name}"
-        create_target_table
-        load_target_table
-
-        @loaded = true
-      end
-
-
-      def connection
-        @connection ||= PG.connect(
-          host:     @credentials[:host] || 'localhost',
-          port:     @credentials[:port] || 5432,
-          dbname:   @credentials[:dbname],
-          user:     @credentials[:user] || `whoami`.chomp,
-          password: @credentials[:password],
-          sslmode:  @credentials[:sslmode] || 'allow'
-        )
-      end
-
-
-      def fields_with_type_ddl
-        @fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
-      end
-
-      def create_target_table
-        connection.exec <<-EOT
-          CREATE TEMPORARY TABLE #{@table_name} (
-            #{fields_with_type_ddl}
-          )
-        EOT
-      end
-
-      def load_target_table
-        connection.copy_data "COPY #{@table_name} (#{@fields.keys.join(', ')}) FROM STDIN" do
-          df.each(:row) do |row|
-            row_str = @fields.keys.map do |field|
-              field = row[field]
-              case
-              when field.respond_to?(:strftime)
-                field.strftime('%Y-%m-%d %H:%M:%S')
-              when field.respond_to?(:map)
-                field.to_json.gsub("\t", '\t')
-              when field.blank? && !field.nil?
-                ''
-              when field.nil?
-                '\N'
-              else
-                field.to_s.gsub("\t", '\t')
-              end
-            end.join("\t")
-
-            connection.put_copy_data row_str + "\n"
-          end
-        end
-      end
-
-    end
-  end
-end

data/lib/remi/data_target/salesforce.rb

@@ -1,54 +0,0 @@
-module Remi
-  module DataTarget
-    class Salesforce
-      include DataTarget
-
-      def initialize(object:, operation:, credentials:, external_id: 'Id', api: :bulk, logger: Remi::Settings.logger)
-        @sfo = object
-        @operation = operation
-        @external_id = external_id
-        @credentials = credentials
-        @api = api
-        @logger = logger
-      end
-
-      def field_symbolizer
-        Remi::FieldSymbolizers[:salesforce]
-      end
-
-      def load
-        return true if @loaded || df.size == 0
-
-        @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
-
-        if @operation == :update
-          Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
-        elsif @operation == :create
-          Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
-        elsif @operation == :upsert
-          Remi::SfBulkHelper::SfBulkUpsert.upsert(restforce_client, @sfo, df_as_array_of_hashes, external_id: @external_id, logger: @logger)
-        else
-          raise "Unknown operation: #{@operation}"
-        end
-
-        @loaded = true
-      end
-
-      def restforce_client
-        @restforce_client ||= begin
-          client = Restforce.new(@credentials)
-
-          #run a dummy query to initiate a connection. Workaround for Bulk API problem
-          # https://github.com/yatish27/salesforce_bulk_api/issues/33
-          client.query('SELECT Id FROM Contact LIMIT 1')
-          client
-        end
-      end
-
-      def df_as_array_of_hashes
-        df.to_a[0]
-      end
-
-    end
-  end
-end

data/lib/remi/data_target/sftp_file.rb

@@ -1,54 +0,0 @@
-module Remi
-  module DataTarget
-    class SftpFile
-      include DataTarget
-
-      def initialize(credentials:, local_path:, remote_path: File.basename(local_path), logger: Remi::Settings.logger)
-        @credentials = credentials
-        @local_path = local_path
-        @remote_path = remote_path
-        @logger = logger
-      end
-
-      attr_reader :local_path
-      attr_reader :remote_path
-
-      def load
-        return true if @loaded
-
-        connection do |sftp|
-          retry_upload { sftp.upload! @local_path, @remote_path }
-        end
-
-        @loaded = true
-      end
-
-
-
-      private
-
-      def connection(&block)
-        result = nil
-        Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @credentials[:port] || '22') do |sftp|
-          result = yield sftp
-        end
-        result
-      end
-
-      def retry_upload(ntry=2, &block)
-        1.upto(ntry).each do |itry|
-          begin
-            block.call
-          rescue RuntimeError => err
-            raise err unless itry < ntry
-            @logger.error "Upload failed with error: #{err.message}"
-            @logger.error "Retry attempt #{itry}/#{ntry-1}"
-            sleep(1)
-          end
-        end
-      end
-
-
-    end
-  end
-end

data/lib/remi/refinements/daru.rb

@@ -1,85 +0,0 @@
-module Remi
-  module Refinements
-    module Daru
-      refine ::Daru::DataFrame do
-
-        # Public: Fixes dup issues in the Daru library (vectors not being duped).
-        def dup
-          dupdf = ::Daru::DataFrame.new([], index: self.index)
-          self.vectors.each do |v|
-            dupdf[v] = self[v]
-          end
-
-          dupdf
-        end
-
-        # Public: Allows for combining dataframes with different columns
-        def concat other_df
-          vectors = @vectors.to_a
-          data = []
-
-          vectors.each do |v|
-            other_vec = other_df.vectors.include?(v) ? other_df[v].to_a : [nil] * other_df.size
-            data << self[v].dup.to_a.concat(other_vec)
-          end
-
-          other_df.vectors.each do |v|
-            next if vectors.include?(v)
-            vectors << v
-            data << ([nil] * self.size).concat(other_df[v].to_a)
-          end
-
-          ::Daru::DataFrame.new(data, order: vectors)
-        end
-
-        # Public: Saves a Dataframe to a file.
-        def hash_dump(filename)
-          File.binwrite(filename, Marshal.dump(self.to_hash))
-        end
-
-        # Public: Allows the user to define an arbitrary aggregation function.
-        #
-        # by   - The name of the DataFrame vector to use to group records.
-        # func - A lambda function that accepts three arguments - the
-        #        first argument is the DataFrame, the second is the
-        #        key to the current group, and the third is the index
-        #        of the elements belonging to a group.
-        #
-        # Example:
-        #   df = Daru::DataFrame.new( { a: ['a','a','a','b','b'], year: ['2018','2015','2019', '2014', '2013'] })
-        #
-        #   mymin = lambda do |field, df, group_key, indices|
-        #     values = indices.map { |idx| df.row[idx][field] }
-        #     "Group #{group_key} has a minimum value of #{values.min}"
-        #   end
-        #
-        #   df.aggregate(by: :a, func: mymin.curry.(:year))
-        #
-        #
-        # Returns a Daru::Vector.
-        def aggregate(by:, func:)
-          grouped = self.group_by(by)
-          df_indices = self.index.to_a
-          ::Daru::Vector.new(
-            grouped.groups.reduce({}) do |h, (key, indices)|
-              # Daru groups don't use the index of the dataframe when returning groups (WTF?).
-              # Instead they return the position of the record in the dataframe.  Here, we
-              group_df_indices = indices.map { |v| df_indices[v] }
-              group_key = key.size == 1 ? key.first : key
-              h[group_key] = func.(self, group_key, group_df_indices)
-              h
-            end
-          )
-        end
-
-      end
-
-      refine ::Daru::DataFrame.singleton_class do
-        # Public: Creates a DataFrame by reading the dumped version from a file.
-        def from_hash_dump(filename)
-          ::Daru::DataFrame.new(Marshal.load(File.binread(filename)))
-        end
-      end
-    end
-  end
-end