remi 0.2.42 → 0.3.0

data/lib/remi/data_subject/sftp_file.rb

@@ -1,65 +0,0 @@
-module Remi
-
-  class DataTarget::SftpFile < DataTarget
-
-    def initialize(*args, **kargs, &block)
-      super
-      init_sftp_file(*args, **kargs, &block)
-    end
-
-    attr_reader :local_path
-    attr_reader :remote_path
-
-    # Public: Performs the load operation, regardless of whether it has
-    # already executed.
-    #
-    # Returns true if the load operation was successful
-    def load!
-      @logger.info "Uploading #{@local_path} to #{@credentials[:username]}@#{@credentials[:host]}: #{@remote_path}"
-      connection do |sftp|
-        retry_upload { sftp.upload! @local_path, @remote_path }
-      end
-
-      true
-    end
-
-
-    private
-
-    def init_sftp_file(*args, credentials:, local_path:, remote_path: File.basename(local_path), **kargs, &block)
-      @credentials = credentials
-      @local_path = local_path
-      @remote_path = remote_path
-      init_df
-    end
-
-    def init_df
-      parameter_df = Daru::DataFrame.new(
-        local_path: Array(@local_path),
-        remote_path: Array(@remote_path)
-      )
-      self.df = parameter_df
-    end
-
-    def connection(&block)
-      result = nil
-      Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @credentials[:port] || '22') do |sftp|
-        result = yield sftp
-      end
-      result
-    end
-
-    def retry_upload(ntry=2, &block)
-      1.upto(ntry).each do |itry|
-        begin
-          block.call
-        rescue RuntimeError => err
-          raise err unless itry < ntry
-          @logger.error "Upload failed with error: #{err.message}"
-          @logger.error "Retry attempt #{itry}/#{ntry-1}"
-          sleep(1)
-        end
-      end
-    end
-  end
-end

data/lib/remi/extractor/file_system.rb

@@ -1,92 +0,0 @@
-module Remi
-  module Extractor
-
-    class FileSystemEntry
-      def initialize(pathname:, create_time:, modified_time:, raw: nil)
-        @pathname = Pathname.new(pathname)
-        @create_time = create_time
-        @modified_time = modified_time
-        @raw = raw
-      end
-
-      attr_reader :pathname, :create_time, :modified_time, :raw
-
-      def name
-        @pathname.basename.to_s
-      end
-    end
-
-
-    class FileSystem
-      class FileNotFoundError < StandardError; end
-
-      def initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block)
-        @remote_path = Pathname.new(remote_path)
-        @pattern = pattern
-        @local_path = Pathname.new(local_path)
-        @most_recent_only = most_recent_only
-        @group_by = group_by
-        @most_recent_by = most_recent_by
-        @logger = logger
-      end
-
-      attr_reader :remote_path
-      attr_reader :pattern
-      attr_reader :local_path
-      attr_reader :most_recent_only
-      attr_reader :group_by
-      attr_reader :most_recent_by
-      attr_reader :logger
-
-      # Public: Called to extract files from the source filesystem.
-      #
-      # Returns an array with containing the paths to all files extracted.
-      def extract
-        raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
-      end
-
-      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
-      # NOTE: all_entries is responsible for matching the path using @remote_path
-      def all_entries
-        raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
-      end
-
-      # Public: Returns just the entries that are to be extracted.
-      def entries
-        if @group_by
-          most_recent_matching_entry_in_group
-        elsif @most_recent_only
-          Array(most_recent_matching_entry)
-        else
-          matching_entries
-        end
-      end
-
-      def matching_entries
-        all_entries.select { |e| @pattern.match e.name }
-      end
-
-      def most_recent_matching_entry
-        matching_entries.sort_by { |e| e.send(@most_recent_by) }.reverse.first
-      end
-
-      def most_recent_matching_entry_in_group
-        entries_with_group = matching_entries.map do |entry|
-          match = entry.name.match(@group_by)
-          next unless match
-
-          group = match.to_a[1..-1]
-          { group: group, entry: entry }
-        end.compact
-        sorted_entries_with_group = entries_with_group.sort_by { |e| [e[:group], e[:entry].send(@most_recent_by)] }.reverse
-
-        last_group = nil
-        sorted_entries_with_group.map do |entry|
-          next unless entry[:group] != last_group
-          last_group = entry[:group]
-          entry[:entry]
-        end.compact
-      end
-    end
-  end
-end

data/lib/remi/extractor/local_file.rb

@@ -1,43 +0,0 @@
-module Remi
-  module Extractor
-
-    class LocalFile < FileSystem
-      def initialize(*args, **kargs)
-        super
-        init_local_file(*args, **kargs)
-      end
-
-      # Public: Called to extract files from the source filesystem.
-      #
-      # Returns an array with containing the paths to all files extracted.
-      def extract
-        entries.map(&:pathname)
-      end
-
-      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
-      def all_entries
-        @all_entries ||= all_entries!
-      end
-
-      def all_entries!
-        dir = @remote_path.directory? ? @remote_path + '*' : @remote_path
-        Dir[dir].map do |entry|
-          path = Pathname.new(entry)
-          if path.file?
-            FileSystemEntry.new(
-              pathname: path.realpath.to_s,
-              create_time: path.ctime,
-              modified_time: path.mtime
-            )
-          end
-        end.compact
-      end
-
-      private
-
-      def init_local_file(*args, **kargs)
-      end
-
-    end
-  end
-end

data/lib/remi/extractor/s3_file.rb

@@ -1,57 +0,0 @@
-module Remi
-  module Extractor
-
-    class S3File < FileSystem
-
-      def initialize(*args, **kargs, &block)
-        super
-        init_s3_file(*args, **kargs, &block)
-      end
-
-      # Public: Called to extract files from the source filesystem.
-      #
-      # Returns an array with containing the paths to all files extracted.
-      def extract
-        entries.map do |entry|
-          local_file = File.join(@local_path, entry.name)
-          @logger.info "Downloading #{entry.pathname} from S3 to #{local_file}"
-          File.open(local_file, 'wb') { |file| entry.raw.get(response_target: file) }
-          local_file
-        end
-      end
-
-      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
-      def all_entries
-        @all_entries ||= all_entries!
-      end
-
-      def all_entries!
-        # S3 does not track anything like a create time, so use last modified for both
-        bucket.objects(prefix: @remote_path.to_s).map do |entry|
-          FileSystemEntry.new(
-            pathname: entry.key,
-            create_time: entry.last_modified,
-            modified_time: entry.last_modified,
-            raw: entry
-          )
-        end
-      end
-
-      def s3_client
-        @s3_client ||= Aws::S3::Client.new
-      end
-
-      private
-
-      def init_s3_file(*args, bucket:, **kargs)
-        @bucket_name = bucket
-      end
-
-      def bucket
-        @bucket ||= Aws::S3::Bucket.new(@bucket_name, client: s3_client)
-      end
-
-    end
-
-  end
-end

data/lib/remi/extractor/sftp_file.rb

@@ -1,83 +0,0 @@
-module Remi
-  module Extractor
-
-    class SftpFile < FileSystem
-
-      N_RETRY = 3
-
-      def initialize(*args, **kargs)
-        super
-        init_sftp_file(*args, **kargs)
-      end
-
-      attr_reader :host
-      attr_reader :username
-      attr_reader :password
-      attr_reader :port
-
-      # Public: Called to extract files from the source filesystem.
-      #
-      # Returns an array with containing the paths to all files extracted.
-      def extract
-        connection do |sftp|
-          entries.map do |entry|
-            local_file = File.join(@local_path, entry.name)
-            @logger.info "Downloading #{entry.name} to #{local_file}"
-            retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
-            local_file
-          end
-        end
-      end
-
-      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
-      def all_entries
-        @all_entries ||= all_entries!
-      end
-
-      def all_entries!
-        sftp_entries = connection { |sftp| sftp.dir.entries(@remote_path) }
-        sftp_entries.map do |entry|
-          # Early versions of the protocol don't support create time, fake it with modified time?
-          FileSystemEntry.new(
-            pathname: File.join(@remote_path, entry.name),
-            create_time: entry.attributes.respond_to?(:createtime) ? entry.attributes.createtime : entry.attributes.mtime,
-            modified_time: entry.attributes.mtime
-          )
-        end
-      end
-
-
-      private
-
-      def init_sftp_file(*args, credentials:, **kargs)
-        @host     = credentials.fetch(:host)
-        @username = credentials.fetch(:username)
-        @password = credentials.fetch(:password)
-        @port     = credentials.fetch(:port, '22')
-      end
-
-      def connection(&block)
-        result = nil
-        Net::SFTP.start(@host, @username, password: @password, port: @port) do |sftp|
-          result = yield sftp
-        end
-        result
-      end
-
-      def retry_download(&block)
-        1.upto(N_RETRY).each do |itry|
-          begin
-            block.call
-            break
-          rescue RuntimeError => err
-            raise err unless itry < N_RETRY
-            @logger.error "Download failed with error: #{err.message}"
-            @logger.error "Retry attempt #{itry}/#{N_RETRY-1}"
-            sleep(1)
-          end
-        end
-      end
-    end
-
-  end
-end

data/spec/data_subject/csv_file_spec.rb

@@ -1,79 +0,0 @@
-require_relative '../remi_spec'
-
-describe DataSource::CsvFile do
-
-  it "converts a CSV into a dataframe" do
-    csv = Remi::DataSource::CsvFile.new(
-      extractor: 'spec/fixtures/basic.csv'
-    )
-
-    expected_df = Remi::DataFrame::Daru.new(
-      {
-        column_a: ['value 1A', 'value 2A'],
-        column_b: ['value 1B', 'value 2B']
-      }
-    )
-    expect(csv.df.to_a).to eq expected_df.to_a
-  end
-
-  it "adds filename when requested" do
-    csv = Remi::DataSource::CsvFile.new(
-      extractor: 'spec/fixtures/basic.csv',
-      filename_field: :from_file
-    )
-
-    expected_files = [Pathname.new('spec/fixtures/basic.csv').realpath.to_s] * 2
-    expect(csv.df[:from_file].to_a).to eq expected_files
-  end
-
-  it "preprocesses records when required" do
-    csv = Remi::DataSource::CsvFile.new(
-      extractor: 'spec/fixtures/unsupported_escape.csv',
-      preprocessor: ->(line) { line.gsub(/\\"/,'""') }
-    )
-
-    expected_df = Remi::DataFrame::Daru.new(
-      {
-        column_a: ['value 1A', 'value 2A'],
-        column_b: ['value "1B"', 'value "2B"']
-      }
-    )
-    expect(csv.df.to_a).to eq expected_df.to_a
-  end
-
-  it "accepts standard Ruby CSV options" do
-    csv = Remi::DataSource::CsvFile.new(
-      extractor: 'spec/fixtures/basic.csv',
-      preprocessor: ->(line) { line.gsub(/,/,'|') },
-      csv_options: { col_sep: '|' }
-    )
-
-    expected_df = Remi::DataFrame::Daru.new(
-      {
-        column_a: ['value 1A', 'value 2A'],
-        column_b: ['value 1B', 'value 2B']
-      }
-    )
-    expect(csv.df.to_a).to eq expected_df.to_a
-  end
-
-  it "combines multiple csv files into a single dataframe" do
-    csv = Remi::DataSource::CsvFile.new(
-      extractor: Remi::Extractor::LocalFile.new(
-        remote_path: 'spec/fixtures',
-        pattern: /basic(|2)\.csv/
-      )
-    )
-
-    expected_df = Remi::DataFrame::Daru.new(
-      {
-        column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
-        column_b: ['value 1B', 'value 2B', nil, nil],
-        column_c: [nil, nil, 'value 1C', 'value 2C']
-      }
-    )
-
-    expect(csv.df.to_a).to eq expected_df.to_a
-  end
-
-end

data/spec/data_subject/data_frame.rb

@@ -1,27 +0,0 @@
-require_relative '../remi_spec'
-
-describe DataSource::DataFrame do
-  it "converts data into a dataframe" do
-    source_dataframe = Remi::DataSource::DataFrame.new(
-      fields: {
-        :col1 => {},
-        :col2 => {}
-      },
-      data: [
-        ['11', '12'],
-        ['21', '22'],
-        ['31', '32']
-      ]
-    )
-
-    expected_df = Remi::DataFrame::Daru.new(
-      {
-        col1: ['11', '21', '31'],
-        col2: ['12', '22', '32']
-      }
-    )
-
-    expect(source_dataframe.df).to be_a Remi::DataFrame
-    expect(source_dataframe.df.to_a).to eq expected_df.to_a
-  end
-end