remi 0.2.35 → 0.2.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ac6de18abdeb2269c391a732e283e54c995344f
-  data.tar.gz: 51f64d79e4b176bb4f2dbd15a4407fe0823aa13f
+  metadata.gz: 0b43c0f454a9c3df185534347b76d5d7ac0a37c4
+  data.tar.gz: 024381d3a8b2da98b1de66fddd14c0929a304d3d
 SHA512:
-  metadata.gz: 9978c92842fc114224f06c72aacaca3175f9b6467b2189fcce5d8f60f26335da324f4f5daf4ae770f33b266ba5f1757f032dd0de7c3ea25f926cdd8f64b20c50
-  data.tar.gz: 8390273e8305709141f4823d837cfcbf7b0fcf891ad2f9cdecdb418bdee5be11cc25da8e3daa9afa5ad14ec42e0a813b23ba2d220133dbce1e58ea82c551f99e
+  metadata.gz: bc680e8048b300f645c013a3f261506b6c4e1efdef0f240f37eebb0d393f2cc948de06d53d1d3912122011ba8d2755ac8e851c6399af79c13d68b467389a1118
+  data.tar.gz: cb5f72b9db98cdf7b79fcaee6a1597e9259b805779b93655ba0738b5e8a14240a0e57cc6078b92cdc2c414f21eb3af034eb78a185e974f3c28707667ea80f792

data/Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.35)
+    remi (0.2.36)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/sample_job.feature

@@ -33,10 +33,6 @@ Feature: This is a sample feature file.
     And files with names matching the pattern /^SampleFile_(\d+)\.txt/
     Then the file with the latest date stamp will be downloaded for processing
 
-    Given files with names that do not match the pattern /^SampleFile_(\d+)\.txt/
-    Then no files will be downloaded for processing
-
-
   Scenario: In order to be parsed and properly processed, the file must conform
     to expectations about its structure and content.
 

data/features/step_definitions/remi_step.rb

@@ -44,34 +44,23 @@ end
 ### Source file processing
 
 Given /^files with names matching the pattern \/(.*)\/$/ do |pattern|
-  @brt.filestore.pattern(Regexp.new(pattern))
+  expect(@brt.source.data_subject.extractor.pattern).to eq Regexp.new(pattern)
 end
 
-Given /^files with names that do not match the pattern \/(.*)\/$/ do |pattern|
-  @brt.filestore.anti_pattern(Regexp.new(pattern))
-end
-
-Given /^files delivered within the last (\d+) hours$/ do |hours|
-  @brt.filestore.delivered_since(Time.now - hours.to_i * 3600)
-end
-
-Given /^files were delivered more than (\d+) hours ago$/ do |hours|
-  @brt.filestore.delivered_before(Time.now - hours.to_i * 3600)
+Given /^download groups defined by the pattern \/(.*)\/$/ do |pattern|
+  expect(@brt.source.data_subject.extractor.group_by).to eq Regexp.new(pattern)
 end
 
 Then /^the file with the latest date stamp will be downloaded for processing$/ do
-  @brt.filestore.generate
-  @brt.source.mock_extractor(@brt.filestore)
-  expect(@brt.source.extract).to match_array Array(@brt.filestore.latest)
+  expect(@brt.source.data_subject.extractor.most_recent_by).to eq :create_time
 end
 
-Then /^files will be downloaded for processing$/ do
+Then /^all files matching the pattern will be downloaded for processing$/ do
+  expect(@brt.source.data_subject.extractor.most_recent_only).to eq false
 end
 
-Then /^no files will be downloaded for processing$/ do
-  @brt.filestore.generate
-  @brt.source.mock_extractor(@brt.filestore)
-  expect { @brt.source.extract }.to raise_error Remi::Extractor::SftpFile::FileNotFoundError
+Then /^the file that comes last in an alphanumeric sort by group will be downloaded for processing$/ do
+  expect(@brt.source.data_subject.extractor.most_recent_by).to eq :name
 end
 
 Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
@@ -150,6 +139,19 @@ Given /^the source field (?:has|is set to) the value "([^"]*)"$/ do |value|
   end
 end
 
+Given /^the source field '([^']+)' (?:has|is set to) the json value$/ do |source_field, value|
+  step "the source field '#{source_field}'"
+
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+  @brt.sources[source_name].fields[source_field_name].value = JSON.parse(value)
+end
+
+Given /^the source field (?:has|is set to) the json value$/ do |value|
+  @brt.sources.fields.each do |field|
+    step "the source field '#{field.full_name}' is set to the json value \"#{value}\""
+  end
+end
+
 When /^the source field '([^']+)' (?:has an empty value|is blank)$/ do |source_field|
   step "the source field '#{source_field}'"
 

data/features/support/env.rb

@@ -6,11 +6,20 @@ require 'remi'
 require 'remi/cucumber'
 
 Remi::Settings.log_level = Logger::ERROR
+Remi::Settings.jobs_dir = File.join(__dir__, '../../jobs')
 
 Before do
   # Restart the random number generator prior to each scenario to
   # ensure we have reproducibility of random output
   Kernel.srand(35983958269835333)
+
+  # Monkey patch faker gem so that dummy random dates and ranges are generated consistently
+  class Faker::Base
+    def self.rand_in_range(from, to)
+      from, to = to, from if to < from
+      Random.rand(from..to)
+    end
+  end
 end
 
 After do

data/jobs/sample_job.rb

@@ -27,8 +27,8 @@ class SampleJob
   define_source :sample_file, Remi::DataSource::CsvFile,
     extractor: Remi::Extractor::SftpFile.new(
       credentials: params[:sftp],
-      remote_file: /^SampleFile_(\d+)\.txt/,
-      remote_folder: '/',
+      remote_path: '/',
+      pattern: /^SampleFile_(\d+)\.txt/,
       most_recent_only: true
     ),
     csv_options: {

data/lib/remi.rb

@@ -42,8 +42,9 @@ require 'remi/field_symbolizers'
 
 require 'remi/refinements/symbolizer'
 
-require 'remi/extractor/sftp_file' # deprecated
 require 'remi/extractor/file_system'
+require 'remi/extractor/local_file'
+require 'remi/extractor/sftp_file'
 require 'remi/extractor/s3_file'
 
 

data/lib/remi/cucumber/business_rules.rb

@@ -113,6 +113,7 @@ module Remi::BusinessRules
 
     def initialize(job_name)
       job_class_name = "#{job_name.gsub(/\s/,'')}Job"
+      require_job_file(job_class_name)
       @job = Object.const_get(job_class_name).new
 
       @job_sources = DataSubjectCollection.new
@@ -121,8 +122,6 @@ module Remi::BusinessRules
       @sources = DataSubjectCollection.new
       @targets = DataSubjectCollection.new
       @examples = DataExampleCollection.new
-
-      @filestore = Filestore.new
     end
 
     attr_reader :job
@@ -131,7 +130,13 @@ module Remi::BusinessRules
     attr_reader :sources
     attr_reader :targets
     attr_reader :examples
-    attr_reader :filestore
+
+    def require_job_file(job_class_name)
+      job_file = Dir["#{Remi::Settings.jobs_dir}/**/*_job.rb"].map do |fname|
+        fname if File.basename(fname) == "#{job_class_name.underscore}.rb"
+      end.compact.pop
+      require job_file
+    end
 
     def add_job_source(name)
       raise "Unknown source #{name} for job" unless @job.methods.include? name.symbolize
@@ -396,17 +401,6 @@ module Remi::BusinessRules
       @data_subject.df[vector_name].recode! { |v| i += 1 }
     end
 
-    def mock_extractor(filestore)
-      extractor = class << @data_subject.extractor; self; end
-
-      extractor.send(:define_method, :all_entries, ->() { filestore.sftp_entries })
-      extractor.send(:define_method, :download, ->(to_download) { to_download.map { |e| e.name } })
-    end
-
-    def extract
-      @data_subject.extractor.extract
-    end
-
     def csv_options
       @data_subject.csv_options
     end
@@ -558,75 +552,4 @@ module Remi::BusinessRules
       end
     end
   end
-
-
-  class Filestore
-    def initialize
-      @files = []
-      @delivered = {}
-    end
-
-    attr_reader :sftp_entries
-
-    def pattern(pattern)
-      @pattern = pattern
-    end
-
-    def anti_pattern(pattern)
-      @pattern = /^ThisBetterNeverMatchAnythingOrIWillShootYou\d{8}Times$/
-    end
-
-    def delivered_since(date_time)
-      @delivered = { :since => date_time }
-    end
-
-    def delivered_before(date_time)
-      @delivered = { :before => date_time }
-    end
-
-    def latest
-      @files.max_by { |f| f[:attributes][:createdtime] }[:name]
-    end
-
-    def generate
-      psuedorand = Random.new(4985674985672348954987589429)
-
-      generate_files_with_pattern
-      @files.map! do |file|
-        date_method = @delivered.keys.first
-        if date_method == :since
-          file[:attributes][:createdtime] = @delivered[:since] + 10 + psuedorand.rand * 100
-        elsif date_method == :before
-          file[:attributes][:createdtime] = @delivered[:since] - 10 - psuedorand.rand * 100
-        else
-          file[:attributes][:createdtime] = Time.now - 10 - psuedorand.rand * 100
-        end
-        file
-      end
-    end
-
-    def sftp_entries
-      @files.map do |file|
-        Net::SFTP::Protocol::V04::Name.new(
-          file[:name],
-          Net::SFTP::Protocol::V04::Attributes.new(createtime: file[:attributes][:createdtime])
-        )
-      end
-    end
-
-    private
-
-    def generate_files_with_pattern
-      filenames = 1.upto(5).map { |f| @pattern.random_example }.uniq
-
-      @files = filenames.map do |fname|
-        {
-          name: fname,
-          attributes: {
-            createdtime: nil
-          }
-        }
-      end
-    end
-  end
 end

data/lib/remi/data_subject/csv_file.rb

@@ -53,6 +53,8 @@ module Remi
       # Assumes that each file has exactly the same structure
       result_df = nil
       extract.each_with_index do |filename, idx|
+        filename = filename.to_s
+
         @logger.info "Converting #{filename} to a dataframe"
         processed_filename = preprocess(filename)
         csv_df = Daru::DataFrame.from_csv processed_filename, @csv_options
@@ -71,16 +73,7 @@ module Remi
 
 
     def extractor=(arg)
-      case arg
-      when Extractor::SftpFile, Extractor::LocalFile, Extractor::S3File
-        @extractor = arg
-      when String
-        @extractor = Extractor::LocalFile.new(path: arg)
-      when Regexp
-        raise "Adding regex matching to local files would be easy, not done yet"
-      else
-        raise "Unknown extractor of type #{arg.class}: #{arg}"
-      end
+      @extractor = arg.respond_to?(:extract) ? arg : Extractor::LocalFile.new(remote_path: arg.to_s)
     end
 
     # Only going to support single file for now

data/lib/remi/extractor/file_system.rb

@@ -18,7 +18,6 @@ module Remi
 
 
     class FileSystem
-
       class FileNotFoundError < StandardError; end
 
       def initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block)
@@ -31,6 +30,12 @@ module Remi
         @logger = logger
       end
 
+      attr_reader :remote_path
+      attr_reader :pattern
+      attr_reader :local_path
+      attr_reader :most_recent_only
+      attr_reader :group_by
+      attr_reader :most_recent_by
       attr_reader :logger
 
       # Public: Called to extract files from the source filesystem.

data/lib/remi/extractor/local_file.rb

@@ -0,0 +1,43 @@
+module Remi
+  module Extractor
+
+    class LocalFile < FileSystem
+      def initialize(*args, **kargs)
+        super
+        init_local_file(*args, **kargs)
+      end
+
+      # Public: Called to extract files from the source filesystem.
+      #
+      # Returns an array with containing the paths to all files extracted.
+      def extract
+        entries.map(&:pathname)
+      end
+
+      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
+      def all_entries
+        @all_entries ||= all_entries!
+      end
+
+      def all_entries!
+        dir = @remote_path.directory? ? @remote_path + '*' : @remote_path
+        Dir[dir].map do |entry|
+          path = Pathname.new(entry)
+          if path.file?
+            FileSystemEntry.new(
+              pathname: path.realpath.to_s,
+              create_time: path.ctime,
+              modified_time: path.mtime
+            )
+          end
+        end.compact
+      end
+
+      private
+
+      def init_local_file(*args, **kargs)
+      end
+
+    end
+  end
+end

data/lib/remi/extractor/sftp_file.rb

@@ -1,125 +1,71 @@
 module Remi
   module Extractor
 
-    class LocalFile
-      def initialize(path:, folder: nil)
-        @path = path
-        @folder = folder
-      end
-
-      def extract
-        if @folder
-          Dir.entries(@folder).map do |entry|
-            next unless entry.match(@path)
-            File.join(@folder,entry)
-          end.compact
-        else
-          @path
-        end
-      end
-    end
-
-    class SftpFile
-
-      class FileNotFoundError < StandardError; end
+    class SftpFile < FileSystem
 
-      SortDesc = Struct.new(:value) do
-        def <=> (target)
-          -(self.value <=> target.value)
-        end
-      end
+      N_RETRY = 3
 
-      def initialize(credentials:, remote_file:, remote_folder: '', local_folder: Settings.work_dir, port: nil, most_recent_only: false, group_by: nil, most_recent_by: :createtime, logger: Remi::Settings.logger)
-        @credentials = credentials
-        @remote_file = remote_file
-        @remote_folder = remote_folder
-        @local_folder = local_folder
-        @port = port || (credentials && credentials[:port]) || '22'
-        @most_recent_only = most_recent_only
-        @group_by = group_by
-        @most_recent_by = most_recent_by
-        @logger = logger
+      def initialize(*args, **kargs)
+        super
+        init_sftp_file(*args, **kargs)
       end
 
-      attr_reader :logger
+      attr_reader :host
+      attr_reader :username
+      attr_reader :password
+      attr_reader :port
 
+      # Public: Called to extract files from the source filesystem.
+      #
+      # Returns an array with containing the paths to all files extracted.
       def extract
-        raise FileNotFoundError, "File not found: #{@remote_file}" if to_download.size == 0
-        download(to_download)
-      end
-
-      def to_download
-        if @group_by
-          most_recent_in_group
-        elsif @most_recent_only
-          Array(most_recent_entry(matching_entries))
-        else
-          matching_entries
+        connection do |sftp|
+          entries.map do |entry|
+            local_file = File.join(@local_path, entry.name)
+            @logger.info "Downloading #{entry.name} to #{local_file}"
+            retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
+            local_file
+          end
         end
       end
 
-      def all_entries(remote_folder = @remote_folder)
-        @all_entries ||= connection { |sftp| sftp.dir.entries(File.join("/", remote_folder)) }
+      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
+      def all_entries
+        @all_entries ||= all_entries!
       end
 
-      def matching_entries(match_name = @remote_file)
-        all_entries.select { |e| match_name.match e.name }
-      end
-
-      def most_recent_entry(entries = matching_entries)
-        entries.sort_by { |e| sort_files_by(e) }.reverse!.first
-      end
-
-      def sort_files_by(entry)
-        if @most_recent_by == :filename
-          entry.name
-        else
-          entry.attributes.send(@most_recent_by)
+      def all_entries!
+        sftp_entries = connection { |sftp| sftp.dir.entries(@remote_path) }
+        sftp_entries.map do |entry|
+          # Early versions of the protocol don't support create time, fake it with modified time?
+          FileSystemEntry.new(
+            pathname: File.join(@remote_path, entry.name),
+            create_time: entry.attributes.respond_to?(:createtime) ? entry.attributes.createtime : entry.attributes.mtime,
+            modified_time: entry.attributes.mtime
+          )
         end
       end
 
-      def most_recent_in_group(match_group = @group_by)
-        entries_with_group = matching_entries.map do |entry|
-          match = entry.name.match(match_group)
-          next unless match
 
-          group = match.to_a[1..-1]
-          { group: group, entry: entry }
-        end.compact
-        entries_with_group.sort_by! { |e| [e[:group], SortDesc.new(sort_files_by(e[:entry]))] }
-
-        last_group = nil
-        entries_with_group.map do |entry|
-          next unless entry[:group] != last_group
-          last_group = entry[:group]
-          entry[:entry]
-        end.compact
-      end
+      private
 
-      def download(entries_to_download, remote_folder: @remote_folder, local_folder: @local_folder, ntry: 3)
-        connection do |sftp|
-          entries_to_download.map do |entry|
-            local_file = File.join(local_folder, entry.name)
-            @logger.info "Downloading #{entry.name} to #{local_file}"
-            retry_download(ntry) { sftp.download!(File.join(remote_folder, entry.name), local_file) }
-            local_file
-          end
-        end
+      def init_sftp_file(*args, credentials:, **kargs)
+        @host     = credentials.fetch(:host)
+        @username = credentials.fetch(:username)
+        @password = credentials.fetch(:password)
+        @port     = credentials.fetch(:port, '22')
       end
 
-
-      private
-
       def connection(&block)
         result = nil
-        Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @port) do |sftp|
+        Net::SFTP.start(@host, @username, password: @password, port: @port) do |sftp|
           result = yield sftp
         end
         result
       end
 
-      def retry_download(ntry=2, &block)
-        1.upto(ntry).each do |itry|
+      def retry_download(&block)
+        1.upto(N_RETRY).each do |itry|
           begin
             block.call
             break
@@ -132,5 +78,6 @@ module Remi
         end
       end
     end
+
   end
 end

data/lib/remi/fields.rb

@@ -4,5 +4,26 @@ module Remi
       @fields = Hash.new({}).merge fields
       super(@fields)
     end
+
+
+    def dup
+      Fields.new(@fields.dup)
+    end
+
+    def merge(other_fields, prefix: nil)
+      dup.merge!(other_fields, prefix: prefix)
+    end
+
+    def merge!(other_fields, prefix: nil)
+      @fields.merge!(other_fields) do |key, this_val, other_val|
+        if prefix
+          @fields["#{prefix}#{key}".to_sym] = other_val
+          this_val
+        else
+          this_val.merge other_val
+        end
+      end
+    end
+
   end
 end

data/lib/remi/settings.rb

@@ -10,6 +10,14 @@ module Remi
       @work_dir = arg
     end
 
+    def jobs_dir
+      @jobs_dir ||= Pathname.new('jobs').realpath
+    end
+
+    def jobs_dir=(arg)
+      @jobs_dir = Pathname.new(arg).realpath
+    end
+
     def log_level
       @log_level ||= Logger::INFO
     end

data/lib/remi/transform.rb

@@ -18,7 +18,7 @@ module Remi
     attr_accessor :target_metadata
 
     # Public: Set to true if the transform expects multiple arguments (default: false)
-    attr_reader :multi_arg
+    attr_reader :multi_args
 
     # Public: Defines the operation of this transform class.
     #
@@ -36,7 +36,7 @@ module Remi
     #
     # Returns the transformed value.
     def call(*values)
-      if @multi_arg
+      if @multi_args
         to_proc.call(*values)
       else
         to_proc.call(Array(values).first)

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.35'
+  VERSION = '0.2.36'
 end

data/spec/data_subject/csv_file_spec.rb

@@ -22,7 +22,8 @@ describe DataSource::CsvFile do
       filename_field: :from_file
     )
 
-    expect(csv.df[:from_file].to_a).to eq ['spec/fixtures/basic.csv'] * 2
+    expected_files = [Pathname.new('spec/fixtures/basic.csv').realpath.to_s] * 2
+    expect(csv.df[:from_file].to_a).to eq expected_files
   end
 
   it "preprocesses records when required" do
@@ -56,12 +57,11 @@ describe DataSource::CsvFile do
     expect(csv.df.to_a).to eq expected_df.to_a
   end
 
-  # Do this when I retire the old LocalFile
-  it "combines multiple csv files into a single dataframe", skip: 'TODO' do
+  it "combines multiple csv files into a single dataframe" do
     csv = Remi::DataSource::CsvFile.new(
       extractor: Remi::Extractor::LocalFile.new(
         remote_path: 'spec/fixtures',
-        pattern: 'basic(|2)\.csv'
+        pattern: /basic(|2)\.csv/
       )
     )
 
@@ -69,9 +69,10 @@ describe DataSource::CsvFile do
       {
         column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
         column_b: ['value 1B', 'value 2B', nil, nil],
-        columb_c: [nil, nil, 'value 1C', 'value 2C']
+        column_c: [nil, nil, 'value 1C', 'value 2C']
       }
     )
+
     expect(csv.df.to_a).to eq expected_df.to_a
   end
 

data/spec/extractor/local_file_spec.rb

@@ -0,0 +1,31 @@
+require 'remi_spec'
+
+describe Extractor::LocalFile do
+  let(:remote_path) { "#{Pathname.new(__FILE__).dirname}" }
+
+  let(:local_file) {
+    Extractor::LocalFile.new(
+      remote_path: remote_path
+    )
+  }
+
+  let(:remote_filenames) { Dir[remote_path + '/*'].map { |f| Pathname.new(f).basename.to_s } }
+
+  context '.new' do
+    it 'creates an instance with valid parameters' do
+      local_file
+    end
+  end
+
+  context '#all_entires' do
+    it 'returns all entries' do
+      expect(local_file.all_entries.map(&:name)).to eq remote_filenames
+    end
+  end
+
+  context '#extract' do
+    it 'references local files with the right names' do
+      expect(local_file.extract.map { |f| Pathname.new(f).basename.to_s }).to eq remote_filenames
+    end
+  end
+end

data/spec/extractor/sftp_file_spec.rb

@@ -1,125 +1,84 @@
 require 'remi_spec'
 
 describe Extractor::SftpFile do
-  before do
-    now = Time.new
+  let(:remote_path) { '' }
+  let(:credentials) {
+    {
+      host: 'host',
+      username: 'username',
+      password: 'password'
+    }
+  }
+
+  let(:sftp_file) {
+    Extractor::SftpFile.new(
+      credentials: credentials,
+      remote_path: remote_path
+    )
+  }
+
+  let(:remote_filenames) { ['file1.csv', 'file2.csv'] }
+  let(:sftp_session) { instance_double('Net:SFTP::Session') }
 
-    example_files = [
-      { name: "ApplicantsA-9.csv", createtime: now - 10.minutes },
-      { name: "ApplicantsA-3.csv", createtime: now - 5.minutes },
-      { name: "ApplicantsA-5.csv", createtime: now - 1.minutes },
-      { name: "ApplicantsB-7.csv", createtime: now - 10.minutes },
-      { name: "ApplicantsB-6.csv", createtime: now - 5.minutes },
-      { name: "ApplicantsB-2.csv", createtime: now - 1.minutes },
-      { name: "ApplicantsB-2.txt", createtime: now - 0.minutes },
-      { name: "Apples.csv", createtime: now - 1.minutes },
-    ]
+  before do
+    sftp_dir = instance_double('Net::SFTP::Operations::Dir')
 
-    allow_any_instance_of(Extractor::SftpFile).to receive(:all_entries) do
-      example_files.map do |file|
-        Net::SFTP::Protocol::V04::Name.new(
-          file[:name],
-          Net::SFTP::Protocol::V04::Attributes.new(createtime: file[:createtime])
-        )
-      end
-    end
+    allow(Net::SFTP).to receive(:start).and_yield sftp_session
+    allow(sftp_session).to receive(:dir).and_return sftp_dir
 
-    @params = { credentials: nil }
+    allow(sftp_dir).to receive(:entries).and_return(remote_filenames.map { |fname|
+      Net::SFTP::Protocol::V04::Name.new(
+        fname,
+        Net::SFTP::Protocol::V04::Attributes.new(createtime: Time.new.to_i, mtime: Time.new.to_i)
+      )
+    })
   end
 
-  let(:sftpfile) { Extractor::SftpFile.new(**@params) }
-
-
-
-
-  context 'extracting all files matching a pattern' do
-    before do
-      @params[:remote_file] = /ApplicantsA-\d+\.csv/
+  context '.new' do
+    it 'creates an instance with valid parameters' do
+      sftp_file
     end
 
-    it 'does not extract non-matching files' do
-      expect(sftpfile.to_download.map(&:name)).not_to include "Apples.csv"
+    it 'requires a hostname' do
+      credentials.delete(:host)
+      expect { sftp_file }.to raise_error KeyError
     end
 
-    it 'extracts all matching files' do
-      expect(sftpfile.to_download.map(&:name)).to match_array([
-       "ApplicantsA-9.csv",
-       "ApplicantsA-3.csv",
-       "ApplicantsA-5.csv"
-      ])
+    it 'requires a username' do
+      credentials.delete(:username)
+      expect { sftp_file }.to raise_error KeyError
     end
-  end
-
 
-  context 'extracting only the most recent matching a pattern' do
-    before do
-      @params.merge!({
-        remote_file: /ApplicantsA-\d+\.csv/,
-        most_recent_only: true
-      })
+    it 'requires a password' do
+      credentials.delete(:password)
+      expect { sftp_file }.to raise_error KeyError
     end
 
-    it 'extracts only the most recent matching file' do
-      expect(sftpfile.to_download.map(&:name)).to match_array([
-       "ApplicantsA-5.csv"
-      ])
+    it 'defaults to using port 22' do
+      expect(sftp_file.port).to eq '22'
     end
 
-    context 'using filename instead of createtime' do
-      before do
-        @params[:most_recent_by] = :filename
-      end
-
-      it 'extracts only the most recent matching file' do
-        expect(sftpfile.to_download.map(&:name)).to match_array([
-         "ApplicantsA-9.csv"
-        ])
-      end
+    it 'allows the port to be defined in the credentials' do
+      credentials[:port] = '1234'
+      expect(sftp_file.port).to eq '1234'
     end
   end
 
-
-  context 'extracting files matching a pattern with a by group' do
-    before do
-      @params.merge!({
-        credentials: nil,
-        remote_file: /^Applicants(A|B)-\d+\.csv/,
-        group_by: /^Applicants(A|B)/
-      })
-    end
-
-    it 'extracts the most recent file that matches a particular regex' do
-      expect(sftpfile.to_download.map(&:name)).to match_array([
-       "ApplicantsA-5.csv",
-       "ApplicantsB-2.csv"
-      ])
+  context '#all_entires' do
+    it 'returns all entries' do
+      expect(sftp_file.all_entries.map(&:name)).to eq remote_filenames
     end
+  end
 
-    context 'with a minimally selective pre-filter' do
-      before do
-        @params[:remote_file] = /^Applicants/
-      end
-
-      it 'extracts the most recent file that matches a particular regex' do
-        expect(sftpfile.to_download.map(&:name)).to match_array([
-         "ApplicantsA-5.csv",
-         "ApplicantsB-2.txt"
-        ])
-      end
+  context '#extract' do
+    it 'downloads files from the ftp' do
+      expect(sftp_session).to receive(:download!).exactly(remote_filenames.size).times
+      sftp_file.extract
     end
 
-    context 'using filename instead of createtime' do
-      before do
-        @params[:most_recent_by] = :filename
-      end
-
-      it 'extracts only the most recent matching file' do
-        expect(sftpfile.to_download.map(&:name)).to match_array([
-         "ApplicantsA-9.csv",
-         "ApplicantsB-7.csv"
-        ])
-      end
+    it 'creates local files with the right names' do
+      allow(sftp_session).to receive(:download!)
+      expect(sftp_file.extract.map { |f| Pathname.new(f).basename.to_s }).to eq remote_filenames
     end
-
   end
 end

data/spec/fields_spec.rb

@@ -0,0 +1,97 @@
+require_relative 'remi_spec'
+
+describe Fields do
+
+  let :base_fields do
+    Fields.new(
+      {
+        col1: { from: :base, base: true },
+        col2: { from: :base, base: true }
+      }
+    )
+  end
+
+  let :fields2 do
+
+  end
+
+  context "merging field sets" do
+
+    context "when there is no overlap" do
+      it "unions field sets" do
+        other_fields = Fields.new(
+          {
+            col3: {},
+            col4: {}
+          }
+        )
+
+        merged_fields = base_fields.merge other_fields
+
+        expect(merged_fields.keys).to eq [:col1, :col2, :col3, :col4]
+      end
+    end
+
+    context "when there is overlap" do
+      let :other_fields do
+        Fields.new(
+          {
+            col2: { from: :other, other: true },
+            col3: { from: :other, other: true }
+          }
+        )
+      end
+
+      it "unions field sets when there is overlap" do
+        merged_fields = base_fields.merge other_fields
+        expect(merged_fields.keys).to eq [:col1, :col2, :col3]
+      end
+
+      it "merges overlapping metadata" do
+        merged_fields = base_fields.merge other_fields
+
+        expect(merged_fields).to eq(
+          {
+            col1: { from: :base, base: true },
+            col2: { from: :other, base: true, other: true },
+            col3: { from: :other, other: true }
+          }
+        )
+      end
+
+      it "does not affect the original field sets" do
+        merged_fields = base_fields.merge other_fields
+
+        expect(base_fields).to eq(
+          {
+            col1: { from: :base, base: true },
+            col2: { from: :base, base: true }
+          }
+        )
+
+        expect(other_fields).to eq(
+          {
+            col2: { from: :other, other: true },
+            col3: { from: :other, other: true }
+          }
+        )
+      end
+
+      context "with a prefix" do
+        it "creates new fields for names that conflict" do
+          merged_fields = base_fields.merge other_fields, prefix: :other_
+
+          expect(merged_fields).to eq(
+            {
+              col1: { from: :base, base: true },
+              col2: { from: :base, base: true },
+              other_col2: { from: :other, other: true },
+              col3: { from: :other, other: true }
+            }
+          )
+        end
+      end
+
+    end
+  end
+end

data/spec/transform_spec.rb

@@ -0,0 +1,53 @@
+require_relative 'remi_spec'
+
+describe Transform do
+
+  context 'a transform with a single argument' do
+    before do
+      class SingleArgument < Transform
+        def initialize(*args, **kargs, &block)
+          super
+        end
+
+        def transform(value)
+          value
+        end
+      end
+    end
+
+    let(:transform) { SingleArgument.new }
+
+    it 'can be converted into a proc and called' do
+      expect(transform.to_proc.call(5)).to eq 5
+    end
+
+    it 'can be called directly' do
+      expect(transform.call(5)).to eq 5
+    end
+  end
+
+  context 'a transform that accepts multiple arguments' do
+    before do
+      class MultipleArgument < Transform
+        def initialize(*args, **kargs, &block)
+          super
+          @multi_args = true
+        end
+
+        def transform(*values)
+          Array(values)
+        end
+      end
+    end
+
+    let(:transform) { MultipleArgument.new }
+
+    it 'can be converted into a proc and called' do
+      expect(transform.to_proc.call(1, 2)).to eq [1, 2]
+    end
+
+    it 'can be called directly' do
+      expect(transform.call(1, 2)).to eq [1, 2]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.35
+  version: 0.2.36
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-08 00:00:00.000000000 Z
+date: 2016-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -232,9 +232,9 @@ files:
 - lib/remi/data_subject/salesforce.rb
 - lib/remi/data_subject/sftp_file.rb
 - lib/remi/extractor/file_system.rb
+- lib/remi/extractor/local_file.rb
 - lib/remi/extractor/s3_file.rb
 - lib/remi/extractor/sftp_file.rb
-- lib/remi/extractor/sftp_file_new.rb
 - lib/remi/field_symbolizers.rb
 - lib/remi/fields.rb
 - lib/remi/job.rb
@@ -248,13 +248,16 @@ files:
 - spec/data_subject/csv_file_spec.rb
 - spec/data_subject/data_frame.rb
 - spec/extractor/file_system_spec.rb
+- spec/extractor/local_file_spec.rb
 - spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
+- spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
 - spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
+- spec/transform_spec.rb
 - workbooks/sample_workbook.ipynb
 - workbooks/workbook_helper.rb
 homepage: https://github.com/inside-track/remi
@@ -304,10 +307,13 @@ test_files:
 - spec/data_subject/csv_file_spec.rb
 - spec/data_subject/data_frame.rb
 - spec/extractor/file_system_spec.rb
+- spec/extractor/local_file_spec.rb
 - spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
+- spec/fields_spec.rb
 - spec/fixtures/basic.csv
 - spec/fixtures/basic2.csv
 - spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
+- spec/transform_spec.rb

data/lib/remi/extractor/sftp_file_new.rb

@@ -1,78 +0,0 @@
-module Remi
-  module Extractor
-
-    class SftpFileNew < FileSystem
-
-      N_RETRY = 3
-
-      def initialize(*args, **kargs)
-        super
-        init_sftp_file(*args, **kargs)
-      end
-
-      # Public: Called to extract files from the source filesystem.
-      #
-      # Returns an array with containing the paths to all files extracted.
-      def extract
-        connection do |sftp|
-          entries.map do |entry|
-            local_file = File.join(@local_path, entry.name)
-            @logger.info "Downloading #{entry.name} to #{local_file}"
-            retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
-            local_file
-          end
-        end
-      end
-
-      # Public: Returns an array of all FileSystemEntry instances that are in the remote_path.
-      def all_entries
-        @all_entries ||= all_entries!
-      end
-
-      def all_entries!
-        sftp_entries = connection { |sftp| sftp.dir.entries(@remote_path.dirname) }
-        sftp_entries.map do |entry|
-          # Early versions of the protocol don't support create time, fake it with modified time?
-          FileSystemEntry.new(
-            name: File.join(@remote_path.dirname, entry.name),
-            create_time: entry.respond_to?(:createtime) ? entry.createtime : entry.mtime,
-            modified_time: entry.mtime
-          )
-        end
-      end
-
-
-      private
-
-      def init_sftp_file(*args, credentials:, **kargs)
-        @host = credentials.fetch(:host)
-        @username = credentials.fetch(:username)
-        @password = credentials.fetch(:password)
-        @port = credentials.fetch(:port, '22')
-      end
-
-      def connection(&block)
-        result = nil
-        Net::SFTP.start(@host, @username, password: @password, port: @port) do |sftp|
-          result = yield sftp
-        end
-        result
-      end
-
-      def retry_download(&block)
-        1.upto(N_RETRY).each do |itry|
-          begin
-            block.call
-            break
-          rescue RuntimeError => err
-            raise err unless itry < ntry
-            @logger.error "Download failed with error: #{err.message}"
-            @logger.error "Retry attempt #{itry}/#{ntry-1}"
-            sleep(1)
-          end
-        end
-      end
-    end
-
-  end
-end