remi 0.2.19 → 0.2.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b1844e0c62caecbd5d416c5b2ea035f38dae670
-  data.tar.gz: 5246aa674a9f203aef16ff6d775625cc0441fe40
+  metadata.gz: ea1259debd08b1c32c279209ab9a87eb5b0607ac
+  data.tar.gz: 624f19339fae31fe5951dc31001f913a5f834c1c
 SHA512:
-  metadata.gz: 774b90bd45ff5474a8e402928b5b40979c0a1e9dc6b2d08dced42dee25c567c5806aeb2b2d68a0f98efb1388ef5f3b2be6d47bcc80da1b7b3842152dfafce653
-  data.tar.gz: 3f2ff9109ea794f6f2006ca66d7f9cc55ba574e826391d2a57c15172fcc1a25047c28709d270266e2f9abf03c38ce031fd693f70d07964208b34a5ad60eaf2dd
+  metadata.gz: 08513f0eba7dca6fc95c90f288ca1581d592cda4d5c2f32073c4ec85ffa53b2521c72aaceabdd9a4482171caf1935c4714ca85d785b2a10a22f65d5325ee8eb4
+  data.tar.gz: a74eecaf4bfce033db38fddf4daef177320f447c01f15cabab39cb494db485943177e3385b3dc1a2d3b8f6e428289e410659f8d1aa9e907e464c6ac779d64ce4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    remi (0.2.19)
+    remi (0.2.20)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/transforms/truncate.feature

@@ -12,3 +12,8 @@ Feature: Test the truncate transformer.
 
     And the source field 'My Field' is set to the value "something"
     Then the target field 'Truncated Field' is set to the value "somet"
+
+    And the job parameter 'truncate_len' is "7"
+
+    And the source field 'My Field' is set to the value "something"
+    Then the target field 'Truncated Field' is set to the value "somethi"

data/lib/remi/data_source/csv_file.rb

@@ -3,6 +3,8 @@ module Remi
     class CsvFile
       include DataSource
 
+      using Remi::Refinements::Daru
+
       def self.default_csv_options
         CSV::DEFAULT_OPTIONS.merge({
           headers: true,
@@ -14,10 +16,11 @@ module Remi
       end
 
 
-      def initialize(fields: {}, extractor:, csv_options: {}, logger: Remi::Settings.logger)
+      def initialize(fields: {}, extractor:, csv_options: {}, filename_field: nil, logger: Remi::Settings.logger)
         @fields = fields
         self.extractor = extractor
         @csv_options = self.class.default_csv_options.merge(csv_options)
+        @filename_field = filename_field
         @logger = logger
       end
 
@@ -30,7 +33,11 @@ module Remi
       end
 
       def extract
-        Array(@extractor.extract).tap { |x| raise "Multiple files not supported" if x.size > 1 }
+        @extracted = Array(@extractor.extract)
+      end
+
+      def extracted
+        @extracted || extract
       end
 
       def extractor=(arg)
@@ -38,7 +45,7 @@ module Remi
         when Extractor::SftpFile, Extractor::LocalFile
           @extractor = arg
         when String
-          @extractor = Extractor::LocalFile.new(arg)
+          @extractor = Extractor::LocalFile.new(path: arg)
         when Regexp
           raise "Adding regex matching to local files would be easy, not done yet"
         else
@@ -48,7 +55,8 @@ module Remi
 
       # Only going to support single file for now
       def source_filename
-        @source_filename ||= extract.first
+        raise "Multiple source files detected" if extracted.size > 1
+        @source_filename ||= extracted.first
       end
 
       def first_line
@@ -67,8 +75,21 @@ module Remi
       end
 
       def to_dataframe
-        @logger.info "Converting #{source_filename} to a dataframe"
-        Daru::DataFrame.from_csv source_filename, @csv_options
+        # Assumes that each file has exactly the same structure
+        result_df = nil
+        extracted.each_with_index do |filename, idx|
+          @logger.info "Converting #{filename} to a dataframe"
+          csv_df = Daru::DataFrame.from_csv filename, @csv_options
+
+          csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
+          if idx == 0
+            result_df = csv_df
+          else
+            result_df = result_df.concat csv_df
+          end
+        end
+
+        result_df
       end
 
       def df

data/lib/remi/extractor/sftp_file.rb

@@ -2,12 +2,20 @@ module Remi
   module Extractor
 
     class LocalFile
-      def initialize(path)
+      def initialize(path:, folder: nil)
         @path = path
+        @folder = folder
       end
 
       def extract
-        @path
+        if @folder
+          Dir.entries(@folder).map do |entry|
+            next unless entry.match(@path)
+            File.join(@folder,entry)
+          end.compact
+        else
+          @path
+        end
       end
     end
 
@@ -15,24 +23,41 @@ module Remi
 
       class FileNotFoundError < StandardError; end
 
-      def initialize(credentials:, remote_file:, remote_folder: '', local_folder: Settings.work_dir, port: '22', most_recent_only: false, logger: Remi::Settings.logger)
+      SortDesc = Struct.new(:value) do
+        def <=> (target)
+          -(self.value <=> target.value)
+        end
+      end
+
+      def initialize(credentials:, remote_file:, remote_folder: '', local_folder: Settings.work_dir, port: nil, most_recent_only: false, group_by: nil, most_recent_by: :createtime, logger: Remi::Settings.logger)
         @credentials = credentials
         @remote_file = remote_file
         @remote_folder = remote_folder
         @local_folder = local_folder
-        @port = port
+        @port = port || (credentials && credentials[:port]) || '22'
         @most_recent_only = most_recent_only
+        @group_by = group_by
+        @most_recent_by = most_recent_by
         @logger = logger
       end
 
       attr_reader :logger
 
       def extract
-        to_download = @most_recent_only ? Array(most_recent_entry(matching_entries)) : matching_entries
         raise FileNotFoundError, "File not found: #{@remote_file}" if to_download.size == 0
         download(to_download)
       end
 
+      def to_download
+        if @group_by
+          most_recent_in_group
+        elsif @most_recent_only
+          Array(most_recent_entry(matching_entries))
+        else
+          matching_entries
+        end
+      end
+
       def all_entries(remote_folder = @remote_folder)
         @all_entries ||= connection { |sftp| sftp.dir.entries(File.join("/", remote_folder)) }
       end
@@ -42,15 +67,41 @@ module Remi
       end
 
       def most_recent_entry(entries = matching_entries)
-        entries.sort_by { |e| e.attributes.createtime }.reverse!.first
+        entries.sort_by { |e| sort_files_by(e) }.reverse!.first
+      end
+
+      def sort_files_by(entry)
+        if @most_recent_by == :filename
+          entry.name
+        else
+          entry.attributes.send(@most_recent_by)
+        end
+      end
+
+      def most_recent_in_group(match_group = @group_by)
+        entries_with_group = matching_entries.map do |entry|
+          match = entry.name.match(match_group)
+          next unless match
+
+          group = match.to_a[1..-1]
+          { group: group, entry: entry }
+        end.compact
+        entries_with_group.sort_by! { |e| [e[:group], SortDesc.new(sort_files_by(e[:entry]))] }
+
+        last_group = nil
+        entries_with_group.map do |entry|
+          next unless entry[:group] != last_group
+          last_group = entry[:group]
+          entry[:entry]
+        end.compact
       end
 
-      def download(to_download = matching_entries, local_folder: @local_folder, ntry: 3)
+      def download(entries_to_download, remote_folder: @remote_folder, local_folder: @local_folder, ntry: 3)
         connection do |sftp|
-          to_download.map do |entry|
+          entries_to_download.map do |entry|
             local_file = File.join(local_folder, entry.name)
             @logger.info "Downloading #{entry.name} to #{local_file}"
-            retry_download(ntry) { sftp.download!(entry.name, local_file) }
+            retry_download(ntry) { sftp.download!(File.join(remote_folder, entry.name), local_file) }
             local_file
           end
         end
@@ -71,6 +122,7 @@ module Remi
         1.upto(ntry).each do |itry|
           begin
             block.call
+            break
           rescue RuntimeError => err
             raise err unless itry < ntry
             @logger.error "Download failed with error: #{err.message}"

data/lib/remi/refinements/daru.rb

@@ -13,15 +13,23 @@ module Remi
           dupdf
         end
 
-        # Public: Fixes a bug where the dataframe on the left side of the
-        # concatenation is accidentally modified.
+        # Public: Allows for combining dataframes with different columns
         def concat other_df
-          vectors = []
-          @vectors.each do |v|
-            vectors << self[v].dup.to_a.concat(other_df[v].to_a)
+          vectors = @vectors.to_a
+          data = []
+
+          vectors.each do |v|
+            other_vec = other_df.vectors.include?(v) ? other_df[v].to_a : [nil] * other_df.size
+            data << self[v].dup.to_a.concat(other_vec)
+          end
+
+          other_df.vectors.each do |v|
+            next if vectors.include?(v)
+            vectors << v
+            data << ([nil] * self.size).concat(other_df[v].to_a)
           end
 
-          ::Daru::DataFrame.new(vectors, order: @vectors)
+          ::Daru::DataFrame.new(data, order: vectors)
         end
 
         # Public: Saves a Dataframe to a file.

data/lib/remi/transform.rb

@@ -40,7 +40,7 @@ module Remi
 
     def truncate(len)
       memoize_as_lambda(__method__, len) do |(mlen), larg|
-        larg.slice(0,len)
+        larg.slice(0,mlen)
       end
     end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.19'
+  VERSION = '0.2.20'
 end

data/spec/extractor/sftp_file_spec.rb

@@ -0,0 +1,125 @@
+require 'remi_spec'
+
+describe Extractor::SftpFile do
+  before do
+    now = Time.new
+
+    example_files = [
+      { name: "ApplicantsA-9.csv", createtime: now - 10.minutes },
+      { name: "ApplicantsA-3.csv", createtime: now - 5.minutes },
+      { name: "ApplicantsA-5.csv", createtime: now - 1.minutes },
+      { name: "ApplicantsB-7.csv", createtime: now - 10.minutes },
+      { name: "ApplicantsB-6.csv", createtime: now - 5.minutes },
+      { name: "ApplicantsB-2.csv", createtime: now - 1.minutes },
+      { name: "ApplicantsB-2.txt", createtime: now - 0.minutes },
+      { name: "Apples.csv", createtime: now - 1.minutes },
+    ]
+
+    allow_any_instance_of(Extractor::SftpFile).to receive(:all_entries) do
+      example_files.map do |file|
+        Net::SFTP::Protocol::V04::Name.new(
+          file[:name],
+          Net::SFTP::Protocol::V04::Attributes.new(createtime: file[:createtime])
+        )
+      end
+    end
+
+    @params = { credentials: nil }
+  end
+
+  let(:sftpfile) { Extractor::SftpFile.new(**@params) }
+
+
+
+
+  context 'extracting all files matching a pattern' do
+    before do
+      @params[:remote_file] = /ApplicantsA-\d+\.csv/
+    end
+
+    it 'does not extract non-matching files' do
+      expect(sftpfile.to_download.map(&:name)).not_to include "Apples.csv"
+    end
+
+    it 'extracts all matching files' do
+      expect(sftpfile.to_download.map(&:name)).to match_array([
+       "ApplicantsA-9.csv",
+       "ApplicantsA-3.csv",
+       "ApplicantsA-5.csv"
+      ])
+    end
+  end
+
+
+  context 'extracting only the most recent matching a pattern' do
+    before do
+      @params.merge!({
+        remote_file: /ApplicantsA-\d+\.csv/,
+        most_recent_only: true
+      })
+    end
+
+    it 'extracts only the most recent matching file' do
+      expect(sftpfile.to_download.map(&:name)).to match_array([
+       "ApplicantsA-5.csv"
+      ])
+    end
+
+    context 'using filename instead of createtime' do
+      before do
+        @params[:most_recent_by] = :filename
+      end
+
+      it 'extracts only the most recent matching file' do
+        expect(sftpfile.to_download.map(&:name)).to match_array([
+         "ApplicantsA-9.csv"
+        ])
+      end
+    end
+  end
+
+
+  context 'extracting files matching a pattern with a by group' do
+    before do
+      @params.merge!({
+        credentials: nil,
+        remote_file: /^Applicants(A|B)-\d+\.csv/,
+        group_by: /^Applicants(A|B)/
+      })
+    end
+
+    it 'extracts the most recent file that matches a particular regex' do
+      expect(sftpfile.to_download.map(&:name)).to match_array([
+       "ApplicantsA-5.csv",
+       "ApplicantsB-2.csv"
+      ])
+    end
+
+    context 'with a minimally selective pre-filter' do
+      before do
+        @params[:remote_file] = /^Applicants/
+      end
+
+      it 'extracts the most recent file that matches a particular regex' do
+        expect(sftpfile.to_download.map(&:name)).to match_array([
+         "ApplicantsA-5.csv",
+         "ApplicantsB-2.txt"
+        ])
+      end
+    end
+
+    context 'using filename instead of createtime' do
+      before do
+        @params[:most_recent_by] = :filename
+      end
+
+      it 'extracts only the most recent matching file' do
+        expect(sftpfile.to_download.map(&:name)).to match_array([
+         "ApplicantsA-9.csv",
+         "ApplicantsB-7.csv"
+        ])
+      end
+    end
+
+  end
+end

data/spec/remi_spec.rb

@@ -0,0 +1,8 @@
+File.expand_path(File.join(File.dirname(__FILE__),'../lib')).tap {|pwd| $LOAD_PATH.unshift(pwd) unless $LOAD_PATH.include?(pwd)}
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'remi'
+
+include Remi

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.19
+  version: 0.2.20
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
@@ -279,6 +279,8 @@ files:
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
+- spec/extractor/sftp_file_spec.rb
+- spec/remi_spec.rb
 - workbooks/sample_workbook.ipynb
 - workbooks/workbook_helper.rb
 homepage: https://github.com/inside-track/remi
@@ -321,3 +323,5 @@ test_files:
 - features/transforms/parse_date.feature
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
+- spec/extractor/sftp_file_spec.rb
+- spec/remi_spec.rb