remi 0.2.33 → 0.2.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a91453040868eaabb19fb50db4aafa0f0da4a643
-  data.tar.gz: 8530db1145f89b1abcb8450fe8bcb4eb5325409d
+  metadata.gz: 49fe8ceb344f11f09d99cc4c93cfb03a23e4ba48
+  data.tar.gz: 320de69f750fc9695209f23413a8f6761de8bf3f
 SHA512:
-  metadata.gz: 866bd205d40ea549fc8b11fd06d599a54a12bae6cdc3ddbb746c377c5bd17617ddf4eb48b0e88e2869df529eeafe26043b8fba7bbea8641ecc81eb2455b8480f
-  data.tar.gz: fc7906b2ffd875e42519a065a21212237cf48c0ece1a0ad393f58b83a02ea6f40811290e4fc21068a0322d9aa2828d8e0e5ea5bec61cb835ed9602eba0c87a6f
+  metadata.gz: 7253a752da9a2dfeed5b27128bef939d042e0fc67af42a6bb4720cf9e37f8826d703ad4aee1a676e52f803093f37914dbacba7bd5844fab678373c11febf9064
+  data.tar.gz: bd5440f12c67f16f4190d56147130d619aee83c76f8fe93da156653ae7d7d1ffa93372767aceca06db2869bcb91d6731191cb4f7e0f77b223e971b651145dbcb

data/Gemfile.lock

@@ -19,7 +19,7 @@ GIT
 PATH
   remote: .
   specs:
-    remi (0.2.33)
+    remi (0.2.34)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/step_definitions/remi_step.rb

@@ -656,6 +656,13 @@ Then /^the target has (\d+) record(?:s|)$/ do |nrecords|
   step "the target '#{target_name}' has #{nrecords} records"
 end
 
+Then /^the targets have (\d+) record(?:s|)$/ do |nrecords|
+  @brt.run_transforms
+
+  obs_nrecords = @brt.targets.keys.reduce(0) { |sum, target_name| sum += @brt.targets[target_name].size }
+  expect(obs_nrecords).to eq nrecords.to_i
+end
+
 Then /^the target '([[:alnum:]\s\-_]+)' has (\d+) record(?:s|)$/ do |target_name, nrecords|
   @brt.run_transforms
   expect(@brt.targets[target_name].size).to eq nrecords.to_i

data/lib/remi/data_subject/csv_file.rb

@@ -54,7 +54,8 @@ module Remi
       result_df = nil
       extract.each_with_index do |filename, idx|
         @logger.info "Converting #{filename} to a dataframe"
-        csv_df = Daru::DataFrame.from_csv filename, @csv_options
+        processed_filename = preprocess(filename)
+        csv_df = Daru::DataFrame.from_csv processed_filename, @csv_options
 
         csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
         if idx == 0
@@ -104,12 +105,31 @@ module Remi
     end
 
 
+
     private
 
-    def init_csv_file(*args, extractor:, csv_options: {}, filename_field: nil, **kargs, &block)
+    def preprocess(filename)
+      return filename unless @preprocessor
+      @logger.info "Preprocessing #{filename}"
+      tmp_filename = File.join(Remi::Settings.work_dir, "#{Pathname.new(filename).basename}-#{SecureRandom.uuid}")
+
+      dirname = Pathname.new(tmp_filename).dirname
+      FileUtils.mkdir_p(dirname) unless File.directory? dirname
+
+      File.open(tmp_filename, 'w') do |outfile|
+        File.foreach(filename) do |in_line|
+          outfile.write @preprocessor.call(in_line)
+        end
+      end
+
+      tmp_filename
+    end
+
+    def init_csv_file(*args, extractor:, csv_options: {}, filename_field: nil, preprocessor: nil, **kargs, &block)
       self.extractor = extractor
       @csv_options = self.class.default_csv_options.merge(csv_options)
       @filename_field = filename_field
+      @preprocessor = preprocessor
     end
   end
 

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.33'
+  VERSION = '0.2.34'
 end

data/spec/data_subject/csv_file_spec.rb

@@ -0,0 +1,78 @@
+require_relative '../remi_spec'
+
+describe DataSource::CsvFile do
+
+  it "converts a CSV into a dataframe" do
+    csv = Remi::DataSource::CsvFile.new(
+      extractor: 'spec/fixtures/basic.csv'
+    )
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B']
+      }
+    )
+    expect(csv.df.to_a).to eq expected_df.to_a
+  end
+
+  it "adds filename when requested" do
+    csv = Remi::DataSource::CsvFile.new(
+      extractor: 'spec/fixtures/basic.csv',
+      filename_field: :from_file
+    )
+
+    expect(csv.df[:from_file].to_a).to eq ['spec/fixtures/basic.csv'] * 2
+  end
+
+  it "preprocesses records when required" do
+    csv = Remi::DataSource::CsvFile.new(
+      extractor: 'spec/fixtures/unsupported_escape.csv',
+      preprocessor: ->(line) { line.gsub(/\\"/,'""') }
+    )
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value "1B"', 'value "2B"']
+      }
+    )
+    expect(csv.df.to_a).to eq expected_df.to_a
+  end
+
+  it "accepts standard Ruby CSV options" do
+    csv = Remi::DataSource::CsvFile.new(
+      extractor: 'spec/fixtures/basic.csv',
+      preprocessor: ->(line) { line.gsub(/,/,'|') },
+      csv_options: { col_sep: '|' }
+    )
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B']
+      }
+    )
+    expect(csv.df.to_a).to eq expected_df.to_a
+  end
+
+  # Do this when I retire the old LocalFile
+  it "combines multiple csv files into a single dataframe", skip: 'TODO' do
+    csv = Remi::DataSource::CsvFile.new(
+      extractor: Remi::Extractor::LocalFile.new(
+        remote_path: 'spec/fixtures',
+        pattern: 'basic(|2)\.csv'
+      )
+    )
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B', nil, nil],
+        columb_c: [nil, nil, 'value 1C', 'value 2C']
+      }
+    )
+    expect(csv.df.to_a).to eq expected_df.to_a
+  end
+
+end

data/spec/fixtures/basic.csv

@@ -0,0 +1,3 @@
+column A,column B
+value 1A,value 1B
+value 2A,value 2B

data/spec/fixtures/basic2.csv

@@ -0,0 +1,3 @@
+column A,column C
+value 1A,value 1C
+value 2A,value 2C

data/spec/fixtures/unsupported_escape.csv

@@ -0,0 +1,3 @@
+column A,column B
+value 1A,"value \"1B\""
+value 2A,"value \"2B\""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.33
+  version: 0.2.34
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-23 00:00:00.000000000 Z
+date: 2016-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bond
@@ -245,9 +245,13 @@ files:
 - lib/remi/transform.rb
 - lib/remi/version.rb
 - remi.gemspec
+- spec/data_subject/csv_file_spec.rb
 - spec/extractor/file_system_spec.rb
 - spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
+- spec/fixtures/basic.csv
+- spec/fixtures/basic2.csv
+- spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb
 - workbooks/sample_workbook.ipynb
@@ -296,8 +300,12 @@ test_files:
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
 - features/transforms/truthy.feature
+- spec/data_subject/csv_file_spec.rb
 - spec/extractor/file_system_spec.rb
 - spec/extractor/s3_file_spec.rb
 - spec/extractor/sftp_file_spec.rb
+- spec/fixtures/basic.csv
+- spec/fixtures/basic2.csv
+- spec/fixtures/unsupported_escape.csv
 - spec/metadata_spec.rb
 - spec/remi_spec.rb