remi 0.2.15 → 0.2.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89aa19a0e9852eb3fcbf63cd3890102501fc27ff
-  data.tar.gz: ef447ea96c09edd89953aabef5b9e18188d06c50
+  metadata.gz: c08e200bea80edb6451804d752e12d3f820211db
+  data.tar.gz: af6e06a889c02f13586e61c4c65b09cbbac14bb1
 SHA512:
-  metadata.gz: 11db98f16aff0c37c83a02dd44dec3ec6e1f908c61e85aaeb277b789acc58b2eaf0706b10274591004da92e0ea66321b1ee2fe5318422772c251519150d41d80
-  data.tar.gz: 7639ef4a4b9b3aa03385e270b5aa3ea10796dd9e9250819e706be0fa366d95dcb4c886e6ccab54e5546f0aa7a38ea6adbb61909fa6919be3350aa68dc8d6b0c0
+  metadata.gz: fa607c200a4c8f838e31c635fbfcd521ad14c86d734b4176880f1cdc8bf7d8e53dc8c3672af62e735a731ad8d3234f54df9982bd9778efceee1e63e7680d03f5
+  data.tar.gz: 18715e8761212d48bf00d32378949044fa9957650b08491f43733bb9902e753d44f2b68a86706b8e00f59c01b4b0fd68a8192ebee4d2d763279bab74a0e3bbe9

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    remi (0.2.15)
+    remi (0.2.16)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)
-      daru (~> 0.1)
+      daru (= 0.1.2)
       docile (~> 1.1)
       faker (~> 1.6)
       net-sftp (~> 2.1)
@@ -39,12 +39,12 @@ GEM
     cucumber-core (1.4.0)
       gherkin (~> 3.2.0)
     cucumber-wire (0.0.1)
-    daru (0.1.1)
+    daru (0.1.2)
       reportbuilder (~> 1.4)
-      spreadsheet (~> 1.0.3)
+      spreadsheet (~> 1.1.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    faker (1.6.1)
+    faker (1.6.3)
       i18n (~> 0.5)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
@@ -94,7 +94,7 @@ GEM
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
       rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.2)
+    rspec-core (3.4.3)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -107,7 +107,7 @@ GEM
     salesforce_bulk_api (0.0.12)
       json
       xml-simple
-    spreadsheet (1.0.9)
+    spreadsheet (1.1.1)
       ruby-ole (>= 1.0)
     text-table (1.2.4)
     thread_safe (0.3.5)

data/features/csv_file_target_job.feature

@@ -0,0 +1,21 @@
+Feature: Tests targets that are Csv Files.
+
+  Background:
+    Given the job is 'Csv File Target'
+    And the job target 'Some Csv File'
+
+
+  Scenario: Defining target csv options.
+
+    Given the target 'Some Csv File'
+    And the target file is delimited with a pipe
+    And the target file is encoded using "UTF-8" format
+    And the target file uses a double quote to quote embedded delimiters
+    And the target file uses a preceding double quote to escape an embedded quoting character
+    And the target file uses unix line endings
+    And the target file contains a header row
+    And the target file contains all of the following headers in this order:
+      | header |
+      | col3   |
+      | col1   |
+      | col2   |

data/features/sftp_file_target_job.feature

@@ -0,0 +1,9 @@
+Feature: Tests targets that are Sftp Files.
+
+  Background:
+    Given the job is 'Sftp File Target'
+    And the job target 'Some File'
+
+  Scenario: Defining the remote path.
+    Given the target 'Some File'
+    Then the file is uploaded to the remote path "some_file_*Today: %Y%m%d*.csv"

data/features/step_definitions/remi_step.rb

@@ -71,37 +71,52 @@ Then /^no files will be downloaded for processing$/ do
   expect { @brt.source.extract }.to raise_error Remi::Extractor::SftpFile::FileNotFoundError
 end
 
+Then /^the file is uploaded to the remote path "([^"]+)"$/ do |remote_path|
+  expect(@brt.target.get_attrib(:remote_path)).to eq Remi::BusinessRules::ParseFormula.parse(remote_path)
+end
+
+## CSV Options
+
+Given /^the (source|target) file is delimited with a (\w+)$/ do |st, delimiter|
+  expect(@brt.send(st.to_sym).csv_options[:col_sep]).to eq Remi::BusinessRules.csv_opt_map[delimiter]
+end
 
-Given /^the source file is delimited with a (\w+)$/ do |delimiter|
-  expect(@brt.source.csv_options[:col_sep]).to eq Remi::BusinessRules.csv_opt_map[delimiter]
+Given /^the (source|target) file is encoded using "([^"]+)" format$/ do |st, encoding|
+  expect(@brt.send(st.to_sym).csv_options[:encoding].split(':').first).to eq encoding
 end
 
-Given /^the source file is encoded using "([^"]+)" format$/ do |encoding|
-  expect(@brt.source.csv_options[:encoding].split(':').first).to eq encoding
+Given /^the (source|target) file uses a ([\w ]+) to quote embedded delimiters$/ do |st, quote_char|
+  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[quote_char]
 end
 
-Given /^the source file uses a ([\w ]+) to quote embedded delimiters$/ do |quote_char|
-  expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[quote_char]
+Given /^the (source|target) file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |st, escape_char|
+  expect(@brt.send(st.to_sym).csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[escape_char]
 end
 
-Given /^the source file uses a preceding ([\w ]+) to escape an embedded quoting character$/ do |escape_char|
-  expect(@brt.source.csv_options[:quote_char]).to eq Remi::BusinessRules.csv_opt_map[escape_char]
+Given /^the (source|target) file uses ([\w ]+) line endings$/ do |st, line_endings|
+  expect(@brt.send(st.to_sym).csv_options[:row_sep]).to eq Remi::BusinessRules.csv_opt_map[line_endings]
 end
 
-Given /^the source file uses ([\w ]+) line endings$/ do |line_endings|
-  expect(@brt.source.csv_options[:row_sep]).to eq Remi::BusinessRules.csv_opt_map[line_endings]
+Given /^the (source|target) file (contains|does not contain) a header row$/ do |st, header|
+  expect(@brt.send(st.to_sym).csv_options[:headers]).to eq (header == 'contains')
 end
 
-Given /^the source file (contains|does not contain) a header row$/ do |header|
-  expect(@brt.source.csv_options[:headers]).to eq (header == 'contains')
+Given /^the (source|target) file contains at least the following headers in no particular order:$/ do |st, table|
+  table.rows.each do |row|
+    field = row.first
+    step "the #{st} field '#{field}'"
+  end
+  expect(@brt.send(st.to_sym).data_obj.fields.keys).to include(*@brt.send(st.to_sym).fields.names)
 end
 
-Given /^the source file contains at least the following headers in no particular order:$/ do |table|
+Given /^the (source|target) file contains all of the following headers in this order:$/ do |st, table|
   table.rows.each do |row|
     field = row.first
-    step "the source field '#{field}'"
+    step "the #{st} field '#{field}'"
   end
-  expect(@brt.source.data_obj.fields.keys).to include(*@brt.source.fields.names)
+
+  @brt.run_transforms if st == 'target'
+  expect(@brt.send(st.to_sym).data_obj.fields.keys).to eq @brt.send(st.to_sym).fields.names
 end
 
 

data/features/support/env_app.rb

@@ -1,5 +1 @@
-require_relative '../../jobs/sample_job'
-require_relative '../../jobs/copy_source_job'
-require_relative '../../jobs/transforms/transform_jobs'
-require_relative '../../jobs/aggregate_job'
-require_relative '../../jobs/parameters_job'
+Dir["#{File.dirname(__FILE__)}/../../jobs/**/*_job.rb"].each { |f| require(f) }

data/jobs/csv_file_target_job.rb

@@ -0,0 +1,20 @@
+require_relative 'all_jobs_shared'
+
+class CsvFileTargetJob
+  include AllJobsShared
+  using Remi::Refinements::Daru
+
+  define_target :some_csv_file, Remi::DataTarget::CsvFile,
+    path: "#{Remi::Settings.work_dir}/some_file.csv",
+    csv_options: {
+      col_sep: '|'
+    }
+
+  define_transform :main do
+    some_csv_file.df = Daru::DataFrame.new({
+      col3: Faker::Hipster.words(10),
+      col1: Faker::Hipster.words(10),
+      col2: ["uh, \"oh"] + Faker::Hipster.words(9)
+    }, order: [:col3, :col1, :col2])
+  end
+end

data/jobs/sftp_file_target_job.rb

@@ -0,0 +1,19 @@
+require_relative 'all_jobs_shared'
+
+class SftpFileTargetJob
+  include AllJobsShared
+  using Remi::Refinements::Daru
+
+
+  define_target :some_file, Remi::DataTarget::SftpFile,
+    credentials: {
+      host: 'example.com',
+      username: 'user',
+      password: 'secret'
+    },
+    local_path: "#{Remi::Settings.work_dir}/some_file.csv",
+    remote_path: "some_file_#{DateTime.current.strftime('%Y%m%d')}.csv"
+
+  define_transform :main do
+  end
+end

data/lib/remi.rb

@@ -57,5 +57,6 @@ require 'remi/data_target.rb'
 require 'remi/data_target/data_frame'
 require 'remi/data_target/salesforce'
 require 'remi/data_target/csv_file'
+require 'remi/data_target/sftp_file'
 
 require 'remi/transform'

data/lib/remi/cucumber/business_rules.rb

@@ -29,9 +29,9 @@ module Remi::BusinessRules
 
     def formulas
       @formulas ||= RegexSieve.new({
-        /(today|yesterday|tomorrow)/i => [:date_reference, :match_single_day],
-        /(this|last|previous|next) (day|month|year|week)/i => [:date_reference, :match_single_unit],
-        /(\d+)\s(day|days|month|months|year|years|week|weeks) (ago|from now)/i => [:date_reference, :match_multiple]
+        /(today|yesterday|tomorrow)(|:[^*]+)\*/i => [:date_reference, :match_single_day],
+        /(this|last|previous|next) (day|month|year|week)(|:[^*]+)\*/i => [:date_reference, :match_single_unit],
+        /(\d+)\s(day|days|month|months|year|years|week|weeks) (ago|from now)(|:[^*]+)\*/i => [:date_reference, :match_multiple]
       })
     end
 
@@ -39,7 +39,7 @@ module Remi::BusinessRules
       return form unless is_formula?(form)
 
       form_opt = formulas[form, :match]
-      raise "Unknown formula #{form}" unless form_opt
+      raise "Unknown formula #{form}" unless form_opt[:match]
 
       to_replace = form.match(base_regex)[0]
       replace_with = if form_opt[:value][0] == :date_reference
@@ -54,30 +54,37 @@ module Remi::BusinessRules
 
     def date_reference(formula, captured)
       parsed = self.send("date_reference_#{formula}", *captured)
-      Date.current.send("#{parsed[:unit]}_#{parsed[:direction]}", parsed[:quantity]).strftime('%Y-%m-%d')
+      Date.current.send("#{parsed[:unit]}_#{parsed[:direction]}", parsed[:quantity]).strftime(parsed[:format])
     end
 
-    def date_reference_match_single_day(form, direction)
+    def parse_colon_date_format(str)
+      str.blank? ? '%Y-%m-%d' : str.slice(1..-1).strip
+    end
+
+    def date_reference_match_single_day(form, direction, format=nil)
       {
         quantity: direction.downcase == 'today' ? 0 : 1,
         unit: 'days',
-        direction: { 'today' => 'ago', 'yesterday' => 'ago', 'tomorrow' => 'since' }[direction.downcase]
+        direction: { 'today' => 'ago', 'yesterday' => 'ago', 'tomorrow' => 'since' }[direction.downcase],
+        format: parse_colon_date_format(format)
       }
     end
 
-    def date_reference_match_single_unit(form, direction, unit)
+    def date_reference_match_single_unit(form, direction, unit, format=nil)
       {
         quantity: direction.downcase == 'this' ? 0 : 1,
         unit: unit.downcase.pluralize,
-        direction: { 'this' => 'ago', 'last' => 'ago', 'previous' => 'ago', 'next' => 'since' }[direction.downcase]
+        direction: { 'this' => 'ago', 'last' => 'ago', 'previous' => 'ago', 'next' => 'since' }[direction.downcase],
+        format: parse_colon_date_format(format)
       }
     end
 
-    def date_reference_match_multiple(form, quantity, unit, direction)
+    def date_reference_match_multiple(form, quantity, unit, direction, format=nil)
       {
         quantity: quantity.to_i,
         unit: unit.downcase.pluralize,
-        direction: { 'ago' => 'ago', 'from now' => 'since' }[direction.downcase]
+        direction: { 'ago' => 'ago', 'from now' => 'since' }[direction.downcase],
+        format: parse_colon_date_format(format)
       }
     end
   end
@@ -240,6 +247,10 @@ module Remi::BusinessRules
       @data_obj.df.size
     end
 
+    def get_attrib(name)
+      @data_obj.send(name)
+    end
+
     # Public: Converts the data subject to a hash where the keys are the table
     # columns and the values are an array for the value of column for each row.
     def column_hash

data/lib/remi/data_target/csv_file.rb

@@ -1,49 +1,42 @@
 module Remi
   module DataTarget
-    class Salesforce
+    class CsvFile
       include DataTarget
 
-      def initialize(object:, operation:, credentials:, api: :bulk, logger: Remi::Settings.logger)
-        @sfo = object
-        @operation = operation
-        @credentials = credentials
-        @api = api
+      def self.default_csv_options
+        CSV::DEFAULT_OPTIONS.merge({
+          headers: true,
+          header_converters: Remi::FieldSymbolizers[:standard],
+          col_sep: ',',
+          encoding: 'UTF-8',
+          quote_char: '"',
+          row_sep: "\n"
+        })
+      end
+
+      def initialize(path:, csv_options: {}, logger: Remi::Settings.logger)
+        @path = path
+        @csv_options = self.class.default_csv_options.merge(csv_options)
         @logger = logger
       end
 
+      attr_reader   :path
+      attr_reader   :csv_options
+
       def field_symbolizer
-        Remi::FieldSymbolizers[:salesforce]
+        self.class.default_csv_options[:header_converters]
       end
 
       def load
         return true if @loaded || df.size == 0
 
-        @logger.info "Performing Salesforce #{@operation} on object #{@sfo}"
+        @logger.info "Writing CSV file #{@path}"
 
-        if @operation == :update
-          Remi::SfBulkHelper::SfBulkUpdate.update(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
-        elsif @operation == :create
-          Remi::SfBulkHelper::SfBulkCreate.create(restforce_client, @sfo, df_as_array_of_hashes, logger: @logger)
-        end
+        df.write_csv @path, @csv_options
 
         @loaded = true
       end
 
-      def restforce_client
-        @restforce_client ||= begin
-          client = Restforce.new(@credentials)
-
-          #run a dummy query to initiate a connection. Workaround for Bulk API problem
-          # https://github.com/yatish27/salesforce_bulk_api/issues/33
-          client.query('SELECT Id FROM Contact LIMIT 1')
-          client
-        end
-      end
-
-      def df_as_array_of_hashes
-        df.to_a[0]
-      end
-
     end
   end
 end

data/lib/remi/data_target/sftp_file.rb

@@ -0,0 +1,54 @@
+module Remi
+  module DataTarget
+    class SftpFile
+      include DataTarget
+
+      def initialize(credentials:, local_path:, remote_path: File.basename(local_path), logger: Remi::Settings.logger)
+        @credentials = credentials
+        @local_path = local_path
+        @remote_path = remote_path
+        @logger = logger
+      end
+
+      attr_reader :local_path
+      attr_reader :remote_path
+
+      def load
+        return true if @loaded
+
+        connection do |sftp|
+          retry_upload { sftp.upload! @local_path, @remote_path }
+        end
+
+        @loaded = true
+      end
+
+
+
+      private
+
+      def connection(&block)
+        result = nil
+        Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @credentials[:port] || '22') do |sftp|
+          result = yield sftp
+        end
+        result
+      end
+
+      def retry_upload(ntry=2, &block)
+        1.upto(ntry).each do |itry|
+          begin
+            block.call
+          rescue RuntimeError => err
+            raise err unless itry < ntry
+            @logger.error "Upload failed with error: #{err.message}"
+            @logger.error "Retry attempt #{itry}/#{ntry-1}"
+            sleep(1)
+          end
+        end
+      end
+
+
+    end
+  end
+end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.15'
+  VERSION = '0.2.16'
 end

data/remi.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.description = "Data manipulation and ETL in Ruby"
 
   s.rubyforge_project = "Remi"
-  s.add_runtime_dependency "daru", ["~> 0.1"]
+  s.add_runtime_dependency "daru", ["0.1.2"]
 
   s.add_runtime_dependency 'bond', ['~> 0.5']
   s.add_runtime_dependency 'docile', ['~> 1.1']

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.15
+  version: 0.2.16
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-23 00:00:00.000000000 Z
+date: 2016-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: bond
   requirement: !ruby/object:Gem::Requirement
@@ -225,10 +225,12 @@ files:
 - bin/remi
 - doc/install-rbenv-os_x.md
 - features/aggregate.feature
+- features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
 - features/parameters.feature
 - features/sample_job.feature
+- features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb
@@ -239,8 +241,10 @@ files:
 - jobs/aggregate_job.rb
 - jobs/all_jobs_shared.rb
 - jobs/copy_source_job.rb
+- jobs/csv_file_target_job.rb
 - jobs/parameters_job.rb
 - jobs/sample_job.rb
+- jobs/sftp_file_target_job.rb
 - jobs/transforms/date_diff_job.rb
 - jobs/transforms/nvl_job.rb
 - jobs/transforms/parse_date_job.rb
@@ -261,6 +265,7 @@ files:
 - lib/remi/data_target/csv_file.rb
 - lib/remi/data_target/data_frame.rb
 - lib/remi/data_target/salesforce.rb
+- lib/remi/data_target/sftp_file.rb
 - lib/remi/extractor/sftp_file.rb
 - lib/remi/field_symbolizers.rb
 - lib/remi/job.rb
@@ -300,10 +305,12 @@ specification_version: 4
 summary: Remi (Ruby Extract Map Integrate)
 test_files:
 - features/aggregate.feature
+- features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
 - features/parameters.feature
 - features/sample_job.feature
+- features/sftp_file_target_job.feature
 - features/step_definitions/remi_step.rb
 - features/support/env.rb
 - features/support/env_app.rb