stockboy 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5fcae8abd12ed8674c56d476860be0ed4469ee13
-  data.tar.gz: b0f49fa1a87c62dfd39184161cea8afcc7185b6c
+  metadata.gz: b36c08f870f1306e9c36cb125438bf8e01b3e6dc
+  data.tar.gz: e997f70d91635b9456ed6c0758e981ea7f2e16ed
 SHA512:
-  metadata.gz: cf10196314b5e62dc785316a95741d3034772ca197defec68e032e570fece15a0de0f87bbc2888a9e35c2696f07d21f0a5dde7398e4ba5d24007029ea1040190
-  data.tar.gz: fba95677551b2a57e5915c79b0cc13432168954772369e36ec54fd30e377aa5b6f728393586d9b190d0624f4e2b1da27af01729064b039bda6ae432caf05aed0
+  metadata.gz: 0a129da6b911fa8812706140192419782ce1e09e5ea26bc8a161dcf1006a918ff7ab6cbd94a94e1b6f4c279d9d092146e87386b5dd64a837980a03ba2682e48a
+  data.tar.gz: e5fd24fb0698cfb2d8488f35587ea49b0ac2d7bc0d39422a17fd5218a0974429fb1baf214e5f00f5295ec4ba309dade51e5a9a1d2b9048e5a9b0db6a51e19773

data/.travis.yml

@@ -3,22 +3,20 @@ bundler_args: --without debug --without doc
 env:
   - CI=true
 rvm:
-  - "1.9.3"
-  - "2.0.0"
-  - "2.1"
-  - "2.2"
-  - "rbx"
-  - "jruby-head"
+  - 2.2
+  - 2.3
+  - 2.4
+  - ruby-head
+  - jruby-head
 before_install:
   - gem install bundler
-script: bundle exec rspec spec
+script:
+  - bundle exec rspec spec
+  - bundle exec codeclimate-test-reporter
 matrix:
   fast_finish: true
   allow_failures:
-    - rvm: "1.9.3"
-    - rvm: "2.0.0"
-    - rvm: "rbx"
-    - rvm: "jruby-head"
+    - rvm: jruby-head
 addons:
   code_climate:
     repo_token: 7591072ad04d7a8ee788ce9259baa284724740cab1ae9ca94c162a583acd8f10

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 1.3.0 / 2017-10-26
+
+* [FEATURE]     Add `secure` option for FTP to use SFTP
+* [FEATURE]     Update test matrix for recent ruby versions
+* [ENHANCEMENT] Log SOAP provider output on debug level
+* [ENHANCEMENT] Let pick accept proc with single arg for list
+* [BUGFIX]      Fix defaults for spreadsheet row options
+* [BUGFIX]      Fix spreadsheet option initialization
+
 ## 1.2.1 / 2016-08-02
 
 * [BUGFIX]      Repeater would error with zero yielded data iterations

data/README.md

@@ -236,7 +236,7 @@ with the "as" option.
 * [`:decimal`][deci]
   Numeric strings to `BigDecimal` (e.g. prices)
 * [`:integer`][intg]
-  Numeric strings to `Fixnum` integers
+  Numeric strings to `Integer` integers
 * [`:string`][stri]
   Clean strings with leading/trailing whitespace trimmed
 * [`:or_empty`][dest]

data/lib/stockboy/configuration.rb

@@ -42,7 +42,7 @@ module Stockboy
       @template_load_paths = []
       @logger = Logger.new(STDOUT)
       @tmp_dir = Dir.tmpdir
-      @translation_error_handler = -> (error) { nil }
+      @translation_error_handler = ->(error) { nil }
       yield self if block_given?
     end
   end

data/lib/stockboy/configurator.rb

@@ -1,4 +1,3 @@
-require 'stockboy/job'
 require 'stockboy/providers'
 require 'stockboy/readers'
 require 'stockboy/filters'

data/lib/stockboy/exceptions.rb

@@ -1,20 +1,24 @@
 module Stockboy
   class OutOfSequence < StandardError; end
 
-  # TranslationError is a wrapper to store the standard error as well as the key and record which caused it
-  class TranslationError < StandardError 
+  class TranslationError < StandardError
 
-     def initialize (key, record)
+    def initialize (key, record)
       @key = key
       @record = record
-     end
-
-     def message
-      "Attribute [#{key}] caused #{cause.message}"
-     end
+      @cause = $!
+    end
 
-     attr_reader :key
-     attr_reader :record
+    def message
+      reason = @cause && @cause.message || super
+      "Attribute [#{key}] caused #{reason}"
+    end
 
+    def backtrace
+      @cause && @cause.backtrace || super
     end
+
+    attr_reader :key
+    attr_reader :record
+  end
 end

data/lib/stockboy/job.rb

@@ -127,7 +127,7 @@ module Stockboy
     # Count of all processed records
     #
     # @!attribute [r] total_records
-    # @return [Fixnum]
+    # @return [Integer]
     #
     def total_records
       @all_records.size
@@ -135,7 +135,7 @@ module Stockboy
 
     # Counts of processed records grouped by filter key
     #
-    # @return [Hash{Symbol=>Fixnum}]
+    # @return [Hash{Symbol=>Integer}]
     #
     def record_counts
       @records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
@@ -267,9 +267,9 @@ module Stockboy
       end
     end
 
-    def with_query_caching(&block)
+    def with_query_caching
       if defined? ActiveRecord
-        ActiveRecord::Base.cache(&block)
+        ActiveRecord::Base.cache { yield }
       else
         yield
       end

data/lib/stockboy/mapped_record.rb

@@ -48,12 +48,5 @@ module Stockboy
       @fields = fields
       freeze
     end
-
-    # @return [String]
-    #
-    def to_s
-      @fields.to_s
-    end
-
   end
 end

data/lib/stockboy/provider.rb

@@ -162,7 +162,7 @@ module Stockboy
       when Symbol
         list.public_send @pick
       when Proc
-        list.reduce &@pick
+        @pick.arity == 1 ? @pick.call(list) : list.reduce(&@pick)
       end
     end
 
@@ -216,14 +216,14 @@ module Stockboy
   #
   #   @!attribute [rw] file_smaller
   #     Validates the maximum data size for the matched file, in bytes
-  #     @return [Fixnum]
+  #     @return [Integer]
   #     @macro provider.pick_validation
   #     @example
   #       file_smaller 1024^3
   #
   #   @!attribute [rw] file_larger
   #     Validates the minimum file size for the matched file, in bytes. This can # help guard against processing zero-byte or truncated files.
-  #     @return [Fixnum]
+  #     @return [Integer]
   #     @macro provider.pick_validation
   #     @example
   #       file_larger 1024

data/lib/stockboy/provider_repeater.rb

@@ -64,7 +64,6 @@ module Stockboy
     def clear
       @base_provider = @orig_provider.dup
       @iterations.clear
-      super
     end
 
     def each

data/lib/stockboy/providers/ftp.rb

@@ -1,5 +1,4 @@
 require 'stockboy/provider'
-require 'net/ftp'
 
 module Stockboy::Providers
 
@@ -23,6 +22,8 @@ module Stockboy::Providers
   #   end
   #
   class FTP < Stockboy::Provider
+    require_relative 'ftp/ftp_adapter'
+    require_relative 'ftp/sftp_adapter'
 
     # @!group Options
 
@@ -71,6 +72,15 @@ module Stockboy::Providers
     #
     dsl_attr :binary
 
+    # Use SFTP protocol for file transfers
+    #
+    # @!attribute [rw] secure
+    # @return [Boolean]
+    # @example
+    #   secure true
+    #
+    dsl_attr :secure
+
     # @macro provider.file_options
     dsl_attr :file_name
     dsl_attr :file_dir
@@ -91,6 +101,7 @@ module Stockboy::Providers
       @passive      = opts[:passive]
       @username     = opts[:username]
       @password     = opts[:password]
+      @secure       = opts[:secure]
       @binary       = opts[:binary]
       @file_dir     = opts[:file_dir]
       @file_name    = opts[:file_name]
@@ -99,21 +110,25 @@ module Stockboy::Providers
       @file_larger  = opts[:file_larger]
       @pick         = opts[:pick] || :last
       DSL.new(self).instance_eval(&block) if block_given?
+      @open_client  = nil
+    end
+
+    def adapter_class
+      secure ? SFTPAdapter : FTPAdapter
     end
 
     def client
       return yield @open_client if @open_client
 
-      Net::FTP.open(host, username, password) do |ftp|
-        ftp.binary = binary
-        ftp.passive = passive
-        ftp.chdir file_dir if file_dir
+      adapter_class.new(self).open do |ftp|
         @open_client = ftp
+        ftp.chdir file_dir if file_dir
         response = yield ftp
         @open_client = nil
         response
       end
-    rescue Net::FTPError => e
+
+    rescue adapter_class.exception_class => e
       errors << e.message
       logger.warn e.message
       nil
@@ -122,7 +137,7 @@ module Stockboy::Providers
     def matching_file
       return @matching_file if @matching_file
       client do |ftp|
-        file_listing = ftp.nlst.sort
+        file_listing = ftp.list_files
         @matching_file = pick_from file_listing.select(&file_name_matcher)
       end
     end
@@ -150,7 +165,7 @@ module Stockboy::Providers
         validate_file(matching_file)
         if valid?
           logger.debug "FTP getting file #{inspect_matching_file}"
-          @data = ftp.get(matching_file, nil)
+          @data = ftp.download(matching_file)
           logger.debug "FTP got file #{inspect_matching_file} (#{data_size} bytes)"
         end
       end
@@ -184,7 +199,7 @@ module Stockboy::Providers
     end
 
     def validate_file_newer(data_file)
-      @data_time ||= client { |ftp| ftp.mtime(data_file) }
+      @data_time ||= client { |ftp| ftp.modification_time(data_file) }
       if file_newer and @data_time < file_newer
         errors << "No new files since #{file_newer}"
       end

data/lib/stockboy/providers/ftp/ftp_adapter.rb

@@ -0,0 +1,50 @@
+require 'net/ftp'
+
+module Stockboy::Providers
+  class FTP::FTPAdapter
+    attr_reader :client
+
+    def initialize(provider)
+      @provider = provider
+    end
+
+    def open
+      result = nil
+      Net::FTP.open(@provider.host, @provider.username, @provider.password) do |ftp|
+        @client = ftp
+        client.binary = @provider.binary
+        client.passive = @provider.passive
+        result = yield self
+      end
+      result
+    end
+
+    def chdir(directory)
+      client.chdir directory
+    end
+
+    def list_files
+      client.nlst.sort
+    end
+
+    def delete(file_name)
+      client.delete file_name
+    end
+
+    def download(file_name)
+      client.get(file_name, nil)
+    end
+
+    def modification_time(file_name)
+      client.mtime file_name
+    end
+
+    def size(file_name)
+      client.size file_name
+    end
+
+    def self.exception_class
+      Net::FTPError
+    end
+  end
+end

data/lib/stockboy/providers/ftp/sftp_adapter.rb

@@ -0,0 +1,57 @@
+require 'net/sftp'
+
+module Stockboy::Providers
+  class FTP::SFTPAdapter
+    attr_reader :client
+
+    def initialize(provider)
+      @provider = provider
+      @file_dir = "."
+    end
+
+    def open
+      result = nil
+      Net::SFTP.start(@provider.host, @provider.username, password: @provider.password) do |sftp|
+        @client = sftp
+        result = yield self
+      end
+      result
+    end
+
+    def chdir(directory)
+      @file_dir = ::File.join(directory, '')
+    end
+
+    def list_files
+      client.dir.entries(@file_dir).map(&:name).sort
+    end
+
+    def delete(file_name)
+      client.remove!(full_path(file_name))
+    end
+
+    def download(file_name)
+      client.download!(full_path(file_name))
+    end
+
+    def full_path(file_name)
+      ::File.join(@file_dir, file_name)
+    end
+
+    def modification_time(file_name)
+      stat(file_name).mtime
+    end
+
+    def size(file_name)
+      stat(file_name).size
+    end
+
+    def stat(file_name)
+      client.file.open(full_path(file_name)).stat
+    end
+
+    def self.exception_class
+      Net::SFTP::Exception
+    end
+  end
+end

data/lib/stockboy/providers/http.rb

@@ -120,14 +120,6 @@ module Stockboy::Providers
       @uri = uri
     end
 
-    def username=(username)
-      @username = username
-    end
-
-    def password=(password)
-      @password = password
-    end
-
     # @!endgroup
 
     # Initialize an HTTP provider

data/lib/stockboy/providers/imap.rb

@@ -143,6 +143,7 @@ module Stockboy::Providers
       @file_larger  = opts[:file_larger]
       @pick         = opts[:pick] || :last
       DSL.new(self).instance_eval(&block) if block_given?
+      @open_client  = nil
     end
 
     # Direct access to the configured +Net::IMAP+ connection
@@ -161,7 +162,7 @@ module Stockboy::Providers
         @open_client.examine(mailbox)
       end
       yield @open_client
-    rescue ::Net::IMAP::Error => e
+    rescue ::Net::IMAP::Error
       errors << "IMAP connection error"
     ensure
       if first_connection && @open_client
@@ -190,7 +191,7 @@ module Stockboy::Providers
     #
     def message_key
       return @message_key if @message_key
-      message_ids = search(default_search_options)
+      message_ids = find_messages(default_search_options)
       @message_key = pick_from(message_ids) unless message_ids.empty?
     end
 
@@ -210,11 +211,11 @@ module Stockboy::Providers
     #   Override default configured search options
     #
     # @example
-    #   provider.search(subject: "Daily Report", before: Date.today)
-    #   provider.search(["SUBJECT", "Daily Report", "BEFORE", "21-DEC-12"])
-    #   provider.search("FLAGGED BEFORE 21-DEC-12")
+    #   provider.find_messages(subject: "Daily Report", before: Date.today)
+    #   provider.find_messages(["SUBJECT", "Daily Report", "BEFORE", "21-DEC-12"])
+    #   provider.find_messages("FLAGGED BEFORE 21-DEC-12")
     #
-    def search(options=nil)
+    def find_messages(options=nil)
       client { |imap| imap.sort(['DATE'], search_keys(options), 'UTF-8') }
     end
 
@@ -261,10 +262,10 @@ module Stockboy::Providers
     end
 
     def open_attachment(mail)
-      part = mail.attachments.detect { |part| validate_attachment(part) }
-      validate_file(part) if part or return
-      yield part.decoded if valid?
-      part
+      file = mail.attachments.detect { |part| validate_attachment(part) }
+      validate_file(file) if file or return
+      yield file.decoded if valid?
+      file
     end
 
     def validate