stockboy 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 025d746c97ca5ded5e263969f5faf3fa215dbaa6
-  data.tar.gz: febec727068f490b15846a93912d625f91ad048b
+  metadata.gz: 64fb63c74d214447769a32a463ccab05bc92516e
+  data.tar.gz: c79abf6d768765673e66503d5ed5cab1d89dade7
 SHA512:
-  metadata.gz: 8fa25cb4d7326baaa6de53c0b4160663a5f0523e5dc7a6132904e6f11bac32d72b69d7328c7a563129b17b37440756d704be45d5f7cfe3b57673f32bdb25e1cd
-  data.tar.gz: 9dd0a0bd18b9b5b198f8aff1a90e649a3ffc8acb4b62b7aa4fd305908de6e8a3826f439ea4894a82f195692e14578513c84ced4942a5818b57eade3afd1f1687
+  metadata.gz: 55c4b8106f01b7f822915b4ce2e655c70afa4770a8e42794dea09345326d88a2a16edd00633fad2822a7054b7f58fee81064c00583ef93ccf0eaf849a63814de
+  data.tar.gz: f28bc29494c073b3b5570dc9294e089c25b15e549f8f4db5349dd0df0df310429a06d39e57dc5acf858ca90035f3bb85bb07c02a27adb4b08f6f92202f203799

data/.rspec

@@ -1,5 +1,3 @@
---backtrace
 --colour
 --format progress
 --order random
---profile

data/.travis.yml

@@ -5,6 +5,7 @@ env:
 rvm:
   - "1.9.3"
   - "2.0.0"
+  - "2.1.1"
   - "jruby-19mode"
   - "rbx"
 script: bundle exec rspec spec

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.7.0 / 2014-03-21
+
+* [FEATURE]     Add individual attribute mappings
+* [FEATURE]     Repeat provider requests to fetch paginated data
+* [ENHANCEMENT] More configurable SOAP options (@markedmondson)
+* Removed ActiveModel errors, configuration errors are simple arrays now
+
 ## 0.6.0 / 2014-02-06
 
 * [FEATURE]     Support HTTP basic authentication (@markedmondson)

data/README.md

@@ -99,10 +99,11 @@ Writing a job template requires you to declare three parts:
       file_pick  :first
     end
 
-    reader :csv do
-      headers true
-      col_sep "|"
-      encoding "Windows-1252"
+    repeat do |inputs, provider|
+      0.upto 12 do |m|
+        provider.file_dir = "reports/#{Date.today << 12-m}"
+        inputs << provider
+      end
     end
 
     attributes do
@@ -130,6 +131,29 @@ data, which is returned as a raw string blob. It can handle some complexity to
 determine which file to pick from an email attachment or FTP directory, for
 example.
 
+#### Fetching paginated data
+
+If the provider requires multiple queries to fetch all the data (e.g. http
+`?page=1` query params or a list of files), you can use an optional repeat
+block to specify how to iterate over the data source, and when to stop.
+Although it's possible to process each page as an individual job, the repeat
+option is useful when you need to have all the data in hand before making
+downstream processing decisions such as purging records that are not in the
+data set.
+
+    repeat do |inputs, http_provider|
+      loop do
+        inputs << http_provider
+        break if http_provider.data.split("\n").size < 100
+        http_provider.query["page"] += 1 
+      end
+    end
+
+For each iteration, increment the provider settings and push it onto the list
+of inputs. (This uses an Enumerator to yield each iteration's data before
+processing the next one, so it should be memory-efficient for long series of
+data sets.)
+
 See: [File][file], [FTP][ftp], [HTTP][http], [IMAP][imap], [SOAP][soap]
 
 [file]: lib/stockboy/providers/file.rb
@@ -209,6 +233,9 @@ so it's a good idea to have default values at the end of the chain.
 * [`:or_zero`][dzer]
   Returns `0` for blank values
 
+Attributes can be defined in a block as described, or added
+individually as `attribute :name`.
+
 [bool]: lib/stockboy/translators/boolean.rb
 [date]: lib/stockboy/translators/date.rb
 [deci]: lib/stockboy/translators/decimal.rb

data/lib/stockboy/attribute_map.rb

@@ -13,22 +13,11 @@ module Stockboy
     #
     class DSL
       def initialize(instance)
-        @instance = instance
-        @map = @instance.instance_variable_get(:@map)
+        @attribute_map = instance
       end
 
       def method_missing(attr, *args)
-        opts = args.first || {}
-        to = attr.to_sym
-        from = opts.fetch(:from, attr)
-        from = from.to_s.freeze if from.is_a? Symbol
-        translators = Array(opts[:as]).map { |t| Translations.translator_for(to, t) }
-        @map[attr] = Attribute.new(to, from, translators)
-        define_attribute_method(attr)
-      end
-
-      def define_attribute_method(attr)
-        (class << @instance; self end).send(:define_method, attr) { @map[attr] }
+        @attribute_map.insert(attr, *args)
       end
     end
 
@@ -40,7 +29,6 @@ module Stockboy
       if block_given?
         DSL.new(self).instance_eval(&block)
       end
-      freeze
     end
 
     # Retrieve an attribute by symbolic name
@@ -52,6 +40,22 @@ module Stockboy
       @map[key]
     end
 
+    # Add or replace a mapped attribute
+    #
+    # @param [Symbol] key Name of the output attribute
+    # @param [Hash] opts
+    # @option opts [String] from Name of input field from reader
+    # @option opts [Array,Proc,Translator] as One or more translators
+    #
+    def insert(key, opts={})
+      to = key.to_sym
+      from = opts.fetch(:from, key)
+      from = from.to_s.freeze if from.is_a? Symbol
+      translators = Array(opts[:as]).map { |t| Translations.translator_for(to, t) }
+      define_singleton_method(key) { @map[key] }
+      @map[key] = Attribute.new(to, from, translators)
+    end
+
     # Fetch the attribute corresponding to the source field name
     #
     # @param [String] key

data/lib/stockboy/candidate_record.rb

@@ -97,21 +97,20 @@ module Stockboy
     private
 
     def translate(col)
-      return sanitize(@table[col.from]) if col.translators.empty?
-      return @tr_table[col.to] if @tr_table.has_key? col.to
-      fields = self.raw_hash.dup
-      translated = col.translators.inject(input) do |m,t|
-        begin
-          new_value = t.call(m)
-        rescue
-          fields[col.to] = nil
-          break SourceRecord.new(fields, @table)
+      @tr_table.fetch(col.to) do |key|
+        return @tr_table[key] = sanitize(@table[col.from]) if col.translators.empty?
+        fields = raw_hash
+        tr_input = col.translators.reduce(input) do |value, tr|
+          begin
+            fields[col.to] = tr[value]
+            SourceRecord.new(fields, @table)
+          rescue
+            fields[col.to] = nil
+            break SourceRecord.new(fields, @table)
+          end
         end
-
-        fields[col.to] = new_value
-        SourceRecord.new(fields, @table)
+        @tr_table[col.to] = tr_input.public_send(col.to)
       end
-      @tr_table[col.to] = translated.public_send(col.to)
     end
 
     # Clean output values that are a subclass of a standard type

data/lib/stockboy/configurator.rb

@@ -40,11 +40,11 @@ module Stockboy
       end
     end
 
-    # DSL method for configuring the provider
+    # Configure the provider for fetching data
     #
     # The optional block is evaluated in the provider's own DSL context.
     #
-    # @param [Symbol, Class, Provider] provider_class
+    # @param [Symbol, Class, Provider] key
     #   The registered symbol name for the provider, or actual provider
     # @param [Hash] opts
     #   Provider-specific options passed to the provider initializer
@@ -56,24 +56,51 @@ module Stockboy
     #
     # @return [Provider]
     #
-    def provider(provider_class, opts={}, &block)
-      raise ArgumentError unless provider_class
+    def provider(key, opts={}, &block)
+      @config[:provider] = Providers.build(key, opts, block)
+    end
+    alias_method :connection, :provider
 
-      @config[:provider] = case provider_class
-      when Symbol
-        Providers.find(provider_class).new(opts, &block)
-      when Class
-        provider_class.new(opts, &block)
-      else
-        provider_class
+    # Configure repeating the provider for fetching multiple parts
+    #
+    # If the provider needs to give us all the data as a series of requests,
+    # for example multiple HTTP pages or FTP files, the repeat block can be
+    # used to define the iteration for fetching each item.
+    #
+    # The `<<` interface used here is defined by Ruby's Enumerator.new block
+    # syntax. For each page that needs to be fetched, the provider options need
+    # to be altered and pushed on to the output. Control will be yielded to the
+    # reader at each iteration.
+    #
+    # @example
+    #   repeat do |output, provider|
+    #     loop do
+    #       output << provider
+    #       break if provider.data.split("\n").size < 100
+    #       provider.query_params["page"] += 1
+    #     end
+    #   end
+    #
+    # @example
+    #   repeat do |output, provider|
+    #     1.upto 10 do |i|
+    #       provider.file_name = "example-#{i}.log"
+    #       output << provider
+    #     end
+    #   end
+    #
+    def repeat(&block)
+      unless block_given? && block.arity == 2
+        raise ArgumentError, "repeat block must accept |output, provider| arguments"
       end
+
+      @config[:repeat] = block
     end
-    alias_method :connection, :provider
 
-    # DSL method for configuring the reader
+    # Configure the reader for parsing data
     #
-    # @param [Symbol, Class, Reader] reader_class
-    #   The registered symbol name for the reader, or actual reader
+    # @param [Symbol, Class, Reader] key
+    #   The registered symbol name for the reader, or actual reader instance
     # @param [Hash] opts
     #   Provider-specific options passed to the provider initializer
     #
@@ -84,21 +111,14 @@ module Stockboy
     #
     # @return [Reader]
     #
-    def reader(reader_class, opts={}, &block)
-      raise ArgumentError unless reader_class
-
-      @config[:reader] = case reader_class
-      when Symbol
-        Readers.find(reader_class).new(opts, &block)
-      when Class
-        reader_class.new(opts, &block)
-      else
-        reader_class
-      end
+    def reader(key, opts={}, &block)
+      @config[:reader] = Readers.build(key, opts, block)
     end
     alias_method :format, :reader
 
-    # DSL method for configuring the attribute map in a block
+    # Configure the attribute map for data records
+    #
+    # This will replace any existing attributes with a new set.
     #
     # @example
     #   attributes do
@@ -113,7 +133,20 @@ module Stockboy
       @config[:attributes] = AttributeMap.new(&block)
     end
 
-    # DSL method to add a filter to the filter chain
+    # Add individual attribute mapping rules
+    #
+    # @param [Symbol] key Name of the output attribute
+    # @param [Hash] opts
+    # @option opts [String] from Name of input field from reader
+    # @option opts [Array,Proc,Translator] as One or more translators
+    #
+    #
+    def attribute(key, opts={})
+      @config[:attributes] ||= AttributeMap.new
+      @config[:attributes].insert(key, opts)
+    end
+
+    # Add a filter to the processing filter chain
     #
     # * Must be called with either a callable argument (proc) or a block.
     # * Must be called in the order that filters should be applied.
@@ -129,17 +162,10 @@ module Stockboy
     #   filter :update, proc{ true } # capture all remaining items
     #
     def filter(key, callable=nil, *args, &block)
-      raise ArgumentError unless key
-      if callable.is_a?(Symbol)
-        callable = Filters.find(callable)
-        callable = callable.new(*args) if callable.is_a? Class
-      end
-      raise ArgumentError unless callable.respond_to?(:call) ^ block_given?
-
-      @config[:filters][key] = block || callable
+      @config[:filters][key] = block || Filters.build(callable, args)
     end
 
-    # DSL method to register a trigger to notify the job of an event
+    # Register a trigger to notify the job of external events
     #
     # Useful for adding generic control over the job's resources from your app.
     # For example, if you need to record stats or clean up data after your
@@ -162,6 +188,7 @@ module Stockboy
     #
     def on(key, &block)
       raise(ArgumentError, "no block given") unless block_given?
+
       @config[:triggers][key] << block
     end
 
@@ -170,7 +197,18 @@ module Stockboy
     # @return [Job]
     #
     def to_job
-      Job.new(@config)
+      Job.new(config_for_job)
+    end
+
+    private
+
+    def config_for_job
+      config.dup.tap { |config| wrap_provider(config) }
+    end
+
+    def wrap_provider(config)
+      return unless (repeat = config.delete(:repeat))
+      config[:provider] = ProviderRepeater.new(config[:provider], &repeat)
     end
 
   end

data/lib/stockboy/filters.rb

@@ -6,6 +6,15 @@ module Stockboy
   #
   module Filters
     extend Stockboy::Registry
+
+    def self.build(callable, args)
+      if callable.is_a?(Symbol)
+        callable = find(callable)
+        callable = callable.new(*args) if callable.is_a? Class
+      end
+      callable
+    end
+
   end
 
 end

data/lib/stockboy/job.rb

@@ -109,7 +109,6 @@ module Stockboy
     # @return [Boolean] Success or failure
     #
     def process
-      reset
       with_query_caching do
         load_records
         yield @records if block_given?
@@ -208,17 +207,41 @@ module Stockboy
     end
 
     def load_records
-      return unless provider.data
+      reset
+      load_all_records
+      partition_all_records
+      @processed = true
+    end
 
-      @all_records = reader.parse(provider.data).map do |row|
-        CandidateRecord.new(row, @attributes)
+    def load_all_records
+      each_reader_row do |row|
+        @all_records << CandidateRecord.new(row, @attributes)
       end
+    end
 
+    def partition_all_records
       @all_records.each do |record|
         record_partition(record) << record
       end
+    end
 
-      @processed = true
+    def each_reader_row
+      return to_enum(__method__) unless block_given?
+      with_provider_data do |data|
+        reader.parse(data).each do |row|
+          yield row
+        end
+      end
+    end
+
+    def with_provider_data
+      return to_enum(__method__) unless block_given?
+      yielded = nil
+      provider.data do |data|
+        yielded = true
+        yield data
+      end
+      yield provider.data unless yielded
     end
 
     def record_partition(record)

data/lib/stockboy/provider.rb

@@ -1,6 +1,4 @@
 require 'logger'
-require 'active_model/errors'
-require 'active_model/naming'
 require 'stockboy/dsl'
 require 'stockboy/exceptions'
 
@@ -34,7 +32,6 @@ module Stockboy
   #
   class Provider
     extend Stockboy::DSL
-    extend ActiveModel::Naming # Required by ActiveModel::Errors
 
     # Default logger if none is provided to the instance
     #
@@ -48,7 +45,7 @@ module Stockboy
     #
     attr_accessor :logger
 
-    # @return [ActiveModel::Errors]
+    # @return [Array]
     #
     attr_reader :errors
 
@@ -61,7 +58,9 @@ module Stockboy
     # @return [String]
     #
     def inspect
-    "#<#{self.class}:#{self.object_id} data_size=#{@data_size or 'nil'} errors=#{@errors.full_messages}>"
+      "#<#{self.class}:#{self.object_id} "\
+      "data_size=#{@data_size.inspect} "\
+      "errors=[#{errors.join(", ")}]>"
     end
 
     # Must be called by subclasses via +super+ to set up dependencies
@@ -79,8 +78,8 @@ module Stockboy
     # @!attribute [r] data
     #
     def data
-      return @data if @data
-      fetch_data if validate_config?
+      fetch_data if @data.nil? && validate_config?
+      yield @data if block_given?
       @data
     end
 
@@ -92,7 +91,7 @@ module Stockboy
       @data = nil
       @data_time = nil
       @data_size = nil
-      @errors = ActiveModel::Errors.new(self)
+      @errors = []
       true
     end
     alias_method :reset, :clear
@@ -125,7 +124,7 @@ module Stockboy
       raise NoMethodError, "#{self.class}#fetch_data needs implementation"
     end
 
-    # Use errors.add(:attribute, "Message") provided by ActiveModel
+    # Use errors << "'option' is required"
     # for validating required provider parameters before attempting
     # to make connections and retrieve data.
     #
@@ -138,27 +137,12 @@ module Stockboy
     def validate_config?
       unless validation = valid?
         logger.error do
-          "Invalid #{self.class} provider configuration: #{errors.full_messages}"
+          "Invalid #{self.class} provider configuration: #{errors.join(', ')}"
         end
       end
       validation
     end
 
-    # Required by ActiveModel::Errors
-    def read_attribute_for_validation(attr)
-      send(attr)
-    end
-
-    # Required by ActiveModel::Errors
-    def self.human_attribute_name(attr, options = {})
-      attr
-    end
-
-    # Required by ActiveModel::Errors
-    def self.lookup_ancestors
-      [self]
-    end
-
     # When picking files from a list you can supply +:first+ or +:last+ to the
     # provider's +pick+ option, or else a block that can reduce to a single
     # value, like: