stockboy 0.5.0

data/lib/stockboy/dsl.rb

@@ -0,0 +1,68 @@
+module Stockboy
+
+
+  # @api private
+  #
+  class ConfiguratorBlock
+
+    # Initialize a DSL context around an instance
+    #
+    def initialize(instance)
+      @instance = instance
+    end
+
+  end
+
+  # Mixin for defining DSL methods
+  #
+  module DSL
+
+    # Define ambiguous attr reader/writers for DSL readability
+    #
+    # @example
+    #   dsl.some_option = "new value" # => some_option = "new value"
+    #   dsl.some_option "new value"   # => some_option = "new value"
+    #   dsl.some_option               # => some_option
+    #
+    # @visibility private
+    # @scope class
+    #
+    def dsl_attr(attr, options={})
+      if options.fetch(:attr_accessor, true)
+        attr_reader attr if options.fetch(:attr_reader, true)
+        attr_writer attr if options.fetch(:attr_writer, true)
+      end
+
+      class_eval <<-___, __FILE__, __LINE__
+      class DSL < Stockboy::ConfiguratorBlock
+        def #{attr}(*arg)
+          if arg.empty?
+            @instance.#{attr}
+          else
+            @instance.#{attr} = arg.first
+          end
+        end
+        def #{attr}=(arg)
+          @instance.#{attr} = arg
+        end
+      end
+      ___
+
+      if attr_alias = options[:alias]
+        alias_method attr_alias, attr
+        alias_method :"#{attr_alias}=", :"#{attr}="
+
+        class_eval <<-___, __FILE__, __LINE__
+        class DSL < Stockboy::ConfiguratorBlock
+          alias_method :#{attr_alias}, :#{attr}
+          alias_method :#{attr_alias}=, :#{attr}=
+        end
+        ___
+      end
+
+      attr
+    end
+
+  end
+
+end

data/lib/stockboy/exceptions.rb

@@ -0,0 +1,3 @@
+module Stockboy
+  class OutOfSequence < StandardError; end
+end

data/lib/stockboy/filter.rb

@@ -0,0 +1,58 @@
+require 'stockboy/exceptions'
+
+module Stockboy
+
+  # Filters can be any callable object that returns true or false. This
+  # abstract class is a helpful way to define a commonly used filter pattern.
+  #
+  # == Interface
+  #
+  # Filter subclasses must define a +filter+ method that returns true or false
+  # when called with the record context.
+  #
+  # @example
+  #   class Bouncer < Stockboy::Filter
+  #     def initialize(age)
+  #       @age = age
+  #     end
+  #     def filter(input_context, output_context)
+  #       input_context["RawAge"].empty? or output_context.age < @age
+  #     end
+  #   end
+  #
+  #   Stockboy::Filters.register(:bouncer, Bouncer.new(19))
+  #   filter :under_age, :bouncer # in job template
+  #
+  #   Stockboy::Filters.register(:check_id, Bouncer)
+  #   filter :under_age, :bouncer, 19 # in job template
+  #
+  # @abstract
+  #
+  class Filter
+
+    # Return true to capture a filtered record, false to pass it on
+    #
+    # @param [SourceRecord] raw_context
+    #   Unmapped source fields with Hash-like access field names (e.g.
+    #   <tt>input["RawField"]</tt>) or raw values on mapped attributes as
+    #   methods (e.g. <tt>input.email</tt>)
+    # @param [MappedRecord] translated_context
+    #   Mapped and translated fields with access to attributes
+    #   as methods (<tt>output.email</tt>)
+    # @return [Boolean]
+    #
+    def call(raw_context, translated_context)
+      return !!filter(raw_context, translated_context)
+    end
+
+    private
+
+    # @abstract
+    #
+    def filter(raw_context, translated_context)
+      raise NoMethodError, "#{self.class}#filter must be implemented"
+    end
+
+  end
+
+end

data/lib/stockboy/filter_chain.rb

@@ -0,0 +1,41 @@
+module Stockboy
+
+  # A hash for executing items in order with callbacks
+  #
+  class FilterChain < Hash
+
+    # Initialize a new FilterChain with a hash of filters
+    #
+    # @param [Hash{Symbol=>Filter}] hash
+    #
+    def self.new(hash=nil)
+      super().replace(hash || {})
+    end
+
+    # Add filters to the front of the chain
+    #
+    # @param [Hash{Symbol=>Filter}] hash Filters to add
+    #
+    def prepend(hash)
+      replace hash.merge(self)
+    end
+
+    # Call the reset callback on all filters that respond to it
+    #
+    # @return [Hash{Symbol=>Array}] Filter keys point to empty arrays
+    #
+    def reset
+      each do |key, filter|
+        filter.reset if filter.respond_to? :reset
+      end
+      keys_to_arrays
+    end
+
+    # @return [Hash{Symbol=>Array}] Filter keys point to empty arrays
+    #
+    def keys_to_arrays
+      Hash[keys.map { |k| [k, []] }]
+    end
+
+  end
+end

data/lib/stockboy/filters.rb

@@ -0,0 +1,11 @@
+require 'stockboy/registry'
+
+module Stockboy
+
+  # Registry of available named filters
+  #
+  module Filters
+    extend Stockboy::Registry
+  end
+
+end

data/lib/stockboy/filters/missing_email.rb

@@ -0,0 +1,37 @@
+require 'stockboy/filter'
+
+module Stockboy::Filters
+
+  # Very loose matching to pre-screen missing emails.
+  #
+  # Only checks if there is a potential email-like string in the output value,
+  # and does not do any format checking for validity.
+  #
+  # @example
+  #   filter = Stockboy::Filters::MissingEmail.new(:addr)
+  #   model.email = ""
+  #   filter.call(_, model) # => false
+  #   model.email = "@"
+  #   filter.call(_, model) # => true
+  #
+  class MissingEmail < Stockboy::Filter
+
+    # Initialize a new filter for a missing email attribute
+    #
+    # @param [Symbol] attr
+    #   Name of the email attribute to examine on the mapped output record
+    #
+    def initialize(attr)
+      @attr = attr
+    end
+
+    private
+
+    def filter(raw,output)
+      value = output.send(@attr)
+      return true if value.blank?
+      return true unless value =~ /\w@\w/
+    end
+
+  end
+end

data/lib/stockboy/job.rb

@@ -0,0 +1,241 @@
+require 'stockboy/configuration'
+require 'stockboy/exceptions'
+require 'stockboy/configurator'
+require 'stockboy/template_file'
+require 'stockboy/filter_chain'
+require 'stockboy/candidate_record'
+
+module Stockboy
+
+  # This class wraps up the main interface for the process of fetching,
+  # parsing and sorting data. When used with a predefined template file, you
+  # can pass the name of the template to define it. This is the common way
+  # to use Stockboy:
+  #
+  #   job = Stockboy::Job.define('my_template')
+  #   if job.process
+  #     job.records[:update].each do |r|
+  #       # ...
+  #     end
+  #     job.records[:cancel].each do |r|
+  #       # ...
+  #     end
+  #   end
+  #
+  class Job
+
+    # Defines the data source for receiving data
+    #
+    # @return [Provider]
+    #
+    attr_accessor :provider
+
+    # Defines the format for parsing received data
+    #
+    # @return [Reader]
+    #
+    attr_accessor :reader
+
+    # Configures the mapping & translation of raw data fields
+    #
+    # @return [AttributeMap]
+    #
+    attr_reader :attributes
+
+    # List of filters for sorting processed records
+    #
+    # @return [FilterChain]
+    #
+    # Filters are applied in order, first match will capture the record.
+    # Records that don't match any
+    #
+    attr_reader :filters
+
+    attr_reader :triggers
+
+    # Lists of records grouped by filter key
+    #
+    # @return [Hash{Symbol=>Array}]
+    #
+    attr_reader :records
+
+    # List of records not matched by any filter
+    #
+    # @return [Array<CandidateRecord>]
+    #
+    attr_reader :unfiltered_records
+
+    # List of all records, filtered or not
+    #
+    # @return [Array<CandidateRecord>]
+    #
+    attr_reader :all_records
+
+    # Initialize a new job
+    #
+    # @param [Hash] params
+    # @option params [Provider]           :provider
+    # @option params [Reader]             :reader
+    # @option params [AttributeMap]       :attributes
+    # @option params [Array,FilterChain]  :filters
+    # @yield instance for further configuration or processing
+    #
+    def initialize(params={}, &block)
+      @provider   = params[:provider]
+      @reader     = params[:reader]
+      @attributes = params[:attributes]
+      @filters    = FilterChain.new params[:filters]
+      @triggers   = Hash.new { |h,k| h[k] = [] }
+      @triggers.replace params[:triggers] if params[:triggers]
+      yield self if block_given?
+      reset
+    end
+
+    # Instantiate a job configured by DSL template file
+    #
+    # @param template_name [String] File basename from template load path
+    # @yield instance for further configuration or processing
+    # @see Configuration#template_load_paths
+    #
+    def self.define(template_name)
+      return nil unless template = TemplateFile.read(template_name)
+      job = Configurator.new(template, TemplateFile.find(template_name)).to_job
+      yield job if block_given?
+      job
+    end
+
+    # Fetch data and process it into groups of filtered records
+    #
+    # @return [Boolean] Success or failure
+    #
+    def process
+      reset
+      with_query_caching do
+        load_records
+        yield @records if block_given?
+      end
+      provider.errors.empty?
+    end
+
+    # Count of all processed records
+    #
+    # @!attribute [r] total_records
+    # @return [Fixnum]
+    #
+    def total_records
+      @all_records.size
+    end
+
+    # Counts of processed records grouped by filter key
+    #
+    # @return [Hash{Symbol=>Fixnum}]
+    #
+    def record_counts
+      @records.reduce(Hash.new) { |a, (k,v)| a[k] = v.size; a }
+    end
+
+    def triggers=(new_triggers)
+      @triggers.replace new_triggers
+    end
+
+    def trigger(key, *args)
+      return nil unless triggers.key?(key)
+      triggers[key].each do |c|
+        c.call(self, *args)
+      end
+    end
+
+    def method_missing(name, *args)
+      if triggers.key?(name)
+        trigger(name, *args)
+      else
+        super
+      end
+    end
+
+    # Replace existing filters
+    #
+    # @param new_filters [Array]
+    # @return [Stockboy::FilterChain]
+    #
+    def filters=(new_filters)
+      @filters.replace new_filters
+      reset
+      @filters
+    end
+
+    # Replace existing attribute map
+    #
+    # @param new_attributes [Stockboy::AttributeMap]
+    # @return [Stockboy::AttributeMap]
+    #
+    def attributes=(new_attributes)
+      @attributes = new_attributes
+      reset
+      @attributes
+    end
+
+    # Has the job been processed successfully?
+    #
+    # @return [Boolean]
+    #
+    def processed?
+      !!@processed
+    end
+
+    # Overview of the job configuration; tries to be less noisy by hiding
+    # sub-element details.
+    #
+    # @return [String]
+    #
+    def inspect
+      prov = "provider=#{(Stockboy::Providers.all.key(provider.class) || provider.class.to_s).inspect}"
+      read = "reader=#{(Stockboy::Readers.all.key(reader.class) || reader.class.to_s).inspect}"
+      attr = "attributes=#{attributes.map(&:to)}"
+      filt = "filters=#{filters.keys}"
+      cnts = "record_counts=#{record_counts}"
+      "#<#{self.class}:#{self.object_id} #{[prov, read, attr, filt, cnts].join(', ')}>"
+    end
+
+    private
+
+    def reset
+      @records = filters.reset
+      @all_records = []
+      @unfiltered_records = []
+      @processed = false
+      true
+    end
+
+    def load_records
+      return unless provider.data
+
+      @all_records = reader.parse(provider.data).map do |row|
+        CandidateRecord.new(row, @attributes)
+      end
+
+      @all_records.each do |record|
+        record_partition(record) << record
+      end
+
+      @processed = true
+    end
+
+    def record_partition(record)
+      if key = record.partition(filters)
+        @records[key]
+      else
+        @unfiltered_records
+      end
+    end
+
+    def with_query_caching(&block)
+      if defined? ActiveRecord
+        ActiveRecord::Base.cache(&block)
+      else
+        yield
+      end
+    end
+
+  end
+end