stockboy 0.5.0

data/lib/stockboy/attribute.rb

@@ -0,0 +1,11 @@
+module Stockboy
+
+  # Struct-like value object for holding mapping & translation details from
+  # input data fields
+  #
+  class Attribute < Struct.new(:to, :from, :translators)
+    def inspect
+      "#<Stockboy::Attribute to=#{to.inspect}, from=#{from.inspect}, translators=#{translators}>"
+    end
+  end
+end

data/lib/stockboy/attribute_map.rb

@@ -0,0 +1,74 @@
+require 'stockboy/attribute'
+require 'stockboy/translations'
+
+module Stockboy
+
+  # Table of attributes for finding corresponding field/attribute translations
+  #
+  class AttributeMap
+
+    include Enumerable
+
+    # @visibility private
+    #
+    class DSL
+      def initialize(instance)
+        @instance = instance
+        @map = @instance.instance_variable_get(:@map)
+      end
+
+      def method_missing(attr, *args)
+        opts = args.first || {}
+        to = attr.to_sym
+        from = opts.fetch(:from, attr)
+        from = from.to_s.freeze if from.is_a? Symbol
+        translators = Array(opts[:as]).map { |t| Translations.translator_for(to, t) }
+        @map[attr] = Attribute.new(to, from, translators)
+        define_attribute_method(attr)
+      end
+
+      def define_attribute_method(attr)
+        (class << @instance; self end).send(:define_method, attr) { @map[attr] }
+      end
+    end
+
+    # Initialize a new attribute map
+    #
+    def initialize(rows={}, &block)
+      @map = rows
+      @unmapped = Hash.new
+      if block_given?
+        DSL.new(self).instance_eval(&block)
+      end
+      freeze
+    end
+
+    # Retrieve an attribute by symbolic name
+    #
+    # @param [Symbol] key
+    # @return [Attribute]
+    #
+    def [](key)
+      @map[key]
+    end
+
+    # Fetch the attribute corresponding to the source field name
+    #
+    # @param [String] key
+    # @return [Attribute]
+    #
+    def attribute_from(key)
+      find { |a| a.from == key } or @unmapped[key] ||= Attribute.new(nil, key, nil)
+    end
+
+    # Enumerate over attributes
+    #
+    # @return [Enumerator]
+    # @yieldparam [Attribute]
+    #
+    def each(*args, &block)
+      @map.values.each(*args, &block)
+    end
+
+  end
+end

data/lib/stockboy/candidate_record.rb

@@ -0,0 +1,130 @@
+require 'stockboy/attribute_map'
+require 'stockboy/mapped_record'
+require 'stockboy/source_record'
+require 'stockboy/translations'
+
+module Stockboy
+
+  # Joins the raw data values to an attribute mapping to allow comparison of
+  # input/output values, conversion, and filtering
+  #
+  class CandidateRecord
+
+    # Initialize a new candidate record
+    #
+    # @param [Hash] attrs Raw key-values from source data
+    # @param [AttributeMap] map Mapping and translations
+    #
+    def initialize(attrs, map)
+      @map = map
+      @table = use_frozen_keys(attrs, map)
+      @tr_table = Hash.new
+      freeze
+    end
+
+    # Convert the mapped output to a hash
+    #
+    # @return [Hash]
+    #
+    def to_hash
+      Hash.new.tap do |out|
+        @map.each { |col| out[col.to] = translate(col) }
+      end
+    end
+    alias_method :attributes, :to_hash
+
+    # Return the original values mapped to attribute keys
+    #
+    # @return [Hash]
+    #
+    def raw_hash
+      Hash.new.tap do |out|
+        @map.each { |col| out[col.to] = @table[col.from] }
+      end
+    end
+    alias_method :raw_attributes, :raw_hash
+
+    # Wrap the mapped attributes in a new ActiveModel or ActiveRecord object
+    #
+    # @param [Class] model ActiveModel class
+    # @return [Class] ActiveModel class
+    #
+    def to_model(model)
+      model.new(attributes)
+    end
+
+    # Find the filter key that captures this record
+    #
+    # @param [FilterChain] filters List of filters to apply
+    # @return [Symbol] Name of the matched filter
+    #
+    def partition(filters={})
+      input, output = self.input, self.output
+      filters.each_pair do |filter_key, f|
+        if f.call(input, output)
+          return filter_key
+        end
+      end
+      nil
+    end
+
+    # Data structure representing the record's raw input values
+    #
+    # Values can be accessed like hash keys, or attribute names that correspond
+    # to a +:from+ attribute mapping option
+    #
+    # @return [SourceRecord]
+    # @example
+    #   input = candidate.input
+    #   input["RawEmail"] # => "ME@EXAMPLE.COM  "
+    #   input.email       # => "ME@EXAMPLE.COM  "
+    #
+    def input
+      SourceRecord.new(self.raw_hash, @table)
+    end
+
+    # Data structure representing the record's mapped & translated output values
+    #
+    # @return [MappedRecord]
+    # @example
+    #   input = candidate.output
+    #   output.email       # => "me@example.com"
+    #
+    def output
+      MappedRecord.new(self.to_hash)
+    end
+
+    private
+
+    def translate(col)
+      return sanitize(@table[col.from]) if col.translators.empty?
+      return @tr_table[col.to] if @tr_table.has_key? col.to
+      fields = self.raw_hash.dup
+      translated = col.translators.inject(input) do |m,t|
+        begin
+          new_value = t.call(m)
+        rescue
+          fields[col.to] = nil
+          break SourceRecord.new(fields, @table)
+        end
+
+        fields[col.to] = new_value
+        SourceRecord.new(fields, @table)
+      end
+      @tr_table[col.to] = translated.public_send(col.to)
+    end
+
+    def sanitize(value)
+      value.is_a?(String) ? value.to_s : value
+    end
+
+    def use_frozen_keys(attrs, map)
+      attrs.reduce(Hash.new) do |new_hash, (field, value)|
+        key = map.attribute_from(field).from
+        new_hash[key] = value
+        new_hash
+      end
+    end
+
+  end
+end

data/lib/stockboy/configuration.rb

@@ -0,0 +1,62 @@
+module Stockboy
+
+  # Global Stockboy configuration options
+  #
+  class Configuration
+
+    # Directories where Stockboy job template files can be found.
+    #
+    # Needs to be configured with your own paths if running standalone.
+    # When running with Rails, includes +config/stockboy_jobs+ by default.
+    #
+    # @return [Array<String>]
+    #
+    attr_accessor :template_load_paths
+
+    # Path for storing tempfiles during processing
+    #
+    # @return [String]
+    #
+    attr_accessor :tmp_dir
+
+    # Default logger
+    #
+    # @return [Logger]
+    #
+    attr_accessor :logger
+
+    # Initialize a set of global configuration options
+    #
+    # @yield self for configuration
+    #
+    def initialize
+      @template_load_paths = []
+      @logger = Logger.new(STDOUT)
+      @tmp_dir = Dir.tmpdir
+      yield self if block_given?
+    end
+  end
+
+  class << self
+
+    # Stockboy configuration block
+    #
+    # @example
+    #   Stockboy.configure do |config|
+    #     config.template_load_paths << "config/my_templates"
+    #   end
+    #
+    # @scope class
+    # @yield self for configuration
+    # @return [Configuration]
+    #
+    def configure
+      @configuration ||= Configuration.new
+      yield @configuration if block_given?
+      @configuration
+    end
+    alias_method :configuration, :configure
+
+  end
+
+end

data/lib/stockboy/configurator.rb

@@ -0,0 +1,176 @@
+require 'stockboy/job'
+require 'stockboy/providers'
+require 'stockboy/readers'
+require 'stockboy/filters'
+require 'stockboy/attribute_map'
+
+module Stockboy
+
+  # Context for evaluating DSL templates and capturing job options for
+  # initializing a job.
+  #
+  # Wraps up the DSL methods called in job templates and handles the construction
+  # of the job's +provider+, +reader+, +attributes+, and +filters+.
+  #
+  class Configurator
+
+    # Captured job configuration options
+    #
+    # @return [Hash]
+    #
+    attr_reader :config
+
+    # Evaluate DSL and capture configuration for building a job
+    #
+    # @overload new(dsl, file=__FILE__)
+    #   Evaluate DSL from a string
+    #   @param [String] dsl   job template language for evaluation
+    #   @param [String] file  path to original file for reporting errors
+    # @overload new(&block)
+    #   Evaluate DSL in a block
+    #
+    def initialize(dsl='', file=__FILE__, &block)
+      @config = {}
+      @config[:triggers] = Hash.new { |hash, key| hash[key] = [] }
+      @config[:filters] = {}
+      if block_given?
+        instance_eval(&block)
+      else
+        instance_eval(dsl, file)
+      end
+    end
+
+    # DSL method for configuring the provider
+    #
+    # The optional block is evaluated in the provider's own DSL context.
+    #
+    # @param [Symbol, Class, Provider] provider_class
+    #   The registered symbol name for the provider, or actual provider
+    # @param [Hash] opts
+    #   Provider-specific options passed to the provider initializer
+    #
+    # @example
+    #   provider :file, file_dir: "/downloads/@client" do
+    #     file_name "example.csv"
+    #   end
+    #
+    # @return [Provider]
+    #
+    def provider(provider_class, opts={}, &block)
+      raise ArgumentError unless provider_class
+
+      @config[:provider] = case provider_class
+      when Symbol
+        Providers.find(provider_class).new(opts, &block)
+      when Class
+        provider_class.new(opts, &block)
+      else
+        provider_class
+      end
+    end
+    alias_method :connection, :provider
+
+    # DSL method for configuring the reader
+    #
+    # @param [Symbol, Class, Reader] reader_class
+    #   The registered symbol name for the reader, or actual reader
+    # @param [Hash] opts
+    #   Provider-specific options passed to the provider initializer
+    #
+    # @example
+    #   reader :csv do
+    #     col_sep "|"
+    #   end
+    #
+    # @return [Reader]
+    #
+    def reader(reader_class, opts={}, &block)
+      raise ArgumentError unless reader_class
+
+      @config[:reader] = case reader_class
+      when Symbol
+        Readers.find(reader_class).new(opts, &block)
+      when Class
+        reader_class.new(opts, &block)
+      else
+        reader_class
+      end
+    end
+    alias_method :format, :reader
+
+    # DSL method for configuring the attribute map in a block
+    #
+    # @example
+    #   attributes do
+    #     first_name as: ->(raw){ raw["FullName"].split(" ").first }
+    #     email      from: "RawEmail", as: [:string]
+    #     check_in   from: "RawCheckIn", as: [:date]
+    #   end
+    #
+    def attributes(&block)
+      raise ArgumentError unless block_given?
+
+      @config[:attributes] = AttributeMap.new(&block)
+    end
+
+    # DSL method to add a filter to the filter chain
+    #
+    # * Must be called with either a callable argument (proc) or a block.
+    # * Must be called in the order that filters should be applied.
+    #
+    # @example
+    #   filter :missing_email do |raw, out|
+    #     raw["RawEmail"].empty?
+    #   end
+    #   filter :past_due do |raw, out|
+    #     out.check_in < Date.today
+    #   end
+    #   filter :under_age, :check_id
+    #   filter :update, proc{ true } # capture all remaining items
+    #
+    def filter(key, callable=nil, *args, &block)
+      raise ArgumentError unless key
+      if callable.is_a?(Symbol)
+        callable = Filters.find(callable)
+        callable = callable.new(*args) if callable.is_a? Class
+      end
+      raise ArgumentError unless callable.respond_to?(:call) ^ block_given?
+
+      @config[:filters][key] = block || callable
+    end
+
+    # DSL method to register a trigger to notify the job of an event
+    #
+    # Useful for adding generic control over the job's resources from your app.
+    # For example, if you need to record stats or clean up data after your
+    # application has successfully processed the records, these actions can be
+    # defined within the context of each job template.
+    #
+    # @param [Symbol] key Name of the trigger
+    # @param [Trigger, Proc, #call] trigger_class
+    #
+    # @example
+    #   trigger :cleanup do |job, *args|
+    #     job.provider.delete_data
+    #   end
+    #
+    #   # elsewhere:
+    #   if MyProjects.find(123).import_records(job.records[:valid])
+    #     job.cleanup
+    #   end
+    #
+    def on(key, &block)
+      raise(ArgumentError, "no block given") unless block_given?
+      @config[:triggers][key] << block
+    end
+
+    # Initialize a new job with the captured options
+    #
+    # @return [Job]
+    #
+    def to_job
+      Job.new(@config)
+    end
+
+  end
+end