data_miner 1.3.8 → 2.0.1

data/CHANGELOG

@@ -1,3 +1,45 @@
+2.0.1 / 2012-04-18
+
+* Enhancements
+
+  * DataMiner.run -> DataMiner.perform
+  * Some basic tests that don't rely on Earth
+
+* Bug fixes
+
+  * Fix the "call stack" - the thing that keeps infinite loops from occurring
+  * Make sure sources get refreshed every time you re-run data miner
+  * Make sure dictionaries " " " " " " "
+
+2.0.0 / 2012-04-17
+
+* Breaking changes
+
+  * Renamed data_miner_config to data_miner_script (etc. for class/method naming)
+  * Simplify DataMiner.run arguments
+      was: DataMiner.run(:resource_names => ['Country'])
+      now: DataMiner.run(['Country'])
+  * Rename "resources" to "models"
+      was: DataMiner.resource_names
+      now: DataMiner.model_names
+  * Expect procs instead of lambdas (because they are just instance-eval'ed now, Blockenspiel is no longer guessing where to find methods)
+      was: :synthesize => lambda { class_method }
+      now: :synthesize => proc { Klass.class_method }
+  * Use UnicodeUtils to correctly upcase and downcase
+  * Use throw/catch instead of exceptions to signal to force a step to stop successfully
+      was: DataMiner::Succeed
+      now: throw :data_miner_succeed
+  * Import steps no longer accept deprecated :table => RemoteTable or :errata => Errata options
+  * DataMiner::Run structure has changed (in addition to other internals)
+
+* Enhancements
+
+  * Easy to modify data miner scripts using DataMiner::Script#{append|prepend|append_once|prepend_once}
+  * DRYer codebase
+  * No longer depends on Blockenspiel
+  * Uses UnixUtils instead of its own spawning code
+  * Should be threadsafe (no more autoload, has mutexes, more careful/fewer singletons, etc.)
+
 1.1.0
 * fixed dependency issues
 1.0.0

data/Gemfile

@@ -1,4 +1,20 @@
-source "http://rubygems.org"
+source :rubygems
 
-# Specify your gem's dependencies in data_miner.gemspec
-gemspec :path => '.'
+gemspec
+
+# development dependencies
+gem 'fuzzy_match'
+gem 'minitest'
+gem 'minitest-reporters'
+gem 'mysql2'
+gem 'rake'
+gem 'yard'
+gem 'earth'
+if RUBY_VERSION >= '1.9'
+  gem 'unicode_utils'
+end
+# if RUBY_VERSION >= '1.9'
+#  gem 'ruby-debug19' # replace with debugger?
+# else
+#   gem 'ruby-debug'
+# end

data/README.rdoc

@@ -9,7 +9,7 @@ Programmatically import useful data into your ActiveRecord models.
 You define <tt>data_miner</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
 
     class Country < ActiveRecord::Base
-      set_primary_key :iso_3166_code
+      self.primary_key =  :iso_3166_code
   
       data_miner do    
         import 'the official ISO country list',
@@ -62,7 +62,7 @@ This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA
     # Tell ActiveRecord that we want to use a string primary key.
     # This makes it easier to repeatedly truncate and re-import this
     # table without breaking associations.
-    set_primary_key :icao_code
+    self.primary_key =  :icao_code
   
     # Use the mini_record-compat gem to define the database schema in-line.
     # It will destructively and automatically add/remove columns.
@@ -268,7 +268,7 @@ This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA
         update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])).to_sql})"
       end
     
-      # And finally re-run the import of resources that depend on this resource.
+      # And finally re-run the import of resources that depend on this model.
       # Don't worry about calling Aircraft.run_data_miner! at the top of AircraftManufacturer's data_miner block;
       # that's the right way to do dependencies. It won't get called twice in the same run.
       [ AircraftManufacturer ].each do |synthetic_resource|

data/Rakefile

@@ -1,27 +1,25 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
 
 require 'rake'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
+  test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
 end
 
-task :default => :test
-
-begin
-  require 'rake/rdoctask'
-  Rake::RDocTask.new do |rdoc|
-    rdoc.rdoc_dir = 'rdoc'
-    rdoc.title = 'data_miner'
-    rdoc.options << '--line-numbers' << '--inline-source'
-    rdoc.rdoc_files.include('README*')
-    rdoc.rdoc_files.include('lib/**/*.rb')
+task :test_separately do
+  Dir[File.expand_path('../test/**/test_*.rb', __FILE__)].each do |path|
+    system "rake test TEST=#{path}"
   end
-rescue LoadError
-   puts "Rdoc is not available"
+end
+
+task :default => :test_separately
+
+require 'yard'
+YARD::Rake::YardocTask.new do |y|
+  y.options << '--no-private'
 end
 
 gemspec = eval(File.read(Dir["*.gemspec"].first))

data/data_miner.gemspec

@@ -1,11 +1,9 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "data_miner/version"
+require File.expand_path("../lib/data_miner/version", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "data_miner"
   s.version     = DataMiner::VERSION
-  s.platform    = Gem::Platform::RUBY
   s.authors     = ["Seamus Abshere", "Andy Rossmeissl", "Derek Kastner"]
   s.email       = ["seamus@abshere.net"]
   s.homepage    = "https://github.com/seamusabshere/data_miner"
@@ -23,17 +21,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'activerecord', '>=2.3.4'
   s.add_runtime_dependency 'activesupport', '>=2.3.4'
   s.add_runtime_dependency 'conversions', '>=1.4.4'
-  s.add_runtime_dependency 'blockenspiel', '>=0.3.2'
   s.add_runtime_dependency 'errata', '>=1.0.1'
-  s.add_development_dependency 'mini_record-compat'
-  s.add_development_dependency 'loose_tight_dictionary', ">=0.0.5"
-  s.add_development_dependency 'test-unit'
-  s.add_development_dependency 'shoulda'
-  s.add_development_dependency 'mysql'
-  s.add_development_dependency 'rake'
-  # if RUBY_VERSION >= '1.9'
-  #   s.add_development_dependency 'ruby-debug19'
-  # else
-  #   s.add_development_dependency 'ruby-debug'
-  # end
+  s.add_runtime_dependency 'active_record_inline_schema'
+  s.add_runtime_dependency 'aasm'
+  s.add_runtime_dependency 'lock_method', '>=0.5.1'
 end

data/lib/data_miner.rb

@@ -1,91 +1,90 @@
+require 'singleton'
+require 'set'
 require 'active_support'
 require 'active_support/version'
-%w{
-  active_support/core_ext/array/conversions
-  active_support/core_ext/string/access
-  active_support/core_ext/string/multibyte
-}.each do |active_support_3_requirement|
-  require active_support_3_requirement
-end if ::ActiveSupport::VERSION::MAJOR == 3
+if ::ActiveSupport::VERSION::MAJOR >= 3
+  require 'active_support/core_ext'
+end
+require 'active_record'
+if RUBY_VERSION >= '1.9'
+  begin
+    require 'unicode_utils/downcase'
+  rescue LoadError
+    Kernel.warn '[data_miner] You may wish to include unicode_utils in your Gemfile to improve accuracy of downcasing'
+  end
+end
 
-require 'singleton'
+require 'data_miner/active_record_extensions'
+require 'data_miner/attribute'
+require 'data_miner/script'
+require 'data_miner/dictionary'
+require 'data_miner/step'
+require 'data_miner/step/import'
+require 'data_miner/step/tap'
+require 'data_miner/step/process'
+require 'data_miner/run'
 
 class DataMiner
-  include ::Singleton
-  
-  class MissingHashColumn < StandardError; end
-  class Finish < StandardError; end
-  class Skip < StandardError; end
-  
-  autoload :ActiveRecordExtensions, 'data_miner/active_record_extensions'
-  autoload :Attribute, 'data_miner/attribute'
-  autoload :Config, 'data_miner/config'
-  autoload :Dictionary, 'data_miner/dictionary'
-  autoload :Import, 'data_miner/import'
-  autoload :Tap, 'data_miner/tap'
-  autoload :Process, 'data_miner/process'
-  autoload :Run, 'data_miner/run'
-  
   class << self
+    delegate :perform, :to => :instance
+    delegate :run, :to => :instance
     delegate :logger, :to => :instance
     delegate :logger=, :to => :instance
-    delegate :run, :to => :instance
-    delegate :resource_names, :to => :instance
-  end
-    
-  # http://avdi.org/devblog/2009/07/14/recursively-symbolize-keys/
-  def self.recursively_stringify_keys(hash)
-    hash.inject(::Hash.new) do |result, (key, value)|
-      new_key   = case key
-                  when ::Symbol then key.to_s
-                  else key
-                  end
-      new_value = case value
-                  when ::Hash then ::DataMiner.recursively_stringify_keys(value)
-                  else value
-                  end
-      result[new_key] = new_value
-      result
+    delegate :model_names, :to => :instance
+
+    # @private
+    def downcase(str)
+      defined?(::UnicodeUtils) ? ::UnicodeUtils.downcase(str) : str.downcase
+    end
+
+    # @private
+    def upcase(str)
+      defined?(::UnicodeUtils) ? ::UnicodeUtils.upcase(str) : str.upcase
+    end
+
+    # @private
+    def compress_whitespace(str)
+      str.gsub(INNER_SPACE, ' ').strip
     end
   end
-  
+
+  MUTEX = ::Mutex.new
+  INNER_SPACE = /[ ]+/
+
+  include ::Singleton
+
   attr_writer :logger
-  def logger
-    return @logger if @logger
-    if defined?(::Rails)
-      @logger = ::Rails.logger
-    elsif defined?(::ActiveRecord) and active_record_logger = ::ActiveRecord::Base.logger
-      @logger = active_record_logger
-    else
-      require 'logger'
-      @logger = ::Logger.new $stderr
+
+  def perform(model_names = DataMiner.model_names)
+    Script.uniq do
+      model_names.each do |model_name|
+        model_name.constantize.run_data_miner!
+      end
     end
   end
 
-  def resource_names
-    @resource_names ||= []
-  end
+  # legacy
+  alias :run :perform
 
-  def call_stack
-    @call_stack ||= []
-  end
-      
-  # Mine data. Defaults to all resource_names touched by DataMiner.
-  #
-  # Options
-  # * <tt>:resource_names</tt>: array of resource (class) names to mine
-  def run(options = {})
-    options = options.dup
-    options.stringify_keys!
-    options['preserve_call_stack_between_runs'] = true
-    resource_names.each do |resource_name|
-      if options['resource_names'].blank? or options['resource_names'].include?(resource_name)
-        resource_name.constantize.data_miner_config.run options
+  def logger
+    @logger || MUTEX.synchronize do
+      @logger ||= if defined?(::Rails)
+        ::Rails.logger
+      elsif defined?(::ActiveRecord) and active_record_logger = ::ActiveRecord::Base.logger
+        active_record_logger
+      else
+        require 'logger'
+        ::Logger.new $stderr
       end
     end
-    call_stack.clear
   end
+
+  def model_names
+    @model_names || MUTEX.synchronize do
+      @model_names ||= ::Set.new
+    end
+  end
+
 end
 
-require 'active_record'
 ::ActiveRecord::Base.extend ::DataMiner::ActiveRecordExtensions

data/lib/data_miner/active_record_extensions.rb

@@ -1,43 +1,38 @@
 require 'active_record'
-require 'blockenspiel'
+require 'lock_method'
 
 class DataMiner
   module ActiveRecordExtensions
-    def data_miner_config
-      @data_miner_config ||= ::DataMiner::Config.new self
-    end
-    
-    def data_miner_config=(config)
-      @data_miner_config = config
+    MUTEX = ::Mutex.new
+
+    def data_miner_script
+      @data_miner_script || MUTEX.synchronize do
+        @data_miner_script ||= DataMiner::Script.new(self)
+      end
     end
     
     def data_miner_runs
-      ::DataMiner::Run.scoped :conditions => { :resource_name => name }
+      DataMiner::Run.scoped :conditions => { :model_name => name }
     end
 
-    def run_data_miner!(options = {})
-      data_miner_config.run options
+    def run_data_miner!
+      data_miner_script.perform
     end
     
     def run_data_miner_on_parent_associations!
-      reflect_on_all_associations(:belongs_to).each do |assoc|
-        next if assoc.options[:polymorphic]
-        assoc.klass.run_data_miner!
+      reflect_on_all_associations(:belongs_to).reject do |assoc|
+        assoc.options[:polymorphic]
+      end.each do |non_polymorphic_belongs_to_assoc|
+        non_polymorphic_belongs_to_assoc.klass.run_data_miner!
       end
     end
     
     def data_miner(options = {}, &blk)
-      ::DataMiner.instance.resource_names.push name unless ::DataMiner.instance.resource_names.include?(name)
-
+      DataMiner.model_names.add name
       unless options[:append]
-        self.data_miner_config = ::DataMiner::Config.new self
+        @data_miner_script = nil
       end
-
-      ::Blockenspiel.invoke blk, data_miner_config
-
-      data_miner_config.after_invoke
+      data_miner_script.append_block blk
     end
   end
 end
-
-

data/lib/data_miner/attribute.rb

@@ -2,12 +2,20 @@ require 'conversions'
 
 class DataMiner
   class Attribute
-    attr_reader :step
-    attr_reader :name
-    attr_reader :options
-
-    def resource
-      step.resource
+    class << self
+      def check_options(options)
+        errors = []
+        if options[:dictionary].is_a?(Dictionary)
+          errors << %{:dictionary must be a Hash of options}
+        end
+        if (invalid_option_keys = options.keys - VALID_OPTIONS).any?
+          errors << %{Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence}}
+        end
+        if (units_options = options.select { |k, _| k.to_s.include?('units') }).any? and VALID_UNIT_DEFINITION_SETS.none? { |d| d.all? { |required_option| options[required_option].present? } }
+          errors << %{#{units_options.inspect} is not a valid set of units definitions. Please supply a set like #{VALID_UNIT_DEFINITION_SETS.map(&:inspect).to_sentence}".}
+        end
+        errors
+      end
     end
 
     VALID_OPTIONS = %w{
@@ -29,28 +37,114 @@ class DataMiner
       field_number
       chars
       synthesize
-    }
+    }.map(&:to_sym)
 
-    def initialize(step, name, options = {})
-      @options = ::DataMiner.recursively_stringify_keys options
+    VALID_UNIT_DEFINITION_SETS = [
+      [:units],
+      [:from_units, :to_units],
+      [:units_field_name],
+      [:units_field_name, :to_units],
+      [:units_field_number],
+      [:units_field_number, :to_units],
+    ]
+
+    DEFAULT_SPLIT = /\s+/
+    DEFAULT_KEEP = 0
+    DEFAULT_DELIMITER = ', '
+    DEFAULT_NULLIFY = false
+    DEFAULT_UPCASE = false
+    DEFAULT_OVERWRITE = true
+
+    attr_reader :step
+    attr_reader :name
+    attr_reader :synthesize
+    attr_reader :matcher
+    attr_reader :field_number
+    attr_reader :field_name
+    # For use when joining a range of field numbers
+    attr_reader :delimiter
+    attr_reader :chars
+    attr_reader :split
+    attr_reader :to_units
+    attr_reader :from_units
+    attr_reader :units_field_number
+    attr_reader :units_field_name
+    attr_reader :sprintf
+    attr_reader :static
 
+    def initialize(step, name, options = {})
+      options = options.symbolize_keys
+      if (errors = Attribute.check_options(options)).any?
+        raise ::ArgumentError, %{[data_miner] Errors on #{inspect}: #{errors.join(';')}}
+      end
       @step = step
       @name = name
-      
-      invalid_option_keys = @options.keys.select { |k| not VALID_OPTIONS.include? k }
-      raise "Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence} (#{inspect})" if invalid_option_keys.any?
+      @synthesize = options[:synthesize]
+      if @dictionary_boolean = options.has_key?(:dictionary)
+        @dictionary_options = options[:dictionary]
+      end
+      @matcher = options[:matcher].is_a?(::String) ? options[:matcher].constantize.new : options[:matcher]
+      if @static_boolean = options.has_key?(:static)
+        @static = options[:static]
+      end
+      @field_number = options[:field_number]
+      @field_name = options.fetch(:field_name, name).to_sym
+      @delimiter = options.fetch :delimiter, DEFAULT_DELIMITER
+      @chars = options[:chars]
+      if split = options[:split]
+        @split = split.symbolize_keys
+      end
+      @nullify_boolean = options.fetch :nullify, DEFAULT_NULLIFY
+      @upcase_boolean = options.fetch :upcase, DEFAULT_UPCASE
+      @from_units = options[:from_units]
+      @to_units = options[:to_units] || options[:units]
+      @sprintf = options[:sprintf]
+      @overwrite_boolean = options.fetch :overwrite, DEFAULT_OVERWRITE
+      @units_field_name = options[:units_field_name]
+      @units_field_number = options[:units_field_number]
+      @dictionary_mutex = ::Mutex.new
     end
-        
-    def inspect
-      %{#<DataMiner::Attribute(#{resource}##{name})>}
+
+    def model
+      step.model
+    end
+
+    def static?
+      @static_boolean
+    end
+
+    def nullify?
+      @nullify_boolean
     end
 
-    def value_in_dictionary(str)
-      dictionary.lookup str
+    def upcase?
+      @upcase_boolean
     end
-    
-    def value_in_source(row)
-      value = if wants_static?
+
+    def dictionary?
+      @dictionary_boolean
+    end
+
+    def convert?
+      from_units.present? or units_field_name.present? or units_field_number.present?
+    end
+
+    def units?
+      to_units.present? or units_field_name.present? or units_field_number.present?
+    end
+
+    def overwrite?
+      @overwrite_boolean
+    end
+        
+    def read(row)
+      if matcher and matched_row = matcher.match(row)
+        return matched_row
+      end
+      if synthesize
+        return synthesize.call(row)
+      end
+      value = if static?
         static
       elsif field_number
         if field_number.is_a?(::Range)
@@ -58,180 +152,83 @@ class DataMiner
         else
           row[field_number]
         end
-      elsif field_name == 'row_hash'
+      elsif field_name == :row_hash
         row.row_hash
       elsif row.is_a?(::Hash) or row.is_a?(::ActiveSupport::OrderedHash)
-        row[field_name]
+        row[field_name.to_s] # remote_table hash keys are always strings
+      end
+      if value.nil?
+        return
+      end
+      if value.is_a? ::ActiveRecord::Base
+        return value
       end
-      return nil if value.nil?
-      return value if value.is_a?(::ActiveRecord::Base) # escape valve for parsers that look up associations directly
       value = value.to_s
-      value = value[chars] if wants_chars?
-      value = do_split(value) if wants_split?
-      value.gsub! /[ ]+/, ' '
-      value.strip!
-      return nil if value.blank? and wants_nullification?
-      value.upcase! if wants_upcase?
-      value = do_convert row, value if wants_conversion?
-      value = do_sprintf value if wants_sprintf?
-      value
-    end
-    
-    def match_row(row)
-      matcher.match row
-    end
-    
-    def value_from_row(row)
-      return match_row row if wants_matcher?
-      value = value_in_source row
-      return value if value.is_a? ::ActiveRecord::Base # carry through trapdoor
-      value = value_in_dictionary value if wants_dictionary?
-      value = synthesize.call(row) if wants_synthesize?
-      value
-    end
-    
-    def set_record_from_row(record, row)
-      return false if !wants_overwriting? and !record.send(name).nil?
-      record.send "#{name}=", value_from_row(row)
-      if wants_units?
-        unit = (to_units || unit_from_source(row)).to_s
-        unit = nil if unit.blank? and wants_nullification?
-        record.send "#{name}_units=", unit
+      if chars
+        value = value[chars]
       end
-    end
-    
-    def unit_from_source(row)
-      row[units_field_name || units_field_number].to_s.strip.underscore.to_sym
-    end
-    
-    def do_convert(row, value)
-      unless wants_units?
-        raise ::RuntimeError, "[data_miner] If you use 'from_units', you need to set 'to_units' (#{inspect})"
+      if split
+        pattern = split.fetch :pattern, DEFAULT_SPLIT
+        keep = split.fetch :keep, DEFAULT_KEEP
+        value = value.to_s.split(pattern)[keep].to_s
       end
-      final_from_units = (from_units || unit_from_source(row))
-      final_to_units = (to_units || unit_from_source(row))
-      if final_from_units.blank? or final_to_units.blank?
-        raise ::RuntimeError, "[data_miner] Missing units (from=#{final_from_units.inspect}, to=#{final_to_units.inspect}"
+      value = DataMiner.compress_whitespace value
+      if nullify? and value.blank?
+        return
       end
-      value.to_f.convert final_from_units, final_to_units
-    end
-    
-    def do_sprintf(value)
-      if /\%[0-9\.]*f/.match sprintf
-        value = value.to_f
-      elsif /\%[0-9\.]*d/.match sprintf
-        value = value.to_i
+      if upcase?
+        value = DataMiner.upcase value
       end
-      sprintf % value
-    end
-    
-    def do_split(value)
-      pattern = split_options['pattern'] || /\s+/ # default is split on whitespace
-      keep = split_options['keep'] || 0           # default is keep first element
-      value.to_s.split(pattern)[keep].to_s
-    end
-  
-    def column_type
-      resource.columns_hash[name.to_s].type
-    end
-    
-    # Our wants and needs :)
-    def wants_split?
-      split_options.present?
-    end
-    def wants_sprintf?
-      sprintf.present?
-    end
-    def wants_upcase?
-      upcase.present?
-    end
-    def wants_static?
-      options.has_key? 'static'
-    end
-    def wants_nullification?
-      nullify == true
-    end
-    def wants_chars?
-      chars.present?
-    end
-    def wants_synthesize?
-      synthesize.is_a?(::Proc)
-    end
-    def wants_overwriting?
-      overwrite != false
-    end
-    def wants_conversion?
-      from_units.present? or units_field_name.present? or units_field_number.present?
-    end
-    def wants_units?
-      to_units.present? or units_field_name.present? or units_field_number.present?
-    end
-    def wants_dictionary?
-      options['dictionary'].present?
-    end
-    def wants_matcher?
-      options['matcher'].present?
+      if convert?
+        final_from_units = from_units || read_units(row)
+        final_to_units = to_units || read_units(row)
+        if final_from_units.blank? or final_to_units.blank?
+          raise ::RuntimeError, "[data_miner] Missing units (from=#{final_from_units.inspect}, to=#{final_to_units.inspect}"
+        end
+        value = value.to_f.convert final_from_units, final_to_units
+      end
+      if sprintf
+        if sprintf.end_with?('f')
+          value = value.to_f
+        elsif sprintf.end_with?('d')
+          value = value.to_i
+        end
+        value = sprintf % value
+      end
+      if dictionary?
+        value = dictionary.lookup(value)
+      end
+      value
     end
 
-    # Options that always have values
-    def field_name
-      (options['field_name'] || name).to_s
-    end
-    def delimiter
-      (options['delimiter'] || ', ')
-    end
-    
-    # Options that can't be referred to by their names
-    def split_options
-      options['split']
-    end
-    
-    def from_units
-      options['from_units']
-    end
-    def to_units
-      options['to_units'] || options['units']
-    end
-    def sprintf
-      options['sprintf']
-    end
-    def nullify
-      options['nullify']
-    end
-    def overwrite
-      options['overwrite']
-    end
-    def upcase
-      options['upcase']
-    end
-    def units_field_name
-      options['units_field_name']
-    end
-    def units_field_number
-      options['units_field_number']
-    end
-    def field_number
-      options['field_number']
-    end
-    def chars
-      options['chars']
-    end
-    def synthesize
-      options['synthesize']
-    end
-    def static
-      options['static']
+    def set_from_row(target, row)
+      if overwrite? or target.send(name).nil?
+        target.send "#{name}=", read(row)
+      end
+      if units? and ((final_to_units = (to_units || read_units(row))) or nullify?)
+        target.send "#{name}_units=", final_to_units
+      end
     end
-    # must be cleared before every run! (because it relies on remote data)
+
     def dictionary
-      @dictionary ||= (options['dictionary'].is_a?(Dictionary) ? options['dictionary'] : Dictionary.new(options['dictionary']))
+      @dictionary || @dictionary_mutex.synchronize do
+        @dictionary ||= Dictionary.new(@dictionary_options)
+      end
     end
-    def matcher
-      @matcher ||= (options['matcher'].is_a?(::String) ? options['matcher'].constantize.new : options['matcher'])
+
+    def refresh
+      @dictionary = nil
     end
-    
+        
+    private
+
+    def read_units(row)
+      if units = row[units_field_name || units_field_number]
+        DataMiner.compress_whitespace(units).underscore.to_sym
+      end
+    end
+
     def free
-      @dictionary.free if @dictionary.is_a?(Dictionary)
       @dictionary = nil
     end
   end