data_miner 0.2.6 → 0.3.0

data/.gitignore

@@ -4,3 +4,4 @@ coverage
 rdoc
 pkg
 test/test.sqlite3
+data_miner.log

data/CHANGELOG

@@ -1,2 +1,7 @@
 0.2.6
 * Upgrade to remote_table 0.1.6 to handle UTF-8 CSVs and long urls.
+0.3.0
+* Removed association code... now data_miner focuses on just importing.
+* New, simpler DSL
+* Upgrade to remote_table 0.2.1 for row_hashes and better blank row handling
+* Remove all association-related code

data/README.rdoc

@@ -8,15 +8,15 @@ Put this in <tt>config/environment.rb</tt>:
 
   config.gem 'data_miner'
 
-You need to define <tt>mine_data</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
+You need to define <tt>data_miner</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
 
   class Country < ActiveRecord::Base
-    mine_data do |step|
+    data_miner do |step|
       # import country names and country codes
       step.import :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do |attr|
-        attr.key :iso_3166, :name_in_source => 'country code'
-        attr.store :iso_3166, :name_in_source => 'country code'
-        attr.store :name, :name_in_source => 'country'
+        attr.key :iso_3166, :field_name => 'country code'
+        attr.store :iso_3166, :field_name => 'country code'
+        attr.store :name, :field_name => 'country'
       end
     end
   end
@@ -26,7 +26,7 @@ You need to define <tt>mine_data</tt> blocks in your ActiveRecord models. For ex
   class Airport < ActiveRecord::Base
     belongs_to :country
   
-    mine_data do |step|
+    data_miner do |step|
       # import airport iata_code, name, etc.
       step.import(:url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false) do |attr|
         attr.key :iata_code, :field_number => 3
@@ -43,12 +43,8 @@ You need to define <tt>mine_data</tt> blocks in your ActiveRecord models. For ex
 Put this in <tt>lib/tasks/data_miner_tasks.rake</tt>: (unfortunately I don't know a way to automatically include gem tasks, so you have to do this manually for now)
 
   namespace :data_miner do
-    task :mine => :environment do
-      DataMiner.mine :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
-    end
-
-    task :map_to_attrs => :environment do
-      DataMiner.map_to_attrs ENV['METHOD'], :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
+    task :run => :environment do
+      DataMiner.run :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
     end
   end
 
@@ -60,9 +56,9 @@ You need to specify what order to mine data. For example, in <tt>config/initiali
     # etc
   end
 
-Once you have (1) set up the order of data mining and (2) defined <tt>mine_data</tt> blocks in your classes, you can:
+Once you have (1) set up the order of data mining and (2) defined <tt>data_miner</tt> blocks in your classes, you can:
 
-  $ rake data_miner:mine
+  $ rake data_miner:run
 
 ==Complete example
 
@@ -75,7 +71,7 @@ Once you have (1) set up the order of data mining and (2) defined <tt>mine_data<
   [...edit per quick start...]
   ~/testapp $ touch config/initializers/data_miner_config.rake
   [...edit per quick start...]
-  ~/testapp $ rake data_miner:mine
+  ~/testapp $ rake data_miner:run
 
 Now you should have
 

data/Rakefile

@@ -10,8 +10,13 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    %w{ activerecord activesupport andand errata conversions }.each { |name| gem.add_dependency name }
-    gem.add_dependency 'remote_table', '0.1.6'
+    gem.add_dependency 'remote_table', '~>0.2.1'
+    gem.add_dependency 'activerecord', '~>2.3.4'
+    gem.add_dependency 'activesupport', '~>2.3.4'
+    gem.add_dependency 'andand', '~>1.3.1'
+    gem.add_dependency 'errata', '~>0.1.4'
+    gem.add_dependency 'conversions', '~>1.4.3'
+    gem.add_dependency 'blockenspiel', '~>0.3.2'
     gem.require_path = "lib"
     gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
     gem.rdoc_options << '--line-numbers' << '--inline-source'

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.3.0

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.2.6"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2009-11-05}
+  s.date = %q{2010-02-25}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -26,17 +26,13 @@ Gem::Specification.new do |s|
      "VERSION",
      "data_miner.gemspec",
      "lib/data_miner.rb",
-     "lib/data_miner/active_record_ext.rb",
      "lib/data_miner/attribute.rb",
-     "lib/data_miner/attribute_collection.rb",
      "lib/data_miner/configuration.rb",
      "lib/data_miner/dictionary.rb",
-     "lib/data_miner/step.rb",
-     "lib/data_miner/step/associate.rb",
-     "lib/data_miner/step/await.rb",
-     "lib/data_miner/step/callback.rb",
-     "lib/data_miner/step/derive.rb",
-     "lib/data_miner/step/import.rb",
+     "lib/data_miner/import.rb",
+     "lib/data_miner/process.rb",
+     "lib/data_miner/run.rb",
+     "lib/data_miner/target.rb",
      "lib/data_miner/william_james_cartesian_product.rb",
      "test/data_miner_test.rb",
      "test/test_helper.rb"
@@ -57,27 +53,30 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<activerecord>, [">= 0"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
-      s.add_runtime_dependency(%q<andand>, [">= 0"])
-      s.add_runtime_dependency(%q<errata>, [">= 0"])
-      s.add_runtime_dependency(%q<conversions>, [">= 0"])
-      s.add_runtime_dependency(%q<remote_table>, ["= 0.1.6"])
+      s.add_runtime_dependency(%q<remote_table>, ["~> 0.2.1"])
+      s.add_runtime_dependency(%q<activerecord>, ["~> 2.3.4"])
+      s.add_runtime_dependency(%q<activesupport>, ["~> 2.3.4"])
+      s.add_runtime_dependency(%q<andand>, ["~> 1.3.1"])
+      s.add_runtime_dependency(%q<errata>, ["~> 0.1.4"])
+      s.add_runtime_dependency(%q<conversions>, ["~> 1.4.3"])
+      s.add_runtime_dependency(%q<blockenspiel>, ["~> 0.3.2"])
     else
-      s.add_dependency(%q<activerecord>, [">= 0"])
-      s.add_dependency(%q<activesupport>, [">= 0"])
-      s.add_dependency(%q<andand>, [">= 0"])
-      s.add_dependency(%q<errata>, [">= 0"])
-      s.add_dependency(%q<conversions>, [">= 0"])
-      s.add_dependency(%q<remote_table>, ["= 0.1.6"])
+      s.add_dependency(%q<remote_table>, ["~> 0.2.1"])
+      s.add_dependency(%q<activerecord>, ["~> 2.3.4"])
+      s.add_dependency(%q<activesupport>, ["~> 2.3.4"])
+      s.add_dependency(%q<andand>, ["~> 1.3.1"])
+      s.add_dependency(%q<errata>, ["~> 0.1.4"])
+      s.add_dependency(%q<conversions>, ["~> 1.4.3"])
+      s.add_dependency(%q<blockenspiel>, ["~> 0.3.2"])
     end
   else
-    s.add_dependency(%q<activerecord>, [">= 0"])
-    s.add_dependency(%q<activesupport>, [">= 0"])
-    s.add_dependency(%q<andand>, [">= 0"])
-    s.add_dependency(%q<errata>, [">= 0"])
-    s.add_dependency(%q<conversions>, [">= 0"])
-    s.add_dependency(%q<remote_table>, ["= 0.1.6"])
+    s.add_dependency(%q<remote_table>, ["~> 0.2.1"])
+    s.add_dependency(%q<activerecord>, ["~> 2.3.4"])
+    s.add_dependency(%q<activesupport>, ["~> 2.3.4"])
+    s.add_dependency(%q<andand>, ["~> 1.3.1"])
+    s.add_dependency(%q<errata>, ["~> 0.1.4"])
+    s.add_dependency(%q<conversions>, ["~> 1.4.3"])
+    s.add_dependency(%q<blockenspiel>, ["~> 0.3.2"])
   end
 end
 

data/lib/data_miner.rb

@@ -1,43 +1,66 @@
-require 'rubygems'
-require 'activesupport'
-require 'activerecord'
+require 'active_support'
+require 'active_record'
+require 'blockenspiel'
 require 'conversions'
 require 'remote_table'
 require 'errata'
+require 'andand'
+require 'log4r'
 
-require 'data_miner/active_record_ext'
 require 'data_miner/attribute'
-require 'data_miner/attribute_collection'
 require 'data_miner/configuration'
 require 'data_miner/dictionary'
-require 'data_miner/step'
-require 'data_miner/step/associate'
-require 'data_miner/step/await'
-require 'data_miner/step/callback'
-require 'data_miner/step/derive'
-require 'data_miner/step/import'
-require 'data_miner/william_james_cartesian_product' # TODO: move to gem
+require 'data_miner/import'
+require 'data_miner/process'
+require 'data_miner/target'
+require 'data_miner/run'
+
+# TODO: move to gem
+require 'data_miner/william_james_cartesian_product'
 
 module DataMiner
-  class << self
-    def mine(options = {})
-      DataMiner::Configuration.mine options
-    end
-    
-    def map_to_attrs(method, options = {})
-      puts DataMiner::Configuration.map_to_attrs(method, options)
-    end
+  class MissingHashColumn < RuntimeError; end
+  
+  include Log4r
 
-    def enqueue(&block)
-      DataMiner::Configuration.enqueue &block
-    end
-    
-    def classes
-      DataMiner::Configuration.classes
+  mattr_accessor :logger
+  
+  def self.start_logging
+    if defined?(Rails)
+      self.logger = Rails.logger
+    else
+      self.logger = Logger.new 'data_miner'
+      logger.outputters = FileOutputter.new 'f1', :filename => 'data_miner.log'
     end
   end
+  
+  def self.run(options = {})
+    DataMiner::Configuration.run options
+  end
+  
+  def self.enqueue(&block)
+    DataMiner::Configuration.enqueue &block
+  end
+  
+  def self.classes
+    DataMiner::Configuration.classes
+  end
+  
+  def self.create_tables
+    DataMiner::Configuration.create_tables
+  end
 end
 
 ActiveRecord::Base.class_eval do
-  include DataMiner::ActiveRecordExt
+  def self.data_miner(&block)
+    # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
+    class_eval { cattr_accessor :data_miner_config }
+    self.data_miner_config = DataMiner::Configuration.new self
+
+    data_miner_config.before_invoke
+    Blockenspiel.invoke block, data_miner_config
+    data_miner_config.after_invoke
+  end
 end
+
+DataMiner.start_logging

data/lib/data_miner/attribute.rb

@@ -1,299 +1,216 @@
 module DataMiner
   class Attribute
-    attr_accessor :klass, :name, :options_for_step, :affected_by_steps, :key_for_steps
+    attr_accessor :klass, :name, :options_for_import
 
     def initialize(klass, name)
       @klass = klass
-      @name = name.to_sym
-      @options_for_step = {}
-      @affected_by_steps = []
-      @key_for_steps = []
+      @name = name
+      @options_for_import = {}
     end
-    
-    # polling questions
-    def report_find_or_create(step)
-      "Creates parents: #{klass}##{name} is set with #{reflection_klass(step)}.find_or_create_by_#{foreign_key(step)}" if wants_create?(step)
-    end
-    
-    def report_unnatural_order(step)
-      if (
-           (rk = klass.reflect_on_association(weighting_association(step)).andand.klass) or
-           (wants_inline_association? and rk = reflection_klass(step))
-         ) and
-         step.configuration.classes.index(rk) > step.configuration.classes.index(klass) and
-         step.options[:awaiting].andand.klass != klass
-        "Unnatural order: #{klass} comes before #{rk}"
-      end
-    end
-
+        
     def inspect
-      "Attribute(#{klass}.#{name})"
+      "Attribute(#{klass}##{name})"
     end
 
-    def affected_by!(step, options = {})
-      self.options_for_step[step] = options
-      self.affected_by_steps << step
-    end
-
-    def affected_by?(step)
-      affected_by_steps.include?(step)
-    end
-
-    def key_for!(step, options = {})
-      self.options_for_step[step] = options
-      self.key_for_steps << step
-    end
-
-    def key_for?(step)
-      key_for_steps.include?(step)
+    def stored_by?(import)
+      options_for_import.has_key?(import)
     end
     
-    def value_in_dictionary(step, key)
-      return *dictionary(step).lookup(key) # strip the array wrapper if there's only one element
+    def value_in_dictionary(import, key)
+      return *dictionary(import).lookup(key) # strip the array wrapper if there's only one element
     end
     
-    def value_in_source(step, row)
-      if wants_static?(step)
-        value = static(step)
-      elsif field_number(step)
-        if field_number(step).is_a?(Range)
-          value = field_number(step).map { |n| row[n] }.join(delimiter(step))
+    def value_in_source(import, row)
+      if wants_static?(import)
+        value = static(import)
+      elsif field_number(import)
+        if field_number(import).is_a?(Range)
+          value = field_number(import).map { |n| row[n] }.join(delimiter(import))
         else
-          value = row[field_number(step)]
+          value = row[field_number(import)]
         end
       else
-        value = row[name_in_source(step)]
+        value = row[field_name(import)]
       end
       return nil if value.nil?
       return value if value.is_a?(ActiveRecord::Base) # escape valve for parsers that look up associations directly
       value = value.to_s
-      value = value[keep(step)] if wants_keep?(step)
-      value = do_split(step, value) if wants_split?(step)
+      value = value[chars(import)] if wants_chars?(import)
+      value = do_split(import, value) if wants_split?(import)
       # taken from old errata... maybe we want to do this here
       value.gsub!(/[ ]+/, ' ')
       # text.gsub!('- ', '-')
       value.gsub!(/([^\\])~/, '\1 ')
       value.strip!
-      value.upcase! if wants_upcase?(step)
-      value = do_convert(step, row, value) if wants_conversion?(step)
-      value = do_sprintf(step, value) if wants_sprintf?(step)
+      value.upcase! if wants_upcase?(import)
+      value = do_convert(import, row, value) if wants_conversion?(import)
+      value = do_sprintf(import, value) if wants_sprintf?(import)
       value
     end
     
-    def value_from_row(step, row)
-      value = value_in_source(step, row)
+    def value_from_row(import, row)
+      value = value_in_source(import, row)
       return value if value.is_a?(ActiveRecord::Base) # carry through trapdoor
-      value = value_in_dictionary(step, value) if wants_dictionary?(step)
-      value = value_as_association(step, value) if wants_inline_association?
+      value = value_in_dictionary(import, value) if wants_dictionary?(import)
       value
     end
-    
-    def value_as_association(step, value)
-      @_value_as_association ||= {}
-      @_value_as_association[step] ||= {}
-      if !@_value_as_association[step].has_key?(value)
-        dynamic_matcher = wants_create?(step) ? "find_or_create_by_#{foreign_key(step)}" : "find_by_#{foreign_key(step)}"
-        @_value_as_association[step][value] = reflection_klass(step).send(dynamic_matcher, value)
-      end
-      @_value_as_association[step][value]
-    end
-    
-    # this will overwrite nils, even if wants_overwriting?(step) is false
-    def set_record_from_row(step, record, row)
-      return if !wants_overwriting?(step) and !record.send(name).nil?
-      value = value_from_row(step, row)
+        
+    # this will overwrite nils, even if wants_overwriting?(import) is false
+    def set_record_from_row(import, record, row)
+      return if !wants_overwriting?(import) and !record.send(name).nil?
+      value = value_from_row(import, row)
       record.send "#{name}=", value
-      $stderr.puts("ActiveRecord didn't like trying to set #{klass}.#{name} = #{value}") if !value.nil? and record.send(name).nil?
-    end
-
-    def perform(step)
-      case step.variant
-      when :associate
-        perform_association(step)
-      when :derive
-        if wants_update_all?(step)
-          perform_update_all(step)
-        elsif wants_weighted_average?(step)
-          perform_weighted_average(step)
-        else
-          perform_callback(step)
-        end
-      when :import
-        raise "This shouldn't be called, the import step is special"
-      end
-    end
-
-    def perform_association(step)
-      raise "dictionary and prefix don't mix" if wants_dictionary?(step) and wants_prefix?(step)
-      klass.update_all("#{reflection.primary_key_name} = NULL") if wants_nullification?(step)
-      if wants_create?(step)
-        klass.find_in_batches do |batch|
-          batch.each do |record|
-            if wants_prefix?(step)
-              sql = "SELECT reflection_table.id FROM #{reflection_klass(step).quoted_table_name} AS reflection_table INNER JOIN #{klass.quoted_table_name} AS klass_table ON LEFT(klass_table.#{key(step)}, LENGTH(reflection_table.#{foreign_key(step)})) = reflection_table.#{foreign_key(step)} WHERE klass_table.id = #{record.id} ORDER BY LENGTH(reflection_table.#{foreign_key(step)}) DESC"
-              associated_id = ActiveRecord::Base.connection.select_value(sql)
-              next if associated_id.blank?
-              record.send("#{reflection.primary_key_name}=", associated_id)
-            else
-              dynamic_finder_value = record.send(key(step))
-              dynamic_finder_value = value_in_dictionary(step, dynamic_finder_value) if wants_dictionary?(step)
-              next if dynamic_finder_value.blank?
-              associated = reflection_klass(step).send("find_or_create_by_#{foreign_key(step)}", dynamic_finder_value) # TODO cache results
-              record.send("#{name}=", associated)
-            end
-            record.save
-          end
-        end
-      else
-        reflection_klass(step).find_in_batches do |batch|
-          batch.each do |reflection_record|
-            klass.update_all ["#{reflection.primary_key_name} = ?", reflection_record.id], ["#{key(step)} = ?", reflection_record.send(foreign_key(step))]
-          end
-        end
-      end
+      DataMiner.logger.info("ActiveRecord didn't like trying to set #{klass}.#{name} = #{value}") if !value.nil? and record.send(name).nil?
     end
 
-    def perform_update_all(step)
-      klass.update_all("#{name} = #{set(step)}", conditions(step))
+    def unit_from_source(import, row)
+      row[units_field_name(import)].to_s.strip.underscore.to_sym
     end
     
-    def perform_weighted_average(step)
-      # handle weighting by scopes instead of associations
-      if weighting_association(step) and !klass.reflect_on_association(weighting_association(step))
-        klass.find_in_batches do |batch|
-          batch.each do |record|
-            record.send "#{name}=", record.send(weighting_association(step)).weighted_average(name, :by => weighting_column(step), :disaggregator => weighting_disaggregator(step))
-            record.save
-          end
-        end
-      else # there's no weighting association OR there is one and it's a valid association
-        klass.update_all_weighted_averages name, :by => weighting_column(step), :disaggregator => weighting_disaggregator(step), :association => weighting_association(step)
-      end
-    end
-    
-    def perform_callback(step)
-      case klass.method(callback(step)).arity
-      when 0:
-        klass.send(callback(step))
-      when 1:
-        klass.send(callback(step), name)
-      when 2:
-        klass.send(callback(step), name, options_for_step[step])
-      end
-    end
-    
-    def unit_from_source(step, row)
-      row[unit_in_source(step)].to_s.strip.underscore.to_sym
+    def do_convert(import, row, value)
+      value.to_f.convert((from_units(import) || unit_from_source(import, row)), to_units(import))
     end
     
-    def do_convert(step, row, value)
-      from_unit = from(step) || unit_from_source(step, row)
-      value.to_f.convert(from_unit, to(step))
-    end
-    
-    def do_sprintf(step, value)
-      if /\%[0-9\.]*f/.match(sprintf(step))
+    def do_sprintf(import, value)
+      if /\%[0-9\.]*f/.match(sprintf(import))
         value = value.to_f
-      elsif /\%[0-9\.]*d/.match(sprintf(step))
+      elsif /\%[0-9\.]*d/.match(sprintf(import))
         value = value.to_i
       end
-      sprintf(step) % value
+      sprintf(import) % value
     end
     
-    def do_split(step, value)
-      pattern = split_options(step)[:pattern] || /\s+/ # default is split on whitespace
-      keep = split_options(step)[:keep] || 0           # default is keep first element
+    def do_split(import, value)
+      pattern = split_options(import)[:pattern] || /\s+/ # default is split on whitespace
+      keep = split_options(import)[:keep] || 0           # default is keep first element
       value.to_s.split(pattern)[keep].to_s
     end
   
     def column_type
-      @column_type ||= klass.columns_hash[name.to_s].type
+      klass.columns_hash[name.to_s].type
+    end
+    
+    def dictionary(import)
+      raise "shouldn't ask for this" unless wants_dictionary?(import) # don't try to initialize if there are no dictionary options
+      Dictionary.new dictionary_options(import)
+    end
+    
+    # {
+    #   :static => 'options_for_import[import].has_key?(:static)',
+    #   :chars => :chars,
+    #   :upcase => :upcase,
+    #   :conversion => '!from_units(import).nil? or !units_field_name(import).nil?',
+    #   :sprintf => :sprintf,
+    #   :dictionary => :dictionary_options,
+    #   :split => :split_options,
+    #   :nullification => 'nullify(import) != false',
+    #   :overwriting => 'overwrite(import) != false',
+    # }.each do |name, condition|
+    #   condition = "!#{condition}(import).nil?" if condition.is_a?(Symbol)
+    #   puts <<-EOS
+    #     def wants_#{name}?(import)
+    #       #{condition}
+    #     end
+    #   EOS
+    # end
+    def wants_split?(import)
+      !split_options(import).nil?
+    end
+    def wants_sprintf?(import)
+      !sprintf(import).nil?
+    end
+    def wants_upcase?(import)
+      !upcase(import).nil?
+    end
+    def wants_static?(import)
+      options_for_import[import].has_key?(:static)
+    end
+    def wants_nullification?(import)
+      nullify(import) != false
+    end
+    def wants_chars?(import)
+      !chars(import).nil?
+    end
+    def wants_overwriting?(import)
+      overwrite(import) != false
+    end
+    def wants_conversion?(import)
+      !from_units(import).nil? or !units_field_name(import).nil?
+    end
+    def wants_dictionary?(import)
+      !dictionary_options(import).nil?
+    end
+    
+    # {
+    #   :field_name => { :default => :name,                           :stringify => true },
+    #   :delimiter      => { :default => '", "' }
+    # }.each do |name, options|
+    #   puts <<-EOS
+    #     def #{name}(import)
+    #       (options_for_import[import][:#{name}] || #{options[:default]})#{'.to_s' if options[:stringify]}
+    #     end
+    #   EOS
+    # end
+    def field_name(import)
+      (options_for_import[import][:field_name] || name).to_s
+    end
+    def delimiter(import)
+      (options_for_import[import][:delimiter] || ", ")
+    end
+    
+    # %w(dictionary split).each do |name|
+    #   puts <<-EOS
+    #     def #{name}_options(import)
+    #       options_for_import[import][:#{name}]
+    #     end
+    #   EOS
+    # end
+    def dictionary_options(import)
+      options_for_import[import][:dictionary]
+    end
+    def split_options(import)
+      options_for_import[import][:split]
     end
-    
-    {
-      :static => 'options_for_step[step].has_key?(:static)',
-      :prefix => :prefix,
-      :create => :create,
-      :keep => :keep,
-      :upcase => :upcase,
-      :conversion => '!from(step).nil? or !unit_in_source(step).nil?',
-      :sprintf => :sprintf,
-      :dictionary => :dictionary_options,
-      :split => :split_options,
-      :update_all => :set,
-      :nullification => 'nullify(step) != false',
-      :overwriting => 'overwrite(step) != false',
-      :weighted_average => '!weighting_association(step).nil? or !weighting_column(step).nil?'
-    }.each do |name, condition|
-      condition = "!#{condition}(step).nil?" if condition.is_a?(Symbol)
-      eval <<-EOS
-        def wants_#{name}?(step)
-          #{condition}
-        end
-      EOS
+        
+    # %w(from_units to_units conditions sprintf nullify overwrite upcase units_field_name field_number chars static).each do |name|
+    #   puts <<-EOS
+    #     def #{name}(import)
+    #       options_for_import[import][:#{name}]
+    #     end
+    #   EOS
+    # end
+    def from_units(import)
+      options_for_import[import][:from_units]
     end
-    
-    {
-      :name_in_source => { :default => :name,                           :stringify => true },
-      :key            => { :default => :name,                           :stringify => true },
-      :foreign_key    => { :default => 'key(step)',                     :stringify => true },
-      :delimiter      => { :default => '", "' }
-    }.each do |name, options|
-      eval <<-EOS
-        def #{name}(step)
-          (options_for_step[step][:#{name}] || #{options[:default]})#{'.to_s' if options[:stringify]}
-        end
-      EOS
+    def to_units(import)
+      options_for_import[import][:to_units]
     end
-    
-    def reflection
-      if @_reflection.nil?
-        @_reflection = klass.reflect_on_association(name) || :missing
-        reflection
-      elsif @_reflection == :missing
-        nil
-      else
-        @_reflection
-      end
+    def conditions(import)
+      options_for_import[import][:conditions]
     end
-    
-    def reflection_klass(step)
-      return nil unless reflection
-      if reflection.options[:polymorphic]
-        polymorphic_type(step).andand.constantize
-      else
-        reflection.klass
-      end
+    def sprintf(import)
+      options_for_import[import][:sprintf]
     end
-    
-    def wants_inline_association?
-      reflection.present?
+    def nullify(import)
+      options_for_import[import][:nullify]
     end
-    
-    def callback(step)
-      (options_for_step[step][:callback] || "derive_#{name}").to_sym
+    def overwrite(import)
+      options_for_import[import][:overwrite]
     end
-
-    def dictionary(step)
-      raise "shouldn't ask for this" unless wants_dictionary?(step) # don't try to initialize if there are no dictionary options
-      @dictionaries ||= {}
-      @dictionaries[step] ||= Dictionary.new(dictionary_options(step))
+    def upcase(import)
+      options_for_import[import][:upcase]
     end
-    
-    %w(dictionary split).each do |name|
-      eval <<-EOS
-        def #{name}_options(step)
-          options_for_step[step][:#{name}]
-        end
-      EOS
+    def units_field_name(import)
+      options_for_import[import][:units_field_name]
     end
-        
-    %w(from to set conditions weighting_association weighting_column weighting_disaggregator sprintf nullify overwrite upcase prefix unit_in_source field_number keep create static polymorphic_type).each do |name|
-      eval <<-EOS
-        def #{name}(step)
-          options_for_step[step][:#{name}]
-        end
-      EOS
+    def field_number(import)
+      options_for_import[import][:field_number]
+    end
+    def chars(import)
+      options_for_import[import][:chars]
+    end
+    def static(import)
+      options_for_import[import][:static]
     end
   end
 end