data_miner 0.4.38 → 0.4.40

data/Rakefile

@@ -10,7 +10,7 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    gem.add_dependency 'remote_table', '>=0.2.25'
+    gem.add_dependency 'remote_table', '>=0.2.26'
     gem.add_dependency 'escape', '>=0.0.4'
     gem.add_dependency 'activerecord', '>=2.3.4'
     gem.add_dependency 'activesupport', '>=2.3.4'
@@ -20,16 +20,20 @@ begin
     gem.add_dependency 'log4r', '>=1.1.7'
     gem.add_dependency 'errata', '>=0.2.1'
     gem.add_dependency 'taps', '>=0.3.5'
-    gem.add_development_dependency "loose_tight_dictionary", ">=0.0.5"
-    gem.require_path = "lib"
-    gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
+    ## sabshere 5/25/10 i was told not to do this
+    # gem.add_development_dependency "loose_tight_dictionary", ">=0.0.5"
+    ## sabshere 5/25/10 i don't think i need this
+    # gem.require_path = "lib"
+    # gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
     gem.rdoc_options << '--line-numbers' << '--inline-source'
-    gem.rubyforge_project = "dataminer"
+    ## sabshere 5/25/10 obsolete
+    # gem.rubyforge_project = "dataminer"
   end
   Jeweler::GemcutterTasks.new
-  Jeweler::RubyforgeTasks.new do |rubyforge|
-    rubyforge.doc_task = "rdoc"
-  end
+  ## sabshere 5/25/10 obsolete
+  # Jeweler::RubyforgeTasks.new do |rubyforge|
+  #   rubyforge.doc_task = "rdoc"
+  # end
 rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
 end

data/VERSION

@@ -1 +1 @@
-0.4.38
+0.4.40

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.38"
+  s.version = "0.4.40"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-24}
+  s.date = %q{2010-05-25}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
      "data_miner.gemspec",
      "lib/data_miner.rb",
      "lib/data_miner/attribute.rb",
-     "lib/data_miner/configuration.rb",
+     "lib/data_miner/base.rb",
      "lib/data_miner/dictionary.rb",
      "lib/data_miner/import.rb",
      "lib/data_miner/process.rb",
@@ -40,7 +40,6 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/seamusabshere/data_miner}
   s.rdoc_options = ["--charset=UTF-8", "--line-numbers", "--inline-source"]
   s.require_paths = ["lib"]
-  s.rubyforge_project = %q{dataminer}
   s.rubygems_version = %q{1.3.6}
   s.summary = %q{Mine remote data into your ActiveRecord models.}
   s.test_files = [
@@ -53,7 +52,7 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.25"])
+      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.26"])
       s.add_runtime_dependency(%q<escape>, [">= 0.0.4"])
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
@@ -63,9 +62,8 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
       s.add_runtime_dependency(%q<errata>, [">= 0.2.1"])
       s.add_runtime_dependency(%q<taps>, [">= 0.3.5"])
-      s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     else
-      s.add_dependency(%q<remote_table>, [">= 0.2.25"])
+      s.add_dependency(%q<remote_table>, [">= 0.2.26"])
       s.add_dependency(%q<escape>, [">= 0.0.4"])
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<activesupport>, [">= 2.3.4"])
@@ -75,10 +73,9 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<log4r>, [">= 1.1.7"])
       s.add_dependency(%q<errata>, [">= 0.2.1"])
       s.add_dependency(%q<taps>, [">= 0.3.5"])
-      s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     end
   else
-    s.add_dependency(%q<remote_table>, [">= 0.2.25"])
+    s.add_dependency(%q<remote_table>, [">= 0.2.26"])
     s.add_dependency(%q<escape>, [">= 0.0.4"])
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<activesupport>, [">= 2.3.4"])
@@ -88,7 +85,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<log4r>, [">= 1.1.7"])
     s.add_dependency(%q<errata>, [">= 0.2.1"])
     s.add_dependency(%q<taps>, [">= 0.3.5"])
-    s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
   end
 end
 

data/lib/data_miner/{configuration.rb → base.rb}

@@ -1,5 +1,5 @@
 module DataMiner
-  class Configuration
+  class Base
     include Blockenspiel::DSL
     
     attr_accessor :resource, :steps, :step_counter, :attributes
@@ -46,12 +46,13 @@ module DataMiner
     def run(options = {})
       options.symbolize_keys!
       
-      return if DataMiner::Configuration.call_stack.include? resource.name
-      DataMiner::Configuration.call_stack.push resource.name
+      return if DataMiner::Base.call_stack.include? resource.name
+      DataMiner::Base.call_stack.push resource.name
       
       finished = false
+      skipped = false
       if DataMiner::Run.table_exists?
-        run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name if DataMiner::Run.table_exists?
+        run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name, :killed => true
       else
         run = nil
         DataMiner.log_info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
@@ -63,11 +64,15 @@ module DataMiner
           resource.reset_column_information
         end
         finished = true
-      rescue DataMiner::Stop
+      rescue DataMiner::Finish
         finished = true
+      rescue DataMiner::Skip
+        skipped = true
       ensure
-        run.update_attributes! :ended_at => Time.now, :finished => finished if DataMiner::Run.table_exists?
-        DataMiner::Configuration.call_stack.clear if DataMiner::Configuration.call_stack.first == resource.name
+        if DataMiner::Run.table_exists?
+          run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
+        end
+        DataMiner::Base.call_stack.clear if DataMiner::Base.call_stack.first == resource.name
       end
       nil
     end
@@ -118,15 +123,6 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
     
     def suggest_missing_column_migrations
       missing_columns = Array.new
-      non_essential_missing_columns = Array.new
-      unless resource.column_names.include?('data_miner_touch_count')
-        non_essential_missing_columns << 'data_miner_touch_count'
-        DataMiner.log_info "Not counting how many times a row has been touched by data_miner."
-      end
-      unless resource.column_names.include?('data_miner_last_run_id')
-        non_essential_missing_columns << 'data_miner_last_run_id'
-        DataMiner.log_info "Not recording which run touched a row."
-      end
       
       import_steps.each do |step|
         step.attributes.each do |_, attribute|
@@ -156,19 +152,16 @@ and **replace** the resulting file with this:
   class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
     def self.up
 #{missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
-#{non_essential_missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :integer #optional" }.join("\n") }
     end
     
     def self.down
 #{missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
-#{non_essential_missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name} #optional" }.join("\n") }
     end
   end
 
 On the other hand, if you're working directly with create_table, this might be helpful:
 
 #{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
-#{non_essential_missing_columns.map { |column_name| "t.integer '#{column_name}' #optional" }.join("\n") }
 
 ================================
         }
@@ -190,7 +183,7 @@ On the other hand, if you're working directly with create_table, this might be h
         
         resource_names.each do |resource_name|
           if options[:resource_names].blank? or options[:resource_names].include?(resource_name)
-            resource_name.constantize.data_miner_config.run options
+            resource_name.constantize.data_miner_base.run options
           end
         end
       end

data/lib/data_miner/import.rb

@@ -3,15 +3,15 @@ module DataMiner
     include Blockenspiel::DSL
     
     attr_reader :attributes
-    attr_accessor :configuration, :position_in_run, :table
+    attr_accessor :base, :position_in_run, :table
     attr_accessor :description
-    delegate :resource, :to => :configuration
+    delegate :resource, :to => :base
     
-    def initialize(configuration, position_in_run, description, table_options = {})
+    def initialize(base, position_in_run, description, table_options = {})
       table_options.symbolize_keys!
 
       @attributes = ActiveSupport::OrderedHash.new
-      @configuration = configuration
+      @base = base
       @position_in_run = position_in_run
       @description = description
       
@@ -47,8 +47,6 @@ module DataMiner
     end
 
     def run(run)
-      increment_counter = resource.column_names.include?('data_miner_touch_count')
-      log_run = resource.column_names.include?('data_miner_last_run_id')
       primary_key = resource.primary_key
       test_counter = 0
 
@@ -63,11 +61,7 @@ OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_na
         end
       
         record = resource.send "find_or_initialize_by_#{@key}", attributes[@key].value_from_row(row)
-        changes = attributes.map { |_, attr| attr.set_record_from_row record, row }
-        if changes.any?
-          record.increment :data_miner_touch_count if increment_counter
-          record.data_miner_last_run = run if log_run
-        end
+        attributes.each { |_, attr| attr.set_record_from_row record, row }
         record.save! if record.send(primary_key).present?
       end
       DataMiner.log_info "performed #{inspect}"

data/lib/data_miner/process.rb

@@ -1,12 +1,12 @@
 module DataMiner
   class Process
-    attr_accessor :configuration, :position_in_run
+    attr_accessor :base, :position_in_run
     attr_accessor :method_name
     attr_accessor :block_description, :block
-    delegate :resource, :to => :configuration
+    delegate :resource, :to => :base
 
-    def initialize(configuration, position_in_run, method_name_or_block_description, &block)
-      @configuration = configuration
+    def initialize(base, position_in_run, method_name_or_block_description, &block)
+      @base = base
       @position_in_run = position_in_run
       if block_given?
         @block_description = method_name_or_block_description

data/lib/data_miner/run.rb

@@ -5,22 +5,21 @@ module DataMiner
     def resource
       resource_name.constantize
     end
-    
-    def resource_records_last_touched_by_me
-      resource.scoped :conditions => { :data_miner_last_run_id => id }
-    end
-    
+        
     class << self
       def create_tables
-        return if table_exists?
-        connection.create_table 'data_miner_runs' do |t|
+        return if table_exists? and column_names.include?('skipped') # force a drop
+        connection.create_table 'data_miner_runs', :force => true do |t|
           t.string 'resource_name'
+          t.boolean 'killed'
+          t.boolean 'skipped'
           t.boolean 'finished'
           t.datetime 'started_at'
-          t.datetime 'ended_at'
+          t.datetime 'terminated_at'
           t.datetime 'created_at'
           t.datetime 'updated_at'
         end
+        reset_column_information
       end
     end
   end

data/lib/data_miner/schema.rb

@@ -2,13 +2,13 @@ module DataMiner
   class Schema
     include Blockenspiel::DSL
     
-    attr_reader :configuration
+    attr_reader :base
     attr_reader :position_in_run
     attr_reader :create_table_options
-    delegate :resource, :to => :configuration
+    delegate :resource, :to => :base
 
-    def initialize(configuration, position_in_run, create_table_options)
-      @configuration = configuration
+    def initialize(base, position_in_run, create_table_options)
+      @base = base
       @position_in_run = position_in_run
       @create_table_options = create_table_options
       @create_table_options.symbolize_keys!
@@ -169,9 +169,7 @@ module DataMiner
     
     EXTRA_COLUMNS = {
       :updated_at => :datetime,
-      :created_at => :datetime,
-      :data_miner_last_run_id => :integer,
-      :data_miner_touch_count => :integer
+      :created_at => :datetime
     }
     def _add_extra_columns
       EXTRA_COLUMNS.each do |extra_name, extra_type|
@@ -237,4 +235,4 @@ module DataMiner
       end
     end
   end
-end
+end

data/lib/data_miner/tap.rb

@@ -1,17 +1,17 @@
 
 module DataMiner
   class Tap
-    attr_reader :configuration
+    attr_reader :base
     attr_reader :position_in_run
     attr_reader :description
     attr_reader :source
     attr_reader :options
-    delegate :resource, :to => :configuration
+    delegate :resource, :to => :base
 
-    def initialize(configuration, position_in_run, description, source, options = {})
+    def initialize(base, position_in_run, description, source, options = {})
       options.symbolize_keys!
       DataMiner.log_or_raise "Tap has to be the first step." unless position_in_run == 0
-      @configuration = configuration
+      @base = base
       @position_in_run = position_in_run
       @description = description
       @source = source

data/lib/data_miner.rb

@@ -21,7 +21,7 @@ require 'tmpdir'
 require 'zlib'
 
 require 'data_miner/attribute'
-require 'data_miner/configuration'
+require 'data_miner/base'
 require 'data_miner/dictionary'
 require 'data_miner/import'
 require 'data_miner/tap'
@@ -31,7 +31,8 @@ require 'data_miner/schema'
 
 module DataMiner
   class MissingHashColumn < StandardError; end
-  class Stop < StandardError; end
+  class Finish < StandardError; end
+  class Skip < StandardError; end
   
   mattr_accessor :logger
   
@@ -71,11 +72,11 @@ module DataMiner
   end
   
   def self.run(options = {})
-    DataMiner::Configuration.run options
+    DataMiner::Base.run options
   end
   
   def self.resource_names
-    DataMiner::Configuration.resource_names
+    DataMiner::Base.resource_names
   end
   
     # TODO this should probably live somewhere else
@@ -111,22 +112,20 @@ ActiveRecord::Base.class_eval do
     
     DataMiner.resource_names.push self.name unless DataMiner.resource_names.include? self.name
 
-    belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
-    
     # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
     class_eval do
-      cattr_accessor :data_miner_config
+      cattr_accessor :data_miner_base
       def self.data_miner_runs
         DataMiner::Run.scoped :conditions => { :resource_name => name }
       end
       def self.run_data_miner!(options = {})
-        data_miner_config.run options
+        data_miner_base.run options
       end
     end
-    self.data_miner_config = DataMiner::Configuration.new self
+    self.data_miner_base = DataMiner::Base.new self
 
-    Blockenspiel.invoke block, data_miner_config
+    Blockenspiel.invoke block, data_miner_base
     
-    data_miner_config.after_invoke
+    data_miner_base.after_invoke
   end
 end

data/test/data_miner_test.rb

@@ -1114,12 +1114,14 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
       string  "make_year_name"
       datetime "created_at"
       datetime "updated_at"
-      integer  'data_miner_touch_count'
-      integer  'data_miner_last_run_id'
     end
     
-    process "stop if i tell you to" do
-      raise DataMiner::Stop if $stop_stop
+    process "finish if i tell you to" do
+      raise DataMiner::Finish if $force_finish
+    end
+    
+    process "skip if i tell you to" do
+      raise DataMiner::Skip if $force_skip
     end
 
     # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
@@ -1168,15 +1170,28 @@ class DataMinerTest < Test::Unit::TestCase
   end
     
   if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
-    should "stop and finish if it gets a DataMiner::Stop" do
+    should "stop and finish if it gets a DataMiner::Finish" do
+      AutomobileMakeFleetYear.delete_all
+      AutomobileMakeFleetYear.data_miner_runs.delete_all
+      $force_finish = true
+      AutomobileMakeFleetYear.run_data_miner!
+      assert_equal 0, AutomobileMakeFleetYear.count
+      assert_equal true, (AutomobileMakeFleetYear.data_miner_runs.count > 0)
+      assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? and not run.skipped and not run.killed? }
+      $force_finish = false
+      AutomobileMakeFleetYear.run_data_miner!
+      assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
+    end
+    
+    should "stop and register skipped if it gets a DataMiner::Skip" do
       AutomobileMakeFleetYear.delete_all
       AutomobileMakeFleetYear.data_miner_runs.delete_all
-      $stop_stop = true
+      $force_skip = true
       AutomobileMakeFleetYear.run_data_miner!
       assert_equal 0, AutomobileMakeFleetYear.count
       assert_equal true, (AutomobileMakeFleetYear.data_miner_runs.count > 0)
-      assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? }
-      $stop_stop = false
+      assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.skipped? and not run.finished? and not run.killed? }
+      $force_skip = false
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
     end
@@ -1188,12 +1203,10 @@ class DataMinerTest < Test::Unit::TestCase
         # t.datetime 'created_at'
         t.string   'census_region_name'
         # t.integer  'census_region_number'
-        # t.integer 'data_miner_touch_count'
-        # t.integer 'data_miner_last_run_id'
       end
       ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
       CensusDivisionTrois.reset_column_information
-      missing_columns = %w{ updated_at created_at census_region_number data_miner_last_run_id data_miner_touch_count }
+      missing_columns = %w{ updated_at created_at census_region_number }
 
       # sanity check
       missing_columns.each do |column|
@@ -1218,12 +1231,10 @@ class DataMinerTest < Test::Unit::TestCase
         # t.datetime 'created_at'
         t.string   'census_region_name'
         # t.integer  'census_region_number'
-        # t.integer 'data_miner_touch_count'
-        # t.integer 'data_miner_last_run_id'
       end
       ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_fours ADD INDEX (census_region_name)'
       CensusDivisionFour.reset_column_information
-      missing_columns = %w{ updated_at created_at census_region_number data_miner_last_run_id data_miner_touch_count }
+      missing_columns = %w{ updated_at created_at census_region_number }
     
       # sanity check
       missing_columns.each do |column|
@@ -1288,39 +1299,39 @@ class DataMinerTest < Test::Unit::TestCase
     end
     
     should "be idempotent" do
-      Country.data_miner_config.run
+      Country.data_miner_base.run
       a = Country.count
-      Country.data_miner_config.run
+      Country.data_miner_base.run
       b = Country.count
       assert_equal a, b
     
-      CensusRegion.data_miner_config.run
+      CensusRegion.data_miner_base.run
       a = CensusRegion.count
-      CensusRegion.data_miner_config.run
+      CensusRegion.data_miner_base.run
       b = CensusRegion.count
       assert_equal a, b
     end
       
     should "hash things" do
-      AutomobileVariant.data_miner_config.steps[0].run(nil)
+      AutomobileVariant.data_miner_base.steps[0].run(nil)
       assert AutomobileVariant.first.row_hash.present?
     end
   
     should "process a callback block instead of a method" do
       AutomobileVariant.delete_all
-      AutomobileVariant.data_miner_config.steps[0].run(nil)
+      AutomobileVariant.data_miner_base.steps[0].run(nil)
       assert !AutomobileVariant.first.fuel_efficiency_city.present?
-      AutomobileVariant.data_miner_config.steps.last.run(nil)
+      AutomobileVariant.data_miner_base.steps.last.run(nil)
       assert AutomobileVariant.first.fuel_efficiency_city.present?
     end
   
     should "keep a log when it does a run" do
       approx_started_at = Time.now
       DataMiner.run :resource_names => %w{ Country }
-      approx_ended_at = Time.now
+      approx_terminated_at = Time.now
       last_run = DataMiner::Run.first(:conditions => { :resource_name => 'Country' }, :order => 'id DESC')
       assert (last_run.started_at - approx_started_at).abs < 5 # seconds
-      assert (last_run.ended_at - approx_ended_at).abs < 5 # seconds
+      assert (last_run.terminated_at - approx_terminated_at).abs < 5 # seconds
     end
   
     should "request a re-import from scratch" do
@@ -1331,29 +1342,7 @@ class DataMinerTest < Test::Unit::TestCase
       DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
       assert !Country.exists?(:iso_3166 => 'JUNK')
     end
-  
-    should "track how many times a row was touched" do
-      DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
-      assert_equal 1, Country.first.data_miner_touch_count
-      DataMiner.run :resource_names => %w{ Country }
-      assert_equal 1, Country.first.data_miner_touch_count
-    end
-  
-    should "keep track of what the last import run that touched a row was" do
-      DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
-      a = DataMiner::Run.last
-      assert_equal a, Country.first.data_miner_last_run
-      DataMiner.run :resource_names => %w{ Country }
-      b = DataMiner::Run.last
-      assert a != b
-      assert_equal a, Country.first.data_miner_last_run
-    end
-  
-    should "be able to get how many rows affected by a run" do
-      DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
-      assert_equal Country.first.data_miner_last_run.resource_records_last_touched_by_me.count, Country.count
-    end
-  
+    
     should "know what runs were on a resource" do
       DataMiner.run :resource_names => %w{ Country }
       DataMiner.run :resource_names => %w{ Country }

data/test/test_helper.rb

@@ -82,8 +82,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "updated_at"
     
     t.string   "row_hash"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE t100_flight_segments ADD PRIMARY KEY (row_hash);'
 
@@ -100,8 +98,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.float    'longitude'
     t.datetime 'created_at'
     t.datetime 'updated_at'
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE airports ADD PRIMARY KEY (iata_code);'
   
@@ -110,8 +106,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   "name"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE countries ADD PRIMARY KEY (iso_3166);"
   
@@ -120,8 +114,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   "name"
     t.datetime "updated_at"
     t.datetime "created_at"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE census_regions ADD PRIMARY KEY (number);"
   
@@ -133,8 +125,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   'census_region_name'
     t.integer  'census_region_number'
     
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE census_divisions ADD PRIMARY KEY (number);'
   
@@ -146,8 +136,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   'census_region_name'
     t.integer  'census_region_number'
     
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE census_division_deux ADD PRIMARY KEY (number);'
   
@@ -159,8 +147,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   'census_region_name'
     t.integer  'census_region_number'
     
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE crosscalling_census_divisions ADD PRIMARY KEY (number);'
   
@@ -195,8 +181,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string 'fuel_efficiency_city_units'
     
     t.string   "row_hash"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_variants ADD PRIMARY KEY (row_hash);"
   
@@ -207,8 +191,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.float    "emission_factor"
     t.float    "annual_distance"
     t.string   "code"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_fuel_types ADD PRIMARY KEY (code);"
 
@@ -282,8 +264,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.integer 'data_miner_touch_count'
-    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE residential_energy_consumption_survey_responses ADD PRIMARY KEY (department_of_energy_identifier);"
   
@@ -302,8 +282,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     # t.float    'endpoint_fuel'
     t.datetime 'updated_at'
     t.datetime 'created_at'
-    t.integer  'data_miner_touch_count'
-    t.integer  'data_miner_last_run_id'
   end
   execute 'ALTER TABLE aircraft ADD PRIMARY KEY (icao_code);'
   
@@ -322,8 +300,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     # t.float    'endpoint_fuel'
     t.datetime 'updated_at'
     t.datetime 'created_at'
-    t.integer  'data_miner_touch_count'
-    t.integer  'data_miner_last_run_id'
   end
   execute 'ALTER TABLE aircraft_deux ADD PRIMARY KEY (icao_code);'
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 38
-  version: 0.4.38
+  - 40
+  version: 0.4.40
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-24 00:00:00 -04:00
+date: 2010-05-25 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,8 +28,8 @@ dependencies:
         segments: 
         - 0
         - 2
-        - 25
-        version: 0.2.25
+        - 26
+        version: 0.2.26
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -158,20 +158,6 @@ dependencies:
         version: 0.3.5
   type: :runtime
   version_requirements: *id010
-- !ruby/object:Gem::Dependency 
-  name: loose_tight_dictionary
-  prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 0
-        - 5
-        version: 0.0.5
-  type: :development
-  version_requirements: *id011
 description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
 email: seamus@abshere.net
 executables: []
@@ -192,7 +178,7 @@ files:
 - data_miner.gemspec
 - lib/data_miner.rb
 - lib/data_miner/attribute.rb
-- lib/data_miner/configuration.rb
+- lib/data_miner/base.rb
 - lib/data_miner/dictionary.rb
 - lib/data_miner/import.rb
 - lib/data_miner/process.rb
@@ -228,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: "0"
 requirements: []
 
-rubyforge_project: dataminer
+rubyforge_project: 
 rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3