data_miner 0.3.7 → 0.3.8

data/README.rdoc

@@ -44,7 +44,7 @@ Put this in <tt>lib/tasks/data_miner_tasks.rake</tt>: (unfortunately I don't kno
 
   namespace :data_miner do
     task :run => :environment do
-      DataMiner.run :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
+      DataMiner.run :resource_names => ENV['RESOURCES'].to_s.split(/\s*,\s*/).flatten.compact
     end
   end
 

data/VERSION

@@ -1 +1 @@
-0.3.7
+0.3.8

data/data_miner.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.3.7"
+  s.version = "0.3.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
@@ -32,7 +32,6 @@ Gem::Specification.new do |s|
      "lib/data_miner/import.rb",
      "lib/data_miner/process.rb",
      "lib/data_miner/run.rb",
-     "lib/data_miner/target.rb",
      "lib/data_miner/william_james_cartesian_product.rb",
      "test/data_miner_test.rb",
      "test/test_helper.rb"

data/lib/data_miner.rb

@@ -12,7 +12,6 @@ require 'data_miner/configuration'
 require 'data_miner/dictionary'
 require 'data_miner/import'
 require 'data_miner/process'
-require 'data_miner/target'
 require 'data_miner/run'
 
 # TODO: move to gem
@@ -38,8 +37,8 @@ module DataMiner
     DataMiner::Configuration.run options
   end
   
-  def self.classes
-    DataMiner::Configuration.classes
+  def self.resource_names
+    DataMiner::Configuration.resource_names
   end
   
   def self.create_tables
@@ -53,10 +52,9 @@ ActiveRecord::Base.class_eval do
       logger.error "[DataMiner gem] Database table `#{table_name}` doesn't exist. DataMiner probably won't work properly until you run a migration or otherwise fix the schema."
       return
     end
-      
-    DataMiner.classes.add self
+    
+    DataMiner.resource_names.add self.name
     DataMiner.create_tables
-    DataMiner::Target.find_or_create_by_name name
 
     belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
     

data/lib/data_miner/attribute.rb

@@ -1,15 +1,15 @@
 module DataMiner
   class Attribute
-    attr_accessor :klass, :name, :options_for_import
+    attr_accessor :resource, :name, :options_for_import
 
-    def initialize(klass, name)
-      @klass = klass
+    def initialize(resource, name)
+      @resource = resource
       @name = name
       @options_for_import = {}
     end
         
     def inspect
-      "Attribute(#{klass}##{name})"
+      "Attribute(#{resource}##{name})"
     end
 
     def stored_by?(import)
@@ -56,11 +56,21 @@ module DataMiner
     end
         
     # this will overwrite nils, even if wants_overwriting?(import) is false
+    # returns true if an attr was changed, otherwise false
     def set_record_from_row(import, record, row)
-      return if !wants_overwriting?(import) and !record.send(name).nil?
-      value = value_from_row(import, row)
-      record.send "#{name}=", value
-      DataMiner.logger.info("ActiveRecord didn't like trying to set #{klass}.#{name} = #{value}") if !value.nil? and record.send(name).nil?
+      return false if !wants_overwriting?(import) and !record.send(name).nil?
+      what_it_was = record.send name
+      what_it_should_be = value_from_row import, row
+      record.send "#{name}=", what_it_should_be
+      what_it_is = record.send name
+      if what_it_is.nil? and !what_it_should_be.nil?
+        DataMiner.logger.info "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
+        nil
+      elsif what_it_is == what_it_was
+        false
+      else
+        true
+      end
     end
 
     def unit_from_source(import, row)
@@ -87,7 +97,7 @@ module DataMiner
     end
   
     def column_type
-      klass.columns_hash[name.to_s].type
+      resource.columns_hash[name.to_s].type
     end
     
     def dictionary(import)

data/lib/data_miner/configuration.rb

@@ -2,12 +2,12 @@ module DataMiner
   class Configuration
     include Blockenspiel::DSL
     
-    attr_accessor :klass, :runnables, :runnable_counter, :attributes, :unique_indices
+    attr_accessor :resource, :runnables, :runnable_counter, :attributes, :unique_indices
 
-    def initialize(klass)
+    def initialize(resource)
       @runnables = Array.new
       @unique_indices = Set.new
-      @klass = klass
+      @resource = resource
       @runnable_counter = 0
       @attributes = HashWithIndifferentAccess.new
     end
@@ -35,7 +35,7 @@ module DataMiner
         
     def after_invoke
       if unique_indices.empty?
-        raise(MissingHashColumn, "No unique_index defined for #{klass.name}, so you need a row_hash:string column.") unless klass.column_names.include?('row_hash')
+        raise(MissingHashColumn, "No unique_index defined for #{resource.name}, so you need a row_hash:string column.") unless resource.column_names.include?('row_hash')
         unique_indices.add 'row_hash'
       end
       runnables.select { |runnable| runnable.is_a?(Import) }.each { |runnable| unique_indices.each { |unique_index| runnable.store(unique_index) unless runnable.stores?(unique_index) } }
@@ -43,10 +43,9 @@ module DataMiner
 
     # Mine data for this class.
     def run(options = {})
-      target = DataMiner::Target.find(klass.name)
       finished = false
-      run = target.runs.create! :started_at => Time.now
-      klass.delete_all if options[:from_scratch]
+      run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name
+      resource.delete_all if options[:from_scratch]
       begin
         runnables.each { |runnable| runnable.run(run) }
         finished = true
@@ -56,34 +55,26 @@ module DataMiner
       nil
     end
     
-    cattr_accessor :classes
-    self.classes = Set.new
+    cattr_accessor :resource_names
+    self.resource_names = Set.new
     class << self
-      # Mine data. Defaults to all classes touched by DataMiner.
+      # Mine data. Defaults to all resource_names touched by DataMiner.
       #
       # Options
-      # * <tt>:class_names</tt>: provide an array class names to mine
+      # * <tt>:resource_names</tt>: array of resource (class) names to mine
       def run(options = {})
-        classes.each do |klass|
-          if options[:class_names].blank? or options[:class_names].include?(klass.name)
-            klass.data_miner_config.run options
+        resource_names.each do |resource_name|
+          if options[:resource_names].blank? or options[:resource_names].include?(resource_name)
+            resource_name.constantize.data_miner_config.run options
           end
         end
       end
             
       def create_tables
         c = ActiveRecord::Base.connection
-        unless c.table_exists?('data_miner_targets')
-          c.create_table 'data_miner_targets', :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-            t.string 'name'
-            t.datetime 'created_at'
-            t.datetime 'updated_at'
-          end
-          c.execute 'ALTER TABLE data_miner_targets ADD PRIMARY KEY (name);'
-        end
         unless c.table_exists?('data_miner_runs')
           c.create_table 'data_miner_runs', :options => 'ENGINE=InnoDB default charset=utf8' do |t|
-            t.string 'data_miner_target_id'
+            t.string 'resource_name'
             t.boolean 'finished'
             t.datetime 'started_at'
             t.datetime 'ended_at'

data/lib/data_miner/import.rb

@@ -2,7 +2,7 @@ module DataMiner
   class Import
     attr_accessor :configuration, :position_in_run, :options, :table, :errata
     attr_accessor :description
-    delegate :klass, :to => :configuration
+    delegate :resource, :to => :configuration
     delegate :unique_indices, :to => :configuration
     
     def initialize(configuration, position_in_run, description, options = {}, &block)
@@ -11,12 +11,12 @@ module DataMiner
       @description = description
       @options = options
       yield self if block_given? # pull in attributes
-      @errata = Errata.new(:url => options[:errata], :klass => klass) if options[:errata]
+      @errata = Errata.new(:url => options[:errata], :klass => resource) if options[:errata]
       @table = RemoteTable.new(options.slice(:url, :filename, :post_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop))
     end
 
     def inspect
-      "Import(#{klass}) position #{position_in_run} (#{description})"
+      "Import(#{resource}) position #{position_in_run} (#{description})"
     end
 
     def attributes
@@ -28,7 +28,7 @@ module DataMiner
     end
 
     def store(attr_name, attr_options = {})
-      configuration.attributes[attr_name] ||= Attribute.new(klass, attr_name)
+      configuration.attributes[attr_name] ||= Attribute.new(resource, attr_name)
       configuration.attributes[attr_name].options_for_import[self] = attr_options
     end
 
@@ -45,14 +45,16 @@ module DataMiner
         
         record_set = WilliamJamesCartesianProduct.cart_prod(*unifying_values).map do |combination|
           next if combination.include?(nil)
-          klass.send "find_or_initialize_by_#{unique_indices.to_a.join('_and_')}", *combination
+          resource.send "find_or_initialize_by_#{unique_indices.to_a.join('_and_')}", *combination
         end.flatten
 
         Array.wrap(record_set).each do |record|
-          attributes.values.each { |attr| attr.set_record_from_row(self, record, row) }
+          hits = attributes.values.map { |attr| attr.set_record_from_row self, record, row }
           record.data_miner_touch_count ||= 0
-          record.data_miner_touch_count += 1
-          record.data_miner_last_run = run
+          if hits.any?
+            record.data_miner_touch_count += 1
+            record.data_miner_last_run = run
+          end
           record.save!
         end
       end

data/lib/data_miner/process.rb

@@ -3,7 +3,7 @@ module DataMiner
     attr_accessor :configuration, :position_in_run
     attr_accessor :method_name
     attr_accessor :block_description, :block
-    delegate :klass, :to => :configuration
+    delegate :resource, :to => :configuration
 
     def initialize(configuration, position_in_run, method_name_or_block_description, &block)
       @configuration = configuration
@@ -12,16 +12,16 @@ module DataMiner
         @block_description = method_name_or_block_description
         @block = block
       else
-        @method_name = method_name
+        @method_name = method_name_or_block_description
       end
     end
     
     def inspect
-      str = "Process(#{klass}) position #{position_in_run}"
+      str = "Process(#{resource}) position #{position_in_run}"
       if block
-        str << " called :#{method_name}"
-      else
         str << " ran block (#{block_description})"
+      else
+        str << " called :#{method_name}"
       end
     end
     
@@ -29,7 +29,7 @@ module DataMiner
       if block
         block.call
       else
-        klass.send method_name
+        resource.send method_name
       end
       DataMiner.logger.info "ran #{inspect}"
     end

data/lib/data_miner/run.rb

@@ -2,6 +2,5 @@ module DataMiner
   class Run < ActiveRecord::Base
     set_table_name 'data_miner_runs'
     default_scope :order => 'id ASC'
-    belongs_to :target, :class_name => '::DataMiner::Target', :foreign_key => 'data_miner_target_id'
   end
 end

data/test/data_miner_test.rb

@@ -876,25 +876,13 @@ class DataMinerTest < Test::Unit::TestCase
     assert AutomobileVariant.first.fuel_efficiency_city.present?
   end
   
-  # should "mine multiple classes in the correct order" do
-  #   DataMiner.run
-  #   uy = Country.find_by_iso_3166('UY')
-  #   assert_equal 'Uruguay', uy.name
-  # end
-  
-  should "have a target record for every class that is mined" do
-    DataMiner.run :class_names => %w{ Country }
-    assert DataMiner::Target.exists?(:name => 'Country')
-    assert_equal 1, DataMiner::Target.count(:conditions => {:name => 'country'})
-  end
-  
   should "keep a log when it does a run" do
     approx_started_at = Time.now
-    DataMiner.run :class_names => %w{ Country }
+    DataMiner.run :resource_names => %w{ Country }
     approx_ended_at = Time.now
-    target = DataMiner::Target.find_by_name('Country')
-    assert (target.runs.last.started_at - approx_started_at).abs < 5 # seconds
-    assert (target.runs.last.ended_at - approx_ended_at).abs < 5 # seconds
+    last_run = DataMiner::Run.first(:conditions => { :resource_name => 'Country' }, :order => 'id DESC')
+    assert (last_run.started_at - approx_started_at).abs < 5 # seconds
+    assert (last_run.ended_at - approx_ended_at).abs < 5 # seconds
   end
   
   should "request a re-import from scratch" do
@@ -902,31 +890,37 @@ class DataMinerTest < Test::Unit::TestCase
     c.iso_3166 = 'JUNK'
     c.save!
     assert Country.exists?(:iso_3166 => 'JUNK')
-    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
     assert !Country.exists?(:iso_3166 => 'JUNK')
   end
   
   should "track how many times a row was touched" do
-    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
+    assert_equal 1, Country.first.data_miner_touch_count
+    DataMiner.run :resource_names => %w{ Country }
     assert_equal 1, Country.first.data_miner_touch_count
-    DataMiner.run :class_names => %w{ Country }
-    assert_equal 2, Country.first.data_miner_touch_count
   end
   
   should "keep track of what the last import run that touched a row was" do
-    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
     a = DataMiner::Run.last
     assert_equal a, Country.first.data_miner_last_run
-    DataMiner.run :class_names => %w{ Country }
+    DataMiner.run :resource_names => %w{ Country }
     b = DataMiner::Run.last
     assert a != b
-    assert_equal b, Country.first.data_miner_last_run
+    assert_equal a, Country.first.data_miner_last_run
   end
   
   unless ENV['FAST'] == 'true'
     should "import using a dictionary" do
-      DataMiner.run :class_names => %w{ ResidentialEnergyConsumptionSurveyResponse }
+      DataMiner.run :resource_names => %w{ ResidentialEnergyConsumptionSurveyResponse }
       assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
     end
+    
+    should "mine multiple classes in the correct order" do
+      DataMiner.run
+      uy = Country.find_by_iso_3166('UY')
+      assert_equal 'Uruguay', uy.name
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -108,7 +108,6 @@ files:
 - lib/data_miner/import.rb
 - lib/data_miner/process.rb
 - lib/data_miner/run.rb
-- lib/data_miner/target.rb
 - lib/data_miner/william_james_cartesian_product.rb
 - test/data_miner_test.rb
 - test/test_helper.rb

data/lib/data_miner/target.rb

@@ -1,26 +0,0 @@
-module DataMiner
-  class Target < ActiveRecord::Base
-    set_table_name 'data_miner_targets'
-    set_primary_key :name
-    has_many :runs, :class_name => '::DataMiner::Run', :foreign_key => 'data_miner_target_id'
-
-    def klass
-      name.constantize
-    end
-    
-    def run(options = {})
-      klass.data_miner_config.run options
-    end
-
-    def included_in_list_of_targets
-      msg = "must have a data_miner block"
-      unless DataMiner.classes.include?(name.constantize)
-        errors.add :name, msg
-      end
-    rescue NameError
-      errors.add :name, msg
-    end
-    
-    validate :included_in_list_of_targets
-  end
-end