data_miner 0.4.0 → 0.4.1

data/README.rdoc

@@ -50,7 +50,8 @@ Put this in <tt>lib/tasks/data_miner_tasks.rake</tt>: (unfortunately I don't kno
 
   namespace :data_miner do
     task :run => :environment do
-      DataMiner.run :resource_names => ENV['RESOURCES'].to_s.split(/\s*,\s*/).flatten.compact
+      resource_names = %w{R RESOURCES RESOURCE RESOURCE_NAMES}.map { |possible_key| ENV[possible_key].to_s }.join.split(/\s*,\s*/).flatten.compact
+      DataMiner.run :resource_names => resource_names
     end
   end
 

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/data_miner.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]

data/lib/data_miner/attribute.rb

@@ -63,7 +63,10 @@ module DataMiner
       return false if !wants_overwriting? and !record.send(name).nil?
       what_it_was = record.send name
       what_it_should_be = value_from_row row
+
       record.send "#{name}=", what_it_should_be
+      record.send "#{name}_units=", (to_units || unit_from_source(row)).to_s if wants_units?
+      
       what_it_is = record.send name
       if what_it_is.nil? and !what_it_should_be.nil?
         DataMiner.logger.info "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
@@ -76,10 +79,11 @@ module DataMiner
     end
 
     def unit_from_source(row)
-      row[units_field_name].to_s.strip.underscore.to_sym
+      row[units_field_name || units_field_number].to_s.strip.underscore.to_sym
     end
     
     def do_convert(row, value)
+      logger.error "[data_miner gem] If you use :from_units, you need to set :to_units (#{resource.name}##{name})" unless wants_units?
       value.to_f.convert((from_units || unit_from_source(row)), to_units)
     end
     
@@ -125,7 +129,10 @@ module DataMiner
       overwrite != false
     end
     def wants_conversion?
-      from_units.present? or units_field_name.present?
+      from_units.present? or units_field_name.present? or units_field_number.present?
+    end
+    def wants_units?
+      to_units.present? or units_field_name.present? or units_field_number.present?
     end
     def wants_dictionary?
       options[:dictionary].present?
@@ -156,7 +163,7 @@ module DataMiner
       options[:from_units]
     end
     def to_units
-      options[:to_units]
+      options[:to_units] || options[:units]
     end
     def conditions
       options[:conditions]
@@ -176,6 +183,9 @@ module DataMiner
     def units_field_name
       options[:units_field_name]
     end
+    def units_field_number
+      options[:units_field_number]
+    end
     def field_number
       options[:field_number]
     end

data/lib/data_miner/configuration.rb

@@ -2,18 +2,17 @@ module DataMiner
   class Configuration
     include Blockenspiel::DSL
     
-    attr_accessor :resource, :runnables, :runnable_counter, :attributes, :unique_indices
+    attr_accessor :resource, :runnables, :runnable_counter, :attributes
 
     def initialize(resource)
       @runnables = Array.new
-      @unique_indices = Set.new
       @resource = resource
       @runnable_counter = 0
       @attributes = HashWithIndifferentAccess.new
     end
-
-    def unique_index(*args)
-      args.each { |arg| unique_indices.add arg }
+    
+    def logger
+      DataMiner.logger
     end
     
     def process(method_name_or_block_description, &block)
@@ -51,8 +50,97 @@ module DataMiner
       nil
     end
     
+    def import_runnables
+      runnables.select { |runnable| runnable.is_a? Import }
+    end
+    
+    def before_invoke
+      
+    end
+    
+    def after_invoke
+      make_sure_unit_definitions_make_sense
+      suggest_missing_column_migrations
+    end
+    
+    COMPLETE_UNIT_DEFINITIONS = [
+      [:units],
+      [:from_units, :to_units],
+      [:units_field_name],
+      [:units_field_name, :to_units],
+      [:units_field_number],
+      [:units_field_number, :to_units]
+    ]
+    
+    def make_sure_unit_definitions_make_sense
+      import_runnables.each do |runnable|
+        runnable.attributes.each do |_, attribute|
+          if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
+            logger.error %{
+
+================================
+
+[data_miner gem] You don't have a valid unit definition for #{resource.name}##{attribute.name}.
+
+You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.map(&:to_sym).inspect }.
+
+You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".
+
+================================
+            }
+          end
+        end
+      end
+    end
+    
+    def suggest_missing_column_migrations
+      missing_columns = Array.new
+      import_runnables.each do |runnable|
+        runnable.attributes.each do |_, attribute|
+          logger.error "[data_miner gem] You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
+          unless resource.column_names.include? attribute.name
+            missing_columns << attribute.name
+          end
+          if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
+            missing_columns << units_column
+          end
+        end
+      end
+      missing_columns.uniq!
+      if missing_columns.any?
+        logger.error %{
+
+================================
+
+[data_miner gem] On #{resource}, it looks like you're missing some columns...
+
+Please run this...
+
+  ./script/generate migration AddMissingColumnsTo#{resource.name}
+
+and **replace** the resulting file with this:
+
+  class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
+    def self.up
+#{missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :#{column_name.ends_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
+    end
+    
+    def self.down
+#{missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
+    end
+  end
+
+On the other hand, if you're working directly with create_table, this might be helpful:
+
+#{missing_columns.map { |column_name| "t.#{column_name.ends_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
+
+================================
+        }
+      end
+    end
+    
     cattr_accessor :resource_names
-    self.resource_names = Set.new
+    self.resource_names = Array.new
     class << self
       # Mine data. Defaults to all resource_names touched by DataMiner.
       #

data/lib/data_miner/import.rb

@@ -28,7 +28,6 @@ module DataMiner
     end
     
     def store(attr_name, attr_options = {})
-      raise "[data_miner gem] Column #{attr_name} doesn't exist on table #{resource.table_name}" unless resource.column_names.include?(attr_name)
       attributes[attr_name] = Attribute.new self, attr_name, attr_options
     end
     

data/lib/data_miner.rb

@@ -17,16 +17,21 @@ require 'data_miner/run'
 module DataMiner
   class MissingHashColumn < RuntimeError; end
   
-  include Log4r
+  include Log4r unless defined? Rails
 
   mattr_accessor :logger
   
   def self.start_logging
-    if defined?(Rails)
+    if defined? Rails
       self.logger = Rails.logger
     else
+      info_outputter = FileOutputter.new 'f1', :filename => 'data_miner.log'
+      error_outputter = Outputter.stderr
+      info_outputter.only_at DEBUG, INFO
+      error_outputter.only_at WARN, ERROR, FATAL
+      
       self.logger = Logger.new 'data_miner'
-      logger.outputters = FileOutputter.new 'f1', :filename => 'data_miner.log'
+      logger.add info_outputter, error_outputter
     end
   end
   
@@ -50,7 +55,7 @@ ActiveRecord::Base.class_eval do
       return
     end
     
-    DataMiner.resource_names.add self.name
+    DataMiner.resource_names.push self.name unless DataMiner.resource_names.include? self.name
     DataMiner.create_tables
 
     belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
@@ -68,6 +73,8 @@ ActiveRecord::Base.class_eval do
     self.data_miner_config = DataMiner::Configuration.new self
 
     Blockenspiel.invoke block, data_miner_config
+    
+    data_miner_config.after_invoke
   end
 end
 

data/test/data_miner_test.rb

@@ -233,6 +233,8 @@ class AutomobileVariant < ActiveRecord::Base
         store 'model_name', :field_name => 'model'
         store 'year'
         store 'fuel_type_code', :field_name => 'fuel_type'
+        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
+        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
         store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'cylinders', :field_name => 'no_cyc'
@@ -267,6 +269,8 @@ class AutomobileVariant < ActiveRecord::Base
         store 'make_name', :field_name => 'make'
         store 'model_name', :field_name => 'model'
         store 'fuel_type_code', :field_name => 'fl'
+        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
+        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
         store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'cylinders', :field_name => 'cyl'
@@ -297,6 +301,8 @@ class AutomobileVariant < ActiveRecord::Base
         store 'make_name', :field_name => 'make'
         store 'model_name', :field_name => 'model'
         store 'fuel_type_code', :field_name => 'FUEL TYPE'
+        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre # we'll convert these in a later step, just setting the stage
+        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre    # ditto
         store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'raw_fuel_efficiency_city', :field_name => 'UNRND CITY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
         store 'cylinders', :field_name => 'NUMB CYL'
@@ -668,7 +674,7 @@ class DataMinerTest < Test::Unit::TestCase
       assert_equal a, b
     end
       
-    should "hash things if no unique index is listed" do
+    should "hash things" do
       AutomobileVariant.data_miner_config.runnables[0].run(nil)
       assert AutomobileVariant.first.row_hash.present?
     end
@@ -734,9 +740,9 @@ class DataMinerTest < Test::Unit::TestCase
       assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
     end
     
-    should "mine residence survey day" do
-      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
-      assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
-    end
+    # should "mine residence survey day" do
+    #   ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+    #   assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
+    # end
   end
 end

data/test/test_helper.rb

@@ -66,11 +66,6 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
   execute 'ALTER TABLE census_divisions ADD PRIMARY KEY (number);'
   
   create_table "automobile_variants", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-    t.string   "automobile_make_id"
-    t.string   "automobile_model_id"
-    t.string   "automobile_model_year_id"
-    t.string   "automobile_fuel_type_id"
-    
     t.float    "fuel_efficiency_city"
     t.float    "fuel_efficiency_highway"
     t.string   "make_name"
@@ -94,6 +89,12 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.boolean  "injection"
     t.string   "carline_class_name"
     t.string   "speeds"
+    
+    t.string 'raw_fuel_efficiency_highway_units'
+    t.string 'raw_fuel_efficiency_city_units'
+    t.string 'fuel_efficiency_highway_units'
+    t.string 'fuel_efficiency_city_units'
+    
     t.string   "row_hash"
     t.integer 'data_miner_touch_count'
     t.integer 'data_miner_last_run_id'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors: 
 - Seamus Abshere