data_miner 0.4.1 → 0.4.2

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.4.2

data/data_miner.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.1"
+  s.version = "0.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]

data/lib/data_miner/configuration.rb

@@ -11,10 +11,6 @@ module DataMiner
       @attributes = HashWithIndifferentAccess.new
     end
     
-    def logger
-      DataMiner.logger
-    end
-    
     def process(method_name_or_block_description, &block)
       self.runnable_counter += 1
       runnables << DataMiner::Process.new(self, runnable_counter, method_name_or_block_description, &block)
@@ -76,7 +72,7 @@ module DataMiner
       import_runnables.each do |runnable|
         runnable.attributes.each do |_, attribute|
           if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
-            logger.error %{
+            DataMiner.logger.error %{
 
 ================================
 
@@ -97,7 +93,7 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
       missing_columns = Array.new
       import_runnables.each do |runnable|
         runnable.attributes.each do |_, attribute|
-          logger.error "[data_miner gem] You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
+          DataMiner.logger.error "[data_miner gem] You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
           unless resource.column_names.include? attribute.name
             missing_columns << attribute.name
           end
@@ -108,7 +104,7 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
       end
       missing_columns.uniq!
       if missing_columns.any?
-        logger.error %{
+        DataMiner.logger.error %{
 
 ================================
 

data/lib/data_miner/import.rb

@@ -28,10 +28,12 @@ module DataMiner
     end
     
     def store(attr_name, attr_options = {})
+      DataMiner.logger.error "[data_miner gem] You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
       attributes[attr_name] = Attribute.new self, attr_name, attr_options
     end
     
     def key(attr_name, attr_options = {})
+      DataMiner.logger.error "[data_miner gem] You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
       @key = attr_name
       store attr_name, attr_options
     end

data/test/data_miner_test.rb

@@ -412,14 +412,12 @@ class Country < ActiveRecord::Base
   
   data_miner do
     import 'The official ISO country list', :url => 'http://www.iso.org/iso/list-en1-semic-3.txt', :skip => 2, :headers => false, :delimiter => ';' do
-      key 'iso_3166'
-      store 'iso_3166', :field_number => 1
+      key 'iso_3166', :field_number => 1
       store 'name', :field_number => 0
     end
     
     import 'A Princeton dataset with better capitalization', :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do
-      key 'iso_3166'
-      store 'iso_3166', :field_name => 'country code'
+      key 'iso_3166', :field_name => 'country code'
       store 'name', :field_name => 'country'
     end
   end
@@ -430,11 +428,10 @@ class Airport < ActiveRecord::Base
   
   data_miner do
     import :url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false, :select => lambda { |row| row[4].present? } do
-      key 'iata_code'
+      key 'iata_code', :field_number => 4
       store 'name', :field_number => 1
       store 'city', :field_number => 2
       store 'country_name', :field_number => 3
-      store 'iata_code', :field_number => 4
       store 'latitude', :field_number => 6
       store 'longitude', :field_number => 7
     end
@@ -446,17 +443,15 @@ class CensusRegion < ActiveRecord::Base
   
   data_miner do
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Region'].to_i > 0 and row['Division'].to_s.strip == 'X'} do
-      key 'number'
+      key 'number', :field_name => 'Region'
       store 'name', :field_name => 'Name'
-      store 'number', :field_name => 'Region'
     end
     
     # pretend this is a different data source
     # fake! just for testing purposes
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Region'].to_i > 0 and row['Division'].to_s.strip == 'X'} do
-      key 'number'
+      key 'number', :field_name => 'Region'
       store 'name', :field_name => 'Name'
-      store 'number', :field_name => 'Region'
     end
   end
 end
@@ -467,9 +462,8 @@ class CensusDivision < ActiveRecord::Base
   
   data_miner do
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
-      key 'number'
+      key 'number', :field_name => 'Division'
       store 'name', :field_name => 'Name'
-      store 'number', :field_name => 'Division'
       store 'census_region_number', :field_name => 'Region'
       store 'census_region_name', :field_name => 'Region', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv' }
     end
@@ -488,8 +482,7 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
     # conversions are NOT performed here, since we first have to zero out legitimate skips
     # otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
     import 'RECs 2005 (but not converting units to metric just yet)', :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv', :headers => :upcase do
-      key 'department_of_energy_identifier'
-      store 'department_of_energy_identifier', :field_name => 'DOEID'
+      key 'department_of_energy_identifier', :field_name => 'DOEID'
       
       store 'residence_class', :field_name => 'TYPEHUQ', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/typehuq/typehuq.csv' }
       store 'construction_year', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Date in the middle (synthetic)', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/yearmade/yearmade.csv' }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors: 
 - Seamus Abshere