data_miner 0.2.6 → 0.3.0

data/test/test_helper.rb

@@ -1,16 +1,147 @@
 require 'rubygems'
 require 'test/unit'
 require 'shoulda'
-require 'sqlite3'
+require 'ruby-debug'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'data_miner'
 
 ActiveRecord::Base.establish_connection(
-  'adapter' => 'sqlite3',
-  'database' => 'test/test.sqlite3'
+  'adapter' => 'mysql',
+  'database' => 'data_miner_test',
+  'username' => 'root',
+  'password' => ''
 )
 
 class Test::Unit::TestCase
 end
+
+ActiveRecord::Schema.define(:version => 20090819143429) do
+  create_table "airports", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string  "country_id"
+    
+    t.string   "iata_code"
+    t.string   "name"
+    t.string   "city"
+    t.string   "country_name"
+    t.float    "latitude"
+    t.float    "longitude"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+  execute "ALTER TABLE airports ADD PRIMARY KEY (iata_code);"
+  
+  create_table "countries", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "iso_3166"
+    t.string   "name"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+  execute "ALTER TABLE countries ADD PRIMARY KEY (iso_3166);"
+  
+  create_table "census_regions", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "name"
+    t.datetime "updated_at"
+    t.datetime "created_at"
+    t.integer  "number"
+  end
+  execute "ALTER TABLE census_regions ADD PRIMARY KEY (number);"
+  
+  create_table "automobile_variants", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "automobile_make_id"
+    t.string   "automobile_model_id"
+    t.string   "automobile_model_year_id"
+    t.string   "automobile_fuel_type_id"
+    
+    t.float    "fuel_efficiency_city"
+    t.float    "fuel_efficiency_highway"
+    t.string   "make_name"
+    t.string   "model_name"
+    t.string   "year"
+    t.string   "fuel_type_code"
+    t.datetime "updated_at"
+    t.datetime "created_at"
+    t.string   "transmission"
+    t.string   "drive"
+    t.boolean  "turbo"
+    t.boolean  "supercharger"
+    t.integer  "cylinders"
+    t.float    "displacement"
+    t.float    "raw_fuel_efficiency_city"
+    t.float    "raw_fuel_efficiency_highway"
+    t.integer  "carline_mfr_code"
+    t.integer  "vi_mfr_code"
+    t.integer  "carline_code"
+    t.integer  "carline_class_code"
+    t.boolean  "injection"
+    t.string   "carline_class_name"
+    t.string   "speeds"
+    t.string   "row_hash"
+  end
+  execute "ALTER TABLE automobile_variants ADD PRIMARY KEY (row_hash);"
+  
+  create_table "automobile_fuel_types", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "name"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.float    "emission_factor"
+    t.float    "annual_distance"
+    t.string   "code"
+  end
+  execute "ALTER TABLE automobile_fuel_types ADD PRIMARY KEY (code);"
+
+  create_table "automobile_make_fleet_years", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "automobile_make_id"
+    t.string   "automobile_model_year_id"
+    t.integer  "automobile_make_year_id"
+
+    t.string   "fleet"
+    t.string   "make_name"
+    t.string   "year"
+    t.float    "fuel_efficiency"
+    t.integer  "volume"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+
+    t.string   "row_hash"
+  end
+  execute "ALTER TABLE automobile_make_fleet_years ADD PRIMARY KEY (row_hash);"
+
+  create_table "automobile_make_years", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.integer  "automobile_make_id" # user-defined
+    t.integer  "automobile_model_year_id" # user-defined
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.float    "fuel_efficiency"
+    t.integer  "volume"
+    t.string   "row_hash"
+  end
+  execute "ALTER TABLE automobile_make_years ADD PRIMARY KEY (row_hash);"
+
+  create_table "automobile_makes", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "name"
+    t.datetime "updated_at"
+    t.datetime "created_at"
+    t.float    "fuel_efficiency"
+    t.boolean  "major"
+  end
+  execute "ALTER TABLE automobile_makes ADD PRIMARY KEY (name);"
+
+  create_table "automobile_model_years", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.integer  "year"
+    t.datetime "updated_at"
+    t.datetime "created_at"
+    t.float    "fuel_efficiency"
+  end
+  execute "ALTER TABLE automobile_model_years ADD PRIMARY KEY (year);"
+
+  create_table "automobile_models", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string   "name"
+    t.string   "automobile_make_id"
+    t.datetime "updated_at"
+    t.datetime "created_at"
+    t.string   "row_hash"
+  end
+  execute "ALTER TABLE automobile_models ADD PRIMARY KEY (row_hash);"
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -10,18 +10,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-05 00:00:00 -05:00
+date: 2010-02-25 00:00:00 -05:00
 default_executable: 
 dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: remote_table
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.2.1
+    version: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 2.3.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
@@ -29,9 +39,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 2.3.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: andand
@@ -39,9 +49,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.3.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: errata
@@ -49,9 +59,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 0.1.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: conversions
@@ -59,19 +69,19 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: 1.4.3
     version: 
 - !ruby/object:Gem::Dependency 
-  name: remote_table
+  name: blockenspiel
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - "="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.1.6
+        version: 0.3.2
     version: 
 description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
 email: seamus@abshere.net
@@ -92,17 +102,13 @@ files:
 - VERSION
 - data_miner.gemspec
 - lib/data_miner.rb
-- lib/data_miner/active_record_ext.rb
 - lib/data_miner/attribute.rb
-- lib/data_miner/attribute_collection.rb
 - lib/data_miner/configuration.rb
 - lib/data_miner/dictionary.rb
-- lib/data_miner/step.rb
-- lib/data_miner/step/associate.rb
-- lib/data_miner/step/await.rb
-- lib/data_miner/step/callback.rb
-- lib/data_miner/step/derive.rb
-- lib/data_miner/step/import.rb
+- lib/data_miner/import.rb
+- lib/data_miner/process.rb
+- lib/data_miner/run.rb
+- lib/data_miner/target.rb
 - lib/data_miner/william_james_cartesian_product.rb
 - test/data_miner_test.rb
 - test/test_helper.rb

data/lib/data_miner/active_record_ext.rb

@@ -1,25 +0,0 @@
-module DataMiner
-  module ActiveRecordExt
-    def self.included(klass)
-      klass.extend(ClassMethods)
-    end
-
-    module ClassMethods
-      def mine_data(options = {}, &block)
-        if defined?(NO_DATA_MINER) and NO_DATA_MINER == true
-          class_eval do
-            class << self
-              def data_mine
-                raise "NO_DATA_MINER is set to true, so data_mine is not available"
-              end
-            end
-          end
-        else
-          class_eval { cattr_accessor :data_mine }
-          self.data_mine = Configuration.new(self)
-          yield data_mine
-        end
-      end
-    end
-  end
-end

data/lib/data_miner/attribute_collection.rb

@@ -1,51 +0,0 @@
-module DataMiner
-  class AttributeCollection
-    attr_accessor :klass, :attributes
-    
-    def initialize(klass)
-      @klass = klass
-      @attributes = {}
-    end
-
-    def key!(step, attr_name, attr_options = {})
-      find_or_initialize(attr_name).key_for!(step, attr_options)
-    end
-
-    def affect!(step, attr_name, attr_options = {})
-      find_or_initialize(attr_name).affected_by!(step, attr_options)
-    end
-    
-    def affect_all_content_columns!(step, options = {})
-      except = Array.wrap(options[:except]).map(&:to_sym)
-      step.klass.content_columns.map(&:name).reject { |content_column| except.include?(content_column.to_sym) }.each do |content_column|
-        find_or_initialize(content_column).affected_by!(step)
-      end
-    end
-
-    def all_affected_by(step)
-      attributes.values.select { |attr| attr.affected_by?(step) }
-    end
-
-    def all_keys_for(step)
-      attributes.values.select { |attr| attr.key_for?(step) }
-    end
-    
-    def all_for(step)
-      (all_affected_by(step) + all_keys_for(step)).uniq
-    end
-    
-    def has_keys_for?(step)
-      attributes.values.any? { |attr| attr.key_for?(step) }
-    end
-    
-    def has_conditional_writes_for?(step)
-      all_affected_by(step).any? { |attr| !attr.wants_overwriting?(step) }
-    end
-    
-    private
-    
-    def find_or_initialize(attr_name)
-      self.attributes[attr_name] ||= Attribute.new(klass, attr_name)
-    end
-  end
-end

data/lib/data_miner/step.rb

@@ -1,64 +0,0 @@
-module DataMiner
-  class Step
-    attr_accessor :configuration, :number, :options
-    delegate :klass, :to => :configuration
-    delegate :attributes, :to => :configuration
-
-    def initialize(configuration, number, options = {}, &block)
-      @configuration = configuration
-      @number = number
-      @options = options
-      yield self if block_given? # pull in attributes
-      attributes.affect_all_content_columns!(self, :except => options[:except]) if options[:affect_all] == :content_columns
-      affected_attributes.each { |attr| attr.options_for_step[self][:callback] = options[:callback] } if options[:callback]
-      all_attributes.each { |attr| attr.options_for_step[self][:name_in_source] = attr.name_in_source(self).upcase } if options[:headers] == :upcase # TODO remove
-    end
-    
-    def variant
-      self.class.name.demodulize.underscore.to_sym
-    end
-    
-    def awaiting?
-      !options[:awaiting].nil?
-    end
-    
-    def inspect
-      "Step(#{klass} #{variant.to_s.camelcase} #{number})"
-    end
-    
-    def signature
-      "#{klass} step #{number}: #{variant}"
-    end
-    
-    def perform(options = {})
-      return if awaiting? and !options[:force]
-      affected_attributes.each { |attr| attr.perform self }
-      $stderr.puts "performed #{signature}"
-    end
-    
-    def affected_attributes
-      @affected_attributes ||= attributes.all_affected_by self
-    end
-    
-    def key_attributes
-      @key_attributes ||= attributes.all_keys_for self
-    end
-    
-    def all_attributes
-      @all_attributes ||= attributes.all_for self
-    end
-
-    def key(attr_name, attr_options = {})
-      attributes.key! self, attr_name, attr_options
-    end
-
-    def affect(attr_name, attr_options = {})
-      attributes.affect! self, attr_name, attr_options
-    end
-    alias_method :store, :affect 
-    
-    def map_to_attrs(method)
-      affected_attributes.map { |attr| attr.send method, self }.compact
-    end
-  end
-end

data/lib/data_miner/step/associate.rb

@@ -1,9 +0,0 @@
-module DataMiner
-  class Step
-    class Associate < Step
-      def signature
-        "#{super} #{affected_attributes.first.name}"
-      end
-    end
-  end
-end

data/lib/data_miner/step/await.rb

@@ -1,35 +0,0 @@
-module DataMiner
-  class Step
-    class Await < Step
-      attr_accessor :other_class
-      
-      def initialize(configuration, number, options = {}, &block)
-        # doesn't call super
-        @configuration = configuration
-        @number = number
-        @options = options
-        @other_class = options.delete :other_class
-        configuration.awaiting! self
-        yield configuration # pull in steps
-        configuration.stop_awaiting!
-      end
-      
-      def perform(*args)
-        other_class.data_mine.steps << Step::Callback.new(other_class.data_mine, self)
-        $stderr.puts "added #{signature} to callbacks after #{other_class}"
-      end
-      
-      def callback
-        $stderr.puts "starting to perform deferred steps in #{signature}..."
-        all_awaiting.each { |step| step.perform :force => true }
-        $stderr.puts "...done"
-      end
-      
-      private
-      
-      def all_awaiting
-        configuration.steps.select { |step| step.options and step.options[:awaiting] == self }
-      end
-    end
-  end
-end