data_miner 0.2.2

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,6 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+test/test.sqlite3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Brighter Planet
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,96 @@
+=data_miner
+
+Mine remote data into your ActiveRecord models.
+
+==Quick start
+
+Put this in <tt>config/environment.rb</tt>:
+
+  config.gem 'seamusabshere-data_miner', :lib => 'data_miner', :source => 'http://gems.github.com'
+
+You need to define <tt>mine_data</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
+
+  class Country < ActiveRecord::Base
+    mine_data do |step|
+      # import country names and country codes
+      step.import :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do |attr|
+        attr.key :iso_3166, :name_in_source => 'country code'
+        attr.store :iso_3166, :name_in_source => 'country code'
+        attr.store :name, :name_in_source => 'country'
+      end
+    end
+  end
+
+...and in <tt>app/models/airport.rb</tt>:
+
+  class Airport < ActiveRecord::Base
+    belongs_to :country
+  
+    mine_data do |step|
+      # import airport iata_code, name, etc.
+      step.import(:url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false) do |attr|
+        attr.key :iata_code, :field_number => 3
+        attr.store :name, :field_number => 0
+        attr.store :city, :field_number => 1
+        attr.store :country, :field_number => 2, :foreign_key => :name       # will use Country.find_by_name(X)
+        attr.store :iata_code, :field_number => 3
+        attr.store :latitude, :field_number => 5
+        attr.store :longitude, :field_number => 6
+      end
+    end
+  end
+
+Put this in <tt>lib/tasks/data_miner_tasks.rake</tt>: (unfortunately I don't know a way to automatically include gem tasks, so you have to do this manually for now)
+
+  namespace :data_miner do
+    task :mine => :environment do
+      DataMiner.mine :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
+    end
+
+    task :map_to_attrs => :environment do
+      DataMiner.map_to_attrs ENV['METHOD'], :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
+    end
+  end
+
+You need to specify what order to mine data. For example, in <tt>config/initializers/data_miner_config.rb</tt>:
+
+  DataMiner.enqueue do |queue|
+    queue << Country  # class whose data should be mined 1st
+    queue << Airport  # class whose data should be mined 2nd
+    # etc
+  end
+
+Once you have (1) set up the order of data mining and (2) defined <tt>mine_data</tt> blocks in your classes, you can:
+
+  $ rake data_miner:mine
+
+==Complete example
+
+  ~ $ rails testapp
+  ~ $ cd testapp/
+  ~/testapp $ ./script/generate model Airport iata_code:string name:string city:string country_id:integer latitude:float longitude:float
+  ~/testapp $ ./script/generate model Country iso_3166:string name:string
+  ~/testapp $ rake db:migrate
+  ~/testapp $ touch lib/tasks/data_miner_tasks.rb
+  [...edit per quick start...]
+  ~/testapp $ touch config/initializers/data_miner_config.rake
+  [...edit per quick start...]
+  ~/testapp $ rake data_miner:mine
+
+Now you should have
+
+  ~/testapp $ ./script/console 
+  Loading development environment (Rails 2.3.3)
+  >> Airport.first.iata_code
+  => "GKA"
+  >> Airport.first.country.name
+  => "Papua New Guinea"
+
+==Authors
+
+* Seamus Abshere <seamus@abshere.net>
+* Andy Rossmeissl <andy@rossmeissl.net>
+
+==Copyright
+
+Copyright (c) 2009 Brighter Planet. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,65 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "data_miner"
+    gem.summary = %Q{Mine remote data into your ActiveRecord models.}
+    gem.description = %Q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
+    gem.email = "seamus@abshere.net"
+    gem.homepage = "http://github.com/seamusabshere/data_miner"
+    gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
+    %w{ activerecord activesupport andand remote_table seamusabshere-errata seamusabshere-conversions }.each { |name| gem.add_dependency name }
+    gem.require_path = "lib"
+    gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
+    gem.rdoc_options << '--line-numbers' << '--inline-source'
+    # gem.rubyforge_project = "dataminer"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+  Jeweler::RubyforgeTasks.new do |rubyforge|
+    rubyforge.doc_task = "rdoc"
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "data_miner #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/data_miner.gemspec

@@ -0,0 +1,81 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{data_miner}
+  s.version = "0.2.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
+  s.date = %q{2009-10-30}
+  s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
+  s.email = %q{seamus@abshere.net}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "data_miner.gemspec",
+     "lib/data_miner.rb",
+     "lib/data_miner/active_record_ext.rb",
+     "lib/data_miner/attribute.rb",
+     "lib/data_miner/attribute_collection.rb",
+     "lib/data_miner/configuration.rb",
+     "lib/data_miner/dictionary.rb",
+     "lib/data_miner/step.rb",
+     "lib/data_miner/step/associate.rb",
+     "lib/data_miner/step/await.rb",
+     "lib/data_miner/step/callback.rb",
+     "lib/data_miner/step/derive.rb",
+     "lib/data_miner/step/import.rb",
+     "lib/data_miner/william_james_cartesian_product.rb",
+     "test/data_miner_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/seamusabshere/data_miner}
+  s.rdoc_options = ["--charset=UTF-8", "--line-numbers", "--inline-source"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Mine remote data into your ActiveRecord models.}
+  s.test_files = [
+    "test/data_miner_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activerecord>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<andand>, [">= 0"])
+      s.add_runtime_dependency(%q<remote_table>, [">= 0"])
+      s.add_runtime_dependency(%q<seamusabshere-errata>, [">= 0"])
+      s.add_runtime_dependency(%q<seamusabshere-conversions>, [">= 0"])
+    else
+      s.add_dependency(%q<activerecord>, [">= 0"])
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<andand>, [">= 0"])
+      s.add_dependency(%q<remote_table>, [">= 0"])
+      s.add_dependency(%q<seamusabshere-errata>, [">= 0"])
+      s.add_dependency(%q<seamusabshere-conversions>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<activerecord>, [">= 0"])
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<andand>, [">= 0"])
+    s.add_dependency(%q<remote_table>, [">= 0"])
+    s.add_dependency(%q<seamusabshere-errata>, [">= 0"])
+    s.add_dependency(%q<seamusabshere-conversions>, [">= 0"])
+  end
+end
+

data/lib/data_miner.rb

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'activesupport'
+require 'activerecord'
+require 'conversions'
+require 'remote_table'
+require 'errata'
+
+require 'data_miner/active_record_ext'
+require 'data_miner/attribute'
+require 'data_miner/attribute_collection'
+require 'data_miner/configuration'
+require 'data_miner/dictionary'
+require 'data_miner/step'
+require 'data_miner/step/associate'
+require 'data_miner/step/await'
+require 'data_miner/step/callback'
+require 'data_miner/step/derive'
+require 'data_miner/step/import'
+require 'data_miner/william_james_cartesian_product' # TODO: move to gem
+
+module DataMiner
+  class << self
+    def mine(options = {})
+      DataMiner::Configuration.mine options
+    end
+    
+    def map_to_attrs(method, options = {})
+      puts DataMiner::Configuration.map_to_attrs(method, options)
+    end
+
+    def enqueue(&block)
+      DataMiner::Configuration.enqueue &block
+    end
+    
+    def classes
+      DataMiner::Configuration.classes
+    end
+  end
+end
+
+ActiveRecord::Base.class_eval do
+  include DataMiner::ActiveRecordExt
+end

data/lib/data_miner/active_record_ext.rb

@@ -0,0 +1,25 @@
+module DataMiner
+  module ActiveRecordExt
+    def self.included(klass)
+      klass.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def mine_data(options = {}, &block)
+        if defined?(NO_DATA_MINER) and NO_DATA_MINER == true
+          class_eval do
+            class << self
+              def data_mine
+                raise "NO_DATA_MINER is set to true, so data_mine is not available"
+              end
+            end
+          end
+        else
+          class_eval { cattr_accessor :data_mine }
+          self.data_mine = Configuration.new(self)
+          yield data_mine
+        end
+      end
+    end
+  end
+end

data/lib/data_miner/attribute.rb

@@ -0,0 +1,299 @@
+module DataMiner
+  class Attribute
+    attr_accessor :klass, :name, :options_for_step, :affected_by_steps, :key_for_steps
+
+    def initialize(klass, name)
+      @klass = klass
+      @name = name.to_sym
+      @options_for_step = {}
+      @affected_by_steps = []
+      @key_for_steps = []
+    end
+    
+    # polling questions
+    def report_find_or_create(step)
+      "Creates parents: #{klass}##{name} is set with #{reflection_klass(step)}.find_or_create_by_#{foreign_key(step)}" if wants_create?(step)
+    end
+    
+    def report_unnatural_order(step)
+      if (
+           (rk = klass.reflect_on_association(weighting_association(step)).andand.klass) or
+           (wants_inline_association? and rk = reflection_klass(step))
+         ) and
+         step.configuration.classes.index(rk) > step.configuration.classes.index(klass) and
+         step.options[:awaiting].andand.klass != klass
+        "Unnatural order: #{klass} comes before #{rk}"
+      end
+    end
+
+    def inspect
+      "Attribute(#{klass}.#{name})"
+    end
+
+    def affected_by!(step, options = {})
+      self.options_for_step[step] = options
+      self.affected_by_steps << step
+    end
+
+    def affected_by?(step)
+      affected_by_steps.include?(step)
+    end
+
+    def key_for!(step, options = {})
+      self.options_for_step[step] = options
+      self.key_for_steps << step
+    end
+
+    def key_for?(step)
+      key_for_steps.include?(step)
+    end
+    
+    def value_in_dictionary(step, key)
+      return *dictionary(step).lookup(key) # strip the array wrapper if there's only one element
+    end
+    
+    def value_in_source(step, row)
+      if wants_static?(step)
+        value = static(step)
+      elsif field_number(step)
+        if field_number(step).is_a?(Range)
+          value = field_number(step).map { |n| row[n] }.join(delimiter(step))
+        else
+          value = row[field_number(step)]
+        end
+      else
+        value = row[name_in_source(step)]
+      end
+      return nil if value.nil?
+      return value if value.is_a?(ActiveRecord::Base) # escape valve for parsers that look up associations directly
+      value = value.to_s
+      value = value[keep(step)] if wants_keep?(step)
+      value = do_split(step, value) if wants_split?(step)
+      # taken from old errata... maybe we want to do this here
+      value.gsub!(/[ ]+/, ' ')
+      # text.gsub!('- ', '-')
+      value.gsub!(/([^\\])~/, '\1 ')
+      value.strip!
+      value.upcase! if wants_upcase?(step)
+      value = do_convert(step, row, value) if wants_conversion?(step)
+      value = do_sprintf(step, value) if wants_sprintf?(step)
+      value
+    end
+    
+    def value_from_row(step, row)
+      value = value_in_source(step, row)
+      return value if value.is_a?(ActiveRecord::Base) # carry through trapdoor
+      value = value_in_dictionary(step, value) if wants_dictionary?(step)
+      value = value_as_association(step, value) if wants_inline_association?
+      value
+    end
+    
+    def value_as_association(step, value)
+      @_value_as_association ||= {}
+      @_value_as_association[step] ||= {}
+      if !@_value_as_association[step].has_key?(value)
+        dynamic_matcher = wants_create?(step) ? "find_or_create_by_#{foreign_key(step)}" : "find_by_#{foreign_key(step)}"
+        @_value_as_association[step][value] = reflection_klass(step).send(dynamic_matcher, value)
+      end
+      @_value_as_association[step][value]
+    end
+    
+    # this will overwrite nils, even if wants_overwriting?(step) is false
+    def set_record_from_row(step, record, row)
+      return if !wants_overwriting?(step) and !record.send(name).nil?
+      value = value_from_row(step, row)
+      record.send "#{name}=", value
+      $stderr.puts("ActiveRecord didn't like trying to set #{klass}.#{name} = #{value}") if !value.nil? and record.send(name).nil?
+    end
+
+    def perform(step)
+      case step.variant
+      when :associate
+        perform_association(step)
+      when :derive
+        if wants_update_all?(step)
+          perform_update_all(step)
+        elsif wants_weighted_average?(step)
+          perform_weighted_average(step)
+        else
+          perform_callback(step)
+        end
+      when :import
+        raise "This shouldn't be called, the import step is special"
+      end
+    end
+
+    def perform_association(step)
+      raise "dictionary and prefix don't mix" if wants_dictionary?(step) and wants_prefix?(step)
+      klass.update_all("#{reflection.primary_key_name} = NULL") if wants_nullification?(step)
+      if wants_create?(step)
+        klass.find_in_batches do |batch|
+          batch.each do |record|
+            if wants_prefix?(step)
+              sql = "SELECT reflection_table.id FROM #{reflection_klass(step).quoted_table_name} AS reflection_table INNER JOIN #{klass.quoted_table_name} AS klass_table ON LEFT(klass_table.#{key(step)}, LENGTH(reflection_table.#{foreign_key(step)})) = reflection_table.#{foreign_key(step)} WHERE klass_table.id = #{record.id} ORDER BY LENGTH(reflection_table.#{foreign_key(step)}) DESC"
+              associated_id = ActiveRecord::Base.connection.select_value(sql)
+              next if associated_id.blank?
+              record.send("#{reflection.primary_key_name}=", associated_id)
+            else
+              dynamic_finder_value = record.send(key(step))
+              dynamic_finder_value = value_in_dictionary(step, dynamic_finder_value) if wants_dictionary?(step)
+              next if dynamic_finder_value.blank?
+              associated = reflection_klass(step).send("find_or_create_by_#{foreign_key(step)}", dynamic_finder_value) # TODO cache results
+              record.send("#{name}=", associated)
+            end
+            record.save
+          end
+        end
+      else
+        reflection_klass(step).find_in_batches do |batch|
+          batch.each do |reflection_record|
+            klass.update_all ["#{reflection.primary_key_name} = ?", reflection_record.id], ["#{key(step)} = ?", reflection_record.send(foreign_key(step))]
+          end
+        end
+      end
+    end
+
+    def perform_update_all(step)
+      klass.update_all("#{name} = #{set(step)}", conditions(step))
+    end
+    
+    def perform_weighted_average(step)
+      # handle weighting by scopes instead of associations
+      if weighting_association(step) and !klass.reflect_on_association(weighting_association(step))
+        klass.find_in_batches do |batch|
+          batch.each do |record|
+            record.send "#{name}=", record.send(weighting_association(step)).weighted_average(name, :by => weighting_column(step), :disaggregator => weighting_disaggregator(step))
+            record.save
+          end
+        end
+      else # there's no weighting association OR there is one and it's a valid association
+        klass.update_all_weighted_averages name, :by => weighting_column(step), :disaggregator => weighting_disaggregator(step), :association => weighting_association(step)
+      end
+    end
+    
+    def perform_callback(step)
+      case klass.method(callback(step)).arity
+      when 0:
+        klass.send(callback(step))
+      when 1:
+        klass.send(callback(step), name)
+      when 2:
+        klass.send(callback(step), name, options_for_step[step])
+      end
+    end
+    
+    def unit_from_source(step, row)
+      row[unit_in_source(step)].to_s.strip.underscore.to_sym
+    end
+    
+    def do_convert(step, row, value)
+      from_unit = from(step) || unit_from_source(step, row)
+      value.to_f.convert(from_unit, to(step))
+    end
+    
+    def do_sprintf(step, value)
+      if /\%[0-9\.]*f/.match(sprintf(step))
+        value = value.to_f
+      elsif /\%[0-9\.]*d/.match(sprintf(step))
+        value = value.to_i
+      end
+      sprintf(step) % value
+    end
+    
+    def do_split(step, value)
+      pattern = split_options(step)[:pattern] || /\s+/ # default is split on whitespace
+      keep = split_options(step)[:keep] || 0           # default is keep first element
+      value.to_s.split(pattern)[keep].to_s
+    end
+  
+    def column_type
+      @column_type ||= klass.columns_hash[name.to_s].type
+    end
+    
+    {
+      :static => 'options_for_step[step].has_key?(:static)',
+      :prefix => :prefix,
+      :create => :create,
+      :keep => :keep,
+      :upcase => :upcase,
+      :conversion => '!from(step).nil? or !unit_in_source(step).nil?',
+      :sprintf => :sprintf,
+      :dictionary => :dictionary_options,
+      :split => :split_options,
+      :update_all => :set,
+      :nullification => 'nullify(step) != false',
+      :overwriting => 'overwrite(step) != false',
+      :weighted_average => '!weighting_association(step).nil? or !weighting_column(step).nil?'
+    }.each do |name, condition|
+      condition = "!#{condition}(step).nil?" if condition.is_a?(Symbol)
+      eval <<-EOS
+        def wants_#{name}?(step)
+          #{condition}
+        end
+      EOS
+    end
+    
+    {
+      :name_in_source => { :default => :name,                           :stringify => true },
+      :key            => { :default => :name,                           :stringify => true },
+      :foreign_key    => { :default => 'key(step)',                     :stringify => true },
+      :delimiter      => { :default => '", "' }
+    }.each do |name, options|
+      eval <<-EOS
+        def #{name}(step)
+          (options_for_step[step][:#{name}] || #{options[:default]})#{'.to_s' if options[:stringify]}
+        end
+      EOS
+    end
+    
+    def reflection
+      if @_reflection.nil?
+        @_reflection = klass.reflect_on_association(name) || :missing
+        reflection
+      elsif @_reflection == :missing
+        nil
+      else
+        @_reflection
+      end
+    end
+    
+    def reflection_klass(step)
+      return nil unless reflection
+      if reflection.options[:polymorphic]
+        polymorphic_type(step).andand.constantize
+      else
+        reflection.klass
+      end
+    end
+    
+    def wants_inline_association?
+      reflection.present?
+    end
+    
+    def callback(step)
+      (options_for_step[step][:callback] || "derive_#{name}").to_sym
+    end
+
+    def dictionary(step)
+      raise "shouldn't ask for this" unless wants_dictionary?(step) # don't try to initialize if there are no dictionary options
+      @dictionaries ||= {}
+      @dictionaries[step] ||= Dictionary.new(dictionary_options(step))
+    end
+    
+    %w(dictionary split).each do |name|
+      eval <<-EOS
+        def #{name}_options(step)
+          options_for_step[step][:#{name}]
+        end
+      EOS
+    end
+        
+    %w(from to set conditions weighting_association weighting_column weighting_disaggregator sprintf nullify overwrite upcase prefix unit_in_source field_number keep create static polymorphic_type).each do |name|
+      eval <<-EOS
+        def #{name}(step)
+          options_for_step[step][:#{name}]
+        end
+      EOS
+    end
+  end
+end

data/lib/data_miner/attribute_collection.rb

@@ -0,0 +1,51 @@
+module DataMiner
+  class AttributeCollection
+    attr_accessor :klass, :attributes
+    
+    def initialize(klass)
+      @klass = klass
+      @attributes = {}
+    end
+
+    def key!(step, attr_name, attr_options = {})
+      find_or_initialize(attr_name).key_for!(step, attr_options)
+    end
+
+    def affect!(step, attr_name, attr_options = {})
+      find_or_initialize(attr_name).affected_by!(step, attr_options)
+    end
+    
+    def affect_all_content_columns!(step, options = {})
+      except = Array.wrap(options[:except]).map(&:to_sym)
+      step.klass.content_columns.map(&:name).reject { |content_column| except.include?(content_column.to_sym) }.each do |content_column|
+        find_or_initialize(content_column).affected_by!(step)
+      end
+    end
+
+    def all_affected_by(step)
+      attributes.values.select { |attr| attr.affected_by?(step) }
+    end
+
+    def all_keys_for(step)
+      attributes.values.select { |attr| attr.key_for?(step) }
+    end
+    
+    def all_for(step)
+      (all_affected_by(step) + all_keys_for(step)).uniq
+    end
+    
+    def has_keys_for?(step)
+      attributes.values.any? { |attr| attr.key_for?(step) }
+    end
+    
+    def has_conditional_writes_for?(step)
+      all_affected_by(step).any? { |attr| !attr.wants_overwriting?(step) }
+    end
+    
+    private
+    
+    def find_or_initialize(attr_name)
+      self.attributes[attr_name] ||= Attribute.new(klass, attr_name)
+    end
+  end
+end

data/lib/data_miner/configuration.rb

@@ -0,0 +1,94 @@
+module DataMiner
+  class Configuration
+    attr_accessor :steps, :klass, :counter, :attributes, :awaiting
+
+    def initialize(klass)
+      @steps = []
+      @klass = klass
+      @counter = 0
+      @attributes = AttributeCollection.new(klass)
+    end
+
+    %w(import associate derive await).each do |method|
+      eval <<-EOS
+        def #{method}(*args, &block)
+          self.counter += 1
+          if block_given? # FORM C
+            step_options = args[0] || {}
+            set_awaiting!(step_options)
+            self.steps << Step::#{method.camelcase}.new(self, counter, step_options, &block)
+          elsif args[0].is_a?(Hash) # FORM A
+            step_options = args[0]
+            set_awaiting!(step_options)
+            self.steps << Step::#{method.camelcase}.new(self, counter, step_options)
+          else # FORM B
+            attr_name = args[0]
+            attr_options = args[1] || {}
+            step_options = {}
+            set_awaiting!(step_options)
+            self.steps << Step::#{method.camelcase}.new(self, counter, step_options) do |attr|
+              attr.affect attr_name, attr_options
+            end
+          end
+        end
+      EOS
+    end
+
+    def set_awaiting!(step_options)
+      step_options.merge!(:awaiting => awaiting) if !awaiting.nil?
+    end
+
+    def awaiting!(step)
+      self.awaiting = step
+    end
+    
+    def stop_awaiting!
+      self.awaiting = nil
+    end
+
+    # Mine data for this class.
+    def mine(options = {})
+      steps.each { |step| step.perform options }
+    end
+    
+    # Map <tt>method</tt> to attributes
+    def map_to_attrs(method)
+      steps.map { |step| step.map_to_attrs(method) }.compact
+    end
+
+    cattr_accessor :classes
+    self.classes = []
+    class << self
+      # Mine data. Defaults to all classes touched by DataMiner.
+      #
+      # Options
+      # * <tt>:class_names</tt>: provide an array class names to mine
+      def mine(options = {})
+        classes.each do |klass|
+          if options[:class_names].blank? or options[:class_names].include?(klass.name)
+            klass.data_mine.mine options
+          end
+        end
+      end
+      
+      # Map a <tt>method</tt> to attrs. Defaults to all classes touched by DataMiner.
+      #
+      # Options
+      # * <tt>:class_names</tt>: provide an array class names to mine
+      def map_to_attrs(method, options = {})
+        classes.map do |klass|
+          if options[:class_names].blank? or options[:class_names].include?(klass.name)
+            klass.data_mine.map_to_attrs method
+          end
+        end.flatten.compact
+      end
+
+      # Queue up all the ActiveRecord classes that DataMiner should touch.
+      #
+      # Generally done in <tt>config/initializers/data_miner_config.rb</tt>.
+      def enqueue(&block)
+        yield self.classes
+      end
+    end
+  end
+end

data/lib/data_miner/dictionary.rb

@@ -0,0 +1,36 @@
+module DataMiner
+  class Dictionary
+    attr_accessor :key_name, :value_name, :sprintf, :table
+
+    def initialize(options = {})
+      @key_name = options[:key]
+      @value_name = options[:returns]
+      @sprintf = options[:sprintf] || '%s'
+      @table = RemoteTable.new(:url => options[:url])
+    end
+
+    def lookup(key)
+      find(self.key_name, key, self.value_name, :sprintf => self.sprintf)
+    end
+    
+    def find(key_name, key, value_name, options = {})
+      if match = table.rows.detect { |row| normalize_for_comparison(key, options) == normalize_for_comparison(row[key_name], options) }
+        match[value_name].to_s.split(/\s*;\s/)
+      end
+    end
+
+    private
+
+    def normalize_for_comparison(string, options = {})
+      if options[:sprintf]
+        if /\%[0-9\.]*f/.match(options[:sprintf])
+          string = string.to_f
+        elsif /\%[0-9\.]*d/.match(options[:sprintf])
+          string = string.to_i
+        end
+        string = sprintf % string
+      end
+      string.to_s.strip
+    end
+  end
+end

data/lib/data_miner/step.rb

@@ -0,0 +1,64 @@
+module DataMiner
+  class Step
+    attr_accessor :configuration, :number, :options
+    delegate :klass, :to => :configuration
+    delegate :attributes, :to => :configuration
+
+    def initialize(configuration, number, options = {}, &block)
+      @configuration = configuration
+      @number = number
+      @options = options
+      yield self if block_given? # pull in attributes
+      attributes.affect_all_content_columns!(self, :except => options[:except]) if options[:affect_all] == :content_columns
+      affected_attributes.each { |attr| attr.options_for_step[self][:callback] = options[:callback] } if options[:callback]
+      all_attributes.each { |attr| attr.options_for_step[self][:name_in_source] = attr.name_in_source(self).upcase } if options[:headers] == :upcase # TODO remove
+    end
+    
+    def variant
+      self.class.name.demodulize.underscore.to_sym
+    end
+    
+    def awaiting?
+      !options[:awaiting].nil?
+    end
+    
+    def inspect
+      "Step(#{klass} #{variant.to_s.camelcase} #{number})"
+    end
+    
+    def signature
+      "#{klass} step #{number}: #{variant}"
+    end
+    
+    def perform(options = {})
+      return if awaiting? and !options[:force]
+      affected_attributes.each { |attr| attr.perform self }
+      $stderr.puts "performed #{signature}"
+    end
+    
+    def affected_attributes
+      @affected_attributes ||= attributes.all_affected_by self
+    end
+    
+    def key_attributes
+      @key_attributes ||= attributes.all_keys_for self
+    end
+    
+    def all_attributes
+      @all_attributes ||= attributes.all_for self
+    end
+
+    def key(attr_name, attr_options = {})
+      attributes.key! self, attr_name, attr_options
+    end
+
+    def affect(attr_name, attr_options = {})
+      attributes.affect! self, attr_name, attr_options
+    end
+    alias_method :store, :affect 
+    
+    def map_to_attrs(method)
+      affected_attributes.map { |attr| attr.send method, self }.compact
+    end
+  end
+end

data/lib/data_miner/step/associate.rb

@@ -0,0 +1,9 @@
+module DataMiner
+  class Step
+    class Associate < Step
+      def signature
+        "#{super} #{affected_attributes.first.name}"
+      end
+    end
+  end
+end

data/lib/data_miner/step/await.rb

@@ -0,0 +1,35 @@
+module DataMiner
+  class Step
+    class Await < Step
+      attr_accessor :other_class
+      
+      def initialize(configuration, number, options = {}, &block)
+        # doesn't call super
+        @configuration = configuration
+        @number = number
+        @options = options
+        @other_class = options.delete :other_class
+        configuration.awaiting! self
+        yield configuration # pull in steps
+        configuration.stop_awaiting!
+      end
+      
+      def perform(*args)
+        other_class.data_mine.steps << Step::Callback.new(other_class.data_mine, self)
+        $stderr.puts "added #{signature} to callbacks after #{other_class}"
+      end
+      
+      def callback
+        $stderr.puts "starting to perform deferred steps in #{signature}..."
+        all_awaiting.each { |step| step.perform :force => true }
+        $stderr.puts "...done"
+      end
+      
+      private
+      
+      def all_awaiting
+        configuration.steps.select { |step| step.options and step.options[:awaiting] == self }
+      end
+    end
+  end
+end

data/lib/data_miner/step/callback.rb

@@ -0,0 +1,22 @@
+module DataMiner
+  class Step
+    class Callback < Step
+      attr_accessor :foreign_step
+      
+      def initialize(configuration, foreign_step)
+        @configuration = configuration
+        @foreign_step = foreign_step
+        @number = "(last)"
+      end
+      
+      def perform(*args)
+        foreign_step.callback
+        $stderr.puts "performed #{signature}"
+      end
+      
+      def signature
+        "#{super} (on behalf of #{foreign_step.signature})"
+      end
+    end
+  end
+end

data/lib/data_miner/step/derive.rb

@@ -0,0 +1,9 @@
+module DataMiner
+  class Step
+    class Derive < Step
+      def signature
+        "#{super} #{affected_attributes.first.name}"
+      end
+    end
+  end
+end

data/lib/data_miner/step/import.rb

@@ -0,0 +1,57 @@
+module DataMiner
+  class Step
+    class Import < Step
+      attr_accessor :table, :errata
+      
+      def initialize(configuration, number, options = {}, &block)
+        super
+        @errata = Errata.new(:url => options[:errata], :klass => klass) if options[:errata]
+        @table = RemoteTable.new(options.slice(:url, :filename, :post_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop))
+      end
+      
+      def signature
+        "#{super} #{options[:url]}"
+      end
+
+      def perform(*args)
+        ActiveRecord::Base.connection.execute("TRUNCATE #{klass.quoted_table_name}") if wants_truncate?
+        table.each_row do |row|
+          if errata
+            next if errata.rejects?(row)
+            errata.correct!(row)
+          end
+          if uses_existing_data?
+            key_values = key_attributes.map { |key_attr| [ key_attr.value_from_row(self, row) ] }
+            record_set = WilliamJamesCartesianProduct.cart_prod(*key_values).map do |combination|
+              next if combination.include?(nil) and !wants_nil_keys?
+              klass.send(dynamic_finder_name, *combination)
+            end.flatten
+          else
+            record_set = klass.new
+          end
+          Array.wrap(record_set).each do |record|
+            affected_attributes.each { |attr| attr.set_record_from_row(self, record, row) }
+            record.save
+          end
+        end
+        $stderr.puts "performed #{signature}"
+      end
+      
+      def wants_truncate?
+        options[:truncate] == true or (!(options[:truncate] == false) and !uses_existing_data?)
+      end
+      
+      def wants_nil_keys?
+        options[:allow_nil_keys] == true
+      end
+      
+      def uses_existing_data?
+        @uses_existing_data ||= attributes.has_keys_for?(self) or attributes.has_conditional_writes_for?(self)
+      end
+      
+      def dynamic_finder_name
+        "find_or_initialize_by_#{key_attributes.map(&:name).join('_and_')}".to_sym
+      end
+    end
+  end
+end

data/lib/data_miner/william_james_cartesian_product.rb

@@ -0,0 +1,11 @@
+# http://www.ruby-forum.com/topic/95519#200484
+
+module WilliamJamesCartesianProduct
+  def self.cart_prod( *args )
+    args.inject([[]]){|old,lst|
+      new = []
+      lst.each{|e| new += old.map{|c| c.dup << e }}
+      new
+    }
+  end
+end

data/test/data_miner_test.rb

@@ -0,0 +1,78 @@
+require 'test_helper'
+
+ActiveRecord::Schema.define(:version => 20090819143429) do
+  create_table "airports", :force => true do |t|
+    t.string   "iata_code"
+    t.string   "name"
+    t.string   "city"
+    t.integer  "country_id"
+    t.float    "latitude"
+    t.float    "longitude"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+  create_table "countries", :force => true do |t|
+    t.string   "iso_3166"
+    t.string   "name"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+end
+
+class Country < ActiveRecord::Base
+  mine_data do |step|
+    # import country names and country codes
+    step.import :url => 'http://www.cs.princeton.edu/introcs/data/iso3166.csv' do |attr|
+      attr.key :iso_3166, :name_in_source => 'country code'
+      attr.store :iso_3166, :name_in_source => 'country code'
+      attr.store :name, :name_in_source => 'country'
+    end
+  end
+end
+
+class Airport < ActiveRecord::Base
+  belongs_to :country
+  mine_data do |step|
+    # import airport iata_code, name, etc.
+    step.import(:url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat', :headers => false) do |attr|
+      attr.key :iata_code, :field_number => 3
+      attr.store :name, :field_number => 0
+      attr.store :city, :field_number => 1
+      attr.store :country, :field_number => 2, :foreign_key => :name       # will use Country.find_by_name(X)
+      attr.store :iata_code, :field_number => 3
+      attr.store :latitude, :field_number => 5
+      attr.store :longitude, :field_number => 6
+    end
+  end
+end
+
+DataMiner.enqueue do |queue|
+  queue << Country
+  queue << Airport
+end
+
+class DataMinerTest < Test::Unit::TestCase
+  def teardown
+    Airport.delete_all
+    Country.delete_all
+  end
+  
+  should "mine a single class" do
+    Country.data_mine.mine
+    assert_equal 'Uruguay', Country.find_by_iso_3166('UY').name
+    assert_equal 0, Airport.count
+  end
+  
+  should "mine a single class using the API" do
+    DataMiner.mine :class_names => ['Country']
+    assert_equal 'Uruguay', Country.find_by_iso_3166('UY').name
+    assert_equal 0, Airport.count
+  end
+  
+  should "mine all classes" do
+    DataMiner.mine
+    uy = Country.find_by_iso_3166('UY')
+    assert_equal 'Uruguay', uy.name
+    assert_equal uy, Airport.find_by_iata_code('MVD').country
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'sqlite3'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'data_miner'
+
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'sqlite3',
+  'database' => 'test/test.sqlite3'
+)
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification 
+name: data_miner
+version: !ruby/object:Gem::Version 
+  version: 0.2.2
+platform: ruby
+authors: 
+- Seamus Abshere
+- Andy Rossmeissl
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-30 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: andand
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: remote_table
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: seamusabshere-errata
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: seamusabshere-conversions
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
+email: seamus@abshere.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- data_miner.gemspec
+- lib/data_miner.rb
+- lib/data_miner/active_record_ext.rb
+- lib/data_miner/attribute.rb
+- lib/data_miner/attribute_collection.rb
+- lib/data_miner/configuration.rb
+- lib/data_miner/dictionary.rb
+- lib/data_miner/step.rb
+- lib/data_miner/step/associate.rb
+- lib/data_miner/step/await.rb
+- lib/data_miner/step/callback.rb
+- lib/data_miner/step/derive.rb
+- lib/data_miner/step/import.rb
+- lib/data_miner/william_james_cartesian_product.rb
+- test/data_miner_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/seamusabshere/data_miner
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+- --line-numbers
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Mine remote data into your ActiveRecord models.
+test_files: 
+- test/data_miner_test.rb
+- test/test_helper.rb