data_miner 0.5.7 → 1.0.0

data/lib/data_miner/schema.rb

@@ -1,24 +1,28 @@
-module DataMiner
+require 'blockenspiel'
+class DataMiner
   class Schema
-    include Blockenspiel::DSL
+    include ::Blockenspiel::DSL
     
-    attr_reader :base
-    attr_reader :position_in_run
+    attr_reader :config
     attr_reader :create_table_options
-    delegate :resource, :to => :base
 
-    def initialize(base, position_in_run, create_table_options)
-      @base = base
-      @position_in_run = position_in_run
-      @create_table_options = create_table_options
-      @create_table_options.symbolize_keys!
-      DataMiner.log_or_raise ":id => true is not allowed in create_table_options." if @create_table_options[:id] === true
-      DataMiner.log_or_raise ":primary_key is not allowed in create_table_options. Use set_primary_key instead." if @create_table_options.has_key?(:primary_key)
-      @create_table_options[:id] = false # always
+    def initialize(config, create_table_options)
+      @config = config
+      @create_table_options = create_table_options.dup
+      @create_table_options.stringify_keys!
+      raise "'id' => true is not allowed in create_table_options." if @create_table_options['id'] === true
+      raise "'primary_key' is not allowed in create_table_options. Use set_primary_key instead." if @create_table_options.has_key?('primary_key')
+      @create_table_options['id'] = false # always
     end
     
+    def resource
+      config.resource
+    end
+    
+    # sabshere 1/25/11 what if there were multiple connections
+    # blockenspiel doesn't like to delegate this to #resource
     def connection
-      ActiveRecord::Base.connection
+      ::ActiveRecord::Base.connection
     end
     
     def table_name
@@ -26,11 +30,11 @@ module DataMiner
     end
     
     def ideal_table
-      @ideal_table ||= ActiveRecord::ConnectionAdapters::TableDefinition.new(connection)
+      @ideal_table ||= ::ActiveRecord::ConnectionAdapters::TableDefinition.new connection
     end
     
     def ideal_indexes
-      @ideal_indexes ||= Array.new
+      @ideal_indexes ||= []
     end
     
     def actual_indexes
@@ -42,10 +46,10 @@ module DataMiner
     end
     
     def inspect
-      "Schema(#{resource}): #{description}"
+      %{#<DataMiner::Schema(#{resource}): #{description}>}
     end
     
-    # lifted straight from activerecord-3.0.0.beta3/lib/active_record/connection_adapters/abstract/schema_definitions.rb
+    # sabshere 1/25/11 lifted straight from activerecord-3.0.3/lib/active_record/connection_adapters/abstract/schema_definitions.rb
     %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
       class_eval <<-EOV
         def #{column_type}(*args)                                               # def string(*args)
@@ -60,16 +64,18 @@ module DataMiner
       ideal_table.column(*args)
     end
     
-    MAX_INDEX_NAME_LENGTH = 50
+    MAX_INDEX_NAME_LENGTH = 32
     def index(columns, options = {})
-      options.symbolize_keys!
-      columns = Array.wrap columns
-      unless name = options[:name]
-        default_name = connection.index_name(table_name, options.merge(:column => columns))
-        name = default_name.length < MAX_INDEX_NAME_LENGTH ? default_name : default_name[0..MAX_INDEX_NAME_LENGTH-11] + Zlib.crc32(default_name).to_s
+      options = options.dup
+      options.stringify_keys!
+      columns = ::Array.wrap columns
+      unless name = options['name']
+        default_name = connection.index_name(table_name, options.symbolize_keys.merge(:column => columns))
+        name = default_name.length < MAX_INDEX_NAME_LENGTH ? default_name : default_name[0..MAX_INDEX_NAME_LENGTH-11] + ::Zlib.crc32(default_name).to_s
       end
-      index_unique = options.has_key?(:unique) ? options[:unique] : true
-      ideal_indexes.push ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, name, index_unique, columns)
+      index_unique = options.has_key?('unique') ? options['unique'] : true
+      ideal_indexes.push ::ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, name, index_unique, columns)
+      nil
     end
     
     def ideal_primary_key_name
@@ -84,7 +90,7 @@ module DataMiner
     def index_equivalent?(a, b)
       return false unless a and b
       INDEX_PROPERTIES.all? do |property|
-        DataMiner.log_debug "...comparing #{a.send(property).inspect}.to_s <-> #{b.send(property).inspect}.to_s"
+        ::DataMiner.logger.debug "...comparing #{a.send(property).inspect}.to_s <-> #{b.send(property).inspect}.to_s"
         a.send(property).to_s == b.send(property).to_s
       end
     end
@@ -131,13 +137,13 @@ module DataMiner
     def place_column(name)
       remove_column name if actual_column name
       ideal = ideal_column name
-      DataMiner.log_debug "ADDING COLUMN #{name}"
+      ::DataMiner.logger.debug "ADDING COLUMN #{name}"
       connection.add_column table_name, name, ideal.type.to_sym # symbol type!
       resource.reset_column_information
     end
     
     def remove_column(name)
-      DataMiner.log_debug "REMOVING COLUMN #{name}"
+      ::DataMiner.logger.debug "REMOVING COLUMN #{name}"
       connection.remove_column table_name, name
       resource.reset_column_information
     end
@@ -145,43 +151,34 @@ module DataMiner
     def place_index(name)
       remove_index name if actual_index name
       ideal = ideal_index name
-      DataMiner.log_debug "ADDING INDEX #{name}"
+      ::DataMiner.logger.debug "ADDING INDEX #{name}"
       connection.add_index table_name, ideal.columns, :name => ideal.name
       resource.reset_column_information
     end
     
     def remove_index(name)
-      DataMiner.log_debug "REMOVING INDEX #{name}"
+      ::DataMiner.logger.debug "REMOVING INDEX #{name}"
       connection.remove_index table_name, :name => name
       resource.reset_column_information
     end
     
-    def run(run)
-      _add_extra_columns
+    def run
       _create_table
       _set_primary_key
       _remove_columns
       _add_columns
       _remove_indexes
       _add_indexes
-      DataMiner.log_debug "ran #{inspect}"
+      nil
     end
-    
-    EXTRA_COLUMNS = {
-      :updated_at => :datetime,
-      :created_at => :datetime
-    }
-    def _add_extra_columns
-      EXTRA_COLUMNS.each do |extra_name, extra_type|
-        send extra_type, extra_name unless ideal_column extra_name
-      end
-    end
-    
+        
     def _create_table
       if not resource.table_exists?
-        DataMiner.log_debug "CREATING TABLE #{table_name} with #{create_table_options.inspect}"
+        create_table_options = @create_table_options.dup
+        create_table_options.symbolize_keys!
+        ::DataMiner.logger.debug "CREATING TABLE #{table_name} with #{create_table_options.inspect}"
         connection.create_table table_name, create_table_options do |t|
-          t.integer :data_miner_placeholder
+          t.integer 'data_miner_placeholder'
         end
         resource.reset_column_information
       end
@@ -190,23 +187,23 @@ module DataMiner
     # FIXME mysql only
     def _set_primary_key
       if ideal_primary_key_name == 'id' and not ideal_column('id')
-        DataMiner.log_debug "no special primary key set on #{table_name}, so using 'id'"
-        column 'id', :primary_key
+        ::DataMiner.logger.debug "no special primary key set on #{table_name}, so using 'id'"
+        column 'id', :primary_key # needs to be a sym?
       end
       actual = actual_column actual_primary_key_name
       ideal = ideal_column ideal_primary_key_name
       if not column_equivalent? actual, ideal
-        DataMiner.log_debug "looks like #{table_name} has a bad (or missing) primary key"
+        ::DataMiner.logger.debug "looks like #{table_name} has a bad (or missing) primary key"
         if actual
-          DataMiner.log_debug "looks like primary key needs to change from #{actual_primary_key_name} to #{ideal_primary_key_name}, re-creating #{table_name} from scratch"
+          ::DataMiner.logger.debug "looks like primary key needs to change from #{actual_primary_key_name} to #{ideal_primary_key_name}, re-creating #{table_name} from scratch"
           connection.drop_table table_name
           resource.reset_column_information
           _create_table
         end
         place_column ideal_primary_key_name
         unless ideal.type.to_s == 'primary_key'
-          DataMiner.log_debug "SETTING #{ideal_primary_key_name} AS PRIMARY KEY"
-          if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
+          ::DataMiner.logger.debug "SETTING #{ideal_primary_key_name} AS PRIMARY KEY"
+          if connection.adapter_name.downcase == 'sqlite'
             connection.execute "CREATE UNIQUE INDEX IDX_#{table_name}_#{ideal_primary_key_name} ON #{table_name} (#{ideal_primary_key_name} ASC)"
           else
             connection.execute "ALTER TABLE `#{table_name}` ADD PRIMARY KEY (`#{ideal_primary_key_name}`)"

data/lib/data_miner/tap.rb

@@ -1,52 +1,52 @@
-
-module DataMiner
+require 'escape'
+class DataMiner
   class Tap
-    attr_reader :base
-    attr_reader :position_in_run
+    attr_reader :config
     attr_reader :description
     attr_reader :source
     attr_reader :options
-    delegate :resource, :to => :base
 
-    def initialize(base, position_in_run, description, source, options = {})
-      options.symbolize_keys!
-      DataMiner.log_or_raise "Tap has to be the first step." unless position_in_run == 0
-      @base = base
-      @position_in_run = position_in_run
+    def initialize(config, description, source, options = {})
+      @config = config
+      @options = options.dup
+      @options.stringify_keys!
       @description = description
       @source = source
-      @options = options
+    end
+    
+    def resource
+      config.resource
     end
     
     def inspect
-      "Tap(#{resource}): #{description} (#{source})"
+      %{#<DataMiner::Tap(#{resource}): #{description} (#{source})>}
     end
     
-    def run(run)
+    def run
       [ source_table_name, resource.table_name ].each do |possible_obstacle|
-        if connection.table_exists?(possible_obstacle)
+        if connection.table_exists? possible_obstacle
           connection.drop_table possible_obstacle
         end
       end
-      DataMiner.backtick_with_reporting taps_pull_cmd
+      ::DataMiner.backtick_with_reporting taps_pull_cmd
       if needs_table_rename?
         connection.rename_table source_table_name, resource.table_name
       end
-      DataMiner.log_info "ran #{inspect}"
+      nil
     end
     
-    private
-    
+    # sabshere 1/25/11 what if there were multiple connections
+    # blockenspiel doesn't like to delegate this to #resource
     def connection
-      ActiveRecord::Base.connection
+      ::ActiveRecord::Base.connection
     end
     
     def db_config
-      @_db_config ||= connection.instance_variable_get(:@config).dup.merge(options.except(:source_table_name))
+      @db_config ||= connection.instance_variable_get(:@config).stringify_keys.merge(options.except('source_table_name'))
     end
     
     def source_table_name
-      options[:source_table_name] || resource.table_name
+      options['source_table_name'] || resource.table_name
     end
     
     def needs_table_rename?
@@ -66,7 +66,7 @@ module DataMiner
 
     # never optional
     def database
-      db_config[:database]
+      db_config['database']
     end
     
     DEFAULT_PORTS = {
@@ -88,7 +88,7 @@ module DataMiner
     %w{ username password port host }.each do |x|
       module_eval %{
         def #{x}
-          db_config[:#{x}] || DEFAULT_#{x.upcase}S[adapter]
+          db_config['#{x}'] || DEFAULT_#{x.upcase}S[adapter]
         end
       }
     end
@@ -104,7 +104,7 @@ module DataMiner
     
     # taps pull mysql://root:password@localhost/taps_test http://foo:bar@data.brighterplanet.com:5000 --tables aircraft
     def taps_pull_cmd
-      Escape.shell_command [
+      ::Escape.shell_command [
         'taps',
         'pull',
         "#{adapter}://#{db_locator}",

data/lib/data_miner/verify.rb

@@ -1,35 +1,28 @@
-module DataMiner
+class DataMiner
   class Verify
-    class VerificationFailed < RuntimeError; end
-
-    attr_accessor :base, :position_in_run, :check, :description
-    delegate :resource, :to => :base
+    attr_reader :config
+    attr_reader :description
+    attr_reader :blk
+    
+    def initialize(config, description, &blk)
+      @config = config
+      @description = description
+      @blk = blk
+    end
     
-    def initialize(base, position_in_run, description, check)
-      self.base = base
-      self.position_in_run = position_in_run
-      self.description = description
-      self.check = check
+    def resource
+      config.resource
     end
 
     def inspect
-      "Verify(#{resource}) position #{position_in_run} (#{description})"
+      %{#<DataMiner::Verify(#{resource})  (#{description})>}
     end
 
-    def run(run)
-      begin
-        verification = check.call
-      rescue Exception => e  # need this to catch Test::Unit assertions
-        raise VerificationFailed,
-          "#{e.inspect}: #{e.backtrace.join("\n")}"
-      rescue => e
-        raise VerificationFailed,
-          "#{e.inspect}: #{e.backtrace.join("\n")}"
-      end
-      unless verification
-        raise VerificationFailed, "Result of check was false" 
+    def run
+      unless blk.call
+        raise VerificationFailed, "FAILED VERIFICATION: #{inspect}"
       end
-      DataMiner.log_info "performed #{inspect}"
+      nil
     end
   end
 end

data/lib/data_miner/version.rb

@@ -0,0 +1,3 @@
+class DataMiner
+  VERSION = '1.0.0'
+end

data/test/{test_helper.rb → helper.rb}

@@ -1,12 +1,29 @@
 require 'rubygems'
+require 'bundler'
+unless RUBY_VERSION >= '1.9'
+  gem 'fastercsv'
+  require 'fastercsv'
+end
+Bundler.setup
 require 'test/unit'
 require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+unless RUBY_VERSION >= '1.9'
+  require 'ruby-debug'
+end
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'data_miner'
+class Test::Unit::TestCase
+end
+
+test_log = File.open('test.log', 'w')
+test_log.sync = true
+DataMiner.logger = Logger.new test_log
+
+# because some of the test files reference it
+require 'errata'
 
-ENV['WIP'] = true if ENV['ALL'] == 'true'
+ENV['WIP'] = 'true' if ENV['ALL'] == 'true'
 
 ActiveRecord::Base.establish_connection(
   'adapter' => 'mysql',

data/test/{data_miner/attribute_test.rb → test_attribute.rb}

@@ -1,6 +1,6 @@
-require 'test_helper'
+require 'helper'
 
-class AttributeTest < Test::Unit::TestCase
+class TestAttribute < Test::Unit::TestCase
   context '#value_from_row' do
     setup do
       @airport = Airport.new

data/test/{data_miner_test.rb → test_old_syntax.rb}

@@ -1,4 +1,4 @@
-require 'test_helper'
+require 'helper'
 
 class TappedAirport < ActiveRecord::Base
   set_primary_key :iata_code
@@ -609,7 +609,7 @@ class CensusDivisionTrois < ActiveRecord::Base
       string   'census_region_name'
       integer  'census_region_number'
       index    'census_region_name', :name => 'homefry'
-      index   ['number_code', 'name', 'census_region_name', 'census_region_number', 'updated_at', 'created_at']
+      index   ['number_code', 'name', 'census_region_name', 'census_region_number']
     end
   end
 end
@@ -627,7 +627,7 @@ class CensusDivisionFour < ActiveRecord::Base
 end
 
 # todo: have somebody properly organize these
-class DataMinerTest < Test::Unit::TestCase
+class TestOldSyntax < Test::Unit::TestCase
   if ENV['WIP']
     context 'with nullify option' do
       should 'treat blank fields as null values' do
@@ -660,9 +660,9 @@ class DataMinerTest < Test::Unit::TestCase
           end
         end
       end
-      assert_kind_of DataMiner::Import, AutomobileFuelType.data_miner_base.steps.first
-      assert_equal 'http://example.com', AutomobileFuelType.data_miner_base.steps.first.table.package.url
-      assert_equal 1, AutomobileFuelType.data_miner_base.step_counter
+      assert_kind_of DataMiner::Import, AutomobileFuelType.data_miner_config.steps.first
+      assert_equal 'http://example.com', AutomobileFuelType.data_miner_config.steps.first.table.package.url
+      assert_equal 1, AutomobileFuelType.data_miner_config.step_counter
     end
     should "stop and finish if it gets a DataMiner::Finish" do
       AutomobileMakeFleetYear.delete_all
@@ -670,8 +670,8 @@ class DataMinerTest < Test::Unit::TestCase
       $force_finish = true
       AutomobileMakeFleetYear.run_data_miner!
       assert_equal 0, AutomobileMakeFleetYear.count
-      assert_equal true, (AutomobileMakeFleetYear.data_miner_runs.count > 0)
-      assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? and not run.skipped and not run.killed? }
+      assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
+      assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? and not run.skipped and not run.killed? }
       $force_finish = false
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
@@ -683,8 +683,8 @@ class DataMinerTest < Test::Unit::TestCase
       $force_skip = true
       AutomobileMakeFleetYear.run_data_miner!
       assert_equal 0, AutomobileMakeFleetYear.count
-      assert_equal true, (AutomobileMakeFleetYear.data_miner_runs.count > 0)
-      assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.skipped? and not run.finished? and not run.killed? }
+      assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
+      assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.skipped? and not run.finished? and not run.killed? }
       $force_skip = false
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
@@ -693,27 +693,25 @@ class DataMinerTest < Test::Unit::TestCase
     should "eagerly enforce a schema" do
       ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
         t.string   'name'
-        # t.datetime 'updated_at'
-        # t.datetime 'created_at'
         t.string   'census_region_name'
         # t.integer  'census_region_number'
       end
       ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
       CensusDivisionTrois.reset_column_information
-      missing_columns = %w{ updated_at created_at census_region_number }
+      missing_columns = %w{ census_region_number }
 
       # sanity check
       missing_columns.each do |column|
-        assert_equal false, CensusDivisionTrois.column_names.include?(column)
+        assert_false CensusDivisionTrois.column_names.include?(column)
       end
-      assert_equal false, ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
       
       3.times do
         CensusDivisionTrois.run_data_miner!
         missing_columns.each do |column|
-          assert_equal true, CensusDivisionTrois.column_names.include?(column)
+          assert CensusDivisionTrois.column_names.include?(column)
         end
-        assert_equal true, ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+        assert ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
         assert_equal :string, CensusDivisionTrois.columns_hash[CensusDivisionTrois.primary_key].type
       end
     end
@@ -721,27 +719,25 @@ class DataMinerTest < Test::Unit::TestCase
     should "let schemas work with default id primary keys" do
       ActiveRecord::Base.connection.create_table 'census_division_fours', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
         t.string   'name'
-        # t.datetime 'updated_at'
-        # t.datetime 'created_at'
         t.string   'census_region_name'
         # t.integer  'census_region_number'
       end
       ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_fours ADD INDEX (census_region_name)'
       CensusDivisionFour.reset_column_information
-      missing_columns = %w{ updated_at created_at census_region_number }
+      missing_columns = %w{ census_region_number }
     
       # sanity check
       missing_columns.each do |column|
-        assert_equal false, CensusDivisionFour.column_names.include?(column)
+        assert_false CensusDivisionFour.column_names.include?(column)
       end
-      assert_equal false, ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
+      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
       
       3.times do
         CensusDivisionFour.run_data_miner!
         missing_columns.each do |column|
-          assert_equal true, CensusDivisionFour.column_names.include?(column)
+          assert CensusDivisionFour.column_names.include?(column)
         end
-        assert_equal true, ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
+        assert ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
         assert_equal :integer, CensusDivisionFour.columns_hash[CensusDivisionFour.primary_key].type
       end
     end
@@ -793,29 +789,29 @@ class DataMinerTest < Test::Unit::TestCase
     end
     
     should "be idempotent" do
-      Country.data_miner_base.run
+      Country.data_miner_config.run
       a = Country.count
-      Country.data_miner_base.run
+      Country.data_miner_config.run
       b = Country.count
       assert_equal a, b
     
-      CensusRegion.data_miner_base.run
+      CensusRegion.data_miner_config.run
       a = CensusRegion.count
-      CensusRegion.data_miner_base.run
+      CensusRegion.data_miner_config.run
       b = CensusRegion.count
       assert_equal a, b
     end
       
     should "hash things" do
-      AutomobileVariant.data_miner_base.steps[0].run(nil)
+      AutomobileVariant.data_miner_config.steps[0].run
       assert AutomobileVariant.first.row_hash.present?
     end
   
     should "process a callback block instead of a method" do
       AutomobileVariant.delete_all
-      AutomobileVariant.data_miner_base.steps[0].run(nil)
+      AutomobileVariant.data_miner_config.steps[0].run
       assert !AutomobileVariant.first.fuel_efficiency_city.present?
-      AutomobileVariant.data_miner_base.steps.last.run(nil)
+      AutomobileVariant.data_miner_config.steps.last.run
       assert AutomobileVariant.first.fuel_efficiency_city.present?
     end
   
@@ -871,7 +867,7 @@ class DataMinerTest < Test::Unit::TestCase
     end
   end
   should "mark the run as skipped if verification fails" do
-    AutomobileFuelType.data_miner_base.instance_eval do
+    AutomobileFuelType.data_miner_config.instance_eval do
       verify "failure" do
         false
       end