data_miner 0.4.30 → 0.4.31

data/VERSION

@@ -1 +1 @@
-0.4.30
+0.4.31

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.30"
+  s.version = "0.4.31"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-18}
+  s.date = %q{2010-05-19}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
      "lib/data_miner/import.rb",
      "lib/data_miner/process.rb",
      "lib/data_miner/run.rb",
+     "lib/data_miner/schema.rb",
      "lib/data_miner/tap.rb",
      "test/data_miner_test.rb",
      "test/test_helper.rb"

data/lib/data_miner.rb

@@ -25,6 +25,7 @@ require 'data_miner/import'
 require 'data_miner/tap'
 require 'data_miner/process'
 require 'data_miner/run'
+require 'data_miner/schema'
 
 module DataMiner
   class MissingHashColumn < RuntimeError; end
@@ -62,6 +63,10 @@ module DataMiner
     logger.info "[data_miner gem] #{message}"
   end
   
+  def self.log_debug(message)
+    logger.debug "[data_miner gem] #{message}"
+  end
+  
   def self.run(options = {})
     DataMiner::Configuration.run options
   end

data/lib/data_miner/configuration.rb

@@ -11,6 +11,13 @@ module DataMiner
       @attributes = HashWithIndifferentAccess.new
     end
     
+    def schema(create_table_options = {}, &block)
+      step = DataMiner::Schema.new self, step_counter, create_table_options
+      Blockenspiel.invoke block, step
+      steps << step
+      self.step_counter += 1
+    end
+    
     def process(method_name_or_block_description, &block)
       steps << DataMiner::Process.new(self, step_counter, method_name_or_block_description, &block)
       self.step_counter += 1

data/lib/data_miner/schema.rb

@@ -0,0 +1,217 @@
+module DataMiner
+  class Schema
+    include Blockenspiel::DSL
+    
+    attr_reader :configuration
+    attr_reader :position_in_run
+    attr_reader :create_table_options
+    delegate :resource, :to => :configuration
+
+    def initialize(configuration, position_in_run, create_table_options)
+      @configuration = configuration
+      @position_in_run = position_in_run
+      @create_table_options = create_table_options
+    end
+    
+    def connection
+      ActiveRecord::Base.connection
+    end
+    
+    def table_name
+      resource.table_name
+    end
+    
+    def ideal_table
+      @ideal_table ||= ActiveRecord::ConnectionAdapters::TableDefinition.new(connection)
+    end
+    
+    def ideal_indexes
+      @ideal_indexes ||= Array.new
+    end
+    
+    def actual_indexes
+      connection.indexes table_name
+    end
+    
+    def description
+      "Define a table called #{table_name} with primary key #{primary_key}"
+    end
+    
+    def inspect
+      "Block(#{resource}): #{description}"
+    end
+    
+    # lifted straight from activerecord-3.0.0.beta3/lib/active_record/connection_adapters/abstract/schema_definitions.rb
+    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
+      class_eval <<-EOV
+        def #{column_type}(*args)                                               # def string(*args)
+          options = args.extract_options!                                       #   options = args.extract_options!
+          column_names = args                                                   #   column_names = args
+                                                                                #
+          column_names.each { |name| ideal_table.column(name, '#{column_type}', options) }  #   column_names.each { |name| ideal_table.column(name, 'string', options) }
+        end                                                                     # end
+      EOV
+    end
+    def column(*args)
+      ideal_table.column(*args)
+    end
+    # class IndexDefinition < Struct.new(:table, :name, :unique, :columns)
+    def index(columns, options = {})
+      options.symbolize_keys!
+      columns = Array.wrap columns
+      name = connection.index_name table_name, options.merge(:columns => columns)
+      index_unique = options.has_key?(:unique) ? options[:unique] : true
+      ideal_indexes.push ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, name, index_unique, columns)
+    end
+        
+    def primary_key
+      resource.primary_key.to_s
+    end
+    
+    INDEX_PROPERTIES = %w{ name columns }
+    COLUMN_PROPERTIES = %w{ name type }
+    %w{ column index }.each do |i|
+      eval %{
+        def #{i}_needs_to_be_placed?(name)
+          actual = actual_#{i} name
+          return true unless actual
+          ideal = ideal_#{i} name
+          #{i.upcase}_PROPERTIES.any? do |property|
+            DataMiner.log_debug "...comparing \#{actual.send(property).inspect}.to_s <-> \#{ideal.send(property).inspect}.to_s"
+            actual.send(property).to_s != ideal.send(property).to_s
+          end
+        end
+    
+        def #{i}_needs_to_be_removed?(name)
+          ideal_#{i}(name).nil?
+        end
+      }
+    end
+    
+    def ideal_column(name)
+      ideal_table[name.to_s]
+    end
+    
+    def actual_column(name)
+      resource.columns_hash[name.to_s]
+    end
+    
+    def ideal_index(name)
+      ideal_indexes.detect { |ideal| ideal.name == name.to_s }
+    end
+    
+    def actual_index(name)
+      actual_indexes.detect { |actual| actual.name == name.to_s }
+    end
+  
+    def place_column(name)
+      remove_column name if actual_column name
+      ideal = ideal_column name
+      DataMiner.log_info "adding column #{name}"
+      connection.add_column table_name, name, ideal.type.to_sym # symbol type!
+      resource.reset_column_information
+    end
+    
+    def remove_column(name)
+      DataMiner.log_info "removing column #{name}"
+      connection.remove_column table_name, name
+      resource.reset_column_information
+    end
+    
+    def place_index(name)
+      remove_index name if actual_index name
+      ideal = ideal_index name
+      DataMiner.log_info "adding index #{name}"
+      connection.add_index table_name, ideal.columns, :name => ideal.name
+      resource.reset_column_information
+    end
+    
+    def remove_index(name)
+      DataMiner.log_info "removing index #{name}"
+      connection.remove_index table_name, :name => name
+      resource.reset_column_information
+    end
+        
+    def run(run)
+      _add_extra_columns
+      _create_table
+      _set_primary_key
+      _remove_columns
+      _add_columns
+      _remove_indexes
+      _add_indexes
+      DataMiner.log_info "ran #{inspect}"
+    end
+    
+    EXTRA_COLUMNS = {
+      :updated_at => :datetime,
+      :created_at => :datetime,
+      :data_miner_last_run_id => :integer,
+      :data_miner_touch_count => :integer
+    }
+    def _add_extra_columns
+      EXTRA_COLUMNS.each do |extra_name, extra_type|
+        send extra_type, extra_name unless ideal_column extra_name
+      end
+    end
+    
+    def _create_table
+      if not resource.table_exists?
+        DataMiner.log_info "creating table #{table_name} with #{create_table_options.inspect}"
+        connection.create_table table_name, create_table_options do |t|
+          t.integer :data_miner_placeholder
+        end
+        resource.reset_column_information
+      end
+    end
+    
+    # FIXME mysql only
+    def _set_primary_key
+      if column_needs_to_be_placed?(primary_key)
+        retries_allowed = 1
+        begin
+          place_column primary_key
+          DataMiner.log_info "adding primary key #{primary_key}"
+          connection.execute "ALTER TABLE `#{table_name}` ADD PRIMARY KEY (`#{primary_key}`)"
+        rescue
+          if retries_allowed > 0 and $!.message =~ /primary/i
+            DataMiner.log_info "looks like primary key changed, re-creating table from scratch"
+            connection.drop_table table_name
+            resource.reset_column_information
+            _create_table
+            retries_allowed -= 1
+            retry
+          else
+            raise $!
+          end
+        end
+        resource.reset_column_information
+      end
+    end
+    
+    def _remove_columns
+      resource.columns_hash.values.each do |actual|
+        remove_column actual.name if column_needs_to_be_removed? actual.name
+      end
+    end
+    
+    def _add_columns
+      ideal_table.columns.each do |ideal|
+        place_column ideal.name if column_needs_to_be_placed? ideal.name
+      end
+    end
+    
+    def _remove_indexes
+      actual_indexes.each do |actual|
+        remove_index actual.name if index_needs_to_be_removed? actual.name
+      end
+    end
+    
+    def _add_indexes
+      ideal_indexes.each do |ideal|
+        next if ideal.name == primary_key # this should already have been taken care of
+        place_index ideal.name if index_needs_to_be_placed? ideal.name
+      end
+    end
+  end
+end

data/test/data_miner_test.rb

@@ -1091,22 +1091,19 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
   set_primary_key :name
 
   data_miner do
-    process "create a table on the fly" do
-      create_table "automobile_make_fleet_years", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-        t.string "name"
-        t.string   "make_name"
-        t.string   "fleet"
-        t.integer   "year"
-        t.float    "fuel_efficiency"
-        t.string "fuel_efficiency_units"
-        t.integer  "volume"
-        t.string  "make_year_name"
-        t.datetime "created_at"
-        t.datetime "updated_at"
-        t.integer  'data_miner_touch_count'
-        t.integer  'data_miner_last_run_id'
-      end
-      execute 'ALTER TABLE automobile_make_fleet_years ADD PRIMARY KEY (name)'
+    schema :id => false do
+      string "name"
+      string   "make_name"
+      string   "fleet"
+      integer   "year"
+      float    "fuel_efficiency"
+      string "fuel_efficiency_units"
+      integer  "volume"
+      string  "make_year_name"
+      datetime "created_at"
+      datetime "updated_at"
+      integer  'data_miner_touch_count'
+      integer  'data_miner_last_run_id'
     end
 
     # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
@@ -1123,12 +1120,55 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
   end
 end
 
+class CensusDivisionTrois < ActiveRecord::Base
+  set_primary_key :number_code
+  data_miner do
+    schema :options => 'ENGINE=InnoDB default charset=utf8', :id => false do
+      string  'number_code'
+      string   'name'
+      string   'census_region_name'
+      integer  'census_region_number'
+      index    'census_region_name', :name => 'homefry'
+    end
+  end
+end
+
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
   if ENV['ALL'] == 'true' or ENV['NEW'] == 'true'
   end
     
   if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+    should "eagerly enforce a schema" do
+      ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
+        t.string   'name'
+        # t.datetime 'updated_at'
+        # t.datetime 'created_at'
+        t.string   'census_region_name'
+        # t.integer  'census_region_number'
+        # t.integer 'data_miner_touch_count'
+        # t.integer 'data_miner_last_run_id'
+      end
+      ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
+      CensusDivisionTrois.reset_column_information
+      missing_columns = %w{ updated_at created_at census_region_number data_miner_last_run_id data_miner_touch_count }
+
+      # sanity check
+      missing_columns.each do |column|
+        assert_equal false, CensusDivisionTrois.column_names.include?(column)
+      end
+      assert_equal false, ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+      
+      3.times do
+        CensusDivisionTrois.run_data_miner!
+        missing_columns.each do |column|
+          assert_equal true, CensusDivisionTrois.column_names.include?(column)
+        end
+        assert_equal true, ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+        assert_equal :string, CensusDivisionTrois.columns_hash[CensusDivisionTrois.primary_key].type
+      end
+    end
+    
     should "allow specifying dictionaries explicitly" do
       CensusDivisionDeux.run_data_miner!
       assert_equal 'South Region', CensusDivisionDeux.find(5).census_region_name

data/test/test_helper.rb

@@ -15,7 +15,7 @@ ActiveRecord::Base.establish_connection(
 )
 
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w{ aircraft aircraft_deux census_division_deux }
+  inflect.uncountable %w{ aircraft aircraft_deux census_division_deux census_division_trois }
 end
 
 class Test::Unit::TestCase

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 30
-  version: 0.4.30
+  - 31
+  version: 0.4.31
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-18 00:00:00 -04:00
+date: 2010-05-19 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -183,6 +183,7 @@ files:
 - lib/data_miner/import.rb
 - lib/data_miner/process.rb
 - lib/data_miner/run.rb
+- lib/data_miner/schema.rb
 - lib/data_miner/tap.rb
 - test/data_miner_test.rb
 - test/test_helper.rb