data_miner 0.4.14 → 0.4.15

data/VERSION

@@ -1 +1 @@
-0.4.14
+0.4.15

data/data_miner.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.14"
+  s.version = "0.4.15"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
      "data_miner.gemspec",
      "lib/data_miner.rb",
      "lib/data_miner/attribute.rb",
+     "lib/data_miner/clone.rb",
      "lib/data_miner/configuration.rb",
      "lib/data_miner/dictionary.rb",
      "lib/data_miner/import.rb",

data/lib/data_miner/clone.rb

@@ -0,0 +1,88 @@
+module DataMiner
+  class Clone
+    attr_accessor :configuration
+    attr_accessor :position_in_run
+    attr_accessor :description
+    attr_accessor :options
+    delegate :resource, :to => :configuration
+
+    def initialize(configuration, position_in_run, description, options = {})
+      DataMiner.log_or_raise "Clone has to be the first step." unless position_in_run == 0
+      DataMiner.log_or_raise "Clone needs :url" unless options[:url].present?
+      @configuration = configuration
+      @position_in_run = position_in_run
+      @description = description
+      @options = options
+    end
+    
+    def inspect
+      "Clone(#{resource}): #{description}"
+    end
+    
+    def run(run)
+      download_sql_source
+      perform_sanity_check unless options[:sanity_check] == false
+      execute_sql_source
+      DataMiner.log_info "ran #{inspect}"
+    end
+    
+    private
+    
+    # from remote_table
+    def tempfile_path    
+      return @_tempfile_path if @_tempfile_path
+      @_tempfile_path = Tempfile.open(options[:url].gsub(/[^a-z0-9]+/i, '_')[0,100]).path
+      FileUtils.rm_f @_tempfile_path
+      at_exit { FileUtils.rm_rf @_tempfile_path }
+      @_tempfile_path
+    end
+    
+    def download_sql_source
+      cmd = %{
+        curl \
+        --silent \
+        --header "Expect: " \
+        --location \
+        "#{options[:url]}" \
+        --output "#{tempfile_path}"
+      }
+      `#{cmd}`
+    end
+
+    def perform_sanity_check
+      File.open(tempfile_path, 'r') do |infile|
+        while (line = infile.gets)
+          line_essence = line.gsub /[^\-\_\.a-zA-Z0-9]+/, ' '
+          if line_essence =~ /(INSERT\s+INTO|CREATE\s+TABLE|ALTER\s+TABLE|DROP\s+TABLE\s+[^I]|DROP\s+TABLE\s+IF\s+EXISTS)\s+([^\s]+)/i
+            one = $1
+            two = $2
+            unless two.split('.').last == resource.table_name
+              DataMiner.log_or_raise %{
+
+Warning: clone SQL tries to #{one} on `#{two}` instead of `#{resource.table_name}`. (#{line[0,100]}...)
+
+If you want to ignore this, use clone 'X', :url => 'Y', :sanity_check => false
+
+If you need to set a different table name, you could say set_table_name '#{two}' in your ActiveRecord model.
+              }
+            end
+          end
+        end
+      end
+    end
+
+    def execute_sql_source
+      mysql_config = ActiveRecord::Base.connection.instance_variable_get :@config
+      cmd = %{
+        mysql                                   \
+        --batch                                 \
+        #{"--host=\"#{mysql_config[:hostname]}\"" if mysql_config[:hostname].present?} \
+        --user="#{mysql_config[:username]}"     \
+        --password="#{mysql_config[:password]}" \
+        --database="#{mysql_config[:database]}" \
+        --execute="SOURCE #{tempfile_path}"
+      }
+      `#{cmd}`
+    end
+  end
+end

data/lib/data_miner/configuration.rb

@@ -12,8 +12,13 @@ module DataMiner
     end
     
     def process(method_name_or_block_description, &block)
-      self.runnable_counter += 1
       runnables << DataMiner::Process.new(self, runnable_counter, method_name_or_block_description, &block)
+      self.runnable_counter += 1
+    end
+
+    def clone(description, options = {})
+      runnables << DataMiner::Clone.new(self, runnable_counter, description, options)
+      self.runnable_counter += 1
     end
 
     def import(*args, &block)
@@ -24,10 +29,10 @@ module DataMiner
       end
       options = args.last
         
-      self.runnable_counter += 1
       runnable = DataMiner::Import.new self, runnable_counter, description, options
       Blockenspiel.invoke block, runnable
       runnables << runnable
+      self.runnable_counter += 1
     end
 
     # Mine data for this class.

data/lib/data_miner.rb

@@ -12,6 +12,7 @@ require 'data_miner/configuration'
 require 'data_miner/dictionary'
 require 'data_miner/import'
 require 'data_miner/process'
+require 'data_miner/clone'
 require 'data_miner/run'
 
 module DataMiner

data/test/data_miner_test.rb

@@ -438,6 +438,24 @@ class Airport < ActiveRecord::Base
   end
 end
 
+class ClonedAirport < ActiveRecord::Base
+  set_primary_key :iata_code
+  set_table_name 'airports'
+  
+  data_miner do
+    clone 'a sanitized airports table', :url => 'http://data.brighterplanet.com/airports.sql', :sanity_check => true
+  end
+end
+
+class BadlyClonedAirport < ActiveRecord::Base
+  set_primary_key :iata_code
+  set_table_name 'badly_cloned_airports'
+  
+  data_miner do
+    clone 'a sanitized airports table', :url => 'http://data.brighterplanet.com/airports.sql', :sanity_check => true
+  end
+end
+
 class CensusRegion < ActiveRecord::Base
   set_primary_key :number
   
@@ -884,7 +902,21 @@ end
 
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
+  if ENV['NEW'] == 'true'
+  end
+    
   if ENV['FAST'] == 'true'
+    should "clone airports" do
+      ClonedAirport.run_data_miner!
+      assert ClonedAirport.count > 0
+    end
+    
+    should "raise an error when the SQL for cloning doesn't seem to match up" do
+      assert_raises(RuntimeError) do
+        BadlyClonedAirport.run_data_miner!
+      end
+    end
+    
     should "import airports" do
       Airport.run_data_miner!
       assert Airport.count > 0

data/test/test_helper.rb

@@ -83,6 +83,10 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
   end
   execute 'ALTER TABLE t100_flight_segments ADD PRIMARY KEY (row_hash);'
   
+  create_table 'badly_cloned_airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string 'placeholder'
+  end
+  
   create_table 'airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
     t.string   'iata_code'
     t.string   'name'

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 14
-  version: 0.4.14
+  - 15
+  version: 0.4.15
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -150,6 +150,7 @@ files:
 - data_miner.gemspec
 - lib/data_miner.rb
 - lib/data_miner/attribute.rb
+- lib/data_miner/clone.rb
 - lib/data_miner/configuration.rb
 - lib/data_miner/dictionary.rb
 - lib/data_miner/import.rb