data_miner 0.4.26 → 0.4.27

data/Rakefile

@@ -18,6 +18,7 @@ begin
     gem.add_dependency 'blockenspiel', '>=0.3.2'
     gem.add_dependency 'log4r', '>=1.1.7'
     gem.add_dependency 'errata', '>=0.2.1'
+    gem.add_dependency 'taps', '>=0.3.5'
     gem.add_development_dependency "loose_tight_dictionary", ">=0.0.5"
     gem.require_path = "lib"
     gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild

data/VERSION

@@ -1 +1 @@
-0.4.26
+0.4.27

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.26"
+  s.version = "0.4.27"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-06}
+  s.date = %q{2010-05-07}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -27,12 +27,12 @@ Gem::Specification.new do |s|
      "data_miner.gemspec",
      "lib/data_miner.rb",
      "lib/data_miner/attribute.rb",
-     "lib/data_miner/clone.rb",
      "lib/data_miner/configuration.rb",
      "lib/data_miner/dictionary.rb",
      "lib/data_miner/import.rb",
      "lib/data_miner/process.rb",
      "lib/data_miner/run.rb",
+     "lib/data_miner/tap.rb",
      "test/data_miner_test.rb",
      "test/test_helper.rb"
   ]
@@ -60,6 +60,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
       s.add_runtime_dependency(%q<errata>, [">= 0.2.1"])
+      s.add_runtime_dependency(%q<taps>, [">= 0.3.5"])
       s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     else
       s.add_dependency(%q<remote_table>, [">= 0.2.20"])
@@ -70,6 +71,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_dependency(%q<log4r>, [">= 1.1.7"])
       s.add_dependency(%q<errata>, [">= 0.2.1"])
+      s.add_dependency(%q<taps>, [">= 0.3.5"])
       s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     end
   else
@@ -81,6 +83,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
     s.add_dependency(%q<log4r>, [">= 1.1.7"])
     s.add_dependency(%q<errata>, [">= 0.2.1"])
+    s.add_dependency(%q<taps>, [">= 0.3.5"])
     s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
   end
 end

data/lib/data_miner.rb

@@ -22,8 +22,8 @@ require 'data_miner/attribute'
 require 'data_miner/configuration'
 require 'data_miner/dictionary'
 require 'data_miner/import'
+require 'data_miner/tap'
 require 'data_miner/process'
-require 'data_miner/clone'
 require 'data_miner/run'
 
 module DataMiner
@@ -69,6 +69,24 @@ module DataMiner
   def self.resource_names
     DataMiner::Configuration.resource_names
   end
+  
+    # TODO this should probably live somewhere else
+  def self.backtick_with_reporting(cmd, raise_on_error = true)
+    cmd = cmd.gsub /\s+/m, ' '
+    output = `#{cmd}`
+    if raise_on_error and not $?.success?
+      raise %{
+From the data_miner gem...
+
+Command failed:
+#{cmd}
+
+Output:
+#{output}
+}
+    end
+  end
+
 end
 
 ActiveRecord::Base.class_eval do

data/lib/data_miner/configuration.rb

@@ -16,8 +16,8 @@ module DataMiner
       self.step_counter += 1
     end
 
-    def clone(description, options = {})
-      steps << DataMiner::Clone.new(self, step_counter, description, options)
+    def tap(description, source, options = {})
+      steps << DataMiner::Tap.new(self, step_counter, description, source, options)
       self.step_counter += 1
     end
 

data/lib/data_miner/tap.rb

@@ -0,0 +1,137 @@
+
+module DataMiner
+  class Tap
+    attr_reader :configuration
+    attr_reader :position_in_run
+    attr_reader :description
+    attr_reader :source
+    attr_reader :options
+    delegate :resource, :to => :configuration
+
+    def initialize(configuration, position_in_run, description, source, options = {})
+      options.symbolize_keys!
+      DataMiner.log_or_raise "Tap has to be the first step." unless position_in_run == 0
+      @configuration = configuration
+      @position_in_run = position_in_run
+      @description = description
+      @source = source
+      @options = options
+    end
+    
+    def inspect
+      "Tap(#{resource}): #{description} (#{source})"
+    end
+    
+    def run(run)
+      [ source_table_name, resource.table_name ].each do |possible_obstacle|
+        if connection.table_exists?(possible_obstacle)
+          connection.drop_table possible_obstacle
+        end
+      end
+      DataMiner.backtick_with_reporting taps_pull_cmd
+      if needs_table_rename?
+        connection.rename_table source_table_name, resource.table_name
+      end
+      DataMiner.log_info "ran #{inspect}"
+    end
+    
+    private
+    
+    def connection
+      ActiveRecord::Base.connection
+    end
+    
+    def db_config
+      @_db_config ||= connection.instance_variable_get(:@config).dup.merge(options.except(:source_table_name))
+    end
+    
+    def source_table_name
+      options[:source_table_name] || resource.table_name
+    end
+    
+    def needs_table_rename?
+      source_table_name != resource.table_name
+    end
+    
+    def adapter
+      case connection.adapter_name
+      when /mysql/i
+        'mysql'
+      when /postgres/i
+        'postgres'
+      when /sqlite/i
+        'sqlite'
+      end
+    end
+
+    # never optional
+    def database
+      db_config[:database]
+    end
+    
+    DEFAULT_PORTS = {
+      'mysql' => 3306,
+      'postgres' => 5432
+    }
+    
+    DEFAULT_USERNAMES = {
+      'mysql' => 'root',
+      'postgres' => ''
+    }
+    
+    DEFAULT_PASSWORDS = {}
+    DEFAULT_PASSWORDS.default = ''
+    
+    DEFAULT_HOSTS = {}
+    DEFAULT_HOSTS.default = 'localhost'
+
+    %w{ username password port host }.each do |x|
+      module_eval %{
+        def #{x}
+          db_config[:#{x}] || DEFAULT_#{x.upcase}S[adapter]
+        end
+      }
+    end
+    
+    def db_locator
+      case adapter
+      when 'mysql', 'postgres'
+        "#{username}:#{password}@#{host}:#{port}/#{database}"
+      when 'sqlite'
+        database
+      end
+    end
+    
+    # taps pull mysql://root:password@localhost/taps_test http://foo:bar@data.brighterplanet.com:5000 --tables aircraft
+    def taps_pull_cmd
+      "taps pull #{adapter}://#{db_locator} #{source} --tables #{source_table_name}"
+    end
+    
+    # 2.3.5 mysql
+    # * <tt>:host</tt> - Defaults to "localhost".
+    # * <tt>:port</tt> - Defaults to 3306.
+    # * <tt>:socket</tt> - Defaults to "/tmp/mysql.sock".
+    # * <tt>:username</tt> - Defaults to "root"
+    # * <tt>:password</tt> - Defaults to nothing.
+    # * <tt>:database</tt> - The name of the database. No default, must be provided.
+    # * <tt>:encoding</tt> - (Optional) Sets the client encoding by executing "SET NAMES <encoding>" after connection.
+    # * <tt>:reconnect</tt> - Defaults to false (See MySQL documentation: http://dev.mysql.com/doc/refman/5.0/en/auto-reconnect.html).
+    # * <tt>:sslca</tt> - Necessary to use MySQL with an SSL connection.
+    # * <tt>:sslkey</tt> - Necessary to use MySQL with an SSL connection.
+    # * <tt>:sslcert</tt> - Necessary to use MySQL with an SSL connection.
+    # * <tt>:sslcapath</tt> - Necessary to use MySQL with an SSL connection.
+    # * <tt>:sslcipher</tt> - Necessary to use MySQL with an SSL connection.
+    # 2.3.5 mysql
+    # * <tt>:host</tt> - Defaults to "localhost".
+    # * <tt>:port</tt> - Defaults to 5432.
+    # * <tt>:username</tt> - Defaults to nothing.
+    # * <tt>:password</tt> - Defaults to nothing.
+    # * <tt>:database</tt> - The name of the database. No default, must be provided.
+    # * <tt>:schema_search_path</tt> - An optional schema search path for the connection given as a string of comma-separated schema names.  This is backward-compatible with the <tt>:schema_order</tt> option.
+    # * <tt>:encoding</tt> - An optional client encoding that is used in a <tt>SET client_encoding TO <encoding></tt> call on the connection.
+    # * <tt>:min_messages</tt> - An optional client min messages that is used in a <tt>SET client_min_messages TO <min_messages></tt> call on the connection.
+    # * <tt>:allow_concurrency</tt> - If true, use async query methods so Ruby threads don't deadlock; otherwise, use blocking query methods.
+    # 2.3.5 sqlite[3]
+    # * <tt>:database</tt> - Path to the database file.
+  end
+end

data/test/data_miner_test.rb

@@ -441,21 +441,12 @@ class Airport < ActiveRecord::Base
   end
 end
 
-class ClonedAirport < ActiveRecord::Base
+class TappedAirport < ActiveRecord::Base
   set_primary_key :iata_code
-  set_table_name 'airports'
   
   data_miner do
-    clone 'a sanitized airports table', :url => 'http://data.brighterplanet.com/airports.sql', :sanity_check => true
-  end
-end
-
-class BadlyClonedAirport < ActiveRecord::Base
-  set_primary_key :iata_code
-  set_table_name 'badly_cloned_airports'
-  
-  data_miner do
-    clone 'a sanitized airports table', :url => 'http://data.brighterplanet.com/airports.sql', :sanity_check => true
+    tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@data.brighterplanet.com:5001", :source_table_name => 'airports'
+    # tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@localhost:5000", :source_table_name => 'airports'
   end
 end
 
@@ -1063,7 +1054,7 @@ class AircraftDeux < ActiveRecord::Base
     %w{ D }.each do |letter|
       import("ICAO codes starting with letter #{letter} used by the FAA",
               :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
-              :encoding => 'US-ASCII',
+              :encoding => 'windows-1252',
               :errata => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
               :row_xpath => '//table/tr[2]/td/table/tr',
               :column_xpath => 'td') do
@@ -1094,23 +1085,17 @@ class DataMinerTest < Test::Unit::TestCase
       assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
       assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
     end
-    
-    should "clone airports" do
-      ClonedAirport.run_data_miner!
-      assert ClonedAirport.count > 0
-    end
-    
-    should "raise an error when the SQL for cloning doesn't seem to match up" do
-      assert_raises(RuntimeError) do
-        BadlyClonedAirport.run_data_miner!
-      end
-    end
-    
+        
     should "import airports" do
       Airport.run_data_miner!
       assert Airport.count > 0
     end
     
+    should "tap airports" do
+      TappedAirport.run_data_miner!
+      assert TappedAirport.count > 0
+    end
+    
     should "pull in census divisions using a data.brighterplanet.com dictionary" do
       CensusDivision.run_data_miner!
       assert CensusDivision.count > 0

data/test/test_helper.rb

@@ -86,9 +86,9 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.integer 'data_miner_last_run_id'
   end
   execute 'ALTER TABLE t100_flight_segments ADD PRIMARY KEY (row_hash);'
-  
-  create_table 'badly_cloned_airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
-    t.string 'placeholder'
+
+  create_table 'tapped_airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string 'i_am_just_here_to_get_in_the_way'
   end
   
   create_table 'airports', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 26
-  version: 0.4.26
+  - 27
+  version: 0.4.27
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-06 00:00:00 -04:00
+date: 2010-05-07 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -131,9 +131,23 @@ dependencies:
   type: :runtime
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
-  name: loose_tight_dictionary
+  name: taps
   prerelease: false
   requirement: &id009 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 3
+        - 5
+        version: 0.3.5
+  type: :runtime
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: loose_tight_dictionary
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -143,7 +157,7 @@ dependencies:
         - 5
         version: 0.0.5
   type: :development
-  version_requirements: *id009
+  version_requirements: *id010
 description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
 email: seamus@abshere.net
 executables: []
@@ -164,12 +178,12 @@ files:
 - data_miner.gemspec
 - lib/data_miner.rb
 - lib/data_miner/attribute.rb
-- lib/data_miner/clone.rb
 - lib/data_miner/configuration.rb
 - lib/data_miner/dictionary.rb
 - lib/data_miner/import.rb
 - lib/data_miner/process.rb
 - lib/data_miner/run.rb
+- lib/data_miner/tap.rb
 - test/data_miner_test.rb
 - test/test_helper.rb
 has_rdoc: true

data/lib/data_miner/clone.rb

@@ -1,86 +0,0 @@
-module DataMiner
-  class Clone
-    attr_accessor :configuration
-    attr_accessor :position_in_run
-    attr_accessor :description
-    attr_accessor :options
-    delegate :resource, :to => :configuration
-
-    def initialize(configuration, position_in_run, description, options = {})
-      DataMiner.log_or_raise "Clone has to be the first step." unless position_in_run == 0
-      DataMiner.log_or_raise "Clone needs :url" unless options[:url].present?
-      @configuration = configuration
-      @position_in_run = position_in_run
-      @description = description
-      @options = options
-    end
-    
-    def inspect
-      "Clone(#{resource}): #{description}"
-    end
-    
-    def run(run)
-      download_sql_source
-      perform_sanity_check unless options[:sanity_check] == false
-      execute_sql_source
-      DataMiner.log_info "ran #{inspect}"
-    end
-    
-    private
-    
-    def tempfile_path
-      return @_tempfile_path if @_tempfile_path
-      @_tempfile_path = File.join Dir.tmpdir, rand.to_s
-      at_exit { FileUtils.rm_f @_tempfile_path }
-      @_tempfile_path
-    end
-    
-    def download_sql_source
-      cmd = %{
-        curl \
-        --silent \
-        --header "Expect: " \
-        --location \
-        "#{options[:url]}" \
-        --output "#{tempfile_path}"
-      }
-      `#{cmd}`
-    end
-
-    def perform_sanity_check
-      File.open(tempfile_path, 'r') do |infile|
-        while (line = infile.gets)
-          line_essence = line.gsub /[^\-\_\.a-zA-Z0-9]+/, ' '
-          if line_essence =~ /(INSERT\s+INTO|CREATE\s+TABLE|ALTER\s+TABLE|DROP\s+TABLE\s+[^I]|DROP\s+TABLE\s+IF\s+EXISTS)\s+([^\s]+)/i
-            one = $1
-            two = $2
-            unless two.split('.').last == resource.table_name
-              DataMiner.log_or_raise %{
-
-Warning: clone SQL tries to #{one} on `#{two}` instead of `#{resource.table_name}`. (#{line[0,100]}...)
-
-If you want to ignore this, use clone 'X', :url => 'Y', :sanity_check => false
-
-If you need to set a different table name, you could say set_table_name '#{two}' in your ActiveRecord model.
-              }
-            end
-          end
-        end
-      end
-    end
-
-    def execute_sql_source
-      mysql_config = ActiveRecord::Base.connection.instance_variable_get :@config
-      cmd = %{
-        mysql                                   \
-        --batch                                 \
-        #{"--host=\"#{mysql_config[:hostname]}\"" if mysql_config[:hostname].present?} \
-        --user="#{mysql_config[:username]}"     \
-        --password="#{mysql_config[:password]}" \
-        --database="#{mysql_config[:database]}" \
-        --execute="SOURCE #{tempfile_path}"
-      }
-      `#{cmd}`
-    end
-  end
-end