data_miner 1.3.8 → 2.0.1

data/.document

@@ -1,5 +0,0 @@
-README.rdoc
-lib/**/*.rb
-bin/*
-features/**/*.feature
-LICENSE

data/lib/data_miner/config.rb

@@ -1,124 +0,0 @@
-require 'blockenspiel'
-require 'benchmark'
-
-class DataMiner
-  class Config
-    include ::Blockenspiel::DSL
-    
-    attr_reader :resource
-
-    def initialize(resource)
-      @resource = resource
-    end
-    
-    def steps
-      @steps ||= []
-    end
-    
-    # def attributes
-    #   @attributes ||= {}
-    # end
-        
-    def process(method_id_or_description, &blk)
-      step = Process.new self, method_id_or_description, &blk
-      steps.push step
-    end
-
-    def tap(description, source, options = {})
-      step = Tap.new self, description, source, options
-      steps.push step
-    end
-
-    def import(*args, &blk)
-      if args.length == 1
-        description = '(no description)'
-      else
-        description = args[0]
-      end
-      options = args.last
-        
-      step = Import.new self, description, options
-      ::Blockenspiel.invoke blk, step
-      steps.push step
-    end
-
-    # Mine data for this class.
-    def run(options = {})
-      options = options.dup
-      options.stringify_keys!
-      
-      return if ::DataMiner.instance.call_stack.include? resource.name
-      ::DataMiner.instance.call_stack.push resource.name
-      
-      finished = false
-      skipped = false
-      run = if Run.table_exists?
-        Run.create! :started_at => ::Time.now, :resource_name => resource.name, :killed => true
-      end
-      resource.delete_all if options['from_scratch']
-      begin
-        steps.each do |step|
-          time = ::Benchmark.realtime { step.run }
-          resource.reset_column_information
-          DataMiner.logger.info %{Ran #{step.inspect} in #{time.to_i}}
-        end
-        finished = true
-      rescue Finish
-        finished = true
-      rescue Skip
-        skipped = true
-      ensure
-        if Run.table_exists?
-          run.update_attributes! :terminated_at => ::Time.now, :finished => finished, :skipped => skipped, :killed => false
-        end
-        if ::DataMiner.instance.call_stack.first == resource.name and !options['preserve_call_stack_between_runs']
-          ::DataMiner.instance.call_stack.clear
-        end
-      end
-      nil
-    end
-    
-    def import_steps
-      steps.select { |step| step.is_a? Import }
-    end
-    
-    def before_invoke
-      
-    end
-    
-    def after_invoke
-      return unless resource.table_exists?
-      make_sure_unit_definitions_make_sense
-    end
-    
-    COMPLETE_UNIT_DEFINITIONS = [
-      %w{units},
-      %w{from_units to_units},
-      %w{units_field_name},
-      %w{units_field_name to_units},
-      %w{units_field_number},
-      %w{units_field_number to_units}
-    ]
-    
-    def make_sure_unit_definitions_make_sense
-      import_steps.each do |step|
-        step.attributes.each do |_, attribute|
-          if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
-            raise %{
-
-================================
-
-You don't have a valid unit definition for #{resource.name}##{attribute.name}.
-
-You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.inspect }.
-
-You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".
-
-================================
-            }
-          end
-        end
-      end
-    end
-  end
-end

data/lib/data_miner/import.rb

@@ -1,93 +0,0 @@
-require 'blockenspiel'
-require 'errata'
-require 'remote_table'
-class DataMiner
-  class Import
-    include ::Blockenspiel::DSL
-    
-    attr_reader :attributes
-    attr_reader :config
-    attr_reader :options
-    attr_reader :description
-    
-    def initialize(config, description, options = {})
-      @config = config
-      @description = description
-      @options = options.dup
-      @options.stringify_keys!
-      # legacy
-      if @options.has_key? 'table'
-        DataMiner.logger.warn "'table' is no longer an allowed option, taking the url from it and ignoring the rest"
-        table_instance = @options.delete 'table'
-        @options['url'] = table_instance.url
-      end
-      # legacy
-      if @options.has_key?('errata') and not @options['errata'].is_a?(::Hash)
-        DataMiner.logger.warn "'errata' must be a hash of Errata options. taking the URL from the Errata instance you provided and ignoring everything else"
-        errata_instance = @options.delete 'errata'
-        @options['errata'] = { 'url' => errata_instance.options['url'] }
-      end
-    end
-        
-    def attributes
-      @attributes ||= ::ActiveSupport::OrderedHash.new
-    end
-    
-    def resource
-      config.resource
-    end
-
-    def inspect
-      %{#<DataMiner::Import(#{resource}) #{description}>}
-    end
-    
-    def store(attr_name, attr_options = {})
-      raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
-      attributes[attr_name] = Attribute.new self, attr_name, attr_options
-    end
-    
-    def key(attr_name, attr_options = {})
-      raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
-      @_key = attr_name
-      store attr_name, attr_options
-    end
-
-    def primary_key
-      resource.primary_key
-    end
-
-    def table
-      return @table if @table.is_a? ::RemoteTable
-      # don't mess with the originals
-      options = @options.dup
-      options['streaming'] = true
-      if options['errata']
-        errata_options = options['errata'].dup
-        errata_options.stringify_keys!
-        errata_options['responder'] ||= resource
-        options['errata'] = errata_options
-      end
-      @table = ::RemoteTable.new options
-    end
-
-    def free
-      attributes.each { |_, attr| attr.free }
-      @table.free if @table.is_a?(::RemoteTable)
-      @table = nil
-    end
-    
-    def run
-      table.each do |row|
-        record = resource.send "find_or_initialize_by_#{@_key}", attributes[@_key].value_from_row(row)
-        attributes.each { |_, attr| attr.set_record_from_row record, row }
-        begin
-          record.save!
-        rescue
-          DataMiner.logger.warn "[data_miner] Got #{$!.inspect} when trying to save #{row}"
-        end
-      end
-      free
-      nil
-    end
-  end
-end

data/lib/data_miner/process.rb

@@ -1,38 +0,0 @@
-class DataMiner
-  class Process
-    attr_reader :config
-    attr_reader :method_id
-    attr_reader :description
-    attr_reader :blk
-
-    alias :block_description :description
-
-    def initialize(config, method_id_or_description, &blk)
-      @config = config
-      if block_given?
-        @description = method_id_or_description
-        @blk = blk
-      else
-        @description = method_id_or_description
-        @method_id = method_id_or_description
-      end
-    end
-    
-    def resource
-      config.resource
-    end
-    
-    def inspect
-      %{#<DataMiner::Process(#{resource}) #{description}>}
-    end
-    
-    def run
-      if blk
-        blk.call
-      else
-        resource.send method_id
-      end
-      nil
-    end
-  end
-end

data/lib/data_miner/tap.rb

@@ -1,143 +0,0 @@
-require 'uri'
-class DataMiner
-  # Note that you probably shouldn't put taps into your Gemfile, because it depends on sequel and other gems that may not compile on Heroku (etc.)
-  #
-  # This class automatically detects if you have Bundler installed, and if so, executes the `taps` binary with a "clean" environment (i.e. one that will not pay attention to the fact that taps is not in your Gemfile)
-  class Tap
-    attr_reader :config
-    attr_reader :description
-    attr_reader :source
-    attr_reader :options
-
-    def initialize(config, description, source, options = {})
-      @config = config
-      @options = options.dup
-      @options.stringify_keys!
-      @description = description
-      @source = source
-    end
-    
-    def resource
-      config.resource
-    end
-    
-    def inspect
-      %{#<DataMiner::Tap(#{resource}): #{description} (#{source})>}
-    end
-    
-    def run
-      [ source_table_name, resource.table_name ].each do |possible_obstacle|
-        if connection.table_exists? possible_obstacle
-          connection.drop_table possible_obstacle
-        end
-      end
-      taps_pull
-      if needs_table_rename?
-        connection.rename_table source_table_name, resource.table_name
-      end
-      nil
-    end
-    
-    # sabshere 1/25/11 what if there were multiple connections
-    # blockenspiel doesn't like to delegate this to #resource
-    def connection
-      ::ActiveRecord::Base.connection
-    end
-    
-    def db_config
-      @db_config ||= connection.instance_variable_get(:@config).stringify_keys.merge(options.except('source_table_name'))
-    end
-    
-    def source_table_name
-      options['source_table_name'] || resource.table_name
-    end
-    
-    def needs_table_rename?
-      source_table_name != resource.table_name
-    end
-    
-    def adapter
-      case connection.adapter_name
-      when /mysql2/i
-        'mysql2'
-      when /mysql/i
-        'mysql'
-      when /postgres/i
-        'postgres'
-      when /sqlite/i
-        'sqlite'
-      end
-    end
-
-    # never optional
-    def database
-      db_config['database']
-    end
-    
-    DEFAULT_PORTS = {
-      'mysql' => 3306,
-      'mysql2' => 3306,
-      'postgres' => 5432
-    }
-    
-    DEFAULT_USERNAMES = {
-      'mysql' => 'root',
-      'mysql2' => 'root',
-      'postgres' => ''
-    }
-    
-    DEFAULT_PASSWORDS = {}
-    DEFAULT_PASSWORDS.default = ''
-    
-    DEFAULT_HOSTS = {}
-    DEFAULT_HOSTS.default = 'localhost'
-
-    %w{ username password port host }.each do |x|
-      module_eval %{
-        def #{x}
-          db_config['#{x}'] || DEFAULT_#{x.upcase}S[adapter]
-        end
-      }
-    end
-    
-    # "user:pass"
-    # "user"
-    # nil
-    def userinfo
-      if username.present?
-        [username, password].select(&:present?).join(':')
-      end
-    end
-    
-    def db_url
-      case adapter
-      when 'sqlite'
-        "sqlite://#{database}"
-      else
-        ::URI::Generic.new(adapter, userinfo, host, port, nil, "/#{database}", nil, nil, nil).to_s
-      end
-    end
-    
-    def taps_pull
-      args = [
-        'taps',
-        'pull',
-        db_url,
-        source,
-        '--indexes-first',
-        '--tables',
-        source_table_name
-      ]
-      child = nil
-      
-      # https://github.com/carlhuda/bundler/issues/1579
-      if defined?(::Bundler)
-        ::Bundler.with_clean_env do
-          ::Kernel.system args.join(' ')
-        end
-      else
-        ::Kernel.system args.join(' ')
-      end
-    end
-  end
-end

data/test/support/aircraft.rb

@@ -1,102 +0,0 @@
-require 'loose_tight_dictionary'
-
-class Aircraft < ActiveRecord::Base
-  set_primary_key :icao_code
-  set_table_name 'aircraft'
-
-  def self.bts_dictionary
-    @_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv', :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }),
-                                              :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
-                                              :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
-                                              :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
-                                              :left_reader  => lambda { |record| record['Manufacturer'] + ' ' + record['Model'] },
-                                              :right_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
-  end
-
-  class BtsAircraftTypeCodeMatcher
-    def match(left_record)
-      right_record = Aircraft.bts_dictionary.left_to_right left_record
-      right_record['Aircraft Type'] if right_record
-    end
-  end
-  
-  class BtsNameMatcher
-    def match(left_record)
-      right_record = Aircraft.bts_dictionary.left_to_right left_record
-      right_record['Manufacturer'] + ' ' + right_record['Long Name'] if right_record
-    end
-  end
-  
-  class Guru
-    # for errata
-    def is_attributed_to_boeing?(row)
-      row['Manufacturer'] =~ /BOEING/i
-    end
-    
-    def is_not_attributed_to_airbus?(row)
-      row['Manufacturer'] =~ /AIRBUS/i
-    end
-    
-    def is_attributed_to_cessna?(row)
-      row['Manufacturer'] =~ /CESSNA/i
-    end
-    
-    def is_attributed_to_fokker?(row)
-      row['Manufacturer'] =~ /FOKKER/i
-    end
-    
-    def is_not_attributed_to_aerospatiale?(row)
-      not row['Manufacturer'] =~ /AEROSPATIALE/i
-    end
-    
-    def is_not_attributed_to_cessna?(row)
-      not row['Manufacturer'] =~ /CESSNA/i
-    end
-    
-    def is_not_attributed_to_learjet?(row)
-      not row['Manufacturer'] =~ /LEAR/i
-    end
-    
-    def is_not_attributed_to_dehavilland?(row)
-      not row['Manufacturer'] =~ /DE ?HAVILLAND/i
-    end
-    
-    def is_not_attributed_to_mcdonnell_douglas?(row)
-      not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
-    end
-    
-    def is_not_a_dc_plane?(row)
-      not row['Model'] =~ /DC/i
-    end
-    
-    def is_a_crj_900?(row)
-      row['Designator'].downcase == 'crj9'
-    end
-  end
-  
-  data_miner do
-    # ('A'..'Z').each do |letter|
-    # Note: for the purposes of testing, only importing "D"
-    %w{ D }.each do |letter|
-      import("ICAO codes starting with letter #{letter} used by the FAA",
-              :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
-              :encoding => 'windows-1252',
-              :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw', :responder => 'Aircraft::Guru' },
-              :row_xpath => '//table/tr[2]/td/table/tr',
-              :column_xpath => 'td') do
-        key 'icao_code', :field_name => 'Designator'
-        store 'bts_name', :matcher => Aircraft::BtsNameMatcher.new, :nullify => true
-        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher.new, :nullify => true
-        store 'manufacturer_name', :field_name => 'Manufacturer', :nullify => true
-        store 'name', :field_name => 'Model', :nullify => true
-      end
-      
-      import 'Brighter Planet aircraft class codes',
-             :url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
-        key   'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
-        store 'brighter_planet_aircraft_class_code', :nullify => true
-      end
-    end
-  end
-end
-