data_miner 2.0.1 → 2.0.2

data/lib/data_miner/dictionary.rb

@@ -1,49 +1,71 @@
 require 'remote_table'
 
 class DataMiner
+  # An easy way to translate data before importing it using an intermediate table.
   class Dictionary
     DEFAULT_CASE_SENSITIVE = true
 
+    # What field in the dictionary holds the lookup key.
+    #
+    # In other words, the column we scan down to find an entry.
+    #
+    # @return [String]
     attr_reader :key_name
+
+    # What field in the dictionary holds the final value.
+    #
+    # @return [String]
     attr_reader :value_name
+
+    # A +sprintf+-style format to be applied.
+    # @return [String]
     attr_reader :sprintf
+
+    # The URL of the dictionary. It must be a CSV.
+    # @return [String]
     attr_reader :url
+
+    # Whether to be case-sensitive with lookups. Defaults to false.
+    # @return [TrueClass, FalseClass]
     attr_reader :case_sensitive
 
+    # @private
     def initialize(options = {})
       options = options.symbolize_keys
       @url = options[:url]
-      @key_name = options[:input]
-      @value_name = options[:output]
+      @key_name = options[:input].to_s
+      @value_name = options[:output].to_s
       @sprintf = options[:sprintf]
       @case_sensitive = options.fetch :case_sensitive, DEFAULT_CASE_SENSITIVE
       @table_mutex = ::Mutex.new
     end
     
+    # Look up a translation for a value.
+    #
+    # @return [nil, String]
+    def lookup(value)
+      normalized_value = normalize_for_comparison value
+      if match = table.detect { |entry| entry[key_name] == normalized_value }
+        match[value_name].to_s
+      end
+    end
+    
+    private
+
     def table
       @table || @table_mutex.synchronize do
-        @table ||= ::RemoteTable.new(url).to_a # make sure it's fully cached
+        @table ||= ::RemoteTable.new(url).map do |entry|
+          entry[key_name] = normalize_for_comparison entry[key_name]
+          entry
+        end
       end
     end
     
     def refresh
       @table = nil
     end
-    
-    def lookup(key)
-      find key_name, key, value_name, {:sprintf => sprintf, :case_sensitive => case_sensitive}
-    end
-    
-    def find(key_name, key, value_name, options = {})
-      normalized_key = normalize_for_comparison(key, options)
-      if match = table.detect { |row| normalized_key == normalize_for_comparison(row[key_name.to_s], options) }
-        match[value_name.to_s].to_s
-      end
-    end
-    
-    private
 
-    def normalize_for_comparison(str, options = {})
+    def normalize_for_comparison(str)
       if sprintf
         if sprintf.end_with?('f')
           str = str.to_f
@@ -53,7 +75,7 @@ class DataMiner
         str = sprintf % str
       end
       str = DataMiner.compress_whitespace str
-      unless options[:case_sensitive]
+      unless case_sensitive
         str = DataMiner.downcase str
       end
       str

data/lib/data_miner/run.rb

@@ -2,7 +2,39 @@ require 'aasm'
 require 'active_record_inline_schema'
 
 class DataMiner
+  # A record of what happened when you ran a data miner script.
+  #
+  # To create the table, use +DataMiner::Run.auto_upgrade!+, possibly in +db/seeds.rb+ or a database migration.
   class Run < ::ActiveRecord::Base
+    class << self
+      # If a previous run died, you may find yourself getting +LockMethod::Locked+ exceptions.
+      # 
+      # @param [String] model_names What locks to clear.
+      #
+      # @return [nil]
+      def clear_locks(model_names = DataMiner.model_names)
+        model_names.each do |model_name|
+          dummy = new
+          dummy.model_name = model_name
+          dummy.lock_method_clear :perform
+        end
+        nil
+      end
+    end
+    # Raise this exception to skip the current run without causing it to fail.
+    #
+    # @example Avoid running certain data miner scripts too often (because they take too long).
+    #   class FlightSegment < ActiveRecord::Base
+    #     data_miner do
+    #       [...]
+    #       process "don't run this more than once an hour" do
+    #         if (last_ran_at = data_miner_runs.first(:order => 'created_at DESC').try(:created_at)) and (Time.now.utc - last_ran_at) < 3600
+    #           raise DataMiner::Run::Skip
+    #         end
+    #       end
+    #       [...]
+    #     end
+    #   end
     class Skip < ::Exception
     end
 
@@ -29,6 +61,7 @@ class DataMiner
 
     validates_presence_of :model_name
 
+    # @private
     def perform
       save!
       begin
@@ -47,9 +80,11 @@ class DataMiner
         save!
         DataMiner.logger.info %{[data_miner] #{model_name} #{aasm_current_state.to_s.upcase} (#{(stopped_at-created_at).round(2)}s)}
       end
+      self
     end
     lock_method :perform
 
+    # @private
     def as_lock
       [Run.connection.current_database, model_name]
     end

data/lib/data_miner/script.rb

@@ -1,4 +1,5 @@
 class DataMiner
+  # The container that holds each step in the script.
   class Script
     class << self
       # @private
@@ -13,18 +14,22 @@ class DataMiner
         end
       end
 
+      # @private
       def current_stack
         ::Thread.current[STACK_THREAD_VAR] ||= []
       end
 
+      # @private
       def current_stack=(stack)
         ::Thread.current[STACK_THREAD_VAR] = stack
       end
 
+      # @private
       def current_uniq
         ::Thread.current[UNIQ_THREAD_VAR]
       end
 
+      # @private
       def current_uniq=(uniq)
         ::Thread.current[UNIQ_THREAD_VAR] = uniq
       end
@@ -33,52 +38,167 @@ class DataMiner
     UNIQ_THREAD_VAR = 'DataMiner::Script.current_uniq'
     STACK_THREAD_VAR = 'DataMiner::Script.current_stack'
 
+    # @private
     attr_reader :model
+
+    # The steps in the script.
+    # @return [Array<DataMiner::Step>]
     attr_reader :steps
 
+    # @private
     def initialize(model)
       @model = model
       @steps = []
     end
 
+    # @private
     def append_block(blk)
       instance_eval(&blk)
     end
 
+    # Identify a single method or a define block of arbitrary code to be executed.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Step::Process The actual Process class.
+    #
+    # @overload process(method_id)
+    #   Run a class method on the model.
+    #   @param [Symbol] method_id The class method to be run on the model.
+    #
+    # @overload process(description, &blk)
+    #   Run a block of code.
+    #   @param [String] description A description of what the block does.
+    #   @yield [] The block to be evaluated in the context of the model (it's instance_eval'ed on the model class)
+    #
+    # @example Single class method
+    #   data_miner do
+    #     [...]
+    #     process :update_averages!
+    #     [...]
+    #   end
+    #
+    # @example Arbitrary code
+    #   data_miner do
+    #     [...]
+    #     process "do some arbitrary stuff" do
+    #       [...]
+    #     end
+    #     [...]
+    #   end
+    #
+    # @return [nil]
     def process(method_id_or_description, &blk)
       append(:process, method_id_or_description, &blk)
     end
 
+    # Use https://github.com/ricardochimal/taps to pull table structure and data.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Step::Tap The actual Tap class.
+    #
+    # @param [String] description A description of the taps source.
+    # @param [String] source The taps URL, including username, password, domain, and port.
+    # @param [optional, Hash] options
+    # @option options [String] :source_table_name (model.table_name) The source table name, if different.
+    #
+    # @note The source table name will default to the model's table name. If it's different, use the +:source_table_name+ option.
+    # @note +taps+ needs to be installed on your system and in your PATH, but it doesn't have to be in your Gemfile. Sometimes having it in your Gemfile will cause Heroku deploys (etc.) to fail because it requires +sqlite3+.
+    #
+    # @example Tapping Brighter Planet's reference data web service
+    #   data_miner do
+    #     [...]
+    #     tap "Brighter Planet's reference data", "http://carbon:neutral@data.brighterplanet.com:5000"
+    #     [...]
+    #   end
+    #
+    # @return [nil]
     def tap(description, source, options = {})
       append :tap, description, source, options
     end
 
-    def import(description = nil, options = {}, &blk)
-      append(:import, description, options, &blk)
+    # Import rows into your model.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Step::Import The actual Import class.
+    #
+    # @param [String] description A description of the data source.
+    # @param [Hash] table_and_errata_settings Settings, including URL of the data source, that are used to download/parse (using RemoteTable) and (sometimes) correct (using Errata) the data.
+    # @option table_and_errata_settings [String] :url The URL of the data source. Passed directly to +RemoteTable.new+.
+    # @option table_and_errata_settings [Hash] :errata The +:responder+ and +:url+ settings that will be passed to +Errata.new+.
+    # @option table_and_errata_settings [*] anything Any other setting will be passed to +RemoteTable.new+.
+    #
+    # @yield [] A block defining how to +key+ the import (to make it idempotent) and which columns to +store+.
+    #
+    # @note Be sure to check out https://github.com/seamusabshere/remote_table and https://github.com/seamusabshere/errata for available +table_and_errata_settings+.
+    # @note There are hundreds of +import+ examples in https://github.com/brighterplanet/earth
+    # @note We often use string primary keys to make idempotency easier. https://github.com/seamusabshere/active_record_inline_schema supports defining these inline.
+    #
+    # @example From the README
+    #   data_miner do
+    #     [...]
+    #     import("OpenGeoCode.org's Country Codes to Country Names list",
+    #            :url => 'http://opengeocode.org/download/countrynames.txt',
+    #            :format => :delimited,
+    #            :delimiter => '; ',
+    #            :headers => false,
+    #            :skip => 22) do
+    #       key   :iso_3166_code, :field_number => 0
+    #       store :iso_3166_alpha_3_code, :field_number => 1
+    #       store :iso_3166_numeric_code, :field_number => 2
+    #       store :name, :field_number => 5
+    #     end
+    #     [...]
+    #   end
+    #
+    # @return [nil]
+    def import(description, table_and_errata_settings, &blk)
+      append(:import, description, table_and_errata_settings, &blk)
     end
 
+    # Prepend a step to a script unless it's already there. Mostly for internal use.
+    #
+    # @return [nil]
     def prepend_once(*args, &blk)
       step = make(*args, &blk)
       unless steps.include? step
         steps.unshift step
       end
+      nil
     end
 
+    # Prepend a step to a script. Mostly for internal use.
+    #
+    # @return [nil]
     def prepend(*args, &blk)
       steps.unshift make(*args, &blk)
+      nil
     end
 
+    # Append a step to a script unless it's already there. Mostly for internal use.
+    #
+    # @return [nil]
     def append_once(*args, &blk)
       step = make(*args, &blk)
       unless steps.include? step
         steps << step
       end
+      nil
     end
 
+    # Append a step to a script. Mostly for internal use.
+    #
+    # @return [nil]
     def append(*args, &blk)
       steps << make(*args, &blk)
+      nil
     end
 
+    # Run the script for this model. Mostly for internal use.
+    #
+    # @note Normally you should use +Country.run_data_miner!+
+    # @note A primitive "call stack" is kept that will prevent infinite loops. So, if Country's data miner script calls Province's AND vice-versa, each one will only be run once.
+    #
+    # @return [DataMiner::Run]
     def perform
       model_name = model.name
       # $stderr.write "0 - #{model_name}\n"
@@ -103,6 +223,7 @@ class DataMiner
         
     private
 
+    # return [DataMiner::Step]
     def make(*args, &blk)
       klass = Step.const_get(args.shift.to_s.camelcase)
       options = args.extract_options!

data/lib/data_miner/step.rb

@@ -1,5 +1,13 @@
-class DataMiner::Step
-  def ==(other)
-    other.class == self.class and other.description == description
+class DataMiner
+  class Step
+    # @private
+    def ==(other)
+      other.class == self.class and other.description == description
+    end
+
+    # @private
+    def model
+      script.model
+    end
   end
 end

data/lib/data_miner/step/import.rb

@@ -1,74 +1,110 @@
 require 'errata'
 require 'remote_table'
 
-class DataMiner::Step::Import
-  attr_reader :attributes
-  attr_reader :script
-  attr_reader :description
-  attr_reader :attributes
-  
-  def initialize(script, description, options = {}, &blk)
-    options = options.symbolize_keys
-    if options.has_key?(:table)
-      raise ::ArgumentError, %{[data_miner] :table is no longer an allowed option.}
-    end
-    if (errata_options = options[:errata]) and not errata_options.is_a?(::Hash)
-      raise ::ArgumentError, %{[data_miner] :errata must be a hash of initialization options to Errata}
-    end
-    @script = script
-    @mutex = ::Mutex.new
-    @attributes = ::ActiveSupport::OrderedHash.new
-    @description = description
-    if options.has_key? :errata
-      errata_options = options[:errata].symbolize_keys
-      errata_options[:responder] ||= model
-      options[:errata] = errata_options
-    end
-    @table_options = options.dup
-    @table_options[:streaming] = true
-    instance_eval(&blk)
-  end
+class DataMiner
+  class Step
+    # A step that imports data from a remote source.
+    #
+    # Create these by calling +import+ inside a +data_miner+ block.
+    #
+    # @see DataMiner::ActiveRecordClassMethods#data_miner Overview of how to define data miner scripts inside of ActiveRecord models.
+    # @see DataMiner::Script#import
+    class Import < Step
+      # The mappings of local columns to remote data source fields.
+      # @return [Array<DataMiner::Attribute>]
+      attr_reader :attributes
 
-  def model
-    script.model
-  end
+      # @private
+      attr_reader :script
 
-  def store(attr_name, attr_options = {})
-    attr_name = attr_name.to_sym
-    if attributes.has_key? attr_name
-      raise "You should only call store or key once for #{model.name}##{attr_name}"
-    end
-    attributes[attr_name] = DataMiner::Attribute.new self, attr_name, attr_options
-  end
-  
-  def key(attr_name, attr_options = {})
-    attr_name = attr_name.to_sym
-    if attributes.has_key? attr_name
-      raise "You should only call store or key once for #{model.name}##{attr_name}"
-    end
-    @key = attr_name
-    store attr_name, attr_options
-  end
+      # Description of what this step does.
+      # @return [String]
+      attr_reader :description
+      
+      # @private
+      def initialize(script, description, table_and_errata_settings, &blk)
+        table_and_errata_settings = table_and_errata_settings.symbolize_keys
+        if table_and_errata_settings.has_key?(:table)
+          raise ::ArgumentError, %{[data_miner] :table is no longer an allowed setting.}
+        end
+        if (errata_settings = table_and_errata_settings[:errata]) and not errata_settings.is_a?(::Hash)
+          raise ::ArgumentError, %{[data_miner] :errata must be a hash of initialization settings to Errata}
+        end
+        @script = script
+        @attributes = ::ActiveSupport::OrderedHash.new
+        @description = description
+        if table_and_errata_settings.has_key? :errata
+          errata_settings = table_and_errata_settings[:errata].symbolize_keys
+          errata_settings[:responder] ||= model
+          table_and_errata_settings[:errata] = errata_settings
+        end
+        @table_settings = table_and_errata_settings.dup
+        @table_settings[:streaming] = true
+        @table_mutex = ::Mutex.new
+        instance_eval(&blk)
+      end
 
-  def table
-    @table || @mutex.synchronize do
-      @table ||= ::RemoteTable.new(@table_options)
-    end
-  end
+      # Store data into a model column.
+      #
+      # @see DataMiner::Attribute The actual Attribute class.
+      #
+      # @param [Symbol] attr_name The name of the local model column.
+      # @param [optional, Hash] attr_options Options that will be passed to +DataMiner::Attribute.new+
+      # @option attr_options [*] anything Any option for +DataMiner::Attribute+.
+      #
+      # @return [nil]
+      def store(attr_name, attr_options = {})
+        attr_name = attr_name.to_sym
+        if attributes.has_key? attr_name
+          raise "You should only call store or key once for #{model.name}##{attr_name}"
+        end
+        attributes[attr_name] = DataMiner::Attribute.new self, attr_name, attr_options
+      end
 
-  def refresh
-    @table = nil
-    attributes.each { |_, attr| attr.refresh }
-    nil
-  end
-  
-  def perform
-    table.each do |row|
-      record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
-      attributes.each { |_, attr| attr.set_from_row record, row }
-      record.save!
+      # Store data into a model column AND use it as the key.
+      #
+      # @see DataMiner::Attribute The actual Attribute class.
+      #
+      # Enables idempotency. In other words, you can run the data miner script multiple times, get updated data, and not get duplicate rows.
+      #
+      # @param [Symbol] attr_name The name of the local model column.
+      # @param [optional, Hash] attr_options Options that will be passed to +DataMiner::Attribute.new+
+      # @option attr_options [*] anything Any option for +DataMiner::Attribute+.
+      #
+      # @return [nil]
+      def key(attr_name, attr_options = {})
+        attr_name = attr_name.to_sym
+        if attributes.has_key? attr_name
+          raise "You should only call store or key once for #{model.name}##{attr_name}"
+        end
+        @key = attr_name
+        store attr_name, attr_options
+      end
+
+      # @private
+      def perform
+        table.each do |row|
+          record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
+          attributes.each { |_, attr| attr.set_from_row record, row }
+          record.save!
+        end
+        refresh
+        nil
+      end
+
+      private
+
+      def table
+        @table || @table_mutex.synchronize do
+          @table ||= ::RemoteTable.new(@table_settings)
+        end
+      end
+
+      def refresh
+        @table = nil
+        attributes.each { |_, attr| attr.refresh }
+        nil
+      end
     end
-    refresh
-    nil
   end
 end