data_miner 0.5.7 → 1.0.0

data/lib/data_miner/config.rb

@@ -0,0 +1,184 @@
+require 'blockenspiel'
+
+class DataMiner
+  class Config
+    include ::Blockenspiel::DSL
+    
+    attr_reader :resource
+
+    def initialize(resource)
+      @resource = resource
+    end
+    
+    def steps
+      @steps ||= []
+    end
+    
+    # def attributes
+    #   @attributes ||= {}
+    # end
+    
+    def schema(create_table_options = {}, &blk)
+      step = Schema.new self, create_table_options
+      ::Blockenspiel.invoke blk, step
+      steps.push step
+    end
+    
+    def process(method_id_or_block_description, &blk)
+      step = Process.new self, method_id_or_block_description, &blk
+      steps.push step
+    end
+
+    def tap(description, source, options = {})
+      step = Tap.new self, description, source, options
+      steps.push step
+    end
+
+    def import(*args, &blk)
+      if args.length == 1
+        description = '(no description)'
+      else
+        description = args[0]
+      end
+      options = args.last
+        
+      step = Import.new self, description, options
+      ::Blockenspiel.invoke blk, step
+      steps.push step
+    end
+
+    def verify(description = '(no description)', &blk)
+      step = Verify.new self, description, &blk
+      steps.push step
+    end
+
+    # Mine data for this class.
+    def run(options = {})
+      options = options.dup
+      options.stringify_keys!
+      
+      return if ::DataMiner.instance.call_stack.include? resource.name
+      ::DataMiner.instance.call_stack.push resource.name
+      
+      finished = false
+      skipped = false
+      if Run.table_exists?
+        run = Run.create! :started_at => ::Time.now, :resource_name => resource.name, :killed => true
+      else
+        run = nil
+        ::DataMiner.logger.info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
+      end
+      resource.delete_all if options['from_scratch']
+      begin
+        steps.each do |step|
+          step.run# run
+          resource.reset_column_information
+        end
+        finished = true
+      rescue Finish
+        finished = true
+      rescue Skip
+        skipped = true
+      ensure
+        if Run.table_exists?
+          run.update_attributes! :terminated_at => ::Time.now, :finished => finished, :skipped => skipped, :killed => false
+        end
+        if ::DataMiner.instance.call_stack.first == resource.name and !options['preserve_call_stack_between_runs']
+          ::DataMiner.instance.call_stack.clear
+        end
+      end
+      nil
+    end
+    
+    def import_steps
+      steps.select { |step| step.is_a? Import }
+    end
+    
+    def before_invoke
+      
+    end
+    
+    def after_invoke
+      return unless resource.table_exists?
+      make_sure_unit_definitions_make_sense
+      suggest_missing_column_migrations
+    end
+    
+    COMPLETE_UNIT_DEFINITIONS = [
+      %w{units},
+      %w{from_units to_units},
+      %w{units_field_name},
+      %w{units_field_name to_units},
+      %w{units_field_number},
+      %w{units_field_number to_units}
+    ]
+    
+    def make_sure_unit_definitions_make_sense
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
+          if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
+            raise %{
+
+================================
+
+You don't have a valid unit definition for #{resource.name}##{attribute.name}.
+
+You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.inspect }.
+
+You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".
+
+================================
+            }
+          end
+        end
+      end
+    end
+    
+    def suggest_missing_column_migrations
+      missing_columns = []
+      
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
+          raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.end_with? '_units'
+          unless resource.column_names.include? attribute.name
+            missing_columns << attribute.name
+          end
+          if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
+            missing_columns << units_column
+          end
+        end
+      end
+      missing_columns.uniq!
+      if missing_columns.any?
+        ::DataMiner.logger.debug %{
+
+================================
+
+On #{resource}, it looks like you're missing some columns...
+
+Please run this...
+
+  ./script/generate migration AddMissingColumnsTo#{resource.name}
+
+and **replace** the resulting file with this:
+
+  class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
+    def self.up
+#{missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
+    end
+    
+    def self.down
+#{missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
+    end
+  end
+
+On the other hand, if you're working directly with create_table, this might be helpful:
+
+#{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
+
+================================
+        }
+      end
+    end
+  end
+end

data/lib/data_miner/dictionary.rb

@@ -1,20 +1,33 @@
-module DataMiner
+class DataMiner
   class Dictionary
-    attr_accessor :key_name, :value_name, :sprintf, :table
-
+    attr_reader :options
     def initialize(options = {})
-      @key_name = options[:input]
-      @value_name = options[:output]
-      @sprintf = options[:sprintf] || '%s'
-      @table = RemoteTable.new(:url => options[:url])
+      @options = options.dup
+      @options.stringify_keys!
+    end
+    
+    def key_name
+      options['input']
+    end
+    
+    def value_name
+      options['output']
+    end
+    
+    def sprintf
+      options['sprintf'] || '%s'
+    end
+    
+    def table
+      @table ||= ::RemoteTable.new options['url']
     end
 
     def lookup(key)
-      find(self.key_name, key, self.value_name, :sprintf => self.sprintf)
+      find key_name, key, value_name, 'sprintf' => sprintf
     end
     
     def find(key_name, key, value_name, options = {})
-      if match = table.rows.detect { |row| normalize_for_comparison(key, options) == normalize_for_comparison(row[key_name], options) }
+      if match = table.detect { |row| normalize_for_comparison(key, options) == normalize_for_comparison(row[key_name], options) }
         match[value_name].to_s
       end
     end
@@ -22,10 +35,10 @@ module DataMiner
     private
 
     def normalize_for_comparison(string, options = {})
-      if options[:sprintf]
-        if /\%[0-9\.]*f/.match(options[:sprintf])
+      if options['sprintf']
+        if /\%[0-9\.]*f/.match options['sprintf']
           string = string.to_f
-        elsif /\%[0-9\.]*d/.match(options[:sprintf])
+        elsif /\%[0-9\.]*d/.match options['sprintf']
           string = string.to_i
         end
         string = sprintf % string

data/lib/data_miner/import.rb

@@ -1,79 +1,88 @@
-module DataMiner
+require 'blockenspiel'
+class DataMiner
   class Import
-    include Blockenspiel::DSL
+    include ::Blockenspiel::DSL
     
     attr_reader :attributes
-    attr_accessor :base
-    attr_accessor :position_in_run
-    attr_accessor :table_options
-    attr_accessor :description
-    delegate :resource, :to => :base
+    attr_reader :config
+    attr_reader :options
+    attr_reader :description
     
-    def initialize(base, position_in_run, description, table_options = {})
-      @table_options = table_options
-      @table_options.symbolize_keys!
-
-      @attributes = ActiveSupport::OrderedHash.new
-      @base = base
-      @position_in_run = position_in_run
+    def initialize(config, description, options = {})
+      @config = config
       @description = description
-      
-      if @table_options[:errata].is_a?(String)
-        @table_options[:errata] = Errata.new :url => @table_options[:errata], :responder => resource
+      @options = options.dup
+      @options.stringify_keys!
+      # legacy
+      if @options.has_key? 'table'
+        ::DataMiner.logger.info "Warning: 'table' is no longer an allowed option, taking the url from it and ignoring the rest"
+        table_instance = @options.delete 'table'
+        @options['url'] = table_instance.url
       end
-        
-      if @table_options[:table] and @table_options[:url].present?
-        DataMiner.log_or_raise "You should specify :table or :url, but not both"
+      # legacy
+      if @options.has_key?('errata') and not @options['errata'].is_a?(::Hash)
+        ::DataMiner.logger.info "Warning: 'errata' must be a hash of Errata options. taking the URL from the Errata instance you provided and ignoring everything else"
+        errata_instance = @options.delete 'errata'
+        @options['errata'] = { 'url' => errata_instance.options['url'] }
       end
     end
-    
-    def table
-      @table ||= (table_options[:table] || RemoteTable.new(table_options))
+        
+    def attributes
+      @attributes ||= ::ActiveSupport::OrderedHash.new
     end
     
-    def clear_table
-      @table = nil
+    def resource
+      config.resource
     end
 
     def inspect
-      "Import(#{resource}) position #{position_in_run} (#{description})"
-    end
-
-    def stores?(attr_name)
-      attributes.has_key? attr_name
+      %{#<DataMiner::Import(#{resource}) (#{description})>}
     end
     
     def store(attr_name, attr_options = {})
-      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
       attributes[attr_name] = Attribute.new self, attr_name, attr_options
     end
     
     def key(attr_name, attr_options = {})
-      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
-      @key = attr_name
+      raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      @_key = attr_name
       store attr_name, attr_options
     end
 
-    def run(run)
-      primary_key = resource.primary_key
-      test_counter = 0
+    def primary_key
+      resource.primary_key
+    end
+
+    def table
+      return @table if @table.is_a? ::RemoteTable
+      # don't mess with the originals
+      options = @options.dup
+      if options['errata']
+        errata_options = options['errata'].dup
+        errata_options.stringify_keys!
+        errata_options['responder'] ||= resource
+        options['errata'] = errata_options
+      end
+      @table = ::RemoteTable.new options
+    end
 
-      table.each_row do |row|
-        if ENV['DUMP'] == 'true'
-          raise "[data_miner gem] Stopping after 5 rows because TEST=true" if test_counter > 5
-          test_counter += 1
-          DataMiner.log_info %{Row #{test_counter}
-IN:  #{row.inspect}
-OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_name] = attr.value_from_row(row); memo }.inspect}
-          }
-        end
-      
-        record = resource.send "find_or_initialize_by_#{@key}", attributes[@key].value_from_row(row)
+    def expire_remote_data
+      @table = nil
+      attributes.each { |_, attr| attr.instance_variable_set :@dictionary, nil }
+    end
+
+    def run
+      expire_remote_data
+      table.each do |row|
+        record = resource.send "find_or_initialize_by_#{@_key}", attributes[@_key].value_from_row(row)
         attributes.each { |_, attr| attr.set_record_from_row record, row }
-        record.save! if record.send(primary_key).present?
+        if record.send(primary_key).present?
+          record.save!
+        else
+          ::DataMiner.logger.debug "Skipping #{row} because there's no primary key"
+        end
       end
-      DataMiner.log_info "performed #{inspect}"
-      clear_table
       nil
     end
   end

data/lib/data_miner/process.rb

@@ -1,37 +1,42 @@
-module DataMiner
+class DataMiner
   class Process
-    attr_accessor :base, :position_in_run
-    attr_accessor :method_name
-    attr_accessor :block_description, :block
-    delegate :resource, :to => :base
+    attr_reader :config
+    attr_reader :method_id
+    attr_reader :block_description
+    attr_reader :blk
 
-    def initialize(base, position_in_run, method_name_or_block_description, &block)
-      @base = base
-      @position_in_run = position_in_run
+    def initialize(config, method_id_or_block_description, &blk)
+      @config = config
       if block_given?
-        @block_description = method_name_or_block_description
-        @block = block
+        @block_description = method_id_or_block_description
+        @blk = blk
       else
-        @method_name = method_name_or_block_description
+        @method_id = method_id_or_block_description
       end
     end
     
+    def resource
+      config.resource
+    end
+    
     def inspect
-      str = "Process(#{resource}) position #{position_in_run}"
+      str = %{<#Process(#{resource})}
       if block
-        str << " ran block (#{block_description})"
+        str << %{ called block "#{block_description}"}
       else
-        str << " called :#{method_name}"
+        str << %{ called method :#{method_id}}
       end
+      str << ">"
+      str
     end
     
-    def run(run)
-      if block
-        block.call
+    def run
+      if blk
+        blk.call
       else
-        resource.send method_name
+        resource.send method_id
       end
-      DataMiner.log_info "ran #{inspect}"
+      nil
     end
   end
 end

data/lib/data_miner/run.rb

@@ -1,5 +1,5 @@
-module DataMiner
-  class Run < ActiveRecord::Base
+class DataMiner
+  class Run < ::ActiveRecord::Base
     set_table_name 'data_miner_runs'
     
     def resource
@@ -8,7 +8,7 @@ module DataMiner
         
     class << self
       def create_tables
-        return if table_exists? and column_names.include?('skipped') # force a drop
+        return if table_exists?
         connection.create_table 'data_miner_runs', :force => true do |t|
           t.string 'resource_name'
           t.boolean 'killed'