data_miner-ruby19 0.5.4

data/lib/data_miner/base.rb

@@ -0,0 +1,194 @@
+module DataMiner
+  class Base
+    include Blockenspiel::DSL
+    
+    attr_accessor :resource, :steps, :step_counter, :attributes
+
+    def initialize(resource)
+      @steps = Array.new
+      @resource = resource
+      @step_counter = 0
+      @attributes = HashWithIndifferentAccess.new
+    end
+    
+    def schema(create_table_options = {}, &block)
+      step = DataMiner::Schema.new self, step_counter, create_table_options
+      Blockenspiel.invoke block, step
+      steps << step
+      self.step_counter += 1
+    end
+    
+    def process(method_name_or_block_description, &block)
+      steps << DataMiner::Process.new(self, step_counter, method_name_or_block_description, &block)
+      self.step_counter += 1
+    end
+
+    def tap(description, source, options = {})
+      steps << DataMiner::Tap.new(self, step_counter, description, source, options)
+      self.step_counter += 1
+    end
+
+    def import(*args, &block)
+      if args.length == 1
+        description = '(no description)'
+      else
+        description = args.first
+      end
+      options = args.last
+        
+      step = DataMiner::Import.new self, step_counter, description, options
+      Blockenspiel.invoke block, step
+      steps << step
+      self.step_counter += 1
+    end
+
+    # Mine data for this class.
+    def run(options = {})
+      options.symbolize_keys!
+      
+      return if DataMiner::Base.call_stack.include? resource.name
+      DataMiner::Base.call_stack.push resource.name
+      
+      finished = false
+      skipped = false
+      if DataMiner::Run.table_exists?
+        run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name, :killed => true
+      else
+        run = nil
+        DataMiner.log_info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
+      end
+      resource.delete_all if options[:from_scratch]
+      begin
+        steps.each do |step|
+          step.run run
+          resource.reset_column_information
+        end
+        finished = true
+      rescue DataMiner::Finish
+        finished = true
+      rescue DataMiner::Skip
+        skipped = true
+      ensure
+        if DataMiner::Run.table_exists?
+          run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
+        end
+        DataMiner::Base.call_stack.clear if DataMiner::Base.call_stack.first == resource.name and !options[:preserve_call_stack_between_runs]
+      end
+      nil
+    end
+    
+    def import_steps
+      steps.select { |step| step.is_a? Import }
+    end
+    
+    def before_invoke
+      
+    end
+    
+    def after_invoke
+      return unless resource.table_exists?
+      make_sure_unit_definitions_make_sense
+      suggest_missing_column_migrations
+    end
+    
+    COMPLETE_UNIT_DEFINITIONS = [
+      [:units],
+      [:from_units, :to_units],
+      [:units_field_name],
+      [:units_field_name, :to_units],
+      [:units_field_number],
+      [:units_field_number, :to_units]
+    ]
+    
+    def make_sure_unit_definitions_make_sense
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
+          if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
+            DataMiner.log_or_raise %{
+
+================================
+
+You don't have a valid unit definition for #{resource.name}##{attribute.name}.
+
+You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.map(&:to_sym).inspect }.
+
+You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".
+
+================================
+            }
+          end
+        end
+      end
+    end
+    
+    def suggest_missing_column_migrations
+      missing_columns = Array.new
+      
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
+          DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.end_with? '_units'
+          unless resource.column_names.include? attribute.name
+            missing_columns << attribute.name
+          end
+          if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
+            missing_columns << units_column
+          end
+        end
+      end
+      missing_columns.uniq!
+      if missing_columns.any?
+        DataMiner.log_debug %{
+
+================================
+
+On #{resource}, it looks like you're missing some columns...
+
+Please run this...
+
+  ./script/generate migration AddMissingColumnsTo#{resource.name}
+
+and **replace** the resulting file with this:
+
+  class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
+    def self.up
+#{missing_columns.map { |column_name| "      add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
+    end
+    
+    def self.down
+#{missing_columns.map { |column_name| "      remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
+    end
+  end
+
+On the other hand, if you're working directly with create_table, this might be helpful:
+
+#{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
+
+================================
+        }
+      end
+    end
+    
+    cattr_accessor :resource_names
+    self.resource_names = Array.new
+    
+    cattr_accessor :call_stack
+    self.call_stack = Array.new
+    class << self
+      # Mine data. Defaults to all resource_names touched by DataMiner.
+      #
+      # Options
+      # * <tt>:resource_names</tt>: array of resource (class) names to mine
+      def run(options = {})
+        options.symbolize_keys!
+        
+        resource_names.each do |resource_name|
+          if options[:resource_names].blank? or options[:resource_names].include?(resource_name)
+            resource_name.constantize.data_miner_base.run options
+          end
+        end
+      ensure
+        RemoteTable.cleanup
+      end
+    end
+  end
+end

data/lib/data_miner/dictionary.rb

@@ -0,0 +1,36 @@
+module DataMiner
+  class Dictionary
+    attr_accessor :key_name, :value_name, :sprintf, :table
+
+    def initialize(options = {})
+      @key_name = options[:input]
+      @value_name = options[:output]
+      @sprintf = options[:sprintf] || '%s'
+      @table = RemoteTable.new(:url => options[:url])
+    end
+
+    def lookup(key)
+      find(self.key_name, key, self.value_name, :sprintf => self.sprintf)
+    end
+    
+    def find(key_name, key, value_name, options = {})
+      if match = table.rows.detect { |row| normalize_for_comparison(key, options) == normalize_for_comparison(row[key_name], options) }
+        match[value_name].to_s
+      end
+    end
+
+    private
+
+    def normalize_for_comparison(string, options = {})
+      if options[:sprintf]
+        if /\%[0-9\.]*f/.match(options[:sprintf])
+          string = string.to_f
+        elsif /\%[0-9\.]*d/.match(options[:sprintf])
+          string = string.to_i
+        end
+        string = sprintf % string
+      end
+      string.to_s.strip
+    end
+  end
+end

data/lib/data_miner/import.rb

@@ -0,0 +1,70 @@
+module DataMiner
+  class Import
+    include Blockenspiel::DSL
+    
+    attr_reader :attributes
+    attr_accessor :base, :position_in_run, :table
+    attr_accessor :description
+    delegate :resource, :to => :base
+    
+    def initialize(base, position_in_run, description, table_options = {})
+      table_options.symbolize_keys!
+
+      @attributes = ActiveSupport::OrderedHash.new
+      @base = base
+      @position_in_run = position_in_run
+      @description = description
+      
+      if table_options[:errata].is_a?(String)
+        table_options[:errata] = Errata.new :url => table_options[:errata], :responder => resource
+      end
+        
+      if table_options[:table].present?
+        DataMiner.log_or_raise "You should specify :table or :url, but not both" if table_options[:url].present?
+        @table = table_options[:table]
+      else
+        @table = RemoteTable.new table_options
+      end
+    end
+
+    def inspect
+      "Import(#{resource}) position #{position_in_run} (#{description})"
+    end
+
+    def stores?(attr_name)
+      attributes.has_key? attr_name
+    end
+    
+    def store(attr_name, attr_options = {})
+      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      attributes[attr_name] = Attribute.new self, attr_name, attr_options
+    end
+    
+    def key(attr_name, attr_options = {})
+      DataMiner.log_or_raise "You should only call store or key once for #{resource.name}##{attr_name}" if attributes.has_key? attr_name
+      @key = attr_name
+      store attr_name, attr_options
+    end
+
+    def run(run)
+      primary_key = resource.primary_key
+      test_counter = 0
+
+      table.each_row do |row|
+        if ENV['DUMP'] == 'true'
+          raise "[data_miner gem] Stopping after 5 rows because TEST=true" if test_counter > 5
+          test_counter += 1
+          DataMiner.log_info %{Row #{test_counter}
+IN:  #{row.inspect}
+OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_name] = attr.value_from_row(row); memo }.inspect}
+          }
+        end
+      
+        record = resource.send "find_or_initialize_by_#{@key}", attributes[@key].value_from_row(row)
+        attributes.each { |_, attr| attr.set_record_from_row record, row }
+        record.save! if record.send(primary_key).present?
+      end
+      DataMiner.log_info "performed #{inspect}"
+    end
+  end
+end

data/lib/data_miner/process.rb

@@ -0,0 +1,37 @@
+module DataMiner
+  class Process
+    attr_accessor :base, :position_in_run
+    attr_accessor :method_name
+    attr_accessor :block_description, :block
+    delegate :resource, :to => :base
+
+    def initialize(base, position_in_run, method_name_or_block_description, &block)
+      @base = base
+      @position_in_run = position_in_run
+      if block_given?
+        @block_description = method_name_or_block_description
+        @block = block
+      else
+        @method_name = method_name_or_block_description
+      end
+    end
+    
+    def inspect
+      str = "Process(#{resource}) position #{position_in_run}"
+      if block
+        str << " ran block (#{block_description})"
+      else
+        str << " called :#{method_name}"
+      end
+    end
+    
+    def run(run)
+      if block
+        block.call
+      else
+        resource.send method_name
+      end
+      DataMiner.log_info "ran #{inspect}"
+    end
+  end
+end

data/lib/data_miner/run.rb

@@ -0,0 +1,26 @@
+module DataMiner
+  class Run < ActiveRecord::Base
+    set_table_name 'data_miner_runs'
+    
+    def resource
+      resource_name.constantize
+    end
+        
+    class << self
+      def create_tables
+        return if table_exists? and column_names.include?('skipped') # force a drop
+        connection.create_table 'data_miner_runs', :force => true do |t|
+          t.string 'resource_name'
+          t.boolean 'killed'
+          t.boolean 'skipped'
+          t.boolean 'finished'
+          t.datetime 'started_at'
+          t.datetime 'terminated_at'
+          t.datetime 'created_at'
+          t.datetime 'updated_at'
+        end
+        reset_column_information
+      end
+    end
+  end
+end

data/lib/data_miner/schema.rb

@@ -0,0 +1,244 @@
+module DataMiner
+  class Schema
+    include Blockenspiel::DSL
+    
+    attr_reader :base
+    attr_reader :position_in_run
+    attr_reader :create_table_options
+    delegate :resource, :to => :base
+
+    def initialize(base, position_in_run, create_table_options)
+      @base = base
+      @position_in_run = position_in_run
+      @create_table_options = create_table_options
+      @create_table_options.symbolize_keys!
+      DataMiner.log_or_raise ":id => true is not allowed in create_table_options." if @create_table_options[:id] === true
+      DataMiner.log_or_raise ":primary_key is not allowed in create_table_options. Use set_primary_key instead." if @create_table_options.has_key?(:primary_key)
+      @create_table_options[:id] = false # always
+    end
+    
+    def connection
+      ActiveRecord::Base.connection
+    end
+    
+    def table_name
+      resource.table_name
+    end
+    
+    def ideal_table
+      @ideal_table ||= ActiveRecord::ConnectionAdapters::TableDefinition.new(connection)
+    end
+    
+    def ideal_indexes
+      @ideal_indexes ||= Array.new
+    end
+    
+    def actual_indexes
+      connection.indexes table_name
+    end
+    
+    def description
+      "Define a table called #{table_name} with primary key #{ideal_primary_key_name}"
+    end
+    
+    def inspect
+      "Schema(#{resource}): #{description}"
+    end
+    
+    # lifted straight from activerecord-3.0.0.beta3/lib/active_record/connection_adapters/abstract/schema_definitions.rb
+    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
+      class_eval <<-EOV
+        def #{column_type}(*args)                                               # def string(*args)
+          options = args.extract_options!                                       #   options = args.extract_options!
+          column_names = args                                                   #   column_names = args
+                                                                                #
+          column_names.each { |name| ideal_table.column(name, '#{column_type}', options) }  #   column_names.each { |name| ideal_table.column(name, 'string', options) }
+        end                                                                     # end
+      EOV
+    end
+    def column(*args)
+      ideal_table.column(*args)
+    end
+    
+    MAX_INDEX_NAME_LENGTH = 50
+    def index(columns, options = {})
+      options.symbolize_keys!
+      columns = Array.wrap columns
+      unless name = options[:name]
+        default_name = connection.index_name(table_name, options.merge(:column => columns))
+        name = default_name.length < MAX_INDEX_NAME_LENGTH ? default_name : default_name[0..MAX_INDEX_NAME_LENGTH-11] + Zlib.crc32(default_name).to_s
+      end
+      index_unique = options.has_key?(:unique) ? options[:unique] : true
+      ideal_indexes.push ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, name, index_unique, columns)
+    end
+    
+    def ideal_primary_key_name
+      resource.primary_key.to_s
+    end
+    
+    def actual_primary_key_name
+      connection.primary_key(table_name).to_s
+    end
+        
+    INDEX_PROPERTIES = %w{ name columns }
+    def index_equivalent?(a, b)
+      return false unless a and b
+      INDEX_PROPERTIES.all? do |property|
+        DataMiner.log_debug "...comparing #{a.send(property).inspect}.to_s <-> #{b.send(property).inspect}.to_s"
+        a.send(property).to_s == b.send(property).to_s
+      end
+    end
+
+    # FIXME mysql only (assume integer primary keys)
+    def column_equivalent?(a, b)
+      return false unless a and b
+      a_type = a.type.to_s == 'primary_key' ? 'integer' : a.type.to_s
+      b_type = b.type.to_s == 'primary_key' ? 'integer' : b.type.to_s
+      a_type == b_type and a.name.to_s == b.name.to_s
+    end
+
+    %w{ column index }.each do |i|
+      eval %{
+        def #{i}_needs_to_be_placed?(name)
+          actual = actual_#{i} name
+          return true unless actual
+          ideal = ideal_#{i} name
+          not #{i}_equivalent? actual, ideal
+        end
+    
+        def #{i}_needs_to_be_removed?(name)
+          ideal_#{i}(name).nil?
+        end
+      }
+    end
+    
+    def ideal_column(name)
+      ideal_table[name.to_s]
+    end
+    
+    def actual_column(name)
+      resource.columns_hash[name.to_s]
+    end
+    
+    def ideal_index(name)
+      ideal_indexes.detect { |ideal| ideal.name == name.to_s }
+    end
+    
+    def actual_index(name)
+      actual_indexes.detect { |actual| actual.name == name.to_s }
+    end
+  
+    def place_column(name)
+      remove_column name if actual_column name
+      ideal = ideal_column name
+      DataMiner.log_debug "ADDING COLUMN #{name}"
+      connection.add_column table_name, name, ideal.type.to_sym # symbol type!
+      resource.reset_column_information
+    end
+    
+    def remove_column(name)
+      DataMiner.log_debug "REMOVING COLUMN #{name}"
+      connection.remove_column table_name, name
+      resource.reset_column_information
+    end
+    
+    def place_index(name)
+      remove_index name if actual_index name
+      ideal = ideal_index name
+      DataMiner.log_debug "ADDING INDEX #{name}"
+      connection.add_index table_name, ideal.columns, :name => ideal.name
+      resource.reset_column_information
+    end
+    
+    def remove_index(name)
+      DataMiner.log_debug "REMOVING INDEX #{name}"
+      connection.remove_index table_name, :name => name
+      resource.reset_column_information
+    end
+    
+    def run(run)
+      _add_extra_columns
+      _create_table
+      _set_primary_key
+      _remove_columns
+      _add_columns
+      _remove_indexes
+      _add_indexes
+      DataMiner.log_debug "ran #{inspect}"
+    end
+    
+    EXTRA_COLUMNS = {
+      :updated_at => :datetime,
+      :created_at => :datetime
+    }
+    def _add_extra_columns
+      EXTRA_COLUMNS.each do |extra_name, extra_type|
+        send extra_type, extra_name unless ideal_column extra_name
+      end
+    end
+    
+    def _create_table
+      if not resource.table_exists?
+        DataMiner.log_debug "CREATING TABLE #{table_name} with #{create_table_options.inspect}"
+        connection.create_table table_name, create_table_options do |t|
+          t.integer :data_miner_placeholder
+        end
+        resource.reset_column_information
+      end
+    end
+    
+    # FIXME mysql only
+    def _set_primary_key
+      if ideal_primary_key_name == 'id' and not ideal_column('id')
+        DataMiner.log_debug "no special primary key set on #{table_name}, so using 'id'"
+        column 'id', :primary_key
+      end
+      actual = actual_column actual_primary_key_name
+      ideal = ideal_column ideal_primary_key_name
+      if not column_equivalent? actual, ideal
+        DataMiner.log_debug "looks like #{table_name} has a bad (or missing) primary key"
+        if actual
+          DataMiner.log_debug "looks like primary key needs to change from #{actual_primary_key_name} to #{ideal_primary_key_name}, re-creating #{table_name} from scratch"
+          connection.drop_table table_name
+          resource.reset_column_information
+          _create_table
+        end
+        place_column ideal_primary_key_name
+        unless ideal.type.to_s == 'primary_key'
+          DataMiner.log_debug "SETTING #{ideal_primary_key_name} AS PRIMARY KEY"
+          if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
+            connection.execute "CREATE UNIQUE INDEX IDX_#{table_name}_#{ideal_primary_key_name} ON #{table_name} (#{ideal_primary_key_name} ASC)"
+          else
+            connection.execute "ALTER TABLE `#{table_name}` ADD PRIMARY KEY (`#{ideal_primary_key_name}`)"
+          end
+        end
+      end
+      resource.reset_column_information
+    end
+    
+    def _remove_columns
+      resource.columns_hash.values.each do |actual|
+        remove_column actual.name if column_needs_to_be_removed? actual.name
+      end
+    end
+    
+    def _add_columns
+      ideal_table.columns.each do |ideal|
+        place_column ideal.name if column_needs_to_be_placed? ideal.name
+      end
+    end
+    
+    def _remove_indexes
+      actual_indexes.each do |actual|
+        remove_index actual.name if index_needs_to_be_removed? actual.name
+      end
+    end
+    
+    def _add_indexes
+      ideal_indexes.each do |ideal|
+        next if ideal.name == ideal_primary_key_name # this should already have been taken care of
+        place_index ideal.name if index_needs_to_be_placed? ideal.name
+      end
+    end
+  end
+end