data_miner 0.5.7 → 1.0.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,8 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+test/test.sqlite3
+data_miner.log
+Gemfile.lock

data/CHANGELOG

@@ -0,0 +1,7 @@
+0.2.6
+* Upgrade to remote_table 0.1.6 to handle UTF-8 CSVs and long urls.
+0.3.0
+* Removed association code... now data_miner focuses on just importing.
+* New, simpler DSL
+* Upgrade to remote_table 0.2.1 for row_hashes and better blank row handling
+* Remove all association-related code

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in data_miner.gemspec
+gemspec :path => '.'

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Brighter Planet
+Copyright (c) 2011 Brighter Planet
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -0,0 +1,23 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake'
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rake/rdoctask'
+  Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'rdoc'
+    rdoc.title = 'data_miner'
+    rdoc.options << '--line-numbers' << '--inline-source'
+    rdoc.rdoc_files.include('README*')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+  end
+rescue LoadError
+   puts "Rdoc is not available"
+end

data/data_miner.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "data_miner/version"
+
+Gem::Specification.new do |s|
+  s.name        = "data_miner"
+  s.version     = DataMiner::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Seamus Abshere", "Andy Rossmeissl", "Derek Kastner"]
+  s.email       = ["seamus@abshere.net"]
+  s.homepage    = "https://github.com/seamusabshere/data_miner"
+  s.summary     = %{Mine remote data into your ActiveRecord models.}
+  s.description = %q{Mine remote data into your ActiveRecord models. You can also convert units.}
+
+  s.rubyforge_project = "data_miner"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency 'remote_table', '>=1.0.2'
+  s.add_dependency 'escape', '>=0.0.4'
+  s.add_dependency 'activerecord', '>=2.3.4'
+  s.add_dependency 'activesupport', '>=2.3.4'
+  s.add_dependency 'conversions', '>=1.4.4'
+  s.add_dependency 'blockenspiel', '>=0.3.2'
+  s.add_dependency 'taps', '>=0.3.11'
+  s.add_development_dependency 'loose_tight_dictionary', ">=0.0.5"
+  s.add_development_dependency 'errata', '>=1.0.1'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'shoulda'
+  s.add_development_dependency 'mysql'
+  s.add_development_dependency 'ruby-debug'
+end

data/lib/data_miner.rb

@@ -6,82 +6,37 @@ require 'active_support/version'
   active_support/core_ext/string/multibyte
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
-end if ActiveSupport::VERSION::MAJOR == 3
+end if ::ActiveSupport::VERSION::MAJOR == 3
 
-require 'active_record'
-require 'blockenspiel'
-require 'conversions'
-require 'errata'
-require 'remote_table'
-require 'escape'
-require 'andand'
-require 'log4r'
-require 'fileutils'
-require 'tmpdir'
-require 'zlib'
-
-require 'data_miner/attribute'
-require 'data_miner/base'
-require 'data_miner/dictionary'
-require 'data_miner/import'
-require 'data_miner/tap'
-require 'data_miner/process'
-require 'data_miner/run'
-require 'data_miner/schema'
-require 'data_miner/verify'
+require 'singleton'
 
-module DataMiner
+class DataMiner
+  include ::Singleton
+  
   class MissingHashColumn < StandardError; end
   class Finish < StandardError; end
   class Skip < StandardError; end
+  class VerificationFailed < StandardError; end
   
-  mattr_accessor :logger
-  
-  def self.start_logging
-    return if logger
-
-    if defined? Rails
-      self.logger = Rails.logger
-    else
-      class_eval { include Log4r }
-      info_outputter = FileOutputter.new 'f1', :filename => 'data_miner.log'
-      error_outputter = Outputter.stderr
-      info_outputter.only_at DEBUG, INFO
-      error_outputter.only_at WARN, ERROR, FATAL
-      
-      self.logger = Logger.new 'data_miner'
-      logger.add info_outputter, error_outputter
-      ActiveRecord::Base.logger = logger
-    end
-  end
-  
-  def self.log_or_raise(message)
-    message = "[data_miner gem] #{message}"
-    if ENV['RAILS_ENV'] == 'production' or ENV['DONT_RAISE'] == 'true'
-      logger.error message
-    else
-      raise message
-    end
-  end
-  
-  def self.log_info(message)
-    logger.info "[data_miner gem] #{message}"
-  end
-  
-  def self.log_debug(message)
-    logger.debug "[data_miner gem] #{message}"
-  end
-  
-  def self.run(options = {})
-    DataMiner::Base.run options.merge(:preserve_call_stack_between_runs => true)
-    DataMiner::Base.call_stack.clear
-  end
+  autoload :ActiveRecordExtensions, 'data_miner/active_record_extensions'
+  autoload :Attribute, 'data_miner/attribute'
+  autoload :Config, 'data_miner/config'
+  autoload :Dictionary, 'data_miner/dictionary'
+  autoload :Import, 'data_miner/import'
+  autoload :Tap, 'data_miner/tap'
+  autoload :Process, 'data_miner/process'
+  autoload :Run, 'data_miner/run'
+  autoload :Schema, 'data_miner/schema'
+  autoload :Verify, 'data_miner/verify'
   
-  def self.resource_names
-    DataMiner::Base.resource_names
+  class << self
+    delegate :logger, :to => :instance
+    delegate :logger=, :to => :instance
+    delegate :run, :to => :instance
+    delegate :resource_names, :to => :instance
   end
   
-    # TODO this should probably live somewhere else
+  # TODO this should probably live somewhere else
   def self.backtick_with_reporting(cmd)
     cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
     output = `#{cmd}`
@@ -97,41 +52,45 @@ Output:
 }
     end
   end
+  
+  attr_accessor :logger
 
-end
+  def resource_names
+    @resource_names ||= []
+  end
 
-ActiveRecord::Base.class_eval do
-  def self.x_data_miner(&block)
-    DataMiner.start_logging
-    
-    DataMiner.log_debug "Skipping data_miner block in #{self.name} because called as x_data_miner"
+  def call_stack
+    @call_stack ||= []
   end
   
-  def self.data_miner(&block)
-    DataMiner.start_logging
-    
-    DataMiner.log_debug "Database table `#{table_name}` doesn't exist. It might be created in the data_miner block, but if it's not, DataMiner probably won't work properly until you run a migration or otherwise fix the schema." unless table_exists?
-    
-    DataMiner.resource_names.push self.name unless DataMiner.resource_names.include? self.name
-
-    # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
-    class_eval do
-      cattr_accessor :data_miner_base
-      def self.data_miner_runs
-        DataMiner::Run.scoped :conditions => { :resource_name => name }
-      end
-      def self.run_data_miner!(options = {})
-        data_miner_base.run options
-      end
-      def self.execute_schema
-        schema = data_miner_base.steps.find { |s| s.instance_of?(DataMiner::Schema) }
-        schema.run(nil) if schema
+  def start_logging
+    if logger.nil?
+      if defined? ::Rails
+        @logger = ::Rails.logger
+      else
+        @logger = ::Logger.new $stdout
       end
     end
-    self.data_miner_base = DataMiner::Base.new self
-
-    Blockenspiel.invoke block, data_miner_base
+    ::ActiveRecord::Base.logger = logger
+  end
     
-    data_miner_base.after_invoke
+  # Mine data. Defaults to all resource_names touched by DataMiner.
+  #
+  # Options
+  # * <tt>:resource_names</tt>: array of resource (class) names to mine
+  def run(options = {})
+    options = options.dup
+    options.stringify_keys!
+    options['preserve_call_stack_between_runs'] = true
+    resource_names.each do |resource_name|
+      if options['resource_names'].blank? or options['resource_names'].include?(resource_name)
+        resource_name.constantize.data_miner_config.run options
+      end
+    end
+    call_stack.clear
+    # RemoteTable.cleanup
   end
 end
+
+require 'active_record'
+::ActiveRecord::Base.extend ::DataMiner::ActiveRecordExtensions

data/lib/data_miner/active_record_extensions.rb

@@ -0,0 +1,38 @@
+require 'active_record'
+require 'blockenspiel'
+
+class DataMiner
+  module ActiveRecordExtensions
+    def data_miner(&blk)
+      ::DataMiner.instance.start_logging
+
+      ::DataMiner.logger.debug "Database table `#{table_name}` doesn't exist. It might be created in the data_miner block, but if it's not, DataMiner probably won't work properly until you run a migration or otherwise fix the schema." unless table_exists?
+
+      ::DataMiner.instance.resource_names.push self.name unless ::DataMiner.instance.resource_names.include? self.name
+
+      # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
+      class_eval do
+        cattr_accessor :data_miner_config
+        def self.data_miner_runs
+          ::DataMiner::Run.scoped :conditions => { :resource_name => name }
+        end
+        def self.run_data_miner!(options = {})
+          data_miner_config.run options
+        end
+        def self.execute_schema
+          if schema = data_miner_config.steps.detect { |s| s.instance_of?(::DataMiner::Schema) }
+            schema.run
+          end
+        end
+      end
+
+      self.data_miner_config = ::DataMiner::Config.new self
+
+      ::Blockenspiel.invoke blk, data_miner_config
+
+      data_miner_config.after_invoke
+    end
+  end
+end
+
+

data/lib/data_miner/attribute.rb

@@ -1,45 +1,49 @@
-module DataMiner
+require 'conversions'
+
+class DataMiner
   class Attribute
-    attr_accessor :step
-    attr_accessor :name
-    attr_accessor :options
+    attr_reader :step
+    attr_reader :name
+    attr_reader :options
 
-    delegate :resource, :to => :step
+    def resource
+      step.resource
+    end
 
-    VALID_OPTIONS = [
-      :from_units,
-      :to_units,
-      :static,
-      :dictionary,
-      :matcher,
-      :field_name,
-      :delimiter,
-      :split,
-      :units,
-      :sprintf,
-      :nullify,
-      :overwrite,
-      :upcase,
-      :units_field_name,
-      :units_field_number,
-      :field_number,
-      :chars,
-      :synthesize
-    ]
+    VALID_OPTIONS = %w{
+      from_units
+      to_units
+      static
+      dictionary
+      matcher
+      field_name
+      delimiter
+      split
+      units
+      sprintf
+      nullify
+      overwrite
+      upcase
+      units_field_name
+      units_field_number
+      field_number
+      chars
+      synthesize
+    }
 
     def initialize(step, name, options = {})
-      options.symbolize_keys!
+      @options = options.dup
+      @options.stringify_keys!
 
       @step = step
       @name = name
       
-      invalid_option_keys = options.keys.select { |k| not VALID_OPTIONS.include? k }
-      DataMiner.log_or_raise "Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence} (#{inspect})" if invalid_option_keys.any?
-      @options = options
+      invalid_option_keys = @options.keys.select { |k| not VALID_OPTIONS.include? k }
+      raise "Invalid options: #{invalid_option_keys.map(&:inspect).to_sentence} (#{inspect})" if invalid_option_keys.any?
     end
         
     def inspect
-      "Attribute(#{resource}##{name})"
+      %{#<DataMiner::Attribute(#{resource}##{name})>}
     end
 
     def value_in_dictionary(str)
@@ -50,7 +54,7 @@ module DataMiner
       if wants_static?
         value = static
       elsif field_number
-        if field_number.is_a?(Range)
+        if field_number.is_a?(::Range)
           value = field_number.map { |n| row[n] }.join(delimiter)
         else
           value = row[field_number]
@@ -59,7 +63,7 @@ module DataMiner
         value = row[field_name]
       end
       return nil if value.nil?
-      return value if value.is_a?(ActiveRecord::Base) # escape valve for parsers that look up associations directly
+      return value if value.is_a?(::ActiveRecord::Base) # escape valve for parsers that look up associations directly
       value = value.to_s
       value = value[chars] if wants_chars?
       value = do_split(value) if wants_split?
@@ -81,7 +85,7 @@ module DataMiner
     def value_from_row(row)
       return match_row row if wants_matcher?
       value = value_in_source row
-      return value if value.is_a? ActiveRecord::Base # carry through trapdoor
+      return value if value.is_a? ::ActiveRecord::Base # carry through trapdoor
       value = value_in_dictionary value if wants_dictionary?
       value = synthesize.call(row) if wants_synthesize?
       value = nil if value.blank? and wants_nullification?
@@ -100,7 +104,7 @@ module DataMiner
       
       what_it_is = record.send name
       if what_it_is.nil? and !what_it_should_be.nil?
-        DataMiner.log_debug "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
+        ::DataMiner.logger.debug "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
         nil
       elsif what_it_is == what_it_was
         false
@@ -114,7 +118,7 @@ module DataMiner
     end
     
     def do_convert(row, value)
-      DataMiner.log_or_raise "If you use :from_units, you need to set :to_units (#{inspect})" unless wants_units?
+      raise "If you use 'from_units', you need to set 'to_units' (#{inspect})" unless wants_units?
       value.to_f.convert((from_units || unit_from_source(row)), (to_units || unit_from_source(row)))
     end
     
@@ -128,8 +132,8 @@ module DataMiner
     end
     
     def do_split(value)
-      pattern = split_options[:pattern] || /\s+/ # default is split on whitespace
-      keep = split_options[:keep] || 0           # default is keep first element
+      pattern = split_options['pattern'] || /\s+/ # default is split on whitespace
+      keep = split_options['keep'] || 0           # default is keep first element
       value.to_s.split(pattern)[keep].to_s
     end
   
@@ -148,7 +152,7 @@ module DataMiner
       upcase.present?
     end
     def wants_static?
-      options.has_key? :static
+      options.has_key? 'static'
     end
     def wants_nullification?
       nullify == true
@@ -157,7 +161,7 @@ module DataMiner
       chars.present?
     end
     def wants_synthesize?
-      synthesize.is_a?(Proc)
+      synthesize.is_a?(::Proc)
     end
     def wants_overwriting?
       overwrite != false
@@ -169,66 +173,67 @@ module DataMiner
       to_units.present? or units_field_name.present? or units_field_number.present?
     end
     def wants_dictionary?
-      options[:dictionary].present?
+      options['dictionary'].present?
     end
     def wants_matcher?
-      options[:matcher].present?
+      options['matcher'].present?
     end
 
     # Options that always have values
     def field_name
-      (options[:field_name] || name).to_s
+      (options['field_name'] || name).to_s
     end
     def delimiter
-      (options[:delimiter] || ', ')
+      (options['delimiter'] || ', ')
     end
     
     # Options that can't be referred to by their names
     def split_options
-      options[:split]
+      options['split']
     end
     
     def from_units
-      options[:from_units]
+      options['from_units']
     end
     def to_units
-      options[:to_units] || options[:units]
+      options['to_units'] || options['units']
     end
     def sprintf
-      options[:sprintf]
+      options['sprintf']
     end
     def nullify
-      options[:nullify]
+      options['nullify']
     end
     def overwrite
-      options[:overwrite]
+      options['overwrite']
     end
     def upcase
-      options[:upcase]
+      options['upcase']
     end
     def units_field_name
-      options[:units_field_name]
+      options['units_field_name']
     end
     def units_field_number
-      options[:units_field_number]
+      options['units_field_number']
     end
     def field_number
-      options[:field_number]
+      options['field_number']
     end
     def chars
-      options[:chars]
+      options['chars']
     end
     def synthesize
-      options[:synthesize]
+      options['synthesize']
     end
     def static
-      options[:static]
+      options['static']
     end
+    # must be cleared before every run! (because it relies on remote data)
     def dictionary
-      @_dictionary ||= (options[:dictionary].is_a?(Dictionary) ? options[:dictionary] : Dictionary.new(options[:dictionary]))
+      @dictionary ||= (options['dictionary'].is_a?(Dictionary) ? options['dictionary'] : Dictionary.new(options['dictionary']))
     end
     def matcher
-      @_matcher ||= (options[:matcher].is_a?(String) ? options[:matcher].constantize.new : options[:matcher])
+      @matcher ||= (options['matcher'].is_a?(::String) ? options['matcher'].constantize.new : options['matcher'])
     end
   end
 end