data_miner 1.3.8 → 2.0.1

data/lib/data_miner/dictionary.rb

@@ -1,55 +1,62 @@
 require 'remote_table'
+
 class DataMiner
   class Dictionary
-    attr_reader :options
+    DEFAULT_CASE_SENSITIVE = true
+
+    attr_reader :key_name
+    attr_reader :value_name
+    attr_reader :sprintf
+    attr_reader :url
+    attr_reader :case_sensitive
+
     def initialize(options = {})
-      @options = options.dup
-      @options.stringify_keys!
-    end
-    
-    def key_name
-      options['input']
-    end
-    
-    def value_name
-      options['output']
-    end
-    
-    def sprintf
-      options['sprintf'] || '%s'
+      options = options.symbolize_keys
+      @url = options[:url]
+      @key_name = options[:input]
+      @value_name = options[:output]
+      @sprintf = options[:sprintf]
+      @case_sensitive = options.fetch :case_sensitive, DEFAULT_CASE_SENSITIVE
+      @table_mutex = ::Mutex.new
     end
     
     def table
-      @table ||= ::RemoteTable.new(options['url']).to_a # convert to Array immediately
+      @table || @table_mutex.synchronize do
+        @table ||= ::RemoteTable.new(url).to_a # make sure it's fully cached
+      end
     end
     
-    def free
-      @table.free if @table.is_a?(::RemoteTable)
+    def refresh
       @table = nil
     end
-
+    
     def lookup(key)
-      find key_name, key, value_name, 'sprintf' => sprintf
+      find key_name, key, value_name, {:sprintf => sprintf, :case_sensitive => case_sensitive}
     end
     
     def find(key_name, key, value_name, options = {})
-      if match = table.detect { |row| normalize_for_comparison(key, options) == normalize_for_comparison(row[key_name], options) }
-        match[value_name].to_s
+      normalized_key = normalize_for_comparison(key, options)
+      if match = table.detect { |row| normalized_key == normalize_for_comparison(row[key_name.to_s], options) }
+        match[value_name.to_s].to_s
       end
     end
-
+    
     private
 
-    def normalize_for_comparison(string, options = {})
-      if options['sprintf']
-        if /\%[0-9\.]*f/.match options['sprintf']
-          string = string.to_f
-        elsif /\%[0-9\.]*d/.match options['sprintf']
-          string = string.to_i
+    def normalize_for_comparison(str, options = {})
+      if sprintf
+        if sprintf.end_with?('f')
+          str = str.to_f
+        elsif sprintf.end_with?('d')
+          str = str.to_i
         end
-        string = sprintf % string
+        str = sprintf % str
+      end
+      str = DataMiner.compress_whitespace str
+      unless options[:case_sensitive]
+        str = DataMiner.downcase str
       end
-      string.to_s.strip
+      str
     end
   end
 end

data/lib/data_miner/run.rb

@@ -1,26 +1,57 @@
+require 'aasm'
+require 'active_record_inline_schema'
+
 class DataMiner
   class Run < ::ActiveRecord::Base
-    set_table_name 'data_miner_runs'
-    
-    def resource
-      resource_name.constantize
+    class Skip < ::Exception
     end
-        
-    class << self
-      def create_tables
-        return if table_exists?
-        connection.create_table 'data_miner_runs', :force => true do |t|
-          t.string 'resource_name'
-          t.boolean 'killed'
-          t.boolean 'skipped'
-          t.boolean 'finished'
-          t.datetime 'started_at'
-          t.datetime 'terminated_at'
-          t.datetime 'created_at'
-          t.datetime 'updated_at'
+
+    INITIAL_STATE = :limbo
+
+    self.table_name = 'data_miner_runs'
+
+    col :model_name
+    col :aasm_state
+    col :created_at, :type => :datetime
+    col :stopped_at, :type => :datetime
+    col :updated_at, :type => :datetime
+    col :error, :type => :text
+
+    include ::AASM
+    aasm_initial_state INITIAL_STATE
+    aasm_state :limbo
+    aasm_state :skipped
+    aasm_state :succeeded
+    aasm_state :failed
+    aasm_event(:succeed) { transitions :from => :limbo, :to => :succeeded }
+    aasm_event(:skip)    { transitions :from => :limbo, :to => :skipped }
+    aasm_event(:fail)    { transitions :from => :limbo, :to => :failed }
+
+    validates_presence_of :model_name
+
+    def perform
+      save!
+      begin
+        catch :data_miner_succeed do
+          yield
         end
-        reset_column_information
+        succeed!
+      rescue Skip
+        skip!
+      rescue
+        self.error = "#{$!.message}\n#{$!.backtrace.join("\n")}"
+        fail!
+        raise $!
+      ensure
+        self.stopped_at = ::Time.now
+        save!
+        DataMiner.logger.info %{[data_miner] #{model_name} #{aasm_current_state.to_s.upcase} (#{(stopped_at-created_at).round(2)}s)}
       end
     end
+    lock_method :perform
+
+    def as_lock
+      [Run.connection.current_database, model_name]
+    end
   end
 end

data/lib/data_miner/script.rb

@@ -0,0 +1,116 @@
+class DataMiner
+  class Script
+    class << self
+      # @private
+      # activerecord-3.2.3/lib/active_record/scoping.rb
+      def uniq
+        previous_uniq = current_uniq
+        Script.current_uniq = true
+        begin
+          yield
+        ensure
+          Script.current_uniq = previous_uniq
+        end
+      end
+
+      def current_stack
+        ::Thread.current[STACK_THREAD_VAR] ||= []
+      end
+
+      def current_stack=(stack)
+        ::Thread.current[STACK_THREAD_VAR] = stack
+      end
+
+      def current_uniq
+        ::Thread.current[UNIQ_THREAD_VAR]
+      end
+
+      def current_uniq=(uniq)
+        ::Thread.current[UNIQ_THREAD_VAR] = uniq
+      end
+    end
+
+    UNIQ_THREAD_VAR = 'DataMiner::Script.current_uniq'
+    STACK_THREAD_VAR = 'DataMiner::Script.current_stack'
+
+    attr_reader :model
+    attr_reader :steps
+
+    def initialize(model)
+      @model = model
+      @steps = []
+    end
+
+    def append_block(blk)
+      instance_eval(&blk)
+    end
+
+    def process(method_id_or_description, &blk)
+      append(:process, method_id_or_description, &blk)
+    end
+
+    def tap(description, source, options = {})
+      append :tap, description, source, options
+    end
+
+    def import(description = nil, options = {}, &blk)
+      append(:import, description, options, &blk)
+    end
+
+    def prepend_once(*args, &blk)
+      step = make(*args, &blk)
+      unless steps.include? step
+        steps.unshift step
+      end
+    end
+
+    def prepend(*args, &blk)
+      steps.unshift make(*args, &blk)
+    end
+
+    def append_once(*args, &blk)
+      step = make(*args, &blk)
+      unless steps.include? step
+        steps << step
+      end
+    end
+
+    def append(*args, &blk)
+      steps << make(*args, &blk)
+    end
+
+    def perform
+      model_name = model.name
+      # $stderr.write "0 - #{model_name}\n"
+      # $stderr.write "A - current_uniq - #{Script.current_uniq ? 'true' : 'false'}\n"
+      # $stderr.write "B - #{Script.current_stack.join(',')}\n"
+      if Script.current_uniq and Script.current_stack.include?(model_name)
+        # we've already done this in the current stack, so skip it
+        return
+      end
+      if not Script.current_uniq
+        # since we're not trying to uniq, ignore the current contents of the stack
+        Script.current_stack.clear
+      end
+      Script.current_stack << model_name
+      Run.new(:model_name => model_name).perform do
+        steps.each do |step|
+          step.perform
+          model.reset_column_information
+        end
+      end
+    end
+        
+    private
+
+    def make(*args, &blk)
+      klass = Step.const_get(args.shift.to_s.camelcase)
+      options = args.extract_options!
+      if args.empty?
+        args = ["#{klass.name.demodulize} step with no description"]
+      end
+      initializer = [self] + args + [options]
+      klass.new(*initializer, &blk)
+    end
+  end
+end

data/lib/data_miner/step.rb

@@ -0,0 +1,5 @@
+class DataMiner::Step
+  def ==(other)
+    other.class == self.class and other.description == description
+  end
+end

data/lib/data_miner/step/import.rb

@@ -0,0 +1,74 @@
+require 'errata'
+require 'remote_table'
+
+class DataMiner::Step::Import
+  attr_reader :attributes
+  attr_reader :script
+  attr_reader :description
+  attr_reader :attributes
+  
+  def initialize(script, description, options = {}, &blk)
+    options = options.symbolize_keys
+    if options.has_key?(:table)
+      raise ::ArgumentError, %{[data_miner] :table is no longer an allowed option.}
+    end
+    if (errata_options = options[:errata]) and not errata_options.is_a?(::Hash)
+      raise ::ArgumentError, %{[data_miner] :errata must be a hash of initialization options to Errata}
+    end
+    @script = script
+    @mutex = ::Mutex.new
+    @attributes = ::ActiveSupport::OrderedHash.new
+    @description = description
+    if options.has_key? :errata
+      errata_options = options[:errata].symbolize_keys
+      errata_options[:responder] ||= model
+      options[:errata] = errata_options
+    end
+    @table_options = options.dup
+    @table_options[:streaming] = true
+    instance_eval(&blk)
+  end
+
+  def model
+    script.model
+  end
+
+  def store(attr_name, attr_options = {})
+    attr_name = attr_name.to_sym
+    if attributes.has_key? attr_name
+      raise "You should only call store or key once for #{model.name}##{attr_name}"
+    end
+    attributes[attr_name] = DataMiner::Attribute.new self, attr_name, attr_options
+  end
+  
+  def key(attr_name, attr_options = {})
+    attr_name = attr_name.to_sym
+    if attributes.has_key? attr_name
+      raise "You should only call store or key once for #{model.name}##{attr_name}"
+    end
+    @key = attr_name
+    store attr_name, attr_options
+  end
+
+  def table
+    @table || @mutex.synchronize do
+      @table ||= ::RemoteTable.new(@table_options)
+    end
+  end
+
+  def refresh
+    @table = nil
+    attributes.each { |_, attr| attr.refresh }
+    nil
+  end
+  
+  def perform
+    table.each do |row|
+      record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
+      attributes.each { |_, attr| attr.set_from_row record, row }
+      record.save!
+    end
+    refresh
+    nil
+  end
+end

data/lib/data_miner/step/process.rb

@@ -0,0 +1,34 @@
+class DataMiner::Step::Process
+  attr_reader :script
+  attr_reader :method_id
+  attr_reader :description
+  attr_reader :blk
+
+  alias :block_description :description
+
+  def initialize(script, method_id_or_description, ignored_options = {}, &blk)
+    @script = script
+    if block_given?
+      @description = method_id_or_description
+      @blk = blk
+    else
+      @description = method_id_or_description
+      @method_id = method_id_or_description
+    end
+  end
+  
+  def model
+    script.model
+  end
+  
+  def perform
+    DataMiner::Script.uniq do
+      if blk
+        model.instance_eval(&blk)
+      else
+        model.send method_id
+      end
+    end
+    nil
+  end
+end

data/lib/data_miner/step/tap.rb

@@ -0,0 +1,134 @@
+require 'uri'
+# Note that you probably shouldn't put taps into your Gemfile, because it depends on sequel and other gems that may not compile on Heroku (etc.)
+#
+# This class automatically detects if you have Bundler installed, and if so, executes the `taps` binary with a "clean" environment (i.e. one that will not pay attention to the fact that taps is not in your Gemfile)
+class DataMiner::Step::Tap
+  DEFAULT_PORTS = {
+    :mysql => 3306,
+    :mysql2 => 3306,
+    :postgres => 5432
+  }
+  
+  DEFAULT_USERNAMES = {
+    :mysql => 'root',
+    :mysql2 => 'root',
+    :postgres => ''
+  }
+  
+  DEFAULT_PASSWORDS = {}
+  DEFAULT_PASSWORDS.default = ''
+  
+  DEFAULT_HOSTS = {}
+  DEFAULT_HOSTS.default = '127.0.0.1'
+
+  attr_reader :script
+  attr_reader :description
+  attr_reader :source
+  attr_reader :database_options
+  attr_reader :source_table_name
+
+  def initialize(script, description, source, options = {})
+    options = options.symbolize_keys
+    @script = script
+    @description = description
+    @source = source
+    @database_options = options.except(:source_table_name).reverse_merge(active_record_config)
+    @source_table_name = options.fetch :source_table_name, model.table_name
+  end
+  
+  def model
+    script.model
+  end
+  
+  def perform
+    [ source_table_name, model.table_name ].each do |possible_obstacle|
+      if connection.table_exists? possible_obstacle
+        connection.drop_table possible_obstacle
+      end
+    end
+    taps_pull
+    if needs_table_rename?
+      connection.rename_table source_table_name, model.table_name
+    end
+    nil
+  end
+  
+  # sabshere 1/25/11 what if there were multiple connections
+  # blockenspiel doesn't like to delegate this to #model
+  def connection
+    ::ActiveRecord::Base.connection
+  end
+  
+  def needs_table_rename?
+    source_table_name != model.table_name
+  end
+  
+  def adapter
+    case connection.adapter_name
+    when /mysql2/i
+      'mysql2'
+    when /mysql/i
+      'mysql'
+    when /postgres/i
+      'postgres'
+    when /sqlite/i
+      'sqlite'
+    end
+  end
+
+  # never optional
+  def database
+    database_options[:database]
+  end
+  
+  %w{ username password port host }.each do |x|
+    module_eval %{
+      def #{x}
+        database_options[:#{x}] || DEFAULT_#{x.upcase}S[adapter.to_sym]
+      end
+    }
+  end
+  
+  # "user:pass"
+  # "user"
+  # nil
+  def userinfo
+    if username.present?
+      [username, password].select(&:present?).join(':')
+    end
+  end
+  
+  def db_url
+    case adapter
+    when 'sqlite'
+      "sqlite://#{database}"
+    else
+      ::URI::Generic.new(adapter, userinfo, host, port, nil, "/#{database}", nil, nil, nil).to_s
+    end
+  end
+
+  def active_record_config
+    connection.instance_variable_get(:@config).symbolize_keys
+  end
+  
+  def taps_pull
+    args = [
+      'taps',
+      'pull',
+      db_url,
+      source,
+      '--indexes-first',
+      '--tables',
+      source_table_name
+    ]
+    
+    # https://github.com/carlhuda/bundler/issues/1579
+    if defined?(::Bundler)
+      ::Bundler.with_clean_env do
+        ::Kernel.system args.join(' ')
+      end
+    else
+      ::Kernel.system args.join(' ')
+    end
+  end
+end