freelancing-god-thinking-sphinx 0.9.13 → 1.1.0

data/lib/thinking_sphinx.rb

@@ -15,6 +15,7 @@ require 'thinking_sphinx/field'
 require 'thinking_sphinx/index'
 require 'thinking_sphinx/rails_additions'
 require 'thinking_sphinx/search'
+require 'thinking_sphinx/deltas'
 
 require 'thinking_sphinx/adapters/abstract_adapter'
 require 'thinking_sphinx/adapters/mysql_adapter'
@@ -28,9 +29,9 @@ Merb::Plugins.add_rakefiles(
 
 module ThinkingSphinx
   module Version #:nodoc:
-    Major = 0
-    Minor = 9
-    Tiny  = 13
+    Major = 1
+    Minor = 1
+    Tiny  = 0
     
     String = [Major, Minor, Tiny].join('.')
   end

data/lib/thinking_sphinx/active_record.rb

@@ -234,7 +234,7 @@ module ThinkingSphinx
         {self.sphinx_document_id => 1}
       ) if ThinkingSphinx.deltas_enabled? &&
         self.class.sphinx_indexes.any? { |index| index.delta? } &&
-        self.delta?
+        self.delta
     rescue ::ThinkingSphinx::ConnectionError
       # nothing
     end

data/lib/thinking_sphinx/active_record/delta.rb

@@ -42,22 +42,7 @@ module ThinkingSphinx
             # if running in the test environment.
             #
             def index_delta(instance = nil)
-              return true unless ThinkingSphinx.updates_enabled? &&
-                ThinkingSphinx.deltas_enabled?
-              
-              config = ThinkingSphinx::Configuration.instance
-              client = Riddle::Client.new config.address, config.port
-              
-              client.update(
-                "#{self.sphinx_indexes.first.name}_core",
-                ['sphinx_deleted'],
-                {instance.sphinx_document_id => 1}
-              ) if instance && instance.in_core_index?
-              
-              output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{self.sphinx_indexes.first.name}_delta`
-              puts output unless ThinkingSphinx.suppress_delta_output?
-
-              true
+              self.sphinx_indexes.first.delta_object.index(self, instance)
             end
           end
           
@@ -65,7 +50,7 @@ module ThinkingSphinx
           
           # Set the delta value for the model to be true.
           def toggle_delta
-            self.delta = true
+            self.class.sphinx_indexes.first.delta_object.toggle(self)
           end
           
           # Build the delta index for the related model. This won't be called

data/lib/thinking_sphinx/deltas.rb

@@ -0,0 +1,19 @@
+require 'thinking_sphinx/deltas/default_delta'
+require 'thinking_sphinx/deltas/delayed_delta'
+
+module ThinkingSphinx
+  module Deltas
+    def self.parse(index, options)
+      case options.delete(:delta)
+      when TrueClass, :default
+        DefaultDelta.new index, options
+      when :delayed
+        DelayedDelta.new index, options
+      when FalseClass, nil
+        nil
+      else
+        raise "Unknown delta type"
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/default_delta.rb

@@ -0,0 +1,50 @@
+module ThinkingSphinx
+  module Deltas
+    class DefaultDelta
+      attr_accessor :column
+      
+      def initialize(index, options)
+        @index  = index
+        @column = options.delete(:column) || :delta
+      end
+      
+      def index(model, instance = nil)
+        return true unless ThinkingSphinx.updates_enabled? &&
+          ThinkingSphinx.deltas_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        client.update(
+          core_index_name(model),
+          ['sphinx_deleted'],
+          {instance.sphinx_document_id => [1]}
+        ) if instance && ThinkingSphinx.sphinx_running? && instance.in_core_index?
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{delta_index_name model}`
+        puts output unless ThinkingSphinx.suppress_delta_output?
+        
+        true
+      end
+      
+      def toggle(instance)
+        instance.delta = true
+      end
+      
+      def clause(model, toggled)
+        "#{model.quoted_table_name}.#{@index.quote_column(@column.to_s)}" +
+        " = #{@index.db_boolean(toggled)}"
+      end
+      
+      protected
+      
+      def core_index_name(model)
+        "#{model.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_core"
+      end
+      
+      def delta_index_name(model)
+        "#{model.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_delta"
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta.rb

@@ -0,0 +1,25 @@
+require 'delayed/job'
+
+require 'thinking_sphinx/deltas/delayed_delta/delta_job'
+require 'thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job'
+require 'thinking_sphinx/deltas/delayed_delta/job'
+
+module ThinkingSphinx
+  module Deltas
+    class DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
+      def index(model, instance = nil)
+        ThinkingSphinx::Deltas::Job.enqueue(
+          ThinkingSphinx::Deltas::DeltaJob.new(delta_index_name(model))
+        )
+        
+        Delayed::Job.enqueue(
+          ThinkingSphinx::Deltas::FlagAsDeletedJob.new(
+            core_index_name(model), instance.sphinx_document_id
+          )
+        ) if instance
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb

@@ -0,0 +1,24 @@
+module ThinkingSphinx
+  module Deltas
+    class DeltaJob
+      attr_accessor :index
+      
+      def initialize(index)
+        @index = index
+      end
+      
+      def perform
+        return true unless ThinkingSphinx.updates_enabled? &&
+          ThinkingSphinx.deltas_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{index}`
+        puts output unless ThinkingSphinx.suppress_delta_output?
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb

@@ -0,0 +1,27 @@
+module ThinkingSphinx
+  module Deltas
+    class FlagAsDeletedJob
+      attr_accessor :index, :document_id
+      
+      def initialize(index, document_id)
+        @index, @document_id = index, document_id
+      end
+      
+      def perform
+        return true unless ThinkingSphinx.updates_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        client.update(
+          @index,
+          ['sphinx_deleted'],
+          {@document_id => [1]}
+        ) if ThinkingSphinx.sphinx_running? &&
+          ThinkingSphinx::Search.search_for_id(@document_id, @index)
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/job.rb

@@ -0,0 +1,24 @@
+module ThinkingSphinx
+  module Deltas
+    class Job < Delayed::Job
+      def self.enqueue(object, priority = 0)
+        super unless duplicates_exist(object)
+      end
+      
+      def self.cancel_thinking_sphinx_jobs
+        delete_all("handler LIKE '--- !ruby/object:ThinkingSphinx::Deltas::%'")
+      end
+
+      private
+
+      def self.duplicates_exist(object)
+        count(
+          :conditions => {
+            :handler    => object.to_yaml,
+            :locked_at  => nil
+          }
+        ) > 0
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/index.rb

@@ -10,7 +10,7 @@ module ThinkingSphinx
   # 
   class Index
     attr_accessor :model, :fields, :attributes, :conditions, :groupings,
-      :delta, :options
+      :delta_object, :options
     
     # Create a new index instance by passing in the model it is tied to, and
     # a block to build it with (optional but recommended). For documentation
@@ -34,7 +34,7 @@ module ThinkingSphinx
       @conditions   = []
       @groupings    = []
       @options      = {}
-      @delta        = false
+      @delta_object = nil
       
       initialize_from_builder(&block) if block_given?
     end
@@ -127,7 +127,7 @@ module ThinkingSphinx
       
       where_clause = ""
       if self.delta?
-        where_clause << " AND #{@model.quoted_table_name}.#{quote_column('delta')}" +" = #{options[:delta] ? db_boolean(true) : db_boolean(false)}"
+        where_clause << " AND #{@delta_object.clause(@model, options[:delta])}"
       end
       unless @conditions.empty?
         where_clause << " AND " << @conditions.join(" AND ")
@@ -190,23 +190,14 @@ GROUP BY #{ (
       
       sql = "SELECT #{min_statement}, #{max_statement} " +
             "FROM #{@model.quoted_table_name} "
-      sql << "WHERE #{@model.quoted_table_name}.#{quote_column('delta')} " + 
-            "= #{options[:delta] ? db_boolean(true) : db_boolean(false)}" if self.delta?
+      sql << "WHERE #{@delta_object.clause(@model, options[:delta])}" if self.delta?
       sql
     end
     
-    # Returns the SQL query to run before a full index - ie: nothing unless the
-    # index has a delta, and then it's an update statement to set delta values
-    # back to 0.
-    #
-    def to_sql_query_pre
-      self.delta? ? "UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}" : ""
-    end
-    
     # Flag to indicate whether this index has a corresponding delta index.
     #
     def delta?
-      @delta
+      !@delta_object.nil?
     end
     
     def adapter
@@ -244,16 +235,27 @@ GROUP BY #{ (
       all_source_options
     end
     
+    def quote_column(column)
+      @model.connection.quote_column_name(column)
+    end
+    
+    # Returns the proper boolean value string literal for the
+    # current database adapter.
+    #
+    def db_boolean(val)
+      if adapter == :postgres
+        val ? 'TRUE' : 'FALSE'
+      else
+        val ? '1' : '0'
+      end
+    end
+    
     private
     
     def utf8?
       self.index_options[:charset_type] == "utf-8"
     end
     
-    def quote_column(column)
-      @model.connection.quote_column_name(column)
-    end
-    
     # Does all the magic with the block provided to the base #initialize.
     # Creates a new class subclassed from Builder, and evaluates the block
     # on it, then pulls all relevant settings - fields, attributes, conditions,
@@ -272,12 +274,12 @@ GROUP BY #{ (
         builder.where("#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'")
       end
 
-      @fields     = builder.fields
-      @attributes = builder.attributes
-      @conditions = builder.conditions
-      @groupings  = builder.groupings
-      @delta      = builder.properties[:delta]
-      @options    = builder.properties.except(:delta)
+      @fields       = builder.fields
+      @attributes   = builder.attributes
+      @conditions   = builder.conditions
+      @groupings    = builder.groupings
+      @delta_object = ThinkingSphinx::Deltas.parse self, builder.properties
+      @options      = builder.properties
       
       # We want to make sure that if the database doesn't exist, then Thinking
       # Sphinx doesn't mind when running non-TS tasks (like db:create, db:drop
@@ -338,17 +340,6 @@ GROUP BY #{ (
       @associations[key] ||= Association.children(@model, key)
     end
 
-    # Returns the proper boolean value string literal for the
-    # current database adapter.
-    #
-    def db_boolean(val)
-      if adapter == :postgres
-        val ? 'TRUE' : 'FALSE'
-      else
-        val ? '1' : '0'
-      end
-    end
-    
     def crc_column
       if @model.column_names.include?(@model.inheritance_column)
         case adapter
@@ -423,7 +414,7 @@ GROUP BY #{ (
       source.sql_query_range  = to_sql_query_range(:delta => delta)
       source.sql_query_info   = to_sql_query_info(offset)
       
-      source.sql_query_pre += send(delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
+      source.sql_query_pre += send(!delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
       
       if @options[:group_concat_max_len]
         source.sql_query_pre << "SET SESSION group_concat_max_len = #{@options[:group_concat_max_len]}"
@@ -445,8 +436,11 @@ GROUP BY #{ (
     end
     
     def sql_query_pre_for_core
-      delta? ? ["UPDATE #{@model.quoted_table_name} SET #{quote_column('delta')} = #{db_boolean(false)}"] : []
-      []
+      if self.delta?
+        ["UPDATE #{@model.quoted_table_name} SET #{@delta_object.clause(@model, false)}"]
+      else
+        []
+      end
     end
     
     def sql_query_pre_for_delta

data/lib/thinking_sphinx/rails_additions.rb

@@ -65,4 +65,57 @@ end
 
 ActiveRecord::Base.extend(
   ThinkingSphinx::ActiveRecordStoreFullSTIClass
-) unless ActiveRecord::Base.respond_to?(:store_full_sti_class)
+) unless ActiveRecord::Base.respond_to?(:store_full_sti_class)
+
+module ThinkingSphinx
+  module ClassAttributeMethods
+    def cattr_reader(*syms)
+      syms.flatten.each do |sym|
+        next if sym.is_a?(Hash)
+        class_eval(<<-EOS, __FILE__, __LINE__)
+          unless defined? @@#{sym}
+            @@#{sym} = nil
+          end
+
+          def self.#{sym}
+            @@#{sym}
+          end
+
+          def #{sym}
+            @@#{sym}
+          end
+        EOS
+      end
+    end
+
+    def cattr_writer(*syms)
+      options = syms.extract_options!
+      syms.flatten.each do |sym|
+        class_eval(<<-EOS, __FILE__, __LINE__)
+          unless defined? @@#{sym}
+            @@#{sym} = nil
+          end
+
+          def self.#{sym}=(obj)
+            @@#{sym} = obj
+          end
+
+          #{"
+          def #{sym}=(obj)
+            @@#{sym} = obj
+          end
+          " unless options[:instance_writer] == false }
+        EOS
+      end
+    end
+
+    def cattr_accessor(*syms)
+      cattr_reader(*syms)
+      cattr_writer(*syms)
+    end
+  end
+end
+
+Class.extend(
+  ThinkingSphinx::ClassAttributeMethods
+) unless Class.respond_to?(:cattr_reader)

data/spec/unit/thinking_sphinx/active_record/delta_spec.rb

@@ -22,7 +22,7 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do
   describe "suspended_delta method" do
     before :each do
       ThinkingSphinx.stub_method(:deltas_enabled? => true)
-      Person.stub_method(:` => "")
+      Person.sphinx_indexes.first.delta_object.stub_method(:` => "")
     end
 
     it "should execute the argument block with deltas disabled" do
@@ -71,8 +71,8 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do
   describe "index_delta method" do
     before :each do
       ThinkingSphinx::Configuration.stub_method(:environment => "spec")
-      ThinkingSphinx.stub_method(:deltas_enabled? => true)
-      Person.stub_method(:` => "")
+      ThinkingSphinx.stub_method(:deltas_enabled? => true, :sphinx_running? => true)
+      Person.sphinx_indexes.first.delta_object.stub_method(:` => "")
       
       @person = Person.new
       @person.stub_method(
@@ -89,7 +89,7 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do
       
       @person.send(:index_delta)
       
-      Person.should_not have_received(:`)
+      Person.sphinx_indexes.first.delta_object.should_not have_received(:`)
       @client.should_not have_received(:update)
     end
     
@@ -98,7 +98,7 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do
       
       @person.send(:index_delta)
       
-      Person.should_not have_received(:`)
+      Person.sphinx_indexes.first.delta_object.should_not have_received(:`)
     end
     
     it "shouldn't index if the environment is 'test'" do
@@ -108,13 +108,13 @@ describe "ThinkingSphinx::ActiveRecord::Delta" do
       
       @person.send(:index_delta)
       
-      Person.should_not have_received(:`)
+      Person.sphinx_indexes.first.delta_object.should_not have_received(:`)
     end
     
     it "should call indexer for the delta index" do
       @person.send(:index_delta)
       
-      Person.should have_received(:`).with(
+      Person.sphinx_indexes.first.delta_object.should have_received(:`).with(
         "#{ThinkingSphinx::Configuration.instance.bin_path}indexer --config #{ThinkingSphinx::Configuration.instance.config_file} --rotate person_delta"
       )
     end

data/tasks/thinking_sphinx_tasks.rb

@@ -54,6 +54,8 @@ namespace :thinking_sphinx do
   
   desc "Index data for Sphinx using Thinking Sphinx's settings"
   task :index => :app_env do
+    ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
+    
     config = ThinkingSphinx::Configuration.instance
     unless ENV["INDEX_ONLY"] == "true"
       puts "Generating Configuration to #{config.config_file}"
@@ -66,6 +68,16 @@ namespace :thinking_sphinx do
     puts cmd
     system cmd
   end
+  
+  desc "Process stored delta index requests"
+  task :delta => :app_env do
+    require 'delayed/worker'
+    
+    Delayed::Worker.new(
+      :min_priority => ENV['MIN_PRIORITY'],
+      :max_priority => ENV['MAX_PRIORITY']
+    ).start
+  end
 end
 
 namespace :ts do
@@ -85,6 +97,8 @@ namespace :ts do
   task :conf    => "thinking_sphinx:configure"
   desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
   task :config  => "thinking_sphinx:configure"
+  desc "Process stored delta index requests"
+  task :delta   => "thinking_sphinx:delta"
 end
 
 def sphinx_pid

data/vendor/delayed_job/lib/delayed/job.rb

@@ -0,0 +1,251 @@
+module Delayed
+
+  class DeserializationError < StandardError
+  end
+
+  class Job < ActiveRecord::Base
+    MAX_ATTEMPTS = 25
+    MAX_RUN_TIME = 4.hours
+    set_table_name :delayed_jobs
+
+    # By default failed jobs are destroyed after too many attempts.
+    # If you want to keep them around (perhaps to inspect the reason
+    # for the failure), set this to false.
+    cattr_accessor :destroy_failed_jobs
+    self.destroy_failed_jobs = true
+
+    # Every worker has a unique name which by default is the pid of the process.
+    # There are some advantages to overriding this with something which survives worker retarts:
+    # Workers can safely resume working on tasks which are locked by themselves. The worker will assume that it crashed before.
+    cattr_accessor :worker_name
+    self.worker_name = "host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
+
+    NextTaskSQL         = '(run_at <= ? AND (locked_at IS NULL OR locked_at < ?) OR (locked_by = ?)) AND failed_at IS NULL'
+    NextTaskOrder       = 'priority DESC, run_at ASC'
+
+    ParseObjectFromYaml = /\!ruby\/\w+\:([^\s]+)/
+
+    cattr_accessor :min_priority, :max_priority
+    self.min_priority = nil
+    self.max_priority = nil
+
+    class LockError < StandardError
+    end
+
+    def self.clear_locks!
+      update_all("locked_by = null, locked_at = null", ["locked_by = ?", worker_name])
+    end
+
+    def failed?
+      failed_at
+    end
+    alias_method :failed, :failed?
+
+    def payload_object
+      @payload_object ||= deserialize(self['handler'])
+    end
+
+    def name
+      @name ||= begin
+        payload = payload_object
+        if payload.respond_to?(:display_name)
+          payload.display_name
+        else
+          payload.class.name
+        end
+      end
+    end
+
+    def payload_object=(object)
+      self['handler'] = object.to_yaml
+    end
+
+    def reschedule(message, backtrace = [], time = nil)
+      if self.attempts < MAX_ATTEMPTS
+        time ||= Job.db_time_now + (attempts ** 4) + 5
+
+        self.attempts    += 1
+        self.run_at       = time
+        self.last_error   = message + "\n" + backtrace.join("\n")
+        self.unlock
+        save!
+      else
+        logger.info "* [JOB] PERMANENTLY removing #{self.name} because of #{attempts} consequetive failures."
+        destroy_failed_jobs ? destroy : update_attribute(:failed_at, Time.now)
+      end
+    end
+
+    def self.enqueue(*args, &block)
+      object = block_given? ? EvaledJob.new(&block) : args.shift
+
+      unless object.respond_to?(:perform) || block_given?
+        raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
+      end
+    
+      priority = args[0] || 0
+      run_at   = args[1]
+
+      Job.create(:payload_object => object, :priority => priority.to_i, :run_at => run_at)
+    end
+
+    def self.find_available(limit = 5, max_run_time = MAX_RUN_TIME)
+
+      time_now = db_time_now
+
+      sql = NextTaskSQL.dup
+
+      conditions = [time_now, time_now - max_run_time, worker_name]
+
+      if self.min_priority
+        sql << ' AND (priority >= ?)'
+        conditions << min_priority
+      end
+
+      if self.max_priority
+        sql << ' AND (priority <= ?)'
+        conditions << max_priority
+      end
+
+      conditions.unshift(sql)
+
+      records = ActiveRecord::Base.silence do
+        find(:all, :conditions => conditions, :order => NextTaskOrder, :limit => limit)
+      end
+
+      records.sort_by { rand() }
+    end
+
+    # Get the payload of the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def self.reserve(max_run_time = MAX_RUN_TIME, &block)
+
+      # We get up to 5 jobs from the db. In face we cannot get exclusive access to a job we try the next.
+      # this leads to a more even distribution of jobs across the worker processes
+      find_available(5, max_run_time).each do |job|
+        begin
+          logger.info "* [JOB] aquiring lock on #{job.name}"
+          job.lock_exclusively!(max_run_time, worker_name)
+          runtime =  Benchmark.realtime do
+            invoke_job(job.payload_object, &block)
+            job.destroy
+          end
+          logger.info "* [JOB] #{job.name} completed after %.4f" % runtime
+
+          return job
+        rescue LockError
+          # We did not get the lock, some other worker process must have
+          logger.warn "* [JOB] failed to aquire exclusive lock for #{job.name}"
+        rescue StandardError => e
+          job.reschedule e.message, e.backtrace
+          log_exception(job, e)
+          return job
+        end
+      end
+
+      nil
+    end
+
+    # This method is used internally by reserve method to ensure exclusive access
+    # to the given job. It will rise a LockError if it cannot get this lock.
+    def lock_exclusively!(max_run_time, worker = worker_name)
+      now = self.class.db_time_now
+      affected_rows = if locked_by != worker
+        # We don't own this job so we will update the locked_by name and the locked_at
+        self.class.update_all(["locked_at = ?, locked_by = ?", now, worker], ["id = ? and (locked_at is null or locked_at < ?)", id, (now - max_run_time.to_i)])
+      else
+        # We already own this job, this may happen if the job queue crashes.
+        # Simply resume and update the locked_at
+        self.class.update_all(["locked_at = ?", now], ["id = ? and locked_by = ?", id, worker])
+      end
+      raise LockError.new("Attempted to aquire exclusive lock failed") unless affected_rows == 1
+
+      self.locked_at    = now
+      self.locked_by    = worker
+    end
+
+    def unlock
+      self.locked_at    = nil
+      self.locked_by    = nil
+    end
+
+    # This is a good hook if you need to report job processing errors in additional or different ways
+    def self.log_exception(job, error)
+      logger.error "* [JOB] #{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts"
+      logger.error(error)
+    end
+
+    def self.work_off(num = 100)
+      success, failure = 0, 0
+
+      num.times do
+        job = self.reserve do |j|
+          begin
+            j.perform
+            success += 1
+          rescue
+            failure += 1
+            raise
+          end
+        end
+
+        break if job.nil?
+      end
+
+      return [success, failure]
+    end
+
+    # Moved into its own method so that new_relic can trace it.
+    def self.invoke_job(job, &block)
+      block.call(job)
+    end
+
+  private
+
+    def deserialize(source)
+      handler = YAML.load(source) rescue nil
+
+      unless handler.respond_to?(:perform)
+        if handler.nil? && source =~ ParseObjectFromYaml
+          handler_class = $1
+        end
+        attempt_to_load(handler_class || handler.class)
+        handler = YAML.load(source)
+      end
+
+      return handler if handler.respond_to?(:perform)
+
+      raise DeserializationError,
+        'Job failed to load: Unknown handler. Try to manually require the appropiate file.'
+    rescue TypeError, LoadError, NameError => e
+      raise DeserializationError,
+        "Job failed to load: #{e.message}. Try to manually require the required file."
+    end
+
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    def attempt_to_load(klass)
+       klass.constantize
+    end
+
+    def self.db_time_now
+      (ActiveRecord::Base.default_timezone == :utc) ? Time.now.utc : Time.now
+    end
+
+  protected
+
+    def before_save
+      self.run_at ||= self.class.db_time_now
+    end
+
+  end
+
+  class EvaledJob
+    def initialize
+      @job = yield
+    end
+
+    def perform
+      eval(@job)
+    end
+  end
+end

data/vendor/delayed_job/lib/delayed/message_sending.rb

@@ -0,0 +1,7 @@
+module Delayed
+  module MessageSending
+    def send_later(method, *args)
+      Delayed::Job.enqueue Delayed::PerformableMethod.new(self, method.to_sym, args)
+    end
+  end
+end

data/vendor/delayed_job/lib/delayed/performable_method.rb

@@ -0,0 +1,55 @@
+module Delayed
+  class PerformableMethod < Struct.new(:object, :method, :args)
+    CLASS_STRING_FORMAT = /^CLASS\:([A-Z][\w\:]+)$/
+    AR_STRING_FORMAT    = /^AR\:([A-Z][\w\:]+)\:(\d+)$/
+
+    def initialize(object, method, args)
+      raise NoMethodError, "undefined method `#{method}' for #{self.inspect}" unless object.respond_to?(method)
+
+      self.object = dump(object)
+      self.args   = args.map { |a| dump(a) }
+      self.method = method.to_sym
+    end
+    
+    def display_name  
+      case self.object
+      when CLASS_STRING_FORMAT then "#{$1}.#{method}"
+      when AR_STRING_FORMAT    then "#{$1}##{method}"
+      else "Unknown##{method}"
+      end      
+    end    
+
+    def perform
+      load(object).send(method, *args.map{|a| load(a)})
+    rescue ActiveRecord::RecordNotFound
+      # We cannot do anything about objects which were deleted in the meantime
+      true
+    end
+
+    private
+
+    def load(arg)
+      case arg
+      when CLASS_STRING_FORMAT then $1.constantize
+      when AR_STRING_FORMAT    then $1.constantize.find($2)
+      else arg
+      end
+    end
+
+    def dump(arg)
+      case arg
+      when Class              then class_to_string(arg)
+      when ActiveRecord::Base then ar_to_string(arg)
+      else arg
+      end
+    end
+
+    def ar_to_string(obj)
+      "AR:#{obj.class}:#{obj.id}"
+    end
+
+    def class_to_string(obj)
+      "CLASS:#{obj.name}"
+    end
+  end
+end

data/vendor/delayed_job/lib/delayed/worker.rb

@@ -0,0 +1,54 @@
+module Delayed
+  class Worker
+    SLEEP = 5
+
+    cattr_accessor :logger
+    self.logger = if defined?(Merb::Logger)
+                    Merb.logger
+                  elsif defined?(RAILS_DEFAULT_LOGGER)
+                    RAILS_DEFAULT_LOGGER
+                  end
+
+    def initialize(options={})
+      @quiet = options[:quiet]
+      Delayed::Job.min_priority = options[:min_priority] if options.has_key?(:min_priority)
+      Delayed::Job.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+    end
+
+    def start
+      say "*** Starting job worker #{Delayed::Job.worker_name}"
+
+      trap('TERM') { say 'Exiting...'; $exit = true }
+      trap('INT')  { say 'Exiting...'; $exit = true }
+
+      loop do
+        result = nil
+
+        realtime = Benchmark.realtime do
+          result = Delayed::Job.work_off
+        end
+
+        count = result.sum
+
+        break if $exit
+
+        if count.zero?
+          sleep(SLEEP)
+        else
+          say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+        end
+
+        break if $exit
+      end
+
+    ensure
+      Delayed::Job.clear_locks!
+    end
+
+    def say(text)
+      puts text unless @quiet
+      logger.info text if logger
+    end
+
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: freelancing-god-thinking-sphinx
 version: !ruby/object:Gem::Version 
-  version: 0.9.13
+  version: 1.1.0
 platform: ruby
 authors: 
 - Pat Allan
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-27 00:00:00 -08:00
+date: 2009-01-03 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -33,6 +33,12 @@ files:
 - lib/thinking_sphinx/attribute.rb
 - lib/thinking_sphinx/collection.rb
 - lib/thinking_sphinx/configuration.rb
+- lib/thinking_sphinx/deltas/default_delta.rb
+- lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb
+- lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb
+- lib/thinking_sphinx/deltas/delayed_delta/job.rb
+- lib/thinking_sphinx/deltas/delayed_delta.rb
+- lib/thinking_sphinx/deltas.rb
 - lib/thinking_sphinx/field.rb
 - lib/thinking_sphinx/index/builder.rb
 - lib/thinking_sphinx/index/faux_column.rb
@@ -56,6 +62,13 @@ files:
 - vendor/after_commit/README
 - vendor/after_commit/test
 - vendor/after_commit/test/after_commit_test.rb
+- vendor/delayed_job
+- vendor/delayed_job/lib
+- vendor/delayed_job/lib/delayed
+- vendor/delayed_job/lib/delayed/job.rb
+- vendor/delayed_job/lib/delayed/message_sending.rb
+- vendor/delayed_job/lib/delayed/performable_method.rb
+- vendor/delayed_job/lib/delayed/worker.rb
 - vendor/riddle
 - vendor/riddle/lib
 - vendor/riddle/lib/riddle