ryansch-ts-resque-delta 1.1.5.1

data/features/thinking_sphinx/database.example.yml

@@ -0,0 +1,3 @@
+username: root
+host: localhost
+password: 

data/features/thinking_sphinx/db/migrations/create_delayed_betas.rb

@@ -0,0 +1,4 @@
+ActiveRecord::Base.connection.create_table :delayed_betas, :force => true do |t|
+  t.column  :name, :string,  :null => false
+  t.column :delta, :boolean, :null => false, :default => false
+end

data/features/thinking_sphinx/models/delayed_beta.rb

@@ -0,0 +1,6 @@
+class DelayedBeta < ActiveRecord::Base
+  define_index do
+    indexes :name, :sortable => true
+    set_property :delta => ThinkingSphinx::Deltas::ResqueDelta
+  end
+end

data/lib/flying_sphinx/resque_delta.rb

@@ -0,0 +1,38 @@
+require 'thinking_sphinx/deltas/resque_delta'
+
+class FlyingSphinx::ResqueDelta < ThinkingSphinx::Deltas::ResqueDelta
+  def self.job_types
+    [
+      FlyingSphinx::ResqueDelta::DeltaJob,
+      FlyingSphinx::ResqueDelta::FlagAsDeletedJob
+    ]
+  end
+  
+  def self.job_prefix
+    'fs-delta'
+  end
+  
+  def index(model, instance = nil)
+    return true if skip?(instance)
+    
+    model.delta_index_names.each do |delta|
+      next if self.class.locked?(delta)
+      
+      Resque.enqueue(
+        FlyingSphinx::ResqueDelta::DeltaJob,
+        [delta]
+      )
+    end
+    
+    Resque.enqueue(
+      FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
+      model.core_index_names,
+      instance.sphinx_document_id
+    ) if instance
+    
+    true
+  end
+end
+
+require 'flying_sphinx/resque_delta/delta_job'
+require 'flying_sphinx/resque_delta/flag_as_deleted_job'

data/lib/flying_sphinx/resque_delta/delta_job.rb

@@ -0,0 +1,14 @@
+class FlyingSphinx::ResqueDelta::DeltaJob < ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
+  @queue = :fs_delta
+  
+  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
+  # is running.
+  #
+  # @param [String] index the name of the Sphinx index
+  #
+  def self.perform(indices)
+    return if skip?(indices)
+
+    FlyingSphinx::IndexRequest.new(indices).perform
+  end
+end

data/lib/flying_sphinx/resque_delta/flag_as_deleted_job.rb

@@ -0,0 +1,7 @@
+class FlyingSphinx::ResqueDelta::FlagAsDeletedJob
+  @queue = :fs_delta
+  
+  def self.perform(indices, document_id)
+    FlyingSphinx::FlagAsDeletedJob.new(indices, document_id).perform
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta.rb

@@ -0,0 +1,118 @@
+require 'resque'
+require 'thinking_sphinx'
+
+require 'thinking_sphinx/deltas/resque_delta/flag_as_deleted_set'
+require 'thinking_sphinx/deltas/resque_delta/index_utils'
+
+# Delayed Deltas for Thinking Sphinx, using Resque.
+#
+# This documentation is aimed at those reading the code. If you're looking for
+# a guide to Thinking Sphinx and/or deltas, I recommend you start with the
+# Thinking Sphinx site instead - or the README for this library at the very
+# least.
+#
+# @author Patrick Allan
+# @see http://ts.freelancing-gods.com Thinking Sphinx
+#
+class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
+  def self.job_types
+    [
+      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
+    ]
+  end
+  
+  def self.job_prefix
+    'ts-delta'
+  end
+  
+  # LTRIM + LPOP deletes all items from the Resque queue without loading it
+  # into client memory (unlike Resque.dequeue).
+  # WARNING: This will clear ALL jobs in any queue used by a ResqueDelta job.
+  # If you're sharing a queue with other jobs they'll be deleted!
+  def self.clear_thinking_sphinx_queues
+    job_types.collect { |c| c.instance_variable_get(:@queue) }.uniq.each do |q|
+      Resque.redis.ltrim("queue:#{q}", 0, 0)
+      Resque.redis.lpop("queue:#{q}")
+    end
+  end
+
+  # Clear both the resque queues and any other state maintained in redis
+  def self.clear!
+    self.clear_thinking_sphinx_queues
+
+    FlagAsDeletedSet.clear_all!
+  end
+
+  # Use simplistic locking.  We're assuming that the user won't run more than one
+  # `rake ts:si` or `rake ts:in` task at a time.
+  def self.lock(index_name)
+    Resque.redis.set("#{job_prefix}:index:#{index_name}:locked", 'true')
+  end
+
+  def self.unlock(index_name)
+    Resque.redis.del("#{job_prefix}:index:#{index_name}:locked")
+  end
+
+  def self.locked?(index_name)
+    Resque.redis.get("#{job_prefix}:index:#{index_name}:locked") == 'true'
+  end
+
+  def self.prepare_for_core_index(index_name)
+    core = "#{index_name}_core"
+    delta = "#{index_name}_delta"
+
+    FlagAsDeletedSet.clear!(core)
+
+    #clear delta jobs
+    # dequeue is fast for jobs with arguments
+    Resque.dequeue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, delta)
+  end
+
+  # Adds a job to the queue for processing the given model's delta index. A job
+  # for hiding the instance in the core index is also created, if an instance is
+  # provided.
+  #
+  # Neither job will be queued if updates or deltas are disabled, or if the 
+  # instance (when given) is not toggled to be in the delta index. The first two
+  # options are controlled via ThinkingSphinx.updates_enabled? and
+  # ThinkingSphinx.deltas_enabled?.
+  #
+  # @param [Class] model the ActiveRecord model to index.
+  # @param [ActiveRecord::Base] instance the instance of the given model that
+  #   has changed. Optional.
+  # @return [Boolean] true
+  #
+  def index(model, instance = nil)
+    return true if skip?(instance)
+    model.delta_index_names.each do |delta|
+      next if self.class.locked?(delta)
+      Resque.enqueue(
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
+        delta
+      )
+    end
+    if instance
+      model.core_index_names.each do |core|
+        FlagAsDeletedSet.add(core, instance.sphinx_document_id)
+      end
+    end
+    true
+  end
+
+  private
+
+  # Checks whether jobs should be enqueued. Only true if updates and deltas are
+  # enabled, and the instance (if there is one) is toggled.
+  #
+  # @param [ActiveRecord::Base, NilClass] instance
+  # @return [Boolean]
+  #
+  def skip?(instance)
+    !ThinkingSphinx.updates_enabled? ||
+    !ThinkingSphinx.deltas_enabled?  ||
+    (instance && !toggled(instance))
+  end
+end
+
+require 'thinking_sphinx/deltas/resque_delta/delta_job'
+require 'thinking_sphinx/deltas/resque_delta/core_index'

data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb

@@ -0,0 +1,98 @@
+class ThinkingSphinx::Deltas::ResqueDelta::CoreIndex
+  # Public: Lock a delta index against indexing or new index jobs.
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   lock_delta('foo')
+  #
+  # Returns nothing.
+  def lock_delta(index_name)
+    ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
+  end
+
+  # Public: Unlock a delta index for indexing or new index jobs.
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   unlock_delta('foo')
+  #
+  # Returns nothing.
+  def unlock_delta(index_name)
+    ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
+  end
+
+  # Public: Lock all delta indexes against indexing or new index jobs.
+  #
+  # Returns nothing.
+  def lock_deltas
+    sphinx_indices.each { |index_name| lock_delta(index_name) }
+  end
+
+  # Public: Unlock all delta indexes for indexing or new index jobs.
+  #
+  # Returns nothing.
+  def unlock_deltas
+    sphinx_indices.each { |index_name| unlock_delta(index_name) }
+  end
+
+  # Public: Index all indices while locking each delta as we index the corresponding core index.
+  #
+  # Returns true on success; false on failure.
+  def smart_index
+    # Load config like ts:in.
+    unless ENV['INDEX_ONLY'] == 'true'
+      puts "Generating Configuration to #{ts_config.config_file}"
+      ts_config.build
+    end
+    FileUtils.mkdir_p(ts_config.searchd_file_path)
+
+    # Index each core, one at a time. Wrap with delta locking logic.
+    index_prefixes.each do |index_name|
+      ret = nil
+
+      with_delta_index_lock(index_name) do
+        ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index(index_name)
+        ts_config.controller.index("#{index_name}_core", :verbose => true)
+        ret = $?
+      end
+
+      return false if ret.to_i != 0
+
+      Resque.enqueue(
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
+        "#{index_name}_delta"
+      )
+    end
+
+    true
+  end
+
+  # Public: Wraps the passed block with a delta index lock
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   with_delta_index_lock('foo')
+  #
+  # Returns nothing.
+  def with_delta_index_lock(index_name)
+    lock_delta(index_name)
+    yield
+    unlock_delta(index_name)
+  end
+
+  private
+
+  def ts_config
+    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.ts_config
+  end
+
+  def index_prefixes
+    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.index_prefixes
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb

@@ -0,0 +1,90 @@
+require 'resque-lock-timeout'
+
+# A simple job class that processes a given index.
+#
+class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
+
+  extend Resque::Plugins::LockTimeout
+  @queue = :ts_delta
+  @lock_timeout = 240
+
+  # Runs Sphinx's indexer tool to process the index. Currently assumes Sphinx
+  # is running.
+  #
+  # @param [String] index the name of the Sphinx index
+  #
+  def self.perform(index)
+    return if skip?(index)
+
+    config = ThinkingSphinx::Configuration.instance
+
+    # Delta Index
+    output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{index}`
+    puts output unless ThinkingSphinx.suppress_delta_output?
+
+    # Flag As Deleted
+    return unless ThinkingSphinx.sphinx_running?
+
+    index = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.delta_to_core(index)
+
+    # Get the document ids we've saved
+    flag_as_deleted_ids = ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.processing_members(index)
+
+    # Filter out the ids that aren't present in sphinx
+    flag_as_deleted_ids = ThinkingSphinx::Search.bundle_searches(flag_as_deleted_ids) do |sphinx, id|
+      sphinx.search_for_ids([], :index => index, :id_range => id..id)
+    end.map(&:to_a).flatten
+
+    # Each hash element should be of the form { id => [1] }
+    flag_hash = Hash[*flag_as_deleted_ids.collect {|id| [id, [1]] }.flatten(1)]
+
+    config.client.update(index, ['sphinx_deleted'], flag_hash)
+  end
+
+  # Try again later if lock is in use.
+  def self.lock_failed(*args)
+    Resque.enqueue(self, *args)
+  end
+
+  # Run only one DeltaJob at a time regardless of index.
+  #def self.identifier(*args)
+    #nil
+  #end
+
+  # This allows us to have a concurrency safe version of ts-delayed-delta's
+  # duplicates_exist:
+  #
+  # http://github.com/freelancing-god/ts-delayed-delta/blob/master/lib/thinkin
+  # g_sphinx/deltas/delayed_delta/job.rb#L47
+  #
+  # The name of this method ensures that it runs within around_perform_lock.
+  #
+  # We've leveraged resque-lock-timeout to ensure that only one DeltaJob is
+  # running at a time. Now, this around filter essentially ensures that only
+  # one DeltaJob of each index type can sit at the queue at once. If the queue
+  # has more than one, lrem will clear the rest off.
+  #
+  def self.around_perform_lock1(*args)
+    # Remove all other instances of this job (with the same args) from the
+    # queue. Uses LREM (http://code.google.com/p/redis/wiki/LremCommand) which
+    # takes the form: "LREM key count value" and if count == 0 removes all
+    # instances of value from the list.
+    redis_job_value = Resque.encode(:class => self.to_s, :args => args)
+    Resque.redis.lrem("queue:#{@queue}", 0, redis_job_value)
+
+    # Grab the subset of flag as deleted document ids to work on
+    core_index = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.delta_to_core(*args)
+    ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.get_subset_for_processing(core_index)
+
+    yield
+
+    # Clear processing set
+    ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.clear_processing(core_index)
+  end
+
+  protected
+
+  def self.skip?(index)
+    ThinkingSphinx::Deltas::ResqueDelta.locked?(index)
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb

@@ -0,0 +1,56 @@
+class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
+  module FlagAsDeletedSet
+    extend self
+
+    def set_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:set"
+    end
+
+    def temp_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:temp"
+    end
+
+    def processing_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:processing"
+    end
+
+    def add(core_name, document_id)
+      Resque.redis.sadd(set_name(core_name), document_id)
+    end
+
+    def clear!(core_name)
+      Resque.redis.del(set_name(core_name))
+
+      #Clear processing set as well
+      delta_name = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_to_delta(core_name)
+      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.around_perform_lock(delta_name) do
+        Resque.redis.del(processing_name(core_name))
+      end
+    end
+
+    def clear_all!
+      ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_indices.each do |core_index|
+        clear!(core_index)
+      end
+    end
+
+    def get_subset_for_processing(core_name)
+      # Copy set to temp
+      Resque.redis.sunionstore temp_name(core_name), set_name(core_name)
+      # Store (set - temp) into set.  This removes all items we copied into temp from set.
+      Resque.redis.sdiffstore set_name(core_name), set_name(core_name), temp_name(core_name)
+      # Merge processing and temp together and store into processing.
+      Resque.redis.sunionstore processing_name(core_name), processing_name(core_name), temp_name(core_name)
+
+      Resque.redis.del temp_name(core_name)
+    end
+
+    def processing_members(core_name)
+      Resque.redis.smembers(processing_name(core_name)).collect(&:to_i)
+    end
+
+    def clear_processing(core_name)
+      Resque.redis.del(processing_name(core_name))
+    end
+  end
+end