ts-resque-delta 1.1.5 → 1.2.0

data/Gemfile

@@ -2,3 +2,27 @@ source "http://rubygems.org"
 
 # Specify your gem's dependencies in ts-resque-delta.gemspec
 gemspec
+
+group :development do
+  if RUBY_PLATFORM =~ /darwin/i
+    gem 'rb-fsevent'
+    gem 'ruby_gntp'
+  else
+    gem 'rb-fsevent', :require => false
+    gem 'growl', :require => false
+  end
+  if RUBY_PLATFORM =~ /linux/i
+    gem 'rb-inotify'
+    gem 'libnotify'
+  else
+    gem 'rb-inotify', :require => false
+    gem 'libnotify', :require => false
+  end
+end
+
+group :test do
+  if RUBY_VERSION >= '1.9'
+  else
+    gem 'ruby-debug'
+  end
+end

data/Guardfile

@@ -0,0 +1,17 @@
+guard 'bundler' do
+  watch('Gemfile')
+  watch(/^.+\.gemspec/)
+end
+
+guard 'rspec', :version => 2, :cli => "-c --format progress", :all_on_start => false do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+
+guard 'cucumber', :all_on_start => false do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$})          { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end
+

data/README.markdown

@@ -68,6 +68,7 @@ Contributors (for ts-delayed-delta)
 -----------------------------------
 * [Aaron Gibralter](https://github.com/agibralter)
 * [Ryan Schlesinger](https://github.com/ryansch) (Locking/`smart_index`)
+* [Pat Allan](https://github.com/freelancing-god) (FlyingSphinx support)
 
 Original Contributors (for ts-delayed-delta)
 --------------------------------------------

data/Rakefile

@@ -1,4 +1,16 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-require File.join(File.dirname(__FILE__), 'tasks/testing')
-task :default => :spec
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+require 'cucumber'
+require 'cucumber/rake/task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["-c", "--format progress"]
+end
+
+Cucumber::Rake::Task.new(:features) do |t|
+end
+
+task :all_tests => [:spec, :features]
+
+task :default => :all_tests

data/cucumber.yml

@@ -0,0 +1,2 @@
+---
+default: --color --format progress --strict

data/features/smart_indexing.feature

@@ -0,0 +1,43 @@
+Feature: Smart Indexing
+  In order to have core indexing that works well with resque delta indexing
+  Developers
+  Should be able to use smart index to update core indices
+
+  Background:
+    Given Sphinx is running
+    And I am searching on delayed betas
+    And I have data
+
+  Scenario: Smart indexing should update core indices
+    When I run the smart indexer
+    And I wait for Sphinx to catch up
+    And I search for one
+    Then I should get 1 result
+
+  Scenario: Smart indexing should reset the delta index
+    Given I have indexed
+    When I change the name of delayed beta one to eleven
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I change the name of delayed beta eleven to one
+    And I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for eleven
+    Then I should get 0 results
+
+  Scenario: Delta Index running after smart indexing should not hide records
+    When I run the smart indexer
+    And I run the delayed jobs
+    And I wait for Sphinx to catch up
+
+    When I search for two
+    Then I should get 1 result
+
+  Scenario: Smart index should remove existing delta jobs
+    When I run the smart indexer
+    And I run one delayed job
+    And I wait for Sphinx to catch up
+    Then there should be no more DeltaJobs on the Resque queue

data/features/step_definitions/common_steps.rb

@@ -1,6 +1,6 @@
 Before do
   $queries_executed = []
-  ThinkingSphinx::Deltas::ResqueDelta.cancel_thinking_sphinx_jobs
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
   @model      = nil
   @method     = :search
   @query      = ""
@@ -20,7 +20,7 @@ Given /^I am searching on (.+)$/ do |model|
   @model = model.gsub(/\s/, '_').singularize.camelize.constantize
 end
 
-Given "I have data and it has been indexed" do
+Given "I have data" do
   DelayedBeta.create(:name => "one")
   DelayedBeta.create(:name => "two")
   DelayedBeta.create(:name => "three")
@@ -31,11 +31,19 @@ Given "I have data and it has been indexed" do
   DelayedBeta.create(:name => "eight")
   DelayedBeta.create(:name => "nine")
   DelayedBeta.create(:name => "ten")
-  ThinkingSphinx::Deltas::ResqueDelta.cancel_thinking_sphinx_jobs
+end
+
+Given "I have indexed" do
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
   ThinkingSphinx::Configuration.instance.controller.index
   sleep(1.5)
 end
 
+Given "I have data and it has been indexed" do
+  step "I have data"
+  step "I have indexed"
+end
+
 When "I wait for Sphinx to catch up" do
   sleep(0.5)
 end
@@ -49,6 +57,11 @@ Then /^I should get (\d+) results?$/ do |count|
   results.length.should == count.to_i
 end
 
+Then /^I debug$/ do
+  debugger
+  0
+end
+
 def results
   @results ||= (@model || ThinkingSphinx).send(
     @method,

data/features/step_definitions/resque_delta_steps.rb

@@ -18,7 +18,7 @@ When /^I run one delayed job$/ do
 end
 
 When /^I cancel the jobs$/ do
-  ThinkingSphinx::Deltas::ResqueDelta.cancel_thinking_sphinx_jobs
+  ThinkingSphinx::Deltas::ResqueDelta.clear!
 end
 
 When /^I change the name of delayed beta (\w+) to (\w+)$/ do |current, replacement|

data/features/step_definitions/smart_indexing_steps.rb

@@ -0,0 +1,3 @@
+When /^I run the smart indexer$/ do
+  ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index(:verbose => false)
+end

data/features/support/env.rb

@@ -1,7 +1,8 @@
 require 'cucumber'
-require 'spec/expectations'
+require 'rspec/expectations'
 require 'fileutils'
 require 'active_record'
+require 'mock_redis'
 
 PROJECT_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
 
@@ -20,9 +21,7 @@ require 'thinking_sphinx/deltas/resque_delta'
 
 world.setup
 
-require 'redis_test_setup'
-RedisTestSetup.start_redis!(PROJECT_ROOT, :cucumber)
-Resque.redis = '127.0.0.1:6398'
+Resque.redis = MockRedis.new
 Before do
   Resque.redis.flushall
 end

data/lib/thinking_sphinx/deltas/resque_delta.rb

@@ -1,6 +1,9 @@
 require 'resque'
 require 'thinking_sphinx'
 
+require 'thinking_sphinx/deltas/resque_delta/flag_as_deleted_set'
+require 'thinking_sphinx/deltas/resque_delta/index_utils'
+
 # Delayed Deltas for Thinking Sphinx, using Resque.
 #
 # This documentation is aimed at those reading the code. If you're looking for
@@ -14,8 +17,7 @@ require 'thinking_sphinx'
 class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
   def self.job_types
     [
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob
+      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
     ]
   end
   
@@ -25,13 +27,24 @@ class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
   
   # LTRIM + LPOP deletes all items from the Resque queue without loading it
   # into client memory (unlike Resque.dequeue).
-  def self.cancel_thinking_sphinx_jobs
+  # WARNING: This will clear ALL jobs in any queue used by a ResqueDelta job.
+  # If you're sharing a queue with other jobs they'll be deleted!
+  def self.clear_thinking_sphinx_queues
     job_types.collect { |c| c.instance_variable_get(:@queue) }.uniq.each do |q|
       Resque.redis.ltrim("queue:#{q}", 0, 0)
       Resque.redis.lpop("queue:#{q}")
     end
   end
 
+  # Clear both the resque queues and any other state maintained in redis
+  def self.clear!
+    self.clear_thinking_sphinx_queues
+
+    FlagAsDeletedSet.clear_all!
+  end
+
+  # Use simplistic locking.  We're assuming that the user won't run more than one
+  # `rake ts:si` or `rake ts:in` task at a time.
   def self.lock(index_name)
     Resque.redis.set("#{job_prefix}:index:#{index_name}:locked", 'true')
   end
@@ -44,6 +57,17 @@ class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
     Resque.redis.get("#{job_prefix}:index:#{index_name}:locked") == 'true'
   end
 
+  def self.prepare_for_core_index(index_name)
+    core = "#{index_name}_core"
+    delta = "#{index_name}_delta"
+
+    FlagAsDeletedSet.clear!(core)
+
+    #clear delta jobs
+    # dequeue is fast for jobs with arguments
+    Resque.dequeue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, delta)
+  end
+
   # Adds a job to the queue for processing the given model's delta index. A job
   # for hiding the instance in the core index is also created, if an instance is
   # provided.
@@ -64,15 +88,13 @@ class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
       next if self.class.locked?(delta)
       Resque.enqueue(
         ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
-        [delta]
+        delta
       )
     end
     if instance
-      Resque.enqueue(
-        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob,
-        model.core_index_names,
-        instance.sphinx_document_id
-      )
+      model.core_index_names.each do |core|
+        FlagAsDeletedSet.add(core, instance.sphinx_document_id)
+      end
     end
     true
   end
@@ -93,4 +115,4 @@ class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
 end
 
 require 'thinking_sphinx/deltas/resque_delta/delta_job'
-require 'thinking_sphinx/deltas/resque_delta/flag_as_deleted_job'
+require 'thinking_sphinx/deltas/resque_delta/core_index'

data/lib/thinking_sphinx/deltas/resque_delta/core_index.rb

@@ -0,0 +1,101 @@
+class ThinkingSphinx::Deltas::ResqueDelta::CoreIndex
+  # Public: Lock a delta index against indexing or new index jobs.
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   lock_delta('foo')
+  #
+  # Returns nothing.
+  def lock_delta(index_name)
+    ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
+  end
+
+  # Public: Unlock a delta index for indexing or new index jobs.
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   unlock_delta('foo')
+  #
+  # Returns nothing.
+  def unlock_delta(index_name)
+    ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
+  end
+
+  # Public: Lock all delta indexes against indexing or new index jobs.
+  #
+  # Returns nothing.
+  def lock_deltas
+    sphinx_indices.each { |index_name| lock_delta(index_name) }
+  end
+
+  # Public: Unlock all delta indexes for indexing or new index jobs.
+  #
+  # Returns nothing.
+  def unlock_deltas
+    sphinx_indices.each { |index_name| unlock_delta(index_name) }
+  end
+
+  # Public: Index all indices while locking each delta as we index the corresponding core index.
+  #
+  # Returns true on success; false on failure.
+  def smart_index(opts = {})
+    verbose = opts.fetch(:verbose, true)
+    verbose = false if ENV['SILENT'] == 'true'
+
+    # Load config like ts:in.
+    unless ENV['INDEX_ONLY'] == 'true'
+      puts "Generating Configuration to #{ts_config.config_file}" if verbose
+      ts_config.build
+    end
+    FileUtils.mkdir_p(ts_config.searchd_file_path)
+
+    # Index each core, one at a time. Wrap with delta locking logic.
+    index_prefixes.each do |index_name|
+      ret = nil
+
+      with_delta_index_lock(index_name) do
+        ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index(index_name)
+        ts_config.controller.index("#{index_name}_core", :verbose => verbose)
+        ret = $?
+      end
+
+      return false if ret.to_i != 0
+
+      Resque.enqueue(
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
+        "#{index_name}_delta"
+      )
+    end
+
+    true
+  end
+
+  # Public: Wraps the passed block with a delta index lock
+  #
+  # index_name - The String index prefix.
+  #
+  # Examples
+  #
+  #   with_delta_index_lock('foo')
+  #
+  # Returns nothing.
+  def with_delta_index_lock(index_name)
+    lock_delta(index_name)
+    yield
+    unlock_delta(index_name)
+  end
+
+  private
+
+  def ts_config
+    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.ts_config
+  end
+
+  def index_prefixes
+    ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.index_prefixes
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta/delta_job.rb

@@ -13,12 +13,34 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
   #
   # @param [String] index the name of the Sphinx index
   #
-  def self.perform(indices)
-    return if skip?(indices)
+  def self.perform(index)
+    return if skip?(index)
 
     config = ThinkingSphinx::Configuration.instance
-    output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{indices.join(' ')}`
+
+    # Delta Index
+    output = `#{config.bin_path}#{config.indexer_binary_name} --config #{config.config_file} --rotate #{index}`
     puts output unless ThinkingSphinx.suppress_delta_output?
+
+    # Flag As Deleted
+    return unless ThinkingSphinx.sphinx_running?
+
+    index = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.delta_to_core(index)
+
+    # Get the document ids we've saved
+    flag_as_deleted_ids = ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.processing_members(index)
+
+    unless flag_as_deleted_ids.empty?
+      # Filter out the ids that aren't present in sphinx
+      flag_as_deleted_ids = filter_flag_as_deleted_ids(flag_as_deleted_ids, index)
+
+      unless flag_as_deleted_ids.empty?
+        # Each hash element should be of the form { id => [1] }
+        flag_hash = Hash[*flag_as_deleted_ids.collect {|id| [id, [1]] }.flatten(1)]
+
+        config.client.update(index, ['sphinx_deleted'], flag_hash)
+      end
+    end
   end
 
   # Try again later if lock is in use.
@@ -26,10 +48,10 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
     Resque.enqueue(self, *args)
   end
 
-  # Run only one DeltaJob at a time regardless of indices.
-  def self.identifier(*args)
-    nil
-  end
+  # Run only one DeltaJob at a time regardless of index.
+  #def self.identifier(*args)
+    #nil
+  #end
 
   # This allows us to have a concurrency safe version of ts-delayed-delta's
   # duplicates_exist:
@@ -51,14 +73,54 @@ class ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
     # instances of value from the list.
     redis_job_value = Resque.encode(:class => self.to_s, :args => args)
     Resque.redis.lrem("queue:#{@queue}", 0, redis_job_value)
+
+    # Grab the subset of flag as deleted document ids to work on
+    core_index = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.delta_to_core(*args)
+    ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.get_subset_for_processing(core_index)
+
     yield
+
+    # Clear processing set
+    ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.clear_processing(core_index)
   end
 
   protected
 
-  def self.skip?(indices)
-    indices.any? do |index|
-      ThinkingSphinx::Deltas::ResqueDelta.locked?(index)
+  def self.skip?(index)
+    ThinkingSphinx::Deltas::ResqueDelta.locked?(index)
+  end
+
+  def self.filter_flag_as_deleted_ids(ids, index)
+    search_results = []
+    partition_ids(ids, 4096) do |subset|
+      search_results += ThinkingSphinx.search_for_ids(
+        :with => {:@id => subset}, :index => index
+      ).results[:matches].collect { |match| match[:doc] }
+    end
+
+    search_results
+  end
+
+  def self.partition_ids(ids, n)
+    if n > 0 && n < ids.size
+      result = []
+      max_subarray_size = n - 1
+      i = j = 0
+      while i < ids.size && j < ids.size
+        j = i + max_subarray_size
+        result << ids.slice(i..j)
+        i += n
+      end
+    else
+      result = ids
     end
+
+    if block_given?
+      result.each do |ary|
+        yield ary
+      end
+    end
+
+    result
   end
 end