ts-resque-delta 1.1.5 → 1.2.0

data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set.rb

@@ -0,0 +1,56 @@
+class ThinkingSphinx::Deltas::ResqueDelta < ThinkingSphinx::Deltas::DefaultDelta
+  module FlagAsDeletedSet
+    extend self
+
+    def set_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:set"
+    end
+
+    def temp_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:temp"
+    end
+
+    def processing_name(core_name)
+      "#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:flag.deleted:#{core_name}:processing"
+    end
+
+    def add(core_name, document_id)
+      Resque.redis.sadd(set_name(core_name), document_id)
+    end
+
+    def clear!(core_name)
+      Resque.redis.del(set_name(core_name))
+
+      #Clear processing set as well
+      delta_name = ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_to_delta(core_name)
+      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.around_perform_lock(delta_name) do
+        Resque.redis.del(processing_name(core_name))
+      end
+    end
+
+    def clear_all!
+      ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.core_indices.each do |core_index|
+        clear!(core_index)
+      end
+    end
+
+    def get_subset_for_processing(core_name)
+      # Copy set to temp
+      Resque.redis.sunionstore temp_name(core_name), set_name(core_name)
+      # Store (set - temp) into set.  This removes all items we copied into temp from set.
+      Resque.redis.sdiffstore set_name(core_name), set_name(core_name), temp_name(core_name)
+      # Merge processing and temp together and store into processing.
+      Resque.redis.sunionstore processing_name(core_name), processing_name(core_name), temp_name(core_name)
+
+      Resque.redis.del temp_name(core_name)
+    end
+
+    def processing_members(core_name)
+      Resque.redis.smembers(processing_name(core_name)).collect(&:to_i)
+    end
+
+    def clear_processing(core_name)
+      Resque.redis.del(processing_name(core_name))
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb

@@ -0,0 +1,47 @@
+module ThinkingSphinx::Deltas::ResqueDelta::IndexUtils
+  extend self
+
+  # Public: Return a list of index prefixes (i.e. without "_core"/"_delta").
+  #
+  # Examples
+  #
+  #   sphinx_indices
+  #   # => ['foo', 'bar']
+  #
+  # Returns an Array of index prefixes.
+  def index_prefixes
+    @prefixes ||= indices.reject { |i| i =~ /_(core|delta)$/ }
+  end
+
+  def core_indices
+    @core_indices ||= indices.select { |i| i =~ /_core$/ }
+  end
+
+  def delta_indices
+    @delta_indices ||= indices.select { |i| i =~ /_delta$/ }
+  end
+
+  def reload!
+    @ts_config = @indices = @prefixes = @core_indices = @delta_indices = nil
+  end
+
+  def delta_to_core(delta_name)
+    delta_name.sub(/_delta$/, '_core')
+  end
+
+  def core_to_delta(core_name)
+    core_name.sub(/_core$/, '_delta')
+  end
+
+  def ts_config
+    @ts_config ||= ThinkingSphinx::Configuration.instance.tap do |config|
+      config.generate
+    end
+  end
+
+  private
+  def indices
+    @indices ||= ts_config.configuration.indices.collect { |i| i.name }
+  end
+
+end

data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb

@@ -1,67 +1,25 @@
 require 'thinking_sphinx/deltas/resque_delta'
 
 namespace :thinking_sphinx do
-
-  # Return a list of index prefixes (i.e. without "_core"/"_delta").
-  def sphinx_indices
-    unless @sphinx_indices
-      @ts_config ||= ThinkingSphinx::Configuration.instance
-      @ts_config.generate
-      @sphinx_indices = @ts_config.configuration.indices.collect { |i| i.name }
-      # The collected indices look like:
-      # ["foo_core", "foo_delta", "foo", "bar_core", "bar_delta", "bar"]
-      @sphinx_indices.reject! { |i| i =~ /_(core|delta)$/}
-      # Now we have:
-      # ["foo", "bar"]
-    end
-    @sphinx_indices
-  end
-
-  def lock_delta(index_name)
-    ThinkingSphinx::Deltas::ResqueDelta.lock("#{index_name}_delta")
-  end
-
-  def unlock_delta(index_name)
-    ThinkingSphinx::Deltas::ResqueDelta.unlock("#{index_name}_delta")
-  end
-
   desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
   task :lock_deltas do
-    sphinx_indices.each { |index_name| lock_delta(index_name) }
+    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.lock_deltas
   end
 
   desc 'Unlock all delta indices.'
   task :unlock_deltas do
-    sphinx_indices.each { |index_name| unlock_delta(index_name) }
+    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.unlock_deltas
   end
 
   desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
   task :smart_index => :app_env do
-    # Load config like ts:in.
-    @ts_config = ThinkingSphinx::Configuration.instance
-    unless ENV['INDEX_ONLY'] == 'true'
-      puts "Generating Configuration to #{@ts_config.config_file}"
-      @ts_config.build
-    end
-    FileUtils.mkdir_p(@ts_config.searchd_file_path)
+    ret = ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
 
-    # Index each core, one at a time. Wrap with delta locking logic.
-    sphinx_indices.each do |index_name|
-      lock_delta(index_name)
-      @ts_config.controller.index("#{index_name}_core", :verbose => true)
-      ret = $?
-      unlock_delta(index_name)
-      exit(-1) if ret.to_i != 0
-      Resque.enqueue(
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
-        ["#{index_name}_delta"]
-      )
-    end
+    abort("Indexing failed.") if ret != true
   end
 end
 
 namespace :ts do
-
   desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
   task :si => 'thinking_sphinx:smart_index'
 end

data/lib/thinking_sphinx/deltas/resque_delta/version.rb

@@ -1,7 +1,7 @@
 module ThinkingSphinx
   module Deltas
     class ResqueDeltaInfo
-      VERSION = "1.1.5"
+      VERSION = '1.2.0'
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,9 +1,13 @@
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
-
-require 'spec'
-require 'spec/autorun'
-
 require 'thinking_sphinx'
 require 'thinking_sphinx/deltas/resque_delta'
 require 'flying_sphinx'
 require 'flying_sphinx/resque_delta'
+
+require 'mock_redis'
+require 'fakefs/spec_helpers'
+
+RSpec.configure do |c|
+  c.filter_run :focus => true
+  c.run_all_when_everything_filtered = true
+  c.treat_symbols_as_metadata_keys_with_true_values = true
+end

data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb

@@ -0,0 +1,210 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
+  let(:indices) { %w[foo bar] }
+  let(:config) { double('config') }
+
+  describe '#lock_delta' do
+    it 'should lock the delta' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
+
+      subject.lock_delta('foo')
+    end
+
+    it 'should lock the delta for the given index' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
+
+      subject.lock_delta('foo')
+    end
+  end
+
+  describe '#unlock_delta' do
+    it 'should unlock the delta' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
+
+      subject.unlock_delta('foo')
+    end
+
+    it 'should unlock the delta for the given index' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
+
+      subject.unlock_delta('foo')
+    end
+  end
+
+  describe '#lock_deltas' do
+    it 'should lock all delta indices' do
+      subject.stub(:sphinx_indices => indices)
+
+      indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
+
+      subject.lock_deltas
+    end
+  end
+
+  describe '#unlock_deltas' do
+    it 'should unlock all delta indices' do
+      subject.stub(:sphinx_indices => indices)
+
+      indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
+
+      subject.unlock_deltas
+    end
+  end
+
+  describe '#with_delta_index_lock' do
+    before :each do
+      subject.stub(:lock_delta)
+      subject.stub(:unlock_delta)
+
+      subject.stub(:block_called)
+
+      @block_called = false
+      @block = lambda { @block_called = true; subject.block_called }
+    end
+
+    it 'should yield' do
+      subject.with_delta_index_lock('foo', &@block)
+      @block_called.should be_true
+    end
+
+    it 'should lock before yielding' do
+      subject.should_receive(:lock_delta).with('foo').ordered
+      subject.should_receive(:block_called).ordered
+
+      subject.with_delta_index_lock('foo', &@block)
+    end
+
+    it 'should unlock after yielding' do
+      subject.should_receive(:block_called).ordered
+      subject.should_receive(:unlock_delta).with('foo').ordered
+
+      subject.with_delta_index_lock('foo', &@block)
+    end
+  end
+
+  describe '#smart_index' do
+    include FakeFS::SpecHelpers
+
+    let(:test_path) { '/tmp/ts-resque-delta/foo' }
+
+    before :each do
+      subject.stub(:ts_config => config)
+      config.stub(:config_file => 'foo_config')
+      config.stub(:build)
+      config.stub(:searchd_file_path => test_path)
+      config.stub_chain(:controller, :index) do
+        # Set $? to 0
+        `/usr/bin/true`
+      end
+
+      # Silence Generating config message
+      subject.stub(:puts)
+
+      subject.stub(:index_prefixes => indices)
+      subject.stub(:lock_delta)
+      subject.stub(:unlock_delta)
+
+      ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
+      Resque.stub(:enqueue)
+    end
+
+    it 'should not generate sphinx configuration if INDEX_ONLY is true' do
+      ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
+      ENV.stub(:[]).with('SILENT').and_return(nil)
+      config.should_not_receive(:build)
+
+      subject.smart_index
+    end
+
+    it 'should generate sphinx configuration if INDEX_ONLY is not true' do
+      ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
+      ENV.stub(:[]).with('SILENT').and_return(nil)
+      config.should_receive(:build).once
+
+      subject.smart_index
+    end
+
+    it 'should create the sphinx file directory' do
+      subject.smart_index
+
+      File.directory?(test_path).should be_true
+    end
+
+    it 'should index all core indices' do
+      indices.each do |index|
+        config.controller.should_receive(:index).with("#{index}_core", anything)
+      end
+
+      subject.smart_index
+    end
+
+    context "with delta lock" do
+      before :each do
+        subject.stub(:with_delta_index_lock).and_yield
+      end
+
+      it 'should index' do
+        config.controller.should_receive(:index)
+        subject.smart_index
+      end
+
+      it 'should signal resque delta to prepare for the core index' do
+        ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
+        subject.smart_index
+      end
+    end
+
+    context "without delta lock" do
+      before :each do
+        subject.stub(:with_delta_index_lock)
+      end
+
+      it 'should not index without the delta lock' do
+        config.controller.should_not_receive(:index)
+        subject.smart_index
+      end
+
+      it 'should not signal resque delta to prepare for the core index' do
+        ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
+        subject.smart_index
+      end
+    end
+
+    it 'should create a delta job after the delta is unlocked' do
+      # Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
+      subject.stub(:resque_called)
+      Resque.stub(:enqueue) { subject.resque_called }
+
+      Resque.should_receive(:enqueue)
+
+      subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
+      subject.should_receive(:resque_called).ordered.exactly(indices.size)
+
+      subject.smart_index
+    end
+
+    context 'with an error' do
+      before :each do
+        config.stub_chain(:controller, :index) do
+          # Set $? to 1
+          `/usr/bin/false`
+        end
+      end
+
+      it 'should stop processing indexes after an error' do
+        config.controller.should_receive(:index).once
+
+        subject.smart_index
+      end
+
+      it 'should return false on failure' do
+        subject.smart_index.should be_false
+      end
+    end
+
+    it 'should return true on success' do
+      subject.smart_index.should be_true
+    end
+  end
+end

data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb

@@ -1,48 +1,37 @@
 require 'spec_helper'
 
 describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
+  subject do
+    ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.tap do |s|
+      s.stub(:` => true)
+      s.stub(:puts => nil)
+    end
+  end
+
   describe '.perform' do
     before :each do
       ThinkingSphinx.suppress_delta_output = false
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:` => true)
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:puts => nil)
       ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(false)
+      ThinkingSphinx.stub(:sphinx_running? => false)
     end
 
     it "should output the delta indexing by default" do
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:puts)
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-        ['foo_delta']
-      )
+      subject.should_receive(:puts)
+      subject.perform('foo_delta')
     end
 
     it "should not output the delta indexing if requested" do
       ThinkingSphinx.suppress_delta_output = true
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_not_receive(:puts)
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-        ['foo_delta']
-      )
+      subject.should_not_receive(:puts)
+      subject.perform('foo_delta')
     end
 
-    it "should process just the requested indices" do
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
+    it "should process just the requested index" do
+      subject.should_receive(:`) do |c|
         c.should match(/foo_delta/)
         c.should_not match(/--all/)
       end
-      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-        ['foo_delta']
-      )
-    end
-
-    context 'multiple indices' do
-      it "should process all requested indices" do
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:`) do |c|
-          c.should match(/foo_delta bar_delta/)
-        end
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-          ['foo_delta', 'bar_delta']
-        )
-      end
+      subject.perform('foo_delta')
     end
 
     context 'when an index is locked' do
@@ -53,18 +42,132 @@ describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
       end
 
       it "should not start the indexer" do
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_not_receive(:`)
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-          ['foo_delta']
-        )
+        subject.should_not_receive(:`)
+        subject.perform('foo_delta')
+      end
+
+      it "should start the indexer for unlocked indexes" do
+        subject.should_receive(:`)
+        subject.perform('bar_delta')
+      end
+    end
+
+    context 'with flag as deleted document ids' do
+      let(:client) { stub('client', :update => true) }
+      let(:document_ids) { [1, 2, 3] }
+
+      before :each do
+        ThinkingSphinx.updates_enabled = true
+
+        ThinkingSphinx::Configuration.instance.stub(:client => client)
+        ThinkingSphinx.stub(:sphinx_running? => true)
+
+        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:processing_members => document_ids)
+        subject.stub(:filter_flag_as_deleted_ids => document_ids)
       end
 
-      it "should not start the indexer for multiple indices" do
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_not_receive(:`)
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.perform(
-          ['bar_delta', 'foo_delta']
-        )
+      it 'should get the processing set of flag as deleted document ids' do
+        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:processing_members).with('foo_core')
+        subject.perform('foo_delta')
       end
+
+      it "should not update if Sphinx isn't running" do
+        ThinkingSphinx.stub(:sphinx_running? => false)
+        client.should_not_receive(:update)
+        subject.perform('foo_delta')
+      end
+
+      it "should validate the document ids with sphinx" do
+        subject.should_receive(:filter_flag_as_deleted_ids).with(document_ids, 'foo_core')
+
+        subject.perform('foo_delta')
+      end
+
+      context "with invalid ids" do
+        before :each do
+          subject.stub(:filter_flag_as_deleted_ids => document_ids.reject {|x| x == 2} )
+        end
+
+        it "should not update documents that aren't in the index" do
+          client.should_receive(:update) do |index, attributes, values|
+            values.should_not include(2)
+          end
+          subject.perform('foo_delta')
+        end
+
+        it "should update documents that are in the index" do
+          client.should_receive(:update) do |index, attributes, values|
+            values.keys.should eql(document_ids.reject{|x| x == 2})
+          end
+          subject.perform('foo_delta')
+        end
+      end
+
+      it "should update the specified index" do
+        client.should_receive(:update) do |index, attributes, values|
+          index.should == 'foo_core'
+        end
+        subject.perform('foo_delta')
+      end
+
+      it "should update the sphinx_deleted attribute" do
+        client.should_receive(:update) do |index, attributes, values|
+          attributes.should == ['sphinx_deleted']
+        end
+        subject.perform('foo_delta')
+      end
+
+      it "should set sphinx_deleted for valid documents to true" do
+        client.should_receive(:update) do |index, attributes, values|
+          document_ids.each {|id| values[id].should == [1] }
+        end
+        subject.perform('foo_delta')
+      end
+
+      context "without any ids" do
+        let(:document_ids) { [] }
+
+        it "should not validate the ids with sphinx" do
+          subject.should_not_receive(:filter_flag_as_deleted_ids)
+          subject.perform('foo_delta')
+        end
+      end
+    end
+  end
+
+  describe '.around_perform_lock1' do
+    before :each do
+      Resque.stub(:encode => 'DeltaJobsAreAwesome')
+      Resque.stub_chain(:redis, :lrem)
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:get_subset_for_processing)
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear_processing)
+    end
+
+    it 'should clear all other delta jobs' do
+      Resque.redis.should_receive(:lrem).with("queue:#{subject.instance_variable_get(:@queue)}", 0, 'DeltaJobsAreAwesome')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+
+    it 'should set up the processing set of document ids' do
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:get_subset_for_processing).with('foo_core')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+
+    it 'should clear the processing set when finished' do
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear_processing).with('foo_core')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+  end
+
+  describe '.lock_failed' do
+    it 'should enqueue the delta job again' do
+      Resque.stub(:enqueue => true)
+      Resque.should_receive(:enqueue)
+
+      subject.lock_failed('foo_delta')
     end
   end
 end