ryansch-ts-resque-delta 1.1.5.1

data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
+  subject do
+    ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.tap do |s|
+      s.stub(:` => true)
+      s.stub(:puts => nil)
+    end
+  end
+
+  describe '.perform' do
+    before :each do
+      ThinkingSphinx.suppress_delta_output = false
+      ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(false)
+    end
+
+    it "should output the delta indexing by default" do
+      subject.should_receive(:puts)
+      subject.perform('foo_delta')
+    end
+
+    it "should not output the delta indexing if requested" do
+      ThinkingSphinx.suppress_delta_output = true
+      subject.should_not_receive(:puts)
+      subject.perform('foo_delta')
+    end
+
+    it "should process just the requested index" do
+      subject.should_receive(:`) do |c|
+        c.should match(/foo_delta/)
+        c.should_not match(/--all/)
+      end
+      subject.perform('foo_delta')
+    end
+
+    context 'when an index is locked' do
+      before do
+        ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?) do |index_name|
+          index_name == 'foo_delta' ? true : false
+        end
+      end
+
+      it "should not start the indexer" do
+        subject.should_not_receive(:`)
+        subject.perform('foo_delta')
+      end
+
+      it "should start the indexer for unlocked indexes" do
+        subject.should_receive(:`)
+        subject.perform('bar_delta')
+      end
+    end
+
+    context 'with flag as deleted document ids' do
+      let(:client) { stub('client', :update => true) }
+      let(:document_ids) { [1, 2, 3] }
+      let(:bundled_search) { double('bundled_search', :search_for_ids => []) }
+
+      before :each do
+        ThinkingSphinx.updates_enabled = true
+
+        ThinkingSphinx::Configuration.instance.stub(:client => client)
+        ThinkingSphinx.stub(:sphinx_running? => true)
+
+        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:processing_members => document_ids)
+        ThinkingSphinx::Search.stub_chain(:bundle_searches, :map => document_ids)
+      end
+
+      it 'should get the processing set of flag as deleted document ids' do
+        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:processing_members).with('foo_core')
+        subject.perform('foo_delta')
+      end
+
+      it "should not update if Sphinx isn't running" do
+        ThinkingSphinx.stub(:sphinx_running? => false)
+        client.should_not_receive(:update)
+        subject.perform('foo_delta')
+      end
+
+      it "should validate the document ids with sphinx" do
+        ThinkingSphinx::Search.stub(:bundle_searches).tap do |s|
+          document_ids.inject(s) do |s, id|
+            s.and_yield(bundled_search, id)
+          end
+        end
+
+        document_ids.each do |id|
+          bundled_search.should_receive(:search_for_ids).with([], :index => 'foo_core', :id_range => id..id)
+        end
+
+        subject.perform('foo_delta')
+      end
+
+      context "with invalid ids" do
+        before :each do
+          ThinkingSphinx::Search.stub_chain(:bundle_searches, :map => document_ids.reject {|x| x == 2} )
+        end
+
+        it "should not update documents that aren't in the index" do
+          client.should_receive(:update) do |index, attributes, values|
+            values.should_not include(2)
+          end
+          subject.perform('foo_delta')
+        end
+
+        it "should update documents that are in the index" do
+          client.should_receive(:update) do |index, attributes, values|
+            values.keys.should eql(document_ids.reject{|x| x == 2})
+          end
+          subject.perform('foo_delta')
+        end
+      end
+
+      it "should update the specified index" do
+        client.should_receive(:update) do |index, attributes, values|
+          index.should == 'foo_core'
+        end
+        subject.perform('foo_delta')
+      end
+
+      it "should update the sphinx_deleted attribute" do
+        client.should_receive(:update) do |index, attributes, values|
+          attributes.should == ['sphinx_deleted']
+        end
+        subject.perform('foo_delta')
+      end
+
+      it "should set sphinx_deleted for valid documents to true" do
+        client.should_receive(:update) do |index, attributes, values|
+          document_ids.each {|id| values[id].should == [1] }
+        end
+        subject.perform('foo_delta')
+      end
+    end
+  end
+
+  describe '.around_perform_lock1' do
+    before :each do
+      Resque.stub(:encode => 'DeltaJobsAreAwesome')
+      Resque.stub_chain(:redis, :lrem)
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:get_subset_for_processing)
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear_processing)
+    end
+
+    it 'should clear all other delta jobs' do
+      Resque.redis.should_receive(:lrem).with("queue:#{subject.instance_variable_get(:@queue)}", 0, 'DeltaJobsAreAwesome')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+
+    it 'should set up the processing set of document ids' do
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:get_subset_for_processing).with('foo_core')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+
+    it 'should clear the processing set when finished' do
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear_processing).with('foo_core')
+
+      subject.around_perform_lock1('foo_delta') {}
+    end
+  end
+
+  describe '.lock_failed' do
+    it 'should enqueue the delta job again' do
+      Resque.stub(:enqueue => true)
+      Resque.should_receive(:enqueue)
+
+      subject.lock_failed('foo_delta')
+    end
+  end
+end

data/spec/thinking_sphinx/deltas/resque_delta/flag_as_deleted_set_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet do
+  describe '.add' do
+    before :each do
+      Resque.stub_chain(:redis, :sadd => true)
+    end
+
+    it 'should add the document id to the correct set' do
+      Resque.redis.should_receive(:sadd).once.with(subject.set_name('foo_core'), 42)
+      subject.add('foo_core', 42)
+    end
+  end
+
+  describe '.clear!' do
+    before :each do
+      Resque.stub_chain(:redis, :del)
+      ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock)
+    end
+
+    it 'should delete all items in the set' do
+      Resque.redis.should_receive(:del).once.with(subject.set_name('foo_core'))
+      subject.clear!('foo_core')
+    end
+
+    context "with DeltaJob integration" do
+      before :each do
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.stub(:around_perform_lock).and_yield
+      end
+
+      it 'should acquire the DeltaJob lock' do
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.should_receive(:around_perform_lock).once.with('foo_delta')
+        subject.clear!('foo_core')
+      end
+
+      it 'should delete all items in the processing set' do
+        Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
+        subject.clear!('foo_core')
+      end
+    end
+  end
+
+  describe '.clear_all!' do
+    let(:core_indices) { %w[foo_core bar_core] }
+
+    it 'should clear each index' do
+      ThinkingSphinx::Deltas::ResqueDelta::IndexUtils.stub_chain(:core_indices, :each).tap do |s|
+        core_indices.inject(s) do |s, index|
+          s.and_yield(index)
+        end
+      end
+
+      core_indices.each do |index|
+        subject.should_receive(:clear!).with(index)
+      end
+
+      subject.clear_all!
+    end
+  end
+
+  describe '.get_subset_for_processing' do
+    let(:mock_redis) do
+      Resque.redis = mr = MockRedis.new
+      subject.add 'foo_core', 42
+      subject.add 'foo_core', 52
+      subject.add 'foo_core', 100
+      mr
+    end
+
+    before :each do
+      Resque.redis = mock_redis.clone
+    end
+
+    it 'should move all members from the flag as deleted set to the processing set' do
+      subject.get_subset_for_processing('foo_core')
+
+      Resque.redis.scard(subject.set_name('foo_core')).should eql(0)
+      Resque.redis.scard(subject.processing_name('foo_core')).should eql(3)
+    end
+
+    it 'should remove the temp set' do
+      subject.get_subset_for_processing('foo_core')
+
+      Resque.redis.scard(subject.temp_name('foo_core')).should eql(0)
+    end
+
+    it 'should preserve existing members of the processing set' do
+      Resque.redis.sadd(subject.processing_name('foo_core'), 1)
+
+      subject.get_subset_for_processing('foo_core')
+
+      Resque.redis.smembers(subject.processing_name('foo_core')).should == %w[1 100 52 42]
+    end
+  end
+
+  describe '.processing_members' do
+    let(:document_ids) { %w[1, 2, 3] }
+
+    before :each do
+      Resque.stub_chain(:redis, :smembers => document_ids)
+    end
+
+    it 'should get the members of the correct set' do
+      Resque.redis.should_receive(:smembers).once.with(subject.processing_name('foo_core'))
+      subject.processing_members('foo_core')
+    end
+
+    it 'should return a list of integers' do
+      subject.processing_members('foo_core').each do |id|
+        id.class.should == Fixnum
+      end
+    end
+  end
+
+  describe '.clear_processing' do
+    before :each do
+      Resque.stub_chain(:redis, :del)
+    end
+
+    it 'should delete the processing set' do
+      Resque.redis.should_receive(:del).once.with(subject.processing_name('foo_core'))
+
+      subject.clear_processing('foo_core')
+    end
+  end
+end

data/spec/thinking_sphinx/deltas/resque_delta/index_utils_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta::IndexUtils do
+  let(:indices) { %w[foo_core foo_delta foo bar_core bar_delta bar] }
+  let(:config) { double('config') }
+
+  before :each do
+    ThinkingSphinx::Configuration.stub(:instance => config)
+    config.stub(:generate)
+    config.stub_chain(:configuration, :indices, :collect => indices)
+
+    subject.reload!
+  end
+
+  describe '.index_prefixes' do
+    it 'should use a cached value if one exists' do
+      indices = []
+      subject.instance_variable_set(:@prefixes, indices)
+
+      subject.index_prefixes.should be(indices)
+    end
+
+    it 'should return a list of only index prefixes' do
+      subject.index_prefixes.should =~ %w[foo bar]
+    end
+  end
+
+  describe '.core_indices' do
+    it 'should use a cached value if one exists' do
+      indices = []
+      subject.instance_variable_set(:@core_indices, indices)
+
+      subject.core_indices.should be(indices)
+    end
+
+    it 'should return a list of only core indices' do
+      subject.core_indices.should =~ %w[foo_core bar_core]
+    end
+  end
+
+  describe '.delta_indices' do
+    it 'should use a cached value if one exists' do
+      indices = []
+      subject.instance_variable_set(:@delta_indices, indices)
+
+      subject.delta_indices.should be(indices)
+    end
+
+    it 'should return a list of only delta indices' do
+      subject.delta_indices.should =~ %w[foo_delta bar_delta]
+    end
+  end
+
+  describe '.ts_config' do
+    it 'should use a cached value if one exists' do
+      subject.instance_variable_set(:@ts_config, config)
+
+      subject.ts_config.should be(config)
+    end
+
+    it 'should generate the config when fetching the Configuration instance' do
+      config.should_receive(:generate)
+
+      subject.ts_config
+    end
+  end
+end

data/spec/thinking_sphinx/deltas/resque_delta_spec.rb

@@ -0,0 +1,191 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta do
+  before :each do
+    ThinkingSphinx.updates_enabled = true
+    ThinkingSphinx.deltas_enabled  = true
+
+    Resque.redis = MockRedis.new
+  end
+
+  describe '#index' do
+    def flag_as_deleted_document_in_set?
+      Resque.redis.sismember(ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.set_name('foo_core'), 42)
+    end
+
+    subject do
+      ThinkingSphinx::Deltas::ResqueDelta.new(
+        stub('instance'), {}
+      ).tap do |s| 
+        s.stub(:toggled).and_return(true)
+        s.stub(:lock)
+        s.stub(:unlock)
+        s.stub(:locked?).and_return(false)
+      end
+    end
+
+    let(:model) do
+      stub('foo').tap do |m|
+        m.stub(:name => 'foo')
+        m.stub(:source_of_sphinx_index => m)
+        m.stub(:core_index_names  => ['foo_core'])
+        m.stub(:delta_index_names => ['foo_delta'])
+      end
+    end
+
+    let(:instance) do
+      stub('instance').tap do |i|
+        i.stub(:sphinx_document_id => 42)
+      end
+    end
+
+    before :each do
+      Resque.stub(:enqueue => true)
+    end
+
+    context 'updates disabled' do
+      before :each do
+        ThinkingSphinx.updates_enabled = false
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        subject.index(model)
+      end
+
+      it "should not add a flag as deleted document to the set" do
+        subject.index(model, instance)
+        flag_as_deleted_document_in_set?.should be_false
+      end
+    end
+
+    context 'deltas disabled' do
+      before :each do
+        ThinkingSphinx.deltas_enabled = false
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        subject.index(model)
+      end
+
+      it "should not add a flag as deleted document to the set" do
+        subject.index(model, instance)
+        flag_as_deleted_document_in_set?.should be_false
+      end
+    end
+
+    context "instance isn't toggled" do
+      before :each do
+        subject.stub(:toggled => false)
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        subject.index(model, instance)
+      end
+
+      it "should not add a flag as deleted document to the set" do
+        subject.index(model, instance)
+        flag_as_deleted_document_in_set?.should be_false
+      end
+    end
+
+    it "should enqueue a delta job" do
+      Resque.should_receive(:enqueue).once.with(
+        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
+        'foo_delta'
+      )
+      subject.index(model)
+    end
+
+    it "should add the flag as deleted document id to the set" do
+      subject.index(model, instance)
+      flag_as_deleted_document_in_set?.should be_true
+    end
+
+    context "delta index is locked" do
+      before :each do
+        ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(true)
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        subject.index(model, instance)
+      end
+
+      it "should add the flag as deleted document id to the set" do
+        subject.index(model, instance)
+        flag_as_deleted_document_in_set?.should be_true
+      end
+    end
+  end
+
+  describe '.clear_thinking_sphinx_queues' do
+    subject { ThinkingSphinx::Deltas::ResqueDelta.clear_thinking_sphinx_queues }
+
+    before :all do
+      class RandomJob
+        @queue = 'ts_delta'
+      end
+    end
+
+    before :each do
+      Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
+      Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'bar_delta')
+      Resque.enqueue(RandomJob, '1234')
+    end
+
+    it 'should remove all jobs' do
+      subject
+      Resque.size('ts_delta').should eq(0)
+    end
+  end
+
+  describe '.lock' do
+    it 'should set the lock key in redis' do
+      ThinkingSphinx::Deltas::ResqueDelta.lock('foo')
+      Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should eql('true')
+    end
+  end
+
+  describe '.unlock' do
+    it 'should unset the lock key in redis' do
+      Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true')
+      ThinkingSphinx::Deltas::ResqueDelta.unlock('foo')
+      Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should be_nil
+    end
+  end
+
+  describe '.locked?' do
+    subject { ThinkingSphinx::Deltas::ResqueDelta.locked?('foo') }
+
+    context "when lock key in redis is true" do
+      before { Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true') }
+      it { should be_true }
+    end
+
+    context "when lock key in redis is nil" do
+      it { should be_false }
+    end
+  end
+
+  describe '.prepare_for_core_index' do
+    subject { ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index('foo') }
+
+    before :each do
+      Resque.stub(:dequeue)
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear!)
+    end
+
+    it "should call FlagAsDeletedSet.clear!" do
+      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear!).with('foo_core')
+      subject
+    end
+
+    it "should clear delta jobs" do
+      Resque.should_receive(:dequeue).with(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
+      subject
+    end
+  end
+end