ts-resque-delta 1.2.4 → 2.0.0

data/lib/thinking_sphinx/deltas/resque_delta/flag_as_deleted_job.rb

@@ -0,0 +1,7 @@
+class ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob
+  @queue = :ts_delta
+
+  def self.perform(index, document_id)
+    ThinkingSphinx::Deltas::DeleteJob.new(index, document_id).perform
+  end
+end

data/lib/ts-resque-delta.rb

@@ -1,3 +1 @@
 require 'thinking_sphinx/deltas/resque_delta'
-require 'flying_sphinx/resque_delta' if defined?(FlyingSphinx)
-require 'thinking_sphinx/deltas/resque_delta/railtie' if defined?(Rails::Railtie)

data/spec/acceptance/resque_deltas_spec.rb

@@ -0,0 +1,50 @@
+require 'acceptance/spec_helper'
+
+describe 'SQL delta indexing', :live => true do
+  def sleep_for_sphinx
+    sleep ENV['CI'] ? 1.0 : 0.25
+  end
+
+  it "automatically indexes new records" do
+    guards = Book.create(
+      :title => 'Guards! Guards!', :author => 'Terry Pratchett'
+    )
+    index
+
+    Book.search('Terry Pratchett').to_a.should == [guards]
+
+    men = Book.create(
+      :title => 'Men At Arms', :author => 'Terry Pratchett'
+    )
+    work
+    sleep_for_sphinx
+
+    Book.search('Terry Pratchett').to_a.should == [guards, men]
+  end
+
+  it "automatically indexes updated records" do
+    book = Book.create :title => 'Night Watch', :author => 'Harry Pritchett'
+    index
+
+    Book.search('Harry').to_a.should == [book]
+
+    book.reload.update_attributes(:author => 'Terry Pratchett')
+    work
+    sleep_for_sphinx
+
+    Book.search('Terry').to_a.should == [book]
+  end
+
+  it "does not match on old values" do
+    book = Book.create :title => 'Night Watch', :author => 'Harry Pritchett'
+    index
+
+    Book.search('Harry').to_a.should == [book]
+
+    book.reload.update_attributes(:author => 'Terry Pratchett')
+    work
+    sleep_for_sphinx
+
+    Book.search('Harry').should be_empty
+  end
+end

data/spec/acceptance/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+root = File.expand_path File.dirname(__FILE__)
+Dir["#{root}/support/**/*.rb"].each { |file| require file }

data/spec/acceptance/support/database_cleaner.rb

@@ -0,0 +1,11 @@
+RSpec.configure do |config|
+  config.before(:suite) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
+  config.after(:each) do
+    if example.example_group_instance.class.metadata[:live]
+      DatabaseCleaner.clean
+    end
+  end
+end

data/spec/acceptance/support/sphinx_controller.rb

@@ -0,0 +1,61 @@
+class SphinxController
+  def initialize
+    if config.respond_to?(:searchd)
+      config.searchd.mysql41 = 9307
+    else
+      config.port = 9313
+    end
+  end
+
+  def setup
+    if config.respond_to?(:searchd)
+      FileUtils.mkdir_p config.indices_location
+
+      config.render_to_file && index
+
+      ThinkingSphinx::Configuration.reset
+    else
+      FileUtils.mkdir_p config.searchd_file_path
+
+      config.build
+      config.controller.index
+
+      ThinkingSphinx::Configuration.instance.reset
+    end
+
+    ActiveSupport::Dependencies.clear
+
+    if config.respond_to?(:searchd)
+      config.index_paths.each do |path|
+        Dir["#{path}/**/*.rb"].each { |file| $LOADED_FEATURES.delete file }
+      end
+
+      config.searchd.mysql41 = 9307
+      config.settings['quiet_deltas']      = true
+      config.settings['attribute_updates'] = true
+    else
+      config.port = 9313
+      ThinkingSphinx.suppress_delta_output = true
+      ThinkingSphinx.updates_enabled       = true
+      ThinkingSphinx.deltas_enabled        = true
+    end
+  end
+
+  def start
+    config.controller.start
+  end
+
+  def stop
+    config.controller.stop
+  end
+
+  def index(*indices)
+    config.controller.index *indices
+  end
+
+  private
+
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+end

data/spec/acceptance/support/sphinx_helpers.rb

@@ -0,0 +1,36 @@
+module SphinxHelpers
+  def sphinx
+    @sphinx ||= SphinxController.new
+  end
+
+  def index(*indices)
+    yield if block_given?
+
+    ThinkingSphinx.before_index_hooks.each &:call
+    sphinx.index *indices
+    sleep 0.25
+
+    ThinkingSphinx::Connection.pool.clear
+  end
+
+  def work
+    resque_worker = Resque::Worker.new("ts_delta")
+    resque_worker.register_worker
+
+    while job = resque_worker.reserve
+      resque_worker.perform(job)
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include SphinxHelpers
+
+  config.before :all do |group|
+    sphinx.setup && sphinx.start if group.class.metadata[:live]
+  end
+
+  config.after :all do |group|
+    sphinx.stop if group.class.metadata[:live]
+  end
+end

data/spec/internal/.gitignore

@@ -0,0 +1,2 @@
+config/*.sphinx.conf
+db/sphinx/*

data/spec/internal/app/indices/book_index.rb

@@ -0,0 +1,3 @@
+ThinkingSphinx::Index.define :book, :with => :active_record, :delta => ThinkingSphinx::Deltas::ResqueDelta do
+  indexes title, author
+end if ThinkingSphinx::Index.respond_to?(:define)

data/spec/internal/app/models/book.rb

@@ -0,0 +1,7 @@
+class Book < ActiveRecord::Base
+  define_index do
+    indexes title, author
+
+    set_property :delta => ThinkingSphinx::Deltas::ResqueDelta
+  end if respond_to?(:define_index)
+end

data/spec/internal/config/database.yml

@@ -0,0 +1,5 @@
+test:
+  adapter:      <%= ENV['DATABASE'] || 'mysql2' %>
+  database:     thinking_sphinx
+  username:     <%= ENV['DATABASE'] == 'postgresql' ? ENV['USER'] : 'root' %>
+  min_messages: warning

data/spec/internal/db/schema.rb

@@ -0,0 +1,10 @@
+ActiveRecord::Schema.define do
+  create_table(:books, :force => true) do |t|
+    t.string  :title
+    t.string  :author
+    t.integer :year
+    t.string  :blurb_file
+    t.boolean :delta, :default => true, :null => false
+    t.timestamps
+  end
+end

data/spec/internal/log/.gitignore

@@ -0,0 +1 @@
+*.log

data/spec/spec_helper.rb

@@ -1,15 +1,19 @@
-require 'thinking_sphinx'
-require 'thinking_sphinx/deltas/resque_delta'
-require 'flying_sphinx'
-require 'flying_sphinx/resque_delta'
-
-require 'mock_redis'
-require 'fakefs/spec_helpers'
-
-RSpec.configure do |c|
-  c.filter_run :focus => true
-  c.run_all_when_everything_filtered = true
-  c.treat_symbols_as_metadata_keys_with_true_values = true
+require 'rubygems'
+require 'bundler'
+
+Bundler.require :default, :development
+
+require 'thinking_sphinx/railtie'
+
+Combustion.initialize! :active_record
+
+root = File.expand_path File.dirname(__FILE__)
+Dir["#{root}/support/**/*.rb"].each { |file| require file }
+
+RSpec.configure do |config|
+  # enable filtering for examples
+  config.filter_run :wip => nil
+  config.run_all_when_everything_filtered = true
 end
 
 SPEC_BIN_PATH = File.expand_path(File.join(File.dirname(__FILE__), 'bin'))

data/spec/thinking_sphinx/deltas/resque_delta/delta_job_spec.rb

@@ -2,172 +2,28 @@ require 'spec_helper'
 
 describe ThinkingSphinx::Deltas::ResqueDelta::DeltaJob do
   subject do
-    ThinkingSphinx::Deltas::ResqueDelta::DeltaJob.tap do |s|
-      s.stub(:` => true)
-      s.stub(:puts => nil)
-    end
+    ThinkingSphinx::Deltas::ResqueDelta::DeltaJob
   end
 
   describe '.perform' do
-    before :each do
-      ThinkingSphinx.suppress_delta_output = false
-      ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(false)
-      ThinkingSphinx.stub(:sphinx_running? => false)
-    end
-
-    it "should output the delta indexing by default" do
-      subject.should_receive(:puts)
-      subject.perform('foo_delta')
-    end
-
-    it "should not output the delta indexing if requested" do
-      ThinkingSphinx.suppress_delta_output = true
-      subject.should_not_receive(:puts)
-      subject.perform('foo_delta')
-    end
-
-    it "should process just the requested index" do
-      subject.should_receive(:`) do |c|
-        c.should match(/foo_delta/)
-        c.should_not match(/--all/)
-      end
-      subject.perform('foo_delta')
-    end
-
-    context 'when an index is locked' do
-      before do
-        ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?) do |index_name|
-          index_name == 'foo_delta' ? true : false
-        end
-      end
-
-      it "should not start the indexer" do
-        subject.should_not_receive(:`)
-        subject.perform('foo_delta')
-      end
-
-      it "should start the indexer for unlocked indexes" do
-        subject.should_receive(:`)
-        subject.perform('bar_delta')
-      end
-    end
-
-    context 'with flag as deleted document ids' do
-      let(:client) { stub('client', :update => true) }
-      let(:document_ids) { [1, 2, 3] }
-
-      before :each do
-        ThinkingSphinx.updates_enabled = true
-
-        ThinkingSphinx::Configuration.instance.stub(:client => client)
-        ThinkingSphinx.stub(:sphinx_running? => true)
-
-        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:processing_members => document_ids)
-        subject.stub(:filter_flag_as_deleted_ids => document_ids)
-      end
-
-      it 'should get the processing set of flag as deleted document ids' do
-        ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:processing_members).with('foo_core')
-        subject.perform('foo_delta')
-      end
-
-      it "should not update if Sphinx isn't running" do
-        ThinkingSphinx.stub(:sphinx_running? => false)
-        client.should_not_receive(:update)
-        subject.perform('foo_delta')
-      end
+    let(:job) { double 'Job', :perform => true }
 
-      it "should validate the document ids with sphinx" do
-        subject.should_receive(:filter_flag_as_deleted_ids).with(document_ids, 'foo_core')
-
-        subject.perform('foo_delta')
-      end
-
-      context "with invalid ids" do
-        before :each do
-          subject.stub(:filter_flag_as_deleted_ids => document_ids.reject {|x| x == 2} )
-        end
-
-        it "should not update documents that aren't in the index" do
-          client.should_receive(:update) do |index, attributes, values|
-            values.should_not include(2)
-          end
-          subject.perform('foo_delta')
-        end
-
-        it "should update documents that are in the index" do
-          client.should_receive(:update) do |index, attributes, values|
-            values.keys.should eql(document_ids.reject{|x| x == 2})
-          end
-          subject.perform('foo_delta')
-        end
-      end
-
-      it "should update the specified index" do
-        client.should_receive(:update) do |index, attributes, values|
-          index.should == 'foo_core'
-        end
-        subject.perform('foo_delta')
-      end
-
-      it "should update the sphinx_deleted attribute" do
-        client.should_receive(:update) do |index, attributes, values|
-          attributes.should == ['sphinx_deleted']
-        end
-        subject.perform('foo_delta')
-      end
-
-      it "should set sphinx_deleted for valid documents to true" do
-        client.should_receive(:update) do |index, attributes, values|
-          document_ids.each {|id| values[id].should == [1] }
-        end
-        subject.perform('foo_delta')
-      end
-
-      context "without any ids" do
-        let(:document_ids) { [] }
-
-        it "should not validate the ids with sphinx" do
-          subject.should_not_receive(:filter_flag_as_deleted_ids)
-          subject.perform('foo_delta')
-        end
-      end
-    end
-  end
-
-  describe '.around_perform_lock1' do
     before :each do
-      Resque.stub(:encode => 'DeltaJobsAreAwesome')
-      Resque.stub_chain(:redis, :lrem)
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:get_subset_for_processing)
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear_processing)
-    end
-
-    it 'should clear all other delta jobs' do
-      Resque.redis.should_receive(:lrem).with("queue:#{subject.instance_variable_get(:@queue)}", 0, 'DeltaJobsAreAwesome')
-
-      subject.around_perform_lock1('foo_delta') {}
+      ThinkingSphinx::Deltas::ResqueDelta.stub :locked? => false
+      ThinkingSphinx::Deltas::IndexJob.stub :new => job
     end
 
-    it 'should set up the processing set of document ids' do
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:get_subset_for_processing).with('foo_core')
+    it "sets up the internal Thinking Sphinx job with the provided index" do
+      ThinkingSphinx::Deltas::IndexJob.should_receive(:new).with('foo_delta').
+        and_return(job)
 
-      subject.around_perform_lock1('foo_delta') {}
+      subject.perform 'foo_delta'
     end
 
-    it 'should clear the processing set when finished' do
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear_processing).with('foo_core')
-
-      subject.around_perform_lock1('foo_delta') {}
-    end
-  end
-
-  describe '.lock_failed' do
-    it 'should enqueue the delta job again' do
-      Resque.stub(:enqueue => true)
-      Resque.should_receive(:enqueue)
+    it "should execute the internal job" do
+      job.should_receive :perform
 
-      subject.lock_failed('foo_delta')
+      subject.perform 'foo_delta'
     end
   end
 end

data/spec/thinking_sphinx/deltas/resque_delta_spec.rb

@@ -2,127 +2,11 @@ require 'spec_helper'
 
 describe ThinkingSphinx::Deltas::ResqueDelta do
   before :each do
-    ThinkingSphinx.updates_enabled = true
-    ThinkingSphinx.deltas_enabled  = true
-
     Resque.redis = MockRedis.new
   end
 
-  describe '#index' do
-    def flag_as_deleted_document_in_set?
-      Resque.redis.sismember(ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.set_name('foo_core'), 42)
-    end
-
-    subject do
-      ThinkingSphinx::Deltas::ResqueDelta.new(
-        stub('instance'), {}
-      ).tap do |s| 
-        s.stub(:toggled).and_return(true)
-        s.stub(:lock)
-        s.stub(:unlock)
-        s.stub(:locked?).and_return(false)
-      end
-    end
-
-    let(:model) do
-      stub('foo').tap do |m|
-        m.stub(:name => 'foo')
-        m.stub(:source_of_sphinx_index => m)
-        m.stub(:core_index_names  => ['foo_core'])
-        m.stub(:delta_index_names => ['foo_delta'])
-      end
-    end
-
-    let(:instance) do
-      stub('instance').tap do |i|
-        i.stub(:sphinx_document_id => 42)
-      end
-    end
-
-    before :each do
-      Resque.stub(:enqueue => true)
-    end
-
-    context 'updates disabled' do
-      before :each do
-        ThinkingSphinx.updates_enabled = false
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        subject.index(model)
-      end
-
-      it "should not add a flag as deleted document to the set" do
-        subject.index(model, instance)
-        flag_as_deleted_document_in_set?.should be_false
-      end
-    end
-
-    context 'deltas disabled' do
-      before :each do
-        ThinkingSphinx.deltas_enabled = false
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        subject.index(model)
-      end
-
-      it "should not add a flag as deleted document to the set" do
-        subject.index(model, instance)
-        flag_as_deleted_document_in_set?.should be_false
-      end
-    end
-
-    context "instance isn't toggled" do
-      before :each do
-        subject.stub(:toggled => false)
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        subject.index(model, instance)
-      end
-
-      it "should not add a flag as deleted document to the set" do
-        subject.index(model, instance)
-        flag_as_deleted_document_in_set?.should be_false
-      end
-    end
-
-    it "should enqueue a delta job" do
-      Resque.should_receive(:enqueue).once.with(
-        ThinkingSphinx::Deltas::ResqueDelta::DeltaJob,
-        'foo_delta'
-      )
-      subject.index(model)
-    end
-
-    it "should add the flag as deleted document id to the set" do
-      subject.index(model, instance)
-      flag_as_deleted_document_in_set?.should be_true
-    end
-
-    context "delta index is locked" do
-      before :each do
-        ThinkingSphinx::Deltas::ResqueDelta.stub(:locked?).and_return(true)
-      end
-
-      it "should not enqueue a delta job" do
-        Resque.should_not_receive(:enqueue)
-        subject.index(model, instance)
-      end
-
-      it "should add the flag as deleted document id to the set" do
-        subject.index(model, instance)
-        flag_as_deleted_document_in_set?.should be_true
-      end
-    end
-  end
-
-  describe '.clear_thinking_sphinx_queues' do
-    subject { ThinkingSphinx::Deltas::ResqueDelta.clear_thinking_sphinx_queues }
+  describe '.cancel_jobs' do
+    subject { ThinkingSphinx::Deltas::ResqueDelta.cancel_jobs }
 
     before :all do
       class RandomJob
@@ -132,7 +16,7 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
 
     before :each do
       Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
-      Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'bar_delta')
+      Resque.enqueue(ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedJob, 'bar_delta')
       Resque.enqueue(RandomJob, '1234')
     end
 
@@ -141,51 +25,4 @@ describe ThinkingSphinx::Deltas::ResqueDelta do
       Resque.size('ts_delta').should eq(0)
     end
   end
-
-  describe '.lock' do
-    it 'should set the lock key in redis' do
-      ThinkingSphinx::Deltas::ResqueDelta.lock('foo')
-      Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should eql('true')
-    end
-  end
-
-  describe '.unlock' do
-    it 'should unset the lock key in redis' do
-      Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true')
-      ThinkingSphinx::Deltas::ResqueDelta.unlock('foo')
-      Resque.redis.get("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked").should be_nil
-    end
-  end
-
-  describe '.locked?' do
-    subject { ThinkingSphinx::Deltas::ResqueDelta.locked?('foo') }
-
-    context "when lock key in redis is true" do
-      before { Resque.redis.set("#{ThinkingSphinx::Deltas::ResqueDelta.job_prefix}:index:foo:locked", 'true') }
-      it { should be_true }
-    end
-
-    context "when lock key in redis is nil" do
-      it { should be_false }
-    end
-  end
-
-  describe '.prepare_for_core_index' do
-    subject { ThinkingSphinx::Deltas::ResqueDelta.prepare_for_core_index('foo') }
-
-    before :each do
-      Resque.stub(:dequeue)
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.stub(:clear!)
-    end
-
-    it "should call FlagAsDeletedSet.clear!" do
-      ThinkingSphinx::Deltas::ResqueDelta::FlagAsDeletedSet.should_receive(:clear!).with('foo_core')
-      subject
-    end
-
-    it "should clear delta jobs" do
-      Resque.should_receive(:dequeue).with(ThinkingSphinx::Deltas::ResqueDelta::DeltaJob, 'foo_delta')
-      subject
-    end
-  end
 end