ryansch-ts-resque-delta 1.1.5.1

data/lib/thinking_sphinx/deltas/resque_delta/index_utils.rb

@@ -0,0 +1,47 @@
+module ThinkingSphinx::Deltas::ResqueDelta::IndexUtils
+  extend self
+
+  # Public: Return a list of index prefixes (i.e. without "_core"/"_delta").
+  #
+  # Examples
+  #
+  #   sphinx_indices
+  #   # => ['foo', 'bar']
+  #
+  # Returns an Array of index prefixes.
+  def index_prefixes
+    @prefixes ||= indices.reject { |i| i =~ /_(core|delta)$/ }
+  end
+
+  def core_indices
+    @core_indices ||= indices.select { |i| i =~ /_core$/ }
+  end
+
+  def delta_indices
+    @delta_indices ||= indices.select { |i| i =~ /_delta$/ }
+  end
+
+  def reload!
+    @ts_config = @indices = @prefixes = @core_indices = @delta_indices = nil
+  end
+
+  def delta_to_core(delta_name)
+    delta_name.sub(/_delta$/, '_core')
+  end
+
+  def core_to_delta(core_name)
+    core_name.sub(/_core$/, '_delta')
+  end
+
+  def ts_config
+    @ts_config ||= ThinkingSphinx::Configuration.instance.tap do |config|
+      config.generate
+    end
+  end
+
+  private
+  def indices
+    @indices ||= ts_config.configuration.indices.collect { |i| i.name }
+  end
+
+end

data/lib/thinking_sphinx/deltas/resque_delta/railtie.rb

@@ -0,0 +1,8 @@
+require 'rails'
+
+class ThinkingSphinx::Deltas::ResqueDelta::Railtie < Rails::Railtie
+
+  rake_tasks do
+    load File.expand_path('../tasks.rb', __FILE__)
+  end
+end

data/lib/thinking_sphinx/deltas/resque_delta/tasks.rb

@@ -0,0 +1,38 @@
+require 'thinking_sphinx/deltas/resque_delta'
+
+namespace :thinking_sphinx do
+  desc 'Lock all delta indices (Resque will not run indexer or place new jobs on the :ts_delta queue).'
+  task :lock_deltas do
+    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.lock_deltas
+  end
+
+  desc 'Unlock all delta indices.'
+  task :unlock_deltas do
+    ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.unlock_deltas
+  end
+
+  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
+  task :smart_index => :app_env do
+    ret = ThinkingSphinx::Deltas::ResqueDelta::CoreIndex.new.smart_index
+
+    abort("Indexing failed.") if ret != true
+  end
+end
+
+namespace :ts do
+  desc 'Like `rake thinking_sphinx:index`, but locks one index at a time.'
+  task :si => 'thinking_sphinx:smart_index'
+end
+
+unless Rake::Task.task_defined?('thinking_sphinx:index')
+  require 'thinking_sphinx/tasks'
+end
+
+# Ensure that indexing does not conflict with ts-resque-delta delta jobs.
+Rake::Task['thinking_sphinx:index'].enhance ['thinking_sphinx:lock_deltas'] do
+  Rake::Task['thinking_sphinx:unlock_deltas'].invoke
+end
+
+Rake::Task['thinking_sphinx:reindex'].enhance ['thinking_sphinx:lock_deltas'] do
+  Rake::Task['thinking_sphinx:unlock_deltas'].invoke
+end

data/lib/thinking_sphinx/deltas/resque_delta/version.rb

@@ -0,0 +1,7 @@
+module ThinkingSphinx
+  module Deltas
+    class ResqueDeltaInfo
+      VERSION = "1.1.5.1"
+    end
+  end
+end

data/lib/ts-resque-delta.rb

@@ -0,0 +1,2 @@
+require 'thinking_sphinx/deltas/resque_delta'
+require 'thinking_sphinx/deltas/resque_delta/railtie' if defined?(Rails::Railtie)

data/spec/flying_sphinx/resque_delta/delta_job_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe FlyingSphinx::ResqueDelta::DeltaJob do
+  describe '@queue' do
+    it "uses the fs_delta queue" do
+      FlyingSphinx::ResqueDelta::DeltaJob.instance_variable_get(:@queue).
+        should == :fs_delta
+    end
+  end
+  
+  describe '.perform' do
+    it "doesn't create an index request when skipping" do
+      FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => true)
+      
+      FlyingSphinx::IndexRequest.should_not_receive(:new)
+      
+      FlyingSphinx::ResqueDelta::DeltaJob.perform ['foo_delta']
+    end
+    
+    it "performs an index request when not skipping" do
+      request = double('index request', :perform => true)
+      FlyingSphinx::ResqueDelta::DeltaJob.stub!(:skip? => false)
+      
+      FlyingSphinx::IndexRequest.should_receive(:new).
+        with(['foo_delta']).
+        and_return(request)
+      request.should_receive(:perform)
+      
+      FlyingSphinx::ResqueDelta::DeltaJob.perform ['foo_delta']
+    end
+  end
+end

data/spec/flying_sphinx/resque_delta/flag_as_deleted_job_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe FlyingSphinx::ResqueDelta::FlagAsDeletedJob do
+  describe '@queue' do
+    it "uses the fs_delta queue" do
+      FlyingSphinx::ResqueDelta::FlagAsDeletedJob.
+        instance_variable_get(:@queue).should == :fs_delta
+    end
+  end
+  
+  describe '.perform' do
+    it "performs a flag-as-deleted job" do
+      job = double('flag as deleted job', :perform => true)
+      
+      FlyingSphinx::FlagAsDeletedJob.should_receive(:new).
+        with(['foo_core'], 5).
+        and_return(job)
+      job.should_receive(:perform)
+      
+      FlyingSphinx::ResqueDelta::FlagAsDeletedJob.perform ['foo_core'], 5
+    end
+  end
+end

data/spec/flying_sphinx/resque_delta_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+
+describe FlyingSphinx::ResqueDelta do
+  describe '.job_types' do
+    it "contains just the Flying Sphinx delta and delete jobs" do
+      FlyingSphinx::ResqueDelta.job_types.should == [
+        FlyingSphinx::ResqueDelta::DeltaJob,
+        FlyingSphinx::ResqueDelta::FlagAsDeletedJob
+      ]
+    end
+  end
+  
+  describe '.job_prefix' do
+    it "is fs-delta" do
+      FlyingSphinx::ResqueDelta.job_prefix.should == 'fs-delta'
+    end
+  end
+  
+  describe '#index' do
+    before :each do
+      ThinkingSphinx.updates_enabled = true
+      ThinkingSphinx.deltas_enabled  = true
+
+      Resque.stub(:enqueue => true)
+
+      @delayed_delta = FlyingSphinx::ResqueDelta.new(
+        stub('instance'), {}
+      )
+      @delayed_delta.stub(:toggled).and_return(true)
+
+      FlyingSphinx::ResqueDelta.stub(:lock)
+      FlyingSphinx::ResqueDelta.stub(:unlock)
+      FlyingSphinx::ResqueDelta.stub(:locked?).and_return(false)
+
+      @model = stub('foo')
+      @model.stub(:core_index_names  => ['foo_core'])
+      @model.stub(:delta_index_names => ['foo_delta'])
+
+      @instance = stub('instance')
+      @instance.stub(:sphinx_document_id => 42)
+    end
+
+    context 'updates disabled' do
+      before :each do
+        ThinkingSphinx.updates_enabled = false
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model)
+      end
+
+      it "should not enqueue a flag as deleted job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model)
+      end
+    end
+
+    context 'deltas disabled' do
+      before :each do
+        ThinkingSphinx.deltas_enabled = false
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model)
+      end
+
+      it "should not enqueue a flag as deleted job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model)
+      end
+    end
+
+    context "instance isn't toggled" do
+      before :each do
+        @delayed_delta.stub(:toggled => false)
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model, @instance)
+      end
+
+      it "should not enqueue a flag as deleted job" do
+        Resque.should_not_receive(:enqueue)
+        @delayed_delta.index(@model, @instance)
+      end
+    end
+
+    it "should enqueue a delta job" do
+      Resque.should_receive(:enqueue).at_least(:once).with(
+        FlyingSphinx::ResqueDelta::DeltaJob,
+        ['foo_delta']
+      )
+      @delayed_delta.index(@model)
+    end
+
+    it "should enqueue a flag-as-deleted job" do
+      Resque.should_receive(:enqueue).at_least(:once).with(
+        FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
+        ['foo_core'],
+        42
+      )
+      @delayed_delta.index(@model, @instance)
+    end
+
+    context "delta index is locked" do
+      before :each do
+        FlyingSphinx::ResqueDelta.stub(:locked?).and_return(true)
+      end
+
+      it "should not enqueue a delta job" do
+        Resque.should_not_receive(:enqueue).with(
+          FlyingSphinx::ResqueDelta::DeltaJob,
+          ['foo_delta']
+        )
+        @delayed_delta.index(@model, @instance)
+      end
+
+      it "should enqueue a flag-as-deleted job" do
+        Resque.should_receive(:enqueue).at_least(:once).with(
+          FlyingSphinx::ResqueDelta::FlagAsDeletedJob,
+          ['foo_core'],
+          42
+        )
+        @delayed_delta.index(@model, @instance)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'thinking_sphinx'
+require 'thinking_sphinx/deltas/resque_delta'
+require 'flying_sphinx'
+require 'flying_sphinx/resque_delta'
+
+require 'mock_redis'
+require 'fakefs/spec_helpers'
+
+RSpec.configure do |c|
+  c.filter_run :focus => true
+  c.run_all_when_everything_filtered = true
+  c.treat_symbols_as_metadata_keys_with_true_values = true
+end

data/spec/thinking_sphinx/deltas/resque_delta/core_index_spec.rb

@@ -0,0 +1,208 @@
+require 'spec_helper'
+
+describe ThinkingSphinx::Deltas::ResqueDelta::CoreIndex do
+  let(:indices) { %w[foo bar] }
+  let(:config) { double('config') }
+
+  describe '#lock_delta' do
+    it 'should lock the delta' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock)
+
+      subject.lock_delta('foo')
+    end
+
+    it 'should lock the delta for the given index' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:lock).with('foo_delta')
+
+      subject.lock_delta('foo')
+    end
+  end
+
+  describe '#unlock_delta' do
+    it 'should unlock the delta' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock)
+
+      subject.unlock_delta('foo')
+    end
+
+    it 'should unlock the delta for the given index' do
+      ThinkingSphinx::Deltas::ResqueDelta.should_receive(:unlock).with('foo_delta')
+
+      subject.unlock_delta('foo')
+    end
+  end
+
+  describe '#lock_deltas' do
+    it 'should lock all delta indices' do
+      subject.stub(:sphinx_indices => indices)
+
+      indices.each {|index| subject.should_receive(:lock_delta).once.with(index) }
+
+      subject.lock_deltas
+    end
+  end
+
+  describe '#unlock_deltas' do
+    it 'should unlock all delta indices' do
+      subject.stub(:sphinx_indices => indices)
+
+      indices.each {|index| subject.should_receive(:unlock_delta).once.with(index) }
+
+      subject.unlock_deltas
+    end
+  end
+
+  describe '#with_delta_index_lock' do
+    before :each do
+      subject.stub(:lock_delta)
+      subject.stub(:unlock_delta)
+
+      subject.stub(:block_called)
+
+      @block_called = false
+      @block = lambda { @block_called = true; subject.block_called }
+    end
+
+    it 'should yield' do
+      subject.with_delta_index_lock('foo', &@block)
+      @block_called.should be_true
+    end
+
+    it 'should lock before yielding' do
+      subject.should_receive(:lock_delta).with('foo').ordered
+      subject.should_receive(:block_called).ordered
+
+      subject.with_delta_index_lock('foo', &@block)
+    end
+
+    it 'should unlock after yielding' do
+      subject.should_receive(:block_called).ordered
+      subject.should_receive(:unlock_delta).with('foo').ordered
+
+      subject.with_delta_index_lock('foo', &@block)
+    end
+  end
+
+  describe '#smart_index' do
+    include FakeFS::SpecHelpers
+
+    let(:test_path) { '/tmp/ts-resque-delta/foo' }
+
+    before :each do
+      subject.stub(:ts_config => config)
+      config.stub(:config_file => 'foo_config')
+      config.stub(:build)
+      config.stub(:searchd_file_path => test_path)
+      config.stub_chain(:controller, :index) do
+        # Set $? to 0
+        `/usr/bin/true`
+      end
+
+      # Silence Generating config message
+      subject.stub(:puts)
+
+      subject.stub(:index_prefixes => indices)
+      subject.stub(:lock_delta)
+      subject.stub(:unlock_delta)
+
+      ThinkingSphinx::Deltas::ResqueDelta.stub(:prepare_for_core_index)
+      Resque.stub(:enqueue)
+    end
+
+    it 'should not generate sphinx configuration if INDEX_ONLY is true' do
+      ENV.stub(:[]).with('INDEX_ONLY').and_return('true')
+      config.should_not_receive(:build)
+
+      subject.smart_index
+    end
+
+    it 'should generate sphinx configuration if INDEX_ONLY is not true' do
+      ENV.stub(:[]).with('INDEX_ONLY').and_return(nil)
+      config.should_receive(:build).once
+
+      subject.smart_index
+    end
+
+    it 'should create the sphinx file directory' do
+      subject.smart_index
+
+      File.directory?(test_path).should be_true
+    end
+
+    it 'should index all core indices' do
+      indices.each do |index|
+        config.controller.should_receive(:index).with("#{index}_core", anything)
+      end
+
+      subject.smart_index
+    end
+
+    context "with delta lock" do
+      before :each do
+        subject.stub(:with_delta_index_lock).and_yield
+      end
+
+      it 'should index' do
+        config.controller.should_receive(:index)
+        subject.smart_index
+      end
+
+      it 'should signal resque delta to prepare for the core index' do
+        ThinkingSphinx::Deltas::ResqueDelta.should_receive(:prepare_for_core_index)
+        subject.smart_index
+      end
+    end
+
+    context "without delta lock" do
+      before :each do
+        subject.stub(:with_delta_index_lock)
+      end
+
+      it 'should not index without the delta lock' do
+        config.controller.should_not_receive(:index)
+        subject.smart_index
+      end
+
+      it 'should not signal resque delta to prepare for the core index' do
+        ThinkingSphinx::Deltas::ResqueDelta.should_not_receive(:prepare_for_core_index)
+        subject.smart_index
+      end
+    end
+
+    it 'should create a delta job after the delta is unlocked' do
+      # Create a dummy method on subject that's called when Resque.enqueue is called so we can enforce order.
+      subject.stub(:resque_called)
+      Resque.stub(:enqueue) { subject.resque_called }
+
+      Resque.should_receive(:enqueue)
+
+      subject.should_receive(:with_delta_index_lock).ordered.exactly(indices.size)
+      subject.should_receive(:resque_called).ordered.exactly(indices.size)
+
+      subject.smart_index
+    end
+
+    context 'with an error' do
+      before :each do
+        config.stub_chain(:controller, :index) do
+          # Set $? to 1
+          `/usr/bin/false`
+        end
+      end
+
+      it 'should stop processing indexes after an error' do
+        config.controller.should_receive(:index).once
+
+        subject.smart_index
+      end
+
+      it 'should return false on failure' do
+        subject.smart_index.should be_false
+      end
+    end
+
+    it 'should return true on success' do
+      subject.smart_index.should be_true
+    end
+  end
+end