sunspot_index_queue 1.0.0

data/lib/sunspot/index_queue/session_proxy.rb

@@ -0,0 +1,111 @@
+require 'sunspot/session_proxy/abstract_session_proxy'
+
+module Sunspot
+  class IndexQueue
+    # This is a Sunspot::SessionProxy that works with the IndexQueue class. Most update requests will
+    # be added to the queue and be processed asynchronously. The exceptions are the +remove+ method with
+    # a block and the +remove_all+ method. These will send their commands directly to Solr since the queue
+    # cannot handle delete by query. You should avoid calling these methods 
+    class SessionProxy < Sunspot::SessionProxy::AbstractSessionProxy
+      attr_reader :queue, :session
+      
+      delegate :new_search, :search, :new_more_like_this, :more_like_this, :config, :to => :session
+      
+      # Create a new session proxy for a particular queue (default to a queue for all classes bound to the
+      # default session configuration). You can specify the session argument if the session used for queries should be
+      # different than the one the queue is bound to.
+      def initialize (queue = nil, session = nil)
+        @queue = queue || IndexQueue.new
+        @session = session || @queue.session
+      end
+      
+      # Does nothing in this implementation.
+      def batch
+        yield if block_given?
+      end
+      
+      # Does nothing in this implementation.
+      def commit
+        # no op
+      end
+      
+      # Does nothing in this implementation.
+      def commit_if_delete_dirty
+        # no op
+      end
+
+      # Does nothing in this implementation.
+      def commit_if_dirty
+        # no op
+      end
+      
+      # Always returns false in this implementation.
+      def delete_dirty?
+        false
+      end
+      
+      # Always returns false in this implementation.
+      def dirty?
+        false
+      end
+      
+      # Queues up the index operation for later.
+      def index (*objects)
+        objects.flatten.each do |object|
+          queue.index(object)
+        end
+      end
+      
+      # Queues up the index operation for later.
+      def index! (*objects)
+        index(*objects)
+      end
+      
+      # Queues up the remove operation for later unless a block is passed. In that case it will
+      # be performed immediately.
+      def remove (*objects, &block)
+        if block
+          # Delete by query not supported by queue, so send to server
+          queue.session.remove(*objects, &block)
+        else
+          objects.flatten.each do |object|
+            queue.remove(object)
+          end
+        end
+      end
+      
+      # Queues up the remove operation for later unless a block is passed. In that case it will
+      # be performed immediately.
+      def remove! (*objects, &block)
+        if block
+          # Delete by query not supported by queue, so send to server
+          queue.session.remove!(*objects, &block)
+        else
+          remove(*objects)
+        end
+      end
+      
+      # Proxies remove_all to the queue session.
+      def remove_all (*classes)
+        # Delete by query not supported by queue, so send to server
+        queue.session.remove_all(*classes)
+      end
+      
+      # Proxies remove_all! to the queue session.
+      def remove_all! (*classes)
+        # Delete by query not supported by queue, so send to server
+        queue.session.remove_all!(*classes)
+      end
+      
+      # Queues up the index operation for later.
+      def remove_by_id (clazz, id)
+        queue.remove(:class => clazz, :id => id)
+      end
+      
+      # Queues up the index operation for later.
+      def remove_by_id! (clazz, id)
+        remove_by_id(clazz, id)
+      end
+    end
+  end
+end

data/lib/sunspot_index_queue.rb

@@ -0,0 +1,5 @@
+require 'sunspot'
+
+module Sunspot
+  autoload :IndexQueue, File.expand_path('../sunspot/index_queue', __FILE__)
+end

data/spec/active_record_impl_spec.rb

@@ -0,0 +1,44 @@
+require File.expand_path('../spec_helper', __FILE__)
+require 'sqlite3'
+require File.expand_path('../entry_impl_examples', __FILE__)
+
+describe Sunspot::IndexQueue::Entry::ActiveRecordImpl do
+
+  before :all do
+    db_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp'))
+    Dir.mkdir(db_dir) unless File.exist?(db_dir)
+    db = File.join(db_dir, 'sunspot_index_queue_test.sqlite3')
+    ActiveRecord::Base.establish_connection("adapter" => "sqlite3", "database" => db)
+    Sunspot::IndexQueue::Entry.implementation = :active_record
+    Sunspot::IndexQueue::Entry::ActiveRecordImpl.create_table
+  end
+  
+  after :all do
+    db_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp'))
+    db = File.join(db_dir, 'sunspot_index_queue_test.sqlite3')
+    ActiveRecord::Base.connection.disconnect!
+    File.delete(db) if File.exist?(db)
+    Dir.delete(db_dir) if File.exist?(db_dir) and Dir.entries(db_dir).reject{|f| f.match(/^\.+$/)}.empty?
+    Sunspot::IndexQueue::Entry.implementation = nil
+  end
+  
+  let(:factory) do
+    factory = Object.new
+    def factory.create (attributes)
+      Sunspot::IndexQueue::Entry::ActiveRecordImpl.create!(attributes)
+    end
+    
+    def factory.delete_all
+      Sunspot::IndexQueue::Entry::ActiveRecordImpl.delete_all
+    end
+    
+    def factory.find (id)
+      Sunspot::IndexQueue::Entry::ActiveRecordImpl.find_by_id(id)
+    end
+    
+    factory
+  end
+    
+  it_should_behave_like "Entry implementation"
+
+end

data/spec/batch_spec.rb

@@ -0,0 +1,118 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Sunspot::IndexQueue::Batch do
+  
+  before :all do
+    Sunspot::IndexQueue::Entry.implementation = :mock
+  end
+  
+  after :all do
+    Sunspot::IndexQueue::Entry.implementation = nil
+  end
+  
+  subject { Sunspot::IndexQueue::Batch.new(queue, [entry_1, entry_2]) }
+  let(:entry_1) { Sunspot::IndexQueue::Entry::MockImpl.new(:record => record_1, :delete => false) }
+  let(:entry_2) { Sunspot::IndexQueue::Entry::MockImpl.new(:record => record_2, :delete => true) }
+  let(:record_1) { Sunspot::IndexQueue::Test::Searchable.new(1) }
+  let(:record_2) { Sunspot::IndexQueue::Test::Searchable.new(2) }
+  let(:queue) { Sunspot::IndexQueue.new(:session => session) }
+  let(:session) { Sunspot::Session.new }
+  
+  it "should submit all entries in a batch to Solr and commit them" do
+    entry_1.stub!(:record).and_return(record_1)
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).with(record_1)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id)
+    session.should_receive(:commit)
+    Sunspot::IndexQueue::Entry.implementation.should_receive(:delete_entries).with([entry_1, entry_2])
+    subject.submit!
+    entry_1.processed?.should == true
+    entry_2.processed?.should == true
+  end
+  
+  it "should submit all entries individually and commit them if the batch errors out" do
+    entry_1.stub!(:record).and_return(record_1)
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).with(record_1).twice
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id).twice
+    session.should_receive(:commit).and_raise("boom")
+    session.should_receive(:commit)
+    Sunspot::IndexQueue::Entry.implementation.should_receive(:delete_entries).with([entry_1, entry_2])
+    subject.submit!
+    entry_1.processed?.should == true
+    entry_2.processed?.should == true
+  end
+  
+  
+  it "should add error messages to each entry that errors out" do
+    entry_1.stub!(:record).and_return(record_1)
+    error = StandardError.new("boom")
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).and_raise(error)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id)
+    session.should_receive(:commit)
+    entry_1.should_receive(:set_error!).with(error, queue.retry_interval)
+    Sunspot::IndexQueue::Entry.implementation.should_receive(:delete_entries).with([entry_2])
+    subject.submit!
+    entry_1.processed?.should == false
+    entry_2.processed?.should == true
+  end
+  
+  it "should add error messages to all entries when a commit fails" do
+    entry_1.stub!(:record).and_return(record_1)
+    error = StandardError.new("boom")
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).with(record_1).twice
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id).twice
+    session.should_receive(:commit).twice.and_raise(error)
+    Sunspot::IndexQueue::Entry.implementation.should_not_receive(:delete_entries)
+    entry_1.should_receive(:set_error!).with(error, queue.retry_interval)
+    entry_2.should_receive(:set_error!).with(error, queue.retry_interval)
+    subject.submit!
+    entry_1.processed?.should == false
+    entry_2.processed?.should == false
+  end
+  
+  it "should silently ignore entries that no longer have a record" do
+    entry_1.stub!(:record).and_return(nil)
+    session.should_receive(:batch).and_yield
+    session.should_not_receive(:index)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id)
+    session.should_receive(:commit)
+    Sunspot::IndexQueue::Entry.implementation.should_receive(:delete_entries).with([entry_1, entry_2])
+    subject.submit!
+    entry_1.processed?.should == true
+    entry_2.processed?.should == true
+  end
+  
+  it "should raise an error and reset entries if the Solr server is not responding for an entry" do
+    entry_1.stub!(:record).and_return(record_1)
+    error = Errno::ECONNREFUSED.new
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).with(record_1)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id).and_raise(error)
+    session.should_not_receive(:commit)
+    Sunspot::IndexQueue::Entry.implementation.should_not_receive(:delete_entries)
+    entry_1.should_receive(:reset!)
+    entry_2.should_receive(:reset!)
+    lambda{subject.submit!}.should raise_error(Sunspot::IndexQueue::SolrNotResponding)
+    entry_1.processed?.should == false
+    entry_2.processed?.should == false
+  end
+  
+  it "should raise an error and reset entries if the Solr server is not responding on a commit" do
+    entry_1.stub!(:record).and_return(record_1)
+    error = Errno::ECONNREFUSED.new
+    session.should_receive(:batch).and_yield
+    session.should_receive(:index).with(record_1)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id)
+    session.should_receive(:commit).and_raise(error)
+    Sunspot::IndexQueue::Entry.implementation.should_not_receive(:delete_entries)
+    entry_1.should_receive(:reset!)
+    entry_2.should_receive(:reset!)
+    lambda{subject.submit!}.should raise_error(Sunspot::IndexQueue::SolrNotResponding)
+    entry_1.processed?.should == false
+    entry_2.processed?.should == false
+  end
+  
+end

data/spec/data_mapper_impl_spec.rb

@@ -0,0 +1,37 @@
+require File.expand_path('../spec_helper', __FILE__)
+require 'sqlite3'
+require  'dm-migrations'
+require File.expand_path('../entry_impl_examples', __FILE__)
+
+describe Sunspot::IndexQueue::Entry::DataMapperImpl do
+
+  before :all do
+    DataMapper.setup(:default, 'sqlite::memory:')
+    Sunspot::IndexQueue::Entry.implementation = :data_mapper
+    Sunspot::IndexQueue::Entry::DataMapperImpl.auto_migrate!
+  end
+  
+  after :all do
+    Sunspot::IndexQueue::Entry.implementation = nil
+  end
+  
+  let(:factory) do
+    factory = Object.new
+    def factory.create (attributes)
+      Sunspot::IndexQueue::Entry::DataMapperImpl.create!(attributes)
+    end
+    
+    def factory.delete_all
+      Sunspot::IndexQueue::Entry::DataMapperImpl.all.destroy!
+    end
+    
+    def factory.find (id)
+      Sunspot::IndexQueue::Entry::DataMapperImpl.get(id)
+    end
+    
+    factory
+  end
+    
+  it_should_behave_like "Entry implementation"
+
+end

data/spec/entry_impl_examples.rb

@@ -0,0 +1,184 @@
+require 'spec_helper'
+
+# Shared examples for Entry implementations. In order to use these examples, the example group should define
+# a block for :factory that will create an entry when yielded to with a hash of attributes..
+shared_examples_for "Entry implementation" do
+
+  after :each do
+    factory.delete_all
+  end
+  
+  context "class methods" do
+    before :each do
+      test_class = "Sunspot::IndexQueue::Test::Searchable"
+      @entry_1 = factory.create('record_class_name' => test_class, 'record_id' => '1', 'is_delete' => false, 'priority' => 0, 'run_at' => Time.now.utc)
+      @entry_2 = factory.create('record_class_name' => test_class, 'record_id' => '2', 'is_delete' => false, 'priority' => 10, 'run_at' => Time.now.utc, 'error' => "boom!", 'attempts' => 1)
+      @entry_3 = factory.create('record_class_name' => "Object", 'record_id' => '3', 'is_delete' => false, 'priority' => 5, 'run_at' => Time.now.utc, 'error' => "boom!", 'attempts' => 1)
+      @entry_4 = factory.create('record_class_name' => test_class, 'record_id' => '4', 'is_delete' => true, 'priority' => 0, 'run_at' => Time.now.utc + 60)
+      @entry_5 = factory.create('record_class_name' => test_class, 'record_id' => '5', 'is_delete' => false, 'priority' => -10, 'run_at' => Time.now.utc - 60)
+      @entry_6 = factory.create('record_class_name' => test_class, 'record_id' => '6', 'is_delete' => false, 'priority' => 0, 'run_at' => Time.now.utc - 3600)
+      @entry_7 = factory.create('record_class_name' => test_class, 'record_id' => '7', 'is_delete' => true, 'priority' => 10, 'run_at' => Time.now.utc - 60)
+    end
+    
+    context "without class_names filter" do
+      let(:queue) { Sunspot::IndexQueue.new(:batch_size => 3, :retry_interval => 5)}
+
+      it "should get the total_count" do
+        Sunspot::IndexQueue::Entry.implementation.total_count(queue).should == 7
+      end
+
+      it "should get the ready_count" do
+        Sunspot::IndexQueue::Entry.implementation.ready_count(queue).should == 6
+      end
+
+      it "should get the error_count" do
+        Sunspot::IndexQueue::Entry.implementation.error_count(queue).should == 2
+      end
+
+      it "should get the errors" do
+        errors = Sunspot::IndexQueue::Entry.implementation.errors(queue, 2, 0)
+        errors.collect{|e| e.record_id}.sort.should == [@entry_2.record_id, @entry_3.record_id]
+
+        errors = Sunspot::IndexQueue::Entry.implementation.errors(queue, 1, 1)
+        errors.collect{|e| e.record_id}.sort.should == [@entry_3.record_id]
+      end
+
+      it "should reset all entries" do
+        Sunspot::IndexQueue::Entry.implementation.reset!(queue)
+        Sunspot::IndexQueue::Entry.implementation.error_count(queue).should == 0
+        @entry_2 = factory.find(@entry_2.id)
+        @entry_2.error.should == nil
+        @entry_2.attempts.should == 0
+
+        @entry_3 = factory.find(@entry_3.id)
+        @entry_3.error.should == nil
+        @entry_3.attempts.should == 0
+      end
+
+      it "should get the next_batch! by index time and priority" do
+        batch = Sunspot::IndexQueue::Entry.implementation.next_batch!(queue)
+        batch.collect{|e| e.record_id}.sort.should == [@entry_2.record_id, @entry_3.record_id, @entry_7.record_id]
+        batch = Sunspot::IndexQueue::Entry.implementation.next_batch!(queue)
+        batch.collect{|e| e.record_id}.sort.should == [@entry_1.record_id, @entry_5.record_id, @entry_6.record_id]
+      end
+    end
+    
+    context "with class_names filter" do
+      let(:queue) { Sunspot::IndexQueue.new(:batch_size => 3, :retry_interval => 5, :class_names => "Sunspot::IndexQueue::Test::Searchable")}
+  
+      it "should get the total_count" do
+        Sunspot::IndexQueue::Entry.implementation.total_count(queue).should == 6
+      end
+  
+      it "should get the ready_count" do
+        Sunspot::IndexQueue::Entry.implementation.ready_count(queue).should == 5
+      end
+  
+      it "should get the error_count" do
+        Sunspot::IndexQueue::Entry.implementation.error_count(queue).should == 1
+      end
+  
+      it "should get the errors" do
+        errors = Sunspot::IndexQueue::Entry.implementation.errors(queue, 2, 0)
+        errors.collect{|e| e.record_id}.sort.should == [@entry_2.record_id]
+        errors = Sunspot::IndexQueue::Entry.implementation.errors(queue, 1, 1)
+        errors.collect{|e| e.record_id}.sort.should == []
+      end
+    
+      it "should reset all entries" do
+        Sunspot::IndexQueue::Entry.implementation.reset!(queue)
+        Sunspot::IndexQueue::Entry.implementation.error_count(queue).should == 0
+        @entry_2 = factory.find(@entry_2.id)
+        @entry_2.error.should == nil
+        @entry_2.attempts.should == 0
+    
+        @entry_3 = factory.find(@entry_3.id)
+        @entry_3.error.should_not == nil
+        @entry_3.attempts.should == 1
+      end
+  
+      it "should get the next_batch! by index time and priority" do
+        batch = Sunspot::IndexQueue::Entry.implementation.next_batch!(queue)
+        batch.collect{|e| e.record_id}.sort.should == [@entry_2.record_id, @entry_6.record_id, @entry_7.record_id]
+        batch = Sunspot::IndexQueue::Entry.implementation.next_batch!(queue)
+        batch.collect{|e| e.record_id}.sort.should == [@entry_1.record_id, @entry_5.record_id]
+      end
+    end
+    
+    context "add and remove" do
+      it "should add an entry" do
+        Sunspot::IndexQueue::Entry.implementation.add(Sunspot::IndexQueue::Test::Searchable, "10", false, 100)
+        entry = Sunspot::IndexQueue::Entry.implementation.next_batch!(Sunspot::IndexQueue.new).detect{|e| e.priority == 100}
+        entry.record_class_name.should == "Sunspot::IndexQueue::Test::Searchable"
+        entry.record_id.should == "10"
+        entry.is_delete?.should == false
+        entry.priority.should == 100
+      end
+  
+      it "should delete a list of entry ids" do
+        Sunspot::IndexQueue::Entry.implementation.delete_entries([@entry_1.id, @entry_2.id])
+        factory.find(@entry_1.id).should == nil
+        factory.find(@entry_2.id).should == nil
+        factory.find(@entry_4.id).id.should == @entry_4.id
+      end
+      
+      it "should not add multiple entries unless a row is being processed" do
+        Sunspot::IndexQueue::Entry.implementation.add(Sunspot::IndexQueue::Test::Searchable, "10", false, 80)
+        Sunspot::IndexQueue::Entry.implementation.next_batch!(Sunspot::IndexQueue.new)
+        Sunspot::IndexQueue::Entry.implementation.add(Sunspot::IndexQueue::Test::Searchable, "10", false, 100)
+        Sunspot::IndexQueue::Entry.implementation.add(Sunspot::IndexQueue::Test::Searchable, "10", false, 110)
+        Sunspot::IndexQueue::Entry.implementation.add(Sunspot::IndexQueue::Test::Searchable, "10", true, 90)
+        Sunspot::IndexQueue::Entry.implementation.reset!(Sunspot::IndexQueue.new)
+        entries = Sunspot::IndexQueue::Entry.implementation.next_batch!(Sunspot::IndexQueue.new)
+        entries.detect{|e| e.priority == 80}.record_id.should == "10"
+        entries.detect{|e| e.priority == 100}.should == nil
+        entries.detect{|e| e.priority == 90}.should == nil
+        entry = entries.detect{|e| e.priority == 110}
+        entry.is_delete?.should == true
+      end
+    end
+  end
+
+  context "instance methods" do
+  
+    it "should get the record_class_name" do
+      entry = Sunspot::IndexQueue::Entry.implementation.new('record_class_name' => "Test")
+      entry.record_class_name.should == "Test"
+    end
+  
+    it "should get the record_id" do
+      entry = Sunspot::IndexQueue::Entry.implementation.new('record_id' => "1")
+      entry.record_id.should == "1"
+    end
+  
+    it "should determine if the entry is an delete" do
+      entry = Sunspot::IndexQueue::Entry.implementation.new('is_delete' => false)
+      entry.is_delete?.should == false
+      entry = Sunspot::IndexQueue::Entry.implementation.new('is_delete' => true)
+      entry.is_delete?.should == true
+    end
+    
+    it "should reset an entry to be indexed immediately" do
+      entry = factory.create('record_class_name' => "Test", 'record_id' => 1, 'is_delete' => false, 'priority' => 10, 'run_at' => Time.now.utc + 600, 'error' => "boom!", 'attempts' => 2)
+      queue = Sunspot::IndexQueue.new
+      queue.error_count.should == 1
+      queue.ready_count.should == 0
+      entry.reset!
+      queue.error_count.should == 0
+      queue.ready_count.should == 1
+      factory.find(entry.id).attempts.should == 0
+    end
+  
+    it "should set the error on an entry" do
+      entry = factory.create('record_class_name' => "Test", 'record_id' => 1, 'is_delete' => false, 'priority' => 10, 'run_at' => Time.now.utc + 600, 'attempts' => 1)
+      error = ArgumentError.new("boom")
+      error.stub!(:backtrace).and_return(["line 1", "line 2"])
+      entry.set_error!(error)
+      entry = factory.find(entry.id)
+      entry.error.should include("ArgumentError")
+      entry.error.should include("boom")
+      entry.error.should include("line 1")
+      entry.error.should include("line 2")
+    end
+  end
+end