sunspot_index_queue 1.1.0 → 1.1.1

data/CHANGE_LOG.txt

@@ -1,3 +1,9 @@
+1.1.1
+
+- Fix bug where entries were lost if they failed to be submitted individually after they failed to be submitted in a batch.
+
+- Fix compatibility with latest mongo gem.
+
 1.1.0
 
 - Change column type for ActiveRecord and DataMapper implementation queue table for record_id from String to Integer. This makes it far more efficient in the normal case at the expense of being less flexible. If you do end up needing String record id's, just create the table definition yourself. To upgrade, you should clear the queue and then rerun the migration.

data/Rakefile

@@ -6,18 +6,17 @@ desc 'Default: run unit tests.'
 task :default => :test
 
 begin
-  require 'spec/rake/spectask'
-  desc 'Test the gem.'
-  Spec::Rake::SpecTask.new(:test) do |t|
-    t.spec_files = FileList.new('spec/**/*_spec.rb')
-  end
+  require 'rspec'
+  require 'rspec/core/rake_task'
+  desc 'Run the unit tests'
+  RSpec::Core::RakeTask.new(:test)
 rescue LoadError
   task :test do
-    STDERR.puts "You must have rspec >= 1.3.0 to run the tests"
+    STDERR.puts "You must have rspec 2.0 installed to run the tests"
   end
 end
 
-desc 'Generate documentation for sunspot_index_queue.'
+desc 'Generate documentation.'
 Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.options << '--title' << 'Sunspot Index Queue' << '--line-numbers' << '--inline-source' << '--main' << 'README.rdoc'
@@ -34,7 +33,7 @@ begin
     gem.email = "brian@embellishedvisions.com"
     gem.homepage = "http://github.com/bdurand/sunspot_index_queue"
     gem.authors = ["Brian Durand"]
-    gem.rdoc_options = ["--charset=UTF-8", "--main", "README.rdoc"]
+    gem.rdoc_options = ["--charset=UTF-8", "--main", "README.rdoc", "MIT_LICENSE"]
     
     gem.add_dependency('sunspot', '>= 1.1.0')
     gem.add_development_dependency('sqlite3')
@@ -43,7 +42,7 @@ begin
     gem.add_development_dependency('dm-aggregates', '>=1.0.0')
     gem.add_development_dependency('dm-migrations', '>=1.0.0')
     gem.add_development_dependency('mongo')
-    gem.add_development_dependency('rspec', '>= 1.3.0')
+    gem.add_development_dependency('rspec', '>= 2.0.0')
     gem.add_development_dependency('jeweler')
   end
 

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.1.1

data/lib/sunspot/index_queue.rb

@@ -19,7 +19,7 @@ module Sunspot
     
     class << self
       # Set the default priority for indexing items within a block. Higher priority items will be processed first.
-      def set_priority (priority, &block)
+      def set_priority(priority, &block)
         save_val = Thread.current[:sunspot_index_queue_priority]
         begin
           Thread.current[:sunspot_index_queue_priority] = priority.to_i
@@ -48,7 +48,7 @@ module Sunspot
     # queues process different classes of records when they need to different configurations.
     #
     # +:session+ - The Sunspot::Session object to use for communicating with Solr (defaults to a session with the default config).
-    def initialize (options = {})
+    def initialize(options = {})
       @retry_interval = options[:retry_interval] || 60
       @batch_size = options[:batch_size] || 100
       @batch_handler = nil
@@ -71,14 +71,14 @@ module Sunspot
     #       batch.submit!
     #     end
     #   end
-    def batch_handler (&block)
+    def batch_handler(&block)
       @batch_handler = block
     end
     
     # Add a record to be indexed to the queue. The record can be specified as either an indexable object or as
     # as hash with :class and :id keys. The priority to be indexed can be passed in the options as +:priority+
     # (defaults to 0).
-    def index (record_or_hash, options = {})
+    def index(record_or_hash, options = {})
       klass, id = class_and_id(record_or_hash)
       Entry.enqueue(self, klass, id, false, options[:priority] || self.class.default_priority)
     end
@@ -86,20 +86,20 @@ module Sunspot
     # Add a record to be removed to the queue. The record can be specified as either an indexable object or as
     # as hash with :class and :id keys. The priority to be indexed can be passed in the options as +:priority+
     # (defaults to 0).
-    def remove (record_or_hash, options = {})
+    def remove(record_or_hash, options = {})
       klass, id = class_and_id(record_or_hash)
       Entry.enqueue(self, klass, id, true, options[:priority] || self.class.default_priority)
     end
 
     # Add a list of records to be indexed to the queue. The priority to be indexed can be passed in the
     # options as +:priority+ (defaults to 0).
-    def index_all (klass, ids, options = {})
+    def index_all(klass, ids, options = {})
       Entry.enqueue(self, klass, ids, false, options[:priority] || self.class.default_priority)
     end
 
     # Add a list of records to be removed to the queue. The priority to be indexed can be passed in the
     # options as +:priority+ (defaults to 0).
-    def remove_all (klass, ids, options = {})
+    def remove_all(klass, ids, options = {})
       Entry.enqueue(self, klass, ids, true, options[:priority] || self.class.default_priority)
     end
     
@@ -119,7 +119,7 @@ module Sunspot
     end
     
     # Get the entries in the queue that have errors. Supported options are +:limit+ (default 50) and +:offset+ (default 0).
-    def errors (options = {})
+    def errors(options = {})
       limit = options[:limit] ? options[:limit].to_i : 50
       Entry.errors(self, limit, options[:offset].to_i)
     end
@@ -157,7 +157,7 @@ module Sunspot
     private
     
     # Get the class and id for either a record or a hash containing +:class+ and +:id+ options
-    def class_and_id (record_or_hash)
+    def class_and_id(record_or_hash)
       if record_or_hash.is_a?(Hash)
         [record_or_hash[:class], record_or_hash[:id]]
       else

data/lib/sunspot/index_queue/batch.rb

@@ -9,7 +9,7 @@ module Sunspot
       # be processed.
       PASS_THROUGH_EXCEPTIONS = [SystemExit, NoMemoryError, Interrupt, SignalException, Errno::ECONNREFUSED]
       
-      def initialize (queue, entries = nil)
+      def initialize(queue, entries = nil)
         @queue = queue
         @entries = []
         @entries.concat(entries) if entries
@@ -32,6 +32,8 @@ module Sunspot
           end
           commit!
         rescue Exception => e
+          @delete_entries.clear
+          entries.each{|entry| entry.processed = false}
           if PASS_THROUGH_EXCEPTIONS.include?(e.class)
             raise e
           else
@@ -56,7 +58,7 @@ module Sunspot
       end
       
       # Clear the processed flag on all entries.
-      def clear_processed (entries)
+      def clear_processed(entries)
         entries.each{|entry| entry.processed = false}
       end
       
@@ -66,8 +68,9 @@ module Sunspot
         Entry.delete_entries(@delete_entries) unless @delete_entries.empty?
       rescue Exception => e
         clear_processed(entries)
-        @delete_entries.clear
         raise e
+      ensure
+        @delete_entries.clear
       end
       
       # Submit all entries to Solr individually and then commit.
@@ -90,7 +93,7 @@ module Sunspot
       end
       
       # Send an entry to Solr doing an update or delete as necessary.
-      def submit_entry (entry)
+      def submit_entry(entry)
         log_entry_error(entry) do
           if entry.is_delete?
             session.remove_by_id(entry.record_class_name, entry.record_id)
@@ -102,7 +105,7 @@ module Sunspot
       end
       
       # Update an entry with an error message if a block fails.
-      def log_entry_error (entry)
+      def log_entry_error(entry)
         begin
           yield
           entry.processed = true

data/lib/sunspot/index_queue/entry.rb

@@ -24,7 +24,7 @@ module Sunspot
         # Implementations should support pulling entries in batches by a priority where higher priority
         # entries are processed first. Errors should be automatically retried after an interval specified
         # by the IndexQueue. The batch size set by the IndexQueue should also be honored.
-        def implementation= (klass)
+        def implementation=(klass)
           unless klass.is_a?(Class) || klass.nil?
             class_name = klass.to_s
             class_name = Sunspot::Util.camel_case(class_name).gsub('/', '::') unless class_name.include?('::')
@@ -43,42 +43,42 @@ module Sunspot
         end
         
         # Get a count of the queue entries for an IndexQueue. Implementations must implement this method.
-        def total_count (queue)
+        def total_count(queue)
           implementation.total_count(queue)
         end
         
         # Get a count of the entries ready to be processed for an IndexQueue. Implementations must implement this method.
-        def ready_count (queue)
+        def ready_count(queue)
           implementation.ready_count(queue)
         end
         
         # Get a count of the error entries for an IndexQueue. Implementations must implement this method.
-        def error_count (queue)
+        def error_count(queue)
           implementation.error_count(queue)
         end
         
         # Get the specified number of error entries for an IndexQueue. Implementations must implement this method.
-        def errors (queue, limit, offset)
+        def errors(queue, limit, offset)
           implementation.errors(queue, limit, offset)
         end
         
         # Get the next batch of entries to process for IndexQueue. Implementations must implement this method.
-        def next_batch! (queue)
+        def next_batch!(queue)
           implementation.next_batch!(queue)
         end
         
         # Reset the entries in the queue to be excuted again immediately and clear any errors.
-        def reset! (queue)
+        def reset!(queue)
           implementation.reset!(queue)
         end
         
         # Add an entry the queue. +is_delete+ will be true if the entry is a delete. Implementations must implement this method.
-        def add (klass, id, delete, options = {})
+        def add(klass, id, delete, options = {})
           raise NotImplementedError.new("add")
         end
         
         # Add multiple entries to the queue. +delete+ will be true if the entry is a delete.
-        def enqueue (queue, klass, ids, delete, priority)
+        def enqueue(queue, klass, ids, delete, priority)
           klass = Sunspot::Util.full_const_get(klass.to_s) unless klass.is_a?(Class)
           unless queue.class_names.empty? || queue.class_names.include?(klass.name)
             raise ArgumentError.new("Class #{klass.name} is not in the class names allowed for the queue")
@@ -94,13 +94,13 @@ module Sunspot
         end
         
         # Delete entries from the queue. Implementations must implement this method.
-        def delete_entries (entries)
+        def delete_entries(entries)
           implementation.delete_entries(entries)
         end
         
         # Load all records in an array of entries. This can be faster than calling load on each DataAccessor
         # depending on them implementation
-        def load_all_records (entries)
+        def load_all_records(entries)
           classes = entries.collect{|entry| entry.record_class_name}.uniq.collect{|name| Sunspot::Util.full_const_get(name) rescue nil}.compact
           map = entries.inject({}){|hash, entry| hash[entry.record_id.to_s] = entry; hash}
           classes.each do |klass|
@@ -124,7 +124,7 @@ module Sunspot
       end
 
       # Set the error message on an entry. Implementations must implement this method.
-      def set_error! (error, retry_interval = nil)
+      def set_error!(error, retry_interval = nil)
         raise NotImplementedError.new("set_error!")
       end
 

data/lib/sunspot/index_queue/entry/active_record_impl.rb

@@ -25,13 +25,13 @@ module Sunspot
 
         class << self
           # Implementation of the total_count method.
-          def total_count (queue)
+          def total_count(queue)
             conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
             count(:conditions => conditions)
           end
           
           # Implementation of the ready_count method.
-          def ready_count (queue)
+          def ready_count(queue)
             conditions = ["#{connection.quote_column_name('run_at')} <= ?", Time.now.utc]
             unless queue.class_names.empty?
               conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
@@ -41,7 +41,7 @@ module Sunspot
           end
 
           # Implementation of the error_count method.
-          def error_count (queue)
+          def error_count(queue)
             conditions = ["#{connection.quote_column_name('error')} IS NOT NULL"]
             unless queue.class_names.empty?
               conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
@@ -51,7 +51,7 @@ module Sunspot
           end
 
           # Implementation of the errors method.
-          def errors (queue, limit, offset)
+          def errors(queue, limit, offset)
             conditions = ["#{connection.quote_column_name('error')} IS NOT NULL"]
             unless queue.class_names.empty?
               conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
@@ -67,7 +67,7 @@ module Sunspot
           end
           
           # Implementation of the next_batch! method.
-          def next_batch! (queue)
+          def next_batch!(queue)
             conditions = ["#{connection.quote_column_name('run_at')} <= ?", Time.now.utc]
             unless queue.class_names.empty?
               conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
@@ -82,7 +82,7 @@ module Sunspot
           end
 
           # Implementation of the add method.
-          def add (klass, id, delete, priority)
+          def add(klass, id, delete, priority)
             queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
             queue_entry = first(:conditions => queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
             queue_entry.is_delete = delete
@@ -92,8 +92,8 @@ module Sunspot
           end
           
           # Implementation of the delete_entries method.
-          def delete_entries (ids)
-            delete_all(:id => ids)
+          def delete_entries(entries)
+            delete_all(:id => entries)
           end
           
           # Create the table used to store the queue in the database.
@@ -115,7 +115,7 @@ module Sunspot
         end
 
         # Implementation of the set_error! method.
-        def set_error! (error, retry_interval = nil)
+        def set_error!(error, retry_interval = nil)
           self.attempts += 1
           self.run_at = (retry_interval * attempts).from_now.utc if retry_interval
           self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"

data/lib/sunspot/index_queue/entry/data_mapper_impl.rb

@@ -28,40 +28,40 @@ module Sunspot
         
         class << self
           # Implementation of the total_count method.
-          def total_count (queue)
+          def total_count(queue)
             conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
             count(conditions)
           end
           
           # Implementation of the ready_count method.
-          def ready_count (queue)
+          def ready_count(queue)
             conditions = {:run_at.lte => Time.now.utc}
             conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
             count(conditions)
           end
 
           # Implementation of the error_count method.
-          def error_count (queue)
+          def error_count(queue)
             conditions = {:error.not => nil}
             conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
             count(conditions)
           end
 
           # Implementation of the errors method.
-          def errors (queue, limit, offset)
+          def errors(queue, limit, offset)
             conditions = {:error.not => nil}
             conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
             all(conditions.merge(:limit => limit, :offset => offset, :order => :id))
           end
 
           # Implementation of the reset! method.
-          def reset! (queue)
+          def reset!(queue)
             conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
             all(conditions).update!(:run_at => Time.now.utc, :attempts => 0, :error => nil, :lock => nil)
           end
           
           # Implementation of the next_batch! method.
-          def next_batch! (queue)
+          def next_batch!(queue)
             conditions = {:run_at.lte => Time.now.utc}
             conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
             batch_entries = all(conditions.merge(:fields => [:id], :limit => queue.batch_size, :order => [:priority.desc, :run_at]))
@@ -73,7 +73,7 @@ module Sunspot
           end
 
           # Implementation of the add method.
-          def add (klass, id, delete, priority)
+          def add(klass, id, delete, priority)
             queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
             queue_entry = first(:conditions => queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
             queue_entry.is_delete = delete
@@ -83,13 +83,13 @@ module Sunspot
           end
           
           # Implementation of the delete_entries method.
-          def delete_entries (ids)
-            all(:id => ids).destroy!
+          def delete_entries(entries)
+            all(:id => entries.map(&:id)).destroy!
           end
         end
 
         # Implementation of the set_error! method.
-        def set_error! (error, retry_interval = nil)
+        def set_error!(error, retry_interval = nil)
           self.attempts += 1
           self.run_at = Time.now.utc + (retry_interval * attempts) if retry_interval
           self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"

data/lib/sunspot/index_queue/entry/mongo_impl.rb

@@ -9,17 +9,17 @@ module Sunspot
       # To set it up, you need to set the connection and database that it will use.
       #
       #   Sunspot::IndexQueue::Entry::MongoImpl.connection = 'localhost'
-      #   Sunspot::IndexQueue::Entry::MongoImpl.database = 'my_database'
+      #   Sunspot::IndexQueue::Entry::MongoImpl.database_name = 'my_database'
       #   # or
       #   Sunspot::IndexQueue::Entry::MongoImpl.connection = Mongo::Connection.new('localhost', 27017)
-      #   Sunspot::IndexQueue::Entry::MongoImpl.database = 'my_database'
+      #   Sunspot::IndexQueue::Entry::MongoImpl.database_name = 'my_database'
       class MongoImpl
         include Entry
 
         class << self
           # Set the connection to MongoDB. The args can either be a Mongo::Connection object, or the args
           # that can be used to create a new Mongo::Connection.
-          def connection= (*args)
+          def connection=(*args)
             @connection = args.first.is_a?(Mongo::Connection) ? args.first : Mongo::Connection.new(*args)
           end
           
@@ -29,7 +29,7 @@ module Sunspot
           end
           
           # Set the name of the database which will contain the queue collection.
-          def database_name= (name)
+          def database_name=(name)
             @collection = nil
             @database_name = name
           end
@@ -45,20 +45,20 @@ module Sunspot
           end
           
           # Create a new entry.
-          def create (attributes)
+          def create(attributes)
             entry = new(attributes)
             entry.save
             entry
           end
           
           # Find one entry given a selector or object id.
-          def find_one (spec_or_object_id=nil, opts={})
+          def find_one(spec_or_object_id=nil, opts={})
             doc = collection.find_one(spec_or_object_id, opts)
             doc ? new(doc) : nil
           end
           
           # Find an array of entries given a selector.
-          def find (selector={}, opts={})
+          def find(selector={}, opts={})
             collection.find(selector, opts).collect{|doc| new(doc)}
           end
           
@@ -68,18 +68,18 @@ module Sunspot
           end
           
           # Set the logger used to log errors.
-          def logger= (logger)
+          def logger=(logger)
             @logger = logger
           end
           
           # Implementation of the total_count method.
-          def total_count (queue)
+          def total_count(queue)
             conditions = queue.class_names.empty? ? {} : {:record_class_name => {'$in' => queue.class_names}}
             collection.find(conditions).count
           end
           
           # Implementation of the ready_count method.
-          def ready_count (queue)
+          def ready_count(queue)
             conditions = {:run_at => {'$lte' => Time.now.utc}}
             unless queue.class_names.empty?
               conditions[:record_class_name] = {'$in' => queue.class_names}
@@ -88,7 +88,7 @@ module Sunspot
           end
 
           # Implementation of the error_count method.
-          def error_count (queue)
+          def error_count(queue)
             conditions = {:error => {'$ne' => nil}}
             unless queue.class_names.empty?
               conditions[:record_class_name] = {'$in' => queue.class_names}
@@ -97,7 +97,7 @@ module Sunspot
           end
 
           # Implementation of the errors method.
-          def errors (queue, limit, offset)
+          def errors(queue, limit, offset)
             conditions = {:error => {'$ne' => nil}}
             unless queue.class_names.empty?
               conditions[:record_class_name] = {'$in' => queue.class_names}
@@ -106,13 +106,13 @@ module Sunspot
           end
 
           # Implementation of the reset! method.
-          def reset! (queue)
+          def reset!(queue)
             conditions = queue.class_names.empty? ? {} : {:record_class_name => {'$in' => queue.class_names}}
             collection.update(conditions, {"$set" => {:run_at => Time.now.utc, :attempts => 0, :error => nil}}, :multi => true)
           end
           
           # Implementation of the next_batch! method.
-          def next_batch! (queue)
+          def next_batch!(queue)
             conditions = {:run_at => {'$lte' => Time.now.utc}}
             unless queue.class_names.empty?
               conditions[:record_class_name] = {'$in' => queue.class_names}
@@ -131,7 +131,7 @@ module Sunspot
           end
 
           # Implementation of the add method.
-          def add (klass, id, delete, priority)
+          def add(klass, id, delete, priority)
             queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
             queue_entry = find_one(queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
             queue_entry.is_delete = delete
@@ -141,15 +141,15 @@ module Sunspot
           end
           
           # Implementation of the delete_entries method.
-          def delete_entries (ids)
-            collection.remove(:_id => {'$in' => ids})
+          def delete_entries(entries)
+            collection.remove(:_id => {'$in' => entries.map(&:id)})
           end
         end
         
         attr_reader :doc
         
         # Create a new entry from a document hash.
-        def initialize (attributes = {})
+        def initialize(attributes = {})
           @doc = {}
           attributes.each do |key, value|
             @doc[key.to_s] = value
@@ -169,7 +169,7 @@ module Sunspot
         end
         
         # Set the entry record_class_name.
-        def record_class_name= (value)
+        def record_class_name=(value)
           doc['record_class_name'] =  value.nil? ? nil : value.to_s
         end
         
@@ -179,7 +179,7 @@ module Sunspot
         end
         
         # Set the entry record_id.
-        def record_id= (value)
+        def record_id=(value)
           doc['record_id'] =  value
         end
         
@@ -189,7 +189,7 @@ module Sunspot
         end
         
         # Set the entry run_at time.
-        def run_at= (value)
+        def run_at=(value)
           value = Time.parse(value.to_s) unless value.nil? || value.is_a?(Time)
           doc['run_at'] =  value.nil? ? nil : value.utc
         end
@@ -200,7 +200,7 @@ module Sunspot
         end
         
         # Set the entry priority.
-        def priority= (value)
+        def priority=(value)
           doc['priority'] =  value.to_i
         end
         
@@ -210,7 +210,7 @@ module Sunspot
         end
         
         # Set the entry attempts.
-        def attempts= (value)
+        def attempts=(value)
           doc['attempts'] =  value.to_i
         end
         
@@ -220,7 +220,7 @@ module Sunspot
         end
         
         # Set the entry error.
-        def error= (value)
+        def error=(value)
           doc['error'] =  value.nil? ? nil : value.to_s
         end
         
@@ -230,7 +230,7 @@ module Sunspot
         end
         
         # Set the entry delete entry flag.
-        def is_delete= (value)
+        def is_delete=(value)
           doc['is_delete'] =  !!value
         end
         
@@ -241,7 +241,7 @@ module Sunspot
         end
 
         # Implementation of the set_error! method.
-        def set_error! (error, retry_interval = nil)
+        def set_error!(error, retry_interval = nil)
           self.attempts += 1
           self.run_at = (retry_interval * attempts).from_now.utc if retry_interval
           self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"

data/lib/sunspot/index_queue/session_proxy.rb

@@ -14,7 +14,7 @@ module Sunspot
       # Create a new session proxy for a particular queue (default to a queue for all classes bound to the
       # default session configuration). You can specify the session argument if the session used for queries should be
       # different than the one the queue is bound to.
-      def initialize (queue = nil, session = nil)
+      def initialize(queue = nil, session = nil)
         @queue = queue || IndexQueue.new
         @session = session || @queue.session
       end
@@ -50,20 +50,20 @@ module Sunspot
       end
       
       # Queues up the index operation for later.
-      def index (*objects)
+      def index(*objects)
         objects.flatten.each do |object|
           queue.index(object)
         end
       end
       
       # Queues up the index operation for later.
-      def index! (*objects)
+      def index!(*objects)
         index(*objects)
       end
       
       # Queues up the remove operation for later unless a block is passed. In that case it will
       # be performed immediately.
-      def remove (*objects, &block)
+      def remove(*objects, &block)
         if block
           # Delete by query not supported by queue, so send to server
           queue.session.remove(*objects, &block)
@@ -76,7 +76,7 @@ module Sunspot
       
       # Queues up the remove operation for later unless a block is passed. In that case it will
       # be performed immediately.
-      def remove! (*objects, &block)
+      def remove!(*objects, &block)
         if block
           # Delete by query not supported by queue, so send to server
           queue.session.remove!(*objects, &block)
@@ -86,24 +86,24 @@ module Sunspot
       end
       
       # Proxies remove_all to the queue session.
-      def remove_all (*classes)
+      def remove_all(*classes)
         # Delete by query not supported by queue, so send to server
         queue.session.remove_all(*classes)
       end
       
       # Proxies remove_all! to the queue session.
-      def remove_all! (*classes)
+      def remove_all!(*classes)
         # Delete by query not supported by queue, so send to server
         queue.session.remove_all!(*classes)
       end
       
       # Queues up the index operation for later.
-      def remove_by_id (clazz, id)
+      def remove_by_id(clazz, id)
         queue.remove(:class => clazz, :id => id)
       end
       
       # Queues up the index operation for later.
-      def remove_by_id! (clazz, id)
+      def remove_by_id!(clazz, id)
         remove_by_id(clazz, id)
       end
     end

data/spec/batch_spec.rb

@@ -43,6 +43,22 @@ describe Sunspot::IndexQueue::Batch do
     entry_2.processed?.should == true
   end
   
+  it "should only delete entries that are successfully committed" do
+    entry_1.stub!(:record).and_return(record_1)
+    def session.batch
+      yield
+      raise("solr rejects malformed batch")
+    end
+    session.should_receive(:index).with(record_1).twice
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id)
+    session.should_receive(:remove_by_id).with(entry_2.record_class_name, entry_2.record_id).and_raise("boom")
+    session.should_receive(:commit)
+    Sunspot::IndexQueue::Entry.implementation.should_receive(:delete_entries).with([entry_1])
+    subject.submit!
+    entry_1.processed?.should == true
+    entry_2.processed?.should == false
+    entry_2.error.should_not == nil
+  end
   
   it "should add error messages to each entry that errors out" do
     entry_1.stub!(:record).and_return(record_1)

data/spec/entry_impl_examples.rb

@@ -116,7 +116,7 @@ shared_examples_for "Entry implementation" do
       end
   
       it "should delete a list of entry ids" do
-        Sunspot::IndexQueue::Entry.implementation.delete_entries([@entry_1.id, @entry_2.id])
+        Sunspot::IndexQueue::Entry.implementation.delete_entries([@entry_1, @entry_2])
         factory.find(@entry_1.id).should == nil
         factory.find(@entry_2.id).should == nil
         factory.find(@entry_4.id).id.should == @entry_4.id

data/spec/integration_spec.rb

@@ -3,6 +3,10 @@ require 'active_record'
 
 describe "Sunspot::IndexQueue integration tests" do
   before :all do
+    ActiveRecord::Base.establish_connection("adapter" => "sqlite3", "database" => ":memory:")
+    Sunspot::IndexQueue::Entry.implementation = :active_record
+    Sunspot::IndexQueue::Entry::ActiveRecordImpl.create_table
+    
     data_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'tmp'))
     FileUtils.rm_rf(data_dir) if File.exist?(data_dir)
     Dir.mkdir(data_dir)
@@ -10,7 +14,7 @@ describe "Sunspot::IndexQueue integration tests" do
       `sunspot-solr start --port=18983 --data-directory=. --pid-dir=. --log-file=./solr.log --max-memory=64m`
       raise "Failed to start Solr on port 18983" unless $? == 0
       # Wait until the server is responding
-      ping_uri = URI.parse("http://127.0.0.1:18983/solr/ping")
+      ping_uri = URI.parse("http://127.0.0.1:18983/solr/admin/ping")
       solr_started = false
       100.times do
         begin
@@ -23,10 +27,6 @@ describe "Sunspot::IndexQueue integration tests" do
       end
       raise "Solr failed to start after 10 seconds" unless solr_started
     end
-    ActiveRecord::Base.establish_connection("adapter" => "sqlite3", "database" => ":memory:")
-    Sunspot::IndexQueue::Entry.implementation = :active_record
-    Sunspot::IndexQueue::Entry::ActiveRecordImpl.create_table
-    
     @solr_session = Sunspot::Session.new do |config|
       config.solr.url = 'http://127.0.0.1:18983/solr'
     end
@@ -40,7 +40,7 @@ describe "Sunspot::IndexQueue integration tests" do
       end
       FileUtils.rm_rf(data_dir)
     end
-    Sunspot::IndexQueue::Entry::ActiveRecordImpl.connection.drop_table(Sunspot::IndexQueue::Entry::ActiveRecordImpl.table_name)
+    Sunspot::IndexQueue::Entry::ActiveRecordImpl.connection.drop_table(Sunspot::IndexQueue::Entry::ActiveRecordImpl.table_name) if Sunspot::IndexQueue::Entry::ActiveRecordImpl.table_exists?
     ActiveRecord::Base.connection.disconnect!
     Sunspot::IndexQueue::Entry.implementation = nil
   end

data/spec/spec_helper.rb

@@ -1,5 +1,9 @@
 require 'rubygems'
 
+require 'uri'
+require 'fileutils'
+require 'net/http'
+
 if ENV["ACTIVE_RECORD_VERSION"]
   gem 'activerecord', ENV["ACTIVE_RECORD_VERSION"]
 else
@@ -41,7 +45,7 @@ module Sunspot
             Thread.current[:mock_db]
           end
           
-          def save (*objects)
+          def save(*objects)
             objects.each do |obj|
               db[obj.id.to_s] = obj.dup
             end
@@ -50,21 +54,21 @@ module Sunspot
         
         attr_reader :id
         attr_accessor :value
-        def initialize (id, value=nil)
+        def initialize(id, value=nil)
           @id = id
           @value = value
         end
         
-        def == (value)
+        def ==(value)
           value.is_a?(self.class) && @id == value.id
         end
         
         class DataAccessor < Sunspot::Adapters::DataAccessor
-          def load (id)
+          def load(id)
             Searchable.db ? Searchable.db[id.to_s] : Searchable.new(id)
           end
           
-          def load_all (ids)
+          def load_all(ids)
             ids.collect{|id| load(id)}.compact
           end
         end
@@ -92,7 +96,7 @@ module Sunspot
         
         attr_reader :record_class_name, :record_id, :error, :attempts
         
-        def initialize (options = {})
+        def initialize(options = {})
           if options[:record]
             @record_class_name = options[:record].class.name
             @record_id = options[:record].id
@@ -110,6 +114,10 @@ module Sunspot
         def id
           object_id
         end
+        
+        def set_error!(message, retry_interval = nil)
+          @error = message
+        end
       end
     end
   end

data/sunspot_index_queue.gemspec

@@ -1,67 +1,65 @@
 # Generated by jeweler
 # DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{sunspot_index_queue}
-  s.version = "1.1.0"
+  s.version = "1.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brian Durand"]
-  s.date = %q{2011-01-01}
+  s.date = %q{2011-04-08}
   s.description = %q{This gem provides asynchronous indexing to Solr for the sunspot gem. It uses a pluggable model for the backing queue and provides support for ActiveRecord and MongoDB out of the box.}
   s.email = %q{brian@embellishedvisions.com}
   s.extra_rdoc_files = [
     "README.rdoc"
   ]
   s.files = [
-    ".gitignore",
-     "CHANGE_LOG.txt",
-     "MIT_LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "lib/sunspot/index_queue.rb",
-     "lib/sunspot/index_queue/batch.rb",
-     "lib/sunspot/index_queue/entry.rb",
-     "lib/sunspot/index_queue/entry/active_record_impl.rb",
-     "lib/sunspot/index_queue/entry/data_mapper_impl.rb",
-     "lib/sunspot/index_queue/entry/mongo_impl.rb",
-     "lib/sunspot/index_queue/session_proxy.rb",
-     "lib/sunspot_index_queue.rb",
-     "spec/active_record_impl_spec.rb",
-     "spec/batch_spec.rb",
-     "spec/data_mapper_impl_spec.rb",
-     "spec/entry_impl_examples.rb",
-     "spec/entry_spec.rb",
-     "spec/index_queue_spec.rb",
-     "spec/integration_spec.rb",
-     "spec/mongo_impl_spec.rb",
-     "spec/session_proxy_spec.rb",
-     "spec/spec_helper.rb",
-     "sunspot_index_queue.gemspec"
+    "CHANGE_LOG.txt",
+    "MIT_LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/sunspot/index_queue.rb",
+    "lib/sunspot/index_queue/batch.rb",
+    "lib/sunspot/index_queue/entry.rb",
+    "lib/sunspot/index_queue/entry/active_record_impl.rb",
+    "lib/sunspot/index_queue/entry/data_mapper_impl.rb",
+    "lib/sunspot/index_queue/entry/mongo_impl.rb",
+    "lib/sunspot/index_queue/session_proxy.rb",
+    "lib/sunspot_index_queue.rb",
+    "spec/active_record_impl_spec.rb",
+    "spec/batch_spec.rb",
+    "spec/data_mapper_impl_spec.rb",
+    "spec/entry_impl_examples.rb",
+    "spec/entry_spec.rb",
+    "spec/index_queue_spec.rb",
+    "spec/integration_spec.rb",
+    "spec/mongo_impl_spec.rb",
+    "spec/session_proxy_spec.rb",
+    "spec/spec_helper.rb",
+    "sunspot_index_queue.gemspec"
   ]
   s.homepage = %q{http://github.com/bdurand/sunspot_index_queue}
-  s.rdoc_options = ["--charset=UTF-8", "--main", "README.rdoc"]
+  s.rdoc_options = ["--charset=UTF-8", "--main", "README.rdoc", "MIT_LICENSE"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{Asynchronous Solr indexing support for the sunspot gem with an emphasis on reliablity and throughput.}
   s.test_files = [
     "spec/active_record_impl_spec.rb",
-     "spec/batch_spec.rb",
-     "spec/data_mapper_impl_spec.rb",
-     "spec/entry_impl_examples.rb",
-     "spec/entry_spec.rb",
-     "spec/index_queue_spec.rb",
-     "spec/integration_spec.rb",
-     "spec/mongo_impl_spec.rb",
-     "spec/session_proxy_spec.rb",
-     "spec/spec_helper.rb"
+    "spec/batch_spec.rb",
+    "spec/data_mapper_impl_spec.rb",
+    "spec/entry_impl_examples.rb",
+    "spec/entry_spec.rb",
+    "spec/index_queue_spec.rb",
+    "spec/integration_spec.rb",
+    "spec/mongo_impl_spec.rb",
+    "spec/session_proxy_spec.rb",
+    "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
@@ -72,7 +70,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<dm-aggregates>, [">= 1.0.0"])
       s.add_development_dependency(%q<dm-migrations>, [">= 1.0.0"])
       s.add_development_dependency(%q<mongo>, [">= 0"])
-      s.add_development_dependency(%q<rspec>, [">= 1.3.0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.0.0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
       s.add_dependency(%q<sunspot>, [">= 1.1.0"])
@@ -82,7 +80,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<dm-aggregates>, [">= 1.0.0"])
       s.add_dependency(%q<dm-migrations>, [">= 1.0.0"])
       s.add_dependency(%q<mongo>, [">= 0"])
-      s.add_dependency(%q<rspec>, [">= 1.3.0"])
+      s.add_dependency(%q<rspec>, [">= 2.0.0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
     end
   else
@@ -93,7 +91,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<dm-aggregates>, [">= 1.0.0"])
     s.add_dependency(%q<dm-migrations>, [">= 1.0.0"])
     s.add_dependency(%q<mongo>, [">= 0"])
-    s.add_dependency(%q<rspec>, [">= 1.3.0"])
+    s.add_dependency(%q<rspec>, [">= 2.0.0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sunspot_index_queue
 version: !ruby/object:Gem::Version 
-  hash: 19
-  prerelease: false
+  hash: 17
+  prerelease: 
   segments: 
   - 1
   - 1
-  - 0
-  version: 1.1.0
+  - 1
+  version: 1.1.1
 platform: ruby
 authors: 
 - Brian Durand
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-01 00:00:00 -06:00
+date: 2011-04-08 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -133,12 +133,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 15
         segments: 
-        - 1
-        - 3
+        - 2
+        - 0
         - 0
-        version: 1.3.0
+        version: 2.0.0
   type: :development
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
@@ -164,7 +164,6 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
-- .gitignore
 - CHANGE_LOG.txt
 - MIT_LICENSE
 - README.rdoc
@@ -198,6 +197,7 @@ rdoc_options:
 - --charset=UTF-8
 - --main
 - README.rdoc
+- MIT_LICENSE
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -221,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: Asynchronous Solr indexing support for the sunspot gem with an emphasis on reliablity and throughput.

data/.gitignore

@@ -1,4 +0,0 @@
-pkg
-rdoc
-tmp
-*.rbc