sunspot_index_queue 1.0.0

data/lib/sunspot/index_queue/entry.rb

@@ -0,0 +1,137 @@
+module Sunspot
+  class IndexQueue
+    # Abstract queue entry interface. All the gory details on actually handling the queue are handled by a
+    # specific implementation class. The default implementation will use ActiveRecord as the backing queue.
+    #
+    # Implementing classes must define attribute readers for +id+, +record_class_name+, +record_id+, +error+,
+    # +attempts+, and +is_delete?+.
+    module Entry
+      autoload :ActiveRecordImpl, File.expand_path('../entry/active_record_impl', __FILE__)
+      autoload :DataMapperImpl, File.expand_path('../entry/data_mapper_impl', __FILE__)
+      autoload :MongoImpl, File.expand_path('../entry/mongo_impl', __FILE__)
+      
+      attr_writer :processed
+
+      class << self
+        # Set the implementation class to use for the queue. This can be set as either a class object,
+        # full class name, or a symbol representing one of the default implementations.
+        #
+        #   # These all set the implementation to use the default ActiveRecord queue.
+        #   Sunspot::IndexQueue::Entry.implementation = :active_record
+        #   Sunspot::IndexQueue::Entry.implementation = "Sunspot::IndexQueue::Entry::ActiveRecordImpl"
+        #   Sunspot::IndexQueue::Entry.implementation = Sunspot::IndexQueue::Entry::ActiveRecordImpl
+        #
+        # Implementations should support pulling entries in batches by a priority where higher priority
+        # entries are processed first. Errors should be automatically retried after an interval specified
+        # by the IndexQueue. The batch size set by the IndexQueue should also be honored.
+        def implementation= (klass)
+          unless klass.is_a?(Class) || klass.nil?
+            class_name = klass.to_s
+            class_name = Sunspot::Util.camel_case(class_name).gsub('/', '::') unless class_name.include?('::')
+            if class_name.include?('::') || !const_defined?("#{class_name}Impl")
+              klass = Sunspot::Util.full_const_get(class_name)
+            else
+              klass = const_get("#{class_name}Impl")
+            end
+          end
+          @implementation = klass
+        end
+        
+        # The implementation class used for the queue.
+        def implementation
+          @implementation ||= ActiveRecordImpl
+        end
+        
+        # Get a count of the queue entries for an IndexQueue. Implementations must implement this method.
+        def total_count (queue)
+          implementation.total_count(queue)
+        end
+        
+        # Get a count of the entries ready to be processed for an IndexQueue. Implementations must implement this method.
+        def ready_count (queue)
+          implementation.ready_count(queue)
+        end
+        
+        # Get a count of the error entries for an IndexQueue. Implementations must implement this method.
+        def error_count (queue)
+          implementation.error_count(queue)
+        end
+        
+        # Get the specified number of error entries for an IndexQueue. Implementations must implement this method.
+        def errors (queue, limit, offset)
+          implementation.errors(queue, limit, offset)
+        end
+        
+        # Get the next batch of entries to process for IndexQueue. Implementations must implement this method.
+        def next_batch! (queue)
+          implementation.next_batch!(queue)
+        end
+        
+        # Reset the entries in the queue to be excuted again immediately and clear any errors.
+        def reset! (queue)
+          implementation.reset!(queue)
+        end
+        
+        # Add an entry the queue. +is_delete+ will be true if the entry is a delete. Implementations must implement this method.
+        def add (klass, id, delete, options = {})
+          raise NotImplementedError.new("add")
+        end
+        
+        # Add multiple entries to the queue. +delete+ will be true if the entry is a delete.
+        def enqueue (queue, klass, ids, delete, priority)
+          klass = Sunspot::Util.full_const_get(klass.to_s) unless klass.is_a?(Class)
+          unless queue.class_names.empty? || queue.class_names.include?(klass.name)
+            raise ArgumentError.new("Class #{klass.name} is not in the class names allowed for the queue")
+          end
+          priority = priority.to_i
+          if ids.is_a?(Array)
+            ids.each do |id|
+              implementation.add(klass, id, delete, priority)
+            end
+          else
+            implementation.add(klass, ids, delete, priority)
+          end
+        end
+        
+        # Delete entries from the queue. Implementations must implement this method.
+        def delete_entries (entries)
+          implementation.delete_entries(entries)
+        end
+        
+        # Load all records in an array of entries. This can be faster than calling load on each DataAccessor
+        # depending on them implementation
+        def load_all_records (entries)
+          classes = entries.collect{|entry| entry.record_class_name}.uniq.collect{|name| Sunspot::Util.full_const_get(name) rescue nil}.compact
+          map = entries.inject({}){|hash, entry| hash[entry.record_id.to_s] = entry; hash}
+          classes.each do |klass|
+            ids = entries.collect{|entry| entry.record_id}
+            Sunspot::Adapters::DataAccessor.create(klass).load_all(ids).each do |record|
+              entry = map[Sunspot::Adapters::InstanceAdapter.adapt(record).id.to_s]
+              entry.instance_variable_set(:@record, record) if entry
+            end
+          end
+        end
+      end
+      
+      def processed?
+        @processed = false unless defined?(@processed)
+        @processed
+      end
+      
+      # Get the record represented by this entry.
+      def record
+        @record ||= Sunspot::Adapters::DataAccessor.create(Sunspot::Util.full_const_get(record_class_name)).load_all([record_id]).first
+      end
+
+      # Set the error message on an entry. Implementations must implement this method.
+      def set_error! (error, retry_interval = nil)
+        raise NotImplementedError.new("set_error!")
+      end
+
+      # Reset an entry to be executed again immediatel and clear any errors. Implementations must implement this method.
+      def reset!
+        raise NotImplementedError.new("reset!")
+      end
+    end
+  end
+end

data/lib/sunspot/index_queue/entry/active_record_impl.rb

@@ -0,0 +1,140 @@
+require 'active_record'
+
+module Sunspot
+  class IndexQueue
+    module Entry
+      # Implementation of an indexing queue backed by ActiveRecord.
+      #
+      # To create the table, you should have a migration containing the following:
+      #
+      #   self.up
+      #     Sunspot::IndexQueue::Entry::ActiveRecordImpl.create_table
+      #   end
+      #
+      #   self.down
+      #     drop_table Sunspot::IndexQueue::Entry::ActiveRecordImpl.table_name
+      #   end
+      class ActiveRecordImpl < ActiveRecord::Base
+        include Entry
+        
+        set_table_name :sunspot_index_queue_entries
+
+        class << self
+          # Implementation of the total_count method.
+          def total_count (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
+            count(:conditions => conditions)
+          end
+          
+          # Implementation of the ready_count method.
+          def ready_count (queue)
+            conditions = ["#{connection.quote_column_name('run_at')} <= ?", Time.now.utc]
+            unless queue.class_names.empty?
+              conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
+              conditions << queue.class_names
+            end
+            count(:conditions => conditions)
+          end
+
+          # Implementation of the error_count method.
+          def error_count (queue)
+            conditions = ["#{connection.quote_column_name('error')} IS NOT NULL"]
+            unless queue.class_names.empty?
+              conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
+              conditions << queue.class_names
+            end
+            count(:conditions => conditions)
+          end
+
+          # Implementation of the errors method.
+          def errors (queue, limit, offset)
+            conditions = ["#{connection.quote_column_name('error')} IS NOT NULL"]
+            unless queue.class_names.empty?
+              conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
+              conditions << queue.class_names
+            end
+            all(:conditions => conditions, :limit => limit, :offset => offset, :order => :id)
+          end
+
+          # Implementation of the reset! method.
+          def reset! (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
+            update_all({:run_at => Time.now.utc, :attempts => 0, :error => nil, :lock => nil}, conditions)
+          end
+          
+          # Implementation of the next_batch! method.
+          def next_batch! (queue)
+            conditions = ["#{connection.quote_column_name('run_at')} <= ?", Time.now.utc]
+            unless queue.class_names.empty?
+              conditions.first << " AND #{connection.quote_column_name('record_class_name')} IN (?)"
+              conditions << queue.class_names
+            end
+            batch_entries = all(:select => "id", :conditions => conditions, :limit => queue.batch_size, :order => 'priority DESC, run_at')
+            queue_entry_ids = batch_entries.collect{|entry| entry.id}
+            return [] if queue_entry_ids.empty?
+            lock = rand(0x7FFFFFFF)
+            update_all({:run_at => queue.retry_interval.from_now.utc, :lock => lock, :error => nil}, :id => queue_entry_ids)
+            all(:conditions => {:id => queue_entry_ids, :lock => lock})
+          end
+
+          # Implementation of the add method.
+          def add (klass, id, delete, priority)
+            queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
+            queue_entry = first(:conditions => queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
+            queue_entry.is_delete = delete
+            queue_entry.priority = priority if priority > queue_entry.priority
+            queue_entry.run_at = Time.now.utc
+            queue_entry.save!
+          end
+          
+          # Implementation of the delete_entries method.
+          def delete_entries (ids)
+            delete_all(:id => ids)
+          end
+          
+          # Create the table used to store the queue in the database.
+          def create_table
+            connection.create_table table_name do |t|
+              t.string :record_class_name, :null => false
+              t.string :record_id, :null => false
+              t.boolean :is_delete, :null => false, :default => false
+              t.datetime :run_at, :null => false
+              t.integer :priority, :null => false, :default => 0
+              t.integer :lock, :null => true
+              t.string :error, :null => true, :limit => 4000
+              t.integer :attempts, :null => false, :default => 0
+            end
+
+            connection.add_index table_name, [:record_class_name, :record_id], :name => "#{table_name}_record"
+            connection.add_index table_name, [:run_at, :record_class_name, :priority], :name => "#{table_name}_run_at"
+          end
+        end
+
+        # Implementation of the set_error! method.
+        def set_error! (error, retry_interval = nil)
+          self.attempts += 1
+          self.run_at = (retry_interval * attempts).from_now.utc if retry_interval
+          self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"
+          self.lock = nil
+          begin
+            save!
+          rescue => e
+            if logger
+              logger.warn(error)
+              logger.warn(e)
+            end
+          end
+        end
+
+        # Implementation of the reset! method.
+        def reset!
+          begin
+            update_attributes!(:attempts => 0, :error => nil, :lock => nil, :run_at => Time.now.utc)
+          rescue => e
+            logger.warn(e)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sunspot/index_queue/entry/data_mapper_impl.rb

@@ -0,0 +1,122 @@
+require 'dm-core'
+require 'dm-aggregates'
+
+module Sunspot
+  class IndexQueue
+    module Entry
+      # Implementation of an indexing queue backed by ActiveRecord.
+      #
+      # To create the table, you should have a migration containing the following:
+      #
+      #   self.up
+      #     Sunspot::IndexQueue::Entry::ActiveRecordImpl.create_table
+      #   end
+      #
+      #   self.down
+      #     drop_table Sunspot::IndexQueue::Entry::ActiveRecordImpl.table_name
+      #   end
+      class DataMapperImpl
+        include DataMapper::Resource
+        include Entry
+        
+        storage_names[:default] = "sunspot_index_queue_entries"
+        property :id, Serial
+        property :run_at, Time, :index => :run_at
+        property :record_class_name, String, :index => [:record, :run_at]
+        property :record_id, String, :index => [:record]
+        property :priority, Integer, :default => 0, :index => :run_at
+        property :is_delete, Boolean
+        property :lock, Integer
+        property :error, String
+        property :attempts, Integer, :default => 0
+        
+        class << self
+          # Implementation of the total_count method.
+          def total_count (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
+            count(conditions)
+          end
+          
+          # Implementation of the ready_count method.
+          def ready_count (queue)
+            conditions = {:run_at.lte => Time.now.utc}
+            conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
+            count(conditions)
+          end
+
+          # Implementation of the error_count method.
+          def error_count (queue)
+            conditions = {:error.not => nil}
+            conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
+            count(conditions)
+          end
+
+          # Implementation of the errors method.
+          def errors (queue, limit, offset)
+            conditions = {:error.not => nil}
+            conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
+            all(conditions.merge(:limit => limit, :offset => offset, :order => :id))
+          end
+
+          # Implementation of the reset! method.
+          def reset! (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => queue.class_names}
+            all(conditions).update!(:run_at => Time.now.utc, :attempts => 0, :error => nil, :lock => nil)
+          end
+          
+          # Implementation of the next_batch! method.
+          def next_batch! (queue)
+            conditions = {:run_at.lte => Time.now.utc}
+            conditions[:record_class_name] = queue.class_names unless queue.class_names.empty?
+            batch_entries = all(conditions.merge(:fields => [:id], :limit => queue.batch_size, :order => [:priority.desc, :run_at]))
+            queue_entry_ids = batch_entries.collect{|entry| entry.id}
+            return [] if queue_entry_ids.empty?
+            lock = rand(0x7FFFFFFF)
+            all(:id => queue_entry_ids).update!(:run_at => Time.now.utc + queue.retry_interval, :lock => lock, :error => nil)
+            all(:id => queue_entry_ids, :lock => lock)
+          end
+
+          # Implementation of the add method.
+          def add (klass, id, delete, priority)
+            queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
+            queue_entry = first(:conditions => queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
+            queue_entry.is_delete = delete
+            queue_entry.priority = priority if priority > queue_entry.priority
+            queue_entry.run_at = Time.now.utc
+            queue_entry.save!
+          end
+          
+          # Implementation of the delete_entries method.
+          def delete_entries (ids)
+            all(:id => ids).destroy!
+          end
+        end
+
+        # Implementation of the set_error! method.
+        def set_error! (error, retry_interval = nil)
+          self.attempts += 1
+          self.run_at = Time.now.utc + (retry_interval * attempts) if retry_interval
+          self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"
+          self.lock = nil
+          begin
+            save!
+          rescue => e
+            if DataMapper.logger
+              DataMapper.logger.warn(error)
+              DataMapper.logger.warn(e)
+            end
+          end
+        end
+
+        # Implementation of the reset! method.
+        def reset!
+          begin
+            update!(:attempts => 0, :error => nil, :lock => nil, :run_at => Time.now.utc)
+          rescue => e
+            DataMapper.logger.warn(e)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sunspot/index_queue/entry/mongo_impl.rb

@@ -0,0 +1,276 @@
+require 'mongo'
+
+module Sunspot
+  class IndexQueue
+    module Entry
+      # Implementation of an indexing queue backed by MongoDB (http://mongodb.org/). This implementation
+      # uses the mongo gem directly and so is independent of any ORM you may be using.
+      #
+      # To set it up, you need to set the connection and database that it will use.
+      #
+      #   Sunspot::IndexQueue::Entry::MongoImpl.connection = 'localhost'
+      #   Sunspot::IndexQueue::Entry::MongoImpl.database = 'my_database'
+      #   # or
+      #   Sunspot::IndexQueue::Entry::MongoImpl.connection = Mongo::Connection.new('localhost', 27017)
+      #   Sunspot::IndexQueue::Entry::MongoImpl.database = 'my_database'
+      class MongoImpl
+        include Entry
+
+        class << self
+          # Set the connection to MongoDB. The args can either be a Mongo::Connection object, or the args
+          # that can be used to create a new Mongo::Connection.
+          def connection= (*args)
+            @connection = args.first.is_a?(Mongo::Connection) ? args.first : Mongo::Connection.new(*args)
+          end
+          
+          # Get the connection currently in use.
+          def connection
+            @connection
+          end
+          
+          # Set the name of the database which will contain the queue collection.
+          def database_name= (name)
+            @collection = nil
+            @database_name = name
+          end
+          
+          # Get the collection used to store the queue.
+          def collection
+            unless @collection
+              @collection = connection.db(@database_name)["sunspot_index_queue_entries"]
+              @collection.create_index([[:record_class_name, Mongo::ASCENDING], [:record_id, Mongo::ASCENDING]])
+              @collection.create_index([[:run_at, Mongo::ASCENDING], [:record_class_name, Mongo::ASCENDING], [:priority, Mongo::DESCENDING]])
+            end
+            @collection
+          end
+          
+          # Create a new entry.
+          def create (attributes)
+            entry = new(attributes)
+            entry.save
+            entry
+          end
+          
+          # Find one entry given a selector or object id.
+          def find_one (spec_or_object_id=nil, opts={})
+            doc = collection.find_one(spec_or_object_id, opts)
+            doc ? new(doc) : nil
+          end
+          
+          # Find an array of entries given a selector.
+          def find (selector={}, opts={})
+            collection.find(selector, opts).collect{|doc| new(doc)}
+          end
+          
+          # Logger used to log errors.
+          def logger
+            @logger
+          end
+          
+          # Set the logger used to log errors.
+          def logger= (logger)
+            @logger = logger
+          end
+          
+          # Implementation of the total_count method.
+          def total_count (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => {'$in' => queue.class_names}}
+            collection.find(conditions).count
+          end
+          
+          # Implementation of the ready_count method.
+          def ready_count (queue)
+            conditions = {:run_at => {'$lte' => Time.now.utc}}
+            unless queue.class_names.empty?
+              conditions[:record_class_name] = {'$in' => queue.class_names}
+            end
+            collection.find(conditions).count
+          end
+
+          # Implementation of the error_count method.
+          def error_count (queue)
+            conditions = {:error => {'$ne' => nil}}
+            unless queue.class_names.empty?
+              conditions[:record_class_name] = {'$in' => queue.class_names}
+            end
+            collection.find(conditions).count
+          end
+
+          # Implementation of the errors method.
+          def errors (queue, limit, offset)
+            conditions = {:error => {'$ne' => nil}}
+            unless queue.class_names.empty?
+              conditions[:record_class_name] = {'$in' => queue.class_names}
+            end
+            find(conditions, :limit => limit, :skip => offset, :sort => :id)
+          end
+
+          # Implementation of the reset! method.
+          def reset! (queue)
+            conditions = queue.class_names.empty? ? {} : {:record_class_name => {'$in' => queue.class_names}}
+            collection.update(conditions, {"$set" => {:run_at => Time.now.utc, :attempts => 0, :error => nil}}, :multi => true)
+          end
+          
+          # Implementation of the next_batch! method.
+          def next_batch! (queue)
+            conditions = {:run_at => {'$lte' => Time.now.utc}}
+            unless queue.class_names.empty?
+              conditions[:record_class_name] = {'$in' => queue.class_names}
+            end
+            entries = []
+            while entries.size < queue.batch_size
+              begin
+                lock = rand(0x7FFFFFFF)
+                doc = collection.find_and_modify(:update => {"$set" => {:run_at => Time.now.utc + queue.retry_interval, :error => nil, :lock => lock}}, :query => conditions, :limit => queue.batch_size, :sort => [[:priority, Mongo::DESCENDING], [:run_at, Mongo::ASCENDING]])
+                entries << new(doc)
+              rescue Mongo::OperationFailure
+                break
+              end
+            end
+            entries
+          end
+
+          # Implementation of the add method.
+          def add (klass, id, delete, priority)
+            queue_entry_key = {:record_id => id, :record_class_name => klass.name, :lock => nil}
+            queue_entry = find_one(queue_entry_key) || new(queue_entry_key.merge(:priority => priority))
+            queue_entry.is_delete = delete
+            queue_entry.priority = priority if priority > queue_entry.priority
+            queue_entry.run_at = Time.now.utc
+            queue_entry.save
+          end
+          
+          # Implementation of the delete_entries method.
+          def delete_entries (ids)
+            collection.remove(:_id => {'$in' => ids})
+          end
+        end
+        
+        attr_reader :doc
+        
+        # Create a new entry from a document hash.
+        def initialize (attributes = {})
+          @doc = {}
+          attributes.each do |key, value|
+            @doc[key.to_s] = value
+          end
+          @doc['priority'] = 0 unless doc['priority']
+          @doc['attempts'] = 0 unless doc['attempts']
+        end
+        
+        # Get the entry id.
+        def id
+          doc['_id']
+        end
+        
+        # Get the entry id.
+        def record_class_name
+          doc['record_class_name']
+        end
+        
+        # Set the entry record_class_name.
+        def record_class_name= (value)
+          doc['record_class_name'] =  value.nil? ? nil : value.to_s
+        end
+        
+        # Get the entry id.
+        def record_id
+          doc['record_id']
+        end
+        
+        # Set the entry record_id.
+        def record_id= (value)
+          doc['record_id'] =  value
+        end
+        
+        # Get the entry run_at time.
+        def run_at
+          doc['run_at']
+        end
+        
+        # Set the entry run_at time.
+        def run_at= (value)
+          value = Time.parse(value.to_s) unless value.nil? || value.is_a?(Time)
+          doc['run_at'] =  value.nil? ? nil : value.utc
+        end
+        
+        # Get the entry priority.
+        def priority
+          doc['priority']
+        end
+        
+        # Set the entry priority.
+        def priority= (value)
+          doc['priority'] =  value.to_i
+        end
+        
+        # Get the entry attempts.
+        def attempts
+          doc['attempts'] || 0
+        end
+        
+        # Set the entry attempts.
+        def attempts= (value)
+          doc['attempts'] =  value.to_i
+        end
+        
+        # Get the entry error.
+        def error
+          doc['error']
+        end
+        
+        # Set the entry error.
+        def error= (value)
+          doc['error'] =  value.nil? ? nil : value.to_s
+        end
+        
+        # Get the entry delete entry flag.
+        def is_delete?
+          doc['is_delete']
+        end
+        
+        # Set the entry delete entry flag.
+        def is_delete= (value)
+          doc['is_delete'] =  !!value
+        end
+        
+        # Save the entry to the database.
+        def save
+          id = self.class.collection.save(doc)
+          doc['_id'] = id if id
+        end
+
+        # Implementation of the set_error! method.
+        def set_error! (error, retry_interval = nil)
+          self.attempts += 1
+          self.run_at = (retry_interval * attempts).from_now.utc if retry_interval
+          self.error = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")[0, 4000]}"
+          begin
+            save
+          rescue => e
+            if self.class.logger
+              self.class.logger.warn(error)
+              self.class.logger.warn(e)
+            end
+          end
+        end
+
+        # Implementation of the reset! method.
+        def reset!
+          begin
+            self.error = nil
+            self.attempts = 0
+            self.run_at = Time.now.utc
+            self.save
+          rescue => e
+            self.class.logger.warn(e) if self.class.logger
+          end
+        end
+        
+        def == (value)
+          value.is_a?(self.class) && ((id && id == value.id) || (doc == value.doc))
+        end
+      end
+    end
+  end
+end