acts_as_background_solr 0.5

data/README

@@ -0,0 +1,86 @@
+= acts_as_background_solr Rails plugin
+This plugin extends the functionality of the acts_as_solr plugin to
+provide for a disconnected background job that synchronizes data with:
+Solr in batch. acts_as_solr works by sending changes do Solr for each
+object immediately following any change. While this is nice as changes
+are immediately viewable, it has a few drawbacks:
+
+  * Invoking commit on solr requires a new searcher to be opened which
+    is slow
+
+  * There is no way to keep track of an object that was saved when
+    notification to solr failed (or when the database transaction
+    rolled back)
+
+Acts as background solr extends the acts_as_solr plugin to focus on
+background processing.
+
+There is one other major changes. Acts as solr calls Model.find for
+each result from the result set. Acts as background solr will
+reconstitute your objects from the attributes stored in solr
+completely avoiding any required database hits when searching against
+solr. This requires that you modify schema.xml to store the fields you
+are indexing.
+
+== Installation
+Use this in place of acts_as_solr as in your models, e.g. 
+
+  acts_as_background_solr
+
+The options :background, :if and :auto_commit will be automatically
+overridden.
+
+Each model can track changes in one of two ways:
+
+  * Default: Explicitly log changes from the model using listeners
+
+  * Database triggers: You can instead use db triggers on your model
+    tables to track changes. If you use this method, set the option
+    :db_triggers => true
+
+Example:
+
+class User < ActiveRecord::Bae
+
+  acts_as_background_solr :additional_fields => [:first_name, :last_name], :exclude_fields => ['encrypted_password'], :db_triggers => true 
+
+end
+
+This plugin depends on the following table structure (this is written
+for postgresql):
+
+create sequence solr_sync_records_seq start with 1;
+create table solr_sync_records (
+  id                integer constraint solr_sync_records_id_pk primary key default nextval('solr_sync_records_seq'),
+  model             varchar(50) not null,
+  model_id          integer not null,
+  created_at        timestamp default now() not null
+);
+
+-- common access path
+create index solr_sync_records_model_id_idx on solr_sync_records(model, model_id);
+
+To update the actual data stored in Solr, you need to invoke the
+following method:
+
+  SolrBatch.process_all
+
+We're using openwfe to schedule this job to run every few minutes, but
+any scheduler should work.
+
+This method updates records in bulk, issuing a single commit when
+records are updated. The current algorithm updates up to 5,000 records
+per model in a single call to this method. The way this works is each
+call to SolrBatch.process_all has a default batch size of 500. Each
+call will attempt up to 10 iterations, w/ each iteration updating up
+to 500 records.
+
+If you want to change the batch size, provide your own implementation
+of SolrBatch.process_all
+
+== Authors
+Michael Bryzek
+mbryzek<at>alum.mit.edu
+
+== Release Information
+Released under the MIT license.

data/init.rb

@@ -0,0 +1,21 @@
+# Copyright (c) 2007 Michael Bryzek
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'acts_as_background_solr'

data/lib/acts_as_background_solr.rb

@@ -0,0 +1,177 @@
+# Copyright (c) 2007 Michael Bryzek
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require File.dirname(__FILE__) + '/solr_batch'
+require File.dirname(__FILE__) + '/class_methods'
+require File.dirname(__FILE__) + '/instance_methods'
+
+# Example:
+# class User < CachedModel
+#  acts_as_background_solr :additional_fields => [:first_name, :last_name], :exclude_fields => ['encrypted_password'], :db_triggers => true
+# end
+
+module ActsAsBackgroundSolr
+  module ActsMethods
+
+    def acts_as_background_solr(options={}, solr_options={})
+      extend ClassMethods
+      include InstanceMethods
+
+      options[:auto_commit] = false
+      options[:if] = true
+      options[:background] = false
+
+      acts_as_solr(options, solr_options)
+
+      unless options[:db_triggers] 
+        ## Allows use to setup database triggers to monitor changes to the objects
+        after_save    :solr_flag_record
+        after_destroy :solr_flag_record
+      end
+
+
+      ## Override implementations of acts_as_solr/parse_* - there were
+      ## no easy hooks to override how to prevent the database queries
+      ## from occurring.
+
+      # Method used by mostly all the ClassMethods when doing a search
+      def self.parse_query(query=nil, options={}, models=nil)
+        valid_options = [:offset, :limit, :facets, :models, :results_format, :order, :scores, :operator]
+        query_options = {}
+        return if query.nil?
+        raise "Invalid parameters: #{(options.keys - valid_options).join(',')}" unless (options.keys - valid_options).empty?
+        begin
+          ActsAsSolr::Deprecation.validate_query(options)
+          query_options[:start] = options[:offset]
+          query_options[:rows] = options[:limit]
+          query_options[:operator] = options[:operator]
+          
+          # first steps on the facet parameter processing
+          if options[:facets]
+            query_options[:facets] = {}
+            query_options[:facets][:limit] = -1  # TODO: make this configurable
+            query_options[:facets][:sort] = :count if options[:facets][:sort]
+            query_options[:facets][:mincount] = 0
+            query_options[:facets][:mincount] = 1 if options[:facets][:zeros] == false
+            query_options[:facets][:fields] = options[:facets][:fields].collect{|k| "#{k}_facet"} if options[:facets][:fields]
+            query_options[:filter_queries] = replace_types(options[:facets][:browse].collect{|k| "#{k.sub!(/ *: */,"_facet:")}"}) if options[:facets][:browse]
+            query_options[:facets][:queries] = replace_types(options[:facets][:query].collect{|k| "#{k.sub!(/ *: */,"_t:")}"}) if options[:facets][:query]
+          end
+          
+          query_options[:field_list] = ['*', 'score']
+          query = "(#{query.gsub(/ *: */,"_t:")}) #{models}"
+          query.gsub!('pk_t','pk_i')
+          ## Automatically filter to this model
+          query = "type_t:#{self} AND #{query}"  unless query.match(/type_t:/)
+          logger.debug("SOLR QUERY: #{query}")
+          order = options[:order].split(/\s*,\s*/).collect{|e| e.gsub(/\s+/,'_t ')  }.join(',') if options[:order] 
+          query_options[:query] = replace_types([query])[0] # TODO adjust replace_types to work with String or Array  
+          
+          if options[:order]
+            # TODO: set the sort parameter instead of the old ;order. style.
+            query_options[:query] << ';' << replace_types([order], false)[0]
+          end
+          
+          ActsAsSolr::Post.execute(Solr::Request::Standard.new(query_options))
+        rescue
+          raise "There was a problem executing your search: #{$!}"
+        end            
+      end
+
+    
+      ## Override parse_results to reconstruct options from the actual
+      ## query results instead of having to hit the database
+      def self.parse_results(solr_data, options = {})
+        results = {
+          :docs => [],
+          :total => 0
+        }
+        configuration = {
+          :format => :objects
+        }
+        results.update(:facets => {'facet_fields' => []}) if options[:facets]
+        return ActsAsSolr::SearchResults.new(results) if solr_data.total == 0
+        configuration.update(options) if options.is_a?(Hash)
+        return super unless configuration[:format] == :objects
+
+        result = []
+        solr_data.docs.each do |doc| 
+          object = self.send(:new)
+          doc.each do |k,v| 
+            next if k == "id"
+            if (k == 'pk_i')
+              ## TODO: Not sure if there is a potential for more than one value
+              object.id = v[0]
+            else
+              field = k.sub(/\_[^_]+$/, '')
+              object.send("#{field}=", convert_value(object.class, field, v)) if object.respond_to?("#{field}=")
+            end
+          end
+          result << object
+        end
+        
+        # ids = solr_data.docs.collect {|doc| doc["#{solr_configuration[:primary_key_field]}"]}.flatten
+        # conditions = [ "#{self.table_name}.#{primary_key} in (?)", ids ]
+        # result = configuration[:format] == :objects ? reorder(self.find(:all, :conditions => conditions), ids) : ids
+        add_scores(result, solr_data) if configuration[:format] == :objects && options[:scores]
+        
+        results.update(:facets => solr_data.data['facet_counts']) if options[:facets]
+        results.update({:docs => result, :total => solr_data.total, :max_score => solr_data.max_score})
+        ActsAsSolr::SearchResults.new(results)
+        
+      end
+    end
+
+    private
+    def convert_value(klass, field, value)
+      return nil unless value
+      if value.is_a?(Array)
+        return value if value.size != 1
+        value = value[0]
+      end
+      return value unless value.is_a?(String)
+      column = klass.columns_hash[field]
+      return value.to_str unless column ## Occurs for included fields
+      
+      case column.sql_type.intern
+      when :integer:    return value.to_i
+      when :numeric:    return BigDecimal.new(value)
+      when :float:      return value.to_f
+      when :date:       return Date.parse(value)
+      when :datetime:   return parse_datetime(value)
+      when :time:       return parse_datetime(value)
+      when :boolean:    return Boolean.new(value)
+      end
+      return value.to_str
+    end
+
+    def parse_datetime(value)
+      begin
+        return Time.parse(value)
+      rescue Exception => e
+        raise "Error parsing date \"#{value}\": #{e}"
+      end
+    end
+
+  end
+end
+
+# reopen ActiveRecord and include the acts_as_background_solr method
+ActiveRecord::Base.extend ActsAsBackgroundSolr::ActsMethods

data/lib/class_methods.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2007 Michael Bryzek
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+module ActsAsBackgroundSolr #:nodoc:
+
+  module ClassMethods
+    def solr_sync_pending_records(batch_size=500)
+      SolrBatch.sync_pending_records(self, { :batch_size => batch_size } );
+    end
+
+    def solr_requeue_all
+      SolrBatch.requeue_all(self)
+    end
+
+  end
+  
+end

data/lib/instance_methods.rb

@@ -0,0 +1,11 @@
+module ActsAsBackgroundSolr #:nodoc:
+  
+  module InstanceMethods
+
+    def solr_flag_record
+      SolrBatch.flag_record(self.class, id)
+    end
+    
+  end
+
+end

data/lib/solr_batch.rb

@@ -0,0 +1,136 @@
+class SolrBatch
+
+  ## This procedure should be run in the background with whatever job
+  ## scheduler you are comfortable with
+  def SolrBatch.process_all
+    models = ActiveRecord::Base.connection.select_values("select distinct model from solr_sync_records")
+    if models.size == 0
+      RAILS_DEFAULT_LOGGER.debug("SolrBatch.process_all: No records pending synchronization")
+    else
+      first_klass = nil
+      models.each do |model|
+        klass = eval(model)
+        first_klass = klass unless first_klass
+        SolrBatch.sync_pending_records(klass, { :do_commit => false, :do_optimize => true })
+      end
+
+      ## Commit/optimize after ALL records have been processed
+      first_klass.send(:solr_commit)
+      first_klass.send(:solr_optimize)
+    end
+  end
+
+  ## Flags an individual record as pending synchronization with Solr
+  def SolrBatch.flag_record(model_name, model_id)
+    sql = 'insert into solr_sync_records (model, model_id) values (?, ?)'
+    ActiveRecord::Base.connection.execute(sanitize_sql_array([sql, model_name, model_id]))
+  end
+
+  ## Marks all records for this model as pending synchronization with Solr
+  def SolrBatch.requeue_all(ar_model)
+    sql = <<-endsql
+        insert into solr_sync_records 
+        (model, model_id) 
+        select ?, id from #{ar_model.send(:table_name)} 
+        EXCEPT 
+        select ?, model_id from solr_sync_records where model = ?
+    endsql
+    ActiveRecord::Base.connection.execute(sanitize_sql_array([sql, ar_model, ar_model, ar_model]))
+  end
+
+  # Ex: from within ActiveRecord Model, call:
+  # SolrBatch.sync_pending_records(self, { :batch_size => 500 } );
+  def SolrBatch.sync_pending_records(ar_model, opts = {}, iteration=1)
+    batch_size = opts.delete(:batch_size)
+    batch_size = 500 if batch_size.nil? || batch_size < 1
+    do_commit   = opts.delete(:do_commit) || true
+    do_optimize = opts.delete(:do_optimize) || true
+    raise "Invalid option keys: #{opts.keys}" unless opts.keys.size == 0
+
+    RAILS_DEFAULT_LOGGER.debug("SolrBatch.sync_pending_records(#{ar_model})")
+    items = get_pending_records(ar_model, batch_size)
+    solr_log(ar_model, 'No records to process') and return if items.size == 0
+
+    delete_batch = []
+    add_batch = []
+    processed = {}
+    items.each do |o|
+      solr_id = o.send(:solr_id)
+      if !processed.has_key?(solr_id)  ## Duplicates are possible in the log table
+        processed[solr_id] = true
+        if o.solr_sync_record_deleted_p == 't'
+          delete_batch << solr_id
+        else
+          add_batch << o.to_solr_doc
+        end
+      end
+    end
+    
+    # We delete in bulk with a single query
+    ActsAsSolr::Post.execute(Solr::Request::Delete.new(:query => id_query(delete_batch))) if delete_batch.size > 0
+    ar_model.send(:solr_add, add_batch) if add_batch.size > 0
+    SolrBatch.delete_from_queue(ar_model, items.last.solr_sync_records_log_id)
+
+    solr_log(ar_model, "#{items.size} items have been batch added to index.")
+
+    if (items.size >= batch_size && iteration < 10)
+      ## More records of this class to process
+      SolrBatch.sync_pending_records(ar_model, opts, iteration + 1)
+    else 
+      ar_model.send(:solr_commit)   if do_commit
+      ar_model.send(:solr_optimize) if do_optimize
+    end
+  end
+  
+  private
+  def SolrBatch.solr_log(ar_model, message)
+    RAILS_DEFAULT_LOGGER.debug("Solr Sync for #{ar_model.name}: #{message}")
+  end
+
+  ## Removes all records from the solr pending sync queue for this
+  ## model up to the specified sync id
+  def SolrBatch.delete_from_queue(ar_model, last_sync_id)
+    sql = 'delete from solr_sync_records where model = ? and id <= ?'
+    ActiveRecord::Base.connection.execute(sanitize_sql_array([sql, ar_model, last_sync_id]))
+  end
+  
+  ## Turns a list of IDs into a single solr query matching all those
+  ## IDs
+  def SolrBatch.id_query(solr_ids)
+    s = nil
+    solr_ids.each do |id|
+      if s
+        s << ' OR '
+      else
+        s = ''
+      end
+      s << "id:" << id.sub(/:/, '\:')
+    end
+    s
+  end
+
+  ## Returns an array of records for the specified active record model
+  ## that are pending synchronization
+  def SolrBatch.get_pending_records(ar_model, batch_size)
+    sql = <<-endsql
+      select log.id as solr_sync_records_log_id, log.model_id as solr_sync_records_model_id, 
+             case when ar.id is null then 't' else 'f' end as solr_sync_record_deleted_p,
+             ar.*
+        from solr_sync_records log
+        left outer join #{ar_model.send('table_name')} ar on ar.id = log.model_id
+       order by log.id asc
+       limit #{batch_size}
+    endsql
+      
+    ar_model.send(:find_by_sql, sql)
+  end
+
+  private
+  def SolrBatch.sanitize_sql_array(ary)
+    statement = ary.shift
+    connection = ActiveRecord::Base.connection
+    ary.each { |v| statement.sub!('?', "'#{connection.quote_string(v.to_s)}'") }
+    statement
+  end
+
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: acts_as_background_solr
+version: !ruby/object:Gem::Version 
+  version: "0.5"
+date: 2007-09-09 00:00:00 -04:00
+summary: Extends the functionality of the acts_as_solr plugin to provide for disconnected background job processing
+require_paths: 
+- lib
+email: mbryzek@alum.mit.edu
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: acts_as_backbround_solr
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Michael Bryzek
+files: 
+- init.rb
+- lib/acts_as_background_solr.rb
+- lib/class_methods.rb
+- lib/instance_methods.rb
+- lib/solr_batch.rb
+- README
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+