watson-acts_as_ferret 0.4.8.2

data/lib/ar_mysql_auto_reconnect_patch.rb

@@ -0,0 +1,41 @@
+# Source: http://pastie.caboo.se/154842
+#
+# in /etc/my.cnf on the MySQL server, you can set the interactive-timeout parameter,
+# for example, 12 hours = 28800 sec
+# interactive-timeout=28800
+
+# in ActiveRecord, setting the verification_timeout to something less than
+# the interactive-timeout parameter; 14400 sec = 6 hours
+ActiveRecord::Base.verification_timeout = 14400
+ActiveRecord::Base.establish_connection
+
+# Below is a monkey patch for keeping ActiveRecord connections alive.
+# http://www.sparecycles.org/2007/7/2/saying-goodbye-to-lost-connections-in-rails
+
+module ActiveRecord
+  module ConnectionAdapters
+    class MysqlAdapter
+      def execute(sql, name = nil) #:nodoc:
+        reconnect_lost_connections = true
+        begin
+          log(sql, name) { @connection.query(sql) }
+        rescue ActiveRecord::StatementInvalid => exception
+          if reconnect_lost_connections and exception.message =~ /(Lost connection to MySQL server during query
+|MySQL server has gone away)/
+            reconnect_lost_connections = false
+            reconnect!
+            retry
+          elsif exception.message.split(":").first =~ /Packets out of order/
+            raise ActiveRecord::StatementInvalid, "'Packets out of order' error was received from the database.
+ Please update your mysql bindings (gem install mysql) and read http://dev.mysql.com/doc/mysql/en/password-hash
+ing.html for more information.  If you're on Windows, use the Instant Rails installer to get the updated mysql 
+bindings." 
+          else
+            raise
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/blank_slate.rb

@@ -0,0 +1,54 @@
+module ActsAsFerret
+  if defined?(BasicObject)
+    # Ruby 1.9.x
+    class BlankSlate < BasicObject
+    end
+  elsif defined?(BlankSlate)
+    # Rails 2.x has it already
+    class BlankSlate < ::BlankSlate
+    end
+  else
+    # 'backported' for Rails pre 2.0
+    #
+    #--
+    # Copyright 2004, 2006 by Jim Weirich (jim@weirichhouse.org).
+    # All rights reserved.
+
+    # Permission is granted for use, copying, modification, distribution,
+    # and distribution of modified versions of this work as long as the
+    # above copyright notice is included.
+    #++
+
+    ######################################################################
+    # BlankSlate provides an abstract base class with no predefined
+    # methods (except for <tt>\_\_send__</tt> and <tt>\_\_id__</tt>).
+    # BlankSlate is useful as a base class when writing classes that
+    # depend upon <tt>method_missing</tt> (e.g. dynamic proxies).
+    #
+    class BlankSlate
+      class << self
+        # Hide the method named +name+ in the BlankSlate class.  Don't
+        # hide +instance_eval+ or any method beginning with "__".
+        def hide(name)
+          if instance_methods.include?(name.to_s) and name !~ /^(__|instance_eval|methods)/
+            @hidden_methods ||= {}
+            @hidden_methods[name.to_sym] = instance_method(name)
+            undef_method name
+          end
+        end
+
+        # Redefine a previously hidden method so that it may be called on a blank
+        # slate object.
+        #
+        # no-op here since we don't hide the methods we reveal where this is
+        # used in this implementation
+        def reveal(name)
+        end
+      end
+
+      instance_methods.each { |m| hide(m) }
+
+    end
+
+  end
+end

data/lib/bulk_indexer.rb

@@ -0,0 +1,56 @@
+module ActsAsFerret
+  class BulkIndexer
+    def initialize(args = {})
+      @batch_size = args[:batch_size] || 1000
+      @logger = args[:logger]
+      @model = args[:model]
+      @work_done = 0
+      @indexed_records = 0
+      @total_time = 0.0
+      @index = args[:index]
+      if args[:reindex]
+        @reindex = true
+        @model_count  = @model.count.to_f
+      else
+        @model_count = args[:total]
+      end
+    end
+
+    def index_records(records, offset)
+      batch_time = measure_time {
+        docs = []
+        records.each { |rec| docs << [rec.to_doc, rec.ferret_analyzer] if rec.ferret_enabled?(true) }
+        @index.update_batch(docs)
+      }.to_f
+      rec_count = records.size
+      @indexed_records += rec_count
+      @total_time += batch_time
+      @work_done = @indexed_records.to_f / @model_count * 100.0 if @model_count > 0
+      @logger.debug "took #{batch_time} to index last #{rec_count} records. #{records_waiting} records to go. Avg time per record: #{avg_time_per_record}"
+      remaining_time = avg_time_per_record * records_waiting
+      @logger.info "#{@reindex ? 're' : 'bulk '}index model #{@model.name} : #{'%.2f' % @work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+    end
+    
+    def measure_time
+      t1 = Time.now
+      yield
+      Time.now - t1
+    end
+    
+    protected
+    
+    def avg_time_per_record
+      if @indexed_records > 0
+        @total_time / @indexed_records
+      else
+        0
+      end
+    end
+    
+    def records_waiting
+      @model_count - @indexed_records
+    end
+
+  end
+
+end

data/lib/class_methods.rb

@@ -0,0 +1,279 @@
+module ActsAsFerret
+        
+  module ClassMethods
+
+    # Disables ferret index updates for this model. When a block is given,
+    # Ferret will be re-enabled again after executing the block.
+    def disable_ferret
+      aaf_configuration[:enabled] = false
+      if block_given?
+        yield
+        enable_ferret
+      end
+    end
+
+    def enable_ferret
+      aaf_configuration[:enabled] = true
+    end
+
+    def ferret_enabled?
+      aaf_configuration[:enabled]
+    end
+
+    # rebuild the index from all data stored for this model, and any other
+    # model classes associated with the same index.
+    # This is called automatically when no index exists yet.
+    #
+    def rebuild_index
+      aaf_index.rebuild_index
+    end
+
+    # re-index a number records specified by the given ids. Use for large
+    # indexing jobs i.e. after modifying a lot of records with Ferret disabled.
+    # Please note that the state of Ferret (enabled or disabled at class or
+    # record level) is not checked by this method, so if you need to do so
+    # (e.g. because of a custom ferret_enabled? implementation), you have to do
+    # so yourself.
+    def bulk_index(*ids)
+      options = Hash === ids.last ? ids.pop : {}
+      ids = ids.first if ids.size == 1 && ids.first.is_a?(Enumerable)
+      aaf_index.bulk_index(self.name, ids, options)
+    end
+
+    # true if our db and table appear to be suitable for the mysql fast batch
+    # hack (see
+    # http://weblog.jamisbuck.org/2007/4/6/faking-cursors-in-activerecord)
+    def use_fast_batches?
+      if connection.class.name =~ /Mysql/ && primary_key == 'id' && aaf_configuration[:mysql_fast_batches]
+        logger.info "using mysql specific batched find :all. Turn off with  :mysql_fast_batches => false if you encounter problems (i.e. because of non-integer UUIDs in the id column)"
+        true
+      end
+    end
+
+    # Returns all records modified or created after the specified time.
+    # Used by the rake rebuild task to find models that need to be updated in
+    # the index after the rebuild finished because they changed while the
+    # rebuild was running.
+    # Override if your models don't stick to the created_at/updated_at
+    # convention.
+    def records_modified_since(time)
+      condition = []
+      %w(updated_at created_at).each do |col|
+        condition << "#{col} >= ?" if column_names.include? col
+      end
+      if condition.empty?
+        logger.warn "#{self.name}: Override records_modified_since(time) to keep the index up to date with records changed during rebuild."
+        []
+      else
+        find :all, :conditions => [ condition.join(' AND '), *([time]*condition.size) ]
+      end
+    end
+
+    # runs across all records yielding those to be indexed when the index is rebuilt
+    def records_for_rebuild(batch_size = 1000)
+      transaction do
+        if use_fast_batches?
+          offset = 0
+          while (rows = find :all, :conditions => [ "#{table_name}.id > ?", offset ], :limit => batch_size).any?
+            offset = rows.last.id
+            yield rows, offset
+          end
+        else
+          order = "#{primary_key} ASC" # fixes #212
+          0.step(self.count, batch_size) do |offset|
+            yield find( :all, :limit => batch_size, :offset => offset, :order => order ), offset
+          end
+        end
+      end
+    end
+
+    # yields the records with the given ids, in batches of batch_size
+    def records_for_bulk_index(ids, batch_size = 1000)
+      transaction do
+        offset = 0
+        ids.each_slice(batch_size) do |id_slice|
+          records = find( :all, :conditions => ["id in (?)", id_slice] )
+          #yield records, offset
+          yield find( :all, :conditions => ["id in (?)", id_slice] ), offset
+          offset += batch_size
+        end
+      end
+    end
+
+    # Retrieve the index instance for this model class. This can either be a
+    # LocalIndex, or a RemoteIndex instance.
+    # 
+    def aaf_index
+      @index ||= ActsAsFerret::get_index(aaf_configuration[:name])
+    end 
+    
+    # Finds instances by searching the Ferret index. Terms are ANDed by default, use 
+    # OR between terms for ORed queries. Or specify +:or_default => true+ in the
+    # +:ferret+ options hash of acts_as_ferret.
+    #
+    # You may either use the +offset+ and +limit+ options to implement your own
+    # pagination logic, or use the +page+ and +per_page+ options to use the
+    # built in pagination support which is compatible with will_paginate's view
+    # helpers. If +page+ and +per_page+ are given, +offset+ and +limit+ will be
+    # ignored.
+    #
+    # == options:
+    # page::        page of search results to retrieve
+    # per_page::    number of search results that are displayed per page
+    # offset::      first hit to retrieve (useful for paging)
+    # limit::       number of hits to retrieve, or :all to retrieve
+    #               all results
+    # lazy::        Array of field names whose contents should be read directly
+    #               from the index. Those fields have to be marked 
+    #               +:store => :yes+ in their field options. Give true to get all
+    #               stored fields. Note that if you have a shared index, you have 
+    #               to explicitly state the fields you want to fetch, true won't
+    #               work here)
+    #
+    # +find_options+ is a hash passed on to active_record's find when
+    # retrieving the data from db, useful to i.e. prefetch relationships with
+    # :include or to specify additional filter criteria with :conditions (only string and array syntax supported).
+    # You can also call find_with_ferret inside named or dynamic scopes, if you like the conditions hash syntax more.
+    #
+    # This method returns a +SearchResults+ instance, which really is an Array that has 
+    # been decorated with a total_hits attribute holding the total number of hits.
+    # Additionally, SearchResults is compatible with the pagination helper
+    # methods of the will_paginate plugin.
+    #
+    # Please keep in mind that the number of results delivered might be less than 
+    # +limit+ if you specify any active record conditions that further limit 
+    # the result. Use +limit+ and +offset+ as AR find_options instead.
+    # +page+ and +per_page+ are supposed to work regardless of any 
+    # +conditions+ present in +find_options+.
+    def find_with_ferret(q, options = {}, find_options = {})
+      if respond_to?(:scope) && scope(:find, :conditions)
+        find_options[:conditions] ||= '1=1' # treat external scope the same as if :conditions present (i.e. when it comes to counting results)
+      end
+      return ActsAsFerret::find q, self, options, find_options
+    end 
+
+
+    # Returns the total number of hits for the given query 
+    #
+    # Note that since we don't query the database here, this method won't deliver 
+    # the expected results when used on an AR association.
+    #
+    def total_hits(q, options={})
+      aaf_index.total_hits(q, options)
+    end
+
+    # Finds instance model name, ids and scores by contents. 
+    # Useful e.g. if you want to search across models or do not want to fetch
+    # all result records (yet).
+    #
+    # Options are the same as for find_with_ferret
+    #
+    # A block can be given too, it will be executed with every result:
+    # find_ids_with_ferret(q, options) do |model, id, score|
+    #    id_array << id
+    #    scores_by_id[id] = score 
+    # end
+    # NOTE: in case a block is given, only the total_hits value will be returned
+    # instead of the [total_hits, results] array!
+    # 
+    def find_ids_with_ferret(q, options = {}, &block)
+      aaf_index.find_ids(q, options, &block)
+    end
+
+    # An implementation of http://rm.jkraemer.net/issues/show/161
+    def highlight(id, query, options = {})
+      aaf_index.highlight(id, query, options)
+    end
+
+    def document_number(id)
+      aaf_index.document_number(id)
+    end
+
+    def query_for_record(id)
+      aaf_index.query_for_record(id)
+    end
+    
+    protected
+
+#    def find_records_lazy_or_not(q, options = {}, find_options = {})
+#      if options[:lazy]
+#        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
+#        lazy_find_by_contents q, options
+#      else
+#        ar_find_by_contents q, options, find_options
+#      end
+#    end
+#
+#    def ar_find_by_contents(q, options = {}, find_options = {})
+#      result_ids = {}
+#      total_hits = find_ids_with_ferret(q, options) do |model, id, score, data|
+#        # stores ids, index and score of each hit for later ordering of
+#        # results
+#        result_ids[id] = [ result_ids.size + 1, score ]
+#      end
+#
+#      result = ActsAsFerret::retrieve_records( { self.name => result_ids }, find_options )
+#      
+#      # count total_hits via sql when using conditions or when we're called
+#      # from an ActiveRecord association.
+#      if find_options[:conditions] or caller.find{ |call| call =~ %r{active_record/associations} }
+#        # chances are the ferret result count is not our total_hits value, so
+#        # we correct this here.
+#        if options[:limit] != :all || options[:page] || options[:offset] || find_options[:limit] || find_options[:offset]
+#          # our ferret result has been limited, so we need to re-run that
+#          # search to get the full result set from ferret.
+#          result_ids = {}
+#          find_ids_with_ferret(q, options.update(:limit => :all, :offset => 0)) do |model, id, score, data|
+#            result_ids[id] = [ result_ids.size + 1, score ]
+#          end
+#          # Now ask the database for the total size of the final result set.
+#          total_hits = count_records( { self.name => result_ids }, find_options )
+#        else
+#          # what we got from the database is our full result set, so take
+#          # it's size
+#          total_hits = result.length
+#        end
+#      end
+#
+#      [ total_hits, result ]
+#    end
+#
+#    def lazy_find_by_contents(q, options = {})
+#      logger.debug "lazy_find_by_contents: #{q}"
+#      result = []
+#      rank   = 0
+#      total_hits = find_ids_with_ferret(q, options) do |model, id, score, data|
+#        logger.debug "model: #{model}, id: #{id}, data: #{data}"
+#        result << FerretResult.new(model, id, score, rank += 1, data)
+#      end
+#      [ total_hits, result ]
+#    end
+
+
+    def model_find(model, id, find_options = {})
+      model.constantize.find(id, find_options)
+    end
+
+
+#    def count_records(id_arrays, find_options = {})
+#      count_options = find_options.dup
+#      count_options.delete :limit
+#      count_options.delete :offset
+#      count = 0
+#      id_arrays.each do |model, id_array|
+#        next if id_array.empty?
+#        model = model.constantize
+#        # merge conditions
+#        conditions = ActsAsFerret::combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], 
+#                                        find_options[:conditions])
+#        opts = find_options.merge :conditions => conditions
+#        opts.delete :limit; opts.delete :offset
+#        count += model.count opts
+#      end
+#      count
+#    end
+
+  end
+  
+end
+

data/lib/ferret_extensions.rb

@@ -0,0 +1,192 @@
+module Ferret
+
+  module Analysis
+  
+    # = PerFieldAnalyzer
+    #
+    # This PerFieldAnalyzer is a workaround to a memory leak in 
+    # ferret 0.11.4. It does basically do the same as the original
+    # Ferret::Analysis::PerFieldAnalyzer, but without the leak :)
+    # 
+    # http://ferret.davebalmain.com/api/classes/Ferret/Analysis/PerFieldAnalyzer.html
+    #
+    # Thanks to Ben from omdb.org for tracking this down and creating this
+    # workaround.
+    # You can read more about the issue there:
+    # http://blog.omdb-beta.org/2007/7/29/tracking-down-a-memory-leak-in-ferret-0-11-4
+    class PerFieldAnalyzer < ::Ferret::Analysis::Analyzer
+      def initialize( default_analyzer = StandardAnalyzer.new )
+        @analyzers = {}
+        @default_analyzer = default_analyzer
+      end
+            
+      def add_field( field, analyzer )
+        @analyzers[field] = analyzer
+      end
+      alias []= add_field
+                
+      def token_stream(field, string)
+        @analyzers.has_key?(field) ? @analyzers[field].token_stream(field, string) : 
+          @default_analyzer.token_stream(field, string)
+      end
+    end
+  end
+
+  class Index::Index
+    attr_accessor :batch_size, :logger
+
+    def index_models(models)
+      models.each { |model| index_model model }
+      flush
+      optimize
+      close
+      ActsAsFerret::close_multi_indexes
+    end
+
+    def index_model(model)
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :reindex => true)
+      logger.info "reindexing model #{model.name}"
+
+      model.records_for_rebuild(@batch_size) do |records, offset|
+        bulk_indexer.index_records(records, offset)
+      end
+    end
+
+    def bulk_index(model, ids, options = {})
+      options.reverse_merge! :optimize => true
+      orig_flush = @auto_flush
+      @auto_flush = false
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :total => ids.size)
+      model.records_for_bulk_index(ids, @batch_size) do |records, offset|
+        logger.debug "#{model} bulk indexing #{records.size} at #{offset}"
+        bulk_indexer.index_records(records, offset)
+      end
+      logger.info 'finishing bulk index...'
+      flush
+      if options[:optimize]
+        logger.info 'optimizing...'
+        optimize 
+      end
+      @auto_flush = orig_flush
+    end
+    
+
+    # bulk-inserts a number of ferret documents.
+    # The argument has to be an array of two-element arrays each holding the document data and the analyzer to 
+    # use for this document (which may be nil).
+    def update_batch(document_analyzer_pairs)
+      ids = document_analyzer_pairs.collect {|da| da.first[@id_field] }
+      @dir.synchronize do
+        batch_delete(ids)
+        ensure_writer_open()
+        document_analyzer_pairs.each do |doc, analyzer|
+          if analyzer
+            old_analyzer = @writer.analyzer
+            @writer.analyzer = analyzer
+            @writer.add_document(doc)
+            @writer.analyzer = old_analyzer
+          else
+            @writer.add_document(doc)
+          end
+        end
+        flush()
+      end      
+    end
+
+    # search for the first document with +arg+ in the +id+ field and return it's internal document number. 
+    # The +id+ field is either :id or whatever you set
+    # :id_field parameter to when you create the Index object.
+    def doc_number(id)
+      @dir.synchronize do
+        ensure_reader_open()
+        term_doc_enum = @reader.term_docs_for(@id_field, id.to_s)
+        return term_doc_enum.next? ? term_doc_enum.doc : nil
+      end
+    end
+    
+    private
+    
+    
+    # If +docs+ is a Hash or an Array then a batch delete will be performed.
+    # If +docs+ is an Array then it will be considered an array of +id+'s. If
+    # it is a Hash, then its keys will be used instead as the Array of
+    # document +id+'s. If the +id+ is an Integers then it is considered a
+    # Ferret document number and the corresponding document will be deleted.
+    # If the +id+ is a String or a Symbol then the +id+ will be considered a
+    # term and the documents that contain that term in the +:id_field+ will
+    # be deleted.
+    #
+    # docs:: An Array of docs to be deleted, or a Hash (in which case the keys
+    # are used)
+    #
+    # ripped from Ferret trunk.
+    def batch_delete(docs)
+      docs = docs.keys if docs.is_a?(Hash)
+      raise ArgumentError, "must pass Array or Hash" unless docs.is_a? Array
+      ids = []
+      terms = []
+      docs.each do |doc|
+        case doc
+        when String   then terms << doc
+        when Symbol   then terms << doc.to_s
+        when Integer  then ids << doc
+        else
+          raise ArgumentError, "Cannot delete for arg of type #{id.class}"
+        end
+      end
+      if ids.size > 0
+        ensure_reader_open
+        ids.each {|id| @reader.delete(id)}
+      end
+      if terms.size > 0
+        ensure_writer_open()
+        terms.each { |t| @writer.delete(@id_field, t) }
+        # TODO with Ferret trunk this would work:
+        # @writer.delete(@id_field, terms)
+      end
+      return self
+    end
+    
+  end
+
+  # add marshalling support to SortFields
+  class Search::SortField
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      case string
+        when /<DOC(_ID)?>!/         then Ferret::Search::SortField::DOC_ID_REV
+        when /<DOC(_ID)?>/          then Ferret::Search::SortField::DOC_ID
+        when '<SCORE>!'             then Ferret::Search::SortField::SCORE_REV
+        when '<SCORE>'              then Ferret::Search::SortField::SCORE
+        when /^(\w+):<(\w+)>(!)?$/  then new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
+        else raise "invalid value: #{string}"
+      end
+    end
+  end
+
+  # add marshalling support to Sort
+  class Search::Sort
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      # we exclude the last <DOC> sorting as it is appended by new anyway
+      if string =~ /^Sort\[(.*?)(<DOC>(!)?)?\]$/
+        sort_fields = $1.split(',').map do |value| 
+          value.strip!
+          Ferret::Search::SortField._load value unless value.blank?
+        end
+        new sort_fields.compact
+      else
+        raise "invalid value: #{string}"
+      end
+    end
+  end
+
+end