acts_as_ferret 0.4.3 → 0.4.4

data/lib/ar_mysql_auto_reconnect_patch.rb

@@ -0,0 +1,41 @@
+# Source: http://pastie.caboo.se/154842
+#
+# in /etc/my.cnf on the MySQL server, you can set the interactive-timeout parameter,
+# for example, 12 hours = 28800 sec
+# interactive-timeout=28800
+
+# in ActiveRecord, setting the verification_timeout to something less than
+# the interactive-timeout parameter; 14400 sec = 6 hours
+ActiveRecord::Base.verification_timeout = 14400
+ActiveRecord::Base.establish_connection
+
+# Below is a monkey patch for keeping ActiveRecord connections alive.
+# http://www.sparecycles.org/2007/7/2/saying-goodbye-to-lost-connections-in-rails
+
+module ActiveRecord
+  module ConnectionAdapters
+    class MysqlAdapter
+      def execute(sql, name = nil) #:nodoc:
+        reconnect_lost_connections = true
+        begin
+          log(sql, name) { @connection.query(sql) }
+        rescue ActiveRecord::StatementInvalid => exception
+          if reconnect_lost_connections and exception.message =~ /(Lost connection to MySQL server during query
+|MySQL server has gone away)/
+            reconnect_lost_connections = false
+            reconnect!
+            retry
+          elsif exception.message.split(":").first =~ /Packets out of order/
+            raise ActiveRecord::StatementInvalid, "'Packets out of order' error was received from the database.
+ Please update your mysql bindings (gem install mysql) and read http://dev.mysql.com/doc/mysql/en/password-hash
+ing.html for more information.  If you're on Windows, use the Instant Rails installer to get the updated mysql 
+bindings." 
+          else
+            raise
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/blank_slate.rb

@@ -0,0 +1,53 @@
+if defined?(BlankSlate)
+  # Rails 2.x has it already
+  module ActsAsFerret
+    class BlankSlate < ::BlankSlate
+    end
+  end
+else
+  module ActsAsFerret
+    # 'backported' for Rails pre 2.0
+    #
+    #--
+    # Copyright 2004, 2006 by Jim Weirich (jim@weirichhouse.org).
+    # All rights reserved.
+
+    # Permission is granted for use, copying, modification, distribution,
+    # and distribution of modified versions of this work as long as the
+    # above copyright notice is included.
+    #++
+
+    ######################################################################
+    # BlankSlate provides an abstract base class with no predefined
+    # methods (except for <tt>\_\_send__</tt> and <tt>\_\_id__</tt>).
+    # BlankSlate is useful as a base class when writing classes that
+    # depend upon <tt>method_missing</tt> (e.g. dynamic proxies).
+    #
+    class BlankSlate
+      class << self
+        # Hide the method named +name+ in the BlankSlate class.  Don't
+        # hide +instance_eval+ or any method beginning with "__".
+        def hide(name)
+          if instance_methods.include?(name.to_s) and name !~ /^(__|instance_eval|methods)/
+            @hidden_methods ||= {}
+            @hidden_methods[name.to_sym] = instance_method(name)
+            undef_method name
+          end
+        end
+
+        # Redefine a previously hidden method so that it may be called on a blank
+        # slate object.
+        #
+        # no-op here since we don't hide the methods we reveal where this is
+        # used in this implementation
+        def reveal(name)
+        end
+      end
+
+      instance_methods.each { |m| hide(m) }
+
+    end
+  end
+
+end
+

data/lib/bulk_indexer.rb

@@ -16,7 +16,10 @@ module ActsAsFerret
 
     def index_records(records, offset)
       batch_time = measure_time {
-        records.each { |rec| @index << rec.to_doc if rec.ferret_enabled?(true) }
+        docs = []
+        records.each { |rec| docs << [rec.to_doc, rec.ferret_analyzer] if rec.ferret_enabled?(true) }
+        @index.update_batch(docs)
+        # records.each { |rec| @index.add_document(rec.to_doc, rec.ferret_analyzer) if rec.ferret_enabled?(true) }
       }.to_f
       @work_done = offset.to_f / @model_count * 100.0 if @model_count > 0
       remaining_time = ( batch_time / @batch_size ) * ( @model_count - offset + @batch_size )

data/lib/class_methods.rb

@@ -20,18 +20,12 @@ module ActsAsFerret
       aaf_configuration[:enabled]
     end
 
-    # rebuild the index from all data stored for this model.
+    # rebuild the index from all data stored for this model, and any other
+    # model classes associated with the same index.
     # This is called automatically when no index exists yet.
     #
-    # When calling this method manually, you can give any additional 
-    # model classes that should also go into this index as parameters. 
-    # Useful when using the :single_index option.
-    # Note that attributes named the same in different models will share
-    # the same field options in the shared index.
-    def rebuild_index(*models)
-      models << self unless models.include?(self)
-      aaf_index.rebuild_index models.map(&:to_s)
-      index_dir = find_last_index_version(aaf_configuration[:index_base_dir]) unless aaf_configuration[:remote]
+    def rebuild_index
+      aaf_index.rebuild_index
     end
 
     # re-index a number records specified by the given ids. Use for large
@@ -43,7 +37,7 @@ module ActsAsFerret
     def bulk_index(*ids)
       options = Hash === ids.last ? ids.pop : {}
       ids = ids.first if ids.size == 1 && ids.first.is_a?(Enumerable)
-      aaf_index.bulk_index(ids, options)
+      aaf_index.bulk_index(self.name, ids, options)
     end
 
     # true if our db and table appear to be suitable for the mysql fast batch
@@ -56,6 +50,25 @@ module ActsAsFerret
       end
     end
 
+    # Returns all records modified or created after the specified time.
+    # Used by the rake rebuild task to find models that need to be updated in
+    # the index after the rebuild finished because they changed while the
+    # rebuild was running.
+    # Override if your models don't stick to the created_at/updated_at
+    # convention.
+    def records_modified_since(time)
+      condition = []
+      %w(updated_at created_at).each do |col|
+        condition << "#{col} >= ?" if column_names.include? col
+      end
+      if condition.empty?
+        logger.warn "#{self.name}: Override records_modified_since(time) to keep the index up to date with records changed during rebuild."
+        []
+      else
+        find :all, :conditions => [ condition.join(' AND '), *([time]*condition.size) ]
+      end
+    end
+
     # runs across all records yielding those to be indexed when the index is rebuilt
     def records_for_rebuild(batch_size = 1000)
       transaction do
@@ -66,8 +79,7 @@ module ActsAsFerret
             yield rows, offset
           end
         else
-          # sql server adapter won't batch correctly without defined ordering
-          order = "#{primary_key} ASC" if connection.class.name =~ /SQLServer/
+          order = "#{primary_key} ASC" # fixes #212
           0.step(self.count, batch_size) do |offset|
             yield find( :all, :limit => batch_size, :offset => offset, :order => order ), offset
           end
@@ -80,9 +92,7 @@ module ActsAsFerret
       transaction do
         offset = 0
         ids.each_slice(batch_size) do |id_slice|
-          logger.debug "########## slice: #{id_slice.join(',')}"
           records = find( :all, :conditions => ["id in (?)", id_slice] )
-          logger.debug "########## slice records: #{records.inspect}"
           #yield records, offset
           yield find( :all, :conditions => ["id in (?)", id_slice] ), offset
           offset += batch_size
@@ -90,23 +100,11 @@ module ActsAsFerret
       end
     end
 
-    # Switches this class to a new index located in dir.
-    # Used by the DRb server when switching to a new index version.
-    def index_dir=(dir)
-      logger.debug "changing index dir to #{dir}"
-      aaf_configuration[:index_dir] = aaf_configuration[:ferret][:path] = dir
-      aaf_index.reopen!
-      logger.debug "index dir is now #{dir}"
-    end
-    
     # Retrieve the index instance for this model class. This can either be a
     # LocalIndex, or a RemoteIndex instance.
     # 
-    # Index instances are stored in a hash, using the index directory
-    # as the key. So model classes sharing a single index will share their
-    # Index object, too.
     def aaf_index
-      ActsAsFerret::ferret_indexes[aaf_configuration[:index_dir]] ||= create_index_instance
+      @index ||= ActsAsFerret::get_index(aaf_configuration[:name])
     end 
     
     # Finds instances by searching the Ferret index. Terms are ANDed by default, use 
@@ -131,12 +129,6 @@ module ActsAsFerret
     #               stored fields. Note that if you have a shared index, you have 
     #               to explicitly state the fields you want to fetch, true won't
     #               work here)
-    # models::      only for single_index scenarios: an Array of other Model classes to 
-    #               include in this search. Use :all to query all models.
-    # multi::       Specify additional model classes to search through. Each of
-    #               these, as well as this class, has to have the
-    #               :store_class_name option set to true. This option replaces the
-    #               multi_search method.
     #
     # +find_options+ is a hash passed on to active_record's find when
     # retrieving the data from db, useful to i.e. prefetch relationships with
@@ -153,61 +145,23 @@ module ActsAsFerret
     # +page+ and +per_page+ are supposed to work regardless of any 
     # +conitions+ present in +find_options+.
     def find_with_ferret(q, options = {}, find_options = {})
-      if options[:per_page]
-        options[:page] = options[:page] ? options[:page].to_i : 1
-        limit = options[:per_page]
-        offset = (options[:page] - 1) * limit
-        if find_options[:conditions] && !options[:multi]
-          find_options[:limit] = limit
-          find_options[:offset] = offset
-          options[:limit] = :all
-          options.delete :offset
-        else
-          # do pagination with ferret (or after everything is done in the case
-          # of multi_search)
-          options[:limit] = limit
-          options[:offset] = offset
-        end
-      elsif find_options[:conditions]
-        if options[:multi]
-          # multisearch ignores find_options limit and offset
-          options[:limit] ||= find_options.delete(:limit)
-          options[:offset] ||= find_options.delete(:offset)
+      if respond_to?(:scope) && scope(:find, :conditions)
+        if find_options[:conditions]
+          find_options[:conditions] = "(#{find_options[:conditions]}) AND (#{scope(:find, :conditions)})"
         else
-          # let the db do the limiting and offsetting for single-table searches
-          find_options[:limit] ||= options.delete(:limit)
-          find_options[:offset] ||= options.delete(:offset)
-          options[:limit] = :all
+          find_options[:conditions] = scope(:find, :conditions)
         end
       end
-
-      total_hits, result = if options[:multi].blank?
-        find_records_lazy_or_not q, options, find_options
-      else
-        _multi_search q, options.delete(:multi), options, find_options
-      end
-      logger.debug "Query: #{q}\ntotal hits: #{total_hits}, results delivered: #{result.size}"
-      SearchResults.new(result, total_hits, options[:page], options[:per_page])
+      return ActsAsFerret::find q, self, options, find_options
     end 
-    alias find_by_contents find_with_ferret
 
-   
 
     # Returns the total number of hits for the given query 
-    # To count the results of a query across multiple models, specify an array of 
-    # class names with the :multi option.
     #
     # Note that since we don't query the database here, this method won't deliver 
     # the expected results when used on an AR association.
+    #
     def total_hits(q, options={})
-      if options[:models]
-        # backwards compatibility
-        logger.warn "the :models option of total_hits is deprecated, please use :multi instead"
-        options[:multi] = options[:models] 
-      end
-      if models = options[:multi]
-        options[:multi] = add_self_to_model_list_if_necessary(models).map(&:to_s)
-      end
       aaf_index.total_hits(q, options)
     end
 
@@ -215,243 +169,100 @@ module ActsAsFerret
     # Useful e.g. if you want to search across models or do not want to fetch
     # all result records (yet).
     #
-    # Options are the same as for find_by_contents
+    # Options are the same as for find_with_ferret
     #
     # A block can be given too, it will be executed with every result:
-    # find_id_by_contents(q, options) do |model, id, score|
+    # find_ids_with_ferret(q, options) do |model, id, score|
     #    id_array << id
     #    scores_by_id[id] = score 
     # end
     # NOTE: in case a block is given, only the total_hits value will be returned
     # instead of the [total_hits, results] array!
     # 
-    def find_id_by_contents(q, options = {}, &block)
-      deprecated_options_support(options)
-      aaf_index.find_id_by_contents(q, options, &block)
+    def find_ids_with_ferret(q, options = {}, &block)
+      aaf_index.find_ids(q, options, &block)
     end
 
     
-    # returns an array of hashes, each containing :class_name,
-    # :id and :score for a hit.
-    #
-    # if a block is given, class_name, id and score of each hit will 
-    # be yielded, and the total number of hits is returned.
-    def id_multi_search(query, additional_models = [], options = {}, &proc)
-      deprecated_options_support(options)
-      models = add_self_to_model_list_if_necessary(additional_models)
-      aaf_index.id_multi_search(query, models.map(&:to_s), options, &proc)
-    end
-    
-
     protected
 
-    def _multi_search(query, additional_models = [], options = {}, find_options = {})
-      result = []
-
-      if options[:lazy]
-        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
-        total_hits = id_multi_search(query, additional_models, options) do |model, id, score, data|
-          result << FerretResult.new(model, id, score, data)
-        end
-      else
-        id_arrays = {}
-        rank = 0
-
-        limit = options.delete(:limit)
-        offset = options.delete(:offset) || 0
-        options[:limit] = :all
-        total_hits = id_multi_search(query, additional_models, options) do |model, id, score, data|
-          id_arrays[model] ||= {}
-          id_arrays[model][id] = [ rank += 1, score ]
-        end
-        result = retrieve_records(id_arrays, find_options)
-        total_hits = result.size if find_options[:conditions]
- #       total_hits += offset if offset
-        if limit && limit != :all
-          result = result[offset..limit+offset-1]
-        end
-      end
-      [total_hits, result]
-    end
-
-    def add_self_to_model_list_if_necessary(models)
-      models = [ models ] unless models.is_a? Array
-      models << self unless models.include?(self)
-      models
-    end
-
-    def find_records_lazy_or_not(q, options = {}, find_options = {})
-      if options[:lazy]
-        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
-        lazy_find_by_contents q, options
-      else
-        ar_find_by_contents q, options, find_options
-      end
-    end
-
-    def ar_find_by_contents(q, options = {}, find_options = {})
-      result_ids = {}
-      total_hits = find_id_by_contents(q, options) do |model, id, score, data|
-        # stores ids, index and score of each hit for later ordering of
-        # results
-        result_ids[id] = [ result_ids.size + 1, score ]
-      end
-
-      result = retrieve_records( { self.name => result_ids }, find_options )
-      
-      # count total_hits via sql when using conditions or when we're called
-      # from an ActiveRecord association.
-      if find_options[:conditions] or caller.find{ |call| call =~ %r{active_record/associations} }
-        # chances are the ferret result count is not our total_hits value, so
-        # we correct this here.
-        if options[:limit] != :all || options[:page] || options[:offset] || find_options[:limit] || find_options[:offset]
-          # our ferret result has been limited, so we need to re-run that
-          # search to get the full result set from ferret.
-          result_ids = {}
-          find_id_by_contents(q, options.update(:limit => :all, :offset => 0)) do |model, id, score, data|
-            result_ids[id] = [ result_ids.size + 1, score ]
-          end
-          # Now ask the database for the total size of the final result set.
-          total_hits = count_records( { self.name => result_ids }, find_options )
-        else
-          # what we got from the database is our full result set, so take
-          # it's size
-          total_hits = result.length
-        end
-      end
-
-      [ total_hits, result ]
-    end
-
-    def lazy_find_by_contents(q, options = {})
-      result = []
-      total_hits = find_id_by_contents(q, options) do |model, id, score, data|
-        result << FerretResult.new(model, id, score, data)
-      end
-      [ total_hits, result ]
-    end
+#    def find_records_lazy_or_not(q, options = {}, find_options = {})
+#      if options[:lazy]
+#        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
+#        lazy_find_by_contents q, options
+#      else
+#        ar_find_by_contents q, options, find_options
+#      end
+#    end
+#
+#    def ar_find_by_contents(q, options = {}, find_options = {})
+#      result_ids = {}
+#      total_hits = find_ids_with_ferret(q, options) do |model, id, score, data|
+#        # stores ids, index and score of each hit for later ordering of
+#        # results
+#        result_ids[id] = [ result_ids.size + 1, score ]
+#      end
+#
+#      result = ActsAsFerret::retrieve_records( { self.name => result_ids }, find_options )
+#      
+#      # count total_hits via sql when using conditions or when we're called
+#      # from an ActiveRecord association.
+#      if find_options[:conditions] or caller.find{ |call| call =~ %r{active_record/associations} }
+#        # chances are the ferret result count is not our total_hits value, so
+#        # we correct this here.
+#        if options[:limit] != :all || options[:page] || options[:offset] || find_options[:limit] || find_options[:offset]
+#          # our ferret result has been limited, so we need to re-run that
+#          # search to get the full result set from ferret.
+#          result_ids = {}
+#          find_ids_with_ferret(q, options.update(:limit => :all, :offset => 0)) do |model, id, score, data|
+#            result_ids[id] = [ result_ids.size + 1, score ]
+#          end
+#          # Now ask the database for the total size of the final result set.
+#          total_hits = count_records( { self.name => result_ids }, find_options )
+#        else
+#          # what we got from the database is our full result set, so take
+#          # it's size
+#          total_hits = result.length
+#        end
+#      end
+#
+#      [ total_hits, result ]
+#    end
+#
+#    def lazy_find_by_contents(q, options = {})
+#      logger.debug "lazy_find_by_contents: #{q}"
+#      result = []
+#      rank   = 0
+#      total_hits = find_ids_with_ferret(q, options) do |model, id, score, data|
+#        logger.debug "model: #{model}, id: #{id}, data: #{data}"
+#        result << FerretResult.new(model, id, score, rank += 1, data)
+#      end
+#      [ total_hits, result ]
+#    end
 
 
     def model_find(model, id, find_options = {})
       model.constantize.find(id, find_options)
     end
 
-    # retrieves search result records from a data structure like this:
-    # { 'Model1' => { '1' => [ rank, score ], '2' => [ rank, score ] }
-    #
-    # TODO: in case of STI AR will filter out hits from other 
-    # classes for us, but this
-    # will lead to less results retrieved --> scoping of ferret query
-    # to self.class is still needed.
-    # from the ferret ML (thanks Curtis Hatter)
-    # > I created a method in my base STI class so I can scope my query. For scoping
-    # > I used something like the following line:
-    # > 
-    # > query << " role:#{self.class.eql?(Contents) '*' : self.class}"
-    # > 
-    # > Though you could make it more generic by simply asking
-    # > "self.descends_from_active_record?" which is how rails decides if it should
-    # > scope your "find" query for STI models. You can check out "base.rb" in
-    # > activerecord to see that.
-    # but maybe better do the scoping in find_id_by_contents...
-    def retrieve_records(id_arrays, find_options = {})
-      result = []
-      # get objects for each model
-      id_arrays.each do |model, id_array|
-        next if id_array.empty?
-        begin
-          model = model.constantize
-        rescue
-          raise "Please use ':store_class_name => true' if you want to use multi_search.\n#{$!}"
-        end
 
-        # merge conditions
-        conditions = combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", 
-                                          id_array.keys ], 
-                                        find_options[:conditions])
-
-        # check for include association that might only exist on some models in case of multi_search
-        filtered_include_options = []
-        if include_options = find_options[:include]
-          include_options = [ include_options ] unless include_options.respond_to?(:each)
-          include_options.each do |include_option|
-            filtered_include_options << include_option if model.reflections.has_key?(include_option.is_a?(Hash) ? include_option.keys[0].to_sym : include_option.to_sym)
-          end
-        end
-        filtered_include_options = nil if filtered_include_options.empty?
-
-        # fetch
-        tmp_result = model.find(:all, find_options.merge(:conditions => conditions, 
-                                                         :include => filtered_include_options))
-
-        # set scores and rank
-        tmp_result.each do |record|
-          record.ferret_rank, record.ferret_score = id_array[record.id.to_s]
-        end
-        # merge with result array
-        result.concat tmp_result
-      end
-      
-      # order results as they were found by ferret, unless an AR :order
-      # option was given
-      result.sort! { |a, b| a.ferret_rank <=> b.ferret_rank } unless find_options[:order]
-      return result
-    end
-
-    def count_records(id_arrays, find_options = {})
-      count_options = find_options.dup
-      count_options.delete :limit
-      count_options.delete :offset
-      count = 0
-      id_arrays.each do |model, id_array|
-        next if id_array.empty?
-        begin
-          model = model.constantize
-          # merge conditions
-          conditions = combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], 
-                                          find_options[:conditions])
-          opts = find_options.merge :conditions => conditions
-          opts.delete :limit; opts.delete :offset
-          count += model.count opts
-        rescue TypeError
-          raise "#{model} must use :store_class_name option if you want to use multi_search against it.\n#{$!}"
-        end
-      end
-      count
-    end
-
-    def deprecated_options_support(options)
-      if options[:num_docs]
-        logger.warn ":num_docs is deprecated, use :limit instead!"
-        options[:limit] ||= options[:num_docs]
-      end
-      if options[:first_doc]
-        logger.warn ":first_doc is deprecated, use :offset instead!"
-        options[:offset] ||= options[:first_doc]
-      end
-    end
-
-    # creates a new Index instance.
-    def create_index_instance
-      if aaf_configuration[:remote]
-       RemoteIndex
-      elsif aaf_configuration[:single_index]
-        SharedIndex
-      else
-        LocalIndex
-      end.new(aaf_configuration)
-    end
-
-    # combine our conditions with those given by user, if any
-    def combine_conditions(conditions, additional_conditions = [])
-      returning conditions do
-        if additional_conditions && additional_conditions.any?
-          cust_opts = additional_conditions.respond_to?(:shift) ? additional_conditions.dup : [ additional_conditions ]
-          conditions.first << " and " << cust_opts.shift
-          conditions.concat(cust_opts)
-        end
-      end
-    end
+#    def count_records(id_arrays, find_options = {})
+#      count_options = find_options.dup
+#      count_options.delete :limit
+#      count_options.delete :offset
+#      count = 0
+#      id_arrays.each do |model, id_array|
+#        next if id_array.empty?
+#        model = model.constantize
+#        # merge conditions
+#        conditions = ActsAsFerret::combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], 
+#                                        find_options[:conditions])
+#        opts = find_options.merge :conditions => conditions
+#        opts.delete :limit; opts.delete :offset
+#        count += model.count opts
+#      end
+#      count
+#    end
 
   end