sdsykes_acts_as_ferret 0.4.3.1

data/lib/acts_as_ferret.rb

@@ -0,0 +1,151 @@
+# Copyright (c) 2006 Kasper Weibel Nielsen-Refs, Thomas Lockney, Jens Krämer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'active_support'
+require 'active_record'
+require 'set'
+require 'enumerator'
+require 'ferret'
+
+require 'bulk_indexer'
+require 'ferret_extensions'
+require 'act_methods'
+require 'search_results'
+require 'class_methods'
+require 'shared_index_class_methods'
+require 'ferret_result'
+require 'instance_methods'
+
+require 'multi_index'
+require 'more_like_this'
+
+require 'index'
+require 'local_index'
+require 'shared_index'
+require 'remote_index'
+
+require 'ferret_server'
+
+
+# The Rails ActiveRecord Ferret Mixin.
+#
+# This mixin adds full text search capabilities to any Rails model.
+#
+# The current version emerged from on the original acts_as_ferret plugin done by
+# Kasper Weibel and a modified version done by Thomas Lockney, which  both can be 
+# found on the Ferret Wiki: http://ferret.davebalmain.com/trac/wiki/FerretOnRails.
+#
+# basic usage:
+# include the following in your model class (specifiying the fields you want to get indexed):
+# acts_as_ferret :fields => [ :title, :description ]
+#
+# now you can use ModelClass.find_by_contents(query) to find instances of your model
+# whose indexed fields match a given query. All query terms are required by default, but 
+# explicit OR queries are possible. This differs from the ferret default, but imho is the more
+# often needed/expected behaviour (more query terms result in less results).
+#
+# Released under the MIT license.
+#
+# Authors: 
+# Kasper Weibel Nielsen-Refs (original author)
+# Jens Kraemer <jk@jkraemer.net> (active maintainer)
+#
+module ActsAsFerret
+
+  # global Hash containing all multi indexes created by all classes using the plugin
+  # key is the concatenation of alphabetically sorted names of the classes the
+  # searcher searches.
+  @@multi_indexes = Hash.new
+  def self.multi_indexes; @@multi_indexes end
+
+  # global Hash containing the ferret indexes of all classes using the plugin
+  # key is the index directory.
+  @@ferret_indexes = Hash.new
+  def self.ferret_indexes; @@ferret_indexes end
+
+ 
+  
+  def self.ensure_directory(dir)
+    FileUtils.mkdir_p dir unless (File.directory?(dir) || File.symlink?(dir))
+  end
+  
+  # make sure the default index base dir exists. by default, all indexes are created
+  # under RAILS_ROOT/index/RAILS_ENV
+  def self.init_index_basedir
+    index_base = "#{RAILS_ROOT}/index"
+    @@index_dir = "#{index_base}/#{RAILS_ENV}"
+  end
+  
+  mattr_accessor :index_dir
+  init_index_basedir
+  
+  def self.append_features(base)
+    super
+    base.extend(ClassMethods)
+  end
+  
+  # builds a FieldInfos instance for creation of an index containing fields
+  # for the given model classes.
+  def self.field_infos(models)
+    # default attributes for fields
+    fi = Ferret::Index::FieldInfos.new(:store => :no, 
+                                        :index => :yes, 
+                                        :term_vector => :no,
+                                        :boost => 1.0)
+    # primary key
+    fi.add_field(:id, :store => :yes, :index => :untokenized) 
+    fields = {}
+    have_class_name = false
+    models.each do |model|
+      fields.update(model.aaf_configuration[:ferret_fields])
+      # class_name
+      if !have_class_name && model.aaf_configuration[:store_class_name]
+        fi.add_field(:class_name, :store => :yes, :index => :untokenized) 
+        have_class_name = true
+      end
+    end
+    fields.each_pair do |field, options|
+      options = options.dup
+      options.delete(:boost) if options[:boost].is_a?(Symbol)
+      fi.add_field(field, { :store => :no, 
+                            :index => :yes }.update(options)) 
+    end
+    return fi
+  end
+
+  def self.close_multi_indexes
+    # close combined index readers, just in case
+    # this seems to fix a strange test failure that seems to relate to a
+    # multi_index looking at an old version of the content_base index.
+    multi_indexes.each_pair do |key, index|
+      # puts "#{key} -- #{self.name}"
+      # TODO only close those where necessary (watch inheritance, where
+      # self.name is base class of a class where key is made from)
+      index.close #if key =~ /#{self.name}/
+    end
+    multi_indexes.clear
+  end
+
+end
+
+# include acts_as_ferret method into ActiveRecord::Base
+ActiveRecord::Base.extend ActsAsFerret::ActMethods
+
+

data/lib/bulk_indexer.rb

@@ -0,0 +1,35 @@
+module ActsAsFerret
+  class BulkIndexer
+    def initialize(args = {})
+      @batch_size = args[:batch_size] || 1000
+      @logger = args[:logger]
+      @model = args[:model]
+      @work_done = 0
+      @index = args[:index]
+      if args[:reindex]
+        @reindex = true
+        @model_count  = @model.count.to_f
+      else
+        @model_count = args[:total]
+      end
+    end
+
+    def index_records(records, offset)
+      batch_time = measure_time {
+        records.each { |rec| @index << rec.to_doc if rec.ferret_enabled?(true) }
+      }.to_f
+      @work_done = offset.to_f / @model_count * 100.0 if @model_count > 0
+      remaining_time = ( batch_time / @batch_size ) * ( @model_count - offset + @batch_size )
+      @logger.info "#{@reindex ? 're' : 'bulk '}index model #{@model.name} : #{'%.2f' % @work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+
+    end
+
+    def measure_time
+      t1 = Time.now
+      yield
+      Time.now - t1
+    end
+
+  end
+
+end

data/lib/class_methods.rb

@@ -0,0 +1,459 @@
+module ActsAsFerret
+        
+  module ClassMethods
+
+    # Disables ferret index updates for this model. When a block is given,
+    # Ferret will be re-enabled again after executing the block.
+    def disable_ferret
+      aaf_configuration[:enabled] = false
+      if block_given?
+        yield
+        enable_ferret
+      end
+    end
+
+    def enable_ferret
+      aaf_configuration[:enabled] = true
+    end
+
+    def ferret_enabled?
+      aaf_configuration[:enabled]
+    end
+
+    # rebuild the index from all data stored for this model.
+    # This is called automatically when no index exists yet.
+    #
+    # When calling this method manually, you can give any additional 
+    # model classes that should also go into this index as parameters. 
+    # Useful when using the :single_index option.
+    # Note that attributes named the same in different models will share
+    # the same field options in the shared index.
+    def rebuild_index(*models)
+      models << self unless models.include?(self)
+      aaf_index.rebuild_index models.map(&:to_s)
+      index_dir = find_last_index_version(aaf_configuration[:index_base_dir]) unless aaf_configuration[:remote]
+    end
+
+    # re-index a number records specified by the given ids. Use for large
+    # indexing jobs i.e. after modifying a lot of records with Ferret disabled.
+    # Please note that the state of Ferret (enabled or disabled at class or
+    # record level) is not checked by this method, so if you need to do so
+    # (e.g. because of a custom ferret_enabled? implementation), you have to do
+    # so yourself.
+    def bulk_index(*ids)
+      options = Hash === ids.last ? ids.pop : {}
+      ids = ids.first if ids.size == 1 && ids.first.is_a?(Enumerable)
+      aaf_index.bulk_index(ids, options)
+    end
+
+    # true if our db and table appear to be suitable for the mysql fast batch
+    # hack (see
+    # http://weblog.jamisbuck.org/2007/4/6/faking-cursors-in-activerecord)
+    def use_fast_batches?
+      if connection.class.name =~ /Mysql/ && primary_key == 'id' && aaf_configuration[:mysql_fast_batches]
+        logger.info "using mysql specific batched find :all. Turn off with  :mysql_fast_batches => false if you encounter problems (i.e. because of non-integer UUIDs in the id column)"
+        true
+      end
+    end
+
+    # runs across all records yielding those to be indexed when the index is rebuilt
+    def records_for_rebuild(batch_size = 1000)
+      transaction do
+        if use_fast_batches?
+          offset = 0
+          while (rows = find :all, :conditions => [ "#{table_name}.id > ?", offset ], :limit => batch_size).any?
+            offset = rows.last.id
+            yield rows, offset
+          end
+        else
+          # sql server adapter won't batch correctly without defined ordering
+          order = "#{primary_key} ASC" if connection.class.name =~ /SQLServer/
+          0.step(self.count, batch_size) do |offset|
+            yield find( :all, :limit => batch_size, :offset => offset, :order => order ), offset
+          end
+        end
+      end
+    end
+
+    # yields the records with the given ids, in batches of batch_size
+    def records_for_bulk_index(ids, batch_size = 1000)
+      transaction do
+        offset = 0
+        ids.each_slice(batch_size) do |id_slice|
+          logger.debug "########## slice: #{id_slice.join(',')}"
+          records = find( :all, :conditions => ["id in (?)", id_slice] )
+          logger.debug "########## slice records: #{records.inspect}"
+          #yield records, offset
+          yield find( :all, :conditions => ["id in (?)", id_slice] ), offset
+          offset += batch_size
+        end
+      end
+    end
+
+    # Switches this class to a new index located in dir.
+    # Used by the DRb server when switching to a new index version.
+    def index_dir=(dir)
+      logger.debug "changing index dir to #{dir}"
+      aaf_configuration[:index_dir] = aaf_configuration[:ferret][:path] = dir
+      aaf_index.reopen!
+      logger.debug "index dir is now #{dir}"
+    end
+    
+    # Retrieve the index instance for this model class. This can either be a
+    # LocalIndex, or a RemoteIndex instance.
+    # 
+    # Index instances are stored in a hash, using the index directory
+    # as the key. So model classes sharing a single index will share their
+    # Index object, too.
+    def aaf_index
+      ActsAsFerret::ferret_indexes[aaf_configuration[:index_dir]] ||= create_index_instance
+    end 
+    
+    # Finds instances by searching the Ferret index. Terms are ANDed by default, use 
+    # OR between terms for ORed queries. Or specify +:or_default => true+ in the
+    # +:ferret+ options hash of acts_as_ferret.
+    #
+    # You may either use the +offset+ and +limit+ options to implement your own
+    # pagination logic, or use the +page+ and +per_page+ options to use the
+    # built in pagination support which is compatible with will_paginate's view
+    # helpers. If +page+ and +per_page+ are given, +offset+ and +limit+ will be
+    # ignored.
+    #
+    # == options:
+    # page::        page of search results to retrieve
+    # per_page::    number of search results that are displayed per page
+    # offset::      first hit to retrieve (useful for paging)
+    # limit::       number of hits to retrieve, or :all to retrieve
+    #               all results
+    # lazy::        Array of field names whose contents should be read directly
+    #               from the index. Those fields have to be marked 
+    #               +:store => :yes+ in their field options. Give true to get all
+    #               stored fields. Note that if you have a shared index, you have 
+    #               to explicitly state the fields you want to fetch, true won't
+    #               work here)
+    # models::      only for single_index scenarios: an Array of other Model classes to 
+    #               include in this search. Use :all to query all models.
+    # multi::       Specify additional model classes to search through. Each of
+    #               these, as well as this class, has to have the
+    #               :store_class_name option set to true. This option replaces the
+    #               multi_search method.
+    #
+    # +find_options+ is a hash passed on to active_record's find when
+    # retrieving the data from db, useful to i.e. prefetch relationships with
+    # :include or to specify additional filter criteria with :conditions.
+    #
+    # This method returns a +SearchResults+ instance, which really is an Array that has 
+    # been decorated with a total_hits attribute holding the total number of hits.
+    # Additionally, SearchResults is compatible with the pagination helper
+    # methods of the will_paginate plugin.
+    #
+    # Please keep in mind that the number of results delivered might be less than 
+    # +limit+ if you specify any active record conditions that further limit 
+    # the result. Use +limit+ and +offset+ as AR find_options instead.
+    # +page+ and +per_page+ are supposed to work regardless of any 
+    # +conitions+ present in +find_options+.
+    def find_with_ferret(q, options = {}, find_options = {})
+      if options[:per_page]
+        options[:page] = options[:page] ? options[:page].to_i : 1
+        limit = options[:per_page]
+        offset = (options[:page] - 1) * limit
+        if find_options[:conditions] && !options[:multi]
+          find_options[:limit] = limit
+          find_options[:offset] = offset
+          options[:limit] = :all
+          options.delete :offset
+        else
+          # do pagination with ferret (or after everything is done in the case
+          # of multi_search)
+          options[:limit] = limit
+          options[:offset] = offset
+        end
+      elsif find_options[:conditions]
+        if options[:multi]
+          # multisearch ignores find_options limit and offset
+          options[:limit] ||= find_options.delete(:limit)
+          options[:offset] ||= find_options.delete(:offset)
+        else
+          # let the db do the limiting and offsetting for single-table searches
+          find_options[:limit] ||= options.delete(:limit)
+          find_options[:offset] ||= options.delete(:offset)
+          options[:limit] = :all
+        end
+      end
+
+      total_hits, result = if options[:multi].blank?
+        find_records_lazy_or_not q, options, find_options
+      else
+        _multi_search q, options.delete(:multi), options, find_options
+      end
+      logger.debug "Query: #{q}\ntotal hits: #{total_hits}, results delivered: #{result.size}"
+      SearchResults.new(result, total_hits, options[:page], options[:per_page])
+    end 
+    alias find_by_contents find_with_ferret
+
+   
+
+    # Returns the total number of hits for the given query 
+    # To count the results of a query across multiple models, specify an array of 
+    # class names with the :multi option.
+    #
+    # Note that since we don't query the database here, this method won't deliver 
+    # the expected results when used on an AR association.
+    def total_hits(q, options={})
+      if options[:models]
+        # backwards compatibility
+        logger.warn "the :models option of total_hits is deprecated, please use :multi instead"
+        options[:multi] = options[:models] 
+      end
+      if models = options[:multi]
+        options[:multi] = add_self_to_model_list_if_necessary(models).map(&:to_s)
+      end
+      aaf_index.total_hits(q, options)
+    end
+
+    # Finds instance model name, ids and scores by contents. 
+    # Useful e.g. if you want to search across models or do not want to fetch
+    # all result records (yet).
+    #
+    # Options are the same as for find_by_contents
+    #
+    # A block can be given too, it will be executed with every result:
+    # find_id_by_contents(q, options) do |model, id, score|
+    #    id_array << id
+    #    scores_by_id[id] = score 
+    # end
+    # NOTE: in case a block is given, only the total_hits value will be returned
+    # instead of the [total_hits, results] array!
+    # 
+    def find_id_by_contents(q, options = {}, &block)
+      deprecated_options_support(options)
+      aaf_index.find_id_by_contents(q, options, &block)
+    end
+
+    
+    # returns an array of hashes, each containing :class_name,
+    # :id and :score for a hit.
+    #
+    # if a block is given, class_name, id and score of each hit will 
+    # be yielded, and the total number of hits is returned.
+    def id_multi_search(query, additional_models = [], options = {}, &proc)
+      deprecated_options_support(options)
+      models = add_self_to_model_list_if_necessary(additional_models)
+      aaf_index.id_multi_search(query, models.map(&:to_s), options, &proc)
+    end
+    
+
+    protected
+
+    def _multi_search(query, additional_models = [], options = {}, find_options = {})
+      result = []
+
+      if options[:lazy]
+        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
+        total_hits = id_multi_search(query, additional_models, options) do |model, id, score, data|
+          result << FerretResult.new(model, id, score, data)
+        end
+      else
+        id_arrays = {}
+        rank = 0
+
+        limit = options.delete(:limit)
+        offset = options.delete(:offset) || 0
+        options[:limit] = :all
+        total_hits = id_multi_search(query, additional_models, options) do |model, id, score, data|
+          id_arrays[model] ||= {}
+          id_arrays[model][id] = [ rank += 1, score ]
+        end
+        result = retrieve_records(id_arrays, find_options)
+        total_hits = result.size if find_options[:conditions]
+ #       total_hits += offset if offset
+        if limit && limit != :all
+          result = result[offset..limit+offset-1]
+        end
+      end
+      [total_hits, result]
+    end
+
+    def add_self_to_model_list_if_necessary(models)
+      models = [ models ] unless models.is_a? Array
+      models << self unless models.include?(self)
+      models
+    end
+
+    def find_records_lazy_or_not(q, options = {}, find_options = {})
+      if options[:lazy]
+        logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
+        lazy_find_by_contents q, options
+      else
+        ar_find_by_contents q, options, find_options
+      end
+    end
+
+    def ar_find_by_contents(q, options = {}, find_options = {})
+      result_ids = {}
+      total_hits = find_id_by_contents(q, options) do |model, id, score, data|
+        # stores ids, index and score of each hit for later ordering of
+        # results
+        result_ids[id] = [ result_ids.size + 1, score ]
+      end
+
+      result = retrieve_records( { self.name => result_ids }, find_options )
+      
+      # count total_hits via sql when using conditions or when we're called
+      # from an ActiveRecord association.
+      if find_options[:conditions] or caller.find{ |call| call =~ %r{active_record/associations} }
+        # chances are the ferret result count is not our total_hits value, so
+        # we correct this here.
+        if options[:limit] != :all || options[:page] || options[:offset] || find_options[:limit] || find_options[:offset]
+          # our ferret result has been limited, so we need to re-run that
+          # search to get the full result set from ferret.
+          result_ids = {}
+          find_id_by_contents(q, options.update(:limit => :all, :offset => 0)) do |model, id, score, data|
+            result_ids[id] = [ result_ids.size + 1, score ]
+          end
+          # Now ask the database for the total size of the final result set.
+          total_hits = count_records( { self.name => result_ids }, find_options )
+        else
+          # what we got from the database is our full result set, so take
+          # it's size
+          total_hits = result.length
+        end
+      end
+
+      [ total_hits, result ]
+    end
+
+    def lazy_find_by_contents(q, options = {})
+      result = []
+      total_hits = find_id_by_contents(q, options) do |model, id, score, data|
+        result << FerretResult.new(model, id, score, data)
+      end
+      [ total_hits, result ]
+    end
+
+
+    def model_find(model, id, find_options = {})
+      model.constantize.find(id, find_options)
+    end
+
+    # retrieves search result records from a data structure like this:
+    # { 'Model1' => { '1' => [ rank, score ], '2' => [ rank, score ] }
+    #
+    # TODO: in case of STI AR will filter out hits from other 
+    # classes for us, but this
+    # will lead to less results retrieved --> scoping of ferret query
+    # to self.class is still needed.
+    # from the ferret ML (thanks Curtis Hatter)
+    # > I created a method in my base STI class so I can scope my query. For scoping
+    # > I used something like the following line:
+    # > 
+    # > query << " role:#{self.class.eql?(Contents) '*' : self.class}"
+    # > 
+    # > Though you could make it more generic by simply asking
+    # > "self.descends_from_active_record?" which is how rails decides if it should
+    # > scope your "find" query for STI models. You can check out "base.rb" in
+    # > activerecord to see that.
+    # but maybe better do the scoping in find_id_by_contents...
+    def retrieve_records(id_arrays, find_options = {})
+      result = []
+      # get objects for each model
+      id_arrays.each do |model, id_array|
+        next if id_array.empty?
+        begin
+          model = model.constantize
+        rescue
+          raise "Please use ':store_class_name => true' if you want to use multi_search.\n#{$!}"
+        end
+
+        # merge conditions
+        conditions = combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", 
+                                          id_array.keys ], 
+                                        find_options[:conditions])
+
+        # check for include association that might only exist on some models in case of multi_search
+        filtered_include_options = []
+        if include_options = find_options[:include]
+          include_options = [ include_options ] unless include_options.respond_to?(:each)
+          include_options.each do |include_option|
+            filtered_include_options << include_option if model.reflections.has_key?(include_option.is_a?(Hash) ? include_option.keys[0].to_sym : include_option.to_sym)
+          end
+        end
+        filtered_include_options = nil if filtered_include_options.empty?
+
+        # fetch
+        tmp_result = model.find(:all, find_options.merge(:conditions => conditions, 
+                                                         :include => filtered_include_options))
+
+        # set scores and rank
+        tmp_result.each do |record|
+          record.ferret_rank, record.ferret_score = id_array[record.id.to_s]
+        end
+        # merge with result array
+        result.concat tmp_result
+      end
+      
+      # order results as they were found by ferret, unless an AR :order
+      # option was given
+      result.sort! { |a, b| a.ferret_rank <=> b.ferret_rank } unless find_options[:order]
+      return result
+    end
+
+    def count_records(id_arrays, find_options = {})
+      count_options = find_options.dup
+      count_options.delete :limit
+      count_options.delete :offset
+      count = 0
+      id_arrays.each do |model, id_array|
+        next if id_array.empty?
+        begin
+          model = model.constantize
+          # merge conditions
+          conditions = combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], 
+                                          find_options[:conditions])
+          opts = find_options.merge :conditions => conditions
+          opts.delete :limit; opts.delete :offset
+          count += model.count opts
+        rescue TypeError
+          raise "#{model} must use :store_class_name option if you want to use multi_search against it.\n#{$!}"
+        end
+      end
+      count
+    end
+
+    def deprecated_options_support(options)
+      if options[:num_docs]
+        logger.warn ":num_docs is deprecated, use :limit instead!"
+        options[:limit] ||= options[:num_docs]
+      end
+      if options[:first_doc]
+        logger.warn ":first_doc is deprecated, use :offset instead!"
+        options[:offset] ||= options[:first_doc]
+      end
+    end
+
+    # creates a new Index instance.
+    def create_index_instance
+      if aaf_configuration[:remote]
+       RemoteIndex
+      elsif aaf_configuration[:single_index]
+        SharedIndex
+      else
+        LocalIndex
+      end.new(aaf_configuration)
+    end
+
+    # combine our conditions with those given by user, if any
+    def combine_conditions(conditions, additional_conditions = [])
+      conditions.tap do
+        if additional_conditions && additional_conditions.any?
+          cust_opts = additional_conditions.respond_to?(:shift) ? additional_conditions.dup : [ additional_conditions ]
+          conditions.first << " and " << cust_opts.shift
+          conditions.concat(cust_opts)
+        end
+      end
+    end
+
+  end
+  
+end
+