acts_as_ferret 0.4.0 → 0.4.1

data/lib/act_methods.rb

@@ -37,33 +37,54 @@ module ActsAsFerret #:nodoc:
     #                    this to true. the model class name will be stored in a keyword field 
     #                    named class_name
     #
-    # ====ferret_options:
-    # or_default:: whether query terms are required by
-    #              default (the default, false), or not (true)
+    # reindex_batch_size:: reindexing is done in batches of this size, default is 1000
+    #
+    # ferret:: Hash of Options that directly influence the way the Ferret engine works. You 
+    #          can use most of the options the Ferret::I class accepts here, too. Among the 
+    #          more useful are:
+    #
+    #     or_default:: whether query terms are required by
+    #                  default (the default, false), or not (true)
     # 
-    # analyzer:: the analyzer to use for query parsing (default: nil,
-    #            which means the ferret StandardAnalyzer gets used)
+    #     analyzer:: the analyzer to use for query parsing (default: nil,
+    #                which means the ferret StandardAnalyzer gets used)
     #
-    # default_field:: use to set one or more fields that are searched for query terms
-    #                 that don't have an explicit field list. This list should *not*
-    #                 contain any untokenized fields. If it does, you're asking
-    #                 for trouble (i.e. not getting results for queries having
-    #                 stop words in them). Aaf by default initializes the default field 
-    #                 list to contain all tokenized fields. If you use :single_index => true, 
-    #                 you really should set this option specifying your default field
-    #                 list (which should be equal in all your classes sharing the index).
-    #                 Otherwise you might get incorrect search results and you won't get 
-    #                 any lazy loading of stored field data.
+    #     default_field:: use to set one or more fields that are searched for query terms
+    #                     that don't have an explicit field list. This list should *not*
+    #                     contain any untokenized fields. If it does, you're asking
+    #                     for trouble (i.e. not getting results for queries having
+    #                     stop words in them). Aaf by default initializes the default field 
+    #                     list to contain all tokenized fields. If you use :single_index => true, 
+    #                     you really should set this option specifying your default field
+    #                     list (which should be equal in all your classes sharing the index).
+    #                     Otherwise you might get incorrect search results and you won't get 
+    #                     any lazy loading of stored field data.
     #
+    # For downwards compatibility reasons you can also specify the Ferret options in the 
+    # last Hash argument.
     def acts_as_ferret(options={}, ferret_options={})
 
       # force local mode if running *inside* the Ferret server - somewhere the
       # real indexing has to be done after all :-)
-      options.delete(:remote) if ActsAsFerret::Remote::Server.running
+      # Usually the automatic detection of server mode works fine, however if you 
+      # require your model classes in environment.rb they will get loaded before the 
+      # DRb server is started, so this code is executed too early and detection won't 
+      # work. In this case you'll get endless loops resulting in "stack level too deep" 
+      # errors. 
+      # To get around this, start the server with the environment variable 
+      # FERRET_USE_LOCAL_INDEX set to '1'.
+      logger.debug "Asked for a remote server ? #{options[:remote].inspect}, ENV[\"FERRET_USE_LOCAL_INDEX\"] is #{ENV["FERRET_USE_LOCAL_INDEX"].inspect}, looks like we are#{ActsAsFerret::Remote::Server.running || ENV['FERRET_USE_LOCAL_INDEX'] ? '' : ' not'} the server"
+      options.delete(:remote) if ENV["FERRET_USE_LOCAL_INDEX"] || ActsAsFerret::Remote::Server.running
 
       if options[:remote] && options[:remote] !~ /^druby/
         # read server location from config/ferret_server.yml
-        options[:remote] = ActsAsFerret::Remote::Config.load("#{RAILS_ROOT}/config/ferret_server.yml")[:uri]
+        options[:remote] = ActsAsFerret::Remote::Config.load("#{RAILS_ROOT}/config/ferret_server.yml")[:uri] rescue nil
+      end
+
+      if options[:remote]
+        logger.debug "Will use remote index server which should be available at #{options[:remote]}"
+      else
+        logger.debug "Will use local index."
       end
 
 
@@ -87,31 +108,38 @@ module ActsAsFerret #:nodoc:
         :name => self.table_name,
         :class_name => self.name,
         :single_index => false,
-        :ferret => {
-          :or_default => false, 
-          :handle_parse_errors => true,
-          :default_field => nil # will be set later on
-          #:max_clauses => 512,
-          #:analyzer => Ferret::Analysis::StandardAnalyzer.new,
-          # :wild_card_downcase => true
-        }
+        :reindex_batch_size => 1000,
+        :ferret => {},                    # Ferret config Hash
+        :ferret_fields => {}              # list of indexed fields that will be filled later
       }
 
       # merge aaf options with args
       aaf_configuration.update(options) if options.is_a?(Hash)
-
-      # list of indexed fields will be filled later
-      aaf_configuration[:ferret_fields] = Hash.new
-
       # apply appropriate settings for shared index
       if aaf_configuration[:single_index] 
         aaf_configuration[:index_dir] = "#{ActsAsFerret::index_dir}/shared" 
         aaf_configuration[:store_class_name] = true 
       end
 
-      # merge default ferret options with args
+      # set ferret default options
+      aaf_configuration[:ferret].reverse_merge!( :or_default => false, 
+                                                 :handle_parse_errors => true,
+                                                 :default_field => nil # will be set later on
+                                                 #:max_clauses => 512,
+                                                 #:analyzer => Ferret::Analysis::StandardAnalyzer.new,
+                                                 # :wild_card_downcase => true
+                                               )
+
+      # merge ferret options with those from second parameter hash
       aaf_configuration[:ferret].update(ferret_options) if ferret_options.is_a?(Hash)
 
+      unless options[:remote]
+        ActsAsFerret::ensure_directory aaf_configuration[:index_dir] 
+        aaf_configuration[:index_base_dir] = aaf_configuration[:index_dir]
+        aaf_configuration[:index_dir] = find_last_index_version(aaf_configuration[:index_dir])
+        logger.debug "using index in #{aaf_configuration[:index_dir]}"
+      end
+
       # these properties are somewhat vital to the plugin and shouldn't
       # be overwritten by the user:
       aaf_configuration[:ferret].update(
@@ -128,8 +156,6 @@ module ActsAsFerret #:nodoc:
         add_fields(aaf_configuration[:additional_fields])
       end
 
-      ActsAsFerret::ensure_directory aaf_configuration[:index_dir] unless options[:remote]
-
       # now that all fields have been added, we can initialize the default
       # field list to be used by the query parser.
       # It will include all content fields *not* marked as :untokenized.
@@ -151,11 +177,32 @@ module ActsAsFerret #:nodoc:
         end
       end
       logger.info "default field list: #{aaf_configuration[:ferret][:default_field].inspect}"
+
+      if options[:remote]
+        aaf_index.ensure_index_exists
+      end
     end
 
 
     protected
     
+    # find the most recent version of an index
+    def find_last_index_version(basedir)
+      # check for versioned index
+      versions = Dir.entries(basedir).select do |f| 
+        dir = File.join(basedir, f)
+        File.directory?(dir) && File.file?(File.join(dir, 'segments')) && f =~ /^\d+(_\d+)?$/
+      end
+      if versions.any?
+        # select latest version
+        versions.sort!
+        File.join basedir, versions.last
+      else
+        basedir
+      end
+    end
+
+
     # helper that defines a method that adds the given field to a ferret 
     # document instance
     def define_to_field_method(field, options = {})
@@ -164,6 +211,7 @@ module ActsAsFerret #:nodoc:
                               :index       => :yes, 
                               :term_vector => :with_positions_offsets,
                               :boost       => 1.0 )
+      options[:term_vector] = :no if options[:index] == :no
       aaf_configuration[:ferret_fields][field] = options
       define_method("#{field}_to_ferret".to_sym) do
         begin
@@ -178,7 +226,7 @@ module ActsAsFerret #:nodoc:
     end
 
     def add_fields(field_config)
-      if field_config.respond_to?(:each_pair)
+      if field_config.is_a? Hash
         field_config.each_pair do |key,val|
           define_to_field_method(key,val)                  
         end

data/lib/acts_as_ferret.rb

@@ -23,6 +23,7 @@ require 'active_record'
 require 'set'
 require 'ferret'
 
+require 'ferret_extensions'
 require 'act_methods'
 require 'class_methods'
 require 'shared_index_class_methods'
@@ -75,6 +76,7 @@ module ActsAsFerret
     # key is the index directory.
     @@ferret_indexes = Hash.new
     def self.ferret_indexes; @@ferret_indexes end
+
  
   # decorator that adds a total_hits accessor to search result arrays
   class SearchResults
@@ -92,7 +94,7 @@ module ActsAsFerret
   end
   
   def self.ensure_directory(dir)
-    FileUtils.mkdir_p dir unless File.directory? dir
+    FileUtils.mkdir_p dir unless (File.directory?(dir) || File.symlink?(dir))
   end
   
   # make sure the default index base dir exists. by default, all indexes are created
@@ -110,37 +112,49 @@ module ActsAsFerret
     base.extend(ClassMethods)
   end
   
-end
-
-# include acts_as_ferret method into ActiveRecord::Base
-ActiveRecord::Base.extend ActsAsFerret::ActMethods
-
-
-# small Ferret monkey patch
-# TODO check if this is still necessary
-class Ferret::Index::MultiReader
-  def latest?
-    # TODO: Exception handling added to resolve ticket #6. 
-    # It should be clarified wether this is a bug in Ferret
-    # in which case a bug report should be posted on the Ferret Trac. 
-    begin
-      @sub_readers.each { |r| return false unless r.latest? }
-    rescue
-      return false
+  # builds a FieldInfos instance for creation of an index containing fields
+  # for the given model classes.
+  def self.field_infos(models)
+    # default attributes for fields
+    fi = Ferret::Index::FieldInfos.new(:store => :no, 
+                                        :index => :yes, 
+                                        :term_vector => :no,
+                                        :boost => 1.0)
+    # primary key
+    fi.add_field(:id, :store => :yes, :index => :untokenized) 
+    fields = {}
+    have_class_name = false
+    models.each do |model|
+      fields.update(model.aaf_configuration[:ferret_fields])
+      # class_name
+      if !have_class_name && model.aaf_configuration[:store_class_name]
+        fi.add_field(:class_name, :store => :yes, :index => :untokenized) 
+        have_class_name = true
+      end
+    end
+    fields.each_pair do |field, options|
+      fi.add_field(field, { :store => :no, 
+                            :index => :yes }.update(options)) 
     end
-    true
+    return fi
   end
-end
 
-# add marshalling support to SortFields
-class Ferret::Search::SortField
-  def _dump(depth)
-    to_s
+  def self.close_multi_indexes
+    # close combined index readers, just in case
+    # this seems to fix a strange test failure that seems to relate to a
+    # multi_index looking at an old version of the content_base index.
+    multi_indexes.each_pair do |key, index|
+      # puts "#{key} -- #{self.name}"
+      # TODO only close those where necessary (watch inheritance, where
+      # self.name is base class of a class where key is made from)
+      index.close #if key =~ /#{self.name}/
+    end
+    multi_indexes.clear
   end
 
-  def self._load(string)
-    raise "invalid value: #{string}" unless string =~ /^(\w+):<(\w+)>(\!)?$/
-    new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
-  end
 end
 
+# include acts_as_ferret method into ActiveRecord::Base
+ActiveRecord::Base.extend ActsAsFerret::ActMethods
+
+

data/lib/class_methods.rb

@@ -12,8 +12,38 @@ module ActsAsFerret
     # the same field options in the shared index.
     def rebuild_index(*models)
       models << self unless models.include?(self)
-      aaf_index.rebuild_index(models.map(&:to_s))
-    end                                                            
+      aaf_index.rebuild_index models.map(&:to_s)
+      index_dir = find_last_index_version(aaf_configuration[:index_base_dir]) unless aaf_configuration[:remote]
+    end
+
+    # runs across all records yielding those to be indexed when the index is rebuilt
+    def records_for_rebuild(batch_size = 1000)
+      transaction do
+        if connection.class.name =~ /Mysql/ && primary_key == 'id'
+          logger.info "using mysql specific batched find :all"
+          offset = 0
+          while (rows = find :all, :conditions => ["id > ?", offset ], :limit => batch_size).any?
+            offset = rows.last.id
+            yield rows, offset
+          end
+        else
+          # sql server adapter won't batch correctly without defined ordering
+          order = "#{primary_key} ASC" if connection.class.name =~ /SQLServer/
+          0.step(self.count, batch_size) do |offset|
+            yield find( :all, :limit => batch_size, :offset => offset, :order => order ), offset
+          end
+        end
+      end
+    end
+
+    # Switches this class to a new index located in dir.
+    # Used by the DRb server when switching to a new index version.
+    def index_dir=(dir)
+      logger.debug "changing index dir to #{dir}"
+      aaf_configuration[:index_dir] = aaf_configuration[:ferret][:path] = dir
+      aaf_index.reopen!
+      logger.debug "index dir is now #{dir}"
+    end
     
     # Retrieve the index instance for this model class. This can either be a
     # LocalIndex, or a RemoteIndex instance.
@@ -25,41 +55,49 @@ module ActsAsFerret
       ActsAsFerret::ferret_indexes[aaf_configuration[:index_dir]] ||= create_index_instance
     end 
     
-    # Finds instances by contents. Terms are ANDed by default, can be circumvented 
-    # by using OR between terms. 
-    # options:
+    # Finds instances by searching the Ferret index. Terms are ANDed by default, use 
+    # OR between terms for ORed queries. Or specify +:or_default => true+ in the
+    # +:ferret+ options hash of acts_as_ferret.
+    #
+    # == options:
     # offset::      first hit to retrieve (useful for paging)
     # limit::       number of hits to retrieve, or :all to retrieve
     #               all results
     # lazy::        Array of field names whose contents should be read directly
     #               from the index. Those fields have to be marked 
-    #               :store => :yes in their field options. Give true to get all
-    #               stored fields (if you have a shared index, you have to
-    #               explicitly state the fields you want to fetch, true won't
-    #               work)
+    #               +:store => :yes+ in their field options. Give true to get all
+    #               stored fields. Note that if you have a shared index, you have 
+    #               to explicitly state the fields you want to fetch, true won't
+    #               work here)
     # models::      only for single_index scenarios: an Array of other Model classes to 
     #               include in this search. Use :all to query all models.
     #
-    # find_options is a hash passed on to active_record's find when
-    # retrieving the data from db, useful to i.e. prefetch relationships.
+    # +find_options+ is a hash passed on to active_record's find when
+    # retrieving the data from db, useful to i.e. prefetch relationships with
+    # :include or to specify additional filter criteria with :conditions.
+    #
+    # This method returns a +SearchResults+ instance, which really is an Array that has 
+    # been decorated with a total_hits attribute holding the total number of hits.
     #
-    # this method returns a SearchResults instance, which really is an Array that has 
-    # been decorated with a total_hits accessor that delivers the total
-    # number of hits (including those not fetched because of a low num_docs
-    # value).
     # Please keep in mind that the number of total hits might be wrong if you specify 
     # both ferret options and active record find_options that somehow limit the result 
-    # set (e.g. :num_docs and some :conditions).
-    def find_by_contents(q, options = {}, find_options = {})
+    # set (e.g. +:num_docs+ and some +:conditions+).
+    def find_with_ferret(q, options = {}, find_options = {})
       total_hits, result = find_records_lazy_or_not q, options, find_options
       logger.debug "Query: #{q}\ntotal hits: #{total_hits}, results delivered: #{result.size}"
       return SearchResults.new(result, total_hits)
     end 
+    alias find_by_contents find_with_ferret
 
    
 
-    # return the total number of hits for the given query 
+    # Returns the total number of hits for the given query 
+    # To count the results of a multi_search query, specify an array of 
+    # class names with the :models option.
     def total_hits(q, options={})
+      if models = options[:models]
+        options[:models] = add_self_to_model_list_if_necessary(models).map(&:to_s)
+      end
       aaf_index.total_hits(q, options)
     end
 
@@ -81,7 +119,7 @@ module ActsAsFerret
       deprecated_options_support(options)
       aaf_index.find_id_by_contents(q, options, &block)
     end
-    
+
     # requires the store_class_name option of acts_as_ferret to be true
     # for all models queried this way.
     def multi_search(query, additional_models = [], options = {}, find_options = {})
@@ -112,14 +150,18 @@ module ActsAsFerret
     # be yielded, and the total number of hits is returned.
     def id_multi_search(query, additional_models = [], options = {}, &proc)
       deprecated_options_support(options)
-      additional_models = [ additional_models ] unless additional_models.is_a? Array
-      additional_models << self
+      additional_models = add_self_to_model_list_if_necessary(additional_models)
       aaf_index.id_multi_search(query, additional_models.map(&:to_s), options, &proc)
     end
     
 
     protected
 
+    def add_self_to_model_list_if_necessary(models)
+      models = [ models ] unless models.is_a? Array
+      models << self unless models.include?(self)
+    end
+
     def find_records_lazy_or_not(q, options = {}, find_options = {})
       if options[:lazy]
         logger.warn "find_options #{find_options} are ignored because :lazy => true" unless find_options.empty?
@@ -138,12 +180,20 @@ module ActsAsFerret
       end
 
       result = retrieve_records( { self.name => result_ids }, find_options )
-      # correct result size if the user specified conditions
-      total_hits = result.length if find_options[:conditions]
-
-      # order results as they were found by ferret, unless an AR :order
-      # option was given
-      result.sort! { |a, b| a.ferret_rank <=> b.ferret_rank } unless find_options[:order]
+      
+      if find_options[:conditions]
+        if options[:limit] != :all
+          # correct result size if the user specified conditions
+          #  wenn conditions: options[:limit] != :all --> ferret-query mit :all wiederholen und select count machen
+          result_ids = {}
+          find_id_by_contents(q, options.update(:limit => :all)) do |model, id, score, data|
+            result_ids[id] = [ result_ids.size + 1, score ]
+          end
+          total_hits = count_records( { self.name => result_ids }, find_options )
+        else
+          total_hits = result.length
+        end
+      end
 
       [ total_hits, result ]
     end
@@ -186,22 +236,56 @@ module ActsAsFerret
         next if id_array.empty?
         begin
           model = model.constantize
-          # merge conditions
-          conditions = combine_conditions([ "#{model.table_name}.#{primary_key} in (?)", id_array.keys ], 
-                                          find_options[:conditions])
-          # fetch
-          tmp_result = model.find(:all, find_options.merge(:conditions => conditions))
-          # set scores and rank
-          tmp_result.each do |record|
-            record.ferret_rank, record.ferret_score = id_array[record.id.to_s]
+        rescue
+          raise "Please use ':store_class_name => true' if you want to use multi_search.\n#{$!}"
+        end
+
+        # check for include association that might only exist on some models in case of multi_search
+        filtered_include_options = []
+        if include_options = find_options[:include]
+          include_options.each do |include_option|
+            filtered_include_options << include_option if model.reflections.has_key?(include_option.is_a?(Hash) ? include_option.keys[0].to_sym : include_option.to_sym)
+          end
+        end
+        filtered_include_options=nil if filtered_include_options.empty?
+
+        # fetch
+        tmp_result = nil
+        model.send(:with_scope, :find => find_options) do 
+          tmp_result = model.find( :all, :conditions => [ 
+            "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ],  
+            :include => filtered_include_options ) 
+        end
+
+        # set scores and rank
+        tmp_result.each do |record|
+          record.ferret_rank, record.ferret_score = id_array[record.id.to_s]
+        end
+        # merge with result array
+        result.concat tmp_result
+      end
+      
+      # order results as they were found by ferret, unless an AR :order
+      # option was given
+      result.sort! { |a, b| a.ferret_rank <=> b.ferret_rank } unless find_options[:order]
+      return result
+    end
+
+    def count_records(id_arrays, find_options = {})
+      count = 0
+      id_arrays.each do |model, id_array|
+        next if id_array.empty?
+        begin
+          model = model.constantize
+          model.send(:with_scope, :find => find_options) do 
+            count += model.count(:conditions => [ "#{model.table_name}.#{model.primary_key} in (?)",
+                                                  id_array.keys ]) 
           end
-          # merge with result array
-          result.concat tmp_result
         rescue TypeError
           raise "#{model} must use :store_class_name option if you want to use multi_search against it.\n#{$!}"
         end
       end
-      return result
+      count
     end
 
     def deprecated_options_support(options)
@@ -215,20 +299,7 @@ module ActsAsFerret
       end
     end
 
-    # combine our conditions with those given by user, if any
-    def combine_conditions(conditions, *additional_conditions)
-      returning conditions do
-        if additional_conditions.any?
-          cust_opts = additional_conditions.dup.flatten
-          conditions.first << " and " << cust_opts.shift
-          conditions.concat(cust_opts)
-        end
-      end
-    end
-
-    # creates a new Index::Index instance. Before that, a check is done
-    # to see if the index exists in the file system. If not, index rebuild
-    # from all model data retrieved by find(:all) is triggered.
+    # creates a new Index instance.
     def create_index_instance
       if aaf_configuration[:remote]
        RemoteIndex

data/lib/ferret_cap_tasks.rb

@@ -0,0 +1,21 @@
+# Ferret DRb server Capistrano tasks
+# Usage:
+# Add require 'vendor/plugins/acts_as_ferret/lib/ferret_cap_tasks' to your
+# config/deploy.rb
+# call ferret.restart where you restart your Mongrels.
+# ferret.stop and ferret.start are available, too.
+module FerretCapTasks
+  def start
+    run "cd #{current_path}; RAILS_ENV=production script/ferret_start"
+  end
+
+  def stop
+    run "cd #{current_path}; RAILS_ENV=production script/ferret_stop"
+  end
+
+  def restart
+    stop
+    start
+  end
+end
+Capistrano.plugin :ferret, FerretCapTasks

data/lib/ferret_extensions.rb

@@ -0,0 +1,81 @@
+module Ferret
+
+
+  class Index::Index
+    attr_accessor :batch_size
+    attr_accessor :logger
+
+    def index_models(models)
+      models.each { |model| index_model model }
+      flush
+      optimize
+      close
+      ActsAsFerret::close_multi_indexes
+    end
+
+    def index_model(model)
+      @batch_size ||= 0
+      work_done = 0
+      batch_time = 0
+      logger.info "reindexing model #{model.name}"
+
+      model_count  = model.count.to_f
+      model.records_for_rebuild(@batch_size) do |records, offset|
+        #records = [ records ] unless records.is_a?(Array)
+        batch_time = measure_time {
+          records.each { |rec| self << rec.to_doc if rec.ferret_enabled?(true) }
+        }.to_f
+        work_done = offset.to_f / model_count * 100.0 if model_count > 0
+        remaining_time = ( batch_time / @batch_size ) * ( model_count - offset + @batch_size )
+        logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+      end
+    end
+
+    def measure_time
+      t1 = Time.now
+      yield
+      Time.now - t1
+    end
+
+  end
+
+
+  # add marshalling support to SortFields
+  class Search::SortField
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      case string
+        when /<DOC(_ID)?>!/         : Ferret::Search::SortField::DOC_ID_REV
+        when /<DOC(_ID)?>/          : Ferret::Search::SortField::DOC_ID
+        when '<SCORE>!'             : Ferret::Search::SortField::SCORE_REV
+        when '<SCORE>'              : Ferret::Search::SortField::SCORE
+        when /^(\w+):<(\w+)>(!)?$/ : new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
+        else raise "invalid value: #{string}"
+      end
+    end
+  end
+
+  # add marshalling support to Sort
+  class Search::Sort
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      # we exclude the last <DOC> sorting as it is appended by new anyway
+      if string =~ /^Sort\[(.*?)(<DOC>(!)?)?\]$/
+        sort_fields = $1.split(',').map do |value| 
+        value.strip!
+          Ferret::Search::SortField._load value unless value.blank?
+        end
+        new sort_fields.compact
+      else
+        raise "invalid value: #{string}"
+      end
+    end
+  end
+
+end

data/lib/ferret_server.rb

@@ -6,84 +6,126 @@ require 'erb'
 
 module ActsAsFerret
 
-module Remote
-
-  module Config
-    class << self
-      DEFAULTS = {
-        'host' => 'localhost',
-        'port' => '9009'
-      }
-      # reads connection settings from config file
-      def load(file = "#{RAILS_ROOT}/config/ferret_server.yml")
-        config = DEFAULTS.merge(YAML.load(ERB.new(IO.read(file)).result))
-        if config = config[RAILS_ENV]
-          config[:uri] = "druby://#{config['host']}:#{config['port']}"
-          return config
+  module Remote
+
+    module Config
+      class << self
+        DEFAULTS = {
+          'host' => 'localhost',
+          'port' => '9009'
+        }
+        # read connection settings from config file
+        def load(file = "#{RAILS_ROOT}/config/ferret_server.yml")
+          config = DEFAULTS.merge(YAML.load(ERB.new(IO.read(file)).result))
+          if config = config[RAILS_ENV]
+            config[:uri] = "druby://#{config['host']}:#{config['port']}"
+            return config
+          end
+          {}
         end
       end
     end
-  end
-
-  # This class acts as a drb server listening for indexing and
-  # search requests from models declared to 'acts_as_ferret :remote => true'
-  #
-  # Usage: 
-  # - copy doc/ferret_server.yml to RAILS_ROOT/config and modify to suit
-  # your needs.
-  # - run script/ferret_server (in the plugin directory) via script/runner:
-  # RAILS_ENV=production script/runner vendor/plugins/acts_as_ferret/script/ferret_server
-  #
-  # TODO: automate installation of files to script/ and config/
-  class Server
-
-    cattr_accessor :running
-
-    def self.start(uri = nil)
-      ActiveRecord::Base.allow_concurrency = true
-      uri ||= ActsAsFerret::Remote::Config.load[:uri]
-      DRb.start_service(uri, ActsAsFerret::Remote::Server.new)
-      self.running = true
-    end
 
-    def initialize
-      @logger = Logger.new("#{RAILS_ROOT}/log/ferret_server.log")
-    end
-
-    # handles all incoming method calls, and sends them on to the LocalIndex
-    # instance of the correct model class.
+    # This class acts as a drb server listening for indexing and
+    # search requests from models declared to 'acts_as_ferret :remote => true'
     #
-    # Calls are not queued atm, so this will block until the call returned.
-    # Might throw the occasional LockError, too, which most probably means that you're 
-    # a) rebuilding your index or 
-    # b) have *really* high load. I wasn't able to reproduce this case until
-    # now, if you do, please contact me.
+    # Usage: 
+    # - modify RAILS_ROOT/config/ferret_server.yml to suit your needs. 
+    # - environments for which no section in the config file exists will use 
+    #   the index locally (good for unit tests/development mode)
+    # - run script/ferret_start to start the server:
+    # RAILS_ENV=production script/ferret_start
     #
-    # TODO: rebuild indexes in separate directory so no lock errors in these
-    # cases.
-    def method_missing(name, *args)
-      clazz = args.shift.constantize
-      begin
-        @logger.debug "call index method: #{name} with #{args.inspect}"
-        clazz.aaf_index.send name, *args
-      rescue NoMethodError
-        @logger.debug "no luck, trying to call class method instead"
-        clazz.send name, *args
+    class Server
+
+      cattr_accessor :running
+
+      def self.start(uri = nil)
+        ActiveRecord::Base.allow_concurrency = true
+        ActiveRecord::Base.logger = Logger.new("#{RAILS_ROOT}/log/ferret_server.log")
+        uri ||= ActsAsFerret::Remote::Config.load[:uri]
+        DRb.start_service(uri, ActsAsFerret::Remote::Server.new)
+        self.running = true
       end
-    rescue
-      @logger.error "ferret server error #{$!}\n#{$!.backtrace.join '\n'}"
-      raise
-    end
 
-    def ferret_index(class_name)
-      # TODO check if in use!
-      class_name.constantize.aaf_index.ferret_index
-    end
+      def initialize
+        @logger = ActiveRecord::Base.logger
+      end
 
-    # the main loop taking stuff from the queue and running it...
-    #def run
-    #end
+      # handles all incoming method calls, and sends them on to the LocalIndex
+      # instance of the correct model class.
+      #
+      # Calls are not queued atm, so this will block until the call returned.
+      #
+      def method_missing(name, *args)
+        @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
+        with_class args.shift do |clazz|
+          begin
+            clazz.aaf_index.send name, *args
+          rescue NoMethodError
+            @logger.debug "no luck, trying to call class method instead"
+            clazz.send name, *args
+          end
+        end
+      rescue
+        @logger.error "ferret server error #{$!}\n#{$!.backtrace.join '\n'}"
+        raise
+      end
 
+      # make sure we have a versioned index in place, building one if necessary
+      def ensure_index_exists(class_name)
+        @logger.debug "DRb server: ensure_index_exists for class #{class_name}"
+        with_class class_name do |clazz|
+          dir = clazz.aaf_configuration[:index_dir]
+          unless File.directory?(dir) && File.file?(File.join(dir, 'segments')) && dir =~ %r{/\d+(_\d+)?$}
+            rebuild_index(clazz)
+          end
+        end
+      end
+
+      # hides LocalIndex#rebuild_index to implement index versioning
+      def rebuild_index(clazz, *models)
+        with_class clazz do |clazz|
+          models = models.flatten.uniq.map(&:constantize)
+          models << clazz unless models.include?(clazz)
+          index = new_index_for(clazz, models)
+          @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
+          index.index_models models
+          new_version = File.join clazz.aaf_configuration[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
+          # create a unique directory name (needed for unit tests where 
+          # multiple rebuilds per second may occur)
+          if File.exists?(new_version)
+            i = 0
+            i+=1 while File.exists?("#{new_version}_#{i}")
+            new_version << "_#{i}"
+          end
+          
+          File.rename index.options[:path], new_version
+          clazz.index_dir = new_version 
+        end
+      end
+
+
+      protected
+
+        def with_class(clazz, *args)
+          clazz = clazz.constantize if String === clazz
+          yield clazz, *args
+        end
+
+        def new_index_for(clazz, models)
+          aaf_configuration = clazz.aaf_configuration
+          ferret_cfg = aaf_configuration[:ferret].dup
+          ferret_cfg.update :auto_flush  => false, 
+                            :create      => true,
+                            :field_infos => ActsAsFerret::field_infos(models),
+                            :path        => File.join(aaf_configuration[:index_base_dir], 'rebuild')
+          returning Ferret::Index::Index.new ferret_cfg do |i|
+            i.batch_size = aaf_configuration[:reindex_batch_size]
+            i.logger = @logger
+          end
+        end
+
+    end
   end
 end
-end

data/lib/index.rb

@@ -4,7 +4,7 @@ module ActsAsFerret
   class AbstractIndex
 
     attr_reader :aaf_configuration
-    attr_reader :logger
+    attr_accessor :logger
     def initialize(aaf_configuration)
       @aaf_configuration = aaf_configuration
       @logger = Logger.new("#{RAILS_ROOT}/log/ferret_index.log")

data/lib/local_index.rb

@@ -9,16 +9,30 @@ module ActsAsFerret
       ensure_index_exists
     end
 
+    def reopen!
+      if @ferret_index
+        @ferret_index.close
+        @ferret_index = nil
+      end
+      logger.debug "reopening index at #{aaf_configuration[:ferret][:path]}"
+      ferret_index
+    end
+
     # The 'real' Ferret Index instance
     def ferret_index
       ensure_index_exists
-      @ferret_index ||= Ferret::Index::Index.new(aaf_configuration[:ferret])
+      returning @ferret_index ||= Ferret::Index::Index.new(aaf_configuration[:ferret]) do
+        @ferret_index.batch_size = aaf_configuration[:reindex_batch_size]
+        @ferret_index.logger = logger
+      end
     end
 
     # Checks for the presence of a segments file in the index directory
     # Rebuilds the index if none exists.
     def ensure_index_exists
+      logger.debug "LocalIndex: ensure_index_exists at #{aaf_configuration[:index_dir]}"
       unless File.file? "#{aaf_configuration[:index_dir]}/segments"
+        ActsAsFerret::ensure_directory(aaf_configuration[:index_dir])
         close
         rebuild_index 
       end
@@ -37,20 +51,15 @@ module ActsAsFerret
     # to. Arguments can be given in shared index scenarios to name multiple
     # model classes to include in the index
     def rebuild_index(*models)
-      logger.debug "rebuild index: #{models.inspect}"
       models << aaf_configuration[:class_name] unless models.include?(aaf_configuration[:class_name])
       models = models.flatten.uniq.map(&:constantize)
+      logger.debug "rebuild index: #{models.inspect}"
       index = Ferret::Index::Index.new(aaf_configuration[:ferret].dup.update(:auto_flush => false, 
-                                                                             :field_infos => field_infos(models),
+                                                                             :field_infos => ActsAsFerret::field_infos(models),
                                                                              :create => true))
-      models.each do |model|
-        reindex_model(index, model)
-      end
-      logger.debug("Created Ferret index in: #{aaf_configuration[:index_dir]}")
-      index.flush
-      index.optimize
-      index.close
-      close_multi_indexes
+      index.batch_size = aaf_configuration[:reindex_batch_size]
+      index.logger = logger
+      index.index_models models
     end
 
     # Parses the given query string into a Ferret Query object.
@@ -64,8 +73,11 @@ module ActsAsFerret
     end
 
     # Total number of hits for the given query. 
+    # To count the results of a multi_search query, specify an array of 
+    # class names with the :models option.
     def total_hits(query, options = {})
-      ferret_index.search(query, options).total_hits
+      index = (models = options.delete(:models)) ? multi_index(models) : ferret_index
+      index.search(query, options).total_hits
     end
 
     def determine_lazy_fields(options = {})
@@ -108,7 +120,6 @@ module ActsAsFerret
     # If a block is given, model, id and score are yielded and the number of 
     # total hits is returned. Otherwise [total_hits, result_array] is returned.
     def id_multi_search(query, models, options = {})
-      models.map!(&:constantize)
       index = multi_index(models)
       result = []
       lazy_fields = determine_lazy_fields options
@@ -117,6 +128,7 @@ module ActsAsFerret
         # fetch stored fields if lazy loading
         data = {}
         lazy_fields.each { |field| data[field] = doc[field] } if lazy_fields
+        raise "':store_class_name => true' required for multi_search to work" if doc[:class_name].blank?
         if block_given?
           yield doc[:class_name], doc[:id], score, doc, data
         else
@@ -184,6 +196,7 @@ module ActsAsFerret
 
     # returns a MultiIndex instance operating on a MultiReader
     def multi_index(model_classes)
+      model_classes.map!(&:constantize) if String === model_classes.first
       model_classes.sort! { |a, b| a.name <=> b.name }
       key = model_classes.inject("") { |s, clazz| s + clazz.name }
       multi_config = aaf_configuration[:ferret].dup
@@ -191,67 +204,6 @@ module ActsAsFerret
       ActsAsFerret::multi_indexes[key] ||= MultiIndex.new(model_classes, multi_config)
     end
  
-    def close_multi_indexes
-      # close combined index readers, just in case
-      # this seems to fix a strange test failure that seems to relate to a
-      # multi_index looking at an old version of the content_base index.
-      ActsAsFerret::multi_indexes.each_pair do |key, index|
-        # puts "#{key} -- #{self.name}"
-        # TODO only close those where necessary (watch inheritance, where
-        # self.name is base class of a class where key is made from)
-        index.close #if key =~ /#{self.name}/
-      end
-      ActsAsFerret::multi_indexes.clear
-    end
-
-    def reindex_model(index, model = aaf_configuration[:class_name].constantize)
-      # index in batches of 1000 to limit memory consumption (fixes #24)
-      # TODO make configurable through options
-      batch_size = 1000
-      model_count = model.count.to_f
-      work_done = 0
-      batch_time = 0
-      logger.info "reindexing model #{model.name}"
-      order = "#{model.primary_key} ASC" # this works around a bug in sqlserver-adapter (where paging only works with an order applied)
-      model.transaction do
-        0.step(model.count, batch_size) do |i|
-          b1 = Time.now.to_f
-          model.find(:all, :limit => batch_size, :offset => i, :order => order).each do |rec|
-            index << rec.to_doc if rec.ferret_enabled?(true)
-          end
-          batch_time = Time.now.to_f - b1
-          work_done = i.to_f / model_count * 100.0 if model_count > 0
-          remaining_time = ( batch_time / batch_size ) * ( model_count - i + batch_size )
-          logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
-        end
-      end
-    end
-
-    # builds a FieldInfos instance for creation of an index containing fields
-    # for the given model classes.
-    def field_infos(models)
-      # default attributes for fields
-      fi = Ferret::Index::FieldInfos.new(:store => :no, 
-                                         :index => :yes, 
-                                         :term_vector => :no,
-                                         :boost => 1.0)
-      # primary key
-      fi.add_field(:id, :store => :yes, :index => :untokenized) 
-      # class_name
-      if aaf_configuration[:store_class_name]
-        fi.add_field(:class_name, :store => :yes, :index => :untokenized) 
-      end
-      fields = {}
-      models.each do |model|
-        fields.update(model.aaf_configuration[:ferret_fields])
-      end
-      fields.each_pair do |field, options|
-        fi.add_field(field, { :store => :no, 
-                              :index => :yes }.update(options)) 
-      end
-      return fi
-    end
-
   end
 
 end

data/lib/more_like_this.rb

@@ -29,8 +29,9 @@ module ActsAsFerret #:nodoc:
         #                                           equals Ferret's internal similarity implementation)
         # :analyzer => 'Ferret::Analysis::StandardAnalyzer' # class name of the analyzer to use
         # :append_to_query => nil # proc taking a query object as argument, which will be called after generating the query. can be used to further manipulate the query used to find related documents, i.e. to constrain the search to a given class in single table inheritance scenarios
-        # find_options : options handed over to find_by_contents
-        def more_like_this(options = {}, find_options = {})
+        # ferret_options : Ferret options handed over to find_by_contents (i.e. for limits and sorting)
+        # ar_options : options handed over to find_by_contents for AR scoping
+        def more_like_this(options = {}, ferret_options = {}, ar_options = {})
           options = {
             :field_names => nil,  # Default field names
             :min_term_freq => 2,  # Ignore terms with less than this frequency in the source doc.
@@ -52,13 +53,16 @@ module ActsAsFerret #:nodoc:
           options[:base_class] = clazz.name
           query = clazz.aaf_index.build_more_like_this_query(self.id, self.class.name, options)
           options[:append_to_query].call(query) if options[:append_to_query]
-          clazz.find_by_contents(query, find_options)
+          clazz.find_by_contents(query, ferret_options, ar_options)
         end
 
       end
 
       module IndexMethods
 
+        # TODO to allow morelikethis for unsaved records, we have to give the
+        # unsaved record's data to this method. check how this will work out
+        # via drb...
         def build_more_like_this_query(id, class_name, options)
           [:similarity, :analyzer].each { |sym| options[sym] = options[sym].constantize.new }
           ferret_index.synchronize do # avoid that concurrent writes close our reader
@@ -103,7 +107,8 @@ module ActsAsFerret #:nodoc:
         # creates a term/term_frequency map for terms from the fields
         # given in options[:field_names]
         def retrieve_terms(id, class_name, reader, options)
-          document_number = document_number(id, class_name)
+          raise "more_like_this atm only works on saved records" if id.nil?
+          document_number = document_number(id, class_name) rescue nil
           field_names = options[:field_names]
           max_num_tokens = options[:max_num_tokens]
           term_freq_map = Hash.new(0)
@@ -111,7 +116,7 @@ module ActsAsFerret #:nodoc:
           record = nil
           field_names.each do |field|
             #puts "field: #{field}"
-            term_freq_vector = reader.term_vector(document_number, field)
+            term_freq_vector = reader.term_vector(document_number, field) if document_number
             #if false
             if term_freq_vector
               # use stored term vector
@@ -123,8 +128,11 @@ module ActsAsFerret #:nodoc:
               # puts 'no stored term vector'
               # no term vector stored, but we have stored the contents in the index
               # -> extract terms from there
-              doc = reader[doc_number]
-              content = doc[field]
+              content = nil
+              if document_number
+                doc = reader[document_number]
+                content = doc[field]
+              end
               unless content
                 # no term vector, no stored content, so try content from this instance
                 record ||= options[:base_class].constantize.find(id)

data/lib/multi_index.rb

@@ -8,7 +8,7 @@ module ActsAsFerret #:nodoc:
           # ensure all models indexes exist
           @model_classes.each { |m| m.aaf_index.ensure_index_exists }
           default_fields = @model_classes.inject([]) do |fields, c| 
-            fields + c.aaf_configuration[:ferret][:default_field] 
+            fields + [ c.aaf_configuration[:ferret][:default_field] ].flatten
           end
           @options = { 
             :default_field => default_fields
@@ -69,9 +69,8 @@ module ActsAsFerret #:nodoc:
                 begin
                   reader = Ferret::Index::IndexReader.new(clazz.aaf_configuration[:index_dir])
                 rescue Exception
-                  puts "error opening #{clazz.aaf_configuration[:index_dir]}: #{$!}"
+                  raise "error opening #{clazz.aaf_configuration[:index_dir]}: #{$!}"
                 end
-                reader
               }
               close
               @reader = Ferret::Index::IndexReader.new(@sub_readers)

data/rakefile

@@ -55,7 +55,8 @@ if PKG_VERSION
     s.platform        = Gem::Platform::RUBY
     s.summary         = "acts_as_ferret - Ferret based full text search for any ActiveRecord model"
     s.files  = Dir.glob('**/*', File::FNM_DOTMATCH).reject do |f| 
-      [ /\.$/, /sqlite$/, /\.log$/, /^pkg/, /\.svn/, /\.swp$/, /^html/, /\~$/, /\/\._/, /\/#/ ].any? {|regex| f =~ regex }
+      [ /\.$/, /sqlite$/, /\.log$/, /^pkg/, /\.svn/, /\.\w+\.sw.$/,
+        /^html/, /\~$/, /\/\._/, /\/#/ ].any? {|regex| f =~ regex }
     end
     #s.files           = FileList["{lib,test}/**/*"].to_a + %w(README MIT-LICENSE CHANGELOG)
   # s.files.delete  ...    
@@ -113,9 +114,7 @@ if PKG_VERSION
   
   # Upload release to rubyforge
   desc "Upload release to rubyforge"
-  #task :prel => [ :tag, :prerelease, :package ] do
-  task :prel => [ :package ] do
-  #task :prel do
+  task :prel => [ :tag, :prerelease, :package ] do
     `rubyforge login`
     release_command = "rubyforge add_release #{RUBYFORGE_PROJECT} #{PKG_NAME} '#{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.gem"
     puts release_command

data/script/ferret_start

@@ -1,5 +1,4 @@
-#!/usr/bin/env script/runner
-
+#!/usr/bin/env ruby
 # Ferret DRb server launcher script
 #
 # Place doc/ferret_server.yml into RAILS_ROOT/config and fit to taste. Start
@@ -18,12 +17,17 @@
 # The server writes a log file in log/ferret_server.log, it's 
 # STDOUT gets redirected to log/ferret_server.out
 
+ENV['FERRET_USE_LOCAL_INDEX'] = 'true'
+require File.dirname(__FILE__) + '/../config/boot'
+require RAILS_ROOT + '/config/environment'
+
 
 config = ActsAsFerret::Remote::Config.load
 @pid_file = config['pid_file']
 
 def write_pid_file
-    open(@pid_file,"w") {|f| f.write(Process.pid) }
+  raise "No PID file defined" if @pid_file.blank?
+  open(@pid_file,"w") {|f| f.write(Process.pid) }
 end
 
 def safefork
@@ -59,7 +63,8 @@ begin
   STDOUT.reopen "#{RAILS_ROOT}/log/ferret_server.out", "a" # point them somewhere sensible
   STDERR.reopen STDOUT                                     # STDOUT/STDERR should go to a logfile
 rescue
-  puts "Error starting ferret DRb server: #{$!}"
+  $stderr.puts "Error starting ferret DRb server: #{$!}"
+  $stderr.puts $!.backtrace
   exit(1)
 end
 DRb.thread.join

data/script/ferret_stop

@@ -17,6 +17,10 @@ end
 
 pid_file = config['pid_file']
 puts "Stopping ferret_server..."
-send_signal("TERM", pid_file)
+if File.file?(pid_file)
+  send_signal("TERM", pid_file) 
+else
+  puts "no pid file found"
+end
 
 # vim:set filetype=ruby:

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: acts_as_ferret
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
-date: 2007-03-24 00:00:00 +01:00
+  version: 0.4.1
+date: 2007-07-17 00:00:00 +02:00
 summary: acts_as_ferret - Ferret based full text search for any ActiveRecord model
 require_paths: 
 - lib
@@ -29,32 +29,34 @@ post_install_message:
 authors: 
 - Jens Kraemer
 files: 
+- script
+- config
+- doc
 - lib
 - LICENSE
 - rakefile
 - init.rb
-- README
-- script
-- doc
-- config
 - install.rb
-- lib/acts_as_ferret.rb
-- lib/multi_index.rb
-- lib/more_like_this.rb
-- lib/instance_methods.rb
-- lib/class_methods.rb
+- README
+- script/ferret_start
+- script/ferret_stop
+- script/ferret_server
+- config/ferret_server.yml
 - lib/ferret_server.rb
+- lib/more_like_this.rb
 - lib/shared_index.rb
+- lib/ferret_cap_tasks.rb
 - lib/local_index.rb
+- lib/multi_index.rb
 - lib/remote_index.rb
+- lib/acts_as_ferret.rb
+- lib/ferret_result.rb
+- lib/shared_index_class_methods.rb
+- lib/ferret_extensions.rb
 - lib/index.rb
+- lib/instance_methods.rb
+- lib/class_methods.rb
 - lib/act_methods.rb
-- lib/shared_index_class_methods.rb
-- lib/ferret_result.rb
-- script/ferret_server
-- script/ferret_start
-- script/ferret_stop
-- config/ferret_server.yml
 test_files: []
 
 rdoc_options: []