acts_as_ferret 0.3.1 → 0.4.0

data/README

@@ -1,4 +1,4 @@
-acts_as_ferret
+= acts_as_ferret
 
 This ActiveRecord mixin adds full text search capabilities to any Rails model.
 
@@ -6,10 +6,28 @@ It is heavily based on the original acts_as_ferret plugin done by
 Kasper Weibel and a modified version done by Thomas Lockney, which 
 both can be found on http://ferret.davebalmain.com/trac/wiki/FerretOnRails
 
-usage:
+== Installation
+
+=== Installation inside your Rails project via script/plugin
+
+script/plugin install svn://projects.jkraemer.net/acts_as_ferret/trunk/plugin/acts_as_ferret
+
+
+=== System-wide installation with Rubygems
+
+<tt>sudo gem install acts_as_ferret</tt>
+
+To use acts_as_ferret in your project, add the following line to your
+project's config/environment.rb:
+
+<tt>require 'acts_as_ferret'</tt>
+
+
+== Usage
+
 include the following in your model class (specifiying the fields you want to get indexed):
 
-acts_as_ferret :fields => [ 'title', 'description' ]
+<tt>acts_as_ferret :fields => [ :title, :description ]</tt>
 
 now you can use ModelClass.find_by_contents(query) to find instances of your model
 whose indexed fields match a given query. All query terms are required by default, 
@@ -17,10 +35,15 @@ but explicit OR queries are possible. This differs from the ferret default, but
 imho is the more often needed/expected behaviour (more query terms result in 
 less results).
 
+Please see ActsAsFerret::ActMethods#acts_as_ferret for more information.
+
+== License
+
 Released under the MIT license.
 
-Authors: 
-Kasper Weibel Nielsen-Refs (original author)
-Jens Kraemer <jk@jkraemer.net>
+== Authors
+
+* Kasper Weibel Nielsen-Refs (original author)
+* Jens Kraemer <jk@jkraemer.net> (current maintainer)
 
 

data/config/ferret_server.yml

@@ -0,0 +1,12 @@
+production:
+  host: ferret.yourdomain.com
+  port: 9010
+  pid_file: log/ferret.pid
+development:
+  host: localhost
+  port: 9010
+  pid_file: log/ferret.pid
+test:
+  host: localhost
+  port: 9009
+  pid_file: log/ferret.pid

data/install.rb

@@ -0,0 +1,19 @@
+# acts_as_ferret install script
+require 'fileutils'
+
+def install(file)
+  puts "Installing: #{file}"
+  target = File.join(File.dirname(__FILE__), '..', '..', '..', file)
+  if File.exists?(target)
+    puts "target #{target} already exists, skipping"
+  else
+    FileUtils.cp File.join(File.dirname(__FILE__), file), target
+  end
+end
+
+install File.join( 'script', 'ferret_start' )
+install File.join( 'script', 'ferret_stop' )
+install File.join( 'config', 'ferret_server.yml' )
+
+puts IO.read(File.join(File.dirname(__FILE__), 'README'))
+

data/lib/act_methods.rb

@@ -0,0 +1,194 @@
+module ActsAsFerret #:nodoc:
+        
+  # This module defines the acts_as_ferret method and is included into 
+  # ActiveRecord::Base
+  module ActMethods
+          
+    
+    def reloadable?; false end
+    
+    # declares a class as ferret-searchable. 
+    #
+    # ====options:
+    # fields:: names all fields to include in the index. If not given,
+    #          all attributes of the class will be indexed. You may also give
+    #          symbols pointing to instance methods of your model here, i.e. 
+    #          to retrieve and index data from a related model. 
+    #
+    # additional_fields:: names fields to include in the index, in addition 
+    #                     to those derived from the db scheme. use if you want 
+    #                     to add custom fields derived from methods to the db 
+    #                     fields (which will be picked by aaf). This option will 
+    #                     be ignored when the fields option is given, in that 
+    #                     case additional fields get specified there.
+    #
+    # index_dir:: declares the directory where to put the index for this class.
+    #             The default is RAILS_ROOT/index/RAILS_ENV/CLASSNAME. 
+    #             The index directory will be created if it doesn't exist.
+    #
+    # single_index:: set this to true to let this class use a Ferret
+    #                index that is shared by all classes having :single_index set to true.
+    #                :store_class_name is set to true implicitly, as well as index_dir, so 
+    #                don't bother setting these when using this option. the shared index
+    #                will be located in index/<RAILS_ENV>/shared .
+    #
+    # store_class_name:: to make search across multiple models (with either
+    #                    single_index or the multi_search method) useful, set
+    #                    this to true. the model class name will be stored in a keyword field 
+    #                    named class_name
+    #
+    # ====ferret_options:
+    # or_default:: whether query terms are required by
+    #              default (the default, false), or not (true)
+    # 
+    # analyzer:: the analyzer to use for query parsing (default: nil,
+    #            which means the ferret StandardAnalyzer gets used)
+    #
+    # default_field:: use to set one or more fields that are searched for query terms
+    #                 that don't have an explicit field list. This list should *not*
+    #                 contain any untokenized fields. If it does, you're asking
+    #                 for trouble (i.e. not getting results for queries having
+    #                 stop words in them). Aaf by default initializes the default field 
+    #                 list to contain all tokenized fields. If you use :single_index => true, 
+    #                 you really should set this option specifying your default field
+    #                 list (which should be equal in all your classes sharing the index).
+    #                 Otherwise you might get incorrect search results and you won't get 
+    #                 any lazy loading of stored field data.
+    #
+    def acts_as_ferret(options={}, ferret_options={})
+
+      # force local mode if running *inside* the Ferret server - somewhere the
+      # real indexing has to be done after all :-)
+      options.delete(:remote) if ActsAsFerret::Remote::Server.running
+
+      if options[:remote] && options[:remote] !~ /^druby/
+        # read server location from config/ferret_server.yml
+        options[:remote] = ActsAsFerret::Remote::Config.load("#{RAILS_ROOT}/config/ferret_server.yml")[:uri]
+      end
+
+
+      extend ClassMethods
+      extend SharedIndexClassMethods if options[:single_index]
+
+      include InstanceMethods
+      include MoreLikeThis::InstanceMethods
+
+      # AR hooks
+      after_create  :ferret_create
+      after_update  :ferret_update
+      after_destroy :ferret_destroy      
+
+      cattr_accessor :aaf_configuration
+
+      # default config
+      self.aaf_configuration = { 
+        :index_dir => "#{ActsAsFerret::index_dir}/#{self.name.underscore}",
+        :store_class_name => false,
+        :name => self.table_name,
+        :class_name => self.name,
+        :single_index => false,
+        :ferret => {
+          :or_default => false, 
+          :handle_parse_errors => true,
+          :default_field => nil # will be set later on
+          #:max_clauses => 512,
+          #:analyzer => Ferret::Analysis::StandardAnalyzer.new,
+          # :wild_card_downcase => true
+        }
+      }
+
+      # merge aaf options with args
+      aaf_configuration.update(options) if options.is_a?(Hash)
+
+      # list of indexed fields will be filled later
+      aaf_configuration[:ferret_fields] = Hash.new
+
+      # apply appropriate settings for shared index
+      if aaf_configuration[:single_index] 
+        aaf_configuration[:index_dir] = "#{ActsAsFerret::index_dir}/shared" 
+        aaf_configuration[:store_class_name] = true 
+      end
+
+      # merge default ferret options with args
+      aaf_configuration[:ferret].update(ferret_options) if ferret_options.is_a?(Hash)
+
+      # these properties are somewhat vital to the plugin and shouldn't
+      # be overwritten by the user:
+      aaf_configuration[:ferret].update(
+        :key               => (aaf_configuration[:single_index] ? [:id, :class_name] : :id),
+        :path              => aaf_configuration[:index_dir],
+        :auto_flush        => true, # slower but more secure in terms of locking problems TODO disable when running in drb mode?
+        :create_if_missing => true
+      )
+      
+      if aaf_configuration[:fields]
+        add_fields(aaf_configuration[:fields])
+      else
+        add_fields(self.new.attributes.keys.map { |k| k.to_sym })
+        add_fields(aaf_configuration[:additional_fields])
+      end
+
+      ActsAsFerret::ensure_directory aaf_configuration[:index_dir] unless options[:remote]
+
+      # now that all fields have been added, we can initialize the default
+      # field list to be used by the query parser.
+      # It will include all content fields *not* marked as :untokenized.
+      # This fixes the otherwise failing CommentTest#test_stopwords. Basically
+      # this means that by default only tokenized fields (which is the default)
+      # will be searched. If you want to search inside the contents of an
+      # untokenized field, you'll have to explicitly specify it in your query.
+      #
+      # Unfortunately this is not very useful with a shared index (see
+      # http://projects.jkraemer.net/acts_as_ferret/ticket/85)
+      # You should consider specifying the default field list to search for as
+      # part of the ferret_options hash in your call to acts_as_ferret.
+      aaf_configuration[:ferret][:default_field] ||= if aaf_configuration[:single_index]
+        logger.warn "You really should set the acts_as_ferret :default_field option when using a shared index!"
+        '*'
+      else
+        aaf_configuration[:ferret_fields].keys.select do |f| 
+          aaf_configuration[:ferret_fields][f][:index] != :untokenized
+        end
+      end
+      logger.info "default field list: #{aaf_configuration[:ferret][:default_field].inspect}"
+    end
+
+
+    protected
+    
+    # helper that defines a method that adds the given field to a ferret 
+    # document instance
+    def define_to_field_method(field, options = {})
+      options.reverse_merge!( :store       => :no, 
+                              :highlight   => :yes, 
+                              :index       => :yes, 
+                              :term_vector => :with_positions_offsets,
+                              :boost       => 1.0 )
+      aaf_configuration[:ferret_fields][field] = options
+      define_method("#{field}_to_ferret".to_sym) do
+        begin
+          val = content_for_field_name(field)
+        rescue
+          logger.warn("Error retrieving value for field #{field}: #{$!}")
+          val = ''
+        end
+        logger.debug("Adding field #{field} with value '#{val}' to index")
+        val
+      end
+    end
+
+    def add_fields(field_config)
+      if field_config.respond_to?(:each_pair)
+        field_config.each_pair do |key,val|
+          define_to_field_method(key,val)                  
+        end
+      elsif field_config.respond_to?(:each)
+        field_config.each do |field| 
+          define_to_field_method(field)
+        end                
+      end
+    end
+
+  end
+
+end

data/lib/acts_as_ferret.rb

@@ -18,18 +18,27 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+require 'active_support'
 require 'active_record'
 require 'set'
 require 'ferret'
 
-require 'multi_index'
-require 'more_like_this'
+require 'act_methods'
 require 'class_methods'
+require 'shared_index_class_methods'
+require 'ferret_result'
 require 'instance_methods'
 
-# 0.10 problems
-# Ferret::Search::Similarity, Ferret::Search::Similarity.default missing
-# IndexReader#latest? segfaults when used on multiple indexes
+require 'multi_index'
+require 'more_like_this'
+
+require 'index'
+require 'local_index'
+require 'shared_index'
+require 'remote_index'
+
+require 'ferret_server'
+
 
 # The Rails ActiveRecord Ferret Mixin.
 #
@@ -41,7 +50,7 @@ require 'instance_methods'
 #
 # basic usage:
 # include the following in your model class (specifiying the fields you want to get indexed):
-# acts_as_ferret :fields => [ 'title', 'description' ]
+# acts_as_ferret :fields => [ :title, :description ]
 #
 # now you can use ModelClass.find_by_contents(query) to find instances of your model
 # whose indexed fields match a given query. All query terms are required by default, but 
@@ -54,59 +63,61 @@ require 'instance_methods'
 # Kasper Weibel Nielsen-Refs (original author)
 # Jens Kraemer <jk@jkraemer.net> (active maintainer)
 #
-module FerretMixin
-  module Acts #:nodoc:
-    module ARFerret #:nodoc:
+module ActsAsFerret
 
-      # decorator that adds a total_hits accessor to search result arrays
-      class SearchResults
-        attr_reader :total_hits
-        def initialize(results, total_hits)
-          @results = results
-          @total_hits = total_hits
-        end
-        def method_missing(symbol, *args, &block)
-          @results.send(symbol, *args, &block)
-        end
-        def respond_to?(name)
-          self.methods.include?(name) || @results.respond_to?(name)
-        end
-      end
-      
-      def self.ensure_directory(dir)
-        FileUtils.mkdir_p dir unless File.directory? dir
-      end
-      
-      # make sure the default index base dir exists. by default, all indexes are created
-      # under RAILS_ROOT/index/RAILS_ENV
-      def self.init_index_basedir
-        index_base = "#{RAILS_ROOT}/index"
-        ensure_directory index_base
-        @@index_dir = "#{index_base}/#{RAILS_ENV}"
-        ensure_directory @@index_dir
-      end
-      
-      mattr_accessor :index_dir
-      init_index_basedir
-      
-      def self.append_features(base)
-        super
-        base.extend(ClassMethods)
-      end
+    # global Hash containing all multi indexes created by all classes using the plugin
+    # key is the concatenation of alphabetically sorted names of the classes the
+    # searcher searches.
+    @@multi_indexes = Hash.new
+    def self.multi_indexes; @@multi_indexes end
 
-      
-      
+    # global Hash containing the ferret indexes of all classes using the plugin
+    # key is the index directory.
+    @@ferret_indexes = Hash.new
+    def self.ferret_indexes; @@ferret_indexes end
+ 
+  # decorator that adds a total_hits accessor to search result arrays
+  class SearchResults
+    attr_reader :total_hits
+    def initialize(results, total_hits)
+      @results = results
+      @total_hits = total_hits
+    end
+    def method_missing(symbol, *args, &block)
+      @results.send(symbol, *args, &block)
+    end
+    def respond_to?(name)
+      self.methods.include?(name) || @results.respond_to?(name)
     end
   end
+  
+  def self.ensure_directory(dir)
+    FileUtils.mkdir_p dir unless File.directory? dir
+  end
+  
+  # make sure the default index base dir exists. by default, all indexes are created
+  # under RAILS_ROOT/index/RAILS_ENV
+  def self.init_index_basedir
+    index_base = "#{RAILS_ROOT}/index"
+    @@index_dir = "#{index_base}/#{RAILS_ENV}"
+  end
+  
+  mattr_accessor :index_dir
+  init_index_basedir
+  
+  def self.append_features(base)
+    super
+    base.extend(ClassMethods)
+  end
+  
 end
 
-# reopen ActiveRecord and include all the above to make
-# them available to all our models if they want it
-ActiveRecord::Base.class_eval do
-  include FerretMixin::Acts::ARFerret
-end
+# include acts_as_ferret method into ActiveRecord::Base
+ActiveRecord::Base.extend ActsAsFerret::ActMethods
 
 
+# small Ferret monkey patch
+# TODO check if this is still necessary
 class Ferret::Index::MultiReader
   def latest?
     # TODO: Exception handling added to resolve ticket #6. 
@@ -121,4 +132,15 @@ class Ferret::Index::MultiReader
   end
 end
 
-# END acts_as_ferret.rb
+# add marshalling support to SortFields
+class Ferret::Search::SortField
+  def _dump(depth)
+    to_s
+  end
+
+  def self._load(string)
+    raise "invalid value: #{string}" unless string =~ /^(\w+):<(\w+)>(\!)?$/
+    new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
+  end
+end
+