acts_as_ferret 0.3.1 → 0.4.0

data/lib/ferret_result.rb

@@ -0,0 +1,36 @@
+module ActsAsFerret
+
+  # mixed into the FerretResult and AR classes calling acts_as_ferret
+  module ResultAttributes
+    # holds the score this record had when it was found via
+    # acts_as_ferret
+    attr_accessor :ferret_score
+
+    attr_accessor :ferret_rank
+  end
+
+  class FerretResult
+    include ResultAttributes
+    attr_accessor :id
+
+    def initialize(model, id, score, data = {})
+      @model = model.constantize
+      @id = id
+      @ferret_score = score
+      @data = data
+    end
+    
+    def method_missing(method, *args)
+      if @ar_record || @data[method].nil?
+        ferret_load_record unless @ar_record
+        @ar_record.send method, *args
+      else
+        @data[method]
+      end
+    end
+
+    def ferret_load_record
+      @ar_record = @model.find(id)
+    end
+  end
+end

data/lib/ferret_server.rb

@@ -0,0 +1,89 @@
+require 'drb'
+require 'thread'
+require 'yaml'
+require 'erb'
+
+
+module ActsAsFerret
+
+module Remote
+
+  module Config
+    class << self
+      DEFAULTS = {
+        'host' => 'localhost',
+        'port' => '9009'
+      }
+      # reads connection settings from config file
+      def load(file = "#{RAILS_ROOT}/config/ferret_server.yml")
+        config = DEFAULTS.merge(YAML.load(ERB.new(IO.read(file)).result))
+        if config = config[RAILS_ENV]
+          config[:uri] = "druby://#{config['host']}:#{config['port']}"
+          return config
+        end
+      end
+    end
+  end
+
+  # This class acts as a drb server listening for indexing and
+  # search requests from models declared to 'acts_as_ferret :remote => true'
+  #
+  # Usage: 
+  # - copy doc/ferret_server.yml to RAILS_ROOT/config and modify to suit
+  # your needs.
+  # - run script/ferret_server (in the plugin directory) via script/runner:
+  # RAILS_ENV=production script/runner vendor/plugins/acts_as_ferret/script/ferret_server
+  #
+  # TODO: automate installation of files to script/ and config/
+  class Server
+
+    cattr_accessor :running
+
+    def self.start(uri = nil)
+      ActiveRecord::Base.allow_concurrency = true
+      uri ||= ActsAsFerret::Remote::Config.load[:uri]
+      DRb.start_service(uri, ActsAsFerret::Remote::Server.new)
+      self.running = true
+    end
+
+    def initialize
+      @logger = Logger.new("#{RAILS_ROOT}/log/ferret_server.log")
+    end
+
+    # handles all incoming method calls, and sends them on to the LocalIndex
+    # instance of the correct model class.
+    #
+    # Calls are not queued atm, so this will block until the call returned.
+    # Might throw the occasional LockError, too, which most probably means that you're 
+    # a) rebuilding your index or 
+    # b) have *really* high load. I wasn't able to reproduce this case until
+    # now, if you do, please contact me.
+    #
+    # TODO: rebuild indexes in separate directory so no lock errors in these
+    # cases.
+    def method_missing(name, *args)
+      clazz = args.shift.constantize
+      begin
+        @logger.debug "call index method: #{name} with #{args.inspect}"
+        clazz.aaf_index.send name, *args
+      rescue NoMethodError
+        @logger.debug "no luck, trying to call class method instead"
+        clazz.send name, *args
+      end
+    rescue
+      @logger.error "ferret server error #{$!}\n#{$!.backtrace.join '\n'}"
+      raise
+    end
+
+    def ferret_index(class_name)
+      # TODO check if in use!
+      class_name.constantize.aaf_index.ferret_index
+    end
+
+    # the main loop taking stuff from the queue and running it...
+    #def run
+    #end
+
+  end
+end
+end

data/lib/index.rb

@@ -0,0 +1,31 @@
+module ActsAsFerret
+
+  # base class for local and remote indexes
+  class AbstractIndex
+
+    attr_reader :aaf_configuration
+    attr_reader :logger
+    def initialize(aaf_configuration)
+      @aaf_configuration = aaf_configuration
+      @logger = Logger.new("#{RAILS_ROOT}/log/ferret_index.log")
+    end
+    
+    class << self
+      def proxy_method(name, *args)
+        define_method name do |*args|
+          @server.send name, model_class_name, *args
+        end
+      end
+
+      def index_proxy_method(*names)
+        names.each do |name|
+          define_method name do |*args|
+            @server.send :"index_#{name}", model_class_name, *args
+          end
+        end
+      end
+
+    end
+  end
+
+end

data/lib/instance_methods.rb

@@ -1,157 +1,126 @@
-module FerretMixin
-  module Acts #:nodoc:
-    module ARFerret #:nodoc:
+module ActsAsFerret #:nodoc:
 
-      module InstanceMethods
-        include MoreLikeThis
+  module InstanceMethods
+    include ResultAttributes
 
-          # Returns an array of strings with the matches highlighted. The +query+ can
-          # either a query String or a Ferret::Search::Query object.
-          # 
-          # === Options
-          #
-          # field::            field to take the content from. This field has 
-          #                    to have it's content stored in the index 
-          #                    (:store => :yes in your call to aaf). If not
-          #                    given, all stored fields are searched, and the
-          #                    highlighted content found in all of them is returned.
-          #                    set :highlight => :no in the field options to
-          #                    avoid highlighting of contents from a :stored field.
-          # excerpt_length::   Default: 150. Length of excerpt to show. Highlighted
-          #                    terms will be in the centre of the excerpt.
-          # num_excerpts::     Default: 2. Number of excerpts to return.
-          # pre_tag::          Default: "<em>". Tag to place to the left of the
-          #                    match.  
-          # post_tag::         Default: "</em>". This tag should close the
-          #                    +:pre_tag+.
-          # ellipsis::         Default: "...". This is the string that is appended
-          #                    at the beginning and end of excerpts (unless the
-          #                    excerpt hits the start or end of the field. You'll
-          #                    probably want to change this so a Unicode elipsis
-          #                    character.
-        def highlight(query, options = {})
-          options = { :num_excerpts => 2, :pre_tag => '<em>', :post_tag => '</em>' }.update(options)
-          i = self.class.ferret_index
-          highlights = []
-          i.synchronize do
-            doc_num = self.document_number
-            if options[:field]
-              highlights << i.highlight(query, doc_num, options)
-            else
-              query = i.process_query(query) # process only once
-              fields_for_ferret.each_pair do |field, config|
-                next if config[:store] == :no || config[:highlight] == :no
-                options[:field] = field
-                highlights << i.highlight(query, doc_num, options)
-              end
-            end
-          end
-          return highlights.compact.flatten[0..options[:num_excerpts]-1]
-        end
-        
-        # re-eneable ferret indexing after a call to #disable_ferret
-        def ferret_enable; @ferret_disabled = nil end
-       
-        # returns true if ferret indexing is enabled
-        def ferret_enabled?; @ferret_disabled.nil? end
+    # Returns an array of strings with the matches highlighted. The +query+ can
+    # either be a String or a Ferret::Search::Query object.
+    # 
+    # === Options
+    #
+    # field::            field to take the content from. This field has 
+    #                    to have it's content stored in the index 
+    #                    (:store => :yes in your call to aaf). If not
+    #                    given, all stored fields are searched, and the
+    #                    highlighted content found in all of them is returned.
+    #                    set :highlight => :no in the field options to
+    #                    avoid highlighting of contents from a :stored field.
+    # excerpt_length::   Default: 150. Length of excerpt to show. Highlighted
+    #                    terms will be in the centre of the excerpt.
+    # num_excerpts::     Default: 2. Number of excerpts to return.
+    # pre_tag::          Default: "<em>". Tag to place to the left of the
+    #                    match.  
+    # post_tag::         Default: "</em>". This tag should close the
+    #                    +:pre_tag+.
+    # ellipsis::         Default: "...". This is the string that is appended
+    #                    at the beginning and end of excerpts (unless the
+    #                    excerpt hits the start or end of the field. You'll
+    #                    probably want to change this so a Unicode elipsis
+    #                    character.
+    def highlight(query, options = {})
+      self.class.aaf_index.highlight(id, self.class.name, query, options)
+    end
+    
+    # re-eneable ferret indexing after a call to #disable_ferret
+    def ferret_enable; @ferret_disabled = nil end
+    
+    # returns true if ferret indexing is enabled
+    # the optional parameter will be true if the method is called by rebuild_index, 
+    # and false otherwise. I.e. useful to enable a model only for indexing during 
+    # scheduled reindex runs.
+    def ferret_enabled?(is_rebuild = false); @ferret_disabled.nil? end
 
-        # Disable Ferret for a specified amount of time. ::once will disable
-        # Ferret for the next call to #save (this is the default), ::always will 
-        # do so for all subsequent calls.
-        # To manually trigger reindexing of a record, you can call #ferret_update 
-        # directly. 
-        #
-        # When given a block, this will be executed without any ferret indexing of 
-        # this object taking place. The optional argument in this case can be used 
-        # to indicate if the object should be indexed after executing the block
-        # (::index_when_finished). Automatic Ferret indexing of this object will be 
-        # turned on after the block has been executed. If passed ::index_when_true, 
-        # the index will only be updated if the block evaluated not to false or nil.
-        def disable_ferret(option = :once)
-          if block_given?
-            @ferret_disabled = :always
-            result = yield
-            ferret_enable
-            ferret_update if option == :index_when_finished || (option == :index_when_true && result)
-            result
-          elsif [:once, :always].include?(option)
-            @ferret_disabled = option
-          else
-            raise ArgumentError.new("Invalid Argument #{option}")
-          end
-        end
+    # Disable Ferret for a specified amount of time. ::once will disable
+    # Ferret for the next call to #save (this is the default), ::always will 
+    # do so for all subsequent calls.
+    # To manually trigger reindexing of a record, you can call #ferret_update 
+    # directly. 
+    #
+    # When given a block, this will be executed without any ferret indexing of 
+    # this object taking place. The optional argument in this case can be used 
+    # to indicate if the object should be indexed after executing the block
+    # (::index_when_finished). Automatic Ferret indexing of this object will be 
+    # turned on after the block has been executed. If passed ::index_when_true, 
+    # the index will only be updated if the block evaluated not to false or nil.
+    def disable_ferret(option = :once)
+      if block_given?
+        @ferret_disabled = :always
+        result = yield
+        ferret_enable
+        ferret_update if option == :index_when_finished || (option == :index_when_true && result)
+        result
+      elsif [:once, :always].include?(option)
+        @ferret_disabled = option
+      else
+        raise ArgumentError.new("Invalid Argument #{option}")
+      end
+    end
 
-        # add to index
-        def ferret_create
-          if ferret_enabled?
-            logger.debug "ferret_create/update: #{self.class.name} : #{self.id}"
-            self.class.ferret_index << self.to_doc
-          else
-            ferret_enable if @ferret_disabled == :once
-          end
-          true # signal success to AR
-        end
-        alias :ferret_update :ferret_create
-        
+    # add to index
+    def ferret_create
+      if ferret_enabled?
+        logger.debug "ferret_create/update: #{self.class.name} : #{self.id}"
+        self.class.aaf_index << self
+      else
+        ferret_enable if @ferret_disabled == :once
+      end
+      true # signal success to AR
+    end
+    alias :ferret_update :ferret_create
+    
 
-        # remove from index
-        def ferret_destroy
-          logger.debug "ferret_destroy: #{self.class.name} : #{self.id}"
-          begin
-            self.class.ferret_index.query_delete(query_for_self)
-          rescue
-            logger.warn("Could not find indexed value for this object: #{$!}")
-          end
-          true # signal success to AR
-        end
-        
-        # convert instance to ferret document
-        def to_doc
-          logger.debug "creating doc for class: #{self.class.name}, id: #{self.id}"
-          # Churn through the complete Active Record and add it to the Ferret document
-          doc = Ferret::Document.new
-          # store the id of each item
-          doc[:id] = self.id
+    # remove from index
+    def ferret_destroy
+      logger.debug "ferret_destroy: #{self.class.name} : #{self.id}"
+      begin
+        self.class.aaf_index.remove self.id, self.class.name
+      rescue
+        logger.warn("Could not find indexed value for this object: #{$!}\n#{$!.backtrace}")
+      end
+      true # signal success to AR
+    end
+    
+    # turn this instance into a ferret document (which basically is a hash of
+    # fieldname => value pairs)
+    def to_doc
+      logger.debug "creating doc for class: #{self.class.name}, id: #{self.id}"
+      returning doc = Ferret::Document.new do
+        # store the id of each item
+        doc[:id] = self.id
 
-          # store the class name if configured to do so
-          if configuration[:store_class_name]
-            doc[:class_name] = self.class.name
-          end
-          
-          # iterate through the fields and add them to the document
-          fields_for_ferret.each_pair do |field, config|
-            doc[field] = self.send("#{field}_to_ferret") unless config[:ignore]
-          end
-          return doc
+        # store the class name if configured to do so
+        doc[:class_name] = self.class.name if aaf_configuration[:store_class_name]
+      
+        # iterate through the fields and add them to the document
+        aaf_configuration[:ferret_fields].each_pair do |field, config|
+          doc[field] = self.send("#{field}_to_ferret") unless config[:ignore]
         end
+      end
+    end
 
-        # returns the ferret document number this record has.
-        def document_number
-          hits = self.class.ferret_index.search(query_for_self)
-          return hits.hits.first.doc if hits.total_hits == 1
-          raise "cannot determine document number from primary key: #{self}"
-        end
+    def document_number
+      self.class.aaf_index.document_number(id, self.class.name)
+    end
 
-        # holds the score this record had when it was found via
-        # acts_as_ferret
-        attr_accessor :ferret_score
-          
-        protected
+    def query_for_record
+      self.class.aaf_index.query_for_record(id, self.class.name)
+    end
 
-        # build a ferret query matching only this record
-        def query_for_self
-          query = Ferret::Search::TermQuery.new(:id, self.id.to_s)
-          if self.class.configuration[:single_index]
-            bq = Ferret::Search::BooleanQuery.new
-            bq.add_query(query, :must)
-            bq.add_query(Ferret::Search::TermQuery.new(:class_name, self.class.name), :must)
-            return bq
-          end
-          return query
-        end
+    def content_for_field_name(field)
+      self[field] || self.instance_variable_get("@#{field.to_s}".to_sym) || self.send(field.to_sym)
+    end
 
-      end
 
-    end
   end
+
 end

data/lib/local_index.rb

@@ -0,0 +1,257 @@
+module ActsAsFerret
+  
+  class LocalIndex < AbstractIndex
+    include MoreLikeThis::IndexMethods
+
+
+    def initialize(aaf_configuration)
+      super
+      ensure_index_exists
+    end
+
+    # The 'real' Ferret Index instance
+    def ferret_index
+      ensure_index_exists
+      @ferret_index ||= Ferret::Index::Index.new(aaf_configuration[:ferret])
+    end
+
+    # Checks for the presence of a segments file in the index directory
+    # Rebuilds the index if none exists.
+    def ensure_index_exists
+      unless File.file? "#{aaf_configuration[:index_dir]}/segments"
+        close
+        rebuild_index 
+      end
+    end
+
+    # Closes the underlying index instance
+    def close
+      @ferret_index.close if @ferret_index
+    rescue StandardError 
+      # is raised when index already closed
+    ensure
+      @ferret_index = nil
+    end
+
+    # rebuilds the index from all records of the model class this index belongs
+    # to. Arguments can be given in shared index scenarios to name multiple
+    # model classes to include in the index
+    def rebuild_index(*models)
+      logger.debug "rebuild index: #{models.inspect}"
+      models << aaf_configuration[:class_name] unless models.include?(aaf_configuration[:class_name])
+      models = models.flatten.uniq.map(&:constantize)
+      index = Ferret::Index::Index.new(aaf_configuration[:ferret].dup.update(:auto_flush => false, 
+                                                                             :field_infos => field_infos(models),
+                                                                             :create => true))
+      models.each do |model|
+        reindex_model(index, model)
+      end
+      logger.debug("Created Ferret index in: #{aaf_configuration[:index_dir]}")
+      index.flush
+      index.optimize
+      index.close
+      close_multi_indexes
+    end
+
+    # Parses the given query string into a Ferret Query object.
+    def process_query(query)
+      # work around ferret bug in #process_query (doesn't ensure the
+      # reader is open)
+      ferret_index.synchronize do
+        ferret_index.send(:ensure_reader_open)
+        original_query = ferret_index.process_query(query)
+      end
+    end
+
+    # Total number of hits for the given query. 
+    def total_hits(query, options = {})
+      ferret_index.search(query, options).total_hits
+    end
+
+    def determine_lazy_fields(options = {})
+      stored_fields = options[:lazy]
+      if stored_fields && !(Array === stored_fields)
+        stored_fields = aaf_configuration[:ferret_fields].select { |field, config| config[:store] == :yes }.map(&:first)
+      end
+      logger.debug "stored_fields: #{stored_fields}"
+      return stored_fields
+    end
+
+    # Queries the Ferret index to retrieve model class, id, score and the
+    # values of any fields stored in the index for each hit.
+    # If a block is given, these are yielded and the number of total hits is
+    # returned. Otherwise [total_hits, result_array] is returned.
+    def find_id_by_contents(query, options = {})
+      result = []
+      index = ferret_index
+      logger.debug "query: #{ferret_index.process_query query}" # TODO only enable this for debugging purposes
+      lazy_fields = determine_lazy_fields options
+
+      total_hits = index.search_each(query, options) do |hit, score|
+        doc = index[hit]
+        model = aaf_configuration[:store_class_name] ? doc[:class_name] : aaf_configuration[:class_name]
+        # fetch stored fields if lazy loading
+        data = {}
+        lazy_fields.each { |field| data[field] = doc[field] } if lazy_fields
+        if block_given?
+          yield model, doc[:id], score, data
+        else
+          result << { :model => model, :id => doc[:id], :score => score, :data => data }
+        end
+      end
+      #logger.debug "id_score_model array: #{result.inspect}"
+      return block_given? ? total_hits : [total_hits, result]
+    end
+
+    # Queries multiple Ferret indexes to retrieve model class, id and score for 
+    # each hit. Use the models parameter to give the list of models to search.
+    # If a block is given, model, id and score are yielded and the number of 
+    # total hits is returned. Otherwise [total_hits, result_array] is returned.
+    def id_multi_search(query, models, options = {})
+      models.map!(&:constantize)
+      index = multi_index(models)
+      result = []
+      lazy_fields = determine_lazy_fields options
+      total_hits = index.search_each(query, options) do |hit, score|
+        doc = index[hit]
+        # fetch stored fields if lazy loading
+        data = {}
+        lazy_fields.each { |field| data[field] = doc[field] } if lazy_fields
+        if block_given?
+          yield doc[:class_name], doc[:id], score, doc, data
+        else
+          result << { :model => doc[:class_name], :id => doc[:id], :score => score, :data => data }
+        end
+      end
+      return block_given? ? total_hits : [ total_hits, result ]
+    end
+
+    ######################################
+    # methods working on a single record
+    # called from instance_methods, here to simplify interfacing with the
+    # remote ferret server
+    # TODO having to pass id and class_name around like this isn't nice
+    ######################################
+
+    # add record to index
+    # record may be the full AR object, a Ferret document instance or a Hash
+    def add(record)
+      record = record.to_doc unless Hash === record || Ferret::Document === record
+      ferret_index << record
+    end
+    alias << add
+
+    # delete record from index
+    def remove(id, class_name)
+      ferret_index.query_delete query_for_record(id, class_name)
+    end
+
+    # highlight search terms for the record with the given id.
+    def highlight(id, class_name, query, options = {})
+      options.reverse_merge! :num_excerpts => 2, :pre_tag => '<em>', :post_tag => '</em>'
+      highlights = []
+      ferret_index.synchronize do
+        doc_num = document_number(id, class_name)
+        if options[:field]
+          highlights << ferret_index.highlight(query, doc_num, options)
+        else
+          query = process_query(query) # process only once
+          aaf_configuration[:ferret_fields].each_pair do |field, config|
+            next if config[:store] == :no || config[:highlight] == :no
+            options[:field] = field
+            highlights << ferret_index.highlight(query, doc_num, options)
+          end
+        end
+      end
+      return highlights.compact.flatten[0..options[:num_excerpts]-1]
+    end
+
+    # retrieves the ferret document number of the record with the given id.
+    def document_number(id, class_name)
+      hits = ferret_index.search(query_for_record(id, class_name))
+      return hits.hits.first.doc if hits.total_hits == 1
+      raise "cannot determine document number from primary key: #{id}"
+    end
+
+    # build a ferret query matching only the record with the given id
+    # the class name only needs to be given in case of a shared index configuration
+    def query_for_record(id, class_name = nil)
+      Ferret::Search::TermQuery.new(:id, id.to_s)
+    end
+
+
+    protected
+
+    # returns a MultiIndex instance operating on a MultiReader
+    def multi_index(model_classes)
+      model_classes.sort! { |a, b| a.name <=> b.name }
+      key = model_classes.inject("") { |s, clazz| s + clazz.name }
+      multi_config = aaf_configuration[:ferret].dup
+      multi_config.delete :default_field  # we don't want the default field list of *this* class for multi_searching
+      ActsAsFerret::multi_indexes[key] ||= MultiIndex.new(model_classes, multi_config)
+    end
+ 
+    def close_multi_indexes
+      # close combined index readers, just in case
+      # this seems to fix a strange test failure that seems to relate to a
+      # multi_index looking at an old version of the content_base index.
+      ActsAsFerret::multi_indexes.each_pair do |key, index|
+        # puts "#{key} -- #{self.name}"
+        # TODO only close those where necessary (watch inheritance, where
+        # self.name is base class of a class where key is made from)
+        index.close #if key =~ /#{self.name}/
+      end
+      ActsAsFerret::multi_indexes.clear
+    end
+
+    def reindex_model(index, model = aaf_configuration[:class_name].constantize)
+      # index in batches of 1000 to limit memory consumption (fixes #24)
+      # TODO make configurable through options
+      batch_size = 1000
+      model_count = model.count.to_f
+      work_done = 0
+      batch_time = 0
+      logger.info "reindexing model #{model.name}"
+      order = "#{model.primary_key} ASC" # this works around a bug in sqlserver-adapter (where paging only works with an order applied)
+      model.transaction do
+        0.step(model.count, batch_size) do |i|
+          b1 = Time.now.to_f
+          model.find(:all, :limit => batch_size, :offset => i, :order => order).each do |rec|
+            index << rec.to_doc if rec.ferret_enabled?(true)
+          end
+          batch_time = Time.now.to_f - b1
+          work_done = i.to_f / model_count * 100.0 if model_count > 0
+          remaining_time = ( batch_time / batch_size ) * ( model_count - i + batch_size )
+          logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+        end
+      end
+    end
+
+    # builds a FieldInfos instance for creation of an index containing fields
+    # for the given model classes.
+    def field_infos(models)
+      # default attributes for fields
+      fi = Ferret::Index::FieldInfos.new(:store => :no, 
+                                         :index => :yes, 
+                                         :term_vector => :no,
+                                         :boost => 1.0)
+      # primary key
+      fi.add_field(:id, :store => :yes, :index => :untokenized) 
+      # class_name
+      if aaf_configuration[:store_class_name]
+        fi.add_field(:class_name, :store => :yes, :index => :untokenized) 
+      end
+      fields = {}
+      models.each do |model|
+        fields.update(model.aaf_configuration[:ferret_fields])
+      end
+      fields.each_pair do |field, options|
+        fi.add_field(field, { :store => :no, 
+                              :index => :yes }.update(options)) 
+      end
+      return fi
+    end
+
+  end
+
+end