sdsykes_acts_as_ferret 0.4.3.1

data/lib/ferret_cap_tasks.rb

@@ -0,0 +1,21 @@
+# Ferret DRb server Capistrano tasks
+# Usage:
+# Add require 'vendor/plugins/acts_as_ferret/lib/ferret_cap_tasks' to your
+# config/deploy.rb
+# call ferret.restart where you restart your Mongrels.
+# ferret.stop and ferret.start are available, too.
+module FerretCapTasks
+  def start
+    run "cd #{current_path}; RAILS_ENV=production script/ferret_start"
+  end
+
+  def stop
+    run "cd #{current_path}; RAILS_ENV=production script/ferret_stop"
+  end
+
+  def restart
+    stop
+    start
+  end
+end
+Capistrano.plugin :ferret, FerretCapTasks

data/lib/ferret_extensions.rb

@@ -0,0 +1,115 @@
+module Ferret
+
+  module Analysis
+  
+    # = PerFieldAnalyzer
+    #
+    # This PerFieldAnalyzer is a workaround to a memory leak in 
+    # ferret 0.11.4. It does basically do the same as the original
+    # Ferret::Analysis::PerFieldAnalyzer, but without the leak :)
+    # 
+    # http://ferret.davebalmain.com/api/classes/Ferret/Analysis/PerFieldAnalyzer.html
+    #
+    # Thanks to Ben from omdb.org for tracking this down and creating this
+    # workaround.
+    # You can read more about the issue there:
+    # http://blog.omdb-beta.org/2007/7/29/tracking-down-a-memory-leak-in-ferret-0-11-4
+    class PerFieldAnalyzer < ::Ferret::Analysis::Analyzer
+      def initialize( default_analyzer = StandardAnalyzer.new )
+        @analyzers = {}
+        @default_analyzer = default_analyzer
+      end
+            
+      def add_field( field, analyzer )
+        @analyzers[field] = analyzer
+      end
+      alias []= add_field
+                
+      def token_stream(field, string)
+        @analyzers.has_key?(field) ? @analyzers[field].token_stream(field, string) : 
+          @default_analyzer.token_stream(field, string)
+      end
+    end
+  end
+
+  class Index::Index
+    attr_accessor :batch_size, :logger
+
+    def index_models(models)
+      models.each { |model| index_model model }
+      flush
+      optimize
+      close
+      ActsAsFerret::close_multi_indexes
+    end
+
+    def index_model(model)
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :reindex => true)
+      logger.info "reindexing model #{model.name}"
+
+      model.records_for_rebuild(@batch_size) do |records, offset|
+        bulk_indexer.index_records(records, offset)
+      end
+    end
+
+    def bulk_index(model, ids, options = {})
+      options.reverse_merge! :optimize => true
+      orig_flush = @auto_flush
+      @auto_flush = false
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :total => ids.size)
+      model.records_for_bulk_index(ids, @batch_size) do |records, offset|
+        logger.debug "#{model} bulk indexing #{records.size} at #{offset}"
+        bulk_indexer.index_records(records, offset)
+      end
+      logger.info 'finishing bulk index...'
+      flush
+      if options[:optimize]
+        logger.info 'optimizing...'
+        optimize 
+      end
+      @auto_flush = orig_flush
+    end
+
+  end
+
+  # add marshalling support to SortFields
+  class Search::SortField
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      case string
+        when /<DOC(_ID)?>!/         : Ferret::Search::SortField::DOC_ID_REV
+        when /<DOC(_ID)?>/          : Ferret::Search::SortField::DOC_ID
+        when '<SCORE>!'             : Ferret::Search::SortField::SCORE_REV
+        when '<SCORE>'              : Ferret::Search::SortField::SCORE
+        when /^(\w+):<(\w+)>(!)?$/ : new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
+        else raise "invalid value: #{string}"
+      end
+    end
+  end
+
+  # add marshalling support to Sort
+  class Search::Sort
+    def _dump(depth)
+      to_s
+    end
+
+    def self._load(string)
+      # we exclude the last <DOC> sorting as it is appended by new anyway
+      if string =~ /^Sort\[(.*?)(<DOC>(!)?)?\]$/
+        sort_fields = $1.split(',').map do |value| 
+        value.strip!
+          Ferret::Search::SortField._load value unless value.blank?
+        end
+        new sort_fields.compact
+      else
+        raise "invalid value: #{string}"
+      end
+    end
+  end
+
+end

data/lib/ferret_result.rb

@@ -0,0 +1,36 @@
+module ActsAsFerret
+
+  # mixed into the FerretResult and AR classes calling acts_as_ferret
+  module ResultAttributes
+    # holds the score this record had when it was found via
+    # acts_as_ferret
+    attr_accessor :ferret_score
+
+    attr_accessor :ferret_rank
+  end
+
+  class FerretResult
+    include ResultAttributes
+    attr_accessor :id
+
+    def initialize(model, id, score, data = {})
+      @model = model.constantize
+      @id = id
+      @ferret_score = score
+      @data = data
+    end
+    
+    def method_missing(method, *args)
+      if @ar_record || @data[method].nil?
+        ferret_load_record unless @ar_record
+        @ar_record.send method, *args
+      else
+        @data[method]
+      end
+    end
+
+    def ferret_load_record
+      @ar_record = @model.find(id)
+    end
+  end
+end

data/lib/ferret_server.rb

@@ -0,0 +1,203 @@
+require 'drb'
+require 'thread'
+require 'yaml'
+require 'erb'
+
+################################################################################
+module ActsAsFerret
+  module Remote
+
+    ################################################################################
+    class Config
+
+      ################################################################################
+      DEFAULTS = {
+        'host'      => 'localhost',
+        'port'      => '9009',
+        'cf'        => "#{RAILS_ROOT}/config/ferret_server.yml",
+        'pid_file'  => "#{RAILS_ROOT}/log/ferret_server.pid",
+        'log_file'  => "#{RAILS_ROOT}/log/ferret_server.log",
+        'log_level' => 'debug',
+      }
+
+      ################################################################################
+      # load the configuration file and apply default settings
+      def initialize (file=DEFAULTS['cf'])
+        @everything = YAML.load(ERB.new(IO.read(file)).result)
+        raise "malformed ferret server config" unless @everything.is_a?(Hash)
+        @config = DEFAULTS.merge(@everything[RAILS_ENV] || {})
+        @config['uri'] = "druby://#{host}:#{port}" if @everything[RAILS_ENV]
+      end
+
+      ################################################################################
+      # treat the keys of the config data as methods
+      def method_missing (name, *args)
+        @config.has_key?(name.to_s) ? @config[name.to_s] : super
+      end
+
+    end
+
+    #################################################################################
+    # This class acts as a drb server listening for indexing and
+    # search requests from models declared to 'acts_as_ferret :remote => true'
+    #
+    # Usage: 
+    # - modify RAILS_ROOT/config/ferret_server.yml to suit your needs. 
+    # - environments for which no section in the config file exists will use 
+    #   the index locally (good for unit tests/development mode)
+    # - run script/ferret_server to start the server:
+    # script/ferret_server -e production start
+    # - to stop the server run
+    # script/ferret_server -e production stop
+    #
+    class Server
+
+      #################################################################################
+      # FIXME include detection of OS and include the correct file
+      require 'unix_daemon'
+      include(ActsAsFerret::Remote::UnixDaemon)
+
+      ################################################################################
+      cattr_accessor :running
+
+      ################################################################################
+      def initialize
+        @cfg = ActsAsFerret::Remote::Config.new
+        ActiveRecord::Base.allow_concurrency = true
+        ActiveRecord::Base.logger = @logger = Logger.new(@cfg.log_file)
+        ActiveRecord::Base.logger.level = Logger.const_get(@cfg.log_level.upcase) rescue Logger::DEBUG
+      end
+
+      ################################################################################
+      # start the server
+      def start
+        raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
+        $stdout.puts("starting ferret server...")
+
+        platform_daemon do 
+          self.class.running = true
+          DRb.start_service(@cfg.uri, self)
+          DRb.thread.join
+        end
+      rescue Exception => e
+        @logger.error(e.to_s)
+        raise
+      end
+
+      #################################################################################
+      # handles all incoming method calls, and sends them on to the LocalIndex
+      # instance of the correct model class.
+      #
+      # Calls are not queued atm, so this will block until the call returned.
+      #
+      def method_missing(name, *args)
+        @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
+        retried = false
+        with_class args.shift do |clazz|
+          reconnect_when_needed(clazz) do
+            # using respond_to? here so we not have to catch NoMethodError
+            # which would silently catch those from deep inside the indexing
+            # code, too...
+            if clazz.aaf_index.respond_to?(name)
+              clazz.aaf_index.send name, *args
+            elsif clazz.respond_to?(name)
+              @logger.debug "no luck, trying to call class method instead"
+              clazz.send name, *args
+            else
+              raise NoMethodError.new("method #{name} not supported by DRb server")
+            end
+          end
+        end
+      rescue => e
+        @logger.error "ferret server error #{$!}\n#{$!.backtrace.join "\n"}"
+        raise e
+      end
+
+      # make sure we have a versioned index in place, building one if necessary
+      def ensure_index_exists(class_name)
+        @logger.debug "DRb server: ensure_index_exists for class #{class_name}"
+        with_class class_name do |clazz|
+          dir = clazz.aaf_configuration[:index_dir]
+          unless File.directory?(dir) && File.file?(File.join(dir, 'segments')) && dir =~ %r{/\d+(_\d+)?$}
+            rebuild_index(clazz)
+          end
+        end
+      end
+
+      # disconnects the db connection for the class specified by class_name
+      # used only in unit tests to check the automatic reconnection feature
+      def db_disconnect!(class_name)
+        with_class class_name do |clazz|
+          clazz.connection.disconnect!
+        end
+      end
+
+      # hides LocalIndex#rebuild_index to implement index versioning
+      def rebuild_index(clazz, *models)
+        with_class clazz do |clazz|
+          models = models.flatten.uniq.map(&:constantize)
+          models << clazz unless models.include?(clazz)
+          index = new_index_for(clazz, models)
+          reconnect_when_needed(clazz) do
+            @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
+            index.index_models models
+          end
+          new_version = File.join clazz.aaf_configuration[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
+          # create a unique directory name (needed for unit tests where 
+          # multiple rebuilds per second may occur)
+          if File.exists?(new_version)
+            i = 0
+            i+=1 while File.exists?("#{new_version}_#{i}")
+            new_version << "_#{i}"
+          end
+          
+          File.rename index.options[:path], new_version
+          clazz.index_dir = new_version 
+        end
+      end
+
+
+      protected
+
+        def with_class(clazz, *args)
+          clazz = clazz.constantize if String === clazz
+          yield clazz, *args
+        end
+
+        def reconnect_when_needed(clazz)
+          retried = false
+          begin
+            yield
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message =~ /MySQL server has gone away/
+              if retried
+                raise e
+              else
+                @logger.info "StatementInvalid caught, trying to reconnect..."
+                clazz.connection.reconnect!
+                retried = true
+                retry
+              end
+            else
+              @logger.error "StatementInvalid caught, but unsure what to do with it: #{e}"
+              raise e
+            end
+          end
+        end
+
+        def new_index_for(clazz, models)
+          aaf_configuration = clazz.aaf_configuration
+          ferret_cfg = aaf_configuration[:ferret].dup
+          ferret_cfg.update :auto_flush  => false, 
+                            :create      => true,
+                            :field_infos => ActsAsFerret::field_infos(models),
+                            :path        => File.join(aaf_configuration[:index_base_dir], 'rebuild')
+          Ferret::Index::Index.new(ferret_cfg).tap do |i|
+            i.batch_size = aaf_configuration[:reindex_batch_size]
+            i.logger = @logger
+          end
+        end
+
+    end
+  end
+end

data/lib/index.rb

@@ -0,0 +1,31 @@
+module ActsAsFerret
+
+  # base class for local and remote indexes
+  class AbstractIndex
+
+    attr_reader :aaf_configuration
+    attr_accessor :logger
+    def initialize(aaf_configuration)
+      @aaf_configuration = aaf_configuration
+      @logger = Logger.new("#{RAILS_ROOT}/log/ferret_index.log")
+    end
+    
+    class << self
+      def proxy_method(name, *args)
+        define_method name do |*args|
+          @server.send name, model_class_name, *args
+        end
+      end
+
+      def index_proxy_method(*names)
+        names.each do |name|
+          define_method name do |*args|
+            @server.send :"index_#{name}", model_class_name, *args
+          end
+        end
+      end
+
+    end
+  end
+
+end

data/lib/instance_methods.rb

@@ -0,0 +1,156 @@
+module ActsAsFerret #:nodoc:
+
+  module InstanceMethods
+    include ResultAttributes
+
+    # Returns an array of strings with the matches highlighted. The +query+ can
+    # either be a String or a Ferret::Search::Query object.
+    # 
+    # === Options
+    #
+    # field::            field to take the content from. This field has 
+    #                    to have it's content stored in the index 
+    #                    (:store => :yes in your call to aaf). If not
+    #                    given, all stored fields are searched, and the
+    #                    highlighted content found in all of them is returned.
+    #                    set :highlight => :no in the field options to
+    #                    avoid highlighting of contents from a :stored field.
+    # excerpt_length::   Default: 150. Length of excerpt to show. Highlighted
+    #                    terms will be in the centre of the excerpt.
+    # num_excerpts::     Default: 2. Number of excerpts to return.
+    # pre_tag::          Default: "<em>". Tag to place to the left of the
+    #                    match.  
+    # post_tag::         Default: "</em>". This tag should close the
+    #                    +:pre_tag+.
+    # ellipsis::         Default: "...". This is the string that is appended
+    #                    at the beginning and end of excerpts (unless the
+    #                    excerpt hits the start or end of the field. You'll
+    #                    probably want to change this so a Unicode elipsis
+    #                    character.
+    def highlight(query, options = {})
+      self.class.aaf_index.highlight(id, self.class.name, query, options)
+    end
+    
+    # re-eneable ferret indexing for this instance after a call to #disable_ferret
+    def enable_ferret  
+      @ferret_disabled = nil 
+    end
+    alias ferret_enable enable_ferret  # compatibility
+    
+    # returns true if ferret indexing is enabled for this record.
+    #
+    # The optional is_bulk_index parameter will be true if the method is called
+    # by rebuild_index or bulk_index, and false otherwise.
+    #
+    # If is_bulk_index is true, the class level ferret_enabled state will be
+    # ignored by this method (per-instance ferret_enabled checks however will 
+    # take place, so if you override this method to forbid indexing of certain 
+    # records you're still safe).
+    def ferret_enabled?(is_bulk_index = false)
+      @ferret_disabled.nil? && (is_bulk_index || self.class.ferret_enabled?)
+    end
+
+    # Disable Ferret for this record for a specified amount of time. ::once will 
+    # disable Ferret for the next call to #save (this is the default), ::always 
+    # will do so for all subsequent calls. 
+    #
+    # Note that this will turn off only the create and update hooks, but not the 
+    # destroy hook. I think that's reasonable, if you think the opposite, please 
+    # tell me.
+    #
+    # To manually trigger reindexing of a record after you're finished modifying 
+    # it, you can call #ferret_update directly instead of #save (remember to
+    # enable ferret again before).
+    #
+    # When given a block, this will be executed without any ferret indexing of 
+    # this object taking place. The optional argument in this case can be used 
+    # to indicate if the object should be indexed after executing the block
+    # (::index_when_finished). Automatic Ferret indexing of this object will be 
+    # turned on after the block has been executed. If passed ::index_when_true, 
+    # the index will only be updated if the block evaluated not to false or nil.
+    #
+    def disable_ferret(option = :once)
+      if block_given?
+        @ferret_disabled = :always
+        result = yield
+        ferret_enable
+        ferret_update if option == :index_when_finished || (option == :index_when_true && result)
+        result
+      elsif [:once, :always].include?(option)
+        @ferret_disabled = option
+      else
+        raise ArgumentError.new("Invalid Argument #{option}")
+      end
+    end
+
+    # add to index
+    def ferret_create
+      if ferret_enabled?
+        logger.debug "ferret_create/update: #{self.class.name} : #{self.id}"
+        self.class.aaf_index << self
+      else
+        ferret_enable if @ferret_disabled == :once
+      end
+      true # signal success to AR
+    end
+    alias :ferret_update :ferret_create
+    
+
+    # remove from index
+    def ferret_destroy
+      logger.debug "ferret_destroy: #{self.class.name} : #{self.id}"
+      begin
+        self.class.aaf_index.remove self.id, self.class.name
+      rescue
+        logger.warn("Could not find indexed value for this object: #{$!}\n#{$!.backtrace}")
+      end
+      true # signal success to AR
+    end
+    
+    # turn this instance into a ferret document (which basically is a hash of
+    # fieldname => value pairs)
+    def to_doc
+      logger.debug "creating doc for class: #{self.class.name}, id: #{self.id}"
+      Ferret::Document.new.tap do |doc|
+        # store the id of each item
+        doc[:id] = self.id
+
+        # store the class name if configured to do so
+        doc[:class_name] = self.class.name if aaf_configuration[:store_class_name]
+      
+        # iterate through the fields and add them to the document
+        aaf_configuration[:ferret_fields].each_pair do |field, config|
+          doc[field] = self.send("#{field}_to_ferret") unless config[:ignore]
+        end
+        if aaf_configuration[:boost]
+          if self.respond_to?(aaf_configuration[:boost])
+            boost = self.send aaf_configuration[:boost]
+            doc.boost = boost.to_i if boost
+          else
+            logger.error "boost option should point to an instance method: #{aaf_configuration[:boost]}"
+          end
+        end
+      end
+    end
+
+    def document_number
+      self.class.aaf_index.document_number(id, self.class.name)
+    end
+
+    def query_for_record
+      self.class.aaf_index.query_for_record(id, self.class.name)
+    end
+
+    def content_for_field_name(field, dynamic_boost = nil)
+      field_data = self[field] || self.instance_variable_get("@#{field.to_s}".to_sym) || self.send(field.to_sym)
+      if (dynamic_boost && boost_value = self.send(dynamic_boost))
+        field_data = Ferret::Field.new(field_data)
+        field_data.boost = boost_value.to_i
+      end
+      field_data
+    end
+
+
+  end
+
+end