acts_as_ferret 0.4.1 → 0.4.2

data/lib/ferret_extensions.rb

@@ -1,9 +1,39 @@
 module Ferret
 
+  module Analysis
+  
+    # = PerFieldAnalyzer
+    #
+    # This PerFieldAnalyzer is a workaround to a memory leak in 
+    # ferret 0.11.4. It does basically do the same as the original
+    # Ferret::Analysis::PerFieldAnalyzer, but without the leak :)
+    # 
+    # http://ferret.davebalmain.com/api/classes/Ferret/Analysis/PerFieldAnalyzer.html
+    #
+    # Thanks to Ben from omdb.org for tracking this down and creating this
+    # workaround.
+    # You can read more about the issue there:
+    # http://blog.omdb-beta.org/2007/7/29/tracking-down-a-memory-leak-in-ferret-0-11-4
+    class PerFieldAnalyzer < ::Ferret::Analysis::Analyzer
+      def initialize( default_analyzer = StandardAnalyzer.new )
+        @analyzers = {}
+        @default_analyzer = default_analyzer
+      end
+            
+      def add_field( field, analyzer )
+        @analyzers[field] = analyzer
+      end
+      alias []= add_field
+                
+      def token_stream(field, string)
+        @analyzers.has_key?(field) ? @analyzers[field].token_stream(field, string) : 
+          @default_analyzer.token_stream(field, string)
+      end
+    end
+  end
 
   class Index::Index
-    attr_accessor :batch_size
-    attr_accessor :logger
+    attr_accessor :batch_size, :logger
 
     def index_models(models)
       models.each { |model| index_model model }
@@ -14,32 +44,36 @@ module Ferret
     end
 
     def index_model(model)
-      @batch_size ||= 0
-      work_done = 0
-      batch_time = 0
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :reindex => true)
       logger.info "reindexing model #{model.name}"
 
-      model_count  = model.count.to_f
       model.records_for_rebuild(@batch_size) do |records, offset|
-        #records = [ records ] unless records.is_a?(Array)
-        batch_time = measure_time {
-          records.each { |rec| self << rec.to_doc if rec.ferret_enabled?(true) }
-        }.to_f
-        work_done = offset.to_f / model_count * 100.0 if model_count > 0
-        remaining_time = ( batch_time / @batch_size ) * ( model_count - offset + @batch_size )
-        logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+        bulk_indexer.index_records(records, offset)
       end
     end
 
-    def measure_time
-      t1 = Time.now
-      yield
-      Time.now - t1
+    def bulk_index(model, ids, options = {})
+      options.reverse_merge! :optimize => true
+      orig_flush = @auto_flush
+      @auto_flush = false
+      bulk_indexer = ActsAsFerret::BulkIndexer.new(:batch_size => @batch_size, :logger => logger, 
+                                                   :model => model, :index => self, :total => ids.size)
+      model.records_for_bulk_index(ids, @batch_size) do |records, offset|
+        logger.debug "#{model} bulk indexing #{records.size} at #{offset}"
+        bulk_indexer.index_records(records, offset)
+      end
+      logger.info 'finishing bulk index...'
+      flush
+      if options[:optimize]
+        logger.info 'optimizing...'
+        optimize 
+      end
+      @auto_flush = orig_flush
     end
 
   end
 
-
   # add marshalling support to SortFields
   class Search::SortField
     def _dump(depth)

data/lib/ferret_server.rb

@@ -3,29 +3,41 @@ require 'thread'
 require 'yaml'
 require 'erb'
 
-
+################################################################################
 module ActsAsFerret
-
   module Remote
 
-    module Config
-      class << self
-        DEFAULTS = {
-          'host' => 'localhost',
-          'port' => '9009'
-        }
-        # read connection settings from config file
-        def load(file = "#{RAILS_ROOT}/config/ferret_server.yml")
-          config = DEFAULTS.merge(YAML.load(ERB.new(IO.read(file)).result))
-          if config = config[RAILS_ENV]
-            config[:uri] = "druby://#{config['host']}:#{config['port']}"
-            return config
-          end
-          {}
-        end
+    ################################################################################
+    class Config
+
+      ################################################################################
+      DEFAULTS = {
+        'host'      => 'localhost',
+        'port'      => '9009',
+        'cf'        => "#{RAILS_ROOT}/config/ferret_server.yml",
+        'pid_file'  => "#{RAILS_ROOT}/log/ferret_server.pid",
+        'log_file'  => "#{RAILS_ROOT}/log/ferret_server.log",
+        'log_level' => 'debug',
+      }
+
+      ################################################################################
+      # load the configuration file and apply default settings
+      def initialize (file=DEFAULTS['cf'])
+        @everything = YAML.load(ERB.new(IO.read(file)).result)
+        raise "malformed ferret server config" unless @everything.is_a?(Hash)
+        @config = DEFAULTS.merge(@everything[RAILS_ENV] || {})
+        @config['uri'] = "druby://#{host}:#{port}" if @everything[RAILS_ENV]
+      end
+
+      ################################################################################
+      # treat the keys of the config data as methods
+      def method_missing (name, *args)
+        @config.has_key?(name.to_s) ? @config[name.to_s] : super
       end
+
     end
 
+    #################################################################################
     # This class acts as a drb server listening for indexing and
     # search requests from models declared to 'acts_as_ferret :remote => true'
     #
@@ -33,25 +45,46 @@ module ActsAsFerret
     # - modify RAILS_ROOT/config/ferret_server.yml to suit your needs. 
     # - environments for which no section in the config file exists will use 
     #   the index locally (good for unit tests/development mode)
-    # - run script/ferret_start to start the server:
-    # RAILS_ENV=production script/ferret_start
+    # - run script/ferret_server to start the server:
+    # script/ferret_server -e production start
+    # - to stop the server run
+    # script/ferret_server -e production stop
     #
     class Server
 
+      #################################################################################
+      # FIXME include detection of OS and include the correct file
+      require 'unix_daemon'
+      include(ActsAsFerret::Remote::UnixDaemon)
+
+      ################################################################################
       cattr_accessor :running
 
-      def self.start(uri = nil)
+      ################################################################################
+      def initialize
+        @cfg = ActsAsFerret::Remote::Config.new
         ActiveRecord::Base.allow_concurrency = true
-        ActiveRecord::Base.logger = Logger.new("#{RAILS_ROOT}/log/ferret_server.log")
-        uri ||= ActsAsFerret::Remote::Config.load[:uri]
-        DRb.start_service(uri, ActsAsFerret::Remote::Server.new)
-        self.running = true
+        ActiveRecord::Base.logger = @logger = Logger.new(@cfg.log_file)
+        ActiveRecord::Base.logger.level = Logger.const_get(@cfg.log_level.upcase) rescue Logger::DEBUG
       end
 
-      def initialize
-        @logger = ActiveRecord::Base.logger
+      ################################################################################
+      # start the server
+      def start
+        raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
+        $stdout.puts("starting ferret server...")
+
+        platform_daemon do 
+          self.class.running = true
+          DRb.start_service(@cfg.uri, self)
+          DRb.thread.join
+        end
+      rescue Exception => e
+        @logger.error(e.to_s)
+        raise
       end
 
+      #################################################################################
       # handles all incoming method calls, and sends them on to the LocalIndex
       # instance of the correct model class.
       #
@@ -59,17 +92,25 @@ module ActsAsFerret
       #
       def method_missing(name, *args)
         @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
+        retried = false
         with_class args.shift do |clazz|
-          begin
-            clazz.aaf_index.send name, *args
-          rescue NoMethodError
-            @logger.debug "no luck, trying to call class method instead"
-            clazz.send name, *args
+          reconnect_when_needed(clazz) do
+            # using respond_to? here so we not have to catch NoMethodError
+            # which would silently catch those from deep inside the indexing
+            # code, too...
+            if clazz.aaf_index.respond_to?(name)
+              clazz.aaf_index.send name, *args
+            elsif clazz.respond_to?(name)
+              @logger.debug "no luck, trying to call class method instead"
+              clazz.send name, *args
+            else
+              raise NoMethodError.new("method #{name} not supported by DRb server")
+            end
           end
         end
-      rescue
-        @logger.error "ferret server error #{$!}\n#{$!.backtrace.join '\n'}"
-        raise
+      rescue => e
+        @logger.error "ferret server error #{$!}\n#{$!.backtrace.join "\n"}"
+        raise e
       end
 
       # make sure we have a versioned index in place, building one if necessary
@@ -83,14 +124,24 @@ module ActsAsFerret
         end
       end
 
+      # disconnects the db connection for the class specified by class_name
+      # used only in unit tests to check the automatic reconnection feature
+      def db_disconnect!(class_name)
+        with_class class_name do |clazz|
+          clazz.connection.disconnect!
+        end
+      end
+
       # hides LocalIndex#rebuild_index to implement index versioning
       def rebuild_index(clazz, *models)
         with_class clazz do |clazz|
           models = models.flatten.uniq.map(&:constantize)
           models << clazz unless models.include?(clazz)
           index = new_index_for(clazz, models)
-          @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
-          index.index_models models
+          reconnect_when_needed(clazz) do
+            @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
+            index.index_models models
+          end
           new_version = File.join clazz.aaf_configuration[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
           # create a unique directory name (needed for unit tests where 
           # multiple rebuilds per second may occur)
@@ -113,6 +164,27 @@ module ActsAsFerret
           yield clazz, *args
         end
 
+        def reconnect_when_needed(clazz)
+          retried = false
+          begin
+            yield
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message =~ /MySQL server has gone away/
+              if retried
+                raise e
+              else
+                @logger.info "StatementInvalid caught, trying to reconnect..."
+                clazz.connection.reconnect!
+                retried = true
+                retry
+              end
+            else
+              @logger.error "StatementInvalid caught, but unsure what to do with it: #{e}"
+              raise e
+            end
+          end
+        end
+
         def new_index_for(clazz, models)
           aaf_configuration = clazz.aaf_configuration
           ferret_cfg = aaf_configuration[:ferret].dup
@@ -120,7 +192,7 @@ module ActsAsFerret
                             :create      => true,
                             :field_infos => ActsAsFerret::field_infos(models),
                             :path        => File.join(aaf_configuration[:index_base_dir], 'rebuild')
-          returning Ferret::Index::Index.new ferret_cfg do |i|
+          returning Ferret::Index::Index.new(ferret_cfg) do |i|
             i.batch_size = aaf_configuration[:reindex_batch_size]
             i.logger = @logger
           end

data/lib/instance_methods.rb

@@ -31,20 +31,36 @@ module ActsAsFerret #:nodoc:
       self.class.aaf_index.highlight(id, self.class.name, query, options)
     end
     
-    # re-eneable ferret indexing after a call to #disable_ferret
-    def ferret_enable; @ferret_disabled = nil end
+    # re-eneable ferret indexing for this instance after a call to #disable_ferret
+    def enable_ferret  
+      @ferret_disabled = nil 
+    end
+    alias ferret_enable enable_ferret  # compatibility
     
-    # returns true if ferret indexing is enabled
-    # the optional parameter will be true if the method is called by rebuild_index, 
-    # and false otherwise. I.e. useful to enable a model only for indexing during 
-    # scheduled reindex runs.
-    def ferret_enabled?(is_rebuild = false); @ferret_disabled.nil? end
+    # returns true if ferret indexing is enabled for this record.
+    #
+    # The optional is_bulk_index parameter will be true if the method is called
+    # by rebuild_index or bulk_index, and false otherwise.
+    #
+    # If is_bulk_index is true, the class level ferret_enabled state will be
+    # ignored by this method (per-instance ferret_enabled checks however will 
+    # take place, so if you override this method to forbid indexing of certain 
+    # records you're still safe).
+    def ferret_enabled?(is_bulk_index = false)
+      @ferret_disabled.nil? && (is_bulk_index || self.class.ferret_enabled?)
+    end
 
-    # Disable Ferret for a specified amount of time. ::once will disable
-    # Ferret for the next call to #save (this is the default), ::always will 
-    # do so for all subsequent calls.
-    # To manually trigger reindexing of a record, you can call #ferret_update 
-    # directly. 
+    # Disable Ferret for this record for a specified amount of time. ::once will 
+    # disable Ferret for the next call to #save (this is the default), ::always 
+    # will do so for all subsequent calls. 
+    #
+    # Note that this will turn off only the create and update hooks, but not the 
+    # destroy hook. I think that's reasonable, if you think the opposite, please 
+    # tell me.
+    #
+    # To manually trigger reindexing of a record after you're finished modifying 
+    # it, you can call #ferret_update directly instead of #save (remember to
+    # enable ferret again before).
     #
     # When given a block, this will be executed without any ferret indexing of 
     # this object taking place. The optional argument in this case can be used 
@@ -52,6 +68,7 @@ module ActsAsFerret #:nodoc:
     # (::index_when_finished). Automatic Ferret indexing of this object will be 
     # turned on after the block has been executed. If passed ::index_when_true, 
     # the index will only be updated if the block evaluated not to false or nil.
+    #
     def disable_ferret(option = :once)
       if block_given?
         @ferret_disabled = :always
@@ -94,7 +111,7 @@ module ActsAsFerret #:nodoc:
     # fieldname => value pairs)
     def to_doc
       logger.debug "creating doc for class: #{self.class.name}, id: #{self.id}"
-      returning doc = Ferret::Document.new do
+      returning Ferret::Document.new do |doc|
         # store the id of each item
         doc[:id] = self.id
 
@@ -105,6 +122,14 @@ module ActsAsFerret #:nodoc:
         aaf_configuration[:ferret_fields].each_pair do |field, config|
           doc[field] = self.send("#{field}_to_ferret") unless config[:ignore]
         end
+        if aaf_configuration[:boost]
+          if self.respond_to?(aaf_configuration[:boost])
+            boost = self.send aaf_configuration[:boost]
+            doc.boost = boost.to_i if boost
+          else
+            logger.error "boost option should point to an instance method: #{aaf_configuration[:boost]}"
+          end
+        end
       end
     end
 
@@ -116,8 +141,13 @@ module ActsAsFerret #:nodoc:
       self.class.aaf_index.query_for_record(id, self.class.name)
     end
 
-    def content_for_field_name(field)
-      self[field] || self.instance_variable_get("@#{field.to_s}".to_sym) || self.send(field.to_sym)
+    def content_for_field_name(field, dynamic_boost = nil)
+      field_data = self[field] || self.instance_variable_get("@#{field.to_s}".to_sym) || self.send(field.to_sym)
+      if (dynamic_boost && boost_value = self.send(dynamic_boost))
+        field_data = Ferret::Field.new(field_data)
+        field_data.boost = boost_value.to_i
+      end
+      field_data
     end
 
 

data/lib/local_index.rb

@@ -1,9 +1,7 @@
 module ActsAsFerret
-  
   class LocalIndex < AbstractIndex
     include MoreLikeThis::IndexMethods
 
-
     def initialize(aaf_configuration)
       super
       ensure_index_exists
@@ -62,6 +60,10 @@ module ActsAsFerret
       index.index_models models
     end
 
+    def bulk_index(ids, options)
+      ferret_index.bulk_index(aaf_configuration[:class_name].constantize, ids, options)
+    end
+
     # Parses the given query string into a Ferret Query object.
     def process_query(query)
       # work around ferret bug in #process_query (doesn't ensure the
@@ -74,9 +76,9 @@ module ActsAsFerret
 
     # Total number of hits for the given query. 
     # To count the results of a multi_search query, specify an array of 
-    # class names with the :models option.
+    # class names with the :multi option.
     def total_hits(query, options = {})
-      index = (models = options.delete(:models)) ? multi_index(models) : ferret_index
+      index = (models = options.delete(:multi)) ? multi_index(models) : ferret_index
       index.search(query, options).total_hits
     end
 
@@ -96,7 +98,7 @@ module ActsAsFerret
     def find_id_by_contents(query, options = {})
       result = []
       index = ferret_index
-      logger.debug "query: #{ferret_index.process_query query}" # TODO only enable this for debugging purposes
+      logger.debug "query: #{ferret_index.process_query query}" if logger.debug?
       lazy_fields = determine_lazy_fields options
 
       total_hits = index.search_each(query, options) do |hit, score|

data/lib/search_results.rb

@@ -0,0 +1,53 @@
+module ActsAsFerret
+
+  # decorator that adds a total_hits accessor and will_paginate compatible 
+  # paging support to search result arrays
+  class SearchResults
+    attr_reader :current_page, :per_page, :total_hits
+
+    def initialize(results, total_hits, current_page = 1, per_page = nil)
+      @results = results
+      @total_hits = total_hits
+      @current_page = current_page
+      @per_page = (per_page || total_hits)
+      @total_pages   = @per_page > 0 ? (@total_hits / @per_page.to_f).ceil : 0
+    end
+
+    def method_missing(symbol, *args, &block)
+      @results.send(symbol, *args, &block)
+    end
+
+    def respond_to?(name)
+      self.methods.include?(name) || @results.respond_to?(name)
+    end
+
+
+    # code from here on was directly taken from will_paginate's collection.rb
+
+    #
+    # The total number of pages.
+    def page_count
+      @total_pages
+    end
+
+    # Current offset of the paginated collection. If we're on the first page,
+    # it is always 0. If we're on the 2nd page and there are 30 entries per page,
+    # the offset is 30. This property is useful if you want to render ordinals
+    # besides your records: simply start with offset + 1.
+    #
+    def offset
+      (current_page - 1) * per_page
+    end
+
+    # current_page - 1 or nil if there is no previous page
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+
+    # current_page + 1 or nil if there is no next page
+    def next_page
+      current_page < page_count ? (current_page + 1) : nil
+    end
+  end
+
+end

data/lib/server_manager.rb

@@ -0,0 +1,44 @@
+################################################################################
+require 'optparse'
+
+################################################################################
+$ferret_server_options = {
+  'environment' => nil,
+  'debug'       => nil,
+}
+
+################################################################################
+OptionParser.new do |optparser|
+  optparser.banner = "Usage: #{File.basename($0)} [options] {start|stop}"
+
+  optparser.on('-h', '--help', "This message") do
+    puts optparser
+    exit
+  end
+
+  optparser.on('-e', '--environment=NAME', 'Set RAILS_ENV to the given string') do |e|
+    $ferret_server_options['environment'] = e
+  end
+
+  optparser.on('--debug', 'Include full stack traces on exceptions') do
+    $ferret_server_options['debug'] = true
+  end
+
+  $ferret_server_action = optparser.permute!(ARGV)
+  (puts optparser; exit(1)) unless $ferret_server_action.size == 1
+
+  $ferret_server_action = $ferret_server_action.first
+  (puts optparser; exit(1)) unless %w(start stop).include?($ferret_server_action)
+end
+
+################################################################################
+begin
+  ENV['FERRET_USE_LOCAL_INDEX'] = 'true'
+  ENV['RAILS_ENV'] = $ferret_server_options['environment']
+  require(File.join(File.dirname(__FILE__), '../../../../config/environment'))
+  ActsAsFerret::Remote::Server.new.send($ferret_server_action)
+rescue Exception => e
+  $stderr.puts(e.message)
+  $stderr.puts(e.backtrace.join("\n")) if $ferret_server_options['debug']
+  exit(1)
+end

data/lib/shared_index_class_methods.rb

@@ -14,7 +14,7 @@ module ActsAsFerret
         
         if original_query.is_a? String
           model_query = options[:models].map(&:name).join '|'
-          q << %{ +class_name:"#{model_query}"}
+          q += %{ +class_name:"#{model_query}"}
         else
           q = Ferret::Search::BooleanQuery.new
           q.add_query(original_query, :must)

data/lib/unix_daemon.rb

@@ -0,0 +1,63 @@
+################################################################################
+module ActsAsFerret
+  module Remote
+
+    ################################################################################
+    # methods for becoming a daemon on Unix-like operating systems
+    module UnixDaemon
+
+      ################################################################################
+      def platform_daemon (&block)
+        safefork do
+          write_pid_file
+          trap("TERM") { exit(0) }
+          sess_id = Process.setsid
+          STDIN.reopen("/dev/null")
+          STDOUT.reopen("#{RAILS_ROOT}/log/ferret_server.out", "a")
+          STDERR.reopen(STDOUT)
+          block.call
+        end
+      end
+
+      ################################################################################
+      # stop the daemon, nicely at first, and then forcefully if necessary
+      def stop
+        pid = read_pid_file
+        raise "ferret_server doesn't appear to be running" unless pid
+        $stdout.puts("stopping ferret server...")
+        Process.kill("TERM", pid)
+        30.times { Process.kill(0, pid); sleep(0.5) }
+        $stdout.puts("using kill -9 #{pid}")
+        Process.kill(9, pid)
+      rescue Errno::ESRCH => e
+        $stdout.puts("process #{pid} has stopped")
+      ensure
+        File.unlink(@cfg.pid_file) if File.exist?(@cfg.pid_file)
+      end
+
+      ################################################################################
+      def safefork (&block)
+        @fork_tries ||= 0
+        fork(&block)
+      rescue Errno::EWOULDBLOCK
+        raise if @fork_tries >= 20
+        @fork_tries += 1
+        sleep 5
+        retry
+      end
+
+      #################################################################################
+      # create the PID file and install an at_exit handler
+      def write_pid_file
+        open(@cfg.pid_file, "w") {|f| f << Process.pid << "\n"}
+        at_exit { File.unlink(@cfg.pid_file) if read_pid_file == Process.pid }
+      end
+
+      #################################################################################
+      def read_pid_file
+        File.read(@cfg.pid_file).to_i if File.exist?(@cfg.pid_file)
+      end
+
+    end
+  end
+end

data/rakefile

@@ -61,6 +61,9 @@ if PKG_VERSION
     #s.files           = FileList["{lib,test}/**/*"].to_a + %w(README MIT-LICENSE CHANGELOG)
   # s.files.delete  ...    
     s.require_path    = 'lib'
+    s.bindir          = "bin"
+    s.executables     = ["aaf_install"]
+    s.default_executable = "aaf_install"
     s.autorequire     = 'acts_as_ferret'
     s.has_rdoc        = true
   # s.test_files      = Dir['test/**/*_test.rb']
@@ -92,8 +95,8 @@ if PKG_VERSION
         fail "SVK status is not clean ... do you have unchecked-in files?"
       end
       announce "No outstanding checkins found ... OK"
-      announce "Pushing to svn..."
-      `svk push .`
+#      announce "Pushing to svn..."
+#      `svk push .`
     end
   end