acts_as_ferret 0.4.3 → 0.4.4

data/lib/ferret_extensions.rb

@@ -71,7 +71,80 @@ module Ferret
       end
       @auto_flush = orig_flush
     end
+    
 
+    # bulk-inserts a number of ferret documents.
+    # The argument has to be an array of two-element arrays each holding the document data and the analyzer to 
+    # use for this document (which may be nil).
+    def update_batch(document_analyzer_pairs)
+      ids = document_analyzer_pairs.collect {|da| da.first[@id_field] }
+      @dir.synchrolock do
+        batch_delete(ids)
+        ensure_writer_open()
+        document_analyzer_pairs.each do |doc, analyzer|
+          if analyzer
+            old_analyzer = @writer.analyzer
+            @writer.analyzer = analyzer
+            @writer.add_document(doc)
+            @writer.analyzer = old_analyzer
+          else
+            @writer.add_document(doc)
+          end
+        end
+        flush()
+      end      
+    end
+    
+    # If +docs+ is a Hash or an Array then a batch delete will be performed.
+    # If +docs+ is an Array then it will be considered an array of +id+'s. If
+    # it is a Hash, then its keys will be used instead as the Array of
+    # document +id+'s. If the +id+ is an Integers then it is considered a
+    # Ferret document number and the corresponding document will be deleted.
+    # If the +id+ is a String or a Symbol then the +id+ will be considered a
+    # term and the documents that contain that term in the +:id_field+ will
+    # be deleted.
+    #
+    # docs:: An Array of docs to be deleted, or a Hash (in which case the keys
+    # are used)
+    #
+    # ripped from Ferret trunk.
+    def batch_delete(docs)
+      docs = docs.keys if docs.is_a?(Hash)
+      raise ArgumentError, "must pass Array or Hash" unless docs.is_a? Array
+      ids = []
+      terms = []
+      docs.each do |doc|
+        case doc
+        when String   then terms << doc
+        when Symbol   then terms << doc.to_s
+        when Integer  then ids << doc
+        else
+          raise ArgumentError, "Cannot delete for arg of type #{id.class}"
+        end
+      end
+      if ids.size > 0
+        ensure_reader_open
+        ids.each {|id| @reader.delete(id)}
+      end
+      if terms.size > 0
+        ensure_writer_open()
+        terms.each { |t| @writer.delete(@id_field, t) }
+        # TODO with Ferret trunk this would work:
+        # @writer.delete(@id_field, terms)
+      end
+      return self
+    end
+
+    # search for the first document with +arg+ in the +id+ field and return it's internal document number. 
+    # The +id+ field is either :id or whatever you set
+    # :id_field parameter to when you create the Index object.
+    def doc_number(id)
+      @dir.synchronize do
+        ensure_reader_open()
+        term_doc_enum = @reader.term_docs_for(@id_field, id.to_s)
+        return term_doc_enum.next? ? term_doc_enum.doc : nil
+      end
+    end
   end
 
   # add marshalling support to SortFields
@@ -82,11 +155,11 @@ module Ferret
 
     def self._load(string)
       case string
-        when /<DOC(_ID)?>!/         : Ferret::Search::SortField::DOC_ID_REV
-        when /<DOC(_ID)?>/          : Ferret::Search::SortField::DOC_ID
-        when '<SCORE>!'             : Ferret::Search::SortField::SCORE_REV
-        when '<SCORE>'              : Ferret::Search::SortField::SCORE
-        when /^(\w+):<(\w+)>(!)?$/ : new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
+        when /<DOC(_ID)?>!/         then Ferret::Search::SortField::DOC_ID_REV
+        when /<DOC(_ID)?>/          then Ferret::Search::SortField::DOC_ID
+        when '<SCORE>!'             then Ferret::Search::SortField::SCORE_REV
+        when '<SCORE>'              then Ferret::Search::SortField::SCORE
+        when /^(\w+):<(\w+)>(!)?$/  then new($1.to_sym, :type => $2.to_sym, :reverse => !$3.nil?)
         else raise "invalid value: #{string}"
       end
     end

data/lib/ferret_find_methods.rb

@@ -0,0 +1,141 @@
+module ActsAsFerret
+  # Ferret search logic common to single-class indexes, shared indexes and
+  # multi indexes.
+  module FerretFindMethods
+
+    def find_records(q, options = {}, ar_options = {})
+      late_pagination = options.delete :late_pagination
+      total_hits, result = if options[:lazy]
+        logger.warn "find_options #{ar_options} are ignored because :lazy => true" unless ar_options.empty?
+        lazy_find q, options
+      else
+        ar_find q, options, ar_options
+      end
+      if late_pagination
+        limit = late_pagination[:limit]
+        offset = late_pagination[:offset] || 0
+        end_index = limit == :all ? -1 : limit+offset-1
+        # puts "late pagination: #{offset} : #{end_index}"
+        result = result[offset..end_index]
+      end
+      return [total_hits, result]
+    end
+
+    def lazy_find(q, options = {})
+      logger.debug "lazy_find: #{q}"
+      result = []
+      rank   = 0
+      total_hits = find_ids(q, options) do |model, id, score, data|
+        logger.debug "model: #{model}, id: #{id}, data: #{data}"
+        result << FerretResult.new(model, id, score, rank += 1, data)
+      end
+      [ total_hits, result ]
+    end
+
+    def ar_find(q, options = {}, ar_options = {})
+      ferret_options = options.dup
+      if ar_options[:conditions] or ar_options[:order]
+        ferret_options[:limit] = :all
+        ferret_options.delete :offset
+      end
+      total_hits, id_arrays = find_id_model_arrays q, ferret_options
+      logger.debug "now retrieving records from AR with options: #{ar_options.inspect}"
+      result = ActsAsFerret::retrieve_records(id_arrays, ar_options)
+      logger.debug "#{result.size} results from AR: #{result.inspect}"
+      
+      # count total_hits via sql when using conditions, multiple models, or when we're called
+      # from an ActiveRecord association.
+      if id_arrays.size > 1 or ar_options[:conditions]
+        # chances are the ferret result count is not our total_hits value, so
+        # we correct this here.
+        if options[:limit] != :all || options[:page] || options[:offset] || ar_options[:limit] || ar_options[:offset]
+          # our ferret result has been limited, so we need to re-run that
+          # search to get the full result set from ferret.
+          new_th, id_arrays = find_id_model_arrays( q, options.merge(:limit => :all, :offset => 0) )
+          # Now ask the database for the total size of the final result set.
+          total_hits = count_records( id_arrays, ar_options )
+        else
+          # what we got from the database is our full result set, so take
+          # it's size
+          total_hits = result.length
+        end
+      end
+      [ total_hits, result ]
+    end
+
+    def count_records(id_arrays, ar_options = {})
+      count_options = ar_options.dup
+      count_options.delete :limit
+      count_options.delete :offset
+      count_options.delete :order
+      count = 0
+      id_arrays.each do |model, id_array|
+        next if id_array.empty?
+        model = model.constantize
+        # merge conditions
+        conditions = ActsAsFerret::conditions_for_model model, ar_options[:conditions]
+        count_options[:conditions] = ActsAsFerret::combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], conditions)
+        count_options[:include] = ActsAsFerret::filter_include_list_for_model(model, ar_options[:include]) if ar_options[:include]
+        cnt = model.count count_options
+        if cnt.is_a?(ActiveSupport::OrderedHash) # fixes #227
+          count += cnt.size
+        else
+          count += cnt
+        end
+      end
+      count
+    end
+
+    def find_id_model_arrays(q, options)
+      id_arrays = {}
+      rank = 0
+      total_hits = find_ids(q, options) do |model, id, score, data|
+        id_arrays[model] ||= {}
+        id_arrays[model][id] = [ rank += 1, score ]
+      end
+      [total_hits, id_arrays]
+    end
+
+    # Queries the Ferret index to retrieve model class, id, score and the
+    # values of any fields stored in the index for each hit.
+    # If a block is given, these are yielded and the number of total hits is
+    # returned. Otherwise [total_hits, result_array] is returned.
+    def find_ids(query, options = {})
+
+      result = []
+      stored_fields = determine_stored_fields options
+
+      q = process_query(query, options)
+      q = scope_query_to_models q, options[:models] #if shared?
+      logger.debug "query: #{query}\n-->#{q}"
+      s = searcher
+      total_hits = s.search_each(q, options) do |hit, score|
+        doc = s[hit]
+        model = doc[:class_name]
+        # fetch stored fields if lazy loading
+        data = extract_stored_fields(doc, stored_fields)
+        if block_given?
+          yield model, doc[:id], score, data
+        else
+          result << { :model => model, :id => doc[:id], :score => score, :data => data }
+        end
+      end
+      #logger.debug "id_score_model array: #{result.inspect}"
+      return block_given? ? total_hits : [total_hits, result]
+    end
+
+    def scope_query_to_models(query, models)
+      return query if models.nil? or models == :all
+      models = [ models ] if Class === models
+      q = Ferret::Search::BooleanQuery.new
+      q.add_query(query, :must)
+      model_query = Ferret::Search::BooleanQuery.new
+      models.each do |model|
+        model_query.add_query(Ferret::Search::TermQuery.new(:class_name, model.name), :should)
+      end
+      q.add_query(model_query, :must)
+      return q
+    end
+
+  end
+end

data/lib/ferret_result.rb

@@ -9,28 +9,45 @@ module ActsAsFerret
     attr_accessor :ferret_rank
   end
 
-  class FerretResult
+  class FerretResult < ActsAsFerret::BlankSlate
     include ResultAttributes
     attr_accessor :id
+    reveal :methods
 
-    def initialize(model, id, score, data = {})
+    def initialize(model, id, score, rank, data = {})
       @model = model.constantize
       @id = id
       @ferret_score = score
+      @ferret_rank  = rank
       @data = data
+      @use_record = false
     end
-    
-    def method_missing(method, *args)
-      if @ar_record || @data[method].nil?
-        ferret_load_record unless @ar_record
-        @ar_record.send method, *args
+
+    def inspect
+      "#<FerretResult wrapper for #{@model} with id #{@id}"
+    end
+
+    def method_missing(method, *args, &block)
+      if (@ar_record && @use_record) || !@data.has_key?(method)
+        to_record.send method, *args, &block
       else
         @data[method]
       end
     end
 
-    def ferret_load_record
-      @ar_record = @model.find(id)
+    def respond_to?(name)
+      methods.include?(name.to_s) || @data.has_key?(name.to_sym) || to_record.respond_to?(name)
+    end
+
+    def to_record
+      unless @ar_record
+        @ar_record = @model.find(id)
+        @ar_record.ferret_rank  = ferret_rank
+        @ar_record.ferret_score = ferret_score
+        # don't try to fetch attributes from RDig based records
+        @use_record = !@ar_record.class.included_modules.include?(ActsAsFerret::RdigAdapter)
+      end
+      @ar_record
     end
   end
 end

data/lib/ferret_server.rb

@@ -18,6 +18,8 @@ module ActsAsFerret
         'pid_file'  => "#{RAILS_ROOT}/log/ferret_server.pid",
         'log_file'  => "#{RAILS_ROOT}/log/ferret_server.log",
         'log_level' => 'debug',
+        'socket'    => nil,
+        'script'    => nil
       }
 
       ################################################################################
@@ -26,7 +28,9 @@ module ActsAsFerret
         @everything = YAML.load(ERB.new(IO.read(file)).result)
         raise "malformed ferret server config" unless @everything.is_a?(Hash)
         @config = DEFAULTS.merge(@everything[RAILS_ENV] || {})
-        @config['uri'] = "druby://#{host}:#{port}" if @everything[RAILS_ENV]
+        if @everything[RAILS_ENV]
+          @config['uri'] = socket.nil? ? "druby://#{host}:#{port}" : "drbunix:#{socket}"
+        end
       end
 
       ################################################################################
@@ -57,70 +61,108 @@ module ActsAsFerret
       require 'unix_daemon'
       include(ActsAsFerret::Remote::UnixDaemon)
 
+
       ################################################################################
       cattr_accessor :running
 
       ################################################################################
       def initialize
-        @cfg = ActsAsFerret::Remote::Config.new
         ActiveRecord::Base.allow_concurrency = true
+        require 'ar_mysql_auto_reconnect_patch'
+        @cfg = ActsAsFerret::Remote::Config.new
         ActiveRecord::Base.logger = @logger = Logger.new(@cfg.log_file)
         ActiveRecord::Base.logger.level = Logger.const_get(@cfg.log_level.upcase) rescue Logger::DEBUG
+        if @cfg.script
+          path = File.join(RAILS_ROOT, @cfg.script) 
+          load path
+          @logger.info "loaded custom startup script from #{path}"
+        end
       end
 
       ################################################################################
-      # start the server
+      # start the server as a daemon process
       def start
         raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
-        $stdout.puts("starting ferret server...")
+        platform_daemon { run_drb_service }
+      end
 
-        platform_daemon do 
-          self.class.running = true
-          DRb.start_service(@cfg.uri, self)
-          DRb.thread.join
-        end
+      ################################################################################
+      # run the server and block until it exits
+      def run
+        raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
+        run_drb_service
+      end
+
+      def run_drb_service
+        $stdout.puts("starting ferret server...")
+        self.class.running = true
+        DRb.start_service(@cfg.uri, self)
+        DRb.thread.join
       rescue Exception => e
         @logger.error(e.to_s)
         raise
       end
 
       #################################################################################
-      # handles all incoming method calls, and sends them on to the LocalIndex
-      # instance of the correct model class.
+      # handles all incoming method calls, and sends them on to the correct local index
+      # instance.
       #
-      # Calls are not queued atm, so this will block until the call returned.
+      # Calls are not queued, so this will block until the call returned.
       #
       def method_missing(name, *args)
         @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
-        retried = false
-        with_class args.shift do |clazz|
-          reconnect_when_needed(clazz) do
-            # using respond_to? here so we not have to catch NoMethodError
-            # which would silently catch those from deep inside the indexing
-            # code, too...
-            if clazz.aaf_index.respond_to?(name)
-              clazz.aaf_index.send name, *args
-            elsif clazz.respond_to?(name)
-              @logger.debug "no luck, trying to call class method instead"
-              clazz.send name, *args
-            else
-              raise NoMethodError.new("method #{name} not supported by DRb server")
-            end
-          end
+
+
+        index_name = args.shift
+        index = if name.to_s =~ /^multi_(.+)/
+          name = $1
+          ActsAsFerret::multi_index(index_name)
+        else
+          ActsAsFerret::get_index(index_name)
+        end
+
+        if index.nil?
+          @logger.error "\#index with name #{index_name} not found in call to #{name} with args #{args.inspect}"
+          raise ActsAsFerret::IndexNotDefined.new(index_name)
+        end
+
+
+        # TODO find another way to implement the reconnection logic (maybe in
+        # local_index or class_methods)
+        #  reconnect_when_needed(clazz) do
+        
+        # using respond_to? here so we not have to catch NoMethodError
+        # which would silently catch those from deep inside the indexing
+        # code, too...
+
+        if index.respond_to?(name)
+          index.send name, *args
+        # TODO check where we need this:
+        #elsif clazz.respond_to?(name)
+        #      @logger.debug "no luck, trying to call class method instead"
+        #      clazz.send name, *args
+        else
+          raise NoMethodError.new("method #{name} not supported by DRb server")
         end
       rescue => e
         @logger.error "ferret server error #{$!}\n#{$!.backtrace.join "\n"}"
         raise e
       end
 
+      def register_class(class_name)
+        @logger.debug "############ registerclass #{class_name}"
+        class_name.constantize
+        @logger.debug "index for class #{class_name}: #{ActsAsFerret::ferret_indexes[class_name.underscore.to_sym]}"
+
+      end
+
       # make sure we have a versioned index in place, building one if necessary
-      def ensure_index_exists(class_name)
-        @logger.debug "DRb server: ensure_index_exists for class #{class_name}"
-        with_class class_name do |clazz|
-          dir = clazz.aaf_configuration[:index_dir]
-          unless File.directory?(dir) && File.file?(File.join(dir, 'segments')) && dir =~ %r{/\d+(_\d+)?$}
-            rebuild_index(clazz)
-          end
+      def ensure_index_exists(index_name)
+        @logger.debug "DRb server: ensure_index_exists for index #{index_name}"
+        definition = ActsAsFerret::get_index(index_name).index_definition
+        dir = definition[:index_dir]
+        unless File.directory?(dir) && File.file?(File.join(dir, 'segments')) && dir =~ %r{/\d+(_\d+)?$}
+          rebuild_index(index_name)
         end
       end
 
@@ -133,37 +175,31 @@ module ActsAsFerret
       end
 
       # hides LocalIndex#rebuild_index to implement index versioning
-      def rebuild_index(clazz, *models)
-        with_class clazz do |clazz|
-          models = models.flatten.uniq.map(&:constantize)
-          models << clazz unless models.include?(clazz)
-          index = new_index_for(clazz, models)
-          reconnect_when_needed(clazz) do
-            @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
-            index.index_models models
-          end
-          new_version = File.join clazz.aaf_configuration[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
-          # create a unique directory name (needed for unit tests where 
-          # multiple rebuilds per second may occur)
-          if File.exists?(new_version)
-            i = 0
-            i+=1 while File.exists?("#{new_version}_#{i}")
-            new_version << "_#{i}"
-          end
-          
-          File.rename index.options[:path], new_version
-          clazz.index_dir = new_version 
+      def rebuild_index(index_name)
+        definition = ActsAsFerret::get_index(index_name).index_definition.dup
+        models = definition[:registered_models]
+        index = new_index_for(definition)
+        # TODO fix reconnection stuff
+        #  reconnect_when_needed(clazz) do
+        #    @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
+        index.index_models models
+        #  end
+        new_version = File.join definition[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
+        # create a unique directory name (needed for unit tests where 
+        # multiple rebuilds per second may occur)
+        if File.exists?(new_version)
+          i = 0
+          i+=1 while File.exists?("#{new_version}_#{i}")
+          new_version << "_#{i}"
         end
+          
+        File.rename index.options[:path], new_version
+        ActsAsFerret::change_index_dir index_name, new_version 
       end
 
 
       protected
 
-        def with_class(clazz, *args)
-          clazz = clazz.constantize if String === clazz
-          yield clazz, *args
-        end
-
         def reconnect_when_needed(clazz)
           retried = false
           begin
@@ -185,15 +221,14 @@ module ActsAsFerret
           end
         end
 
-        def new_index_for(clazz, models)
-          aaf_configuration = clazz.aaf_configuration
-          ferret_cfg = aaf_configuration[:ferret].dup
+        def new_index_for(index_definition)
+          ferret_cfg = index_definition[:ferret].dup
           ferret_cfg.update :auto_flush  => false, 
                             :create      => true,
-                            :field_infos => ActsAsFerret::field_infos(models),
-                            :path        => File.join(aaf_configuration[:index_base_dir], 'rebuild')
+                            :field_infos => ActsAsFerret::field_infos(index_definition),
+                            :path        => File.join(index_definition[:index_base_dir], 'rebuild')
           returning Ferret::Index::Index.new(ferret_cfg) do |i|
-            i.batch_size = aaf_configuration[:reindex_batch_size]
+            i.batch_size = index_definition[:reindex_batch_size]
             i.logger = @logger
           end
         end