watson-acts_as_ferret 0.4.8.2

data/lib/multi_index.rb

@@ -0,0 +1,133 @@
+module ActsAsFerret #:nodoc:
+
+  # Base class for remote and local multi-indexes
+  class MultiIndexBase
+    include FerretFindMethods
+    attr_accessor :logger
+
+    def initialize(indexes, options = {})
+      # ensure all models indexes exist
+      @indexes = indexes
+      indexes.each { |i| i.ensure_index_exists }
+      default_fields = indexes.inject([]) do |fields, idx| 
+        fields + [ idx.index_definition[:ferret][:default_field] ]
+      end.flatten.uniq
+      # Patch to pass an or_default setting on to ferret.
+      # Without an or_default set, ferret will use OR queries by default.
+      # This implementation will use OR if any model asks for OR, otherwise 
+      # AND. -- Paul Fitzpatrick, 11/1/2010 (contributed to public domain)
+      or_default = 
+        indexes.select{|idx| idx.index_definition[:ferret][:or_default]}.size>0
+      @options = {
+        :default_field => default_fields,
+        :or_default => or_default
+      }.update(options)
+      @logger = IndexLogger.new(ActsAsFerret::logger, "multi: #{indexes.map(&:index_name).join(',')}")
+    end
+
+    def ar_find(query, options = {}, ar_options = {})
+      limit = options.delete(:limit)
+      offset = options.delete(:offset) || 0
+      options[:limit] = :all
+      total_hits, result = super query, options, ar_options  
+      total_hits = result.size if ar_options[:conditions]
+      # if limit && limit != :all
+      #   result = result[offset..limit+offset-1]
+      # end
+      [total_hits, result]
+    end
+    
+    def determine_stored_fields(options)
+      return nil unless options.has_key?(:lazy)
+      stored_fields = []
+      @indexes.each do |index|
+        stored_fields += index.determine_stored_fields(options)
+      end
+      return stored_fields.uniq
+    end
+
+    def shared?
+      false
+    end
+      
+  end
+  
+  # This class can be used to search multiple physical indexes at once.
+  class MultiIndex < MultiIndexBase
+    
+    def extract_stored_fields(doc, stored_fields)
+      ActsAsFerret::get_index_for(doc[:class_name]).extract_stored_fields(doc, stored_fields) unless stored_fields.blank?
+    end
+
+    def total_hits(q, options = {})
+      search(q, options).total_hits
+    end
+    
+    def search(query, options={})
+      query = process_query(query, options)
+      logger.debug "parsed query: #{query.to_s}"
+      searcher.search(query, options)
+    end
+
+    def search_each(query, options = {}, &block)
+      query = process_query(query, options)
+      searcher.search_each(query, options, &block)
+    end
+
+    # checks if all our sub-searchers still are up to date
+    def latest?
+      #return false unless @reader
+      # segfaults with 0.10.4 --> TODO report as bug @reader.latest?
+      @reader and @reader.latest?
+      #@sub_readers.each do |r| 
+      #  return false unless r.latest? 
+      #end
+      #true
+    end
+
+    def searcher
+      ensure_searcher
+      @searcher
+    end
+    
+    def doc(i)
+      searcher[i]
+    end
+    alias :[] :doc
+    
+    def query_parser
+      @query_parser ||= Ferret::QueryParser.new(@options)
+    end
+    
+    def process_query(query, options = {})
+      query = query_parser.parse(query) if query.is_a?(String)
+      return query
+    end
+
+    def close
+      @searcher.close if @searcher
+      @reader.close if @reader
+    end
+
+    protected
+
+      def ensure_searcher
+        unless latest?
+          @sub_readers = @indexes.map { |idx| 
+            begin
+              reader = Ferret::Index::IndexReader.new(idx.index_definition[:index_dir])
+              logger.debug "sub-reader opened: #{reader}"
+              reader
+            rescue Exception
+              raise "error opening reader on index for class #{clazz.inspect}: #{$!}"
+            end
+          }
+          close
+          @reader = Ferret::Index::IndexReader.new(@sub_readers)
+          @searcher = Ferret::Search::Searcher.new(@reader)
+        end
+      end
+
+  end # of class MultiIndex
+
+end

data/lib/rdig_adapter.rb

@@ -0,0 +1,149 @@
+begin
+  require 'rdig'
+rescue LoadError
+  puts "RDig gem not found, searching and indexing static pages won't work."
+end
+require 'digest/md5'
+
+module ActsAsFerret
+
+  # The RdigAdapter is automatically included into your model if you specify
+  # the +:rdig+ options hash in your call to acts_as_ferret. It overrides
+  # several methods declared by aaf to retrieve documents with the help of
+  # RDig's http crawler when you call rebuild_index.
+  module RdigAdapter
+
+    if defined?(RDig)
+
+      def self.included(target)
+        target.extend ClassMethods
+        target.send :include, InstanceMethods
+        target.alias_method_chain :ferret_key, :md5
+      end
+
+      # Indexer class to replace RDig's original indexer
+      class Indexer
+        include MonitorMixin
+        def initialize(batch_size, model_class, &block)
+          @batch_size = batch_size
+          @model_class = model_class
+          @documents = []
+          @offset = 0
+          @block = block
+          super()
+        end
+
+        def add(doc)
+          synchronize do
+            @documents << @model_class.new(doc.uri.to_s, doc)
+            process_batch if @documents.size >= @batch_size
+          end
+        end
+        alias << add
+
+        def close
+          synchronize do
+            process_batch
+          end
+        end
+
+        protected
+        def process_batch
+          ActsAsFerret::logger.info "RdigAdapter::Indexer#process_batch: #{@documents.size} docs in queue, offset #{@offset}"
+          @block.call @documents, @offset
+          @offset += @documents.size
+          @documents = []
+        end
+      end
+      
+      module ClassMethods
+        # overriding aaf to return the documents fetched via RDig
+        def records_for_rebuild(batch_size = 1000, &block)
+          indexer = Indexer.new(batch_size, self, &block)
+          configure_rdig do
+            crawler = RDig::Crawler.new RDig.configuration, ActsAsFerret::logger
+            crawler.instance_variable_set '@indexer', indexer
+            ActsAsFerret::logger.debug "now crawling..."
+            crawler.crawl
+          end
+        rescue => e
+          ActsAsFerret::logger.error e
+          ActsAsFerret::logger.debug e.backtrace.join("\n")
+        ensure
+          indexer.close if indexer
+        end
+
+        # overriding aaf to skip reindexing records changed during the rebuild
+        # when rebuilding with the rake task
+        def records_modified_since(time)
+          []
+        end
+
+        # unfortunately need to modify global RDig.configuration because it's
+        # used everywhere in RDig
+        def configure_rdig
+          # back up original config
+          old_logger = RDig.logger
+          old_cfg = RDig.configuration.dup
+          RDig.logger = ActsAsFerret.logger
+          rdig_configuration[:crawler].each { |k,v| RDig.configuration.crawler.send :"#{k}=", v } if rdig_configuration[:crawler]
+          if ce_config = rdig_configuration[:content_extraction]
+            RDig.configuration.content_extraction = OpenStruct.new( :hpricot => OpenStruct.new( ce_config ) )
+          end
+          yield
+        ensure
+          # restore original config
+          RDig.configuration.crawler = old_cfg.crawler
+          RDig.configuration.content_extraction = old_cfg.content_extraction
+          RDig.logger = old_logger
+        end
+
+        # overriding aaf to enforce loading page title and content from the
+        # ferret index
+        def find_with_ferret(q, options = {}, find_options = {})
+          options[:lazy] = true
+          super
+        end
+
+        def find_for_id(id)
+          new id
+        end
+      end
+
+      module InstanceMethods
+        def initialize(uri, rdig_document = nil)
+          @id = uri
+          @rdig_document = rdig_document
+        end
+
+        # Title of the document.
+        # Use the +:title_tag_selector+ option to declare the hpricot expression
+        # that should be used for selecting the content for this field.
+        def title
+          @rdig_document.title
+        end
+
+        # Content of the document.
+        # Use the +:content_tag_selector+ option to declare the hpricot expression
+        # that should be used for selecting the content for this field.
+        def content
+          @rdig_document.body
+        end
+
+        # Url of this document.
+        def id
+          @id
+        end
+
+        def ferret_key_with_md5
+          Digest::MD5.hexdigest(ferret_key_without_md5)
+        end
+        
+        def to_s
+          "Page at #{id}, title: #{title}"
+        end
+      end
+    end
+  end
+  
+end

data/lib/remote_functions.rb

@@ -0,0 +1,43 @@
+module ActsAsFerret
+  module RemoteFunctions
+
+    private
+
+    def yield_results(total_hits, results)
+      results.each do |result|
+        yield result[:model], result[:id], result[:score], result[:data]
+      end
+      total_hits
+    end
+
+
+    def handle_drb_error(return_value_in_case_of_error = false)
+      yield
+    rescue DRb::DRbConnError => e
+      logger.error "DRb connection error: #{e}"
+      logger.warn e.backtrace.join("\n")
+      raise e if ActsAsFerret::raise_drb_errors?
+      return_value_in_case_of_error
+    end
+
+    alias :old_handle_drb_error :handle_drb_error 
+    def handle_drb_error(return_value_in_case_of_error = false)
+      handle_drb_restart do
+        old_handle_drb_error(return_value_in_case_of_error) { yield }
+      end
+    end
+
+    def handle_drb_restart
+      trys = 1
+      begin
+        return yield
+      rescue ActsAsFerret::IndexNotDefined
+        logger.warn "Recovering from ActsAsFerret::IndexNotDefined exception"
+        ActsAsFerret::ferret_indexes[index_name] = ActsAsFerret::create_index_instance( index_definition )
+        ActsAsFerret::ferret_indexes[index_name].register_class ActsAsFerret::index_using_classes.index(index_name).constantize, {}
+        retry if (trys -= 1) > 0
+      end
+      yield
+    end
+  end
+end

data/lib/remote_index.rb

@@ -0,0 +1,54 @@
+require 'drb'
+module ActsAsFerret
+
+  # This index implementation connects to a remote ferret server instance. It
+  # basically forwards all calls to the remote server.
+  class RemoteIndex < AbstractIndex
+    include RemoteFunctions
+
+    def initialize(config)
+      super
+      @server = DRbObject.new(nil, ActsAsFerret::remote)
+    end
+
+    # Cause model classes to be loaded (and indexes get declared) on the DRb
+    # side of things.
+    def register_class(clazz, options)
+      handle_drb_error { @server.register_class clazz.name }
+    end
+
+    def method_missing(method_name, *args)
+      args.unshift index_name
+      handle_drb_error { @server.send(method_name, *args) }
+    end
+
+    # Proxy any methods that require special return values in case of errors
+    { 
+      :highlight => [] 
+    }.each do |method_name, default_result|
+      define_method method_name do |*args|
+        args.unshift index_name
+        handle_drb_error(default_result) { @server.send method_name, *args }
+      end
+    end
+
+    def find_ids(q, options = {}, &proc)
+      total_hits, results = handle_drb_error([0, []]) { @server.find_ids(index_name, q, options) }
+      block_given? ? yield_results(total_hits, results, &proc) : [ total_hits, results ]
+    end
+
+    # add record to index
+    def add(record)
+      handle_drb_error { @server.add index_name, record.to_doc }
+    end
+    alias << add
+
+    private
+
+    #def model_class_name
+    #  index_definition[:class_name]
+    #end
+
+  end
+
+end

data/lib/remote_multi_index.rb

@@ -0,0 +1,20 @@
+module ActsAsFerret
+  class RemoteMultiIndex < MultiIndexBase
+    include RemoteFunctions
+
+    def initialize(indexes, options = {})
+      @index_names = indexes.map(&:index_name)
+      @server = DRbObject.new(nil, ActsAsFerret::remote)
+      super
+    end
+
+    def find_ids(query, options, &proc)
+      total_hits, results = handle_drb_error([0, []]) { @server.multi_find_ids(@index_names, query, options) }
+      block_given? ? yield_results(total_hits, results, &proc) : [ total_hits, results ]
+    end
+
+    def method_missing(name, *args)
+      handle_drb_error { @server.send(:"multi_#{name}", @index_names, *args) }
+    end
+  end
+end

data/lib/search_results.rb

@@ -0,0 +1,50 @@
+module ActsAsFerret
+
+  # decorator that adds a total_hits accessor and will_paginate compatible 
+  # paging support to search result arrays
+  class SearchResults < ActsAsFerret::BlankSlate
+    reveal :methods
+    attr_reader :current_page, :per_page, :total_hits, :total_pages
+    alias total_entries total_hits  # will_paginate compatibility
+    alias page_count total_pages    # will_paginate backwards compatibility
+
+    def initialize(results, total_hits, current_page = 1, per_page = nil)
+      @results = results
+      @total_hits = total_hits
+      @current_page = current_page
+      @per_page = (per_page || total_hits)
+      @total_pages   = @per_page > 0 ? (@total_hits / @per_page.to_f).ceil : 0
+    end
+
+    def method_missing(symbol, *args, &block)
+      @results.send(symbol, *args, &block)
+    end
+
+    def respond_to?(name)
+      methods.include?(name.to_s) || @results.respond_to?(name)
+    end
+
+
+    # code from here on was directly taken from will_paginate's collection.rb
+
+    # Current offset of the paginated collection. If we're on the first page,
+    # it is always 0. If we're on the 2nd page and there are 30 entries per page,
+    # the offset is 30. This property is useful if you want to render ordinals
+    # besides your records: simply start with offset + 1.
+    #
+    def offset
+      (current_page - 1) * per_page
+    end
+
+    # current_page - 1 or nil if there is no previous page
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+
+    # current_page + 1 or nil if there is no next page
+    def next_page
+      current_page < total_pages ? (current_page + 1) : nil
+    end
+  end
+
+end