watson-acts_as_ferret 0.4.8.2

data/lib/ferret_find_methods.rb

@@ -0,0 +1,142 @@
+module ActsAsFerret
+  # Ferret search logic common to single-class indexes, shared indexes and
+  # multi indexes.
+  module FerretFindMethods
+
+    def find_records(q, options = {}, ar_options = {})
+      late_pagination = options.delete :late_pagination
+      total_hits, result = if options[:lazy]
+        logger.warn "find_options #{ar_options} are ignored because :lazy => true" unless ar_options.empty?
+        lazy_find q, options
+      else
+        ar_find q, options, ar_options
+      end
+      if late_pagination
+        limit = late_pagination[:limit]
+        offset = late_pagination[:offset] || 0
+        end_index = limit == :all ? -1 : limit+offset-1
+        # puts "late pagination: #{offset} : #{end_index}"
+        result = result[offset..end_index]
+      end
+      return [total_hits, result]
+    end
+
+    def lazy_find(q, options = {})
+      logger.debug "lazy_find: #{q}"
+      result = []
+      rank   = 0
+      total_hits = find_ids(q, options) do |model, id, score, data|
+        logger.debug "model: #{model}, id: #{id}, data: #{data}"
+        result << FerretResult.new(model, id, score, rank += 1, data)
+      end
+      [ total_hits, result ]
+    end
+
+    def ar_find(q, options = {}, ar_options = {})
+      ferret_options = options.dup
+      if ar_options[:conditions] or ar_options[:order]
+        ferret_options[:limit] = :all
+        ferret_options.delete :offset
+      end
+      total_hits, id_arrays = find_id_model_arrays q, ferret_options
+      logger.debug "now retrieving records from AR with options: #{ar_options.inspect}"
+      result = ActsAsFerret::retrieve_records(id_arrays, ar_options)
+      logger.debug "#{result.size} results from AR: #{result.inspect}"
+      
+      # count total_hits via sql when using conditions, multiple models, or when we're called
+      # from an ActiveRecord association.
+      if id_arrays.size > 1 or ar_options[:conditions]
+        # chances are the ferret result count is not our total_hits value, so
+        # we correct this here.
+        if options[:limit] != :all || options[:page] || options[:offset] || ar_options[:limit] || ar_options[:offset]
+          # our ferret result has been limited, so we need to re-run that
+          # search to get the full result set from ferret.
+          new_th, id_arrays = find_id_model_arrays( q, options.merge(:limit => :all, :offset => 0) )
+          # Now ask the database for the total size of the final result set.
+          total_hits = count_records( id_arrays, ar_options )
+        else
+          # what we got from the database is our full result set, so take
+          # it's size
+          total_hits = result.length
+        end
+      end
+      [ total_hits, result ]
+    end
+
+    def count_records(id_arrays, ar_options = {})
+      count_options = ar_options.dup
+      count_options.delete :limit
+      count_options.delete :offset
+      count_options.delete :order
+      count_options.delete :select
+      count = 0
+      id_arrays.each do |model, id_array|
+        next if id_array.empty?
+        model = model.constantize
+        # merge conditions
+        conditions = ActsAsFerret::conditions_for_model model, ar_options[:conditions]
+        count_options[:conditions] = ActsAsFerret::combine_conditions([ "#{model.table_name}.#{model.primary_key} in (?)", id_array.keys ], conditions)
+        count_options[:include] = ActsAsFerret::filter_include_list_for_model(model, ar_options[:include]) if ar_options[:include]
+        cnt = model.count count_options
+        if cnt.is_a?(ActiveSupport::OrderedHash) # fixes #227
+          count += cnt.size
+        else
+          count += cnt
+        end
+      end
+      count
+    end
+
+    def find_id_model_arrays(q, options)
+      id_arrays = {}
+      rank = 0
+      total_hits = find_ids(q, options) do |model, id, score, data|
+        id_arrays[model] ||= {}
+        id_arrays[model][id] = [ rank += 1, score ]
+      end
+      [total_hits, id_arrays]
+    end
+
+    # Queries the Ferret index to retrieve model class, id, score and the
+    # values of any fields stored in the index for each hit.
+    # If a block is given, these are yielded and the number of total hits is
+    # returned. Otherwise [total_hits, result_array] is returned.
+    def find_ids(query, options = {})
+
+      result = []
+      stored_fields = determine_stored_fields options
+
+      q = process_query(query, options)
+      q = scope_query_to_models q, options[:models] #if shared?
+      logger.debug "query: #{query}\n-->#{q}"
+      s = searcher
+      total_hits = s.search_each(q, options) do |hit, score|
+        doc = s[hit]
+        model = doc[:class_name]
+        # fetch stored fields if lazy loading
+        data = extract_stored_fields(doc, stored_fields)
+        if block_given?
+          yield model, doc[:id], score, data
+        else
+          result << { :model => model, :id => doc[:id], :score => score, :data => data }
+        end
+      end
+      #logger.debug "id_score_model array: #{result.inspect}"
+      return block_given? ? total_hits : [total_hits, result]
+    end
+
+    def scope_query_to_models(query, models)
+      return query if models.nil? or models == :all
+      models = [ models ] if Class === models
+      q = Ferret::Search::BooleanQuery.new
+      q.add_query(query, :must)
+      model_query = Ferret::Search::BooleanQuery.new
+      models.each do |model|
+        model_query.add_query(Ferret::Search::TermQuery.new(:class_name, model.name), :should)
+      end
+      q.add_query(model_query, :must)
+      return q
+    end
+
+  end
+end

data/lib/ferret_result.rb

@@ -0,0 +1,58 @@
+module ActsAsFerret
+
+  # mixed into the FerretResult and AR classes calling acts_as_ferret
+  module ResultAttributes
+    # holds the score this record had when it was found via
+    # acts_as_ferret
+    attr_accessor :ferret_score
+
+    attr_accessor :ferret_rank
+  end
+
+  class FerretResult < ActsAsFerret::BlankSlate
+    include ResultAttributes
+    attr_accessor :id
+    reveal :methods
+
+    def initialize(model, id, score, rank, data = {})
+      @model = model.constantize
+      @id = id
+      @ferret_score = score
+      @ferret_rank  = rank
+      @data = data
+      @use_record = false
+    end
+
+    def inspect
+      "#<FerretResult wrapper for #{@model} with id #{@id}"
+    end
+
+    def method_missing(method, *args, &block)
+      if [:highlight, :document_number, :query_for_record].include?(method.to_sym)
+        @model.send method, id, *args
+      elsif (@ar_record && @use_record) || !@data.has_key?(method)
+        to_record.send method, *args, &block
+      else
+        @data[method]
+      end
+    end
+
+    # An implementation of http://rm.jkraemer.net/issues/show/161
+    def [](attr) method_missing(attr) end
+
+    def respond_to?(name)
+      methods.include?(name.to_s) || @data.has_key?(name.to_sym) || to_record.respond_to?(name)
+    end
+
+    def to_record
+      unless @ar_record
+        @ar_record = @model.find(id)
+        @ar_record.ferret_rank  = ferret_rank
+        @ar_record.ferret_score = ferret_score
+        # don't try to fetch attributes from RDig based records
+        @use_record = !@ar_record.class.included_modules.include?(ActsAsFerret::RdigAdapter)
+      end
+      @ar_record
+    end
+  end
+end

data/lib/ferret_server.rb

@@ -0,0 +1,238 @@
+require 'drb'
+require 'thread'
+require 'yaml'
+require 'erb'
+
+################################################################################
+module ActsAsFerret
+  module Remote
+
+    ################################################################################
+    class Config
+
+      ################################################################################
+      DEFAULTS = {
+        'host'      => 'localhost',
+        'port'      => '9009',
+        'cf'        => "#{RAILS_ROOT}/config/ferret_server.yml",
+        'pid_file'  => "#{RAILS_ROOT}/log/ferret_server.pid",
+        'log_file'  => "#{RAILS_ROOT}/log/ferret_server.log",
+        'log_level' => 'debug',
+        'socket'    => nil,
+        'script'    => nil
+      }
+
+      ################################################################################
+      # load the configuration file and apply default settings
+      def initialize (file=DEFAULTS['cf'])
+        @everything = YAML.load(ERB.new(IO.read(file)).result)
+        raise "malformed ferret server config" unless @everything.is_a?(Hash)
+        @config = DEFAULTS.merge(@everything[RAILS_ENV] || {})
+        if @everything[RAILS_ENV]
+          @config['uri'] = socket.nil? ? "druby://#{host}:#{port}" : "drbunix:#{socket}"
+        end
+      end
+
+      ################################################################################
+      # treat the keys of the config data as methods
+      def method_missing (name, *args)
+        @config.has_key?(name.to_s) ? @config[name.to_s] : super
+      end
+
+    end
+
+    #################################################################################
+    # This class acts as a drb server listening for indexing and
+    # search requests from models declared to 'acts_as_ferret :remote => true'
+    #
+    # Usage: 
+    # - modify RAILS_ROOT/config/ferret_server.yml to suit your needs. 
+    # - environments for which no section in the config file exists will use 
+    #   the index locally (good for unit tests/development mode)
+    # - run script/ferret_server to start the server:
+    # script/ferret_server -e production start
+    # - to stop the server run
+    # script/ferret_server -e production stop
+    #
+    class Server
+
+      #################################################################################
+      # FIXME include detection of OS and include the correct file
+      require 'unix_daemon'
+      include(ActsAsFerret::Remote::UnixDaemon)
+
+
+      ################################################################################
+      cattr_accessor :running
+
+      ################################################################################
+      def initialize
+        ActiveRecord::Base.allow_concurrency = true
+        require 'ar_mysql_auto_reconnect_patch'
+        @cfg = ActsAsFerret::Remote::Config.new
+        ActiveRecord::Base.logger = @logger = Logger.new(@cfg.log_file)
+        ActiveRecord::Base.logger.level = Logger.const_get(@cfg.log_level.upcase) rescue Logger::DEBUG
+        if @cfg.script
+          path = File.join(RAILS_ROOT, @cfg.script) 
+          load path
+          @logger.info "loaded custom startup script from #{path}"
+        end
+      end
+
+      ################################################################################
+      # start the server as a daemon process
+      def start
+        raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
+        platform_daemon { run_drb_service }
+      end
+
+      ################################################################################
+      # run the server and block until it exits
+      def run
+        raise "ferret_server not configured for #{RAILS_ENV}" unless (@cfg.uri rescue nil)
+        run_drb_service
+      end
+
+      def run_drb_service
+        $stdout.puts("starting ferret server...")
+        self.class.running = true
+        DRb.start_service(@cfg.uri, self)
+        DRb.thread.join
+      rescue Exception => e
+        @logger.error(e.to_s)
+        raise
+      end
+
+      #################################################################################
+      # handles all incoming method calls, and sends them on to the correct local index
+      # instance.
+      #
+      # Calls are not queued, so this will block until the call returned.
+      #
+      def method_missing(name, *args)
+        @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
+
+
+        index_name = args.shift
+        index = if name.to_s =~ /^multi_(.+)/
+          name = $1
+          ActsAsFerret::multi_index(index_name)
+        else
+          ActsAsFerret::get_index(index_name)
+        end
+
+        if index.nil?
+          @logger.error "\#index with name #{index_name} not found in call to #{name} with args #{args.inspect}"
+          raise ActsAsFerret::IndexNotDefined.new(index_name)
+        end
+
+
+        # TODO find another way to implement the reconnection logic (maybe in
+        # local_index or class_methods)
+        #  reconnect_when_needed(clazz) do
+        
+        # using respond_to? here so we not have to catch NoMethodError
+        # which would silently catch those from deep inside the indexing
+        # code, too...
+
+        if index.respond_to?(name)
+          index.send name, *args
+        # TODO check where we need this:
+        #elsif clazz.respond_to?(name)
+        #      @logger.debug "no luck, trying to call class method instead"
+        #      clazz.send name, *args
+        else
+          raise NoMethodError.new("method #{name} not supported by DRb server")
+        end
+      rescue => e
+        @logger.error "ferret server error #{$!}\n#{$!.backtrace.join "\n"}"
+        raise e
+      end
+
+      def register_class(class_name)
+        @logger.debug "############ registerclass #{class_name}"
+        class_name.constantize
+        @logger.debug "index for class #{class_name}: #{ActsAsFerret::ferret_indexes[class_name.underscore.to_sym]}"
+
+      end
+
+      # make sure we have a versioned index in place, building one if necessary
+      def ensure_index_exists(index_name)
+        @logger.debug "DRb server: ensure_index_exists for index #{index_name}"
+        definition = ActsAsFerret::get_index(index_name).index_definition
+        dir = definition[:index_dir]
+        unless File.directory?(dir) && File.file?(File.join(dir, 'segments')) && dir =~ %r{/\d+(_\d+)?$}
+          rebuild_index(index_name)
+        end
+      end
+
+      # disconnects the db connection for the class specified by class_name
+      # used only in unit tests to check the automatic reconnection feature
+      def db_disconnect!(class_name)
+        with_class class_name do |clazz|
+          clazz.connection.disconnect!
+        end
+      end
+
+      # hides LocalIndex#rebuild_index to implement index versioning
+      def rebuild_index(index_name)
+        definition = ActsAsFerret::get_index(index_name).index_definition.dup
+        models = definition[:registered_models]
+        index = new_index_for(definition)
+        # TODO fix reconnection stuff
+        #  reconnect_when_needed(clazz) do
+        #    @logger.debug "DRb server: rebuild index for class(es) #{models.inspect} in #{index.options[:path]}"
+        index.index_models models
+        #  end
+        new_version = File.join definition[:index_base_dir], Time.now.utc.strftime('%Y%m%d%H%M%S')
+        # create a unique directory name (needed for unit tests where 
+        # multiple rebuilds per second may occur)
+        if File.exists?(new_version)
+          i = 0
+          i+=1 while File.exists?("#{new_version}_#{i}")
+          new_version << "_#{i}"
+        end
+          
+        File.rename index.options[:path], new_version
+        ActsAsFerret::change_index_dir index_name, new_version 
+      end
+
+
+      protected
+
+        def reconnect_when_needed(clazz)
+          retried = false
+          begin
+            yield
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message =~ /MySQL server has gone away/
+              if retried
+                raise e
+              else
+                @logger.info "StatementInvalid caught, trying to reconnect..."
+                clazz.connection.reconnect!
+                retried = true
+                retry
+              end
+            else
+              @logger.error "StatementInvalid caught, but unsure what to do with it: #{e}"
+              raise e
+            end
+          end
+        end
+
+        def new_index_for(index_definition)
+          ferret_cfg = index_definition[:ferret].dup
+          ferret_cfg.update :auto_flush  => false, 
+                            :create      => true,
+                            :field_infos => ActsAsFerret::field_infos(index_definition),
+                            :path        => File.join(index_definition[:index_base_dir], 'rebuild')
+          Ferret::Index::Index.new(ferret_cfg).tap do |i|
+            i.batch_size = index_definition[:reindex_batch_size]
+            i.logger = @logger
+          end
+        end
+
+    end
+  end
+end

data/lib/index.rb

@@ -0,0 +1,99 @@
+module ActsAsFerret
+
+  class IndexLogger
+    def initialize(logger, name)
+      @logger = logger
+      @index_name = name
+    end
+    %w(debug info warn error).each do |m|
+      define_method(m) do |message|
+        @logger.send m, "[#{@index_name}] #{message}"
+      end
+      question = :"#{m}?"
+      define_method(question) do
+        @logger.send question
+      end
+    end
+  end
+
+  # base class for local and remote indexes
+  class AbstractIndex
+    include FerretFindMethods
+
+    attr_reader :logger, :index_name, :index_definition, :registered_models_config
+    def initialize(index_definition)
+      @index_definition = index_definition
+      @registered_models_config = {}
+      @index_name = index_definition[:name]
+      @logger = IndexLogger.new(ActsAsFerret::logger, @index_name)
+    end
+
+    # TODO allow for per-class field configuration (i.e. different via, boosts
+    # for the same field among different models)
+    def register_class(clazz, options = {})
+      logger.info "register class #{clazz} with index #{index_name}"
+
+      if force = options.delete(:force_re_registration)
+        index_definition[:registered_models].delete(clazz)
+      end
+
+      if index_definition[:registered_models].map(&:name).include?(clazz.name)
+        logger.info("refusing re-registration of class #{clazz}")
+      else
+        index_definition[:registered_models] << clazz
+        @registered_models_config[clazz] = options
+
+        # merge fields from this acts_as_ferret call with predefined fields
+        already_defined_fields = index_definition[:ferret_fields]
+        field_config = ActsAsFerret::build_field_config options[:fields]
+        field_config.update ActsAsFerret::build_field_config( options[:additional_fields] )
+        field_config.each do |field, config|
+          if already_defined_fields.has_key?(field)
+            logger.info "ignoring redefinition of ferret field #{field}" if shared? 
+          else
+            already_defined_fields[field] = config
+            logger.info "adding new field #{field} from class #{clazz.name} to index #{index_name}"
+          end
+        end
+
+        # update default field list to be used by the query parser, unless it 
+        # was explicitly given by user.
+        #
+        # It will include all content fields *not* marked as :untokenized.
+        # This fixes the otherwise failing CommentTest#test_stopwords. Basically
+        # this means that by default only tokenized fields (which all fields are
+        # by default) will be searched. If you want to search inside the contents 
+        # of an untokenized field, you'll have to explicitly specify it in your 
+        # query.
+        unless index_definition[:user_default_field]
+          # grab all tokenized fields
+          ferret_fields = index_definition[:ferret_fields]
+          index_definition[:ferret][:default_field] = ferret_fields.keys.select do |field|
+            ferret_fields[field][:index] != :untokenized
+          end
+          logger.info "default field list for index #{index_name}: #{index_definition[:ferret][:default_field].inspect}"
+        end
+      end
+
+      return index_definition
+    end
+
+    # true if this index is used by more than one model class
+    def shared?
+      index_definition[:registered_models].size > 1
+    end
+
+    # Switches the index to a new index directory.
+    # Used by the DRb server when switching to a new index version.
+    def change_index_dir(new_dir)
+      logger.debug "[#{index_name}] changing index dir to #{new_dir}"
+      index_definition[:index_dir] = index_definition[:ferret][:path] = new_dir
+      reopen!
+      logger.debug "[#{index_name}] index dir is now #{new_dir}"
+    end
+
+    protected
+
+  end
+
+end