ferret 0.9.6 → 0.10.0

data/lib/ferret/search/multi_searcher.rb

@@ -1,261 +0,0 @@
-module Ferret::Search
-
-  # Implements searching multiple IndexSearchers at once
-  # 
-  # Applications usually need only call the @link #search(Query)
-  # or @link #search(Query,Filter) methods. For performance reasons it is 
-  # recommended to open only one Searcher and use it for all of your searches.
-  class MultiSearcher
-    include Ferret::Index
-
-    attr_accessor :similarity, :searchers
-
-    # Creates a MultiSearcher searching across all the searchers
-    # in the provided array.
-    #
-    def initialize(args)
-      @searchers = Array.new(args)
-      @similarity = Similarity.default
-      
-      # initialize reader lookup array
-      @max_doc = 0
-      @starts = Array.new(@searchers.size + 1)
-      @searchers.each_with_index { |searcher, i| 
-        @starts[i] = @max_doc
-        @max_doc += searcher.max_doc
-      }
-      @starts[@searchers.size] = @max_doc
-    end
-    
-    # closes all underlying Searchers
-    def close()
-      @searchers.each { |searcher| searcher.close() }
-    end
-
-    # Expert: Returns the number of documents containing +term+.
-    # Called by search code to compute term weights.
-    # See IndexReader#doc_freq
-    def doc_freq(term)
-      return @searchers.inject(0) { |df, searcher| 
-        df + searcher.doc_freq(term) 
-      }
-    end
-
-    # Expert: For each term in the terms array, calculates the number of
-    # documents containing +term+. Returns an array with these
-    # document frequencies. Used to minimize number of remote calls.
-    def doc_freqs(terms)
-      result = Array.new
-      terms.each {|term, i| result << doc_freq(term)}
-      return result
-    end
-
-    # Expert: Returns the stored fields of document +n+.
-    #
-    # See IndexReader#get_document
-    def doc(n)
-      i = sub_searcher(n)
-      return @searchers[i].doc(n - @starts[i])
-    end
-
-    # Returns index of the searcher for document <code>n</code> in the 
-    # array used to construct this searcher. 
-    def sub_searcher(n)
-      lo = 0			            # search starts array
-      hi = @searchers.size - 1  # for first element less
-						                  # than n, return its index
-      while hi >= lo do
-        mid = (lo + hi) >> 1
-        midValue = @starts[mid]
-        if n < midValue
-          hi = mid - 1;
-        elsif n > midValue
-          lo = mid + 1;
-        else                   # found a match
-          while mid+1 < @searchers.size && @starts[mid+1] == midValue do
-            mid += 1                # scan to last match
-          end
-          return mid
-        end
-      end
-      return hi
-    end
-
-    # Returns the document number of document <code>n</code> within its
-    # sub-index.
-    def sub_doc(n)
-      return n - @starts[sub_searcher(n)]
-    end
-    
-    # Expert: Returns one greater than the largest possible document number.
-    # Called by search code to compute term weights.
-    # See IndexReader#max_doc
-    def max_doc
-      return @max_doc
-    end
-
-    # Create weight in multiple index scenario.
-    # 
-    # Distributed query processing is done in the following steps:
-    # 1. rewrite query
-    # 2. extract necessary terms
-    # 3. collect dfs for these terms from the Searchables
-    # 4. create query weight using aggregate dfs.
-    # 5. distribute that weight to Searchables
-    # 6. merge results
-    #
-    # Steps 1-4 are done here, 5+6 in the search() methods
-    def create_weight(query)
-      # step 1
-      rewritten_query = self.rewrite(query)
-
-      # step 2
-      terms = Set.new
-      rewritten_query.extract_terms(terms)
-
-      # step 3
-      aggregated_dfs = Array.new(terms.size, 0)
-      @searchers.each { |searcher|
-        dfs = searcher.doc_freqs(terms)
-        dfs.each_with_index { |df,i|
-          aggregated_dfs[i] += df
-        }
-      }
-
-      df_map = Hash.new
-      terms.each_with_index { |term,i|
-        df_map[term] = aggregated_dfs[i]
-      }
-
-      # step 4
-      cache_sim = CachedDfSource.new(df_map, self.max_doc, self.similarity)
-      
-      return rewritten_query.weight(cache_sim)
-    end
-
-   
-    def search(query, options = {})
-      filter = options[:filter]
-      first_doc = options[:first_doc]||0
-      num_docs = options[:num_docs]||10
-      max_size = first_doc + num_docs
-      sort = options[:sort]
-
-      if (num_docs <= 0)
-        raise ArgumentError, "num_docs must be > 0 to run a search"
-      end
-
-      if (first_doc < 0)
-        raise ArgumentError, "first_doc must be >= 0 to run a search"
-      end
-
-
-      if (sort)
-        raise NotImplementedError
-        #fields = sort.is_a?(Array) ? sort : sort.fields
-        #hq = FieldDocSortedHitQueue.new(fields, max_size)
-      else
-        hq = HitQueue.new(max_size)
-      end
-
-      total_hits = 0
-      weight = create_weight(query)
-      @searchers.each_with_index { |searcher,i|     # search each searcher
-        docs = searcher.search(weight, 
-                               :filter => filter,
-                               #:sort => sort,
-                               :num_docs => max_size,
-                               :first_doc => 0)
-        total_hits += docs.total_hits  # update total_hits
-        docs.score_docs.each { |score_doc|
-          score_doc.doc += @starts[i]   # convert doc
-          break unless hq.insert(score_doc) # no more scores > min_score
-        }
-      }
-      
-      score_docs = []
-      if (hq.size > first_doc)
-        if (hq.size - first_doc) < num_docs 
-          num_docs = hq.size - first_doc
-        end
-        num_docs.times do
-          score_docs.unshift(hq.pop)
-        end
-      end
-      hq.clear
-
-      return TopDocs.new(total_hits, score_docs)
-    end
-
-    def search_each(query, filter = nil, &block)
-      weight = create_weight(query)
-      @searchers.each { |searcher|     # search each searcher
-        searcher.search_each(weight, filter, &block)
-      }
-    end
-
-    # rewrites the query into a query that can be processed by the search
-    # methods. For example, a Fuzzy query is turned into a massive boolean
-    # query.
-    #
-    # original:: The original query to be rewritten.
-    def rewrite(original)
-      #print "multi_searcher#rewrite: #{original}\n"
-      queries = []
-      @searchers.each { |searcher|
-        queries << searcher.rewrite(original)
-      }
-      return queries.first.combine(queries)
-    end
-
-    # Returns an Explanation that describes how +doc+ scored against
-    # +query+.
-    # 
-    # This is intended to be used in developing Similarity implementations,
-    # and, for good performance, should not be displayed with every hit.
-    # Computing an explanation is as expensive as executing the query over the
-    # entire index.
-    def explain(query, doc)
-      i = sub_searcher(doc)
-      return @searchers[i].explain(create_weight(query), doc-@starts[i])
-    end
-
-  end
-
-
-  # Document Frequency cache acting as a Dummy-Searcher.
-  # This class is no full-fledged Searcher, but only supports
-  # the methods necessary to initialize Weights.
-  class CachedDfSource
-
-    attr_reader :max_doc, :similarity
-    
-    def initialize(df_map, max_doc, similarity)
-      @df_map = df_map
-      @max_doc = max_doc
-      @similarity = similarity
-    end
-      
-    def doc_freq(term)
-      return @df_map[term]
-    end
-
-    def doc_freqs(terms)
-      result = Array.new
-      terms.each { |term|
-        result << doc_freq(term)
-      }
-      return result
-    end
-
-    def rewrite(query)
-      # this is a bit of a hack. We know that a query which
-      # creates a Weight based on this Dummy-Searcher is
-      # always already rewritten (see preparedWeight()).
-      # Therefore we just return the unmodified query here
-      return query
-    end
-
-  end
-
-end

data/lib/ferret/search/multi_term_query.rb

@@ -1,65 +0,0 @@
-module Ferret::Search
-  # A Query that matches documents containing a subset of terms provided
-  # by a FilteredTermEnum enumeration.
-  #
-  # +MultiTermQuery+ is not designed to be used by itself. The reason being
-  # that it is not intialized with a FilteredTermEnum enumeration. A
-  # FilteredTermEnum enumeration needs to be provided.
-  #
-  # For example, WildcardQuery and FuzzyQuery extend +MultiTermQuery+ to
-  # provide WildcardTermEnum and FuzzyTermEnum, respectively.
-  class MultiTermQuery < Query 
-    attr_reader :term
-
-    # Constructs a query for terms matching +term+. 
-    def initialize(term) 
-      super()
-      @term = term
-    end
-
-    # Construct the enumeration to be used, expanding the pattern term. 
-    def get_term_enum(reader)
-      raise NotImplementedError
-    end
-     
-
-    def rewrite(reader)
-      enumerator = get_term_enum(reader)
-      bq = BooleanQuery.new(true)
-      begin 
-        begin 
-          t = enumerator.term()
-          if (t != nil)
-            tq = TermQuery.new(t)      # found a match
-            tq.boost = boost() * enumerator.difference()   # set the boost
-            bq.add_query(tq, BooleanClause::Occur::SHOULD) # add to query
-          end
-        end while enumerator.next?
-      ensure 
-        enumerator.close()
-      end
-      return bq
-    end
-
-    # Prints a user-readable version of this query. 
-    def to_s(field = nil) 
-      buffer = ""
-      buffer << "#{@term.field}:" if @term.field != field
-      buffer << @term.text
-      buffer << "^#{boost()}" if (boost() != 1.0) 
-      return buffer
-    end
-
-    def eql?(o) 
-      if not o.instance_of? MultiTermQuery
-        return false
-      end
-      return term == o.term
-    end
-    alias :== :eql?
-
-    def hash() 
-      return term.hash()
-    end
-  end
-end

data/lib/ferret/search/non_matching_scorer.rb

@@ -1,22 +0,0 @@
-module Ferret::Search
-  # A scorer that matches no document at all. 
-  class NonMatchingScorer < Scorer 
-    def initialize()
-      super(nil) # no similarity used
-    end
-    
-   def next?
-     return false
-   end
-
-   def skip_to(target)
-     return false
-   end
-
-   def explain(doc) 
-      e = Explanation.new()
-      e.description = "No document matches."
-      return e
-    end
-  end
-end

data/lib/ferret/search/phrase_positions.rb

@@ -1,55 +0,0 @@
-module Ferret::Search
-  class PhrasePositions 
-    attr_reader :doc, :position
-    attr_accessor :next
-
-    def initialize(tp_enum, offset) 
-      @tp_enum = tp_enum
-      @offset = offset
-      @count = @position = @doc = -1
-      @next = nil
-    end
-
-    def next?()
-      if not @tp_enum.next?
-        @tp_enum.close()          # close stream
-        @doc = Scorer::MAX_DOCS    # sentinel value
-        return false
-      end
-      @doc = @tp_enum.doc
-      @position = 0
-      return true
-    end
-
-    def skip_to(target)
-      if not @tp_enum.skip_to(target)
-        @tp_enum.close()          # close stream
-        @doc = Scorer::MAX_DOCS    # sentinel value
-        return false
-      end
-      @doc = @tp_enum.doc
-      @position = 0
-      return true
-    end
-
-
-    def first_position()
-      @count = @tp_enum.freq       # read first pos
-      next_position()
-    end
-
-    def next_position()
-      @count -= 1
-      if @count >= 0          # read subsequent pos's
-        @position = @tp_enum.next_position() - @offset
-        return true
-      else
-        return false
-      end
-    end
-
-    def to_s
-      "pp->(doc => #{@doc}, position => #{position})"
-    end
-  end
-end

data/lib/ferret/search/phrase_query.rb

@@ -1,214 +0,0 @@
-module Ferret::Search
-  # A Query that matches documents containing a particular sequence of terms.
-  # A PhraseQuery is built by QueryParser for input like +"new york"+.
-  # 
-  # This query may be combined with other terms or queries with a BooleanQuery.
-  class PhraseQuery < Query 
-    def initialize()
-      super
-      @slop = 0
-      @terms = []
-      @positions = []
-      @field = nil
-    end
-
-    # Sets the number of other words permitted between words in query phrase.
-    # If zero, then this is an exact phrase search.  For larger values this
-    # works like a +WITHIN+ or +NEAR+ operator.
-    #
-    # The slop is in fact an edit-distance, where the units correspond to
-    # moves of terms in the query phrase out of position.  For example, to
-    # switch the order of two words requires two moves (the first move places
-    # the words atop one another), so to permit re-orderings of phrases, the
-    # slop must be at least two.
-    #
-    # More exact matches are scored higher than sloppier matches, thus search
-    # results are sorted by exactness.
-    #
-    # The slop is zero by default, requiring exact matches.
-    attr_accessor :slop
-    attr_reader :terms, :positions, :field
-
-    # Adds a term to the end of the query phrase.
-    #
-    # The relative position of the term is the one immediately after the last
-    # term added, unless explicitly specified. By specifying explicitly,
-    # you can have phrases with more than one term at the same position or
-    # phrases with gaps (e.g. in connection with stopwords).
-    #
-    # term:: the term to search for
-    # position:: the relative position of the term to the rest of the terms
-    # int the query.
-    def add(term, position = nil, pos_inc = 1) 
-      if position.nil?
-        position = (@positions.size > 0) ? (@positions[-1] + pos_inc) : 0
-      end
-      
-      if @terms.size == 0
-        @field = term.field
-      elsif (term.field != @field)
-        raise ArgumentError, "All phrase terms must be in the same field: #{term}"
-      end
-      
-      @terms << term
-      @positions << position
-    end
-
-    def <<(term)
-      add(term)
-      return self
-    end
-    
-    class PhraseWeight < Weight 
-      attr_reader :query, :value
-
-      def initialize(query, searcher)
-        @query = query
-        @similarity = query.similarity(searcher)
-        @idf = @similarity.idf_phrase(@query.terms, searcher)
-      end
-
-      def to_s() return "phrase_weight(#{@value})" end
-
-      def sum_of_squared_weights() 
-        @query_weight = @idf * @query.boost()  # compute query weight
-        return @query_weight * @query_weight   # square it
-      end
-
-      def normalize(query_norm) 
-        @query_norm = query_norm
-        @query_weight *= query_norm            # normalize query weight
-        @value = @query_weight * @idf          # idf for document 
-      end
-
-      def scorer(reader)
-        return nil if @query.terms.size == 0   # optimize zero-term case
-
-        tps = []
-        @query.terms.each do |term|
-          tp = reader.term_positions_for(term)
-          return nil if tp.nil?
-          tps << tp
-        end
-
-        if (@query.slop == 0)				  # optimize exact case
-          return ExactPhraseScorer.new(self, tps, @query.positions,
-                                       @similarity,
-                                       reader.get_norms(@query.field))
-        else
-          return SloppyPhraseScorer.new(self, tps, @query.positions,
-                                   @similarity,
-                                   @query.slop,
-                                   reader.get_norms(@query.field))
-        end
-      end
-
-      def explain(reader, doc)
-        result = Explanation.new()
-        result.description = "weight(#{@query} in #{doc}), product of:"
-
-        doc_freqs = @query.terms.map do |term|
-          "#{term.text}=#{reader.doc_freq(term)}"
-        end.join(", ")
-
-        idf_expl = Explanation.new(@idf, "idf(#{@query.field}:<#{doc_freqs}>)")
-        
-        # explain query weight
-        query_expl = Explanation.new()
-        query_expl.description = "query_weight(#{@query}), product of:"
-
-        boost = @query.boost()
-        if boost != 1.0
-          boost_expl = Explanation.new(boost, "boost")
-          query_expl << boost_expl
-        end
-        query_expl << idf_expl
-        
-        query_norm_expl = Explanation.new(@query_norm, "query_norm")
-        query_expl << query_norm_expl
-        
-        query_expl.value = boost * @idf * @query_norm
-
-        result << query_expl
-       
-        # explain field weight
-        field_expl = Explanation.new()
-        field_expl.description =
-          "field_weight(#{query} in #{doc}), product of:"
-
-        tf_expl = scorer(reader).explain(doc)
-        field_expl << tf_expl
-        field_expl << idf_expl
-
-        field_norm_expl = Explanation.new()
-        field_norms = reader.get_norms(@query.field)
-        field_norm =
-          field_norms ? Similarity.decode_norm(field_norms[doc]) : 0.0
-        field_norm_expl.value = field_norm
-        field_norm_expl.description =
-          "field_norm(field=#{@query.field}, doc=#{doc})"
-        field_expl << field_norm_expl
-
-        field_expl.value = tf_expl.value * @idf * field_norm
-        result << field_expl
-
-        if (query_expl.value == 1.0)
-          return field_expl
-        else
-          result.value = query_expl.value * field_expl.value
-          return result
-        end
-      end
-    end
-
-    def create_weight(searcher)
-      if @terms.size == 1 # optimize one-term case
-        term = @terms[0]
-        tq = TermQuery.new(term)
-        tq.boost = boost()
-        return tq.create_weight(searcher)
-      end
-      return PhraseWeight.new(self, searcher)
-    end
-
-    # See Query#extract_terms()
-    def extract_terms(query_terms) 
-      query_terms.merge(@terms)
-    end
-
-    # Prints a user-readable version of this query. 
-    def to_s(f=nil) 
-      buffer = ""
-      buffer << "#{@field}:" if @field != f
-      buffer << '"'
-      last_pos = -1
-      @terms.each_index do |i|
-        term = @terms[i]
-        pos = @positions[i]
-        last_pos.upto(pos-2) {buffer << "<> "}
-        last_pos = pos
-        buffer << "#{term.text} "
-      end
-      buffer.rstrip!
-      buffer << '"'
-      buffer << "~#{slop}" if (slop != 0) 
-      buffer << "^#{boost()}" if boost() != 1.0
-      return buffer
-    end
-
-    # Returns true iff +o+ is equal to this. 
-    def eql?(o) 
-      if not o.instance_of? PhraseQuery
-        return false
-      end
-      return (boost() == o.boost() and @slop == o.slop and
-        @terms == o.terms and @positions == o.positions)
-    end
-    alias :== :eql?
-
-    # Returns a hash code value for this object.
-    def hash() 
-      return boost().hash ^ slop.hash ^ @terms.hash ^ @positions.hash
-    end
-  end
-end