ferret 0.9.6 → 0.10.0

data/lib/ferret/search/scorer.rb

@@ -1,91 +0,0 @@
-module Ferret::Search
-  # Expert: Common scoring functionality for different types of queries.
-  # 
-  # A +Scorer+ either iterates over documents matching a query, or provides an
-  # explanation of the score for a query for a given document.
-  # 
-  # Document scores are computed using a given +Similarity+ implementation.
-  class Scorer 
-    attr_reader :similarity
-    MAX_DOCS = 0x7FFFFFFF
-
-    # Constructs a Scorer.
-    # similarity:: The +Similarity+ implementation used by this scorer.
-    def initialize(similarity) 
-      @similarity = similarity
-    end
-
-    # Expert: Iterates over matching all documents, yielding the document
-    # number and the score.
-    #
-    # returns:: true if more matching documents may remain.
-    def each_hit() # :yields: doc, score
-      while next?
-        yield(doc(), score())
-      end
-    end
- 
-    # Expert: Iterates over matching documents in a range.
-    #
-    # max:: Do not score documents past this. Default will search all documents
-    # avaliable.
-    # returns:: true if more matching documents may remain.
-    def each_hit_up_to(max = MAX_DOCS) # :yields: doc, score
-      while (next? and doc() < max)
-        yield(doc(), score())
-      end
-      return doc() < max
-    end
-
-    # Advances to the next document matching the query.
-    # returns:: true iff there is another document matching the query.
-    # When this method is used the #explain(int) method should not be used.
-    def next?()
-      raise NotImplementedError
-    end
-
-    # Returns the current document number matching the query.
-    # Initially invalid, until #next?() is called the first time.
-    def doc()
-      raise NotImplementedError
-    end
-
-    # Returns the score for the current document matching the query.
-    # Initially invalid, until #next?() is called the first time.
-    def score()
-      raise NotImplementedError
-    end
-
-    # Skips to the first match beyond the current whose document number is
-    # greater than or equal to a given target.
-    #
-    # When this method is used the #explain(int) method should not be used.
-    #
-    # target:: The target document number.
-    # returns:: true iff there is such a match.
-    #
-    # Behaves as if written:
-    #
-    #   def skip_to(target) 
-    #     begin 
-    #       return false if not next?()
-    #     end while (target > doc())
-    #     return true
-    #   end
-    #
-    # Most implementations are considerably more efficient than that.
-    def skip_to(target)
-      raise NotImplementedError
-    end
-
-    # Returns an explanation of the score for a document.
-    #
-    # When this method is used, the #next?(), #skip_to(int) and
-    # #score(HitCollector) methods should not be used.
-    #
-    # doc:: The document number for the explanation.
-    def explain(doc)
-      raise NotImplementedError
-    end
-  end
-end

data/lib/ferret/search/similarity.rb

@@ -1,278 +0,0 @@
-module Ferret::Search
-  # Expert: Scoring API.
-  # Subclasses implement search scoring.
-  #
-  # The score of query *q* for document *d* is defined
-  # in terms of these methods as follows:
-  #
-  # <table cellpadding="0" cellspacing="0" border="0">
-  #  <tr>
-  #    <td valign="middle" align="right" rowspan="2">score(q,d) =<br></td>
-  #    <td valign="middle" align="center">
-  #    <big><big><big><big><big>&Sigma</big></big></big></big></big></td>
-  #    <td valign="middle"><small>
-  #    #tf(int) tf(t in d)#
-  #    #idf_term(Term,Searcher) idf(t)#
-  #    Field#getBoost getBoost(t.field in d)#
-  #    #length_norm(String,int) length_norm(t.field in d)
-  #    </small></td>
-  #    <td valign="middle" rowspan="2">&nbsp*
-  #    #coord(int,int) coord(q,d)#
-  #    #query_norm(float) query_norm(q)
-  #    </td>
-  #  </tr>
-  #  <tr>
-  #   <td valign="top" align="right">
-  #    <small>t in q</small>
-  #    </td>
-  #  </tr>
-  # </table>
-  #
-  # See #set_default
-  # See IndexWriter#set_similarity
-  # See Searcher#set_similarity
-  class Similarity
-
-    def Similarity.byte_to_float(b)
-      if (b == 0)
-        return 0.0
-      end
-      mantissa = b & 0x07           # 0x07 =  7 = 0b00000111
-      exponent = (b >> 3) & 0x1F    # 0x1f = 31 = 0b00011111
-      return [0,0,(mantissa << 5),(exponent+48)].pack("cccc").unpack("e")[0]
-    end
-
-    def Similarity.float_to_byte(f) 
-      if (f <= 0.0) then return 0 end
-
-      bits = [f].pack("e").unpack("cccc")
-      mantissa = (bits[2] & 0xEf) >> 5 
-      exponent = (bits[3] - 48)
-
-      if (exponent > 0x1f)
-        exponent = 0x1f   # 0x1f = 31 = 0b00011111
-        mantissa = 0x07   # 0x07 =  7 = 0b00000111
-      end
-
-      if (exponent < 0)
-        exponent = 0
-        mantissa = 1
-      end
-
-      return ((exponent<<3) | mantissa)
-    end
-
-    # Cache of decoded bytes
-    NORM_TABLE = Array.new(256) { |i| Similarity.byte_to_float(i) }
-
-    # Decodes a normalization factor stored in an index.
-    # See Similarity#encode_norm(float)
-    def Similarity.decode_norm(b) 
-      return NORM_TABLE[b & 0xFF]
-    end
-
-    # Decodes a normalization factor stored in an index.
-    # See Similarity#encode_norm(float)
-    def decode_norm(b)
-      return self.class.decode_norm(b)
-    end
-
-    # Computes the normalization value for a field given the total number of
-    # terms contained in a field.  These values, together with field boosts, are
-    # stored in an index and multipled into scores for hits on each field by the
-    # search code.
-    #
-    # Matches in longer fields are less precise, so implemenations of this
-    # method usually return smaller values when *num_tokens* is large,
-    # and larger values when *num_tokens* is small.
-    #
-    # That these values are computed under 
-    # IndexWriter#add_document and stored then using
-    # #encode_norm(float).  Thus they have limited precision, and documents
-    # must be re-indexed if this method is altered.
-    #
-    # field:: the name of the field
-    # num_tokens:: the total number of tokens contained in fields named
-    #              _field_ of _doc_.
-    #
-    # See Field#set_boost
-    def length_norm
-      raise NotImplementedError
-    end
-
-    # Computes the normalization value for a query given the sum of the squared
-    # weights of each of the query terms.  This value is then multipled into the
-    # weight of each query term.
-    #
-    # This does not affect ranking, but rather just attempts to make scores
-    # from different queries comparable.
-    #
-    # sum_of_squared_weights:: the sum of the squares of query term weights
-    # Return:: a normalization factor for query weights
-    def query_norm
-      raise NotImplementedError
-    end
-
-    # Encodes a normalization factor for storage in an index.
-    #
-    # The encoding uses a five-bit exponent and three-bit mantissa, thus
-    # representing values from around 7x10^9 to 2x10^-9 with about one
-    # significant decimal digit of accuracy.  Zero is also represented.
-    # Negative numbers are rounded up to zero.  Values too large to represent
-    # are rounded down to the largest representable value.  Positive values too
-    # small to represent are rounded up to the smallest positive representable
-    # value.
-    #
-    # See Field#boost=
-    def Similarity.encode_norm(f) 
-      return Similarity.float_to_byte(f)
-    end
-
-    def encode_norm(f) 
-      return self.class.float_to_byte(f)
-    end
-
-    # Computes a score factor based on a term or phrase's frequency in a
-    # document.  This value is multiplied by the #idf_term(Term, Searcher)
-    # factor for each term in the query and these products are then summed to
-    # form the initial score for a document.
-    #
-    # Terms and phrases repeated in a document indicate the topic of the
-    # document, so implementations of this method usually return larger values
-    # when _freq_ is large, and smaller values when _freq_
-    # is small.
-    #
-    # The default implementation calls #tf(float)
-    #
-    # freq:: the frequency of a term within a document
-    # Return:: a score factor based on a term's within-document frequency
-    def tf
-      raise NotImplementedError
-    end
-
-    # Computes the amount of a sloppy phrase match, based on an edit distance.
-    # This value is summed for each sloppy phrase match in a document to form
-    # the frequency that is passed to #tf(float).
-    #
-    # A phrase match with a small edit distance to a document passage more
-    # closely matches the document, so implementations of this method usually
-    # return larger values when the edit distance is small and smaller values
-    # when it is large.
-    #
-    # See PhraseQuery#slop(int)
-    # distance:: the edit distance of this sloppy phrase match
-    # Return:: the frequency increment for this match
-    def sloppy_freq
-      raise NotImplementedError
-    end
-
-    # Computes a score factor for a simple term.
-    #
-    # The default implementation is:
-    #   return idf(searcher.doc_freq(term), searcher.max_doc())
-    #
-    # Note that Searcher#max_doc() is used instead of
-    # IndexReader#num_docs() because it is proportional to
-    # Searcher#doc_freq(Term) , i.e., when one is inaccurate,
-    # so is the other, and in the same direction.
-    #
-    # term:: the term in question
-    # searcher:: the document collection being searched
-    # Return:: a score factor for the term
-    def idf_term(term, searcher)
-      return idf(searcher.doc_freq(term), searcher.max_doc())
-    end
-
-    # Computes a score factor for a phrase.
-    #
-    # The default implementation sums the #idf(Term,Searcher) factor
-    # for each term in the phrase.
-    #
-    # terms:: the terms in the phrase
-    # searcher:: the document collection being searched
-    # Return:: a score factor for the phrase
-    def idf_phrase(terms, searcher)
-      idf = 0.0
-      terms.each { |term| idf += idf_term(term, searcher) }
-      return idf
-    end
-
-    # Computes a score factor based on a term's document frequency (the number
-    # of documents which contain the term).  This value is multiplied by the
-    # #tf(int) factor for each term in the query and these products are
-    # then summed to form the initial score for a document.
-    #
-    # Terms that occur in fewer documents are better indicators of topic, so
-    # implemenations of this method usually return larger values for rare terms,
-    # and smaller values for common terms.
-    #
-    # doc_freq:: the number of documents which contain the term
-    # num_docs:: the total number of documents in the collection
-    # Return:: a score factor based on the term's document frequency
-    def idf
-      raise NotImplementedError
-    end
-
-    # Computes a score factor based on the fraction of all query terms that a
-    # document contains.  This value is multiplied into scores.
-    #
-    # The presence of a large portion of the query terms indicates a better
-    # match with the query, so implemenations of this method usually return
-    # larger values when the ratio between these parameters is large and smaller
-    # values when the ratio between them is small.
-    #
-    # overlap:: the number of query terms matched in the document
-    # max_overlap:: the total number of terms in the query
-    # Return:: a score factor based on term overlap with the query
-    def coord
-      raise NotImplementedError
-    end
-  end
-
-  # Expert: Default scoring implementation.
-  class DefaultSimilarity < Similarity
-    # See source
-    def length_norm(field, num_terms)
-      return 1.0 / Math.sqrt(num_terms)
-    end
-    
-    # See source
-    def query_norm(sum_of_squared_weights)
-      return 1.0 / Math.sqrt(sum_of_squared_weights)
-    end
-
-    # See source
-    def tf(freq)
-      return Math.sqrt(freq)
-    end
-      
-    # See source
-    def sloppy_freq(distance)
-      return 1.0 / (distance + 1)
-    end
-      
-    # See source
-    def idf(doc_freq, num_docs)
-      return 0.0 if num_docs == 0
-      return Math.log(num_docs.to_f/(doc_freq+1)) + 1.0
-    end
-      
-    # See source
-    def coord(overlap, max_overlap)
-      return overlap.to_f / max_overlap
-    end
-  end
-
-  class Similarity
-    # The Similarity implementation used by default.
-    @@default = DefaultSimilarity.new()
-
-    def Similarity.default
-      return @@default
-    end
-
-    def Similarity.default=(default)
-      @@default = default
-    end
-  end
-end

data/lib/ferret/search/sloppy_phrase_scorer.rb

@@ -1,47 +0,0 @@
-module Ferret::Search
-  class SloppyPhraseScorer < PhraseScorer 
-
-    def initialize(weight, tps, positions, similarity, slop, norms) 
-      super(weight, tps, positions, similarity, norms)
-      @slop = slop
-    end
-
-    def phrase_freq()
-      @pq.clear()
-      last_pos = 0
-      each do |pp|
-        pp.first_position()
-        last_pos = pp.position if (pp.position > last_pos)
-        @pq.push(pp)         # build pq from list
-      end
-
-      freq = 0.0
-      done = false
-      begin 
-        pp = @pq.pop()
-        pos = start = pp.position
-        next_pos = @pq.top().position
-        while pos <= next_pos
-          start = pos       # advance pp to min window
-          if not pp.next_position()
-            done = true     # ran out of a term -- done
-            break
-          end
-          pos = pp.position
-        end
-
-        match_length = last_pos - start
-        if (match_length <= @slop)
-          freq += @similarity.sloppy_freq(match_length) # score match
-        end
-
-        if (pp.position > last_pos)
-          last_pos = pp.position
-        end
-        @pq.push(pp)         # restore pq
-      end while (!done)
-
-      return freq
-    end
-  end
-end

data/lib/ferret/search/sort.rb

@@ -1,112 +0,0 @@
-module Ferret::Search
-  # Encapsulates sort criteria for returned hits.
-  # 
-  # The fields used to determine sort order must be carefully chosen.
-  # Documents must contain a single term in such a field, and the value of the
-  # term should indicate the document's relative position in a given sort
-  # order.  The field must be indexed, but should not be tokenized, and does
-  # not need to be stored (unless you happen to want it back with the rest of
-  # your document data).  In other words:
-  # 
-  #   document << Field.new("by_number",
-  #                         x.to_s,
-  #                         Field::Store::NO,
-  #                         Field::Index::UN_TOKENIZED))
-  # 
-  # 
-  # === Valid Types of Values
-  # 
-  # There are three possible kinds of term values which may be put into
-  # sorting fields: Integers, Floats, or Strings.  Unless SortField objects
-  # are specified, the type of value in the field is determined by parsing the
-  # first term in the field.
-  # 
-  # Integer term values should contain only digits and an optional preceeding
-  # negative sign.  Values must be base 10.  Documents which should appear
-  # first in the sort should have low value integers, later documents high
-  # values (i.e. the documents should be numbered +1..n+ where +1+ is the
-  # first and +n+ the last).
-  # 
-  # Float term values should conform to values accepted by String#to_f.
-  # Documents which should appear first in the sort should have low values,
-  # later documents high values.
-  # 
-  # String term values can contain any valid String, but should not be
-  # tokenized.  The values are sorted according to their Comparable natural
-  # order.  Note that using this type of term value has higher memory
-  # requirements than the other two types.
-  # 
-  # === Object Reuse
-  # 
-  # One of these objects can be used multiple times and the sort order changed
-  # between usages.
-  # 
-  # This class is thread safe.
-  # 
-  # === Memory Usage
-  # 
-  # Sorting uses caches of term values maintained by the internal HitQueue(s).
-  # The cache is static and contains an integer or float array of length
-  # +IndexReader#max_doc+ for each field name for which a sort is performed.
-  # In other words, the size of the cache in bytes is:
-  # 
-  #   4 * IndexReader#max_doc * (# of different fields actually used to sort)
-  # 
-  # For String fields, the cache is larger: in addition to the above array,
-  # the value of every term in the field is kept in memory.  If there are many
-  # unique terms in the field, this could be quite large.
-  # 
-  # Note that the size of the cache is not affected by how many fields are in
-  # the index and _might_ be used to sort - only by the ones actually used to
-  # sort a result set.
-  # 
-  # The cache is cleared each time a new +IndexReader+ is passed in, or if the
-  # value returned by +max_doc()+ changes for the current IndexReader.  This
-  # class is not set up to be able to efficiently sort hits from more than one
-  # index simultaneously.
-  class Sort
-
-    attr_accessor :fields
-
-    # Sorts by computed relevance. You can pass a string representing the name
-    # of the field you want to sort on, a SortField, or an array of either
-    # (but not a mixed array). If you pass a string or and array of strings
-    # you can also pass a reverse flag. If you pass a SortField the reverse is
-    # handled by it.
-    #
-    # fields::  The fields you want to sort on. See also SortField
-    # reverse:: pass true if you want the sort order to be reversed. Only
-    #    works if you pass the field names.
-    def initialize(fields = [SortField::FIELD_SCORE, SortField::FIELD_DOC],
-                   reverse = false)
-      fields = [fields] unless fields.is_a?(Array)
-      @fields = fields
-      fields = fields.map {|field| field.is_a?(Symbol) ? field.to_s : field}
-      if fields[0].is_a?(String)
-        @fields = fields.map do |field|
-          if (field.is_a?(String))
-            next SortField.new(field, {:sort_type => SortField::SortType::AUTO,
-                                       :reverse => reverse})
-          else
-            next field
-          end
-        end
-      end
-      doc_sort_added = false
-      @fields.each {|f| doc_sort_added = true if f == SortField::FIELD_DOC }
-      @fields << SortField::FIELD_DOC if not doc_sort_added
-    end
-
-    # Represents sorting by computed relevance. Using this sort criteria returns
-    # the same results as calling Searcher#search(Query) Searcher#search()
-    # without a sort criteria, only with slightly more overhead.
-    RELEVANCE = Sort.new()
-
-    # Represents sorting by index order. 
-    INDEX_ORDER = Sort.new(SortField::FIELD_DOC)
-
-    def to_s() 
-      return "Sort[" + @fields.map {|field| "#{field}"}.join(", ") + "]"
-    end
-  end
-end