ferret 0.1.0

data/lib/ferret/search/req_excl_scorer.rb

@@ -0,0 +1,125 @@
+module Ferret::Search
+  # A Scorer for queries with a required subscorer and an excluding (prohibited)
+  # subscorer.
+  #
+  # This +Scorer+ implements Scorer#skip_to(int), and it uses the skip_to() on
+  # the given scorers.
+  class ReqExclScorer < Scorer 
+    # Construct a +ReqExclScorer+.
+    # req_scorer:: The scorer that must match, except where
+    # excl_scorer:: indicates exclusion.
+    def initialize(req_scorer, excl_scorer) 
+      super(nil) # No similarity used.
+      @req_scorer = req_scorer
+      @excl_scorer = excl_scorer
+
+      @first_time = true
+    end
+
+    
+    def next?
+      if @first_time
+        if not @excl_scorer.next?
+          @excl_scorer = nil # exhausted at start
+        end
+        @first_time = false
+      end
+      if @req_scorer == nil 
+        return false
+      end
+      if not @req_scorer.next?
+        @req_scorer = nil; # exhausted, nothing left
+        return false
+      end
+      if @excl_scorer == nil
+        return true # @req_scorer.next? already returned true
+      end
+      return to_non_excluded()
+    end
+    
+    # Advance to non excluded doc.
+    # On entry:
+    # 
+    # * @req_scorer != nil
+    # * @excl_scorer != nil
+    # * @req_scorer was advanced once via next? or skip_to() and
+    #   @req_scorer.doc() may still be excluded.
+    # 
+    # Advances @req_scorer a non excluded required doc, if any.
+    #
+    # returns:: true iff there is a non excluded required doc.
+    def to_non_excluded()
+      excl_doc = @excl_scorer.doc
+      begin 
+        req_doc = @req_scorer.doc # may be excluded
+        if (req_doc < excl_doc) 
+          return true # @req_scorer advanced to before @excl_scorer, ie. not excluded
+        elsif (req_doc > excl_doc) 
+          unless @excl_scorer.skip_to(req_doc) 
+            @excl_scorer = nil # exhausted, no more exclusions
+            return true
+          end
+          excl_doc = @excl_scorer.doc
+          if excl_doc > req_doc 
+            return true; # not excluded
+          end
+        end
+      end while @req_scorer.next?
+      @req_scorer = nil; # exhausted, nothing left
+      return false
+    end
+
+    # @req_scorer may be nil when next? or skip_to() already return false so
+    # only call when you know that a doc exists
+    def doc() 
+      return @req_scorer.doc
+    end
+
+    # Returns the score of the current document matching the query.
+    #
+    # Initially invalid, until #next? is called the first time.
+    #
+    # returns:: The score of the required scorer.
+    def score()
+      return @req_scorer.score()
+    end
+    
+    # Skips to the first match beyond the current whose document number is
+    # greater than or equal to a given target.
+    #
+    # When this method is used the #explain(int) method should not be used.
+    #
+    # target:: The target document number.
+    # returns:: true iff there is such a match.
+    def skip_to(target)
+      if (@first_time) 
+        @first_time = false
+        if (! @excl_scorer.skip_to(target)) 
+          @excl_scorer = nil; # exhausted
+        end
+      end
+      if (@req_scorer == nil) 
+        return false
+      end
+      if (@excl_scorer == nil) 
+        return @req_scorer.skip_to(target)
+      end
+      if (! @req_scorer.skip_to(target)) 
+        @req_scorer = nil
+        return false
+      end
+      return to_non_excluded()
+    end
+
+    def explain(doc)
+      e = Explanation.new()
+      if @excl_scorer.skip_to(doc) and @excl_scorer.doc == doc
+        e.description = "excluded"
+      else 
+        e.description = "not excluded"
+        e.details << @req_scorer.explain(doc)
+      end
+      return e
+    end
+  end
+end

data/lib/ferret/search/req_opt_sum_scorer.rb

@@ -0,0 +1,70 @@
+module Ferret::Search
+  # A Scorer for queries with a required part and an optional part.
+  # Delays skip_to() on the optional part until a score() is needed.
+  # 
+  # This +Scorer+ implements Scorer#skip_to(int).
+  class ReqOptSumScorer < Scorer 
+    # The scorers passed from the constructor.
+    # These are set to nil as soon as their next? or skip_to() returns false.
+    #
+    # Construct a +ReqOptScorer+.
+    # req_scorer:: The required scorer. This must match.
+    # opt_scorer:: The optional scorer. This is used for scoring only.
+    def initialize(req_scorer, opt_scorer)
+      super(nil) # No similarity used.
+      @req_scorer = req_scorer
+      @opt_scorer = opt_scorer
+
+      @first_time_opt_scorer = true
+    end
+
+
+    def next?
+      return @req_scorer.next?
+    end
+
+    def skip_to(target)
+      return @req_scorer.skip_to(target)
+    end
+
+    def doc() 
+      return @req_scorer.doc()
+    end
+
+    # Returns the score of the current document matching the query.
+    # Initially invalid, until #next? is called the first time.
+    #
+    # returns:: The score of the required scorer, eventually increased by the
+    #           score of the optional scorer when it also matches the current
+    #           document.
+    def score()
+      cur_doc = @req_scorer.doc
+      req_score = @req_scorer.score
+      if @first_time_opt_scorer
+        @first_time_opt_scorer = false
+        if not @opt_scorer.skip_to(cur_doc) 
+          @opt_scorer = nil
+          return req_score
+        end
+      elsif @opt_scorer.nil?
+        return req_score
+      elsif @opt_scorer.doc < cur_doc and not @opt_scorer.skip_to(cur_doc)
+        @opt_scorer = nil
+        return req_score
+      end
+      # assert (@opt_scorer != nil) and (@opt_scorer.doc() >= cur_doc)
+      return (@opt_scorer.doc == cur_doc) ? req_score + @opt_scorer.score() : req_score
+    end
+
+    # Explain the score of a document.
+    # @todo Also show the total score.
+    # See BooleanScorer.explain() on how to do this.
+    def explain(doc)
+      e = Explanation.new()
+      e.description = "required, optional"
+      e.details << @req_scorer.explain(doc)
+      e.details << @opt_scorer.explain(doc)
+      return e
+    end
+  end
+end

data/lib/ferret/search/score_doc.rb

@@ -0,0 +1,38 @@
+module Ferret::Search
+  # Expert: Returned by low-level search implementations.
+  # See TopDocs 
+  class ScoreDoc
+    include Comparable
+    # Expert: The score of this document for the query. 
+    attr_accessor :score
+
+    # Expert: A hit document's number.
+    attr_accessor :doc
+
+    # Expert: Constructs a ScoreDoc. 
+    def initialize(doc, score) 
+      @doc = doc
+      @score = score
+    end
+
+    # returns a hash value for storage in a Hash
+    def hash()
+      return 100 * doc * score
+    end
+
+    # score_docA < score_docB if score_docA.score < score_docB.score or
+    # score_docA.doc > score_docB.doc
+    def <=>(other)
+      result = @score.<=>(other.score)
+      if (result == 0)
+        return other.doc.<=>(@doc)
+      else
+        return result
+      end
+    end
+
+    def to_s
+      "#{@doc} -> %0.2f" % @score
+    end
+  end
+end

data/lib/ferret/search/score_doc_comparator.rb

@@ -0,0 +1,114 @@
+module Ferret::Search
+  # Expert: Compares two ScoreDoc objects for sorting.
+  class ScoreDocComparator 
+
+    # Special comparator for sorting hits according to computed relevance (score). 
+    RELEVANCE = ScoreDocComparator.new() 
+    class <<RELEVANCE
+      def compare(i, j) 
+        return -(i.score <=> j.score)
+      end
+      def sort_value(i) 
+        return i.score
+      end
+      def sort_type() 
+        return SortField::SortType::SCORE
+      end
+    end
+
+
+    # Special comparator for sorting hits according to index order (number). 
+    INDEX_ORDER = ScoreDocComparator.new() 
+    class <<INDEX_ORDER
+      def compare(i, j) 
+        return i.doc <=> j.doc
+      end
+      def sort_value(i) 
+        return i.doc
+      end
+      def sort_type() 
+        return SortField::SortType::DOC
+      end
+    end
+
+
+    # Compares two ScoreDoc objects and returns a result indicating their
+    # sort order.
+    # i:: First ScoreDoc
+    # j:: Second ScoreDoc
+    # returns:: +-1+ if +i+ should come before +j+
+    #           +1+  if +i+ should come after +j+
+    #           +0+  if they are equal
+    def compare(i, j)
+      return NotImplementedError
+    end
+
+
+    # Returns the value used to sort the given document.  The object returned
+    # must implement the java.io.Serializable interface.  This is used by
+    # multisearchers to determine how to collate results from their searchers.
+    #
+    # See FieldDoc
+    # i:: Document
+    # returns:: Serializable object
+    def sort_value(i)
+      return NotImplementedError
+    end
+
+
+    # Returns the type of sort.  Should return +SortField.SCORE+,
+    # +SortField.DOC+, +SortField.STRING+, +SortField.INTEGER+,
+    # +SortField.FLOAT+ or +SortField.CUSTOM+.  It is not valid to return
+    # +SortField.AUTO+.
+    # This is used by multisearchers to determine how to collate results from
+    # their searchers.  returns:: One of the constants in SortField.
+    # See SortField
+    def sort_type()
+      return NotImplementedError
+    end
+  end
+
+  class SimpleFieldComparator < ScoreDocComparator
+    def initialize(index, sort_type)
+      @index = index
+      @sort_type = sort_type
+    end
+
+    def compare(j, i) 
+      return @index[i.doc] <=> @index[j.doc]
+    end
+    def sort_value(i) 
+      return @index[i.doc]
+    end
+    def sort_type() 
+      return @sort_type
+    end
+  end
+
+  class SpecialFieldComparator < SimpleFieldComparator
+    def initialize(index, sort_type, comparator)
+      super(index, sort_type)
+      @comparator = comparator
+    end
+    def compare(j, i) 
+      return @comparator.call(@index[i.doc], @index[j.doc])
+    end
+  end
+
+  class StringFieldComparator < ScoreDocComparator
+    def initialize(index)
+      @str_index = index.str_index
+      @str_map = index.str_map
+    end
+
+    def compare(i, j) 
+      return @str_index[i.doc] <=> @str_index[j.doc]
+    end
+    def sort_value(i) 
+      return @str_map[@str_index[i.doc]]
+    end
+    def sort_type() 
+      return SortField::SortType::STRING
+    end
+  end
+end

data/lib/ferret/search/scorer.rb

@@ -0,0 +1,91 @@
+module Ferret::Search
+  # Expert: Common scoring functionality for different types of queries.
+  # 
+  # A +Scorer+ either iterates over documents matching a query, or provides an
+  # explanation of the score for a query for a given document.
+  # 
+  # Document scores are computed using a given +Similarity+ implementation.
+  class Scorer 
+    attr_reader :similarity
+    MAX_DOCS = 0x7FFFFFFF
+
+    # Constructs a Scorer.
+    # similarity:: The +Similarity+ implementation used by this scorer.
+    def initialize(similarity) 
+      @similarity = similarity
+    end
+
+    # Expert: Iterates over matching all documents, yielding the document
+    # number and the score.
+    #
+    # returns:: true if more matching documents may remain.
+    def each_hit() # :yields: doc, score
+      while next?
+        yield(doc(), score())
+      end
+    end
+ 
+    # Expert: Iterates over matching documents in a range.
+    #
+    # max:: Do not score documents past this. Default will search all documents
+    # avaliable.
+    # returns:: true if more matching documents may remain.
+    def each_hit_up_to(max = MAX_DOCS) # :yields: doc, score
+      while (next? and doc() < max)
+        yield(doc(), score())
+      end
+      return doc() < max
+    end
+
+    # Advances to the next document matching the query.
+    # returns:: true iff there is another document matching the query.
+    # When this method is used the #explain(int) method should not be used.
+    def next?()
+      raise NotImplementedError
+    end
+
+    # Returns the current document number matching the query.
+    # Initially invalid, until #next?() is called the first time.
+    def doc()
+      raise NotImplementedError
+    end
+
+    # Returns the score for the current document matching the query.
+    # Initially invalid, until #next?() is called the first time.
+    def score()
+      raise NotImplementedError
+    end
+
+    # Skips to the first match beyond the current whose document number is
+    # greater than or equal to a given target.
+    #
+    # When this method is used the #explain(int) method should not be used.
+    #
+    # target:: The target document number.
+    # returns:: true iff there is such a match.
+    #
+    # Behaves as if written:
+    #
+    #   def skip_to(target) 
+    #     begin 
+    #       return false if not next?()
+    #     end while (target > doc())
+    #     return true
+    #   end
+    #
+    # Most implementations are considerably more efficient than that.
+    def skip_to(target)
+      raise NotImplementedError
+    end
+
+    # Returns an explanation of the score for a document.
+    #
+    # When this method is used, the #next?(), #skip_to(int) and
+    # #score(HitCollector) methods should not be used.
+    #
+    # doc:: The document number for the explanation.
+    def explain(doc)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/search/similarity.rb

@@ -0,0 +1,278 @@
+module Ferret::Search
+  # Expert: Scoring API.
+  # Subclasses implement search scoring.
+  #
+  # The score of query *q* for document *d* is defined
+  # in terms of these methods as follows:
+  #
+  # <table cellpadding="0" cellspacing="0" border="0">
+  #  <tr>
+  #    <td valign="middle" align="right" rowspan="2">score(q,d) =<br></td>
+  #    <td valign="middle" align="center">
+  #    <big><big><big><big><big>&Sigma</big></big></big></big></big></td>
+  #    <td valign="middle"><small>
+  #    #tf(int) tf(t in d)#
+  #    #idf_term(Term,Searcher) idf(t)#
+  #    Field#getBoost getBoost(t.field in d)#
+  #    #length_norm(String,int) length_norm(t.field in d)
+  #    </small></td>
+  #    <td valign="middle" rowspan="2">&nbsp*
+  #    #coord(int,int) coord(q,d)#
+  #    #query_norm(float) query_norm(q)
+  #    </td>
+  #  </tr>
+  #  <tr>
+  #   <td valign="top" align="right">
+  #    <small>t in q</small>
+  #    </td>
+  #  </tr>
+  # </table>
+  #
+  # See #set_default
+  # See IndexWriter#set_similarity
+  # See Searcher#set_similarity
+  class Similarity
+
+    def Similarity.byte_to_float(b)
+      if (b == 0)
+        return 0.0
+      end
+      mantissa = b & 0x07           # 0x07 =  7 = 0b00000111
+      exponent = (b >> 3) & 0x1F    # 0x1f = 31 = 0b00011111
+      return [0,0,(mantissa << 5),(exponent+48)].pack("cccc").unpack("f")[0]
+    end
+
+    def Similarity.float_to_byte(f) 
+      if (f <= 0.0) then return 0 end
+
+      bits = [f].pack("f").unpack("cccc")
+      mantissa = (bits[2] & 0xEf) >> 5 
+      exponent = (bits[3] - 48)
+
+      if (exponent > 0x1f)
+        exponent = 0x1f   # 0x1f = 31 = 0b00011111
+        mantissa = 0x07   # 0x07 =  7 = 0b00000111
+      end
+
+      if (exponent < 0)
+        exponent = 0
+        mantissa = 1
+      end
+
+      return ((exponent<<3) | mantissa)
+    end
+
+    # Cache of decoded bytes
+    NORM_TABLE = Array.new(256) { |i| Similarity.byte_to_float(i) }
+
+    # Decodes a normalization factor stored in an index.
+    # See Similarity#encode_norm(float)
+    def Similarity.decode_norm(b) 
+      return NORM_TABLE[b & 0xFF]
+    end
+
+    # Decodes a normalization factor stored in an index.
+    # See Similarity#encode_norm(float)
+    def decode_norm(b)
+      return self.class.decode_norm(b)
+    end
+
+    # Computes the normalization value for a field given the total number of
+    # terms contained in a field.  These values, together with field boosts, are
+    # stored in an index and multipled into scores for hits on each field by the
+    # search code.
+    #
+    # Matches in longer fields are less precise, so implemenations of this
+    # method usually return smaller values when *num_tokens* is large,
+    # and larger values when *num_tokens* is small.
+    #
+    # That these values are computed under 
+    # IndexWriter#add_document and stored then using
+    # #encode_norm(float).  Thus they have limited precision, and documents
+    # must be re-indexed if this method is altered.
+    #
+    # field:: the name of the field
+    # num_tokens:: the total number of tokens contained in fields named
+    #              _field_ of _doc_.
+    #
+    # See Field#set_boost
+    def length_norm
+      raise NotImplementedError
+    end
+
+    # Computes the normalization value for a query given the sum of the squared
+    # weights of each of the query terms.  This value is then multipled into the
+    # weight of each query term.
+    #
+    # This does not affect ranking, but rather just attempts to make scores
+    # from different queries comparable.
+    #
+    # sum_of_squared_weights:: the sum of the squares of query term weights
+    # Return:: a normalization factor for query weights
+    def query_norm
+      raise NotImplementedError
+    end
+
+    # Encodes a normalization factor for storage in an index.
+    #
+    # The encoding uses a five-bit exponent and three-bit mantissa, thus
+    # representing values from around 7x10^9 to 2x10^-9 with about one
+    # significant decimal digit of accuracy.  Zero is also represented.
+    # Negative numbers are rounded up to zero.  Values too large to represent
+    # are rounded down to the largest representable value.  Positive values too
+    # small to represent are rounded up to the smallest positive representable
+    # value.
+    #
+    # See Field#boost=
+    def Similarity.encode_norm(f) 
+      return Similarity.float_to_byte(f)
+    end
+
+    def encode_norm(f) 
+      return self.class.float_to_byte(f)
+    end
+
+    # Computes a score factor based on a term or phrase's frequency in a
+    # document.  This value is multiplied by the #idf_term(Term, Searcher)
+    # factor for each term in the query and these products are then summed to
+    # form the initial score for a document.
+    #
+    # Terms and phrases repeated in a document indicate the topic of the
+    # document, so implementations of this method usually return larger values
+    # when _freq_ is large, and smaller values when _freq_
+    # is small.
+    #
+    # The default implementation calls #tf(float)
+    #
+    # freq:: the frequency of a term within a document
+    # Return:: a score factor based on a term's within-document frequency
+    def tf
+      raise NotImplementedError
+    end
+
+    # Computes the amount of a sloppy phrase match, based on an edit distance.
+    # This value is summed for each sloppy phrase match in a document to form
+    # the frequency that is passed to #tf(float).
+    #
+    # A phrase match with a small edit distance to a document passage more
+    # closely matches the document, so implementations of this method usually
+    # return larger values when the edit distance is small and smaller values
+    # when it is large.
+    #
+    # See PhraseQuery#slop(int)
+    # distance:: the edit distance of this sloppy phrase match
+    # Return:: the frequency increment for this match
+    def sloppy_freq
+      raise NotImplementedError
+    end
+
+    # Computes a score factor for a simple term.
+    #
+    # The default implementation is:
+    #   return idf(searcher.doc_freq(term), searcher.max_doc())
+    #
+    # Note that Searcher#max_doc() is used instead of
+    # IndexReader#num_docs() because it is proportional to
+    # Searcher#doc_freq(Term) , i.e., when one is inaccurate,
+    # so is the other, and in the same direction.
+    #
+    # term:: the term in question
+    # searcher:: the document collection being searched
+    # Return:: a score factor for the term
+    def idf_term(term, searcher)
+      return idf(searcher.doc_freq(term), searcher.max_doc())
+    end
+
+    # Computes a score factor for a phrase.
+    #
+    # The default implementation sums the #idf(Term,Searcher) factor
+    # for each term in the phrase.
+    #
+    # terms:: the terms in the phrase
+    # searcher:: the document collection being searched
+    # Return:: a score factor for the phrase
+    def idf_phrase(terms, searcher)
+      idf = 0.0
+      terms.each { |term| idf += idf_term(term, searcher) }
+      return idf
+    end
+
+    # Computes a score factor based on a term's document frequency (the number
+    # of documents which contain the term).  This value is multiplied by the
+    # #tf(int) factor for each term in the query and these products are
+    # then summed to form the initial score for a document.
+    #
+    # Terms that occur in fewer documents are better indicators of topic, so
+    # implemenations of this method usually return larger values for rare terms,
+    # and smaller values for common terms.
+    #
+    # doc_freq:: the number of documents which contain the term
+    # num_docs:: the total number of documents in the collection
+    # Return:: a score factor based on the term's document frequency
+    def idf
+      raise NotImplementedError
+    end
+
+    # Computes a score factor based on the fraction of all query terms that a
+    # document contains.  This value is multiplied into scores.
+    #
+    # The presence of a large portion of the query terms indicates a better
+    # match with the query, so implemenations of this method usually return
+    # larger values when the ratio between these parameters is large and smaller
+    # values when the ratio between them is small.
+    #
+    # overlap:: the number of query terms matched in the document
+    # max_overlap:: the total number of terms in the query
+    # Return:: a score factor based on term overlap with the query
+    def coord
+      raise NotImplementedError
+    end
+  end
+
+  # Expert: Default scoring implementation.
+  class DefaultSimilarity < Similarity
+    # See source
+    def length_norm(field, num_terms)
+      return 1.0 / Math.sqrt(num_terms)
+    end
+    
+    # See source
+    def query_norm(sum_of_squared_weights)
+      return 1.0 / Math.sqrt(sum_of_squared_weights)
+    end
+
+    # See source
+    def tf(freq)
+      return Math.sqrt(freq)
+    end
+      
+    # See source
+    def sloppy_freq(distance)
+      return 1.0 / (distance + 1)
+    end
+      
+    # See source
+    def idf(doc_freq, num_docs)
+      return 0.0 if num_docs == 0
+      return Math.log(num_docs.to_f/(doc_freq+1)) + 1.0
+    end
+      
+    # See source
+    def coord(overlap, max_overlap)
+      return overlap.to_f / max_overlap
+    end
+  end
+
+  class Similarity
+    # The Similarity implementation used by default.
+    @@default = DefaultSimilarity.new()
+
+    def Similarity.default
+      return @@default
+    end
+
+    def Similarity.default=(default)
+      @@default = default
+    end
+  end
+end