ferret 0.1.0

data/lib/ferret/search/multi_term_query.rb

@@ -0,0 +1,65 @@
+module Ferret::Search
+  # A Query that matches documents containing a subset of terms provided
+  # by a FilteredTermEnum enumeration.
+  #
+  # +MultiTermQuery+ is not designed to be used by itself. The reason being
+  # that it is not intialized with a FilteredTermEnum enumeration. A
+  # FilteredTermEnum enumeration needs to be provided.
+  #
+  # For example, WildcardQuery and FuzzyQuery extend +MultiTermQuery+ to
+  # provide WildcardTermEnum and FuzzyTermEnum, respectively.
+  class MultiTermQuery < Query 
+    attr_reader :term
+
+    # Constructs a query for terms matching +term+. 
+    def initialize(term) 
+      super()
+      @term = term
+    end
+
+    # Construct the enumeration to be used, expanding the pattern term. 
+    def get_term_enum(reader)
+      raise NotImplementedError
+    end
+     
+
+    def rewrite(reader)
+      enumerator = get_term_enum(reader)
+      bq = BooleanQuery.new(true)
+      begin 
+        begin 
+          t = enumerator.term()
+          if (t != nil)
+            tq = TermQuery.new(t)      # found a match
+            tq.boost = boost() * enumerator.difference()   # set the boost
+            bq.add_query(tq, BooleanClause::Occur::SHOULD) # add to query
+          end
+        end while enumerator.next?
+      ensure 
+        enumerator.close()
+      end
+      return bq
+    end
+
+    # Prints a user-readable version of this query. 
+    def to_s(field = nil) 
+      buffer = ""
+      buffer << "#{@term.field}:" if @term.field != field
+      buffer << @term.text
+      buffer << "^#{boost()}" if (boost() != 1.0) 
+      return buffer
+    end
+
+    def eql?(o) 
+      if not o.instance_of? MultiTermQuery
+        return false
+      end
+      return term == o.term
+    end
+    alias :== :eql?
+
+    def hash() 
+      return term.hash()
+    end
+  end
+end

data/lib/ferret/search/non_matching_scorer.rb

@@ -0,0 +1,22 @@
+module Ferret::Search
+  # A scorer that matches no document at all. 
+  class NonMatchingScorer < Scorer 
+    def initialize()
+      super(nil) # no similarity used
+    end
+    
+   def next?
+     return false
+   end
+
+   def skip_to(target)
+     return false
+   end
+
+   def explain(doc) 
+      e = Explanation.new()
+      e.description = "No document matches."
+      return e
+    end
+  end
+end

data/lib/ferret/search/phrase_positions.rb

@@ -0,0 +1,55 @@
+module Ferret::Search
+  class PhrasePositions 
+    attr_reader :doc, :position
+    attr_accessor :next
+
+    def initialize(tp_enum, offset) 
+      @tp_enum = tp_enum
+      @offset = offset
+      @count = @position = @doc = -1
+      @next = nil
+    end
+
+    def next?()
+      if not @tp_enum.next?
+        @tp_enum.close()          # close stream
+        @doc = Scorer::MAX_DOCS    # sentinel value
+        return false
+      end
+      @doc = @tp_enum.doc
+      @position = 0
+      return true
+    end
+
+    def skip_to(target)
+      if not @tp_enum.skip_to(target)
+        @tp_enum.close()          # close stream
+        @doc = Scorer::MAX_DOCS    # sentinel value
+        return false
+      end
+      @doc = @tp_enum.doc
+      @position = 0
+      return true
+    end
+
+
+    def first_position()
+      @count = @tp_enum.freq       # read first pos
+      next_position()
+    end
+
+    def next_position()
+      @count -= 1
+      if @count >= 0          # read subsequent pos's
+        @position = @tp_enum.next_position() - @offset
+        return true
+      else
+        return false
+      end
+    end
+
+    def to_s
+      "pp->(doc => #{@doc}, position => #{position})"
+    end
+  end
+end

data/lib/ferret/search/phrase_query.rb

@@ -0,0 +1,217 @@
+module Ferret::Search
+  # A Query that matches documents containing a particular sequence of terms.
+  # A PhraseQuery is built by QueryParser for input like +"new york"+.
+  # 
+  # This query may be combined with other terms or queries with a BooleanQuery.
+  class PhraseQuery < Query 
+    def initialize()
+      super
+      @slop = 0
+      @terms = []
+      @positions = []
+      @field = nil
+    end
+
+    # Sets the number of other words permitted between words in query phrase.
+    # If zero, then this is an exact phrase search.  For larger values this
+    # works like a +WITHIN+ or +NEAR+ operator.
+    #
+    # The slop is in fact an edit-distance, where the units correspond to
+    # moves of terms in the query phrase out of position.  For example, to
+    # switch the order of two words requires two moves (the first move places
+    # the words atop one another), so to permit re-orderings of phrases, the
+    # slop must be at least two.
+    #
+    # More exact matches are scored higher than sloppier matches, thus search
+    # results are sorted by exactness.
+    #
+    # The slop is zero by default, requiring exact matches.
+    attr_accessor :slop
+    attr_reader :terms, :positions, :field
+
+    # Adds a term to the end of the query phrase.
+    #
+    # The relative position of the term is the one immediately after the last
+    # term added, unless explicitly specified. By specifying explicitly,
+    # you can have phrases with more than one term at the same position or
+    # phrases with gaps (e.g. in connection with stopwords).
+    #
+    # term:: the term to search for
+    # position:: the relative position of the term to the rest of the terms
+    # int the query.
+    def add(term, position = nil, pos_inc = 1) 
+      if position.nil?
+        position = (@positions.size > 0) ? (@positions[-1] + pos_inc) : 0
+      end
+      
+      if @terms.size == 0
+        @field = term.field
+      elsif (term.field != @field)
+        raise ArgumentError, "All phrase terms must be in the same field: #{term}"
+      end
+      
+      @terms << term
+      @positions << position
+    end
+
+    def <<(term)
+      add(term)
+      return self
+    end
+    
+    class PhraseWeight < Weight 
+      attr_reader :query, :value
+
+      def initialize(query, searcher)
+        @query = query
+        @similarity = query.similarity(searcher)
+        @idf = @similarity.idf_phrase(@query.terms, searcher)
+      end
+
+      def to_s() return "phrase_weight(#{@value})" end
+
+      def sum_of_squared_weights() 
+        @query_weight = @idf * @query.boost()  # compute query weight
+        return @query_weight * @query_weight   # square it
+      end
+
+      def normalize(query_norm) 
+        @query_norm = query_norm
+        @query_weight *= query_norm            # normalize query weight
+        @value = @query_weight * @idf          # idf for document 
+      end
+
+      def scorer(reader)
+        return nil if @query.terms.size == 0   # optimize zero-term case
+
+        tps = []
+        @query.terms.each do |term|
+          tp = reader.term_positions_for(term)
+          return nil if tp.nil?
+          tps << tp
+        end
+
+        if (@query.slop == 0)				  # optimize exact case
+          return ExactPhraseScorer.new(self, tps, @query.positions,
+                                       @similarity,
+                                       reader.get_norms(@query.field))
+        else
+          return SloppyPhraseScorer.new(self, tps, @query.positions,
+                                   @similarity,
+                                   @query.slop,
+                                   reader.get_norms(@query.field))
+        end
+      end
+
+      def explain(reader, doc)
+        result = Explanation.new()
+        result.description = "weight(#{@query} in #{doc}), product of:"
+
+        doc_freqs = @query.terms.map do |term|
+          "#{term.text}=#{reader.doc_freq(term)}"
+        end.join(", ")
+
+        idf_expl = Explanation.new(@idf, "idf(#{@query.field}:<#{doc_freqs}>)")
+        
+        # explain query weight
+        query_expl = Explanation.new()
+        query_expl.description = "query_weight(#{@query}), product of:"
+
+        boost = @query.boost()
+        if boost != 1.0
+          boost_expl = Explanation.new(boost, "boost")
+          query_expl << boost_expl
+        end
+        query_expl << idf_expl
+        
+        query_norm_expl = Explanation.new(@query_norm, "query_norm")
+        query_expl << query_norm_expl
+        
+        query_expl.value = boost * @idf * query_norm_expl.value
+
+        result << query_expl
+       
+        # explain field weight
+        field_expl = Explanation.new()
+        field_expl.description =
+          "field_weight(#{query} in #{doc}), product of:"
+
+        tf_expl = scorer(reader).explain(doc)
+        field_expl << tf_expl
+        field_expl << idf_expl
+
+        field_norm_expl = Explanation.new()
+        field_norms = reader.get_norms(@query.field)
+        field_norm =
+          field_norms ? Similarity.decode_norm(field_norms[doc]) : 0.0
+        field_norm_expl.value = field_norm
+        field_norm_expl.description =
+          "field_norm(field=#{@query.field}, doc=#{doc})"
+        field_expl << field_norm_expl
+
+        field_expl.value = tf_expl.value * @idf * field_norm
+        
+        result << field_expl
+
+        # combine them
+        result.value = query_expl.value * field_expl.value
+
+        if query_expl.value == 1.0
+          return field_expl
+        else
+          return result
+        end
+      end
+    end
+
+    def create_weight(searcher)
+      if @terms.size == 1 # optimize one-term case
+        term = @terms[0]
+        tq = TermQuery.new(term)
+        tq.boost = boost()
+        return tq.create_weight(searcher)
+      end
+      return PhraseWeight.new(self, searcher)
+    end
+
+    # See Query#extract_terms()
+    def extract_terms(query_terms) 
+      query_terms.add_all(@terms)
+    end
+
+    # Prints a user-readable version of this query. 
+    def to_s(f=nil) 
+      buffer = ""
+      buffer << "#{@field}:" if @field != f
+      buffer << '"'
+      last_pos = -1
+      @terms.each_index do |i|
+        term = @terms[i]
+        pos = @positions[i]
+        last_pos.upto(pos-2) {buffer << "<> "}
+        last_pos = pos
+        buffer << "#{term.text} "
+      end
+      buffer.rstrip!
+      buffer << '"'
+      buffer << "~#{slop}" if (slop != 0) 
+      buffer << "^#{boost()}" if boost() != 1.0
+      return buffer
+    end
+
+    # Returns true iff +o+ is equal to this. 
+    def eql?(o) 
+      if not o.instance_of? PhraseQuery
+        return false
+      end
+      return (boost() == o.boost() and @slop == o.slop and
+        @terms == o.terms and @positions == o.positions)
+    end
+    alias :== :eql?
+
+    # Returns a hash code value for this object.
+    def hash() 
+      return boost().hash ^ slop.hash ^ @terms.hash ^ @positions.hash
+    end
+  end
+end

data/lib/ferret/search/phrase_scorer.rb

@@ -0,0 +1,153 @@
+module Ferret::Search
+  class PhraseScorer < Scorer 
+    attr_reader :first, :last
+    protected   :first, :last
+
+    def initialize(weight, tps, positions, similarity, norms) 
+      super(similarity)
+      @norms = norms
+      @weight = weight
+      @value = weight.value
+      @first_time = true
+      @more = true
+
+      # convert tps to a list
+      tps.length.times do |i|
+        pp = PhrasePositions.new(tps[i], positions[i])
+        if (@last != nil) # add next to end of list
+          @last.next = pp
+        else
+          @first = pp
+        end
+        @last = pp
+      end
+
+      @pq = PhraseQueue.new(tps.length)  # construct empty pq
+    end
+
+    def doc()
+      return @first.doc
+    end
+
+    def next?
+      if (@first_time) 
+        init()
+        @first_time = false
+      elsif (@more) 
+        @more = @last.next?                      # trigger further scanning
+      end
+      return do_next()
+    end
+    
+    # next without initial increment
+    def do_next()
+      while (@more) 
+        while (@more and @first.doc < @last.doc) # find doc w/ all the terms
+          @more = @first.skip_to(@last.doc)      # skip first upto last
+          first_to_last()                        # and move it to the end
+        end
+
+        if (@more) 
+          # found a doc with all of the terms
+          @freq = phrase_freq()                  # check for phrase
+          if (@freq == 0.0)                      # no match
+            @more = @last.next?                  # trigger further scanning
+          else
+            return true                          # found a match
+          end
+        end
+      end
+      return false                               # no more matches
+    end
+
+    def each()
+      pp = @first
+      while (pp != nil)
+        yield pp
+        pp = pp.next
+      end
+    end
+
+    def score()
+      #puts("scoring #{@first.doc}")
+      raw = similarity().tf(@freq) * @value      # raw score
+      return raw * Similarity.decode_norm(@norms[@first.doc])  # normalize
+    end
+
+    def skip_to(target)
+      each() { |pp| break if not @more = pp.skip_to(target) }
+      sort() if @more                            # re-sort
+      return do_next()
+    end
+
+    def phrase_freq()
+      raise NotImplementedError
+    end
+
+    def init()
+      each do |pp|
+        break if not @more = pp.next?
+      end
+      if @more
+        sort()
+      end
+    end
+    
+    def sort() 
+      @pq.clear()
+      each() do |pp|
+        @pq.push(pp)
+      end
+      pq_to_list()
+    end
+
+    def pq_to_list() 
+      @last = @first = nil
+      while (@pq.top() != nil) 
+        pp = @pq.pop()
+        if (@last != nil) # add next to end of list
+          @last.next = pp
+        else
+          @first = pp
+        end
+        @last = pp
+        pp.next = nil
+      end
+    end
+
+    def first_to_last() 
+      @last.next = @first  # move first to end of list
+      @last = @first
+      @first = @first.next
+      @last.next = nil
+    end
+
+    def explain(doc)
+      tf_explanation = Explanation.new()
+
+      while (next? and doc() < doc)
+      end
+
+      phrase_freq = (doc() == doc) ? @freq : 0.0
+      tf_explanation.value = @similarity.tf(phrase_freq)
+      tf_explanation.description = "tf(phrase_freq=#{phrase_freq})"
+
+      return tf_explanation
+    end
+
+    def to_s() return "phrase_scorer(#{@weight})" end
+
+  end
+
+
+  class PhraseQueue < Ferret::Utils::PriorityQueue
+    def less_than(pp1, pp2)
+      if (pp1.doc == pp2.doc) 
+        return pp1.position < pp2.position
+      else
+        return pp1.doc < pp2.doc
+      end
+    end
+  end
+
+end