ferret 0.1.0

data/lib/ferret/search/boolean_scorer.rb

@@ -0,0 +1,294 @@
+module Ferret::Search
+  # An alternative to BooleanScorer.
+  # 
+  # Uses ConjunctionScorer, DisjunctionScorer, ReqOptScorer and ReqExclScorer.
+  # 
+  # Implements skip_to(), and has no limitations on the numbers of added scorers.
+  class BooleanScorer < Scorer 
+    attr_reader :required_scorers, :coordinator
+
+    class Coordinator 
+      attr_accessor :max_coord, :nr_matchers
+
+      def initialize(similarity)
+        @max_coord = 0 # to be increased for each non prohibited scorer
+        @coord_factors = nil
+        @similarity = similarity
+      end
+      
+      
+      def init() # use after all scorers have been added.
+        @coord_factors = Array.new(@max_coord + 1)
+
+        (@max_coord+1).times do |i|
+          @coord_factors[i] = @similarity.coord(i, @max_coord)
+        end
+      end
+      
+
+      def init_doc() 
+        @nr_matchers = 0
+      end
+      
+      def coord_factor() 
+        return @coord_factors[@nr_matchers]
+      end
+    end
+
+    # The scorer to which all scoring will be delegated,
+    # except for computing and using the coordination factor.
+
+    def initialize(similarity) 
+      super(similarity)
+      @required_scorers = []
+      @optional_scorers = []
+      @prohibited_scorers = []
+      @counting_sum_scorer = nil
+      @coordinator = Coordinator.new(similarity)
+    end
+
+    def add_scorer(scorer, occur) 
+      unless occur == BooleanClause::Occur::MUST_NOT
+        @coordinator.max_coord += 1
+      end
+
+      case occur
+      when BooleanClause::Occur::MUST: @required_scorers << scorer
+      when BooleanClause::Occur::SHOULD: @optional_scorers << scorer
+      when BooleanClause::Occur::MUST_NOT: @prohibited_scorers << scorer
+      end
+    end
+
+    # Initialize the match counting scorer that sums all the
+    # scores. 
+    # When "counting" is used in a name it means counting the number
+    # of matching scorers.<br>
+    # When "sum" is used in a name it means score value summing
+    # over the matching scorers
+    def init_counting_sum_scorer() 
+      @coordinator.init()
+      @counting_sum_scorer = make_counting_sum_scorer()
+    end
+
+    # Count a scorer as a single match. 
+    class SingleMatchScorer < Scorer 
+      def initialize(parent_scorer, scorer) 
+        super(scorer.similarity)
+        @scorer = scorer
+        @parent_scorer = parent_scorer
+      end
+      def score()
+        @parent_scorer.coordinator.nr_matchers += 1
+        return @scorer.score
+      end
+      def doc() 
+        return @scorer.doc
+      end
+      def next?
+        return @scorer.next?
+      end
+      def skip_to(doc_nr)
+        return @scorer.skip_to(doc_nr)
+      end
+      def explain(doc_nr)
+        return @scorer.explain(doc_nr)
+      end
+    end
+
+    class CountingDisjunctionSumScorer < DisjunctionSumScorer
+      def initialize(parent_scorer, scorers)
+        super(scorers)
+        @parent_scorer = parent_scorer
+      end
+      def score
+        @parent_scorer.coordinator.nr_matchers += @nr_matchers
+        return super
+      end
+    end
+
+    def counting_disjunction_sum_scorer(scorers)
+    # each scorer from the list counted as a single matcher
+    
+      return CountingDisjunctionSumScorer.new(self, scorers) 
+    end
+
+    class CountingConjunctionScorer < ConjunctionScorer
+      def initialize(parent_scorer, similarity)
+        super(similarity)
+        @parent_scorer = parent_scorer
+        @required_nr_matchers = parent_scorer.required_scorers.size
+        @last_scored_doc = -1
+      end
+      def score
+        if (@parent_scorer.doc() > @last_scored_doc)
+          @last_scored_doc = @parent_scorer.doc()
+          @parent_scorer.coordinator.nr_matchers += @required_nr_matchers
+        end
+
+        return super
+      end
+    end
+
+    def counting_conjunction_sum_scorer(required_scorers)
+      # each scorer from the list counted as a single matcher
+    
+      required_nr_matchers = required_scorers.size
+      ccs = CountingConjunctionScorer.new(self, Similarity.default)
+      @required_scorers.each do |scorer|
+        ccs << scorer
+      end
+      return ccs
+    end
+
+    # Returns the scorer to be used for match counting and score summing.
+    # Uses required_scorers, optional_scorers and prohibited_scorers.
+    def make_counting_sum_scorer()
+    # each scorer counted as a single matcher
+      if @required_scorers.size == 0
+        if @optional_scorers.size == 0
+          return NonMatchingScorer.new # only prohibited scorers
+        elsif @optional_scorers.size == 1
+          return make_counting_sum_scorer2( # the only optional scorer is required
+                    SingleMatchScorer.new(self, @optional_scorers[0]),
+                    []) # no optional scorers left
+        else # more than 1 @optional_scorers, no required scorers
+          return make_counting_sum_scorer2( # at least one optional scorer is required
+                    counting_disjunction_sum_scorer(@optional_scorers), 
+                    []) # no optional scorers left
+        end
+      elsif @required_scorers.size == 1 # 1 required
+        return make_counting_sum_scorer2(
+                    SingleMatchScorer.new(self, @required_scorers[0]),
+                    @optional_scorers)
+      else # more required scorers
+        return make_counting_sum_scorer2(
+                    counting_conjunction_sum_scorer(@required_scorers),
+                    @optional_scorers)
+      end
+    end
+
+    # Returns the scorer to be used for match counting and score summing.
+    # Uses the arguments and prohibited_scorers.
+    # required_counting_sum_scorer:: A required scorer already built.
+    # @optional_scorers:: A list of optional scorers, possibly empty.
+    def make_counting_sum_scorer2(required_counting_sum_scorer, optional_scorers)
+    
+      if (optional_scorers.size == 0)
+        if (@prohibited_scorers.size == 0)
+          return required_counting_sum_scorer
+        elsif (@prohibited_scorers.size == 1)
+          return ReqExclScorer.new(required_counting_sum_scorer,
+                                   @prohibited_scorers[0])
+        else # no optional, more than 1 prohibited
+          return ReqExclScorer.new(
+                        required_counting_sum_scorer,
+                        DisjunctionSumScorer.new(@prohibited_scorers))
+        end
+      elsif (optional_scorers.size == 1)
+        return make_counting_sum_scorer3(
+                        required_counting_sum_scorer,
+                        SingleMatchScorer.new(self, optional_scorers[0]))
+      else # more optional
+        return make_counting_sum_scorer3(
+                        required_counting_sum_scorer,
+                        counting_disjunction_sum_scorer(optional_scorers))
+      end
+    end
+
+    # Returns the scorer to be used for match counting and score summing.
+    # Uses the arguments and prohibited_scorers.
+    # required_counting_sum_scorer:: A required scorer already built.
+    # optional_counting_sum_scorer:: An optional scorer already built.
+    def make_counting_sum_scorer3(required_counting_sum_scorer,
+                                  optional_counting_sum_scorer)
+      if (@prohibited_scorers.size == 0) # no prohibited
+        return ReqOptSumScorer.new(required_counting_sum_scorer,
+                                   optional_counting_sum_scorer)
+      elsif (@prohibited_scorers.size == 1) # 1 prohibited
+        return ReqOptSumScorer.new(
+                  ReqExclScorer.new(required_counting_sum_scorer,
+                                    @prohibited_scorers[0]),
+                  optional_counting_sum_scorer)
+      else # more prohibited
+        return ReqOptSumScorer.new(
+                  ReqExclScorer.new(required_counting_sum_scorer,
+                                    DisjunctionSumScorer.new(@prohibited_scorers)),
+                  optional_counting_sum_scorer)
+      end
+    end
+
+    # Expert: Iterates over matching all documents, yielding the document
+    # number and the score.
+    #
+    # returns:: true if more matching documents may remain.
+    def each_hit() # :yields: doc, score
+      if @counting_sum_scorer.nil?
+        init_counting_sum_scorer()
+      end
+      while @counting_sum_scorer.next?
+        yield(@counting_sum_scorer.doc, score())
+      end
+    end
+
+    # Expert: Iterates over matching documents in a range.
+    #
+    # NOTE: that #next? needs to be called first.
+    #
+    # max:: Do not score documents past this. Default will search all documents
+    # avaliable.
+    # returns:: true if more matching documents may remain.
+    def each_hit_up_to(max = MAX_DOCS) # :yields: doc, score
+      # nil pointer exception when next? was not called before:
+      doc_nr = @counting_sum_scorer.doc()
+      while (doc_nr < max) 
+        yield(doc_nr, score())
+        if not @counting_sum_scorer.next? 
+          return false
+        end
+        doc_nr = @counting_sum_scorer.doc()
+      end
+      return true
+    end
+
+    def doc()
+      return @counting_sum_scorer.doc
+    end
+
+    def next?
+      if (@counting_sum_scorer == nil) 
+        init_counting_sum_scorer()
+      end
+      return @counting_sum_scorer.next?
+    end
+
+    def score()
+      @coordinator.init_doc()
+      sum = @counting_sum_scorer.score()
+      return sum * @coordinator.coord_factor()
+    end
+
+    # Skips to the first match beyond the current whose document number is
+    # greater than or equal to a given target.
+    # 
+    # When this method is used the #explain(int) method should not be used.
+    # 
+    # target:: The target document number.
+    # returns:: true iff there is such a match.
+    def skip_to(target)
+      if (@counting_sum_scorer == nil) 
+        init_counting_sum_scorer()
+      end
+      return @counting_sum_scorer.skip_to(target)
+    end
+
+    # TODO: Implement an explanation of the coordination factor.
+    # doc:: The document number for the explanation.
+    # raises:: UnsupportedOperationException
+    def explain(doc) 
+      raise NotImplementedError
+      # How to explain the coordination factor?
+      #init_counting_sum_scorer()
+      #return @counting_sum_scorer.explain(doc); # misses coord factor. 
+    end
+  end
+end

data/lib/ferret/search/caching_wrapper_filter.rb

@@ -0,0 +1,40 @@
+module Ferret::Search
+  require 'monitor'
+
+  # Wraps another filter's result and caches it.  The caching
+  # behavior is like QueryFilter.  The purpose is to allow
+  # filters to simply filter, and then wrap with this class to add
+  # caching, keeping the two concerns decoupled yet composable.
+  class CachingWrapperFilter < Filter 
+    # filter:: Filter to cache results of
+    def initialize(filter) 
+      @filter = filter
+      @cache = nil
+    end
+
+    def bits(reader)
+      if (@cache == nil) 
+        @cache = Ferret::Utils::WeakKeyHash.new.extend(MonitorMixin)
+      end
+
+      @cache.synchronize() do # check cache
+        bits = @cache[reader]
+        if bits
+          return bits
+        end
+      end
+
+      bits = @filter.bits(reader)
+
+      @cache.synchronize() do # update cache
+        @cache[reader] = bits
+      end
+
+      return bits
+    end
+
+    def to_s() 
+      return "CachingWrapperFilter(#{@filter})"
+    end
+  end
+end

data/lib/ferret/search/conjunction_scorer.rb

@@ -0,0 +1,99 @@
+require 'set'
+module Ferret::Search
+  # Scorer for conjunctions, sets of queries, all of which are required. 
+  class ConjunctionScorer < Scorer 
+
+    def initialize(similarity) 
+      super
+      @scorers = []
+      @first_time = true
+      @more = true
+    end
+
+    def add(scorer) 
+      @scorers << scorer
+    end
+    alias :<< :add
+
+    def first()
+      return @scorers.first
+    end
+
+    def last()
+      return @scorers.last
+    end
+
+    def doc()
+      return first().doc()
+    end
+
+    def next?()
+      if (@first_time) 
+        init(true)
+      elsif (@more) 
+        @more = last().next? # trigger further scanning
+      end
+      return do_next()
+    end
+    
+    def do_next()
+      while @more and first().doc < last().doc # find doc w/ all clauses
+        @more = first().skip_to(last().doc)    # skip first upto last
+        @scorers << @scorers.shift             # move first to last
+      end
+      return @more # found a doc with all clauses
+    end
+
+    def skip_to(target)
+      if(@first_time) 
+        init(false)
+      end
+      
+      @scorers.each do |scorer|
+        break if not @more
+        @more = scorer.skip_to(target)
+      end
+      
+      sort_scorers() if @more # resort the scorers
+      
+      return do_next()
+    end
+
+    # Sums the scores of all of the scorers for the current document.
+    def score()
+      score = 0.0 # sum scores
+      @scorers.each do |scorer|
+        score += scorer.score
+      end
+      score *= @coord
+      return score
+    end
+    
+    def init(init_scorers)
+      #  compute coord factor
+      @coord = similarity().coord(@scorers.size(), @scorers.size())
+     
+      @more = @scorers.size() > 0
+
+      if init_scorers
+        # move each scorer to its first entry
+        @scorers.each do |scorer|
+          break if not @more
+          @more = scorer.next?
+        end
+        sort_scorers() if @more
+      end
+
+      @first_time = false
+    end
+
+    def sort_scorers() 
+      # move @scorers to an array
+      @scorers.sort! {|a,b| a.doc <=> b.doc }
+    end
+
+    def explain(doc) 
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/search/disjunction_sum_scorer.rb

@@ -0,0 +1,203 @@
+module Ferret::Search
+  # A Scorer for OR like queries, counterpart of Lucene's +ConjunctionScorer+.
+  # This Scorer implements Scorer#skip_to(int) and uses skip_to() on the given Scorers. 
+  class DisjunctionSumScorer < Scorer 
+    # the sub-scorers
+    attr_accessor :sub_scorers
+    
+    # Construct a +DisjunctionScorer+.
+    # sub_scorers:: A collection of at least two subscorers.
+    #
+    # minimum_nr_matchers:: The positive minimum number of subscorers that should
+    # match to match this query.
+    # <br>When +@minimum_nr_matchers+ is bigger than
+    # the number of +sub_scorers+,
+    # no matches will be produced.
+    # <br>When @minimum_nr_matchers equals the number of sub_scorers,
+    # it more efficient to use +ConjunctionScorer+.
+    def initialize(sub_scorers, minimum_nr_matchers = 1) 
+      super(nil)
+      
+      # The number of subscorers.  
+      @nr_scorers = sub_scorers.size
+
+      # The document number of the current match. 
+      @current_doc = -1
+      @curret_score = nil
+      # The number of subscorers that provide the current match. 
+      @nr_matchers = -1
+
+      if (minimum_nr_matchers <= 0) 
+        raise ArgumentError, "Minimum nr of matchers must be positive"
+      end
+      if (@nr_scorers <= 1) 
+        raise ArgumentError, "There must be at least 2 sub_scorers"
+      end
+
+      @minimum_nr_matchers = minimum_nr_matchers
+      @sub_scorers = sub_scorers
+
+      # The @scorer_queue contains all subscorers ordered by their current
+      # doc, with the minimum at the top.
+      # 
+      # The @scorer_queue is initialized the first time next? or skip_to() is
+      # called.
+      # 
+      # An exhausted scorer is immediately removed from the @scorer_queue.
+      # 
+      # If less than the @minimum_nr_matchers scorers remain in the
+      # @scorer_queue next? and skip_to() return false.
+      # 
+      # After each to call to next? or skip_to()
+      # +currentSumScore+ is the total score of the current matching doc,
+      # +@nr_matchers+ is the number of matching scorers,
+      # and all scorers are after the matching doc, or are exhausted.
+      @scorer_queue = nil
+    end
+    
+    # Called the first time next? or skip_to() is called to
+    # initialize +@scorer_queue+.
+    def init_scorer_queue()
+      @scorer_queue = ScorerQueue.new(@nr_scorers)
+      @sub_scorers.each do |sub_scorer|
+        if (sub_scorer.next?) # doc() method will be used in @scorer_queue.
+          @scorer_queue.insert(sub_scorer)
+        end
+      end
+    end
+
+    # A +PriorityQueue+ that orders by Scorer#doc(). 
+    class ScorerQueue < Ferret::Utils::PriorityQueue 
+      def less_than(scorer1, scorer2) 
+        return scorer1.doc < scorer2.doc
+      end
+    end
+    
+    def next?
+      if (@scorer_queue == nil) 
+        init_scorer_queue()
+      end
+
+      if (@scorer_queue.size < @minimum_nr_matchers) 
+        return false
+      else 
+        return advance_after_current()
+      end
+    end
+
+
+    # Advance all subscorers after the current document determined by the
+    # top of the +@scorer_queue+.
+    # Repeat until at least the minimum number of subscorers match on the same
+    # document and all subscorers are after that document or are exhausted.
+    # 
+    # On enbegin the +@scorer_queue+ has at least +@minimum_nr_matchers+
+    # available. At least the scorer with the minimum document number will be advanced.
+    # returns:: true iff there is a match.
+    # 
+    # In case there is a match, +@current_doc+, +currentSumScore+,
+    # and +@nr_matchers+ describe the match.
+    # 
+    # TODO Investigate whether it is possible to use skip_to() when
+    # the minimum number of matchers is bigger than one, ie. begin and use the
+    # character of ConjunctionScorer for the minimum number of matchers.
+    def advance_after_current()
+      begin # repeat until minimum nr of matchers
+        top = @scorer_queue.top
+        @current_doc = top.doc
+        @current_score = top.score
+        @nr_matchers = 1
+        begin # Until all subscorers are after @current_doc
+          if top.next?
+            @scorer_queue.adjust_top()
+          else 
+            @scorer_queue.pop()
+            if (@scorer_queue.size < (@minimum_nr_matchers - @nr_matchers)) 
+              # Not enough subscorers left for a match on this document,
+              # and also no more chance of any further match.
+              return false
+            end
+            if (@scorer_queue.size == 0) 
+              break # nothing more to advance, check for last match.
+            end
+          end
+          top = @scorer_queue.top
+          if top.doc != @current_doc
+            break # All remaining subscorers are after @current_doc.
+          else 
+            @current_score += top.score
+            @nr_matchers += 1
+          end
+        end while (true)
+        
+        if (@nr_matchers >= @minimum_nr_matchers) 
+          return true
+        elsif (@scorer_queue.size < @minimum_nr_matchers) 
+          return false
+        end
+      end while (true)
+    end
+    
+    # Returns the score of the current document matching the query.
+    # Initially invalid, until #next? is called the first time.
+    def score()
+      return @current_score
+    end
+     
+    # Returns the document number of the current document matching the query.
+    # Initially invalid, until #next? is called the first time.
+    def doc()
+      return @current_doc
+    end
+
+    # Returns the number of subscorers matching the current document.
+    # Initially invalid, until #next? is called the first time.
+    def number_of_matchers()
+      return @nr_matchers
+    end
+
+    # Skips to the first match beyond the current whose document number is
+    # greater than or equal to a given target.
+    # 
+    # When this method is used the #explain(int) method should not be used.
+    # 
+    # The implementation uses the skip_to() method on the subscorers.
+    # target:: The target document number.
+    # returns:: true iff there is such a match.
+    def skip_to(target)
+      if @scorer_queue.nil?
+        init_scorer_queue()
+      end
+      if @scorer_queue.size < @minimum_nr_matchers
+        return false
+      end
+      if target <= @current_doc
+        target = @current_doc + 1
+      end
+      begin 
+        top = @scorer_queue.top
+        if top.doc >= target 
+          return advance_after_current()
+        elsif top.skip_to(target) 
+          @scorer_queue.adjust_top()
+        else 
+          @scorer_queue.pop()
+          if (@scorer_queue.size < @minimum_nr_matchers) 
+            return false
+          end
+        end
+      end while (true)
+    end
+
+    # Gives and explanation for the score of a given document.
+    # TODO Show the resulting score. See BooleanScorer.explain() on how to do this.
+    def explain(doc)
+      e = Explanation.new()
+      e.description = "At least " + @minimum_nr_matchers + " of"
+      @sub_scorers.each do |sub_scorer|
+        e.details << sub_scorer.explain(doc)
+      end
+      return e
+    end
+  end
+end