ferret 0.1.0

data/lib/ferret/search/hit_collector.rb

@@ -0,0 +1,34 @@
+module Ferret::Search
+  # Lower-level search API.
+  #
+  # HitCollectors are primarily meant to be used to implement queries, sorting
+  # and filtering.
+  #
+  # See Searcher#search(Query, HitCollector)
+  class HitCollector 
+    # Called once for every non-zero scoring document, with the document number
+    # and its score.
+    # 
+    # If, for example, an application wished to collect all of the hits for a
+    # query in a BitSet, then it might:
+    #
+    #   searcher = IndexSearcher.new(index_reader)
+    #   bits = BitSet.new(index_reader.max_doc())
+    #   searcher.search(query, HitCollector.new() 
+    #     def collect(doc, score) 
+    #       bits.set(doc)
+    #     end
+    #   end
+    # 
+    # NOTE: This is called in an inner search loop.  For good search
+    # performance, implementations of this method should not call
+    # Searcher#doc(int) or IndexReader#document(int) on every document number
+    # encountered.  Doing so can slow searches by an order of magnitude or more.
+    #
+    # NOTE: The +score+ passed to this method is a raw score.  In other words,
+    # the score will not necessarily be a float whose value is between 0 and 1.
+    def collect(doc, score)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/search/hit_queue.rb

@@ -0,0 +1,11 @@
+module Ferret::Search
+  class HitQueue < Ferret::Utils::PriorityQueue 
+    def less_than(hit1, hit2) 
+      if (hit1.score == hit2.score)
+        return hit1.doc > hit2.doc
+      else
+        return hit1.score < hit2.score
+      end
+    end
+  end
+end

data/lib/ferret/search/index_searcher.rb

@@ -0,0 +1,173 @@
+module Ferret::Search
+
+  # Implements search over a single IndexReader.
+  # 
+  # Applications usually need only call the inherited @link #search(Query)end
+  # or @link #search(Query,Filter)endmethods. For performance reasons it is 
+  # recommended to open only one IndexSearcher and use it for all of your searches.
+  class IndexSearcher
+    include Ferret::Index
+
+    attr_accessor :similarity, :reader
+
+    # Creates a searcher searching the index in the provided directory. 
+    def initialize(arg)
+      if arg.is_a?(IndexReader)
+        @reader = arg
+      elsif arg.is_a?(Ferret::Store::Directory)
+        @reader = IndexReader.open(arg)
+      elsif arg.is_a?(String)
+        @dir = Ferret::Store::FSDirectory.open(arg)
+        @reader = IndexReader.open(@dir, true)
+      else
+        raise ArgumentError, "Unknown argument passed to initialize IndexReader"
+      end
+
+      @similarity = Similarity.default
+    end
+    
+    # IndexSearcher was constructed with IndexSearcher(r).
+    # If the IndexReader was supplied implicitly by specifying a directory, then
+    # the IndexReader gets closed.
+    def close()
+      @reader.close()
+    end
+
+    # Expert: Returns the number of documents containing +term+.
+    # Called by search code to compute term weights.
+    # See IndexReader#doc_freq
+    def doc_freq(term)
+      return @reader.doc_freq(term)
+    end
+
+    # Expert: For each term in the terms array, calculates the number of
+    # documents containing +term+. Returns an array with these
+    # document frequencies. Used to minimize number of remote calls.
+    def doc_freqs(terms)
+      result = Array.new(terms.length)
+      terms.each_with_index {|term, i| result[i] = doc_freq(term)}
+      return result
+    end
+
+    # Expert: Returns the stored fields of document +i+.
+    # Called by HitCollector implementations.
+    # See IndexReader#get_document
+    def doc(i)
+      return @reader.document(i)
+    end
+
+    # Expert: Returns one greater than the largest possible document number.
+    # Called by search code to compute term weights.
+    # See IndexReader#max_doc
+    def max_doc()
+      return @reader.max_doc()
+    end
+
+    # Creates a weight for +query+
+    # returns:: new weight
+    def create_weight(query)
+      return query.weight(self)
+    end
+
+    # The main search method for the index. You need to create a query to
+    # pass to this method. You can also pass a hash with one or more of the
+    # following; {filter, num_docs, first_doc, sort}
+    #
+    # query::    the query to run on the index
+    # filter::   filters docs from the search result
+    # first_doc:: The index in the results of the first doc retrieved.
+    #    Default is 0
+    # num_docs:: The number of results returned. Default is 10
+    # sort::     an array of SortFields describing how to sort the results.
+    def search(query, options = {})
+      filter = options[:filter]
+      first_doc = options[:first_doc]||0
+      num_docs = options[:num_docs]||10
+      sort = options[:sort]
+
+      if (num_docs <= 0)  # nil might be returned from hq.top() below.
+        raise ArgumentError, "num_docs must be > 0 to run a search"
+      end
+
+      scorer = query.weight(self).scorer(@reader)
+      if (scorer == nil)
+        return TopDocs.new(0, [])
+      end
+
+      bits = (filter.nil? ? nil : filter.bits(@reader))
+      if (sort)
+        fields = sort.is_a?(Array) ? sort : sort.fields
+        hq = FieldSortedHitQueue.new(@reader, fields, num_docs + first_doc)
+      else
+        hq = HitQueue.new(num_docs + first_doc)
+      end
+      total_hits = 0
+      min_score = 0.0
+      scorer.each_hit() do |doc, score|
+        if score > 0.0 and (bits.nil? or bits.get(doc)) # skip docs not in bits
+          total_hits += 1
+          if hq.size < num_docs or score >= min_score 
+            hq.insert(ScoreDoc.new(doc, score))
+            min_score = hq.top.score # maintain min_score
+          end
+        end
+      end
+
+      score_docs = Array.new(hq.size)
+      if (hq.size > first_doc)
+        score_docs = Array.new(hq.size - first_doc)
+        first_doc.times { hq.pop }
+        (hq.size - 1).downto(0) do |i|
+          score_docs[i] = hq.pop
+        end
+      else
+        score_docs = []
+        hq.clear
+      end
+
+      return TopDocs.new(total_hits, score_docs)
+    end
+
+    # Accepts a block and iterates through all of results yielding the doc
+    # number and the score for that hit. The hits are unsorted. This is the
+    # fastest way to get all of the hits from a search. However, you will
+    # usually want your hits sorted at least by score so you should use the
+    # #search method.
+    def search_each(query, filter = nil)
+      scorer = query.weight(self).scorer(@reader)
+      return if scorer == nil
+      bits = (filter.nil? ? nil : filter.bits(@reader))
+      scorer.each_hit() do |doc, score|
+        if score > 0.0 and (bits.nil? or bits.get(doc)) # skip docs not in bits
+          yield(doc, score)
+        end
+      end
+    end
+
+    # rewrites the query into a query that can be processed by the search
+    # methods. For example, a Fuzzy query is turned into a massive boolean
+    # query.
+    #
+    # original:: The original query to be rewritten.
+    def rewrite(original)
+      query = original
+      rewritten_query = query.rewrite(@reader)
+      while query != rewritten_query
+        query = rewritten_query
+        rewritten_query = query.rewrite(@reader)
+      end
+      return query
+    end
+
+    # Returns an Explanation that describes how +doc+ scored against
+    # +query+.
+    # 
+    # This is intended to be used in developing Similarity implementations,
+    # and, for good performance, should not be displayed with every hit.
+    # Computing an explanation is as expensive as executing the query over the
+    # entire index.
+    def explain(query, doc)
+      return query.weight(self).explain(@reader, doc)
+    end
+  end
+end

data/lib/ferret/search/match_all_docs_query.rb

@@ -0,0 +1,104 @@
+module Ferret::Search
+  # A query that matches all documents.
+  class MatchAllDocsQuery < Query 
+
+    def initialize() 
+      super
+    end
+
+    class MatchAllScorer < Scorer 
+
+      def initialize(reader, similarity) 
+        super(similarity)
+        @reader = reader
+        @count = -1
+        @max_doc = reader.max_doc
+      end
+
+      def doc() 
+        return @count
+      end
+
+      def explain(doc) 
+        return Explanation.new(1.0, "MatchAllDocsQuery")
+      end
+
+      def next? 
+        while (@count < (@max_doc - 1)) 
+          @count += 1
+          if (!@reader.deleted?(@count)) 
+            return true
+          end
+        end
+        return false
+      end
+
+      def score() 
+        return 1.0
+      end
+
+      def skip_to(target) 
+        @count = target - 1
+        return next?
+      end
+    end
+
+    class MatchAllDocsWeight < Weight 
+      attr_reader :query
+      def initialize(query, searcher) 
+        @query = query
+        @searcher = searcher
+      end
+
+      def to_s() 
+        return "weight(#{@query})"
+      end
+
+      def value() 
+        return 1.0
+      end
+
+      def sum_of_squared_weights() 
+        return 1.0
+      end
+
+      def normalize(query_norm) 
+      end
+
+      def scorer(reader) 
+        return MatchAllScorer.new(reader, @query.similarity(@searcher))
+      end
+
+      def explain(reader, doc) 
+        # explain query weight
+        query_expl = Explanation.new(1.0, "MatchAllDocsQuery")
+        boost_expl = Explanation.new(@query.boost, "boost")
+        if (boost_expl.value != 1.0)
+          query_expl << boost_expl
+          query_expl.value = boost_expl.value
+        end
+
+        return query_expl
+      end
+    end
+
+    def create_weight(searcher) 
+      return MatchAllDocsWeight.new(self, searcher)
+    end
+
+    def to_s(field) 
+      buffer = "MatchAllDocsQuery"
+      buffer << "^#{boost}" if (boost() != 1.0) 
+      return buffer
+    end
+
+    def eql?(o) 
+      return (o.instance_of?(MatchAllDocsQuery) and boost == o.boost)
+    end
+    alias :== :eql?
+
+    def hash
+      return boost.hash
+    end
+  end
+end

data/lib/ferret/search/multi_phrase_query.rb

@@ -0,0 +1,204 @@
+module Ferret::Search
+  # MultiPhraseQuery is a generalized version of PhraseQuery, with an added
+  # method #add(Term[]).
+  #
+  # To use this class, to search for the phrase "Microsoft app*" first use
+  # add(Term) on the term "Microsoft", then find all terms that have "app" as
+  # prefix using IndexReader.terms(Term), and use MultiPhraseQuery.add(Term[]
+  # terms) to add them to the query.
+  # 
+  # Author Anders Nielsen
+  class MultiPhraseQuery < Query 
+    include Ferret::Index
+
+    attr_accessor :slop
+    attr_reader :positions, :term_arrays, :field
+
+    def initialize()
+      super()
+      @slop = 0
+      @term_arrays = []
+      @positions = []
+      @field = nil
+    end
+
+    # Allows to specify the relative position of terms within the phrase.
+    # 
+    # See PhraseQuery#add(Term, int)
+    # terms:: the array of terms to search for or a single term
+    # position:: the position to search for these terms
+    def add(terms, position = nil, pos_inc = 1) 
+      if position.nil?
+        position = (@positions.size > 0) ? (@positions[-1] + pos_inc) : 0
+      end
+
+      if terms.instance_of?(Term)
+        terms = [terms]
+      end
+
+      if (@term_arrays.size == 0)
+        @field = terms[0].field
+      end
+
+      terms.each do |term|
+        if (term.field != @field) 
+          raise ArgumentError,
+            "All phrase terms must be in the same field (#{@field}): #{term}"
+        end
+      end
+
+      @term_arrays << terms
+      @positions << position
+    end
+    alias :<< :add
+    
+    class MultiPhraseWeight < Weight 
+      include Ferret::Index
+
+      attr_reader :query, :value
+
+      def initialize(query, searcher)
+        @query = query
+        @term_arrays = query.term_arrays
+        @positions = query.positions
+        @similarity = query.similarity(searcher)
+        @idf = 0.0
+
+        # compute idf
+        query.term_arrays.each do |terms|
+          terms.each do |term|
+            @idf += @similarity.idf_term(term, searcher)
+          end
+        end
+      end
+
+      def sum_of_squared_weights() 
+        @query_weight = @idf * @query.boost()      # compute query weight
+        return @query_weight * @query_weight       # square it
+      end
+
+      def normalize(query_norm) 
+        @query_norm = query_norm
+        @query_weight *= query_norm                # normalize query weight
+        @value = @query_weight * @idf              # idf for document 
+      end
+
+      def scorer(reader)
+        return nil if (@term_arrays.size == 0) # optimize zero-term case
+        tps = []
+        @term_arrays.each do |terms|
+          p = []
+          if (terms.length > 1)
+            p = MultipleTermDocPosEnum.new(reader, terms)
+          else
+            p = reader.term_positions_for(terms[0])
+          end
+        
+          return nil if (p == nil)
+        
+          tps << p
+        end
+      
+        if (@query.slop == 0)
+          return ExactPhraseScorer.new(self, tps, @positions, @similarity,
+                                       reader.get_norms(@query.field))
+        else
+          return SloppyPhraseScorer.new(self, tps, @positions, @similarity,
+                                        @query.slop, reader.get_norms(@query.field))
+        end
+      end
+      
+      def explain(reader, doc)
+       
+        result = Explanation.new()
+        result.description = "weight(#{@query} in #{doc}), product of:"
+
+        idf_expl = Explanation.new(@idf, "idf(#{@query})")
+        
+        # explain query weight
+        query_expl = Explanation.new()
+        query_expl.description = "query_weight(#{@query}), product of:"
+
+        boost_expl = Explanation.new(@query.boost(), "boost")
+        (query_expl << boost_expl) if (@query.boost() != 1.0)
+
+        query_expl << idf_expl
+        
+        query_norm_expl = Explanation.new(@query_norm,"query_norm")
+        query_expl << query_norm_expl
+        
+        query_expl.value =
+          boost_expl.value * idf_expl.value * query_norm_expl.value
+
+        result << query_expl
+       
+        # explain field weight
+        field_expl = Explanation.new()
+        field_expl.description =
+          "field_weight(#{@query} in #{doc}), product of:"
+
+        tf_expl = scorer(reader).explain(doc)
+        field_expl << tf_expl
+        field_expl << idf_expl
+
+        field_norm_expl = Explanation.new()
+        field_norms = reader.get_norms(@query.field)
+        field_norm =
+          field_norms ? Similarity.decode_norm(field_norms[doc]) : 0.0
+        field_norm_expl.value = field_norm
+        field_norm_expl.description =
+          "field_norm(field=#{@query.field}, doc=#{doc})"
+        field_expl << field_norm_expl
+
+        field_expl.value = tf_expl.value * idf_expl.value * field_norm_expl.value 
+        result << field_expl
+
+        if (query_expl.value == 1.0)
+          return field_expl
+        else
+          result.value = query_expl.value * field_expl.value
+          return result
+        end
+      end
+    end
+
+    def rewrite(reader) 
+      if (@term_arrays.size() == 1) # optimize one-term case
+        terms = @term_arrays[0]
+        bq = BooleanQuery.new(true)
+        terms.each do |term|
+          bq.add(TermQuery.new(term), BooleanClause::Occur::SHOULD)
+        end
+        bq.boost = boost()
+        return boq
+      else 
+        return self
+      end
+    end
+    
+    def create_weight(searcher)
+      return MultiPhraseWeight.new(self, searcher)
+    end
+
+    # Prints a user-readable version of this query. 
+    def to_s(f = nil) 
+      buffer = ""
+      buffer << "#{@field}:" if @field != f 
+      buffer << '"'
+      last_pos = -1
+      @term_arrays.each_index do |i|
+        terms = @term_arrays[i]
+        pos = @positions[i]
+        last_pos.upto(pos-2) {buffer << "<> "}
+        last_pos = pos
+        buffer << "#{terms.map {|term| term.text}.join("|")} "
+      end
+      buffer.rstrip!
+      buffer << '"'
+
+      buffer << "~#{@slop}" if (@slop != 0) 
+      buffer << "^#{boost()}" if boost() != 1.0
+      return buffer
+    end
+  end
+end