ferret 0.1.0

data/lib/ferret/search.rb

@@ -0,0 +1,49 @@
+require 'ferret/search/similarity.rb'
+require 'ferret/search/boolean_clause.rb'
+require 'ferret/search/scorer.rb'
+require 'ferret/search/score_doc.rb'
+require 'ferret/search/score_doc_comparator.rb'
+require 'ferret/search/weight.rb'
+require 'ferret/search/query.rb'
+require 'ferret/search/term_query.rb'
+require 'ferret/search/term_scorer.rb'
+require 'ferret/search/top_docs.rb'
+require 'ferret/search/boolean_query.rb'
+require 'ferret/search/conjunction_scorer.rb'
+require 'ferret/search/disjunction_sum_scorer.rb'
+require 'ferret/search/multi_term_query.rb'
+require 'ferret/search/phrase_query.rb'
+require 'ferret/search/multi_phrase_query.rb'
+require 'ferret/search/prefix_query.rb'
+require 'ferret/search/range_query.rb'
+require 'ferret/search/filtered_term_enum.rb'
+require 'ferret/search/wildcard_term_enum.rb'
+require 'ferret/search/wildcard_query.rb'
+require 'ferret/search/fuzzy_term_enum.rb'
+require 'ferret/search/fuzzy_query.rb'
+require 'ferret/search/phrase_positions.rb'
+require 'ferret/search/phrase_scorer.rb'
+require 'ferret/search/exact_phrase_scorer.rb'
+require 'ferret/search/sloppy_phrase_scorer.rb'
+require 'ferret/search/boolean_scorer.rb'
+require 'ferret/search/explanation.rb'
+require 'ferret/search/field_doc.rb'
+require 'ferret/search/hit_collector.rb'
+require 'ferret/search/hit_queue.rb'
+require 'ferret/search/non_matching_scorer.rb'
+require 'ferret/search/req_excl_scorer.rb'
+require 'ferret/search/req_opt_sum_scorer.rb'
+require 'ferret/search/score_doc.rb'
+require 'ferret/search/score_doc_comparator.rb'
+require 'ferret/search/sort_field.rb'
+require 'ferret/search/sort.rb'
+require 'ferret/search/field_cache.rb'
+require 'ferret/search/field_sorted_hit_queue.rb'
+require 'ferret/search/filter.rb'
+require 'ferret/search/range_filter.rb'
+require 'ferret/search/query_filter.rb'
+require 'ferret/search/caching_wrapper_filter.rb'
+require 'ferret/search/filtered_query.rb'
+require 'ferret/search/match_all_docs_query.rb'
+require 'ferret/search/spans.rb'
+require 'ferret/search/index_searcher.rb'

data/lib/ferret/search/boolean_clause.rb

@@ -0,0 +1,100 @@
+
+module Ferret::Search
+
+  # A clause in a BooleanQuery. 
+  class BooleanClause
+    
+    class Occur < Ferret::Utils::Parameter
+      
+      def to_s() 
+        return "+" if (self == MUST)
+        return "-" if (self == MUST_NOT)
+        return ""
+      end
+
+      # Use this operator for terms that _must_ appear in the matching
+      # documents. 
+      MUST = Occur.new("MUST")
+
+      # Use this operator for terms that _should_ appear in the matching
+      # documents. For a BooleanQuery with two +SHOULD+ subqueries, at
+      # least one of the queries must appear in the matching documents. 
+      SHOULD = Occur.new("SHOULD")
+
+      # Use this operator for terms that _must not_ appear in the matching
+      # documents.  Note that it is not possible to search for queries that
+      # only consist of a +MUST_NOT+ query. 
+      MUST_NOT = Occur.new("MUST_NOT")
+    end
+
+    # The query whose matching documents are combined by the boolean query.
+    attr_accessor :query
+
+    # If true, documents documents which _do not_ match this sub-query will
+    # _not_ match the boolean query.
+    attr_writer :required
+    def required?
+      @required
+    end
+    
+    # If true, documents documents which _do_ match this sub-query will _not_
+    # match the boolean query.
+    attr_writer :prohibited
+    def prohibited?
+      @prohibited
+    end
+
+    # See BooleanQuery::Occur for values for this attribute
+    attr_reader :occur
+    def occur=(occur) 
+      @occur = occur
+      set_fields(occur)
+    end
+
+    # Constructs a BooleanClause. Default value for occur is Occur::SHOULD
+    def initialize(query, occur = Occur::SHOULD) 
+      @query = query
+      @occur = occur
+      set_fields(occur)
+    end
+
+
+    # Returns true iff +other+ is equal to this. 
+    def eql?(other) 
+      if not other.instance_of?(BooleanClause)
+        return false
+      end
+      return (@query == other.query and
+              @required == other.required? and
+              @prohibited == other.prohibited?)
+    end
+    alias :== :eql?
+
+    # Returns a hash code value for this object.
+    def hash() 
+      return @query.hash() ^ (@required ? 1 : 0) ^ (@prohibited ? 2 : 0)
+    end
+
+    # represent a boolean clause as a string
+    def to_s() 
+      return @occur.to_s() + @query.to_s()
+    end
+    
+    private
+
+      def set_fields(occur) 
+        if (occur == Occur::MUST) 
+          @required = true
+          @prohibited = false
+        elsif (occur == Occur::SHOULD) 
+          @required = false
+          @prohibited = false
+        elsif (occur == Occur::MUST_NOT) 
+          @required = false
+          @prohibited = true
+        else 
+          raise ArgumentError, "Unknown operator " + occur
+        end
+      end
+  end
+end

data/lib/ferret/search/boolean_query.rb

@@ -0,0 +1,303 @@
+module Ferret::Search
+  # A Query that matches documents matching boolean combinations of other
+  # queries, e.g. TermQuerys, PhraseQuerys or other BooleanQuerys.
+  class BooleanQuery < Query 
+    
+    # The maximum number of clauses permitted. Default value is 1024.
+    #
+    # TermQuery clauses are generated from for example prefix queries and
+    # fuzzy queries. Each TermQuery needs some buffer space during search,
+    # so this parameter indirectly controls the maximum buffer requirements
+    # for query search.
+    #
+    # When this parameter becomes a bottleneck for a Query one can use a
+    # Filter. For example instead of a RangeQuery one can use a RangeFilter.
+    #
+    # Attempts to add more than the permitted number of clauses cause
+    # TooManyClauses to be raisen.
+    attr_accessor :max_clause_count
+    attr_accessor :clauses
+    DEFAULT_MAX_CLAUSE_COUNT = 1024
+
+    @@max_clause_count = DEFAULT_MAX_CLAUSE_COUNT
+    def BooleanQuery.max_clause_count
+      return @@max_clause_count
+    end
+    def BooleanQuery.max_clause_count=(mcc)
+      @@max_clause_count = mcc
+    end
+
+    # Thrown when an attempt is made to add more than #max_clause_count()
+    # clauses. This typically happens if a PrefixQuery, FuzzyQuery,
+    # WildcardQuery, or RangeQuery is expanded to many terms during search. 
+    class TooManyClauses < Exception
+    end
+
+    # Constructs an empty boolean query.
+    # 
+    # Similarity#coord(int,int) may be disabled in scoring, as appropriate.
+    # For example, this score factor does not make sense for most automatically
+    # generated queries, like WildcardQuery and FuzzyQuery.
+    # 
+    # coord_disabled:: disables Similarity#coord(int,int) in scoring.
+    def initialize(coord_disabled = false) 
+      super()
+      @coord_disabled = coord_disabled
+      @clauses = []
+    end
+
+    # Returns true iff Similarity#coord(int,int) is disabled in scoring for
+    # this query instance.
+    # See #BooleanQuery(boolean)
+    def coord_disabled?()
+      return @coord_disabled
+    end
+
+    def similarity(searcher) 
+      sim = super
+      if (@coord_disabled) # disable coord as requested
+        class <<sim 
+          def coord(overlap, max_overlap) 
+            return 1.0
+          end
+        end
+      end
+      return sim
+    end
+
+    # Adds a clause to a boolean query.  Clauses may be:
+    # 
+    # required::   which means that documents which _do not_ match this
+    #              sub-query will _not_ match the boolean query
+    # prohibited:: which means that documents which _do_ match this
+    #              sub-query will _not_ match the boolean query; or
+    # neither::    in which case matched documents are neither prohibited
+    #              from nor required to match the sub-query. However, a
+    #              document must match at least 1 sub-query to match the
+    #              boolean query.
+    # 
+    # * For +required+ use add(query, BooleanClause::Occur::MUST)
+    # * For +prohibited+ use add(query, BooleanClause::Occur::MUST_NOT)
+    # * For +neither+ use add(query, BooleanClause::Occur::SHOULD)
+    # 
+    # raises:: TooManyClauses if the new number of clauses exceeds the
+    #          maximum clause number #max_clause_count()
+    def add_query(query, occur) 
+      add_clause(BooleanClause.new(query, occur))
+    end
+
+    # Adds a clause to a boolean query.
+    # raises:: TooManyClauses if the new number of clauses exceeds the
+    #          maximum clause number. See #max_clause_count()
+    def add_clause(clause) 
+      if @clauses.size >= @@max_clause_count
+        raise TooManyClauses
+      end
+
+      @clauses << clause
+    end
+    alias :<< :add_clause
+
+    class BooleanWeight < Weight 
+      attr_accessor :similarity
+      attr_accessor :weights
+      attr_reader :query
+
+      def initialize(query, searcher)
+        @query = query
+        @weights = []
+       
+        @similarity = query.similarity(searcher)
+        query.clauses.each do |clause|
+          @weights << clause.query.create_weight(searcher)
+        end
+      end
+
+      def value()
+        return @query.boost()
+      end
+
+      def sum_of_squared_weights()
+        sum = 0
+        @weights.each_with_index do |weight, i|
+          clause = @query.clauses[i]
+          if not clause.prohibited?
+            sum += weight.sum_of_squared_weights()         # sum sub weights
+          end
+        end
+
+        sum *= @query.boost() * @query.boost()             # boost each sub-weight
+
+        return sum 
+      end
+
+
+      def normalize(norm) 
+        norm *= @query.boost()
+        @weights.each_with_index do |weight, i|
+          clause = @query.clauses[i]
+          if not clause.prohibited?
+            weight.normalize(norm)
+          end
+        end
+      end
+
+      # returns:: An alternative Scorer that uses and provides skip_to(),
+      # and scores documents in document number order.
+      def scorer(reader)
+        result = BooleanScorer.new(@similarity)
+
+        @weights.each_with_index do |weight, i|
+          clause = @query.clauses[i]
+          sub_scorer = weight.scorer(reader)
+          if (sub_scorer != nil)
+            result.add_scorer(sub_scorer, clause.occur)
+          elsif (clause.required?())
+            return nil
+          end
+        end
+
+        return result
+      end
+
+      def explain(reader, doc)
+       
+        sum_expl = Explanation.new()
+        sum_expl.description = "sum of:"
+        coord = 0
+        max_coord = 0
+        sum = 0.0
+
+        @weights.each_with_index do |weight, i|
+          clause = @query.clauses[i]
+          explanation = weight.explain(reader, doc)
+          max_coord += 1 if not clause.prohibited?
+          if explanation.value > 0 
+            if not clause.prohibited?
+              sum_expl << explanation
+              sum += explanation.value
+              coord += 1
+            else 
+              return Explanation.new(0.0, "match prohibited")
+            end
+          elsif clause.required?
+            return Explanation.new(0.0, "match required")
+          end
+        end
+        sum_expl.value = sum
+
+        if (coord == 1)                          # only one clause matched
+          sum_expl = sum_expl.details[0]         # eliminate wrapper
+        end
+
+        coord_factor = @similarity.coord(coord, max_coord)
+        if (coord_factor == 1.0)                # coord is no-op
+          return sum_expl                       # eliminate wrapper
+        else 
+          result = Explanation.new()
+          result.description = "product of:"
+          result << sum_expl
+          result << Explanation.new(coord_factor, "coord(#{coord}/#{max_coord})")
+          result.value = sum * coord_factor
+          return result
+        end
+      end
+    end #end BooleanWeight
+
+    def create_weight(searcher)
+      return BooleanWeight.new(self, searcher)
+    end
+
+    def rewrite(reader)
+      if @clauses.size == 1 # optimize 1-clause queries
+        clause = @clauses[0]
+        if not clause.prohibited? # just return clause
+
+          query = clause.query.rewrite(reader) # rewrite first
+
+          if boost() != 1.0 # incorporate boost
+            if query == clause.query # if rewrite was no-op
+              query = query.clone    # then clone before boost
+            end
+            query.boost = boost() * query.boost()
+          end
+
+          return query
+        end
+      end
+
+      clone = nil # recursively rewrite
+      @clauses.each_with_index do |clause, i|
+        query = clause.query().rewrite(reader)
+        if query != clause.query() # clause rewrote: must clone
+          clone ||= clone()
+          clone.clauses[i] = BooleanClause.new(query, clause.occur)
+        end
+      end
+      if (clone != nil) 
+        return clone # some clauses rewrote
+      else
+        return self  # no clauses rewrote
+      end
+    end
+
+    def extract_terms(terms) 
+      @clauses.each do |clause|
+        clause.query.extract_terms(terms)
+      end
+    end
+
+    def combine(queries) 
+      return Query.merge_boolean_queries(queries)
+    end
+
+    def clone() 
+      clone = super
+      clone.clauses = @clauses.clone
+      return clone
+    end
+
+    # Prints a user-readable version of this query. 
+    def to_s(field = nil) 
+      buffer = ""
+      buffer << "(" if boost != 1.0
+
+      @clauses.each_with_index do |clause, i|
+        if clause.prohibited?
+          buffer << "-"
+        elsif clause.required?
+          buffer << "+"
+        end
+
+        sub_query = clause.query
+        if sub_query.instance_of? BooleanQuery # wrap sub-bools in parens
+          buffer << "(#{clause.query.to_s(field)})"
+        else
+          buffer << clause.query.to_s(field)
+        end
+
+        if i != (@clauses.size - 1)
+          buffer << " "
+        end
+      end
+
+      buffer << ")^#{boost}" if boost() != 1.0 
+
+      return buffer
+    end
+
+    # Returns true iff +o+ is equal to this. 
+    def eql?(other) 
+      if not other.instance_of?(BooleanQuery)
+        return false
+      end
+      return (boost() == other.boost() and @clauses == other.clauses)
+    end
+    alias :== :eql?
+
+    # Returns a hash code value for this object.
+    def hash() 
+      return boost().hash ^ @clauses.hash
+    end
+  end
+end