ferret 0.1.0

data/lib/ferret/search/term_scorer.rb

@@ -0,0 +1,181 @@
+module Ferret::Search
+  # Expert: A +Scorer+ for documents matching a +Term+.
+  class TermScorer < Scorer 
+    SCORE_CACHE_SIZE = 32
+
+    # Returns the current document number matching the query.
+    # Initially invalid, until #next() is called the first time.
+    attr_reader :doc
+
+    # Construct a +TermScorer+.
+    # weight:: The weight of the +Term+ in the query.
+    # td:: An iterator over the documents matching the +Term+.
+    # similarity:: The +Similarity+ implementation to be used for score
+    # computations.
+    # norms:: The field norms of the document fields for the +Term+.
+    def initialize(weight, td, similarity, norms) 
+      super(similarity)
+
+      @doc = 0
+      @docs = Array.new(32, 0) # buffered doc numbers
+      @freqs = Array.new(32, 0) # buffered term freqs
+      @pointer = @pointer_max = 0;
+      @score_cache = Array.new(SCORE_CACHE_SIZE)
+
+      @weight = weight
+      @term_docs = td
+      @norms = norms
+      @weight_value = weight.value
+
+      SCORE_CACHE_SIZE.times do |i|
+        @score_cache[i] = similarity().tf(i) * @weight_value
+      end
+    end
+
+    # Expert: Iterates over matching all documents, yielding the document
+    # number and the score.
+    #
+    # returns:: true if more matching documents may remain.
+    def each_hit() # :yields: doc, score
+      sim = similarity() # cache sim in local
+      while next?
+        f = @freqs[@pointer]
+
+        # compute tf(f)*weight
+        if f < SCORE_CACHE_SIZE                    # check cache
+          score = @score_cache[f]                  # cache hit
+        else
+          score = sim.tf(f) * @weight_value # cache miss
+        end
+
+        score *= sim.decode_norm(@norms[@doc])      # normalize for field
+
+        yield(@doc, score)                         # collect score
+      end
+    end
+
+    # Expert: Iterates over matching documents in a range.
+    #
+    # NOTE: that #next? needs to be called first.
+    #
+    # max:: Do not score documents past this. Default will search all documents
+    # avaliable.
+    # returns:: true if more matching documents may remain.
+    def each_hit_up_to(max = MAX_DOCS) # :yields: doc, score
+      sim = similarity() # cache sim in local
+      while (@doc < max) # for docs in window
+        f = @freqs[@pointer]
+
+        # compute tf(f)*weight
+        if f < SCORE_CACHE_SIZE                    # check cache
+          score = @score_cache[f]                  # cache hit
+        else
+          score = sim.tf(f) * @weight_value # cache miss
+        end
+
+        score *= sim.decode_norm(@norms[@doc])      # normalize for field
+
+        yield(@doc, score)                         # collect score
+        if not next?
+          return false
+        end
+      end
+      return true # false if we didn't find +max+ hits
+    end
+
+
+    # Advances to the next document matching the query.
+    # <br>The iterator over the matching documents is buffered using
+    # TermDocEnum#read(int[],int[]).
+    # returns:: true iff there is another document matching the query.
+    def next?()
+      @pointer += 1
+      if @pointer >= @pointer_max
+        @pointer_max = @term_docs.read(@docs, @freqs) # refill buffer
+        if @pointer_max != 0
+          @pointer = 0
+        else 
+          @term_docs.close()                          # close stream
+          @doc = MAX_DOCS                             # set to sentinel value
+          return false
+        end
+      end
+      @doc = @docs[@pointer]
+      return true
+    end
+
+    def score() 
+      f = @freqs[@pointer]
+      # compute tf(f)*weight
+      if  f < SCORE_CACHE_SIZE                 # check cache
+        raw = @score_cache[f]                  # cache hit
+      else
+        raw = similarity().tf(f) * @weight_value # cache miss
+      end
+
+      return raw * Similarity.decode_norm(@norms[@doc]) # normalize for field
+    end
+
+    # Skips to the first match beyond the current whose document number is
+    # greater than or equal to a given target. 
+    # 
+    # The implementation uses TermDocEnum#skip_to(int).
+    # target:: The target document number.
+    # returns:: true iff there is such a match.
+    def skip_to(target)
+      # first scan in cache
+      while (@pointer += 1) < @pointer_max
+        if @docs[@pointer] >= target
+          @doc = @docs[@pointer]
+          return true
+        end
+      end
+
+      # not found in cache, seek underlying stream
+      result = @term_docs.skip_to(target)
+      if (result) 
+        @pointer_max = 1
+        @pointer = 0
+        @docs[@pointer] = @doc = @term_docs.doc
+        @freqs[@pointer] = @term_docs.freq
+      else 
+        @doc = MAX_DOCS
+      end
+      return result
+    end
+
+    # Returns an explanation of the score for a document.
+    # 
+    # When this method is used, the #next() method and the #score() method
+    # should not be used.
+    #
+    # doc:: The document number for the explanation.
+    # TODO: Modify to make use of TermDocEnum#skip_to(int).
+    def explain(doc)
+      query = @weight.query()
+      tf_explanation = Explanation.new()
+      tf = 0
+      while (@pointer < @pointer_max) 
+        if (@docs[@pointer] == doc)
+          tf = @freqs[@pointer]
+        end
+        @pointer += 1
+      end
+      if (tf == 0) 
+        while (@term_docs.next?) 
+          if (@term_docs.doc() == doc) 
+            tf = @term_docs.freq()
+          end
+        end
+      end
+      @term_docs.close()
+      tf_explanation.value = similarity().tf(tf)
+      tf_explanation.description = "tf(term_freq(#{query.term})=#{tf})"
+      
+      return tf_explanation
+    end
+
+    # Returns a string representation of this +TermScorer+. 
+    def to_s() return "scorer(" + @weight + ")"; end
+  end
+end

data/lib/ferret/search/top_docs.rb

@@ -0,0 +1,24 @@
+module Ferret::Search
+  # Expert: Returned by low-level search implementations.
+  # See Searcher#search
+  class TopDocs
+    # Expert: The total number of hits for the query.
+    # See Hits#length()
+    attr_accessor :score_docs, :total_hits, :fields
+
+    # Expert: Constructs a TopDocs.
+    def initialize(total_hits, score_docs, fields = SortField::FIELD_SCORE) 
+      @total_hits = total_hits
+      @score_docs = score_docs
+      @fields = fields
+    end
+
+    def to_s
+      buffer = "#{total_hits} hits sorted by <"
+      buffer << [fields].flatten.map {|field| "#{@field}" }.join(", ")
+      buffer << ">:\n"
+      score_docs.each {|sd| buffer << "\t#{sd}\n" }
+      return buffer
+    end
+  end
+end

data/lib/ferret/search/top_field_docs.rb

@@ -0,0 +1,17 @@
+module Ferret::Search
+  # Expert: Returned by low-level sorted search implementations.
+  class TopFieldDocs < TopDocs 
+
+    # The fields which were used to sort results by. 
+    attr_accessor :fields
+
+    # Creates one of these objects.
+    # total_hits::  Total number of hits for the query.
+    # score_docs::  The top hits for the query.
+    # fields::     The sort criteria used to find the top hits.
+    def initialize(total_hits, score_docs, fields) 
+      super(total_hits, score_docs)
+      @fields = fields
+    end
+  end
+end

data/lib/ferret/search/weight.rb

@@ -0,0 +1,54 @@
+module Ferret
+  module Search
+    # Expert: Calculate query weights and build query scorers.
+    # 
+    # The purpose of Weight is to make it so that searching does not modify
+    # a Query, so that a Query instance can be reused.
+    #
+    # Searcher dependent state of the query should reside in the Weight.
+    #
+    # IndexReader dependent state should reside in the Scorer.
+    # 
+    # A +Weight+ is used in the following way:
+    # 
+    # 1. A +Weight+ is constructed by a top-level query, given a +Searcher+
+    #    (See Query#create_weight).
+    # 2. The #sum_of_squared_weights() method is called on the +Weight+ to
+    #    compute the query normalization factor Similarity#query_norm(float)
+    #    of the query clauses contained in the query.
+    # 3. The query normalization factor is passed to #normalize().
+    #    At this point the weighting is complete.
+    # 4. A +Scorer+ is constructed by #scorer()
+    class Weight
+      # The query that this concerns. 
+      def query()
+        raise NotImplementedError
+      end
+
+      # The weight for this query. 
+      def  value()
+        raise NotImplementedError
+      end
+
+      # The sum of squared weights of contained query clauses. 
+      def sum_of_squared_weights()
+        raise NotImplementedError
+      end
+
+      # Assigns the query normalization factor to this. 
+      def normalize(norm)
+        raise NotImplementedError
+      end
+
+      # Constructs a scorer for this. 
+      def scorer(reader)
+        raise NotImplementedError
+      end
+
+      # An explanation of the score computation for the named document. 
+      def explain(reader, doc)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/ferret/search/wildcard_query.rb

@@ -0,0 +1,26 @@
+module Ferret::Search
+  # Implements the wildcard search query. Supported wildcards are +*+, which
+  # matches any character sequence (including the empty one), and +?+, which
+  # matches any single character. Note this query can be slow, as it needs to
+  # iterate over many terms. In order to prevent extremely slow
+  # WildcardQueries, a Wildcard term should not start with one of the
+  # wildcards +*+ or +?+.
+  # 
+  # See WildcardTermEnum
+  class WildcardQuery < MultiTermQuery 
+    def initialize(term) 
+      super(term)
+    end
+
+    def get_term_enum(reader)
+      return WildcardTermEnum.new(reader, @term)
+    end
+
+    def eql?(o) 
+      if o.instance_of?(WildcardQuery)
+        return super(o)
+      end
+      return false
+    end
+  end
+end

data/lib/ferret/search/wildcard_term_enum.rb

@@ -0,0 +1,61 @@
+module Ferret::Search
+  # Subclass of FilteredTermEnum for enumerating all terms that match the
+  # specified wildcard filter term.
+  # 
+  # Term enumerations are always ordered by Term.compareTo().  Each term in
+  # the enumeration is greater than all that precede it.
+  # 
+  class WildcardTermEnum < FilteredTermEnum 
+    include Ferret::Index
+
+    attr_reader :end_enum
+
+    WILDCARD_STRING = '*'
+    WILDCARD_CHAR = '?'
+
+    # Creates a new +WildcardTermEnum+.  Passing in a
+    # org.apache.lucene.index.Term Term that does not contain a
+    # +WILDCARD_CHAR+ will cause an exception to be raisen.
+    # 
+    # After calling the constructor the enumeration is already pointing to the first 
+    # valid term if such a term exists.
+    def initialize(reader, term)
+      super()
+      @end_enum = false
+      @search_term = term
+      @field = @search_term.field
+      text = @search_term.text
+      len = text.length
+
+      sidx = text.index(WILDCARD_STRING)||len
+      cidx = text.index(WILDCARD_CHAR)||len
+      idx = [sidx, cidx].min
+
+      @pre = @search_term.text[0,idx]
+      @pre_len = idx
+      @pattern = /^#{Regexp.escape(text[idx..-1]).gsub(/\\([?*])/){".#{$1}"}}$/
+      self.enum = reader.terms_from(Term.new(@search_term.field, @pre))
+    end
+
+    def term_compare(term) 
+      if (@field == term.field) 
+        search_text = term.text
+        if (search_text[0, @pre_len] == @pre) 
+          return (search_text[@pre_len..-1] =~ @pattern)
+        end
+      end
+      @end_enum = true
+      return false
+    end
+
+    def difference() 
+      return 1.0
+    end
+
+    def close()
+      super()
+      @pattern = nil
+      @field = nil
+    end
+  end
+end

data/lib/ferret/stemmers.rb

@@ -0,0 +1 @@
+require 'ferret/stemmers/porter_stemmer'

data/lib/ferret/stemmers/porter_stemmer.rb

@@ -0,0 +1,218 @@
+#!/usr/bin/env ruby
+#
+# $Id: PorterStemmer.rb,v 1.1.1.1 2004/04/17 13:55:20 pragdave Exp $
+#
+# See example usage at the end of this file.
+#
+
+module Stemmable
+
+  STEMMED = {}
+
+  STEP_2_LIST = {
+    'ational'=>'ate', 'tional'=>'tion', 'enci'=>'ence', 'anci'=>'ance',
+    'izer'=>'ize', 'bli'=>'ble',
+    'alli'=>'al', 'entli'=>'ent', 'eli'=>'e', 'ousli'=>'ous',
+    'ization'=>'ize', 'ation'=>'ate',
+    'ator'=>'ate', 'alism'=>'al', 'iveness'=>'ive', 'fulness'=>'ful',
+    'ousness'=>'ous', 'aliti'=>'al',
+    'iviti'=>'ive', 'biliti'=>'ble', 'logi'=>'log'
+  }
+  
+  STEP_3_LIST = {
+    'icate'=>'ic', 'ative'=>'', 'alize'=>'al', 'iciti'=>'ic',
+    'ical'=>'ic', 'ful'=>'', 'ness'=>''
+  }
+
+
+  SUFFIX_1_REGEXP = /(
+                    ational  |
+                    tional   |
+                    enci     |
+                    anci     |
+                    izer     |
+                    bli      |
+                    alli     |
+                    entli    |
+                    eli      |
+                    ousli    |
+                    ization  |
+                    ation    |
+                    ator     |
+                    alism    |
+                    iveness  |
+                    fulness  |
+                    ousness  |
+                    aliti    |
+                    iviti    |
+                    biliti   |
+                    logi)$/x
+
+
+  SUFFIX_2_REGEXP = /(
+                      al       |
+                      ance     |
+                      ence     |
+                      er       |
+                      ic       | 
+                      able     |
+                      ible     |
+                      ant      |
+                      ement    |
+                      ment     |
+                      ent      |
+                      ou       |
+                      ism      |
+                      ate      |
+                      iti      |
+                      ous      |
+                      ive      |
+                      ize)$/x
+
+
+  C = "[^aeiou]"         # consonant
+  V = "[aeiouy]"         # vowel
+  CC = "#{C}(?>[^aeiouy]*)"  # consonant sequence
+  VV = "#{V}(?>[aeiou]*)"    # vowel sequence
+
+  MGR0 = /^(#{CC})?#{VV}#{CC}/o                # [cc]vvcc... is m>0
+  MEQ1 = /^(#{CC})?#{VV}#{CC}(#{VV})?$/o       # [cc]vvcc[vv] is m=1
+  MGR1 = /^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o      # [cc]vvccvvcc... is m>1
+  VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem
+
+  #
+  # Porter stemmer in Ruby.
+  #
+  # This is the Porter stemming algorithm, ported to Ruby from the
+  # version coded up in Perl.  It's easy to follow against the rules
+  # in the original paper in:
+  #
+  #   Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
+  #   no. 3, pp 130-137,
+  #
+  # See also http://www.tartarus.org/~martin/PorterStemmer
+  #
+  # Send comments to raypereda@hotmail.com
+  #
+  
+  def stem_porter(w = self.to_str.dup)
+
+    # make a copy of the given object and convert it to a string.
+    original_word = w
+    
+    return w if w.length < 3
+
+    result = STEMMED[w]
+    return result if result
+
+    # now map initial y to Y so that the patterns never treat it as vowel
+    w[0] = 'Y' if w[0] == ?y
+    
+    # Step 1a
+    if w =~ /(ss|i)es$/
+      w = $` + $1
+    elsif w =~ /([^s])s$/ 
+      w = $` + $1
+    end
+
+    # Step 1b
+    if w =~ /eed$/
+      w.chop! if $` =~ MGR0 
+    elsif w =~ /(ed|ing)$/
+      stem = $`
+      if stem =~ VOWEL_IN_STEM 
+        w = stem
+        case w
+        when /(at|bl|iz)$/             then w << "e"
+        when /([^aeiouylsz])\1$/       then w.chop!
+        when /^#{CC}#{V}[^aeiouwxy]$/o then w << "e"
+        end
+      end
+    end
+
+    if w =~ /y$/ 
+      stem = $`
+      w = stem + "i" if stem =~ VOWEL_IN_STEM 
+    end
+
+    # Step 2
+    if w =~ SUFFIX_1_REGEXP
+      stem = $`
+      suffix = $1
+      # print "stem= " + stem + "\n" + "suffix=" + suffix + "\n"
+      if stem =~ MGR0
+        w = stem + STEP_2_LIST[suffix]
+      end
+    end
+
+    # Step 3
+    if w =~ /(icate|ative|alize|iciti|ical|ful|ness)$/
+      stem = $`
+      suffix = $1
+      if stem =~ MGR0
+        w = stem + STEP_3_LIST[suffix]
+      end
+    end
+
+    # Step 4
+    if w =~ SUFFIX_2_REGEXP
+      stem = $`
+      if stem =~ MGR1
+        w = stem
+      end
+    elsif w =~ /(s|t)(ion)$/
+      stem = $` + $1
+      if stem =~ MGR1
+        w = stem
+      end
+    end
+
+    #  Step 5
+    if w =~ /e$/ 
+      stem = $`
+      if (stem =~ MGR1) ||
+          (stem =~ MEQ1 && stem !~ /^#{CC}#{V}[^aeiouwxy]$/o)
+        w = stem
+      end
+    end
+
+    if w =~ /ll$/ && w =~ MGR1
+      w.chop!
+    end
+
+    # and turn initial Y back to y
+    w[0] = 'y' if w[0] == ?Y
+
+    STEMMED[original_word] = w
+
+    w
+  end
+
+
+  module_function :stem_porter
+  #
+  # make the stem_porter the default stem method, just in case we
+  # feel like having multiple stemmers available later.
+  #
+  alias stem stem_porter
+  public :stem
+
+end
+
+
+#
+# Make this script executable, and send it words on stdin, one per
+# line, and it will output the stemmed versions to stdout.
+#
+if $0 == __FILE__ then
+  class String
+    include Stemmable
+  end
+
+  # the String class, and any subclasses of it you might have, now know
+  # how to stem things.
+
+  $stdin.each do |word|
+    puts word.strip.stem
+  end
+end