ferret 0.9.6 → 0.10.0

data/lib/ferret/search/term_scorer.rb

@@ -1,183 +0,0 @@
-module Ferret::Search
-  # Expert: A +Scorer+ for documents matching a +Term+.
-  class TermScorer < Scorer 
-    SCORE_CACHE_SIZE = 32
-
-    # Returns the current document number matching the query.
-    # Initially invalid, until #next() is called the first time.
-    attr_reader :doc
-
-    # Construct a +TermScorer+.
-    # weight:: The weight of the +Term+ in the query.
-    # td:: An iterator over the documents matching the +Term+.
-    # similarity:: The +Similarity+ implementation to be used for score
-    # computations.
-    # norms:: The field norms of the document fields for the +Term+.
-    def initialize(weight, td, similarity, norms) 
-      super(similarity)
-
-      @doc = 0
-      @docs = Array.new(SCORE_CACHE_SIZE, 0) # buffered doc numbers
-      @freqs = Array.new(SCORE_CACHE_SIZE, 0) # buffered term freqs
-      @pointer = @pointer_max = 0;
-      @score_cache = Array.new(SCORE_CACHE_SIZE)
-
-      @weight = weight
-      @term_docs = td
-      @norms = norms
-      @weight_value = weight.value
-
-      SCORE_CACHE_SIZE.times do |i|
-        @score_cache[i] = similarity().tf(i) * @weight_value
-      end
-    end
-
-    # Expert: Iterates over matching all documents, yielding the document
-    # number and the score.
-    #
-    # returns:: true if more matching documents may remain.
-    def each_hit() # :yields: doc, score
-      sim = similarity() # cache sim in local
-      while next?
-        f = @freqs[@pointer]
-
-        # compute tf(f)*weight
-        if f < SCORE_CACHE_SIZE                    # check cache
-          score = @score_cache[f]                  # cache hit
-        else
-          score = sim.tf(f) * @weight_value # cache miss
-        end
-
-        score *= sim.decode_norm(@norms[@doc])      # normalize for field
-
-        yield(@doc, score)                         # collect score
-      end
-    end
-
-    # Expert: Iterates over matching documents in a range.
-    #
-    # NOTE: that #next? needs to be called first.
-    #
-    # max:: Do not score documents past this. Default will search all documents
-    # avaliable.
-    # returns:: true if more matching documents may remain.
-    def each_hit_up_to(max = MAX_DOCS) # :yields: doc, score
-      sim = similarity() # cache sim in local
-      while (@doc < max) # for docs in window
-        f = @freqs[@pointer]
-
-        # compute tf(f)*weight
-        if f < SCORE_CACHE_SIZE                    # check cache
-          score = @score_cache[f]                  # cache hit
-        else
-          score = sim.tf(f) * @weight_value # cache miss
-        end
-
-        score *= sim.decode_norm(@norms[@doc])      # normalize for field
-
-        yield(@doc, score)                         # collect score
-        if not next?
-          return false
-        end
-      end
-      return true # false if we didn't find +max+ hits
-    end
-
-
-    # Advances to the next document matching the query.
-    # 
-    # The iterator over the matching documents is buffered using
-    # TermDocEnum#read(int[],int[]).
-    #
-    # returns:: true iff there is another document matching the query.
-    def next?()
-      @pointer += 1
-      if @pointer >= @pointer_max
-        @pointer_max = @term_docs.read(@docs, @freqs) # refill buffer
-        if @pointer_max != 0
-          @pointer = 0
-        else 
-          @term_docs.close()                          # close stream
-          @doc = MAX_DOCS                             # set to sentinel value
-          return false
-        end
-      end
-      @doc = @docs[@pointer]
-      return true
-    end
-
-    def score() 
-      f = @freqs[@pointer]
-      # compute tf(f)*weight
-      if  f < SCORE_CACHE_SIZE                 # check cache
-        raw = @score_cache[f]                  # cache hit
-      else
-        raw = similarity().tf(f) * @weight_value # cache miss
-      end
-
-      return raw * Similarity.decode_norm(@norms[@doc]) # normalize for field
-    end
-
-    # Skips to the first match beyond the current whose document number is
-    # greater than or equal to a given target. 
-    # 
-    # The implementation uses TermDocEnum#skip_to(int).
-    # target:: The target document number.
-    # returns:: true iff there is such a match.
-    def skip_to(target)
-      # first scan in cache
-      while (@pointer += 1) < @pointer_max
-        if @docs[@pointer] >= target
-          @doc = @docs[@pointer]
-          return true
-        end
-      end
-
-      # not found in cache, seek underlying stream
-      result = @term_docs.skip_to(target)
-      if (result) 
-        @pointer_max = 1
-        @pointer = 0
-        @docs[@pointer] = @doc = @term_docs.doc
-        @freqs[@pointer] = @term_docs.freq
-      else 
-        @doc = MAX_DOCS
-      end
-      return result
-    end
-
-    # Returns an explanation of the score for a document.
-    # 
-    # When this method is used, the #next() method and the #score() method
-    # should not be used.
-    #
-    # doc:: The document number for the explanation.
-    # TODO: Modify to make use of TermDocEnum#skip_to(int).
-    def explain(doc)
-      query = @weight.query()
-      tf_explanation = Explanation.new()
-      tf = 0
-      while (@pointer < @pointer_max) 
-        if (@docs[@pointer] == doc)
-          tf = @freqs[@pointer]
-        end
-        @pointer += 1
-      end
-      if (tf == 0) 
-        while (@term_docs.next?) 
-          if (@term_docs.doc() == doc) 
-            tf = @term_docs.freq()
-          end
-        end
-      end
-      @term_docs.close()
-      tf_explanation.value = similarity().tf(tf)
-      tf_explanation.description = "tf(term_freq(#{query.term})=#{tf})"
-      
-      return tf_explanation
-    end
-
-    # Returns a string representation of this +TermScorer+. 
-    def to_s() return "scorer(" + @weight + ")"; end
-  end
-end

data/lib/ferret/search/top_docs.rb

@@ -1,36 +0,0 @@
-module Ferret::Search
-  # Expert: Returned by low-level search implementations.
-  # See Searcher#search
-  class TopDocs
-    # Expert: The total number of hits for the query.
-    # See Hits#length()
-    attr_accessor :score_docs, :total_hits, :fields
-    alias :size :total_hits
-
-    # iterate through each of the score docs, yielding the document number and
-    # the score. eg:
-    #
-    #   top_docs.each do |doc, score|
-    #     puts "Doc number #{doc} found with score of #{score}"}
-    #   end
-    #
-    def each
-      score_docs.each {|sd| yield(sd.doc, sd.score) }
-    end
-
-    # Expert: Constructs a TopDocs.
-    def initialize(total_hits, score_docs, fields = SortField::FIELD_SCORE) 
-      @total_hits = total_hits
-      @score_docs = score_docs
-      @fields = fields
-    end
-
-    def to_s
-      buffer = "#{total_hits} hits sorted by <"
-      buffer << [fields].flatten.map {|field| "#{@field}" }.join(", ")
-      buffer << ">:\n"
-      score_docs.each {|sd| buffer << "\t#{sd}\n" }
-      return buffer
-    end
-  end
-end

data/lib/ferret/search/top_field_docs.rb

@@ -1,17 +0,0 @@
-module Ferret::Search
-  # Expert: Returned by low-level sorted search implementations.
-  class TopFieldDocs < TopDocs 
-
-    # The fields which were used to sort results by. 
-    attr_accessor :fields
-
-    # Creates one of these objects.
-    # total_hits::  Total number of hits for the query.
-    # score_docs::  The top hits for the query.
-    # fields::     The sort criteria used to find the top hits.
-    def initialize(total_hits, score_docs, fields) 
-      super(total_hits, score_docs)
-      @fields = fields
-    end
-  end
-end

data/lib/ferret/search/weight.rb

@@ -1,54 +0,0 @@
-module Ferret
-  module Search
-    # Expert: Calculate query weights and build query scorers.
-    # 
-    # The purpose of Weight is to make it so that searching does not modify
-    # a Query, so that a Query instance can be reused.
-    #
-    # Searcher dependent state of the query should reside in the Weight.
-    #
-    # IndexReader dependent state should reside in the Scorer.
-    # 
-    # A +Weight+ is used in the following way:
-    # 
-    # 1. A +Weight+ is constructed by a top-level query, given a +Searcher+
-    #    (See Query#create_weight).
-    # 2. The #sum_of_squared_weights() method is called on the +Weight+ to
-    #    compute the query normalization factor Similarity#query_norm(float)
-    #    of the query clauses contained in the query.
-    # 3. The query normalization factor is passed to #normalize().
-    #    At this point the weighting is complete.
-    # 4. A +Scorer+ is constructed by #scorer()
-    class Weight
-      # The query that this concerns. 
-      def query()
-        raise NotImplementedError
-      end
-
-      # The weight for this query. 
-      def  value()
-        raise NotImplementedError
-      end
-
-      # The sum of squared weights of contained query clauses. 
-      def sum_of_squared_weights()
-        raise NotImplementedError
-      end
-
-      # Assigns the query normalization factor to this. 
-      def normalize(norm)
-        raise NotImplementedError
-      end
-
-      # Constructs a scorer for this. 
-      def scorer(reader)
-        raise NotImplementedError
-      end
-
-      # An explanation of the score computation for the named document. 
-      def explain(reader, doc)
-        raise NotImplementedError
-      end
-    end
-  end
-end

data/lib/ferret/search/wildcard_query.rb

@@ -1,26 +0,0 @@
-module Ferret::Search
-  # Implements the wildcard search query. Supported wildcards are +*+, which
-  # matches any character sequence (including the empty one), and +?+, which
-  # matches any single character. Note this query can be slow, as it needs to
-  # iterate over many terms. In order to prevent extremely slow
-  # WildcardQueries, a Wildcard term should not start with one of the
-  # wildcards +*+ or +?+.
-  # 
-  # See WildcardTermEnum
-  class WildcardQuery < MultiTermQuery 
-    def initialize(term) 
-      super(term)
-    end
-
-    def get_term_enum(reader)
-      return WildcardTermEnum.new(reader, @term)
-    end
-
-    def eql?(o) 
-      if o.instance_of?(WildcardQuery)
-        return super(o)
-      end
-      return false
-    end
-  end
-end

data/lib/ferret/search/wildcard_term_enum.rb

@@ -1,61 +0,0 @@
-module Ferret::Search
-  # Subclass of FilteredTermEnum for enumerating all terms that match the
-  # specified wildcard filter term.
-  # 
-  # Term enumerations are always ordered by Term.compareTo().  Each term in
-  # the enumeration is greater than all that precede it.
-  # 
-  class WildcardTermEnum < FilteredTermEnum 
-    include Ferret::Index
-
-    attr_reader :end_enum
-
-    WILDCARD_STRING = '*'
-    WILDCARD_CHAR = '?'
-
-    # Creates a new +WildcardTermEnum+.  Passing in a
-    # org.apache.lucene.index.Term Term that does not contain a
-    # +WILDCARD_CHAR+ will cause an exception to be raisen.
-    # 
-    # After calling the constructor the enumeration is already pointing to the first 
-    # valid term if such a term exists.
-    def initialize(reader, term)
-      super()
-      @end_enum = false
-      @search_term = term
-      @field = @search_term.field
-      text = @search_term.text
-      len = text.length
-
-      sidx = text.index(WILDCARD_STRING)||len
-      cidx = text.index(WILDCARD_CHAR)||len
-      idx = [sidx, cidx].min
-
-      @pre = @search_term.text[0,idx]
-      @pre_len = idx
-      @pattern = /^#{Regexp.escape(text[idx..-1]).gsub(/\\([?*])/){".#{$1}"}}$/
-      self.enum = reader.terms_from(Term.new(@search_term.field, @pre))
-    end
-
-    def term_compare(term) 
-      if (@field == term.field) 
-        search_text = term.text
-        if (search_text[0, @pre_len] == @pre) 
-          return (search_text[@pre_len..-1] =~ @pattern)
-        end
-      end
-      @end_enum = true
-      return false
-    end
-
-    def difference() 
-      return 1.0
-    end
-
-    def close()
-      super()
-      @pattern = nil
-      @field = nil
-    end
-  end
-end

data/lib/ferret/stemmers.rb

@@ -1 +0,0 @@
-require 'ferret/stemmers/porter_stemmer'

data/lib/ferret/stemmers/porter_stemmer.rb

@@ -1,218 +0,0 @@
-#!/usr/bin/env ruby
-#
-# $Id: PorterStemmer.rb,v 1.1.1.1 2004/04/17 13:55:20 pragdave Exp $
-#
-# See example usage at the end of this file.
-#
-
-module Stemmable
-
-  STEMMED = {}
-
-  STEP_2_LIST = {
-    'ational'=>'ate', 'tional'=>'tion', 'enci'=>'ence', 'anci'=>'ance',
-    'izer'=>'ize', 'bli'=>'ble',
-    'alli'=>'al', 'entli'=>'ent', 'eli'=>'e', 'ousli'=>'ous',
-    'ization'=>'ize', 'ation'=>'ate',
-    'ator'=>'ate', 'alism'=>'al', 'iveness'=>'ive', 'fulness'=>'ful',
-    'ousness'=>'ous', 'aliti'=>'al',
-    'iviti'=>'ive', 'biliti'=>'ble', 'logi'=>'log'
-  }
-  
-  STEP_3_LIST = {
-    'icate'=>'ic', 'ative'=>'', 'alize'=>'al', 'iciti'=>'ic',
-    'ical'=>'ic', 'ful'=>'', 'ness'=>''
-  }
-
-
-  SUFFIX_1_REGEXP = /(
-                    ational  |
-                    tional   |
-                    enci     |
-                    anci     |
-                    izer     |
-                    bli      |
-                    alli     |
-                    entli    |
-                    eli      |
-                    ousli    |
-                    ization  |
-                    ation    |
-                    ator     |
-                    alism    |
-                    iveness  |
-                    fulness  |
-                    ousness  |
-                    aliti    |
-                    iviti    |
-                    biliti   |
-                    logi)$/x
-
-
-  SUFFIX_2_REGEXP = /(
-                      al       |
-                      ance     |
-                      ence     |
-                      er       |
-                      ic       | 
-                      able     |
-                      ible     |
-                      ant      |
-                      ement    |
-                      ment     |
-                      ent      |
-                      ou       |
-                      ism      |
-                      ate      |
-                      iti      |
-                      ous      |
-                      ive      |
-                      ize)$/x
-
-
-  C = "[^aeiou]"         # consonant
-  V = "[aeiouy]"         # vowel
-  CC = "#{C}(?>[^aeiouy]*)"  # consonant sequence
-  VV = "#{V}(?>[aeiou]*)"    # vowel sequence
-
-  MGR0 = /^(#{CC})?#{VV}#{CC}/o                # [cc]vvcc... is m>0
-  MEQ1 = /^(#{CC})?#{VV}#{CC}(#{VV})?$/o       # [cc]vvcc[vv] is m=1
-  MGR1 = /^(#{CC})?#{VV}#{CC}#{VV}#{CC}/o      # [cc]vvccvvcc... is m>1
-  VOWEL_IN_STEM   = /^(#{CC})?#{V}/o                      # vowel in stem
-
-  #
-  # Porter stemmer in Ruby.
-  #
-  # This is the Porter stemming algorithm, ported to Ruby from the
-  # version coded up in Perl.  It's easy to follow against the rules
-  # in the original paper in:
-  #
-  #   Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
-  #   no. 3, pp 130-137,
-  #
-  # See also http://www.tartarus.org/~martin/PorterStemmer
-  #
-  # Send comments to raypereda@hotmail.com
-  #
-  
-  def stem_porter(w = self.to_str.dup)
-
-    # make a copy of the given object and convert it to a string.
-    original_word = w
-    
-    return w if w.length < 3
-
-    result = STEMMED[w]
-    return result if result
-
-    # now map initial y to Y so that the patterns never treat it as vowel
-    w[0] = 'Y' if w[0] == ?y
-    
-    # Step 1a
-    if w =~ /(ss|i)es$/
-      w = $` + $1
-    elsif w =~ /([^s])s$/ 
-      w = $` + $1
-    end
-
-    # Step 1b
-    if w =~ /eed$/
-      w.chop! if $` =~ MGR0 
-    elsif w =~ /(ed|ing)$/
-      stem = $`
-      if stem =~ VOWEL_IN_STEM 
-        w = stem
-        case w
-        when /(at|bl|iz)$/             then w << "e"
-        when /([^aeiouylsz])\1$/       then w.chop!
-        when /^#{CC}#{V}[^aeiouwxy]$/o then w << "e"
-        end
-      end
-    end
-
-    if w =~ /y$/ 
-      stem = $`
-      w = stem + "i" if stem =~ VOWEL_IN_STEM 
-    end
-
-    # Step 2
-    if w =~ SUFFIX_1_REGEXP
-      stem = $`
-      suffix = $1
-      # print "stem= " + stem + "\n" + "suffix=" + suffix + "\n"
-      if stem =~ MGR0
-        w = stem + STEP_2_LIST[suffix]
-      end
-    end
-
-    # Step 3
-    if w =~ /(icate|ative|alize|iciti|ical|ful|ness)$/
-      stem = $`
-      suffix = $1
-      if stem =~ MGR0
-        w = stem + STEP_3_LIST[suffix]
-      end
-    end
-
-    # Step 4
-    if w =~ SUFFIX_2_REGEXP
-      stem = $`
-      if stem =~ MGR1
-        w = stem
-      end
-    elsif w =~ /(s|t)(ion)$/
-      stem = $` + $1
-      if stem =~ MGR1
-        w = stem
-      end
-    end
-
-    #  Step 5
-    if w =~ /e$/ 
-      stem = $`
-      if (stem =~ MGR1) ||
-          (stem =~ MEQ1 && stem !~ /^#{CC}#{V}[^aeiouwxy]$/o)
-        w = stem
-      end
-    end
-
-    if w =~ /ll$/ && w =~ MGR1
-      w.chop!
-    end
-
-    # and turn initial Y back to y
-    w[0] = 'y' if w[0] == ?Y
-
-    STEMMED[original_word] = w
-
-    w
-  end
-
-
-  module_function :stem_porter
-  #
-  # make the stem_porter the default stem method, just in case we
-  # feel like having multiple stemmers available later.
-  #
-  alias stem stem_porter
-  public :stem
-
-end
-
-
-#
-# Make this script executable, and send it words on stdin, one per
-# line, and it will output the stemmed versions to stdout.
-#
-if $0 == __FILE__ then
-  class String
-    include Stemmable
-  end
-
-  # the String class, and any subclasses of it you might have, now know
-  # how to stem things.
-
-  $stdin.each do |word|
-    puts word.strip.stem
-  end
-end