ferret 0.9.6 → 0.10.0

data/lib/ferret/search/filter.rb

@@ -1,11 +0,0 @@
-module Ferret::Search
-  # Abstract base class providing a mechanism to restrict searches to a subset
-  # of an index. 
-  class Filter
-    # Returns a BitSet with true for documents which should be permitted in
-    # search results, and false for those that should not. 
-    def bits(reader)
-      raise NotImplementedError
-    end
-  end
-end

data/lib/ferret/search/filtered_query.rb

@@ -1,130 +0,0 @@
-module Ferret::Search
-  # A query that applies a filter to the results of another query.
-  # 
-  # Note: the bits are retrieved from the filter each time this
-  # query is used in a search - use a CachingWrapperFilter to avoid
-  # regenerating the bits every time.
-  class FilteredQuery < Query 
-    attr_accessor :sub_query
-    attr_reader :filter
-
-    # Constructs a new query which applies a filter to the results of the
-    # original query.
-    #
-    # Filter.bits() will be called every time this query is used in a search.
-    #
-    # query::  Query to be filtered, cannot be +nil+.
-    # filter:: Filter to apply to query results, cannot be +nil+.
-    def initialize(query, filter) 
-      super()
-      @sub_query = query
-      @filter = filter
-    end
-
-    # Returns a Weight that applies the filter to the enclosed query's Weight.
-    # This is accomplished by overriding the Scorer returned by the Weight.
-    def create_weight(searcher)
-      sub_weight = @sub_query.create_weight(searcher)
-      similarity = @sub_query.similarity(searcher)
-      return FilteredWeight.new(self, sub_weight, similarity) 
-    end
-
-    class FilteredScorer < Scorer
-      def initialize(sub_scorer, bits, similarity)
-        super(similarity)
-        @sub_scorer = sub_scorer
-        @bits = bits
-      end
- 
-      # pass these methods through to the enclosed scorer
-      def next?() return @sub_scorer.next?; end
-      def doc() return @sub_scorer.doc; end
-      def skip_to(i) return @sub_scorer.skip_to(i); end
-
-      # if the document has been filtered out, set score to 0.0
-      def score()
-        return (@bits.get(@sub_scorer.doc) ? @sub_scorer.score() : 0.0)
-      end
-
-      # add an explanation about whether the document was filtered
-      def explain(i)
-        exp = @sub_scorer.explain(i)
-        if (@bits.get(i))
-          exp.description = "allowed by filter: #{exp.description}"
-        else
-          exp.description = "removed by filter: #{exp.description}"
-        end
-        return exp
-      end
-    end
-
-    class FilteredWeight < Weight
-      attr_reader :query
-
-      def initialize(query, sub_weight, similarity)
-        @query = query
-        @sub_weight = sub_weight
-        @similarity = similarity
-      end
-
-      # pass these methods through to enclosed query's weight
-      def value()
-        return @sub_weight.value
-      end
-
-      def sum_of_squared_weights()
-        return @sub_weight.sum_of_squared_weights
-      end
-
-      def normalize(v)
-        return @sub_weight.normalize(v)
-      end
-
-      def explain(ir, i)
-        return @sub_weight.explain(ir, i)
-      end
-
-      # return a scorer that overrides the enclosed query's score if
-      # the given hit has been filtered out.
-      def scorer(reader)
-        scorer = @sub_weight.scorer(reader)
-        bits = @query.filter.bits(reader)
-        return FilteredScorer.new(scorer, bits, @similarity) 
-      end
-    end
-
-    # Rewrites the wrapped query. 
-    def rewrite(reader)
-      rewritten = @sub_query.rewrite(reader)
-      if (rewritten != @sub_query) 
-        clone = self.clone()
-        clone.query = rewritten
-        return clone
-      else 
-        return self
-      end
-    end
-
-    # inherit javadoc
-    def extract_terms(terms) 
-      @sub_query.extract_terms(terms)
-    end
-
-    # Prints a user-readable version of this query. 
-    def to_s(f = nil) 
-      return "filtered(#{@sub_query.to_s(f)})->#{@filter}"
-    end
-
-    # Returns true iff +o+ is equal to this. 
-    def eql?(o) 
-      return (o.instance_of?(FilteredQuery) and 
-        (@sub_query == o.sub_query) and (@filter == o.filter))
-    end
-    alias :== :eql?
-
-    # Returns a hash code value for this object. 
-    def hash() 
-      return @sub_query.hash ^ @filter.hash
-    end
-  end
-end

data/lib/ferret/search/filtered_term_enum.rb

@@ -1,79 +0,0 @@
-module Ferret::Search
-
-  # Abstract class for enumerating a subset of all terms. 
-  #
-  # Term enumerations are always ordered by Term.<=>().  Each term in
-  # the enumeration is greater than all that precede it.  
-  class FilteredTermEnum < Ferret::Index::TermEnum 
-      
-    # Returns the current Term in the enumeration.
-    # Returns nil if no Term matches or all terms have been enumerated. 
-    attr_reader :term
-      
-    def initialize()
-      @term = nil
-      @enum = nil
-      @reader = nil
-    end
-
-    # Equality compare on the term 
-    def term_compare(term)
-      raise NotImplementedError
-    end
-      
-    # Equality measure on the term 
-    def difference()
-      raise NotImplementedError
-    end
-
-    # Indiciates the end of the enumeration has been reached 
-    def end_enum()
-      raise NotImplementedError
-    end
-      
-    def enum=(enum)
-      @enum = enum
-      # Find the first term that matches
-      term = @enum.term()
-      if (term != nil and term_compare(term)) 
-        @term = term
-      else
-        next?
-      end
-    end
-      
-    # Returns the doc_freq of the current Term in the enumeration.
-    # Returns -1 if no Term matches or all terms have been enumerated.
-    def doc_freq() 
-      if (@enum == nil)
-        return -1
-      end
-      return @enum.doc_freq()
-    end
-    
-    # Increments the enumeration to the next element.  True if one exists. 
-    def next?()
-      return false if (@enum == nil) # enum not initialized
-      @term = nil
-      while @term.nil? 
-        if end_enum() or ! @enum.next?
-          return false
-        end
-        term = @enum.term()
-        if (term_compare(term)) 
-          @term = term
-          return true
-        end
-      end
-      @term = nil
-      return false
-    end
-
-    # Closes the enumeration to further activity, freeing resources.  
-    def close()
-      @enum.close()
-      @term = nil
-      @enum = nil
-    end
-  end
-end

data/lib/ferret/search/fuzzy_query.rb

@@ -1,154 +0,0 @@
-module Ferret::Search
-  # Implements the fuzzy search query. The similiarity measurement
-  # is based on the Levenshtein (distance) algorithm.
-  class FuzzyQuery < MultiTermQuery 
-    @@default_min_similarity = 0.5
-    @@default_prefix_length = 0
-    
-    def FuzzyQuery.default_min_similarity()
-      return @@default_min_similarity
-    end
-
-    def FuzzyQuery.default_min_similarity=(minimum_similarity)
-      if (minimum_similarity >= 1.0)
-        raise ArgumentError, "minimum_similarity cannot be greater than or equal to 1"
-      elsif (minimum_similarity < 0.0)
-        raise ArgumentError, "minimum_similarity cannot be less than 0"
-      end
-      @@default_min_similarity = minimum_similarity
-    end
-   
-    def FuzzyQuery.default_prefix_length()
-      return @@default_prefix_length
-    end
-
-    def FuzzyQuery.default_prefix_length=(prefix_length)
-      if (prefix_length < 0)
-        raise ArgumentError, "prefix_length cannot be less than 0"
-      end
-      @@default_prefix_length = prefix_length
-    end
-   
-   
-    attr_reader :prefix_length, :minimum_similarity
-    # Create a new FuzzyQuery that will match terms with a similarity 
-    # of at least +minimum_similarity+ to +term+.
-    # If a +prefix_length+ > 0 is specified, a common prefix
-    # of that length is also required.
-    # 
-    # term::               the term to search for
-    # minimum_similarity:: a value between 0 and 1 to set the required
-    #                      similarity between the query term and the matching
-    #                      terms. For example, for a +minimum_similarity+ of
-    #                      <tt>0.5</tt> a term of the same length as the query
-    #                      term is considered similar to the query term if the
-    #                      edit distance between both terms is less than
-    #                      <tt>length(term)*0.5</tt>
-    # prefix_length::      length of common (non-fuzzy) prefix. This is the
-    #                      number of characters at the start of a term that
-    #                      must be identical (fuzzy) to the query term if the
-    #                      query is to match that term. 
-    # raises::             ArgumentError if minimum_similarity is >= 1 or < 0
-    #                      or if prefix_length < 0
-    def initialize(term,
-                   minimum_similarity = @@default_min_similarity,
-                   prefix_length = @@default_prefix_length)
-      super(term)
-      
-      if (minimum_similarity >= 1.0)
-        raise ArgumentError, "minimum_similarity >= 1"
-      elsif (minimum_similarity < 0.0)
-        raise ArgumentError, "minimum_similarity < 0"
-      end
-
-      if (prefix_length < 0)
-        raise ArgumentError, "prefix_length < 0"
-      end
-      
-      @minimum_similarity = minimum_similarity
-      @prefix_length = prefix_length
-    end
-    
-    def get_term_enum(reader)
-      return FuzzyTermEnum.new(reader, @term, @minimum_similarity, @prefix_length)
-    end
-    
-    def rewrite(reader)
-
-      fuzzy_enum = get_term_enum(reader)
-      max_clause_count = BooleanQuery.max_clause_count
-      st_queue = ScoreTermQueue.new(max_clause_count)
-
-      begin 
-        begin 
-          min_score = 0.0
-          score = 0.0
-          t = fuzzy_enum.term()
-          if t
-            score = fuzzy_enum.difference()
-
-            # terms come in alphabetical order, therefore if queue is full and score
-            # not bigger than min_score, we can skip
-            if(st_queue.size < max_clause_count or score > min_score)
-              st_queue.insert(ScoreTerm.new(t, score))
-              min_score = st_queue.top.score # maintain min_score
-            end
-          end
-        end while fuzzy_enum.next?
-      ensure 
-        fuzzy_enum.close()
-      end
-      
-      bq = BooleanQuery.new(true)
-      st_queue.size.times do |i|
-        st = st_queue.pop()
-        tq = TermQuery.new(st.term)                     # found a match
-        tq.boost = boost() * st.score                   # set the boost
-        bq.add_query(tq, BooleanClause::Occur::SHOULD)  # add to query
-      end
-
-      return bq
-    end
-      
-    def to_s(field = nil) 
-      buffer = ""
-      buffer << "#{@term.field}:" if @term.field != field
-      buffer << "#{@term.text}~"
-      buffer << minimum_similarity.to_s if minimum_similarity != 0.5
-      buffer << "^#{boost()}" if (boost() != 1.0) 
-      return buffer
-    end
-    
-    class ScoreTerm
-      attr_accessor :term, :score
-      
-      def initialize(term, score)
-        @term = term
-        @score = score
-      end
-    end
-    
-    class ScoreTermQueue < Ferret::Utils::PriorityQueue 
-      
-      # See PriorityQueue#less_than(o1, o2)
-      def less_than(st1, st2) 
-        if (st1.score == st1.score)
-          return st1.term > st2.term
-        else
-          return st1.score < st2.score
-        end
-      end
-    end
-
-    def eql?(o) 
-      return (o.instance_of?(FuzzyQuery) and super(o) and
-              (@minimum_similarity == o.minimum_similarity) and
-              (@prefix_length == fuzzyQuery.prefix_length))
-    end
-    alias :== :eql?
-
-    def hash() 
-      return super ^ @minimum_similarity.hash ^ @prefix_length.hash
-    end
-  end
-end

data/lib/ferret/search/fuzzy_term_enum.rb

@@ -1,247 +0,0 @@
-require 'monitor'
-
-module Ferret::Search
-  # Subclass of FilteredTermEnum for enumerating all terms that are similiar
-  # to the specified filter term.
-  # 
-  # Term enumerations are always ordered by Term.compareTo().  Each term in
-  # the enumeration is greater than all that precede it.
-  class FuzzyTermEnum < FilteredTermEnum 
-    include MonitorMixin
-
-    include Ferret::Index
-    attr_reader :end_enum
-
-    # This should be somewhere around the average long word.
-    # If it is longer, we waste time and space. If it is shorter, we waste a
-    # little bit of time growing the array as we encounter longer words.
-    TYPICAL_LONGEST_WORD_IN_INDEX = 19
-
-    # Constructor for enumeration of all terms from specified +reader+ which
-    # share a prefix of length +prefix_length+ with +term+ and which have a
-    # fuzzy similarity > +min_similarity+.
-    # 
-    # After calling the constructor the enumeration is already pointing to the
-    # first valid term if such a term exists. 
-    # 
-    # reader:: Delivers terms.
-    # term:: Pattern term.
-    # min_similarity:: Minimum required similarity for terms from the reader.
-    # Default value is 0.5.
-    # prefix_length:: Length of required common prefix. Default value is 0.
-    def initialize(reader, term,
-                   minimum_similarity = FuzzyQuery.default_min_similarity,
-                   prefix_length = FuzzyQuery.default_prefix_length)
-      super()
-      
-      @reader = reader
-      @end_enum = false
-      @max_distances = Array.new(TYPICAL_LONGEST_WORD_IN_INDEX)
-
-      
-      if (minimum_similarity >= 1.0)
-        raise ArgumentError, "minimum_similarity cannot be greater than or equal to 1"
-      elsif (minimum_similarity < 0.0)
-        raise ArgumentError, "minimum_similarity cannot be less than 0"
-      end
-      if(prefix_length < 0)
-        raise ArgumentError, "prefix_length cannot be less than 0"
-      end
-
-      @minimum_similarity = minimum_similarity
-      @scale_factor = 1.0 / (1.0 - @minimum_similarity)
-      @search_term = term
-      @field = @search_term.field
-
-      # The prefix could be longer than the word.
-      # It's kind of silly though.  It means we must match the entire word.
-      term_length = @search_term.text.length
-      if prefix_length > term_length 
-        @prefix_length = term_length 
-      else
-        @prefix_length = prefix_length
-      end
-
-      @text = @search_term.text[@prefix_length..-1]
-      @prefix = @search_term.text[0, @prefix_length]
-
-      initialize_max_distances()
-
-      # Allows us save time required to create a new array
-      # everytime similarity is called.
-      @d = init_distance_array()
-
-      self.enum = reader.terms_from(Term.new(@search_term.field, @prefix))
-    end
-
-    # The term_compare method in FuzzyTermEnum uses Levenshtein distance to 
-    # calculate the distance between the given term and the comparing term. 
-    def term_compare(term) 
-      if (@field == term.field and term.text[0, @prefix_length] == @prefix) 
-        target = term.text[@prefix_length..-1]
-        @similarity = similarity(target)
-        return (@similarity > @minimum_similarity)
-      end
-      @end_enum = true
-      return false
-    end
-    
-    def difference() 
-      return  (@scale_factor * (@similarity - @minimum_similarity))
-    end
-    
-    # ****************************
-    # Compute Levenshtein distance
-    # ****************************
-    
-    # Finds and returns the smallest of three integers 
-    def min(a, b, c) 
-      t = (a < b) ? a : b
-      return (t < c) ? t : c
-    end
-
-    def init_distance_array()
-      return Array.new(@text.length() + 1) {Array.new(TYPICAL_LONGEST_WORD_IN_INDEX)}
-    end
-
-    # Similarity returns a number that is 1.0 or less (including negative
-    # numbers) based on how similar the Term is compared to a target term.  It
-    # returns exactly 0.0 when
-    # 
-    #    edit_distance < maximum_edit_distance
-    #
-    # Otherwise it returns:
-    #
-    #    1 - (edit_distance / length)
-    #
-    # where length is the length of the shortest term (text or target)
-    # including a prefix that are identical and edit_distance is the
-    # Levenshtein distance for the two words.
-    # 
-    # Embedded within this algorithm is a fail-fast Levenshtein distance
-    # algorithm.  The fail-fast algorithm differs from the standard
-    # Levenshtein distance algorithm in that it is aborted if it is discovered
-    # that the mimimum distance between the words is greater than some
-    # threshold.
-    # 
-    # To calculate the maximum distance threshold we use the following formula:
-    #
-    #    (1 - minimum_similarity) * length
-    #
-    # where length is the shortest term including any prefix that is not part
-    # of the similarity comparision.  This formula was derived by solving for
-    # what maximum value of distance returns false for the following
-    # statements:
-    # 
-    #    similarity = 1 - (distance / (prefix_length + [textlen, targetlen].min))
-    #    return (similarity > minimum_similarity)
-    #
-    # where distance is the Levenshtein distance for the two words.
-    # 
-    # Levenshtein distance (also known as edit distance) is a measure of
-    # similiarity between two strings where the distance is measured as the
-    # number of character deletions, insertions or substitutions required to
-    # transform one string to the other string.
-    #
-    # target:: the target word or phrase
-    # returns:: the similarity,  0.0 or less indicates that it matches less
-    #    than the required threshold and 1.0 indicates that the text and
-    #    target are identical
-    def similarity(target) 
-      synchronize do
-        m = target.length
-        n = @text.length
-
-        if (n == 0)  
-          # we don't have anything to compare.  That means if we just add the
-          # letters for m we get the new word
-          return (@prefix_length == 0) ? 0.0 : 1.0 - (m.to_f / @prefix_length)
-        end
-        if (m == 0) 
-          return (@prefix_length == 0) ? 0.0 : 1.0 - (n.to_f / @prefix_length)
-        end
-
-        max_distance = max_distance(m)
-
-        if (max_distance < (m-n).abs) 
-          #just adding the characters of m to n or vice-versa results in
-          #too many edits
-          #for example "pre" length is 3 and "prefixes" length is 8.  We can see that
-          #given this optimal circumstance, the edit distance cannot be less than 5.
-          #which is 8-3 or more precisesly Math.abs(3-8).
-          #if our maximum edit distance is 4, then we can discard this word
-          #without looking at it.
-          return 0.0
-        end
-
-        #let's make sure we have enough room in our array to do the distance calculations.
-        if (@d[0].length <= m) 
-          grow_distance_array(m)
-        end
-
-        # init matrix d
-        (n+1).times {|i| @d[i][0] = i}
-        (m+1).times {|j| @d[0][j] = j}
-        
-        # start computing edit distance
-        1.upto(n) do |i|
-          best_possible_edit_distance = m
-          s_i = @text[i-1]
-          1.upto(m) do |j|
-            if (s_i != target[j-1]) 
-              @d[i][j] = min(@d[i-1][j], @d[i][j-1], @d[i-1][j-1])+1
-            else 
-              @d[i][j] = min(@d[i-1][j]+1, @d[i][j-1]+1, @d[i-1][j-1])
-            end
-            if @d[i][j] < best_possible_edit_distance
-              best_possible_edit_distance = @d[i][j]
-            end
-          end
-
-          # After calculating row i, the best possible edit distance can be
-          # found by found by finding the smallest value in a given column.
-          # If the best_possible_edit_distance is greater than the max distance,
-          # abort.
-          if (i > max_distance and best_possible_edit_distance > max_distance)
-            # equal is okay, but not greater
-            # the closest the target can be to the text is just too far away.
-            # this target is leaving the party early.
-            return 0.0
-          end
-        end
-
-        # this will return less than 0.0 when the edit distance is
-        # greater than the number of characters in the shorter word.
-        # but this was the formula that was previously used in FuzzyTermEnum,
-        # so it has not been changed (even though minimum_similarity must be
-        # greater than 0.0)
-        return 1.0 - (@d[n][m].to_f / (@prefix_length + (n < m ? n : m)))
-      end
-    end
-
-    # Grow the second dimension of the array, so that we can calculate the
-    # Levenshtein difference.
-    def grow_distance_array(m) 
-      @d = @d.map {Array.new(m+1)}
-    end
-
-    # The max Distance is the maximum Levenshtein distance for the text
-    # compared to some other value that results in score that is
-    # better than the minimum similarity.
-    # m:: the length of the "other value"
-    # returns:: the maximum levenshtein distance that we care about
-    def max_distance(m) 
-      return @max_distances[m] ||= calculate_max_distance(m)
-    end
-
-    def initialize_max_distances() 
-      @max_distances.length.times do |i|
-        @max_distances[i] = calculate_max_distance(i)
-      end
-    end
-    
-    def calculate_max_distance(m) 
-      return ((1-@minimum_similarity) * ([@text.length, m].min + @prefix_length))
-    end
-  end
-end