ferret 0.1.0

data/lib/ferret/search/filter.rb

@@ -0,0 +1,11 @@
+module Ferret::Search
+  # Abstract base class providing a mechanism to restrict searches to a subset
+  # of an index. 
+  class Filter
+    # Returns a BitSet with true for documents which should be permitted in
+    # search results, and false for those that should not. 
+    def bits(reader)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/search/filtered_query.rb

@@ -0,0 +1,130 @@
+module Ferret::Search
+  # A query that applies a filter to the results of another query.
+  # 
+  # Note: the bits are retrieved from the filter each time this
+  # query is used in a search - use a CachingWrapperFilter to avoid
+  # regenerating the bits every time.
+  class FilteredQuery < Query 
+    attr_accessor :sub_query
+    attr_reader :filter
+
+    # Constructs a new query which applies a filter to the results of the
+    # original query.
+    #
+    # Filter.bits() will be called every time this query is used in a search.
+    #
+    # query::  Query to be filtered, cannot be +nil+.
+    # filter:: Filter to apply to query results, cannot be +nil+.
+    def initialize(query, filter) 
+      super()
+      @sub_query = query
+      @filter = filter
+    end
+
+    # Returns a Weight that applies the filter to the enclosed query's Weight.
+    # This is accomplished by overriding the Scorer returned by the Weight.
+    def create_weight(searcher)
+      sub_weight = @sub_query.create_weight(searcher)
+      similarity = @sub_query.similarity(searcher)
+      return FilteredWeight.new(self, sub_weight, similarity) 
+    end
+
+    class FilteredScorer < Scorer
+      def initialize(sub_scorer, bits, similarity)
+        super(similarity)
+        @sub_scorer = sub_scorer
+        @bits = bits
+      end
+ 
+      # pass these methods through to the enclosed scorer
+      def next?() return @sub_scorer.next?; end
+      def doc() return @sub_scorer.doc; end
+      def skip_to(i) return @sub_scorer.skip_to(i); end
+
+      # if the document has been filtered out, set score to 0.0
+      def score()
+        return (@bits.get(@sub_scorer.doc) ? @sub_scorer.score() : 0.0)
+      end
+
+      # add an explanation about whether the document was filtered
+      def explain(i)
+        exp = @sub_scorer.explain(i)
+        if (@bits.get(i))
+          exp.description = "allowed by filter: #{exp.description}"
+        else
+          exp.description = "removed by filter: #{exp.description}"
+        end
+        return exp
+      end
+    end
+
+    class FilteredWeight < Weight
+      attr_reader :query
+
+      def initialize(query, sub_weight, similarity)
+        @query = query
+        @sub_weight = sub_weight
+        @similarity = similarity
+      end
+
+      # pass these methods through to enclosed query's weight
+      def value()
+        return @sub_weight.value
+      end
+
+      def sum_of_squared_weights()
+        return @sub_weight.sum_of_squared_weights
+      end
+
+      def normalize(v)
+        return @sub_weight.normalize(v)
+      end
+
+      def explain(ir, i)
+        return @sub_weight.explain(ir, i)
+      end
+
+      # return a scorer that overrides the enclosed query's score if
+      # the given hit has been filtered out.
+      def scorer(reader)
+        scorer = @sub_weight.scorer(reader)
+        bits = @query.filter.bits(reader)
+        return FilteredScorer.new(scorer, bits, @similarity) 
+      end
+    end
+
+    # Rewrites the wrapped query. 
+    def rewrite(reader)
+      rewritten = @sub_query.rewrite(reader)
+      if (rewritten != @sub_query) 
+        clone = self.clone()
+        clone.query = rewritten
+        return clone
+      else 
+        return self
+      end
+    end
+
+    # inherit javadoc
+    def extract_terms(terms) 
+      @sub_query.extract_terms(terms)
+    end
+
+    # Prints a user-readable version of this query. 
+    def to_s(f = nil) 
+      return "filtered(#{@sub_query.to_s(f)})->#{@filter}"
+    end
+
+    # Returns true iff +o+ is equal to this. 
+    def eql?(o) 
+      return (o.instance_of?(FilteredQuery) and 
+        (@sub_query == o.sub_query) and (@filter == o.filter))
+    end
+    alias :== :eql?
+
+    # Returns a hash code value for this object. 
+    def hash() 
+      return @sub_query.hash ^ @filter.hash
+    end
+  end
+end

data/lib/ferret/search/filtered_term_enum.rb

@@ -0,0 +1,79 @@
+module Ferret::Search
+
+  # Abstract class for enumerating a subset of all terms. 
+  #
+  # Term enumerations are always ordered by Term.<=>().  Each term in
+  # the enumeration is greater than all that precede it.  
+  class FilteredTermEnum < Ferret::Index::TermEnum 
+      
+    # Returns the current Term in the enumeration.
+    # Returns nil if no Term matches or all terms have been enumerated. 
+    attr_reader :term
+      
+    def initialize()
+      @term = nil
+      @enum = nil
+      @reader = nil
+    end
+
+    # Equality compare on the term 
+    def term_compare(term)
+      raise NotImplementedError
+    end
+      
+    # Equality measure on the term 
+    def difference()
+      raise NotImplementedError
+    end
+
+    # Indiciates the end of the enumeration has been reached 
+    def end_enum()
+      raise NotImplementedError
+    end
+      
+    def enum=(enum)
+      @enum = enum
+      # Find the first term that matches
+      term = @enum.term()
+      if (term != nil and term_compare(term)) 
+        @term = term
+      else
+        next?
+      end
+    end
+      
+    # Returns the doc_freq of the current Term in the enumeration.
+    # Returns -1 if no Term matches or all terms have been enumerated.
+    def doc_freq() 
+      if (@enum == nil)
+        return -1
+      end
+      return @enum.doc_freq()
+    end
+    
+    # Increments the enumeration to the next element.  True if one exists. 
+    def next?()
+      return false if (@enum == nil) # enum not initialized
+      @term = nil
+      while @term.nil? 
+        if end_enum() or ! @enum.next?
+          return false
+        end
+        term = @enum.term()
+        if (term_compare(term)) 
+          @term = term
+          return true
+        end
+      end
+      @term = nil
+      return false
+    end
+
+    # Closes the enumeration to further activity, freeing resources.  
+    def close()
+      @enum.close()
+      @term = nil
+      @enum = nil
+    end
+  end
+end

data/lib/ferret/search/fuzzy_query.rb

@@ -0,0 +1,153 @@
+module Ferret::Search
+  # Implements the fuzzy search query. The similiarity measurement
+  # is based on the Levenshtein (distance) algorithm.
+  class FuzzyQuery < MultiTermQuery 
+    @@default_min_similarity = 0.5
+    @@default_prefix_length = 0
+    
+    def FuzzyQuery.default_min_similarity()
+      return @@default_min_similarity
+    end
+
+    def FuzzyQuery.default_min_similarity=(minimum_similarity)
+      if (minimum_similarity >= 1.0)
+        raise ArgumentError, "minimum_similarity cannot be greater than or equal to 1"
+      elsif (minimum_similarity < 0.0)
+        raise ArgumentError, "minimum_similarity cannot be less than 0"
+      end
+      @@default_min_similarity = minimum_similarity
+    end
+   
+    def FuzzyQuery.default_prefix_length()
+      return @@default_prefix_length
+    end
+
+    def FuzzyQuery.default_prefix_length=(prefix_length)
+      if (prefix_length < 0)
+        raise ArgumentError, "prefix_length cannot be less than 0"
+      end
+      @@default_prefix_length = prefix_length
+    end
+   
+   
+    attr_reader :prefix_length, :minimum_similarity
+    # Create a new FuzzyQuery that will match terms with a similarity 
+    # of at least +minimum_similarity+ to +term+.
+    # If a +prefix_length+ > 0 is specified, a common prefix
+    # of that length is also required.
+    # 
+    # term::               the term to search for
+    # minimum_similarity:: a value between 0 and 1 to set the required
+    #                      similarity between the query term and the matching
+    #                      terms. For example, for a +minimum_similarity+ of
+    #                      <tt>0.5</tt> a term of the same length as the query
+    #                      term is considered similar to the query term if the
+    #                      edit distance between both terms is less than
+    #                      <tt>length(term)*0.5</tt>
+    # prefix_length::      length of common (non-fuzzy) prefix. This is the
+    #                      number of characters at the start of a term that
+    #                      must be identical (fuzzy) to the query term if the
+    #                      query is to match that term. 
+    # raises::             ArgumentError if minimum_similarity is >= 1 or < 0
+    #                      or if prefix_length < 0
+    def initialize(term,
+                   minimum_similarity = @@default_min_similarity,
+                   prefix_length = @@default_prefix_length)
+      super(term)
+      
+      if (minimum_similarity >= 1.0)
+        raise ArgumentError, "minimum_similarity >= 1"
+      elsif (minimum_similarity < 0.0)
+        raise ArgumentError, "minimum_similarity < 0"
+      end
+
+      if (prefix_length < 0)
+        raise ArgumentError, "prefix_length < 0"
+      end
+      
+      @minimum_similarity = minimum_similarity
+      @prefix_length = prefix_length
+    end
+    
+    def get_term_enum(reader)
+      return FuzzyTermEnum.new(reader, @term, @minimum_similarity, @prefix_length)
+    end
+    
+    def rewrite(reader)
+
+      fuzzy_enum = get_term_enum(reader)
+      max_clause_count = BooleanQuery.max_clause_count
+      st_queue = ScoreTermQueue.new(max_clause_count)
+
+      begin 
+        begin 
+          min_score = 0.0
+          score = 0.0
+          t = fuzzy_enum.term()
+          if t
+            score = fuzzy_enum.difference()
+
+            # terms come in alphabetical order, therefore if queue is full and score
+            # not bigger than min_score, we can skip
+            if(st_queue.size < max_clause_count or score > min_score)
+              st_queue.insert(ScoreTerm.new(t, score))
+              min_score = st_queue.top.score # maintain min_score
+            end
+          end
+        end while fuzzy_enum.next?
+      ensure 
+        fuzzy_enum.close()
+      end
+      
+      bq = BooleanQuery.new(true)
+      st_queue.size.times do |i|
+        st = st_queue.pop()
+        tq = TermQuery.new(st.term)                     # found a match
+        tq.boost = boost() * st.score                   # set the boost
+        bq.add_query(tq, BooleanClause::Occur::SHOULD)  # add to query
+      end
+
+      return bq
+    end
+      
+    def to_s(field = nil) 
+      buffer = ""
+      buffer << "#{@term.field}:" if @term.field != field
+      buffer << "#{@term.text}~#{minimum_similarity}"
+      buffer << "^#{boost()}" if (boost() != 1.0) 
+      return buffer
+    end
+    
+    class ScoreTerm
+      attr_accessor :term, :score
+      
+      def initialize(term, score)
+        @term = term
+        @score = score
+      end
+    end
+    
+    class ScoreTermQueue < Ferret::Utils::PriorityQueue 
+      
+      # See PriorityQueue#less_than(o1, o2)
+      def less_than(st1, st2) 
+        if (st1.score == st1.score)
+          return st1.term > st2.term
+        else
+          return st1.score < st2.score
+        end
+      end
+    end
+
+    def eql?(o) 
+      return (o.instance_of?(FuzzyQuery) and super(o) and
+              (@minimum_similarity == o.minimum_similarity) and
+              (@prefix_length == fuzzyQuery.prefix_length))
+    end
+    alias :== :eql?
+
+    def hash() 
+      return super ^ @minimum_similarity.hash ^ @prefix_length.hash
+    end
+  end
+end

data/lib/ferret/search/fuzzy_term_enum.rb

@@ -0,0 +1,244 @@
+module Ferret::Search
+  # Subclass of FilteredTermEnum for enumerating all terms that are similiar
+  # to the specified filter term.
+  # 
+  # Term enumerations are always ordered by Term.compareTo().  Each term in
+  # the enumeration is greater than all that precede it.
+  class FuzzyTermEnum < FilteredTermEnum 
+    include Ferret::Index
+    attr_reader :end_enum
+
+    # This should be somewhere around the average long word.
+    # If it is longer, we waste time and space. If it is shorter, we waste a
+    # little bit of time growing the array as we encounter longer words.
+    TYPICAL_LONGEST_WORD_IN_INDEX = 19
+
+    # Constructor for enumeration of all terms from specified +reader+ which
+    # share a prefix of length +prefix_length+ with +term+ and which have a
+    # fuzzy similarity > +min_similarity+.
+    # 
+    # After calling the constructor the enumeration is already pointing to the
+    # first valid term if such a term exists. 
+    # 
+    # reader:: Delivers terms.
+    # term:: Pattern term.
+    # min_similarity:: Minimum required similarity for terms from the reader.
+    # Default value is 0.5.
+    # prefix_length:: Length of required common prefix. Default value is 0.
+    def initialize(reader, term,
+                   minimum_similarity = FuzzyQuery.default_min_similarity,
+                   prefix_length = FuzzyQuery.default_prefix_length)
+      super()
+      
+      @reader = reader
+      @end_enum = false
+      @max_distances = Array.new(TYPICAL_LONGEST_WORD_IN_INDEX)
+
+      
+      if (minimum_similarity >= 1.0)
+        raise ArgumentError, "minimum_similarity cannot be greater than or equal to 1"
+      elsif (minimum_similarity < 0.0)
+        raise ArgumentError, "minimum_similarity cannot be less than 0"
+      end
+      if(prefix_length < 0)
+        raise ArgumentError, "prefix_length cannot be less than 0"
+      end
+
+      @minimum_similarity = minimum_similarity
+      @scale_factor = 1.0 / (1.0 - @minimum_similarity)
+      @search_term = term
+      @field = @search_term.field
+
+      # The prefix could be longer than the word.
+      # It's kind of silly though.  It means we must match the entire word.
+      term_length = @search_term.text.length
+      if prefix_length > term_length 
+        @prefix_length = term_length 
+      else
+        @prefix_length = prefix_length
+      end
+
+      @text = @search_term.text[@prefix_length..-1]
+      @prefix = @search_term.text[0, @prefix_length]
+
+      initialize_max_distances()
+
+      # Allows us save time required to create a new array
+      # everytime similarity is called.
+      @d = init_distance_array()
+
+      self.enum = reader.terms_from(Term.new(@search_term.field, @prefix))
+    end
+
+    # The term_compare method in FuzzyTermEnum uses Levenshtein distance to 
+    # calculate the distance between the given term and the comparing term. 
+    def term_compare(term) 
+      if (@field == term.field and term.text[0, @prefix_length] == @prefix) 
+        target = term.text[@prefix_length..-1]
+        @similarity = similarity(target)
+        return (@similarity > @minimum_similarity)
+      end
+      @end_enum = true
+      return false
+    end
+    
+    def difference() 
+      return  (@scale_factor * (@similarity - @minimum_similarity))
+    end
+    
+    # ****************************
+    # Compute Levenshtein distance
+    # ****************************
+    
+    # Finds and returns the smallest of three integers 
+    def min(a, b, c) 
+      t = (a < b) ? a : b
+      return (t < c) ? t : c
+    end
+
+    def init_distance_array()
+      return Array.new(@text.length() + 1) {Array.new(TYPICAL_LONGEST_WORD_IN_INDEX)}
+    end
+
+    # Similarity returns a number that is 1.0 or less (including negative
+    # numbers) based on how similar the Term is compared to a target term.  It
+    # returns exactly 0.0 when
+    # 
+    #    edit_distance < maximum_edit_distance
+    #
+    # Otherwise it returns:
+    #
+    #    1 - (edit_distance / length)
+    #
+    # where length is the length of the shortest term (text or target)
+    # including a prefix that are identical and edit_distance is the
+    # Levenshtein distance for the two words.
+    # 
+    # Embedded within this algorithm is a fail-fast Levenshtein distance
+    # algorithm.  The fail-fast algorithm differs from the standard
+    # Levenshtein distance algorithm in that it is aborted if it is discovered
+    # that the mimimum distance between the words is greater than some
+    # threshold.
+    # 
+    # To calculate the maximum distance threshold we use the following formula:
+    #
+    #    (1 - minimum_similarity) * length
+    #
+    # where length is the shortest term including any prefix that is not part
+    # of the similarity comparision.  This formula was derived by solving for
+    # what maximum value of distance returns false for the following
+    # statements:
+    # 
+    #    similarity = 1 - (distance / (prefix_length + [textlen, targetlen].min))
+    #    return (similarity > minimum_similarity)
+    #
+    # where distance is the Levenshtein distance for the two words.
+    # 
+    # Levenshtein distance (also known as edit distance) is a measure of
+    # similiarity between two strings where the distance is measured as the
+    # number of character deletions, insertions or substitutions required to
+    # transform one string to the other string.
+    #
+    # target:: the target word or phrase
+    # returns:: the similarity,  0.0 or less indicates that it matches less
+    #    than the required threshold and 1.0 indicates that the text and
+    #    target are identical
+    def similarity(target) 
+      m = target.length
+      n = @text.length
+
+      if (n == 0)  
+        # we don't have anything to compare.  That means if we just add the
+        # letters for m we get the new word
+        return (@prefix_length == 0) ? 0.0 : 1.0 - (m.to_f / @prefix_length)
+      end
+      if (m == 0) 
+        return (@prefix_length == 0) ? 0.0 : 1.0 - (n.to_f / @prefix_length)
+      end
+
+      max_distance = max_distance(m)
+
+      if (max_distance < (m-n).abs) 
+        #just adding the characters of m to n or vice-versa results in
+        #too many edits
+        #for example "pre" length is 3 and "prefixes" length is 8.  We can see that
+        #given this optimal circumstance, the edit distance cannot be less than 5.
+        #which is 8-3 or more precisesly Math.abs(3-8).
+        #if our maximum edit distance is 4, then we can discard this word
+        #without looking at it.
+        return 0.0
+      end
+
+      #let's make sure we have enough room in our array to do the distance calculations.
+      if (@d[0].length <= m) 
+        grow_distance_array(m)
+      end
+
+      # init matrix d
+      (n+1).times {|i| @d[i][0] = i}
+      (m+1).times {|j| @d[0][j] = j}
+      
+      # start computing edit distance
+      1.upto(n) do |i|
+        best_possible_edit_distance = m
+        s_i = @text[i-1]
+        1.upto(m) do |j|
+          if (s_i != target[j-1]) 
+            @d[i][j] = min(@d[i-1][j], @d[i][j-1], @d[i-1][j-1])+1
+          else 
+            @d[i][j] = min(@d[i-1][j]+1, @d[i][j-1]+1, @d[i-1][j-1])
+          end
+          if @d[i][j] < best_possible_edit_distance
+            best_possible_edit_distance = @d[i][j]
+          end
+        end
+
+        # After calculating row i, the best possible edit distance can be
+        # found by found by finding the smallest value in a given column.
+        # If the best_possible_edit_distance is greater than the max distance,
+        # abort.
+        if (i > max_distance and best_possible_edit_distance > max_distance)
+          # equal is okay, but not greater
+          # the closest the target can be to the text is just too far away.
+          # this target is leaving the party early.
+          return 0.0
+        end
+      end
+
+      # this will return less than 0.0 when the edit distance is
+      # greater than the number of characters in the shorter word.
+      # but this was the formula that was previously used in FuzzyTermEnum,
+      # so it has not been changed (even though minimum_similarity must be
+      # greater than 0.0)
+      return 1.0 - (@d[n][m].to_f / (@prefix_length + (n < m ? n : m)))
+    end
+
+    # Grow the second dimension of the array, so that we can calculate the
+    # Levenshtein difference.
+    def grow_distance_array(m) 
+      @d = @d.map {Array.new(m+1)}
+    end
+
+    # The max Distance is the maximum Levenshtein distance for the text
+    # compared to some other value that results in score that is
+    # better than the minimum similarity.
+    # m:: the length of the "other value"
+    # returns:: the maximum levenshtein distance that we care about
+    def max_distance(m) 
+      if (m >= @max_distances.length)
+        @max_distances[m] = calculate_max_distance(m)
+      end
+      return @max_distances[m]
+    end
+
+    def initialize_max_distances() 
+      @max_distances.length.times do |i|
+        @max_distances[i] = calculate_max_distance(i)
+      end
+    end
+    
+    def calculate_max_distance(m) 
+      return ((1-@minimum_similarity) * ([@text.length, m].min + @prefix_length))
+    end
+  end
+end