ferret 0.1.0

data/lib/ferret/search/sloppy_phrase_scorer.rb

@@ -0,0 +1,47 @@
+module Ferret::Search
+  class SloppyPhraseScorer < PhraseScorer 
+
+    def initialize(weight, tps, positions, similarity, slop, norms) 
+      super(weight, tps, positions, similarity, norms)
+      @slop = slop
+    end
+
+    def phrase_freq()
+      @pq.clear()
+      last_pos = 0
+      each do |pp|
+        pp.first_position()
+        last_pos = pp.position if (pp.position > last_pos)
+        @pq.push(pp)         # build pq from list
+      end
+
+      freq = 0.0
+      done = false
+      begin 
+        pp = @pq.pop()
+        pos = start = pp.position
+        next_pos = @pq.top().position
+        while pos <= next_pos
+          start = pos       # advance pp to min window
+          if not pp.next_position()
+            done = true     # ran out of a term -- done
+            break
+          end
+          pos = pp.position
+        end
+
+        match_length = last_pos - start
+        if (match_length <= @slop)
+          freq += @similarity.sloppy_freq(match_length) # score match
+        end
+
+        if (pp.position > last_pos)
+          last_pos = pp.position
+        end
+        @pq.push(pp)         # restore pq
+      end while (!done)
+
+      return freq
+    end
+  end
+end

data/lib/ferret/search/sort.rb

@@ -0,0 +1,105 @@
+module Ferret::Search
+  # Encapsulates sort criteria for returned hits.
+  # 
+  # The fields used to determine sort order must be carefully chosen.
+  # Documents must contain a single term in such a field, and the value of the
+  # term should indicate the document's relative position in a given sort
+  # order.  The field must be indexed, but should not be tokenized, and does
+  # not need to be stored (unless you happen to want it back with the rest of
+  # your document data).  In other words:
+  # 
+  #   document << Field.new("by_number",
+  #                         x.to_s,
+  #                         Field::Store::NO,
+  #                         Field::Index::UN_TOKENIZED))
+  # 
+  # 
+  # === Valid Types of Values
+  # 
+  # There are three possible kinds of term values which may be put into
+  # sorting fields: Integers, Floats, or Strings.  Unless SortField objects
+  # are specified, the type of value in the field is determined by parsing the
+  # first term in the field.
+  # 
+  # Integer term values should contain only digits and an optional preceeding
+  # negative sign.  Values must be base 10.  Documents which should appear
+  # first in the sort should have low value integers, later documents high
+  # values (i.e. the documents should be numbered +1..n+ where +1+ is the
+  # first and +n+ the last).
+  # 
+  # Float term values should conform to values accepted by String#to_f.
+  # Documents which should appear first in the sort should have low values,
+  # later documents high values.
+  # 
+  # String term values can contain any valid String, but should not be
+  # tokenized.  The values are sorted according to their Comparable natural
+  # order.  Note that using this type of term value has higher memory
+  # requirements than the other two types.
+  # 
+  # === Object Reuse
+  # 
+  # One of these objects can be used multiple times and the sort order changed
+  # between usages.
+  # 
+  # This class is thread safe.
+  # 
+  # === Memory Usage
+  # 
+  # Sorting uses caches of term values maintained by the internal HitQueue(s).
+  # The cache is static and contains an integer or float array of length
+  # +IndexReader#max_doc+ for each field name for which a sort is performed.
+  # In other words, the size of the cache in bytes is:
+  # 
+  #   4 * IndexReader#max_doc * (# of different fields actually used to sort)
+  # 
+  # For String fields, the cache is larger: in addition to the above array,
+  # the value of every term in the field is kept in memory.  If there are many
+  # unique terms in the field, this could be quite large.
+  # 
+  # Note that the size of the cache is not affected by how many fields are in
+  # the index and _might_ be used to sort - only by the ones actually used to
+  # sort a result set.
+  # 
+  # The cache is cleared each time a new +IndexReader+ is passed in, or if the
+  # value returned by +max_doc()+ changes for the current IndexReader.  This
+  # class is not set up to be able to efficiently sort hits from more than one
+  # index simultaneously.
+  class Sort
+
+    attr_accessor :fields
+
+    # Sorts by computed relevance. You can pass a string representing the name
+    # of the field you want to sort on, a SortField, or an array of either
+    # (but not a mixed array). If you pass a string or and array of strings
+    # you can also pass a reverse flag. If you pass a SortField the reverse is
+    # handled by it.
+    #
+    # fields::  The fields you want to sort on. See also SortField
+    # reverse:: pass true if you want the sort order to be reversed. Only
+    #    works if you pass the field names.
+    def initialize(fields = [SortField::FIELD_SCORE, SortField::FIELD_DOC],
+                   reverse = false)
+      fields = [fields] unless fields.is_a?(Array)
+      @fields = fields
+      if fields[0].is_a?(String)
+        @fields = fields.map do |field|
+          SortField.new(field, {:sort_type => SortField::SortType::AUTO,
+                                :reverse => reverse})
+        end
+        @fields << SortField::FIELD_DOC if @fields.size == 1
+      end
+    end
+
+    # Represents sorting by computed relevance. Using this sort criteria returns
+    # the same results as calling Searcher#search(Query) Searcher#search()
+    # without a sort criteria, only with slightly more overhead.
+    RELEVANCE = Sort.new()
+
+    # Represents sorting by index order. 
+    INDEX_ORDER = Sort.new(SortField::FIELD_DOC)
+
+    def to_s() 
+      return @fields.map {|field| "#{field}"}.join(", ")
+    end
+  end
+end

data/lib/ferret/search/sort_comparator.rb

@@ -0,0 +1,60 @@
+module Ferret::Search
+  # Abstract base class for sorting hits returned by a Query.
+  # 
+  # This class should only be used if the other SortField types (SCORE, DOC,
+  # STRING, INT, FLOAT) do not provide an adequate sorting.  It maintains an
+  # internal cache of values which could be quite large.  The cache is an
+  # array of Comparable, one for each document in the index.  There is a
+  # distinct Comparable for each unique term in the field - if some documents
+  # have the same term in the field, the cache array will have entries which
+  # reference the same Comparable.
+  # 
+  # Author::  Tim Jones
+  class SortComparator
+
+    # Creates a comparator for the field in the given index.
+    #
+    # reader:: Index to create comparator for.
+    # field_name::  Field to create comparator for.
+    # returns:: Comparator of ScoreDoc objects.
+    def new_comparator(reader, field_name)
+      cached_values = FieldCache::DEFAULT.custom(reader, field, self)
+      
+      score_doc_comparator =  ScoreDocComparator.new() 
+
+      class <<score_doc_comparator
+        attr_writer :cache_values
+        def compare(i, j) 
+          return @cached_values[i.doc] <=> @cached_values[j.doc]
+        end
+
+        def sort_value(i) 
+          return @cached_values[i.doc]
+        end
+
+        def sort_type()
+          return SortField::SortType::CUSTOM
+        end
+      end
+      score_doc_comparator.cached_values = cached_values
+      return score_doc_comparator
+    end
+
+    # Returns an object which, when sorted according to natural order, will
+    # order the Term values in the correct order.  For example, if the Terms
+    # contained integer values, this method would return +term_text.to_i+.
+    # Note that this might not always be the most efficient implementation -
+    # for this particular example, a better implementation might be to make a
+    # ScoreDocLookupComparator that uses an internal lookup table of int.
+    #
+    # term_text:: The textual value of the term.
+    #
+    # returns:: An object representing +term_text+ that sorts according to the
+    #           natural order of +term_text+.
+    #
+    # See ScoreDocComparator
+    def get_comparable(term_text)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ferret/search/sort_field.rb

@@ -0,0 +1,87 @@
+module Ferret::Search
+
+  # Stores information about how to sort documents by terms in an individual
+  # field.  Fields must be indexed in order to sort by them.
+  class SortField
+    class SortType < Ferret::Utils::Parameter
+      attr_reader :parser, :comparator
+
+      # Creates a new SortType. A SortType is used to specify how a field is
+      # sorted in a document. Each SortType *MUST* have a unique name. This is
+      # because the SortType object is used to cache a fields values for a
+      # particular reader, so each SortType should be created once only and
+      # stored in a constant. See the standard SortTypes stored hear for
+      # example.
+      def initialize(name, parser = lambda{|str| str}, comparator = nil)
+        super(name)
+        @parser = parser
+        @comparator = comparator
+      end
+
+      # Sort by document score (relevancy).  Sort values are Float and higher
+      # values are at the front. 
+      SCORE = SortType.new("score")
+
+      # Sort by document number (order).  Sort values are Integer and lower
+      # values are at the front. 
+      DOC = SortType.new("doc")
+
+      # Guess sort type of sort based on field contents. We try parsing the
+      # field as an integer and then as a floating point number. If we are
+      # unsuccessful, the field is parsed as a plain string.
+      AUTO = SortType.new("auto")
+
+      # Sort using term values as Strings.  Sort values are String and lower
+      # values are at the front. 
+      STRING = SortType.new("string")
+
+      # Sort using term values as encoded Integers.  Sort values are Integer
+      # and lower values are at the front. 
+      INT = SortType.new("int", lambda{|str| str.to_i})
+
+      # Sort using term values as encoded Floats.  Sort values are Float and
+      # lower values are at the front. 
+      FLOAT = SortType.new("float", lambda{|str| str.to_f})
+    end
+
+    attr_reader :name, :sort_type, :comparator
+
+    def reverse?
+      return @reverse
+    end
+
+    # Creates a SortField which specifies which field the data is sorted on
+    # and how that field is sorted. See SortType.
+    #
+    # name:: Name of field to sort by.  Can be +nil+ if +sort_type+ is SCORE or
+    #     DOC.
+    #
+    # A hash with the followind values can also be supplied;
+    # sort_type:: Type of values in the terms.
+    # reverse:: True if natural order should be reversed.
+    # comparator:: a proc used to compare two values from the index. You can
+    #    also give this value to the SortType object that you pass.
+    def initialize(name = nil, args= {})
+      @name = name
+      @sort_type = args[:sort_type]||SortType::AUTO
+      @reverse = args[:reverse]||false
+      @comparator = args[:comparator]||@sort_type.comparator
+      if (@name == nil and @sort_type != SortType::DOC and
+          @sort_type != SortType::SCORE)
+        raise ArgumentError, "You must supply a field name for your sort field"
+      end
+    end
+
+    # Represents sorting by document score (relevancy). 
+    FIELD_SCORE = SortField.new(nil, {:sort_type => SortType::SCORE})
+
+    # Represents sorting by document number (order). 
+    FIELD_DOC = SortField.new(nil, {:sort_type => SortType::DOC})
+
+    def to_s() 
+      buffer = '"' + (@name||"<#{@sort_type}>") + '"'
+      buffer << '!' if @reverse
+      return buffer
+    end
+  end
+end

data/lib/ferret/search/spans.rb

@@ -0,0 +1,12 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'spans/spans_enum.rb'
+require 'spans/near_spans_enum.rb'
+require 'spans/span_query.rb'
+require 'spans/span_first_query.rb'
+require 'spans/span_near_query.rb'
+require 'spans/span_not_query.rb'
+require 'spans/span_or_query.rb'
+require 'spans/span_scorer.rb'
+require 'spans/span_term_query.rb'
+require 'spans/span_weight.rb'

data/lib/ferret/search/spans/near_spans_enum.rb

@@ -0,0 +1,304 @@
+module Ferret::Search::Spans
+  class NearSpansEnum < SpansEnum
+
+    class CellQueue < Ferret::Utils::PriorityQueue 
+      def less_than(o1, o2) 
+        if (o1.doc == o2.doc) 
+          if (o1.start == o2.start) 
+            if (o1.finish == o2.finish) 
+              return o1.index > o2.index
+            else 
+              return o1.finish < o2.finish
+            end
+          else 
+            return o1.start < o2.start
+          end
+        else 
+          return o1.doc < o2.doc
+        end
+      end
+    end
+
+
+    # Wraps a SpansEnum, and can be used to form a linked list.
+    class SpansCell < SpansEnum
+      attr_accessor :next, :index
+
+      def initialize(parent, spans, index) 
+        @parent = parent
+        @spans = spans
+        @index = index
+        @length = -1
+      end
+
+      def next?()
+        if (@length != -1)                  # subtract old length
+          @parent.total_length -= @length
+        end
+
+        more = @spans.next?                 # move to next
+
+        if more 
+          @length = finish() - start()      # compute new length
+          @parent.total_length += @length   # add new length to total
+
+          if (@parent.max.nil? or doc() > @parent.max.doc or     # maintain max
+              (doc() == @parent.max.doc and finish() > @parent.max.finish))
+            @parent.max = self
+          end
+        end
+
+        return more
+      end
+
+      def skip_to(target)
+        if (@length != -1)                  # subtract old length
+          @parent.total_length -= @length
+        end
+
+        more = @spans.skip_to(target)       # skip
+
+        if (more) 
+          @length = finish() - start()      # compute new length
+          @parent.total_length += @length   # add new length to total
+
+          if (@parent.max == nil or doc() > @parent.max.doc() or   # maintain max
+              (doc() == @parent.max.doc and finish() > @parent.max.finish))
+            @parent.max = self
+          end
+        end
+
+        return more
+      end
+
+      def doc() return @spans.doc() end
+      def start() return @spans.start() end
+      def finish() return @spans.finish() end
+
+      def to_s() return "#{@spans}##{@index}" end
+    end
+
+    attr_accessor :total_length, :max
+
+    def initialize(query, reader)
+      @ordered = []         # spans in query order
+
+      @first = nil          # linked list of spans
+      @last = nil           # sorted by doc only
+
+      @total_length = 0     # sum of current lengths
+
+      @queue = nil          # sorted queue of spans
+      @max = nil            # max element in queue
+
+      @more = true          # true iff not done
+      @first_time = true    # true before first next?
+
+     
+      @query = query
+      @slop = query.slop
+      @in_order = query.in_order?
+
+      clauses = query.clauses # initialize spans & list
+      @queue = CellQueue.new(clauses.length)
+      clauses.length.times do |i|
+        # construct clause spans
+        cell = SpansCell.new(self, clauses[i].spans(reader), i)
+        @ordered << cell    # add to ordered
+      end
+    end
+
+    def next?()
+      if (@first_time) 
+        init_list(true)
+        list_to_queue()                # initialize queue
+        @first_time = false
+      elsif (@more) 
+        @more = min().next?            # trigger further scanning
+        @queue.adjust_top() if (@more) # maintain queue 
+      end
+
+      while (@more) 
+        queue_stale = false
+
+        if (min().doc != @max.doc)     # maintain list
+          queue_to_list()
+          queue_stale = true
+        end
+
+        # skip to doc w/ all clauses
+
+        while (@more and @first.doc < @last.doc) 
+          @more = @first.skip_to(@last.doc) # skip first upto last
+          first_to_last()              # and move it to the end
+          queue_stale = true
+        end
+
+        return false if not @more
+
+        # found doc w/ all clauses
+
+        if (queue_stale) # maintain the queue
+          list_to_queue()
+          queue_stale = false
+        end
+
+        return true if at_match?
+        
+        # trigger further scanning
+        if (@in_order and check_slop?()) 
+          # There is a non ordered match within slop and an ordered match is needed. 
+          @more = first_non_ordered_next_to_partial_list()
+          if (@more) 
+            partial_list_to_queue()
+          end
+        else 
+          @more = min().next?()
+          if (@more) 
+            @queue.adjust_top()        # maintain queue
+          end
+        end
+      end
+      return false                     # no more matches
+    end
+
+    def each()
+      cell = @first
+      while (cell)
+        yield cell
+        cell=cell.next
+      end
+    end
+
+    def skip_to(target)
+      if (@first_time) # initialize
+        init_list(false)
+        each() do |cell|
+          @more = cell.skip_to(target) # skip all
+          break if not @more
+        end
+
+        if (@more) 
+          list_to_queue()
+        end
+        @first_time = false
+
+      else # normal case
+        while (@more and min().doc < target) # skip as needed
+          @more = min().skip_to(target)
+          @queue.adjust_top() if (@more)
+        end
+      end
+
+      if (@more) 
+        return true if (at_match?())              # at a match?
+        return next?                              # no, scan
+      end
+
+      return false
+    end
+
+    def min() @queue.top() end
+
+    def doc() min().doc() end
+    def start() min().start() end
+    def finish() @max.finish() end
+
+
+    def to_s() 
+      buffer = "spans(#{@query})@"
+      if @first_time
+        buffer << "START"
+      else
+        buffer << (@queue.size>0 ? ("#{doc}:#{start()}-#{finish}") : "END")
+      end
+      return buffer
+    end
+
+    def init_list(nxt)
+      @ordered.each do |cell|
+        @more = cell.next? if nxt
+        if @more
+          add_to_list(cell) # add to list
+        else
+          break
+        end
+      end
+    end
+
+    def add_to_list(cell) 
+      if (@last != nil) # add next to end of list
+        @last.next = cell
+      else
+        @first = cell
+      end
+      @last = cell
+      cell.next = nil
+    end
+
+    def first_to_last() 
+      @last.next = @first # move first to end of list
+      @last = @first
+      @first = @first.next
+      @last.next = nil
+    end
+
+    def queue_to_list() 
+      @last = @first = nil
+      while (@queue.top() != nil) 
+        add_to_list(@queue.pop())
+      end
+    end
+    
+    def first_non_ordered_next_to_partial_list()
+      # Creates a partial list consisting of first non ordered and earlier.
+      # Returns first non ordered .next?.
+      @last = @first = nil
+      ordered_index = 0
+      while (@queue.top() != nil) 
+        cell = @queue.pop()
+        add_to_list(cell)
+        if (cell.index == ordered_index) 
+          ordered_index += 1
+        else 
+          return cell.next?()
+          # FIXME: continue here, rename to eg. checkOrderedMatch():
+          # when check_slop?() and not ordered, repeat cell.next?().
+          # when check_slop?() and ordered, add to list and repeat queue.pop()
+          # without check_slop?(): no match, rebuild the queue from the partial list.
+          # When queue is empty and check_slop?() and ordered there is a match.
+        end
+      end
+      raise RuntimeException, "Unexpected: ordered"
+    end
+
+    def list_to_queue() 
+      @queue.clear() # rebuild queue
+      partial_list_to_queue()
+    end
+
+    def partial_list_to_queue() 
+      each() { |cell| @queue.push(cell) } # add to queue from list
+    end
+
+    def at_match?() 
+      return ((min().doc() == @max.doc()) and check_slop?() and
+              (not @in_order or match_is_ordered?()))
+    end
+    
+    def check_slop?() 
+      match_length = @max.finish() - min.start()
+      return ((match_length - @total_length) <= @slop)
+    end
+
+    def match_is_ordered?() 
+      last_start = -1
+      @ordered.each do |cell|
+        start = cell.start
+        return false if start <= last_start
+        last_start = start
+      end
+      return true
+    end
+  end
+end