ferret 0.1.0

data/lib/ferret/search/exact_phrase_scorer.rb

@@ -0,0 +1,32 @@
+module Ferret::Search
+  class ExactPhraseScorer < PhraseScorer 
+
+    def initialize(weight, tps, positions, similarity, norms) 
+      super(weight, tps, positions, similarity, norms)
+    end
+
+    def phrase_freq()
+      # sort list with pq
+      each do |pp|
+        pp.first_position()
+        @pq.push(pp)         # build pq from list
+      end
+      pq_to_list()           # rebuild list from pq
+
+      freq = 0
+      begin # find position w/ all terms
+        while (@first.position < @last.position) # scan forward in first
+          begin
+            if not @first.next_position()
+              return freq
+            end
+          end while (@first.position < @last.position)
+          first_to_last()
+        end
+        freq += 1            # all equal: a match
+      end while @last.next_position()
+    
+      return freq
+    end
+  end
+end

data/lib/ferret/search/explanation.rb

@@ -0,0 +1,41 @@
+module Ferret::Search
+  # Expert: Describes the score computation for document and query. 
+  class Explanation
+    attr_accessor :value, :description, :details
+
+    def initialize(value = nil, description = nil) 
+      @value = value
+      @description = description
+      @details = []
+    end
+
+    def <<(detail)
+      @details << detail
+    end
+
+    # Render an explanation as text. 
+    def to_s(depth = 0) 
+      buffer = "  " * depth
+      buffer << "#{@value} = #{@description}\n"
+
+      @details.each do |detail|
+        buffer << detail.to_s(depth + 1)
+      end
+      return buffer
+    end
+
+    # Render an explanation as HTML. 
+    def to_html() 
+      buffer = "<ul>\n"
+      buffer << "<li>#{@value} = #{@description}</li>\n"
+
+      @details.each do |detail|
+        buffer << detail.to_html
+      end
+
+      buffer << "</ul>\n"
+
+      return buffer
+    end
+  end
+end

data/lib/ferret/search/field_cache.rb

@@ -0,0 +1,216 @@
+module Ferret::Search
+  require 'monitor'
+
+  # Expert: The default cache implementation, storing all values in memory.
+  # A WeakKeyHash is used for storage.
+  class FieldCache
+    include Ferret::Index
+
+    StringIndex = Struct.new(:str_index, :str_map)
+
+    # Expert: Every key in the internal cache is of this type. 
+    class Entry 
+      attr_reader :field, :sort_type, :comparator
+      # Creates one of these objects. 
+      def initialize(field, sort_type, comparator = nil) 
+        @field = field
+        @sort_type = sort_type
+        @comparator = comparator
+      end
+
+      # Two of these are equal iff they reference the same field and sort_type. 
+      def eql?(o) 
+        return (o.instance_of? Entry and o.field == @field and
+                o.sort_type == @sort_type and o.comparator == comparator)
+      end
+      alias :== :eql?
+
+      # Composes a hashcode based on the field and sort_type. 
+      def hash() 
+        return @field.hash ^ @sort_type.hash ^ @comparator.hash
+      end
+    end
+
+    INT_PARSER = lambda {|i| i.to_i}
+
+    FLOAT_PARSER = lambda {|i| i.to_f}
+
+    # The internal cache. Maps Entry to array of interpreted term values.
+    @@cache = Ferret::Utils::WeakKeyHash.new.extend(MonitorMixin)
+
+    # See if an object is in the cache. 
+    def FieldCache.lookup(reader, field, sort_type) 
+      entry = Entry.new(field, sort_type)
+      @@cache.synchronize() do
+        reader_cache = @@cache[reader]
+        return nil if reader_cache.nil?
+        return reader_cache[entry]
+      end
+    end
+
+    # Put an object into the cache. 
+    def FieldCache.store(reader, field, sort_type, value) 
+      entry = Entry.new(field, sort_type)
+      @@cache.synchronize() do
+        reader_cache = @@cache[reader]
+        if (reader_cache == nil) 
+          reader_cache = {}
+          @@cache[reader] = reader_cache
+        end
+        return reader_cache[entry] = value
+      end
+    end
+
+    # Checks the internal cache for an appropriate entry, and if none is found,
+    # reads the terms in +field+ and parses them with the provided parser and
+    # returns an array of size +reader.max_doc+ of the value each document has
+    # in the given field.
+    #
+    # reader::     Used to get field values.
+    # field::      Which field contains the values.
+    # sort_type::  The type of sort to run on the field. Holds the parser
+    # return::     The values in the given field for each document.
+    def FieldCache.get_index(reader, field, sort_type)
+      index = lookup(reader, field, sort_type)
+      if (index == nil) 
+        parser = sort_type.parser
+        index = Array.new(reader.max_doc)
+        if (index.length > 0) 
+          term_docs = reader.term_docs
+          term_enum = reader.terms_from(Term.new(field, ""))
+          begin 
+            if term_enum.term.nil? 
+              raise "no terms in field '#{field}' to sort by"
+            end
+            begin 
+              term = term_enum.term
+              break if (term.field != field)
+              termval = parser.call(term.text)
+              term_docs.seek(term_enum)
+              while term_docs.next? 
+                index[term_docs.doc] = termval
+              end
+            end while term_enum.next?
+          ensure 
+            term_docs.close()
+            term_enum.close()
+          end
+        end
+        store(reader, field, sort_type, index)
+      end
+      return index
+    end
+
+    # Checks the internal cache for an appropriate entry, and if none is found
+    # reads the term values in +field+ and returns an array of them in natural
+    # order, along with an array telling which element in the term array each
+    # document uses.
+    #
+    # reader::  Used to get field values.
+    # field::   Which field contains the strings.
+    # returns:: Array of terms and index into the array for each document.
+    def FieldCache.get_string_index(reader, field)
+      index = lookup(reader, field, SortField::SortType::STRING)
+      if (index == nil) 
+        str_index = Array.new(reader.max_doc)
+        str_map = Array.new(reader.max_doc+1)
+        if (str_index.length > 0) 
+          term_docs = reader.term_docs
+          term_enum = reader.terms_from(Term.new(field,""))
+          t = 0 # current term number
+
+          # an entry for documents that have no terms in this field should a
+          # document with no terms be at top or bottom?
+          #
+          # this puts them at the top - if it is changed, FieldDocSortedHitQueue
+          # needs to change as well.
+          str_map[t] = nil
+          t += 1
+
+          begin 
+            if (term_enum.term() == nil) 
+              raise "no terms in field #{field} to sort by"
+            end
+            begin
+              term = term_enum.term
+              break if (term.field != field)
+
+              # store term text
+              # we expect that there is at most one term per document
+              if (t >= str_map.length)
+                raise "there are more terms than documents in field \"#{field}\", but it's impossible to sort on tokenized fields"
+              end
+              str_map[t] = term.text
+
+              term_docs.seek(term_enum)
+              while term_docs.next?
+                str_index[term_docs.doc] = t
+              end
+
+              t += 1
+            end while term_enum.next?
+          ensure 
+            term_docs.close()
+            term_enum.close()
+          end
+
+          if (t == 0) 
+            # if there are no terms, make the term array
+            # have a single nil entry
+            # str_map = [nil] <= already set above
+          elsif (t < str_map.length) 
+            # if there are less terms than documents,
+            # trim off the dead array space
+            str_map.compact!
+          end
+        end
+        index = StringIndex.new(str_index, str_map)
+        store(reader, field, SortField::SortType::STRING, index)
+      end
+      return index
+    end
+
+    # Checks the internal cache for an appropriate entry, and if none is found
+    # reads +field+ to see if it contains integers, floats or strings, and then
+    # calls one of the other methods in this class to get the values.  For
+    # string values, a StringIndex is returned.  After calling this method,
+    # there is an entry in the cache for both type +AUTO+ and the actual found
+    # type.
+    #
+    # reader:: Used to get field values.
+    # field::  Which field contains the values.
+    # return:: Integer Array, Float Array or StringIndex.
+    def FieldCache.get_auto_index(reader, field)
+      index = lookup(reader, field, SortField::SortType::AUTO)
+      if (index == nil) 
+        term_enum = reader.terms_from(Term.new(field, ""))
+        begin 
+          term = term_enum.term
+          if (term == nil) 
+            raise "no terms in field #{field} to sort by"
+          end
+          if (term.field == field) 
+            termtext = term.text.strip
+
+            if (termtext == termtext.to_i.to_s)
+              index = get_index(reader, field, SortField::SortType::INT)
+            elsif (termtext == termtext.to_f.to_s or termtext == "%f"%termtext.to_f)
+              index = get_index(reader, field, SortField::SortType::FLOAT)
+            else
+              index = get_string_index(reader, field)
+            end
+
+            if (index != nil) 
+              store(reader, field, SortField::SortType::AUTO, index)
+            end
+          else 
+            raise "field \"#{field}\" does not appear to be indexed"
+          end
+        ensure 
+          term_enum.close()
+        end
+      end
+      return index
+    end
+  end
+end

data/lib/ferret/search/field_doc.rb

@@ -0,0 +1,31 @@
+module Ferret::Search
+  # Expert: A ScoreDoc which also contains information about
+  # how to sort the referenced document.  In addition to the
+  # document number and score, this object contains an array
+  # of values for the document from the field(s) used to sort.
+  # For example, if the sort criteria was to sort by fields
+  # "a", "b" then "c", the +fields+ object array
+  # will have three elements, corresponding respectively to
+  # the term values for the document in fields "a", "b" and "c".
+  # The class of each element in the array will be either
+  # Integer, Float or String depending on the type of values
+  # in the terms of each field.
+  # 
+  class FieldDoc < ScoreDoc 
+
+    # Expert: The values which are used to sort the referenced document.
+    # The order of these will match the original sort criteria given by a
+    # Sort object.  Each Object will be either an Integer, Float or String,
+    # depending on the type of values in the terms of the original field.
+    # See Sort
+    # See Searcher#search(Query,Filter,int,Sort)
+    attr_accessor :fields
+
+    # Expert: Creates one of these objects with the given sort information. 
+    def initialize(doc, score, fields = nil)
+      super(doc, score)
+      @fields = fields
+    end
+
+  end
+end

data/lib/ferret/search/field_sorted_hit_queue.rb

@@ -0,0 +1,184 @@
+require 'monitor'
+
+module Ferret::Search
+  # Expert: A hit queue for sorting by hits by terms in more than one field.
+  # Uses +FieldCache+ for maintaining internal term lookup tables.
+  class FieldSortedHitQueue < Ferret::Utils::PriorityQueue 
+    # Stores a comparator corresponding to each field being sorted by 
+    attr_accessor :comparators
+
+    # Stores the sort criteria being used. 
+    attr_accessor :fields
+
+    # Creates a hit queue sorted by the given list of fields.
+    #
+    # reader::  Index to use.
+    # fields:: Field names, in priority order (highest priority first).
+    #          Cannot be +nil+ or empty.  size::  The number of hits to
+    #          retain.  Must be greater than zero.
+    # raises:: IOError
+    def initialize(reader, fields, size)
+      super(size)
+      n = fields.length
+      @comparators = Array.new(n)
+      @fields = Array.new(n)
+      fields.each_with_index do |field, i|
+        @comparators[i] = get_cached_comparator(reader, field)
+        @fields[i] = SortField.new(field.name,
+                                   {:sort_type => comparators[i].sort_type,
+                                    :reverse => field.reverse?})
+      end
+
+      # Stores the maximum score value encountered, for normalizing.
+      # we only care about scores greater than 1.0 - if all the scores
+      # are less than 1.0, we don't have to normalize. 
+      @max_score = 1.0
+    end
+
+
+    # Returns whether +a+ is less relevant than +b+.
+    # sd1:: ScoreDoc
+    # sd2:: ScoreDoc
+    # returns:: +true+ if document +a+ should be sorted after document +b+.
+    def less_than(sd1, sd2) 
+      # keep track of maximum score
+      @max_score = sd1.score if (sd1.score > @max_score)
+      @max_score = sd2.score if (sd2.score > @max_score)
+
+      # run comparators
+      c = 0
+
+      @comparators.length.times do |i|
+        if @fields[i].reverse?
+          c = @comparators[i].compare(sd2, sd1)
+        else
+          c = @comparators[i].compare(sd1, sd2)
+        end
+        break unless c == 0
+      end
+
+      # avoid random sort order that could lead to duplicates
+      if (c == 0)
+        return sd1.doc > sd2.doc
+      end
+      return c > 0
+    end
+
+
+    # Given a FieldDoc object, stores the values used
+    # to sort the given document.  These values are not the raw
+    # values out of the index, but the internal representation
+    # of them.  This is so the given search hit can be collated
+    # by a MultiSearcher with other search hits.
+    # doc:: The FieldDoc to store sort values into.
+    # returns::  The same FieldDoc passed in.
+    # See Searchable#search(Weight,Filter,int,Sort)
+    def fill_fields(doc) 
+      fields = Array.new(@comparators.length)
+      @comparators.each do |comparator|
+        fields[i] = comparator.sort_value(doc)
+      end
+      doc.fields = fields
+    end
+
+    # Internal cache of comparators. Similar to FieldCache, only
+    # caches comparators instead of term values. 
+    @@comparators = Ferret::Utils::WeakKeyHash.new.extend(MonitorMixin)
+
+    # Returns a comparator if it is in the cache. 
+    def lookup(reader, field, sort_type, comproc) 
+      entry = FieldCache::Entry.new(field, sort_type, comproc)
+      @@comparators.synchronize() do
+        reader_cache = @@comparators[reader]
+        return nil if reader_cache.nil?
+        return reader_cache[entry]
+      end
+    end
+
+    # Stores a comparator into the cache. 
+    def store(reader, field, sort_type, comproc, value) 
+      entry = FieldCache::Entry.new(field, sort_type, comproc)
+      @@comparators.synchronize do 
+        reader_cache = @@comparators[reader]
+        if reader_cache.nil?
+          reader_cache = Hash.new()
+          @@comparators[reader] = reader_cache
+        end
+        return reader_cache[entry] = value
+      end
+    end
+
+    def get_cached_comparator(reader, field)
+      if field.sort_type == SortField::SortType::DOC
+        return ScoreDocComparator::INDEX_ORDER
+      end
+      if field.sort_type == SortField::SortType::SCORE
+        return ScoreDocComparator::RELEVANCE 
+      end
+
+      comparator = lookup(reader, field.name, field.sort_type, field.comparator)
+      if (comparator == nil) 
+        case (field.sort_type) 
+        when SortField::SortType::AUTO: 
+          comparator = comparator_auto(reader, field.name)
+        when SortField::SortType::STRING: 
+          comparator = comparator_string(reader, field.name)
+        else 
+          comparator = comparator_simple(reader, field)
+        end
+
+        store(reader, field.name, field.sort_type, field.comparator, comparator)
+      end
+      return comparator
+    end
+
+    # Returns a comparator for sorting hits according to the sort type and the
+    # comparator function passed.
+    # strings. 
+    #
+    # reader::  Index to use.
+    # field::   Lets us know which field to search and how to parse it.
+    # returns:: Comparator for sorting hits.
+    def comparator_simple(reader, field)
+      index = FieldCache.get_index(reader, field.name, field.sort_type)
+      comproc = field.comparator
+      if (comproc)
+        return SpecialFieldComparator.new(index, field.sort_type, comproc)
+      else
+        return SimpleFieldComparator.new(index, field.sort_type)
+      end
+    end
+
+    # Returns a comparator for sorting hits according to a field containing
+    # strings. 
+    #
+    # reader::  Index to use.
+    # field::   Field containing string values.
+    # returns:: Comparator for sorting hits.
+    def comparator_string(reader, field)
+      index = FieldCache.get_string_index(reader, field)
+      return StringFieldComparator.new(index)
+    end
+
+    # Returns a comparator for sorting hits according to values in the given field.
+    # The terms in the field are looked at to determine whether they contain integers,
+    # floats or strings.  Once the type is determined, one of the other static methods
+    # in this class is called to get the comparator.
+    # reader::  Index to use.
+    # field::  Field containg values.
+    # returns::  Comparator for sorting hits.
+    # raises:: IOException If an error occurs reading the index.
+    def comparator_auto(reader, field)
+      index = FieldCache.get_auto_index(reader, field)
+      if (index.is_a?(FieldCache::StringIndex))
+        return StringFieldComparator.new(index)
+      elsif (index[0].is_a?(Integer)) 
+        return SimpleFieldComparator.new(index, SortField::SortType::INT)
+      elsif (index[0].is_a?(Float)) 
+        return SimpleFieldComparator.new(index, SortField::SortType::FLOAT)
+      else 
+        raise "unknown data type in field '#{field}'. Data = #{index[0]}"
+      end
+    end
+  end
+end