ferret 0.9.6 → 0.10.0

data/lib/ferret/search/exact_phrase_scorer.rb

@@ -1,32 +0,0 @@
-module Ferret::Search
-  class ExactPhraseScorer < PhraseScorer 
-
-    def initialize(weight, tps, positions, similarity, norms) 
-      super(weight, tps, positions, similarity, norms)
-    end
-
-    def phrase_freq()
-      # sort list with pq
-      each do |pp|
-        pp.first_position()
-        @pq.push(pp)         # build pq from list
-      end
-      pq_to_list()           # rebuild list from pq
-
-      freq = 0
-      begin # find position w/ all terms
-        while (@first.position < @last.position) # scan forward in first
-          begin
-            if not @first.next_position()
-              return freq
-            end
-          end while (@first.position < @last.position)
-          first_to_last()
-        end
-        freq += 1            # all equal: a match
-      end while @last.next_position()
-    
-      return freq
-    end
-  end
-end

data/lib/ferret/search/explanation.rb

@@ -1,41 +0,0 @@
-module Ferret::Search
-  # Expert: Describes the score computation for document and query. 
-  class Explanation
-    attr_accessor :value, :description, :details
-
-    def initialize(value = nil, description = nil) 
-      @value = value
-      @description = description
-      @details = []
-    end
-
-    def <<(detail)
-      @details << detail
-    end
-
-    # Render an explanation as text. 
-    def to_s(depth = 0) 
-      buffer = "  " * depth
-      buffer << "#{@value} = #{@description}\n"
-
-      @details.each do |detail|
-        buffer << detail.to_s(depth + 1)
-      end
-      return buffer
-    end
-
-    # Render an explanation as HTML. 
-    def to_html() 
-      buffer = "<ul>\n"
-      buffer << "<li>#{@value} = #{@description}</li>\n"
-
-      @details.each do |detail|
-        buffer << detail.to_html
-      end
-
-      buffer << "</ul>\n"
-
-      return buffer
-    end
-  end
-end

data/lib/ferret/search/field_cache.rb

@@ -1,215 +0,0 @@
-module Ferret::Search
-
-  # Expert: The default cache implementation, storing all values in memory.
-  # A WeakKeyHash is used for storage.
-  class FieldCache
-    include Ferret::Index
-
-    StringIndex = Struct.new(:str_index, :str_map)
-
-    # Expert: Every key in the internal cache is of this type. 
-    class Entry 
-      attr_reader :field, :sort_type, :comparator
-      # Creates one of these objects. 
-      def initialize(field, sort_type, comparator = nil) 
-        @field = field
-        @sort_type = sort_type
-        @comparator = comparator
-      end
-
-      # Two of these are equal iff they reference the same field and sort_type. 
-      def eql?(o) 
-        return (o.instance_of? Entry and o.field == @field and
-                o.sort_type == @sort_type and o.comparator == comparator)
-      end
-      alias :== :eql?
-
-      # Composes a hashcode based on the field and sort_type. 
-      def hash() 
-        return @field.hash ^ @sort_type.hash ^ @comparator.hash
-      end
-    end
-
-    INT_PARSER = lambda {|i| i.to_i}
-
-    FLOAT_PARSER = lambda {|i| i.to_f}
-
-    # The internal cache. Maps Entry to array of interpreted term values.
-    @@cache = Ferret::Utils::WeakKeyHash.new
-
-    # See if an object is in the cache. 
-    def FieldCache.lookup(reader, field, sort_type) 
-      entry = Entry.new(field, sort_type)
-      @@cache.synchronize() do
-        reader_cache = @@cache[reader]
-        return nil if reader_cache.nil?
-        return reader_cache[entry]
-      end
-    end
-
-    # Put an object into the cache. 
-    def FieldCache.store(reader, field, sort_type, value) 
-      entry = Entry.new(field, sort_type)
-      @@cache.synchronize() do
-        reader_cache = @@cache[reader]
-        if (reader_cache == nil) 
-          reader_cache = {}
-          @@cache[reader] = reader_cache
-        end
-        return reader_cache[entry] = value
-      end
-    end
-
-    # Checks the internal cache for an appropriate entry, and if none is found,
-    # reads the terms in +field+ and parses them with the provided parser and
-    # returns an array of size +reader.max_doc+ of the value each document has
-    # in the given field.
-    #
-    # reader::     Used to get field values.
-    # field::      Which field contains the values.
-    # sort_type::  The type of sort to run on the field. Holds the parser
-    # return::     The values in the given field for each document.
-    def FieldCache.get_index(reader, field, sort_type)
-      index = lookup(reader, field, sort_type)
-      if (index == nil) 
-        parser = sort_type.parser
-        index = Array.new(reader.max_doc)
-        if (index.length > 0) 
-          term_docs = reader.term_docs
-          term_enum = reader.terms_from(Term.new(field, ""))
-          begin 
-            if term_enum.term.nil? 
-              raise "no terms in field '#{field}' to sort by"
-            end
-            begin 
-              term = term_enum.term
-              break if (term.field != field)
-              termval = parser.call(term.text)
-              term_docs.seek(term)
-              while term_docs.next? 
-                index[term_docs.doc] = termval
-              end
-            end while term_enum.next?
-          ensure 
-            term_docs.close()
-            term_enum.close()
-          end
-        end
-        store(reader, field, sort_type, index)
-      end
-      return index
-    end
-
-    # Checks the internal cache for an appropriate entry, and if none is found
-    # reads the term values in +field+ and returns an array of them in natural
-    # order, along with an array telling which element in the term array each
-    # document uses.
-    #
-    # reader::  Used to get field values.
-    # field::   Which field contains the strings.
-    # returns:: Array of terms and index into the array for each document.
-    def FieldCache.get_string_index(reader, field)
-      index = lookup(reader, field, SortField::SortType::STRING)
-      if (index == nil) 
-        str_index = Array.new(reader.max_doc)
-        str_map = Array.new(reader.max_doc+1)
-        if (str_index.length > 0) 
-          term_docs = reader.term_docs
-          term_enum = reader.terms_from(Term.new(field,""))
-          t = 0 # current term number
-
-          # an entry for documents that have no terms in this field should a
-          # document with no terms be at top or bottom?
-          #
-          # this puts them at the top - if it is changed, FieldDocSortedHitQueue
-          # needs to change as well.
-          str_map[t] = nil
-          t += 1
-
-          begin 
-            if (term_enum.term() == nil) 
-              raise "no terms in field #{field} to sort by"
-            end
-            begin
-              term = term_enum.term
-              break if (term.field != field)
-
-              # store term text
-              # we expect that there is at most one term per document
-              if (t >= str_map.length)
-                raise "there are more terms than documents in field \"#{field}\", but it's impossible to sort on tokenized fields"
-              end
-              str_map[t] = term.text
-
-              term_docs.seek(term)
-              while term_docs.next?
-                str_index[term_docs.doc] = t
-              end
-
-              t += 1
-            end while term_enum.next?
-          ensure 
-            term_docs.close()
-            term_enum.close()
-          end
-
-          if (t == 0) 
-            # if there are no terms, make the term array
-            # have a single nil entry
-            # str_map = [nil] <= already set above
-          elsif (t < str_map.length) 
-            # if there are less terms than documents,
-            # trim off the dead array space
-            str_map.compact!
-          end
-        end
-        index = StringIndex.new(str_index, str_map)
-        store(reader, field, SortField::SortType::STRING, index)
-      end
-      return index
-    end
-
-    # Checks the internal cache for an appropriate entry, and if none is found
-    # reads +field+ to see if it contains integers, floats or strings, and then
-    # calls one of the other methods in this class to get the values.  For
-    # string values, a StringIndex is returned.  After calling this method,
-    # there is an entry in the cache for both type +AUTO+ and the actual found
-    # type.
-    #
-    # reader:: Used to get field values.
-    # field::  Which field contains the values.
-    # return:: Integer Array, Float Array or StringIndex.
-    def FieldCache.get_auto_index(reader, field)
-      index = lookup(reader, field, SortField::SortType::AUTO)
-      if (index == nil) 
-        term_enum = reader.terms_from(Term.new(field, ""))
-        begin 
-          term = term_enum.term
-          if (term == nil) 
-            raise "no terms in field #{field} to sort by"
-          end
-          if (term.field == field) 
-            termtext = term.text.strip
-
-            if (termtext == termtext.to_i.to_s)
-              index = get_index(reader, field, SortField::SortType::INTEGER)
-            elsif (termtext == termtext.to_f.to_s or termtext == "%f"%termtext.to_f)
-              index = get_index(reader, field, SortField::SortType::FLOAT)
-            else
-              index = get_string_index(reader, field)
-            end
-
-            if (index != nil) 
-              store(reader, field, SortField::SortType::AUTO, index)
-            end
-          else 
-            raise "field \"#{field}\" does not appear to be indexed"
-          end
-        ensure 
-          term_enum.close()
-        end
-      end
-      return index
-    end
-  end
-end

data/lib/ferret/search/field_doc.rb

@@ -1,31 +0,0 @@
-module Ferret::Search
-  # Expert: A ScoreDoc which also contains information about
-  # how to sort the referenced document.  In addition to the
-  # document number and score, this object contains an array
-  # of values for the document from the field(s) used to sort.
-  # For example, if the sort criteria was to sort by fields
-  # "a", "b" then "c", the +fields+ object array
-  # will have three elements, corresponding respectively to
-  # the term values for the document in fields "a", "b" and "c".
-  # The class of each element in the array will be either
-  # Integer, Float or String depending on the type of values
-  # in the terms of each field.
-  # 
-  class FieldDoc < ScoreDoc 
-
-    # Expert: The values which are used to sort the referenced document.
-    # The order of these will match the original sort criteria given by a
-    # Sort object.  Each Object will be either an Integer, Float or String,
-    # depending on the type of values in the terms of the original field.
-    # See Sort
-    # See Searcher#search(Query,Filter,int,Sort)
-    attr_accessor :fields
-
-    # Expert: Creates one of these objects with the given sort information. 
-    def initialize(doc, score, fields = nil)
-      super(doc, score)
-      @fields = fields
-    end
-
-  end
-end

data/lib/ferret/search/field_sorted_hit_queue.rb

@@ -1,184 +0,0 @@
-require 'monitor'
-
-module Ferret::Search
-  # Expert: A hit queue for sorting by hits by terms in more than one field.
-  # Uses +FieldCache+ for maintaining internal term lookup tables.
-  class FieldSortedHitQueue < Ferret::Utils::PriorityQueue 
-    # Stores a comparator corresponding to each field being sorted by 
-    attr_accessor :comparators
-
-    # Stores the sort criteria being used. 
-    attr_accessor :fields
-
-    # Creates a hit queue sorted by the given list of fields.
-    #
-    # reader:: Index to use.
-    # fields:: Field names, in priority order (highest priority first).
-    #          Cannot be +nil+ or empty.
-    # size::   The number of hits to retain.  Must be greater than zero.
-    # raises:: IOError
-    def initialize(reader, fields, size)
-      super(size)
-      n = fields.length
-      @comparators = Array.new(n)
-      @fields = Array.new(n)
-      fields.each_with_index do |field, i|
-        @comparators[i] = get_cached_comparator(reader, field)
-        @fields[i] = SortField.new(field.name,
-                                   {:sort_type => comparators[i].sort_type,
-                                    :reverse => field.reverse?})
-      end
-
-      # Stores the maximum score value encountered, for normalizing.
-      # we only care about scores greater than 1.0 - if all the scores
-      # are less than 1.0, we don't have to normalize. 
-      @max_score = 1.0
-    end
-
-
-    # Returns whether +a+ is less relevant than +b+.
-    # sd1:: ScoreDoc
-    # sd2:: ScoreDoc
-    # returns:: +true+ if document +a+ should be sorted after document +b+.
-    def less_than(sd1, sd2) 
-      # keep track of maximum score
-      @max_score = sd1.score if (sd1.score > @max_score)
-      @max_score = sd2.score if (sd2.score > @max_score)
-
-      # run comparators
-      c = 0
-
-      @comparators.length.times do |i|
-        if @fields[i].reverse?
-          c = @comparators[i].compare(sd2, sd1)
-        else
-          c = @comparators[i].compare(sd1, sd2)
-        end
-        break unless c == 0
-      end
-
-      # avoid random sort order that could lead to duplicates
-      if (c == 0)
-        return sd1.doc > sd2.doc
-      end
-      return c > 0
-    end
-
-
-    # Given a FieldDoc object, stores the values used
-    # to sort the given document.  These values are not the raw
-    # values out of the index, but the internal representation
-    # of them.  This is so the given search hit can be collated
-    # by a MultiSearcher with other search hits.
-    # doc:: The FieldDoc to store sort values into.
-    # returns::  The same FieldDoc passed in.
-    # See Searchable#search(Weight,Filter,int,Sort)
-    def fill_fields(doc) 
-      fields = Array.new(@comparators.length)
-      @comparators.each do |comparator|
-        fields[i] = comparator.sort_value(doc)
-      end
-      doc.fields = fields
-    end
-
-    # Internal cache of comparators. Similar to FieldCache, only
-    # caches comparators instead of term values. 
-    @@comparators = Ferret::Utils::WeakKeyHash.new
-
-    # Returns a comparator if it is in the cache. 
-    def lookup(reader, field, sort_type, comproc) 
-      entry = FieldCache::Entry.new(field, sort_type, comproc)
-      @@comparators.synchronize() do
-        reader_cache = @@comparators[reader]
-        return nil if reader_cache.nil?
-        return reader_cache[entry]
-      end
-    end
-
-    # Stores a comparator into the cache. 
-    def store(reader, field, sort_type, comproc, value) 
-      entry = FieldCache::Entry.new(field, sort_type, comproc)
-      @@comparators.synchronize do 
-        reader_cache = @@comparators[reader]
-        if reader_cache.nil?
-          reader_cache = Hash.new()
-          @@comparators[reader] = reader_cache
-        end
-        return reader_cache[entry] = value
-      end
-    end
-
-    def get_cached_comparator(reader, field)
-      if field.sort_type == SortField::SortType::DOC
-        return ScoreDocComparator::INDEX_ORDER
-      end
-      if field.sort_type == SortField::SortType::SCORE
-        return ScoreDocComparator::RELEVANCE 
-      end
-
-      comparator = lookup(reader, field.name, field.sort_type, field.comparator)
-      if (comparator == nil) 
-        case (field.sort_type) 
-        when SortField::SortType::AUTO: 
-          comparator = comparator_auto(reader, field.name)
-        when SortField::SortType::STRING: 
-          comparator = comparator_string(reader, field.name)
-        else 
-          comparator = comparator_simple(reader, field)
-        end
-
-        store(reader, field.name, field.sort_type, field.comparator, comparator)
-      end
-      return comparator
-    end
-
-    # Returns a comparator for sorting hits according to the sort type and the
-    # comparator function passed.
-    # strings. 
-    #
-    # reader::  Index to use.
-    # field::   Lets us know which field to search and how to parse it.
-    # returns:: Comparator for sorting hits.
-    def comparator_simple(reader, field)
-      index = FieldCache.get_index(reader, field.name, field.sort_type)
-      comproc = field.comparator
-      if (comproc)
-        return SpecialFieldComparator.new(index, field.sort_type, comproc)
-      else
-        return SimpleFieldComparator.new(index, field.sort_type)
-      end
-    end
-
-    # Returns a comparator for sorting hits according to a field containing
-    # strings. 
-    #
-    # reader::  Index to use.
-    # field::   Field containing string values.
-    # returns:: Comparator for sorting hits.
-    def comparator_string(reader, field)
-      index = FieldCache.get_string_index(reader, field)
-      return StringFieldComparator.new(index)
-    end
-
-    # Returns a comparator for sorting hits according to values in the given field.
-    # The terms in the field are looked at to determine whether they contain integers,
-    # floats or strings.  Once the type is determined, one of the other static methods
-    # in this class is called to get the comparator.
-    # reader::  Index to use.
-    # field::  Field containg values.
-    # returns::  Comparator for sorting hits.
-    # raises:: IOException If an error occurs reading the index.
-    def comparator_auto(reader, field)
-      index = FieldCache.get_auto_index(reader, field)
-      if (index.is_a?(FieldCache::StringIndex))
-        return StringFieldComparator.new(index)
-      elsif (index[0].is_a?(Integer)) 
-        return SimpleFieldComparator.new(index, SortField::SortType::INTEGER)
-      elsif (index[0].is_a?(Float)) 
-        return SimpleFieldComparator.new(index, SortField::SortType::FLOAT)
-      else 
-        raise "unknown data type in field '#{field}'. Data = #{index[0]}"
-      end
-    end
-  end
-end