ferret 0.3.2 → 0.9.0

data/lib/ferret/index/segment_term_vector.rb

@@ -6,14 +6,14 @@ module Ferret::Index
     # to the terms in the array obtained from _terms_
     # method. Each location in the array contains the number of times this
     # term occurs in the document or the document field.
-    attr_reader :term_frequencies, :positions, :offsets
+    attr_reader :freqs, :positions, :offsets
 
     attr_reader :field, :terms
 
-    def initialize(field, terms, term_freqs, positions=nil, offsets=nil) 
+    def initialize(field, terms, freqs, positions=nil, offsets=nil) 
       @field = field
       @terms = terms
-      @term_frequencies = term_freqs
+      @freqs = freqs
       @positions = positions
       @offsets = offsets
     end
@@ -23,7 +23,7 @@ module Ferret::Index
       if @terms
         terms.each_with_index do |term, i|
           sb << ', ' if i > 0
-          sb << term + '/' + @term_frequencies[i].to_s
+          sb << term + '/' + @freqs[i].to_s
         end
       end
       sb << 'end'

data/lib/ferret/index/term.rb

@@ -19,7 +19,7 @@ module Ferret::Index
     # Constructs a Term with the given field and text
     def initialize(fld_name, txt)
       @field = fld_name
-      @text = txt
+      @text = txt.to_s
     end
 
     # Combines the hash() of the field and the text.
@@ -42,6 +42,10 @@ module Ferret::Index
       initialize(fld_name, txt)
     end
 
+    def text=(text)
+      @text = text.to_s
+    end
+
     def to_s
       @field + ":" + @text
     end

data/lib/ferret/index/term_vector_offset_info.rb

@@ -1,20 +1,20 @@
 module Ferret::Index
   class TermVectorOffsetInfo 
-    attr_accessor :start_offset, :end_offset
+    attr_accessor :start, :end
 
-    def initialize(start_offset, end_offset) 
-      @end_offset = end_offset
-      @start_offset = start_offset
+    def initialize(start, endd) 
+      @end = endd
+      @start = start
     end
 
     def eql?(o) 
       return false if !o.instance_of?(TermVectorOffsetInfo)
-      @end_offset == o.end_offset and @start_offset == o.start_offset
+      @end == o.end and @start == o.start
     end
     alias :== :eql?
 
     def hash() 
-      29 * @start_offset + @end_offset
+      29 * @start + @end
     end
   end
 end

data/lib/ferret/index/term_vectors_io.rb

@@ -145,7 +145,7 @@ module Ferret::Index
 
           vector.size.times do |j|
             add_term_internal(vector.terms[j],
-                              vector.term_frequencies[j],
+                              vector.freqs[j],
                               store_positions ? vector.positions[j] : nil,
                               store_offsets ? vector.offsets[j] : nil)
           end
@@ -247,11 +247,11 @@ module Ferret::Index
             # use delta encoding for offsets
             position = 0
             term.freq.times do |j|
-              @tvf.write_vint(term.offsets[j].start_offset - position)
+              @tvf.write_vint(term.offsets[j].start - position)
               #Save the diff between the two.
-              @tvf.write_vint(term.offsets[j].end_offset -
-                              term.offsets[j].start_offset)
-              position = term.offsets[j].end_offset()
+              @tvf.write_vint(term.offsets[j].end -
+                              term.offsets[j].start)
+              position = term.offsets[j].end()
             end
           end
         end

data/lib/ferret/query_parser/query_parser.tab.rb

@@ -11,7 +11,7 @@ module Ferret
 
   class QueryParser < Racc::Parser
 
-module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20b', 'lib/ferret/query_parser/query_parser.y', 126
+module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id0396ae54ac', 'lib/ferret/query_parser/query_parser.y', 126
   attr_accessor :default_field, :fields, :handle_parse_errors
 
   def initialize(default_field = "*", options = {})
@@ -170,22 +170,23 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
   end
 
   def get_bad_query(field, str)
-    tokens = []
-    stream = @analyzer.token_stream(field, str)
-    while token = stream.next
-      tokens << token
-    end
-    if tokens.length == 0
-      return TermQuery.new(Term.new(field, ""))
-    elsif tokens.length == 1
-      return TermQuery.new(Term.new(field, tokens[0].term_text))
-    else
-      bq = BooleanQuery.new()
-      tokens.each do |token|
-        bq << BooleanClause.new(TermQuery.new(Term.new(field, token.term_text)))
-      end
-      return bq
-    end
+    get_term_query(field, str)
+    #tokens = []
+    #stream = @analyzer.token_stream(field, str)
+    #while token = stream.next
+    #  tokens << token
+    #end
+    #if tokens.length == 0
+    #  return TermQuery.new(Term.new(field, ""))
+    #elsif tokens.length == 1
+    #  return TermQuery.new(Term.new(field, tokens[0].text))
+    #else
+    #  bq = BooleanQuery.new()
+    #  tokens.each do |token|
+    #    bq << BooleanClause.new(TermQuery.new(Term.new(field, token.text)))
+    #  end
+    #  return bq
+    #end
   end
 
   def get_range_query(field, start_word, end_word, inc_upper, inc_lower)
@@ -201,11 +202,11 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
     if tokens.length == 0
       return TermQuery.new(Term.new(field, ""))
     elsif tokens.length == 1
-      return TermQuery.new(Term.new(field, tokens[0].term_text))
+      return TermQuery.new(Term.new(field, tokens[0].text))
     else
       pq = PhraseQuery.new()
       tokens.each do |token|
-        pq.add(Term.new(field, token.term_text), nil, token.position_increment)
+        pq.add(Term.new(field, token.text), nil, token.pos_inc)
       end
       return pq
     end
@@ -216,9 +217,9 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
     stream = @analyzer.token_stream(field, word)
     if token = stream.next # only makes sense to look at one term for fuzzy
       if min_sim
-        return FuzzyQuery.new(Term.new(field, token.term_text), min_sim.to_f)
+        return FuzzyQuery.new(Term.new(field, token.text), min_sim.to_f)
       else
-        return FuzzyQuery.new(Term.new(field, token.term_text))
+        return FuzzyQuery.new(Term.new(field, token.text))
       end
     else
       return TermQuery.new(Term.new(field, ""))
@@ -254,8 +255,8 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
         tokens << token
       end
       tokens.each do |token|
-        pq.add(Term.new(field, token.term_text), nil,
-               token.position_increment + pos_inc)
+        pq.add(Term.new(field, token.text), nil,
+               token.pos_inc + pos_inc)
         pos_inc = 0
       end
     end
@@ -278,7 +279,7 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
         position.each do |word|
           stream = @analyzer.token_stream(field, word)
           if token = stream.next # only put one term per word
-            terms << Term.new(field, token.term_text)
+            terms << Term.new(field, token.text)
           end
         end
         mpq.add(terms, nil, pos_inc + 1) # must go at least one forward
@@ -290,8 +291,8 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
           tokens << token
         end
         tokens.each do |token|
-          mpq.add([Term.new(field, token.term_text)], nil,
-                 token.position_increment + pos_inc)
+          mpq.add([Term.new(field, token.text)], nil,
+                 token.pos_inc + pos_inc)
           pos_inc = 0
         end
       end
@@ -300,8 +301,15 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
   end
 
   def get_phrase_query(positions, slop = nil)
-    if positions.size == 1 and not positions[0].is_a?(Array)
-      return _get_term_query(positions[0])
+    if positions.size == 1 
+      if positions[0].is_a?(Array)
+        clauses = positions[0].map { |word|
+          BooleanClause.new(_get_term_query(word), BooleanClause::Occur::SHOULD)
+        }
+        return get_boolean_query(clauses)
+      else
+        return _get_term_query(positions[0])
+      end
     end
 
     multi_phrase = false
@@ -402,7 +410,7 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20
     return qp.parse(query)
   end
 
-..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20b
+..end lib/ferret/query_parser/query_parser.y modeval..id0396ae54ac
 
 ##### racc 1.4.4 generates ###
 
@@ -416,14 +424,14 @@ racc_reduce_table = [
  2, 28, :_reduce_6,
  2, 28, :_reduce_7,
  1, 28, :_reduce_8,
- 1, 30, :_reduce_none,
- 3, 30, :_reduce_10,
- 1, 29, :_reduce_none,
- 3, 29, :_reduce_12,
- 1, 29, :_reduce_none,
- 1, 29, :_reduce_none,
- 1, 29, :_reduce_none,
  1, 29, :_reduce_none,
+ 3, 29, :_reduce_10,
+ 1, 30, :_reduce_none,
+ 3, 30, :_reduce_12,
+ 1, 30, :_reduce_none,
+ 1, 30, :_reduce_none,
+ 1, 30, :_reduce_none,
+ 1, 30, :_reduce_none,
  1, 31, :_reduce_17,
  3, 31, :_reduce_18,
  2, 31, :_reduce_19,
@@ -461,15 +469,15 @@ racc_reduce_n = 48
 racc_shift_n = 78
 
 racc_action_table = [
-     8,    10,    60,    59,    75,    74,    50,    21,     2,    25,
-   -26,     7,     9,    41,    13,    15,    17,    19,     8,    10,
-     3,    43,    64,    26,   -26,    21,     2,    40,    38,     7,
-     9,    63,    13,    15,    17,    19,     8,    10,     3,    36,
-    46,    53,    37,    21,     2,    49,    34,     7,     9,    45,
+     8,    10,    67,    66,    75,    74,    50,    21,     2,    40,
+    25,     7,     9,    38,    13,    15,    17,    19,     8,    10,
+     3,    53,    46,    39,    26,    21,     2,    37,   -26,     7,
+     9,    45,    13,    15,    17,    19,     8,    10,     3,    43,
+    64,    49,   -26,    21,     2,    60,    59,     7,     9,    63,
     13,    15,    17,    19,    58,    57,     3,     8,    10,    31,
     33,    54,    55,    56,    21,     2,    44,    48,     7,     9,
-    61,    13,    15,    17,    19,    67,    66,     3,     8,    10,
-    31,    33,    62,    42,    65,    21,     2,    39,    30,     7,
+    61,    13,    15,    17,    19,    36,    62,     3,     8,    10,
+    31,    33,    34,    42,    65,    21,     2,    41,    30,     7,
      9,    70,    13,    15,    17,    19,     8,    10,     3,    71,
     72,    73,    24,    21,     2,    77,   nil,     7,     9,   nil,
     13,    15,    17,    19,    21,     2,     3,   nil,     7,     9,
@@ -480,61 +488,57 @@ racc_action_table = [
      3 ]
 
 racc_action_check = [
-     0,     0,    38,    38,    64,    64,    30,     0,     0,     6,
-    21,     0,     0,    17,     0,     0,     0,     0,     2,     2,
-     0,    21,    42,     6,    21,     2,     2,    17,    15,     2,
-     2,    42,     2,     2,     2,     2,    33,    33,     2,    13,
-    24,    34,    15,    33,    33,    28,    13,    33,    33,    24,
+     0,     0,    46,    46,    64,    64,    30,     0,     0,    17,
+     6,     0,     0,    15,     0,     0,     0,     0,     2,     2,
+     0,    34,    24,    17,     6,     2,     2,    15,    21,     2,
+     2,    24,     2,     2,     2,     2,    33,    33,     2,    21,
+    42,    28,    21,    33,    33,    38,    38,    33,    33,    42,
     33,    33,    33,    33,    37,    35,    33,    23,    23,    23,
     23,    35,    35,    35,    23,    23,    23,    26,    23,    23,
-    39,    23,    23,    23,    23,    46,    46,    23,    12,    12,
-    12,    12,    40,    19,    43,    12,    12,    16,    11,    12,
+    39,    23,    23,    23,    23,    13,    41,    23,    12,    12,
+    12,    12,    13,    19,    43,    12,    12,    18,    11,    12,
     12,    53,    12,    12,    12,    12,    31,    31,    12,    54,
     55,    56,     3,    31,    31,    72,   nil,    31,    31,   nil,
-    31,    31,    31,    31,    49,    49,    31,   nil,    49,    49,
-   nil,    49,    49,    49,    49,    25,    25,    49,   nil,    25,
-    25,   nil,    25,    25,    25,    25,     8,     8,    25,   nil,
-     8,     8,   nil,     8,     8,     8,     8,    10,    10,     8,
-   nil,    10,    10,   nil,    10,    10,    10,    10,   nil,   nil,
-    10 ]
+    31,    31,    31,    31,     8,     8,    31,   nil,     8,     8,
+   nil,     8,     8,     8,     8,    25,    25,     8,   nil,    25,
+    25,   nil,    25,    25,    25,    25,    10,    10,    25,   nil,
+    10,    10,   nil,    10,    10,    10,    10,    49,    49,    10,
+   nil,    49,    49,   nil,    49,    49,    49,    49,   nil,   nil,
+    49 ]
 
 racc_action_pointer = [
-    -3,   nil,    15,    92,   nil,   nil,     7,   nil,   126,   nil,
-   137,    88,    75,    29,   nil,    18,    78,     3,   nil,    73,
-   nil,     8,   nil,    54,    30,   115,    57,   nil,    43,   nil,
-     6,    93,   nil,    33,    28,    45,   nil,    44,   -19,    60,
-    72,   nil,    12,    74,   nil,   nil,    54,   nil,   nil,   104,
+    -3,   nil,    15,    92,   nil,   nil,     8,   nil,   104,   nil,
+   126,    88,    75,    65,   nil,     3,   nil,    -1,    78,    73,
+   nil,    26,   nil,    54,    12,   115,    57,   nil,    39,   nil,
+     6,    93,   nil,    33,     8,    45,   nil,    44,    24,    60,
+   nil,    66,    30,    74,   nil,   nil,   -19,   nil,   nil,   137,
    nil,   nil,   nil,    81,    89,    87,    82,   nil,   nil,   nil,
    nil,   nil,   nil,   nil,   -17,   nil,   nil,   nil,   nil,   nil,
    nil,   nil,    95,   nil,   nil,   nil,   nil,   nil ]
 
 racc_action_default = [
    -48,   -14,   -48,   -48,   -15,   -16,   -48,   -20,   -48,   -23,
-   -48,   -48,    -1,   -48,    -2,   -48,    -9,   -48,    -8,   -48,
+   -48,   -48,    -1,   -48,    -2,   -48,    -8,   -48,    -9,   -48,
    -11,   -17,   -13,   -48,   -48,   -48,   -48,    -6,   -48,    -7,
    -48,   -48,    -5,   -48,   -30,   -48,   -32,   -48,   -44,   -48,
-   -48,   -47,   -48,   -19,   -12,   -43,   -48,   -21,   -27,   -48,
+   -47,   -48,   -48,   -19,   -12,   -43,   -48,   -21,   -27,   -48,
     78,    -3,    -4,   -48,   -48,   -28,   -48,   -33,   -45,   -40,
-   -41,   -10,   -46,   -42,   -48,   -18,   -39,   -38,   -22,   -24,
+   -41,   -46,   -10,   -42,   -48,   -18,   -39,   -38,   -22,   -24,
    -31,   -35,   -48,   -34,   -37,   -36,   -25,   -29 ]
 
 racc_goto_table = [
-    27,    32,    29,    12,    68,    23,    11,    28,    76,    35,
-   nil,   nil,    32,   nil,   nil,   nil,   nil,    47,   nil,   nil,
-    51,   nil,    52,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,    69 ]
+    47,    32,    12,    27,    23,    29,    11,    68,    28,    76,
+    35,   nil,    32,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+    51,   nil,    52,   nil,    69 ]
 
 racc_goto_check = [
-     4,     3,     4,     2,    12,     2,     1,    13,    14,    15,
-   nil,   nil,     3,   nil,   nil,   nil,   nil,     4,   nil,   nil,
-     3,   nil,     3,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,     4 ]
+     5,     3,     2,     4,     2,     4,     1,    12,    13,    14,
+    15,   nil,     3,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+     3,   nil,     3,   nil,     5 ]
 
 racc_goto_pointer = [
-   nil,     6,     3,   -11,    -8,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   -43,    -2,   -61,    -4 ]
+   nil,     6,     2,   -11,    -5,   -25,   nil,   nil,   nil,   nil,
+   nil,   nil,   -40,    -1,   -60,    -3 ]
 
 racc_goto_default = [
    nil,   nil,   nil,    14,    16,    18,    20,    22,     1,     4,
@@ -617,8 +621,8 @@ Racc_token_to_s_table = [
 'top_query',
 'bool_query',
 'bool_clause',
-'query',
 'boosted_query',
+'query',
 'term_query',
 'field_query',
 'phrase_query',

data/lib/ferret/search.rb

@@ -44,6 +44,6 @@ require 'ferret/search/range_filter.rb'
 require 'ferret/search/query_filter.rb'
 require 'ferret/search/caching_wrapper_filter.rb'
 require 'ferret/search/filtered_query.rb'
-require 'ferret/search/match_all_docs_query.rb'
+require 'ferret/search/match_all_query.rb'
 require 'ferret/search/spans.rb'
 require 'ferret/search/index_searcher.rb'

data/lib/ferret/search/boolean_query.rb

@@ -82,7 +82,7 @@ module Ferret::Search
     # 
     # raises:: TooManyClauses if the new number of clauses exceeds the
     #          maximum clause number #max_clause_count()
-    def add_query(query, occur) 
+    def add_query(query, occur=BooleanClause::Occur::SHOULD) 
       add_clause(BooleanClause.new(query, occur))
     end
 
@@ -95,6 +95,7 @@ module Ferret::Search
       end
 
       @clauses << clause
+      self
     end
     alias :<< :add_clause
 

data/lib/ferret/search/field_sorted_hit_queue.rb

@@ -12,10 +12,10 @@ module Ferret::Search
 
     # Creates a hit queue sorted by the given list of fields.
     #
-    # reader::  Index to use.
+    # reader:: Index to use.
     # fields:: Field names, in priority order (highest priority first).
-    #          Cannot be +nil+ or empty.  size::  The number of hits to
-    #          retain.  Must be greater than zero.
+    #          Cannot be +nil+ or empty.
+    # size::   The number of hits to retain.  Must be greater than zero.
     # raises:: IOError
     def initialize(reader, fields, size)
       super(size)

data/lib/ferret/search/fuzzy_query.rb

@@ -113,7 +113,8 @@ module Ferret::Search
     def to_s(field = nil) 
       buffer = ""
       buffer << "#{@term.field}:" if @term.field != field
-      buffer << "#{@term.text}~#{minimum_similarity}"
+      buffer << "#{@term.text}~"
+      buffer << minimum_similarity.to_s if minimum_similarity != 0.5
       buffer << "^#{boost()}" if (boost() != 1.0) 
       return buffer
     end

data/lib/ferret/search/index_searcher.rb

@@ -92,6 +92,9 @@ module Ferret::Search
       num_docs = options[:num_docs]||10
       max_size = first_doc + num_docs
       sort = options[:sort]
+      if sort and not sort.kind_of?(Sort)
+        sort = Sort.new(sort)
+      end
 
       if (num_docs <= 0)
         raise ArgumentError, "num_docs must be > 0 to run a search"

data/lib/ferret/search/{match_all_docs_query.rb → match_all_query.rb}

@@ -1,6 +1,6 @@
 module Ferret::Search
   # A query that matches all documents.
-  class MatchAllDocsQuery < Query 
+  class MatchAllQuery < Query 
 
     def initialize() 
       super
@@ -20,7 +20,7 @@ module Ferret::Search
       end
 
       def explain(doc) 
-        return Explanation.new(1.0, "MatchAllDocsQuery")
+        return Explanation.new(1.0, "MatchAllQuery")
       end
 
       def next? 
@@ -43,7 +43,7 @@ module Ferret::Search
       end
     end
 
-    class MatchAllDocsWeight < Weight 
+    class MatchAllWeight < Weight 
       attr_reader :query
       def initialize(query, searcher) 
         @query = query
@@ -71,7 +71,7 @@ module Ferret::Search
 
       def explain(reader, doc) 
         # explain query weight
-        query_expl = Explanation.new(1.0, "MatchAllDocsQuery")
+        query_expl = Explanation.new(1.0, "MatchAllQuery")
         boost_expl = Explanation.new(@query.boost, "boost")
         if (boost_expl.value != 1.0)
           query_expl << boost_expl
@@ -83,17 +83,17 @@ module Ferret::Search
     end
 
     def create_weight(searcher) 
-      return MatchAllDocsWeight.new(self, searcher)
+      return MatchAllWeight.new(self, searcher)
     end
 
     def to_s(field) 
-      buffer = "MatchAllDocsQuery"
+      buffer = "MatchAllQuery"
       buffer << "^#{boost}" if (boost() != 1.0) 
       return buffer
     end
 
     def eql?(o) 
-      return (o.instance_of?(MatchAllDocsQuery) and boost == o.boost)
+      return (o.instance_of?(MatchAllQuery) and boost == o.boost)
     end
     alias :== :eql?