ferret 0.2.2 → 0.3.0

data/lib/ferret/index/segment_merge_info.rb

@@ -1,44 +1,46 @@
 module Ferret
   module Index
     class SegmentMergeInfo 
-      attr_reader :term, :term_enum, :reader, :postings, :doc_map, :base
+      attr_reader :term_enum, :reader, :base, :term_buffer
 
       def initialize(base, term_enum, reader)
         @base = base
         @reader = reader
         @term_enum = term_enum
-        @term = term_enum.term()
-        @postings = @reader.term_positions()
+        @term_buffer = term_enum.term_buffer
+      end
+
+      def positions
+        @postings ||= @reader.term_positions()
+      end
 
-        # build array which maps document numbers around deletions 
-        if (@reader.has_deletions?()) 
-          max_doc = @reader.max_doc()
-          @doc_map = Array.new(max_doc)
-          j = 0
-          max_doc.times do |i|
-            if (@reader.deleted?(i))
-              @doc_map[i] = -1
-            else
-              @doc_map[i] = j
-              j += 1
+      def doc_map
+        if @doc_map.nil?
+          # build array which maps document numbers around deletions 
+          if (@reader.has_deletions?()) 
+            max_doc = @reader.max_doc()
+            @doc_map = Array.new(max_doc)
+            j = 0
+            max_doc.times do |i|
+              if (@reader.deleted?(i))
+                @doc_map[i] = -1
+              else
+                @doc_map[i] = j
+                j += 1
+              end
             end
           end
         end
+        return @doc_map
       end
 
       def next?
-        if @term_enum.next?
-          @term = @term_enum.term
-          return true
-        else 
-          @term = nil
-          return false
-        end
+        @term_enum.next?
       end
 
       def close()
         @term_enum.close()
-        @postings.close()
+        @postings.close() if @postings
         @reader = nil
       end
     end

data/lib/ferret/index/segment_merge_queue.rb

@@ -1,10 +1,10 @@
 module Ferret::Index
   class SegmentMergeQueue < Ferret::Utils::PriorityQueue 
     def less_than(sti_a, sti_b) 
-      if sti_a.term == sti_b.term
+      if sti_a.term_buffer == sti_b.term_buffer
         return sti_a.base < sti_b.base
       else
-        return sti_a.term < sti_b.term
+        return sti_a.term_buffer < sti_b.term_buffer
       end
     end
 

data/lib/ferret/index/segment_merger.rb

@@ -71,7 +71,7 @@ module Ferret::Index
 
       # Field norm files
       @field_infos.each_with_index do |fi, i|
-        if (fi.indexed?) 
+        if (fi.indexed? and not fi.omit_norms?) 
           files << "#{@segment}.f#{i}"
         end
       end
@@ -94,6 +94,21 @@ module Ferret::Index
       return files
     end
 
+    def add_indexed(reader, field_infos, field_names,
+                    store_term_vectors,
+                    store_position_with_term_vector,
+                    store_offset_with_term_vector)
+      field_names.each do |field|
+        field_infos.add(field, true,
+                        store_term_vectors,
+                        store_position_with_term_vector,
+                        store_offset_with_term_vector,
+                        !reader.has_norms?(field))
+      end
+    end
+    private :add_indexed
+  
+
     # 
     # returns:: The number of documents in all of the readers
     # raises:: IOError
@@ -101,11 +116,11 @@ module Ferret::Index
       @field_infos = FieldInfos.new()      # merge field names
       doc_count = 0
       @readers.each do |reader|
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION_OFFSET), true, true, true, true)
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION), true, true, true, false)
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_OFFSET), true, true, false, true)
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR), true, true, false, false)
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::INDEXED), true, false, false, false)
+        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION_OFFSET), true, true, true)
+        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION), true, true, false)
+        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_OFFSET), true, false, true)
+        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR), true, false, false)
+        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::INDEXED), false, false, false)
         @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::UNINDEXED), false)
       end
       @field_infos.write_to_dir(@directory, @segment + ".fnm")
@@ -186,10 +201,12 @@ module Ferret::Index
         match_size = 0     # pop matching terms
         match[match_size] = @queue.pop
         match_size += 1
-        term = match[0].term
+        #term = match[0].term
+        term_buffer = match[0].term_buffer
         top = @queue.top
 
-        while top and term == top.term
+        #while top and term == top.term
+        while top and term_buffer == top.term_buffer
           match[match_size] = @queue.pop
           match_size += 1
           top = @queue.top
@@ -227,7 +244,7 @@ module Ferret::Index
       if (df > 0) 
         # add an enbegin to the dictionary with pointers to prox and freq files
         @term_info.set_values!(df, freq_pointer, prox_pointer, (skip_pointer - freq_pointer))
-        @term_infos_writer.add(smis[0].term, @term_info)
+        @term_infos_writer.add(smis[0].term_buffer.term, @term_info)
       end
     end
 
@@ -244,7 +261,7 @@ module Ferret::Index
       reset_skip()
       n.times do |i|
         smi = smis[i]
-        postings = smi.postings
+        postings = smi.positions
         base = smi.base
         doc_map = smi.doc_map
   
@@ -315,7 +332,7 @@ module Ferret::Index
 
     def merge_norms()
       @field_infos.each_with_index do |fi, i|
-        if (fi.indexed?) 
+        if (fi.indexed? and not fi.omit_norms?) 
           output = @directory.create_output(@segment + ".f" + i.to_s)
           begin 
             @readers.each do |reader|

data/lib/ferret/index/segment_reader.rb

@@ -127,7 +127,7 @@ module Ferret::Index
       end
 
       @field_infos.each_with_index do |fi, i|
-        if (fi.indexed?)
+        if (fi.indexed? and not fi.omit_norms?)
           if @cfs_reader.nil?
             name = "#{@segment}.f#{i}"
           else
@@ -228,10 +228,22 @@ module Ferret::Index
       return field_set
     end
 
+    def has_norms?(field)
+      return @norms.has_key?(field)
+    end
+  
+    def SegmentReader.create_fake_norms(size)
+      Array.new(size, 1).pack("C*")
+    end
+  
+    def fake_norms()
+      return @ones ||= SegmentReader.create_fake_norms(max_doc())
+    end
+
     def get_norms(field)
       synchronize do
         norm = @norms[field]
-        if (norm == nil)               # not an indexed field
+        if (norm == nil)               # not an indexed field or omit norms
           return nil
         end
         if (norm.bytes == nil)         # value not yet read
@@ -258,7 +270,10 @@ module Ferret::Index
     def get_norms_into(field, bytes, offset)
       synchronize do
         norm = @norms[field]
-        return if (norm == nil) # use zeros in array
+        if (norm.nil?) 
+          bytes[offset, max_doc()] = fake_norms[0, max_doc()]
+          return
+        end
 
         if (norm.bytes != nil) # can copy from cache
           bytes[offset, max_doc()] = norm.bytes[0, max_doc()]
@@ -277,7 +292,7 @@ module Ferret::Index
 
     def open_norms(cfs_dir)
       @field_infos.each do |fi|
-        if (fi.indexed?) 
+        if (fi.indexed? and not fi.omit_norms?)
           # look first if there are separate norms in compound format
           file_name = @segment + ".s" + fi.number.to_s
           d = @directory

data/lib/ferret/index/segment_term_enum.rb

@@ -15,11 +15,9 @@ module Ferret::Index
 
       @term_buffer = TermBuffer.new()
       @prev_buffer = TermBuffer.new()
-      @scratch = nil # used for scanning
       @term_info = TermInfo.new()
 
       @index_pointer = 0
-      @format_m1skip_interval = nil
 
       first_int = @input.read_int()
 
@@ -61,8 +59,7 @@ module Ferret::Index
     
     #attr_accessors for the clone method
     attr_accessor :input, :term_buffer, :prev_buffer
-    protected :input, :input=, :term_buffer,
-      :term_buffer=, :prev_buffer, :prev_buffer=
+    protected :input, :input=, :prev_buffer, :prev_buffer=
 
     def initialize_copy(o)
       super
@@ -83,7 +80,7 @@ module Ferret::Index
     # Increments the enumeration to the next element.  True if one exists.
     def next?
       @position += 1
-      if (@position > @size - 1) 
+      if (@position >= @size) 
         @term_buffer.reset()
         return false
       end
@@ -117,13 +114,8 @@ module Ferret::Index
       return true
     end
 
-    # Optimized scan, without allocating new terms. 
     def scan_to(term)
-      if (@scratch == nil)
-        @scratch = TermBuffer.new()
-      end
-      @scratch.term = term
-      while (@scratch > @term_buffer and next?) do
+      while (term > @term_buffer and next?) do
       end
     end
 

data/lib/ferret/index/term_buffer.rb

@@ -2,21 +2,21 @@ module Ferret::Index
   class TermBuffer
     include Comparable
 
-    attr_reader :text, :text_length, :field
+    attr_reader :text_buf, :text_length, :field
 
     def initialize
-      @text = String.new
+      @text_buf = String.new
       @text_length = -1
       @field = nil
     end
 
     def hash()
-      return @text.hash + @field.hash
+      return text.hash + @field.hash
     end
 
     def <=>(other) 
       if (@field == other.field)
-        return text_str <=> other.text_str
+        return text <=> other.text
       end
       @field <=> other.field
     end
@@ -27,7 +27,7 @@ module Ferret::Index
       length = input.read_vint()
       total_length = start + length
       @text_length = total_length
-      input.read_chars(@text, start, length)
+      input.read_chars(@text_buf, start, length)
       @field = field_infos[input.read_vint()].name
     end
 
@@ -38,8 +38,8 @@ module Ferret::Index
       end
 
       # copy text into the buffer
-      @text_length = term.text.length
-      @text = term.text.clone
+      @text_buf = term.text.clone
+      @text_length = @text_buf.length
 
       @field = term.field
       @term = term
@@ -47,14 +47,15 @@ module Ferret::Index
 
     def set!(other) 
       @text_length = other.text_length
-      @text = other.text.clone if other.text
+      @text_buf = other.text_buf.clone if other.text_buf
       @field = other.field
       @term = other.term
     end
+    alias :initialize_copy :set!
 
     def reset() 
       @field = nil
-      @text = String.new
+      @text_buf = ""
       @text_length = 0
       @term = nil
     end
@@ -65,18 +66,14 @@ module Ferret::Index
       end
 
       if @term.nil?
-        @term = Term.new(@field, @text[0,@text_length].to_s)
+        @term = Term.new(@field, @text_buf[0,@text_length].to_s)
       end
       return @term
     end
     alias :term :to_term
 
-    def initialize_copy(o)
-      set!(o)
-    end
-
-    def text_str()
-      @text[0,@text_length]
+    def text()
+      @text_buf[0,@text_length]
     end
 
     def to_s()

data/lib/ferret/index/term_doc_enum.rb

@@ -73,13 +73,16 @@ module Ferret::Index
       if t.instance_of?(Term)
         ti = parent.term_infos[t]
       elsif t.is_a?(TermEnum)
+        ti = t.term_info()
+        # The following is being done in the Java version. I don't think it's
+        # necessary.
         # use comparison of fieldinfos to verify that term enum (t) belongs to the
         # same segment as this SegmentTermDocEnum
-        if (t.instance_of?(SegmentTermEnum) and t.field_infos == parent.field_infos)
-          ti = t.term_info()
-        else                                          # punt case
-          ti = parent.term_infos[t.term]
-        end
+        #if (t.instance_of?(SegmentTermEnum) and t.field_infos == parent.field_infos)
+        #  ti = t.term_info()
+        #else                                          # punt case
+        #  ti = parent.term_infos[t.term]
+        #end
       elsif t.is_a? TermInfo # this one is easy. That's exactly what we're looking for
         ti = t
       else

data/lib/ferret/index/term_enum.rb

@@ -33,7 +33,7 @@ module Ferret
       #
       # Behaves as if written:
       #
-      #   def skip_to(target_term)
+      #   def skip_to(target)
       #     while (target > term)
       #       if (!next()) return false
       #     end
@@ -41,7 +41,7 @@ module Ferret
       #   end
       #
       # Some implementations are considerably more efficient than that.
-      def skip_to(term)
+      def skip_to(target)
         while (target > term)
           return false if not next?
         end

data/lib/ferret/index/term_info.rb

@@ -21,10 +21,6 @@ module Ferret::Index
       @skip_offset = so
     end
 
-    def copy_of()
-      TermInfo.new(doc_freq, freq_pointer, prox_pointer, skip_offset)
-    end
-
     def ==(o)
       return false if !o.instance_of?(TermInfo)
       @doc_freq == o.doc_freq &&
@@ -35,7 +31,7 @@ module Ferret::Index
     alias eql? ==
 
     def to_s()
-      "TermInfo:df=#{@doc_freq}:fp=#{@freq_pointer}:pp=#{@prox_pointer}:so=#{@skip_offset}"
+      "TermInfo:df=#{doc_freq}:fp=#{freq_pointer}:pp=#{prox_pointer}:so=#{skip_offset}"
     end
   end
 end

data/lib/ferret/index/term_infos_io.rb

@@ -252,6 +252,8 @@ module Ferret::Index
 
       # Returns the offset of the greatest index entry which is less than or
       # equal to term.
+      #
+      # This method is rewritten in the C extension.
       def get_index_offset(term) 
         lo = 0            # binary search @index_terms[]
         hi = @index_terms.length - 1

data/lib/ferret/query_parser/query_parser.tab.rb

@@ -11,7 +11,7 @@ module Ferret
 
   class QueryParser < Racc::Parser
 
-module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd43492', 'lib/ferret/query_parser/query_parser.y', 126
+module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20b', 'lib/ferret/query_parser/query_parser.y', 126
   attr_accessor :default_field, :fields, :handle_parse_errors
 
   def initialize(default_field = "*", options = {})
@@ -53,11 +53,11 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd4349
       case str
       when /\A\s+/
         ;
-      when /\A[#{ECHR}]/
+      when /\A([#{EWCHR}]|[*?](?=:))/
         @q.push [ RESERVED[$&]||$&, $& ]
       when /\A(\&\&|\|\|)/
         @q.push [ RESERVED[$&], $& ]
-      when /\A(\\[#{ECHR}]|[^\s#{ECHR}])+[?*](\\[#{EWCHR}]|[^\s#{EWCHR}])*/
+      when /\A(\\[#{ECHR}]|[^\s#{ECHR}])*[?*](\\[#{EWCHR}]|[^\s#{EWCHR}])*/
         str = $'
         unescaped = $&.gsub(/\\(?!\\)/,"")
         @q.push [ :WILD_STRING, unescaped ]
@@ -82,8 +82,8 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd4349
     end
 
     @q.push([ false, '$' ])
-    #p @q
 
+    query = nil
     begin
       query = do_parse
     rescue Racc::ParseError => e
@@ -199,7 +199,7 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd4349
       tokens << token
     end
     if tokens.length == 0
-      return nil
+      return TermQuery.new(Term.new(field, ""))
     elsif tokens.length == 1
       return TermQuery.new(Term.new(field, tokens[0].term_text))
     else
@@ -221,7 +221,7 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd4349
         return FuzzyQuery.new(Term.new(field, token.term_text))
       end
     else
-      return nil
+      return TermQuery.new(Term.new(field, ""))
     end
   end
 
@@ -402,7 +402,7 @@ module_eval <<'..end lib/ferret/query_parser/query_parser.y modeval..id81dbd4349
     return qp.parse(query)
   end
 
-..end lib/ferret/query_parser/query_parser.y modeval..id81dbd43492
+..end lib/ferret/query_parser/query_parser.y modeval..id6e7f6ac20b
 
 ##### racc 1.4.4 generates ###