ferret 0.2.2 → 0.3.0

data/lib/ferret/document/field.rb

@@ -71,6 +71,12 @@ module Ferret::Document
     # field string
     def store_offsets?() return @store_offset end
 
+    # True if the norms are not stored for this field. No norms means that
+    # index-time boosting and field length normalization will be disabled.
+    # The benefit is less memory usage as norms take up one byte per indexed
+    # field for every document in the index.
+    def omit_norms?() return @omit_norms end
+    
     class Store < Ferret::Utils::Parameter
       # Store the original field value in the index in a compressed form.
       # This is useful for long documents and for binary valued fields.
@@ -101,6 +107,13 @@ module Ferret::Document
       # searched.  As no analyzer is used the value will be stored as a
       # single term. This is useful for unique Ids like product numbers.
       UNTOKENIZED = Index.new("UNTOKENIZED")
+
+      # Index the field's value without an Analyzer, and disable the storing
+      # of norms.  No norms means that index-time boosting and field length
+      # normalization will be disabled.  The benefit is less memory usage as
+      # norms take up one byte per indexed field for every document in the
+      # index.
+      NO_NORMS = Index.new("NO_NORMS");
     end
 
     class TermVector < Ferret::Utils::Parameter
@@ -174,13 +187,14 @@ module Ferret::Document
     end
 
     def stored=(stored)
-      if (stored == Store::YES)
+      case stored
+      when Store::YES
         @stored = true
         @compressed = false
-      elsif (stored == Store::COMPRESS)
+      when Store::COMPRESS
         @stored = true
         @compressed = true
-      elsif (stored == Store::NO)
+      when Store::NO
         @stored = false
         @compressed = false
       else
@@ -189,38 +203,45 @@ module Ferret::Document
     end
 
     def index=(index)
-      if (index == Index::NO)
+      @omit_norms = false
+      case index
+      when Index::NO
         @indexed = false
         @tokenized = false
-      elsif (index == Index::TOKENIZED)
+      when Index::TOKENIZED
         @indexed = true
         @tokenized = true
-      elsif (index == Index::UNTOKENIZED)
+      when Index::UNTOKENIZED
+        @indexed = true
+        @tokenized = false
+      when Index::NO_NORMS
         @indexed = true
         @tokenized = false
+        @omit_norms = true
       else
         raise "unknown stored parameter " + index.to_s
       end
     end
 
     def store_term_vector=(store_term_vector)
-      if (store_term_vector == TermVector::NO)
+      case store_term_vector
+      when TermVector::NO
         @store_term_vector = false
         @store_position = false
         @store_offset = false
-      elsif (store_term_vector == TermVector::YES)
+      when TermVector::YES
         @store_term_vector = true
         @store_position = false
         @store_offset = false
-      elsif (store_term_vector == TermVector::WITH_POSITIONS)
+      when TermVector::WITH_POSITIONS
         @store_term_vector = true
         @store_position = true
         @store_offset = false
-      elsif (store_term_vector == TermVector::WITH_OFFSETS)
+      when TermVector::WITH_OFFSETS
         @store_term_vector = true
         @store_position = false
         @store_offset = true
-      elsif (store_term_vector == TermVector::WITH_POSITIONS_OFFSETS)
+      when TermVector::WITH_POSITIONS_OFFSETS
         @store_term_vector = true
         @store_position = true
         @store_offset = true
@@ -284,6 +305,7 @@ module Ferret::Document
       str << "store_term_vector," if (@store_term_vector)
       str << "tv_offset," if (@store_offset)
       str << "tv_position," if (@store_position)
+      str << "omit_norms," if (@omit_norms)
       str << "binary," if (@binary)
       str << "<#{@name}:#{data}>"
     end  

data/lib/ferret/index/document_writer.rb

@@ -92,9 +92,10 @@ module Ferret::Index
 
           length = @field_lengths[field_number]     # length of field
           position = @field_positions[field_number] # position in field
+          position += @analyzer.position_increment_gap(field_name) if length > 0
           offset = @field_offsets[field_number]     # offset field
 
-          if field_info.indexed? 
+          if field_info.indexed?
             if not field.tokenized? # un-tokenized field
               string_value = field.string_value
               if field_info.store_offsets?
@@ -261,7 +262,7 @@ module Ferret::Index
 
       def write_norms(segment)
         @field_infos.each_with_index do |fi, i|
-          if fi.indexed?
+          if fi.indexed? and not fi.omit_norms?
             norm = @field_boosts[i] * @similarity.length_norm(fi.name, @field_lengths[i])
             norms = @directory.create_output(segment + ".f" + i.to_s)
             begin 

data/lib/ferret/index/field_infos.rb

@@ -35,7 +35,8 @@ module Ferret
               field.indexed?,
               field.store_term_vector?,
               field.store_positions?,
-              field.store_offsets?)
+              field.store_offsets?,
+              field.omit_norms?)
         end
       end
       alias :<< :add_doc_fields
@@ -45,9 +46,11 @@ module Ferret
                     indexed = true,
                     store_term_vector = false,
                     store_position = false,
-                    store_offset = false)
+                    store_offset = false,
+                    omit_norms = false)
         names.each do |name|
-          add(name, indexed, store_term_vector, store_position, store_offset)
+          add(name, indexed, store_term_vector, store_position,
+              store_offset, omit_norms)
         end
       end
 
@@ -65,10 +68,12 @@ module Ferret
               indexed = true,
               store_term_vector = false,
               store_position = false,
-              store_offset = false)
+              store_offset = false,
+              omit_norms = false)
         fi = @fi_hash[name]
         if (fi == nil)
-          fi = add_internal(name, indexed, store_term_vector, store_position, store_offset)
+          fi = add_internal(name, indexed, store_term_vector, store_position,
+                            store_offset, omit_norms)
         else
           if (fi.indexed? != indexed)
             fi.indexed = true             # once indexed, always index
@@ -82,6 +87,9 @@ module Ferret
           if (fi.store_offsets? != store_offset)
             fi.store_offset = true   # once vector, always vector
           end
+          if (fi.omit_norms? != omit_norms)
+            fi.omit_norms = false   # once norms are stored, always store norms
+          end
         end
         return fi
       end
@@ -174,7 +182,9 @@ module Ferret
             store_term_vector = (bits & STORE_TERM_VECTOR) != 0
             store_position = (bits & STORE_POSITION) != 0
             store_offset = (bits & STORE_OFFSET) != 0
-            add_internal(name, indexed, store_term_vector, store_position, store_offset)
+            omit_norms = (bits & OMIT_NORMS) != 0
+            add_internal(name, indexed, store_term_vector, store_position,
+                         store_offset, omit_norms)
           end
         end
 
@@ -183,15 +193,18 @@ module Ferret
         STORE_TERM_VECTOR = 0x2;
         STORE_POSITION = 0x4;
         STORE_OFFSET = 0x8;
+        OMIT_NORMS = 0x10;
 
         def add_internal(name, indexed, store_term_vector,
                          store_position = false,
-                         store_offset = false)
+                         store_offset = false,
+                         omit_norms = false)
           fi = FieldInfo.new(name, indexed,
                              @fi_array.size(),
                              store_term_vector,
                              store_position,
-                             store_offset)
+                             store_offset,
+                             omit_norms)
           @fi_array << fi
           @fi_hash[name] = fi
           return fi
@@ -211,13 +224,17 @@ module Ferret
           if (fi.store_offsets?)
             bits |= STORE_OFFSET
           end
+          if (fi.omit_norms?)
+            bits |= OMIT_NORMS
+          end
           return bits
         end
     end
 
     class FieldInfo
       attr_accessor :name, :number
-      attr_writer :indexed, :store_term_vector, :store_offset, :store_position
+      attr_writer :indexed, :store_term_vector, :store_offset,
+        :store_position, :omit_norms
 
       def indexed?()
         return @indexed
@@ -230,23 +247,32 @@ module Ferret
       def store_offsets?()
         return @store_offset
       end
+
       def store_positions?()
         return @store_position
       end
 
-      def set!(indexed, store_term_vector, store_position, store_offset)
+      def omit_norms?()
+        return @omit_norms
+      end
+
+      def set!(indexed, store_term_vector, store_position,
+               store_offset, omit_norms)
         @indexed = indexed
         @store_term_vector = store_term_vector
         @store_position = store_position
         @store_offset = store_offset
+        @omit_norms = omit_norms
       end
 
       def initialize(name, indexed, number, store_term_vector,
                      store_position = false,
-                     store_offset = false)
+                     store_offset = false,
+                     omit_norms = false)
         @name = name
         @number = number
-        set!(indexed, store_term_vector, store_position, store_offset)
+        set!(indexed, store_term_vector, store_position,
+             store_offset, omit_norms)
       end
     end
   end

data/lib/ferret/index/fields_io.rb

@@ -55,10 +55,16 @@ module Ferret::Index
           end
         else 
           store = Field::Store::YES
-          if fi.indexed? and tokenize
-            index = Field::Index::TOKENIZED
-          elsif fi.indexed? and not tokenize
-            index = Field::Index::UNTOKENIZED
+          if fi.indexed?
+            if tokenize
+              index = Field::Index::TOKENIZED
+            else
+              if fi.omit_norms?
+                index = Field::Index::NO_NORMS
+              else
+                index = Field::Index::UNTOKENIZED
+              end
+            end
           else
             index = Field::Index::NO
           end

data/lib/ferret/index/index.rb

@@ -34,7 +34,8 @@ module Ferret::Index
     #                        used when you add a simple string to the index
     #                        using #add_document. This will also be used for
     #                        default_search_field unless you set it
-    #                        explicitly.
+    #                        explicitly. The default for this value is the
+    #                        empty string "".
     # default_search_field:: This specifies the field or fields that will be 
     #                        searched by the query parser. You can use a
     #                        string to specify one field, eg, "title". Or you
@@ -93,13 +94,20 @@ module Ferret::Index
     #                        your query.  This defualts to true. If you set it
     #                        to false a QueryParseException is raised on a
     #                        query parse error.
+    # auto_flush::           Set this option to true if you want the index
+    #                        automatically flushed every time you do a write
+    #                        (includes delete) to the index. This is useful if
+    #                        you have multiple processes accessing the index
+    #                        and you don't want lock errors. This is set to
+    #                        false by default.
     # 
     # Some examples;
     #
     #   index = Index::Index.new(:analyzer => WhiteSpaceAnalyzer.new())
     #
     #   index = Index::Index.new(:path => '/path/to/index',
-    #                            :create_if_missing => false)
+    #                            :create_if_missing => false,
+    #                            :auto_flush => true)
     #
     #   index = Index::Index.new(:dir => directory,
     #                            :close_dir => false
@@ -126,12 +134,15 @@ module Ferret::Index
 
       @dir.synchronize do
         @options = options
-        @writer = IndexWriter.new(@dir, options)
+        @writer = IndexWriter.new(@dir, options) # create the index if need be
         options[:analyzer] = @analyzer = @writer.analyzer
+        @writer.close
+        @writer = nil
         @has_writes = false
         @reader = nil
         @options.delete(:create) # only want to create the first time if at all
         @close_dir = @options.delete(:close_dir) || false # we'll hold this here
+        @auto_flush = @options[:auto_flush] || false
         @default_search_field = (@options[:default_search_field] || \
                                  @options[:default_field] || "*")
         @default_field = @options[:default_field] || ""
@@ -257,6 +268,7 @@ module Ferret::Index
         ensure_writer_open()
         @has_writes = true
         @writer.add_document(fdoc, analyzer || @writer.analyzer)
+        flush() if @auto_flush
       end
     end
     alias :<< :add_document
@@ -334,6 +346,7 @@ module Ferret::Index
         else
           raise ArgumentError, "Cannot delete for id of type #{id.class}"
         end
+        flush() if @auto_flush
       end
     end
 
@@ -349,6 +362,7 @@ module Ferret::Index
         @searcher.search_each(query) do |doc, score|
           @reader.delete(doc)
         end
+        flush() if @auto_flush
       end
     end
 
@@ -393,6 +407,7 @@ module Ferret::Index
         else
           raise ArgumentError, "Cannot update for id of type #{id.class}"
         end
+        flush() if @auto_flush
       end
     end
 
@@ -429,6 +444,7 @@ module Ferret::Index
         docs_to_add.each do |document|
           @writer.add_document(document)
         end
+        flush() if @auto_flush
       end
     end
 
@@ -532,7 +548,7 @@ module Ferret::Index
     #             false.
     def persist(directory, create = true)
       synchronize do
-        flush
+        flush()
         old_dir = @dir
         if directory.is_a?(String)
           @dir = FSDirectory.new(directory, create)

data/lib/ferret/index/index_reader.rb

@@ -143,7 +143,7 @@ module Ferret::Index
     #           stored for the specified document.
     # raises:: IOError if index cannot be accessed
     #
-    # See Field.TermVector
+    # See Field::TermVector
     def get_term_vectors(doc_number)
       raise NotImplementedError
     end
@@ -161,7 +161,7 @@ module Ferret::Index
     # returns:: term vector May be nil if field does not exist in the specified
     #           document or term vector was not stored.
     # raises:: IOError if index cannot be accessed
-    # See Field.TermVector
+    # See Field::TermVector
     def get_term_vector(doc_number, field)
       raise NotImplementedError
     end
@@ -223,12 +223,27 @@ module Ferret::Index
     def has_deletions?()
       raise NotImplementedError
     end
-    
+
+    # Returns true if there are norms stored for this field.
+    def has_norms?(field)
+      # backward compatible implementation.
+      # SegmentReader has an efficient implementation.
+      return (get_norms(field) != nil)
+    end
+   
     # Returns the byte-encoded normalization factor for the named field of
     # every document.  This is used by the search code to score documents.
     # 
     # See Field#boost
-    def get_norms(field, bytes=nil, offset=nil)
+    def get_norms(field)
+      raise NotImplementedError
+    end
+
+    # Read norms into a pre-allocated array. This is used as an optimization
+    # of get_norms.
+    # 
+    # See Field#boost
+    def get_norms_into(field, bytes, offset)
       raise NotImplementedError
     end
 

data/lib/ferret/index/index_writer.rb

@@ -455,7 +455,7 @@ module Index
         write_deleteable_files(deletable) # note files we can't delete
         # This is a great time to start the garbage collector as all of our
         # ram files have just become free
-        GC.start
+        #GC.start
 
 ##############################################################################
 #          objs = {}

data/lib/ferret/index/multi_reader.rb

@@ -110,12 +110,20 @@ module Ferret::Index
       return hi
     end
 
+    def has_norms?(field)
+      @sub_readers.each {|reader| return true if reader.has_norms?(field)}
+      return false
+    end
+  
+    def fake_norms()
+      return @ones ||= SegmentReader.create_fake_norms(max_doc())
+    end
+
     def get_norms(field)
       synchronize do
         bytes = @norms_cache[field]
-        if (bytes != nil)
-          return bytes    # cache hit
-        end
+        return bytes if bytes
+        return fake_norms if not has_norms?(field)
 
         bytes = " " * @max_doc
         @sub_readers.length.times do |i|
@@ -129,7 +137,9 @@ module Ferret::Index
     def get_norms_into(field, buf, offset)
       synchronize do
         bytes = @norms_cache[field]
-        if (bytes != nil)                            # cache hit
+        bytes = fake_norms() if (bytes.nil? and not has_norms?(field))
+
+        if (bytes)                       # cache hit
           buf[offset ,@max_doc] = bytes[0, @max_doc]
           return
         end
@@ -220,14 +230,14 @@ module Ferret::Index
     def next?()
       top = @queue.top()
       if (top == nil) 
-        @term = nil
+        @term_buffer = nil
         return false
       end
 
-      @term = top.term
+      @term = top.term_buffer.term
       @doc_freq = 0
 
-      while top and @term == top.term
+      while top and @term == top.term_buffer
         @queue.pop()
         @doc_freq += top.term_enum.doc_freq() # increment freq
         if (top.next?)
@@ -240,6 +250,10 @@ module Ferret::Index
       return true
     end
 
+    #def term()
+    #  @term_buffer.term if @term_buffer
+    #end
+
     def close()
       @queue.close()
     end