ferret 0.3.2 → 0.9.0

data/lib/ferret.rb

@@ -22,20 +22,19 @@
 #++
 # :include: ../TUTORIAL
 module Ferret
-  VERSION = '0.3.2'
+  VERSION = '0.9.0'
 end
 
-require 'ferret/utils'
-require 'ferret/document'
-require 'ferret/stemmers'
-require 'ferret/analysis'
-require 'ferret/store'
-require 'ferret/index'
-require 'ferret/search'
-require 'ferret/query_parser'
-
 # try and load the C extension but it isn't necessary.
 begin
   require 'ferret_ext'
 rescue Exception => e 
+  require 'ferret/utils'
+  require 'ferret/document'
+  require 'ferret/stemmers'
+  require 'ferret/analysis'
+  require 'ferret/store'
+  require 'ferret/index'
+  require 'ferret/search'
+  require 'ferret/query_parser'
 end

data/lib/ferret/analysis/analyzers.rb

@@ -28,10 +28,10 @@ module Ferret::Analysis
     # exact PhraseQuery matches, for instance, across Field instance boundaries.
     #
     # field_name::             Field name being indexed.
-    # position_increment_gap:: added to the next token emitted from
+    # pos_inc_gap:: added to the next token emitted from
     #                          #token_stream(String,Reader)
     #
-    def position_increment_gap(field_name)
+    def pos_inc_gap(field_name)
       return 0
     end
 

data/lib/ferret/analysis/standard_tokenizer.rb

@@ -64,7 +64,7 @@ if __FILE__ == $0
   $stdin.each do |line|
     stk = Ferret::Analysis::StandardTokenizer.new(line)
     while tk = stk.next()
-      puts "    <" + tk.term_text + "> from " + tk.start_offset.to_s + " to " + tk.end_offset.to_s
+      puts "    <" + tk.text + "> from " + tk.start_offset.to_s + " to " + tk.end_offset.to_s
     end
   end
 end

data/lib/ferret/analysis/token.rb

@@ -18,25 +18,25 @@ module Ferret::Analysis
   # end_offset:: is equal to one greater than the position of the last
   #              character corresponding of this token Note that the
   #              difference between @end_offset and @start_offset may not be
-  #              equal to @term_text.length(), as the term text may have been
+  #              equal to @text.length(), as the term text may have been
   #              altered by a stemmer or some other filter.
   class Token
     include Comparable
-    attr_accessor :term_text
-    attr_reader :position_increment, :start_offset, :end_offset, :type
+    attr_accessor :text
+    attr_reader :pos_inc, :start_offset, :end_offset, :type
 
     # Constructs a Token with the given term text, and start & end offsets.
     # The type defaults to "word."
-    def initialize(txt, so, eo, typ="word", pos_inc=1)
-      @term_text = txt
+    def initialize(txt, so, eo, pos_inc=1, typ="word")
+      @text = txt
       @start_offset = so
       @end_offset = eo
       @type = typ # lexical type
-      @position_increment = pos_inc
+      @pos_inc = pos_inc
     end
 
     def set!(txt, so, eo)
-      @term_text = txt
+      @text = txt
       @start_offset = so
       @end_offset = eo
       self
@@ -44,20 +44,20 @@ module Ferret::Analysis
 
     def eql?(o)
       return (o.instance_of?(Token) and @start_offset == o.start_offset and
-              @end_offset == o.end_offset and @term_text == o.term_text)
+              @end_offset == o.end_offset and @text == o.text)
     end
     alias :== :eql?
 
     # Tokens are sorted by the position in the text at which they occur, ie
     # the start_offset. If two tokens have the same start offset, (see
-    # position_increment=) then, they are sorted by the end_offset and then
+    # pos_inc=) then, they are sorted by the end_offset and then
     # lexically by the token text.
     def <=>(o)
       r = @start_offset <=> o.start_offset
       return r if r != 0
       r = @end_offset <=> o.end_offset
       return r if r != 0
-      r = @term_text <=> o.term_text
+      r = @text <=> o.text
       return r
     end
 
@@ -82,17 +82,17 @@ module Ferret::Analysis
     #   words and also sets the increment to the number of stop words removed
     #   before each non-stop word.  Then exact phrase queries will only match
     #   when the terms occur with no intervening stop words.
-    def position_increment=(pos_inc)
+    def pos_inc=(pos_inc)
       if (pos_inc < 0)
         raise ArgumentError, "Increment must be zero or greater: " + pos_inc
       end
-      @position_increment = pos_inc
+      @pos_inc = pos_inc
     end
 
     # Returns a string representation of the token with all the attributes.
     def to_s
-      buf = "#{term_text}:#{start_offset}->#{end_offset}"
-      buf << "(pos_inc=#{@position_increment})" if (@position_increment != 1)
+      buf = "#{text}:#{start_offset}->#{end_offset}"
+      buf << "(pos_inc=#{@pos_inc})" if (@pos_inc != 1)
       buf << "(type=#{@type})" if (@type != "word")
       buf
     end

data/lib/ferret/analysis/token_filters.rb

@@ -24,7 +24,7 @@ module Ferret::Analysis
         return nil
       end
 
-      t.term_text = t.term_text.downcase()
+      t.text = t.text.downcase()
 
       return t
     end
@@ -50,7 +50,7 @@ module Ferret::Analysis
     def next()
       # return the first non-stop word found
       while token = @input.next()
-        return token if ! @stop_set.include?(token.term_text)
+        return token if ! @stop_set.include?(token.text)
       end
       return nil
     end
@@ -78,7 +78,7 @@ module Ferret::Analysis
       if (token == nil)
         return nil
       else
-        token.term_text = Stemmable.stem_porter(token.term_text)
+        token.text = Stemmable.stem_porter(token.text)
       end
       token
     end

data/lib/ferret/document/field.rb

@@ -27,8 +27,7 @@ module Ferret::Document
     # Documents returned from IndexReader#document(int) and
     # Hits#doc(int) may thus not have the same value present as when this field
     # was indexed.
-    attr_accessor :boost, :data
-
+    attr_accessor :boost, :data 
     attr_reader :name
 
     # True iff the value of the field is to be stored in the index for
@@ -160,34 +159,34 @@ module Ferret::Document
     # this field more important.
     def initialize(name,
                    value,
-                   stored = Store::YES,
+                   store = Store::YES,
                    index = Index::UNTOKENIZED,
-                   store_term_vector = TermVector::NO,
+                   term_vector = TermVector::NO,
                    binary = false,
                    boost = 1.0)
-      if (index == Index::NO and stored == Store::NO)
+      if (index == Index::NO and store == Store::NO)
         raise ArgumentError, "it doesn't make sense to have a field that " +
           "is neither indexed nor stored"
       end
-      if (index == Index::NO && store_term_vector != TermVector::NO)
+      if (index == Index::NO && term_vector != TermVector::NO)
         raise ArgumentError, "cannot store term vector information for a " +
           "field that is not indexed"
       end
 
       # The name of the field (e.g., "date", "subject", "title", or "body")
-      @name = name
+      @name = name.to_s
 
       # the one and only data object for all different kind of field values
       @data = value
-      self.stored = stored
+      self.store = store
       self.index = index
-      self.store_term_vector = store_term_vector
+      self.term_vector = term_vector
       @binary = binary
       @boost = boost
     end
 
-    def stored=(stored)
-      case stored
+    def store=(store)
+      case store
       when Store::YES
         @stored = true
         @compressed = false
@@ -198,7 +197,7 @@ module Ferret::Document
         @stored = false
         @compressed = false
       else
-        raise "unknown stored parameter " + stored.to_s
+        raise "unknown stored parameter " + store.to_s
       end
     end
 
@@ -223,8 +222,8 @@ module Ferret::Document
       end
     end
 
-    def store_term_vector=(store_term_vector)
-      case store_term_vector
+    def term_vector=(term_vector)
+      case term_vector
       when TermVector::NO
         @store_term_vector = false
         @store_position = false
@@ -303,11 +302,11 @@ module Ferret::Document
       str << "indexed," if (@indexed)
       str << "tokenized," if (@tokenized)
       str << "store_term_vector," if (@store_term_vector)
-      str << "tv_offset," if (@store_offset)
-      str << "tv_position," if (@store_position)
+      str << "store_offsets," if (@store_offset)
+      str << "store_positions," if (@store_position)
       str << "omit_norms," if (@omit_norms)
       str << "binary," if (@binary)
-      str << "<#{@name}:#{data}>"
+      str << "<#{@name}:#{@binary ? '=bin_data=' : data}>"
     end  
   end
 end

data/lib/ferret/index/document_writer.rb

@@ -92,7 +92,7 @@ module Ferret::Index
 
           length = @field_lengths[field_number]     # length of field
           position = @field_positions[field_number] # position in field
-          position += @analyzer.position_increment_gap(field_name) if length > 0
+          position += @analyzer.pos_inc_gap(field_name) if length > 0
           offset = @field_offsets[field_number]     # offset field
 
           if field_info.indexed?
@@ -120,18 +120,18 @@ module Ferret::Index
               begin 
                 last_token = nil
                 while token = stream.next
-                  position += (token.position_increment - 1)
+                  position += (token.pos_inc - 1)
                   
                   if(field_info.store_offsets?())
                     add_position(field_name,
-                                 token.term_text(),
+                                 token.text(),
                                  position,
                                  TermVectorOffsetInfo.new(
                                    offset + token.start_offset(),
                                    offset + token.end_offset()))
                     position += 1
                   else
-                    add_position(field_name, token.term_text(), position, nil)
+                    add_position(field_name, token.text(), position, nil)
                     position += 1
                   end
                   

data/lib/ferret/index/index.rb

@@ -32,10 +32,15 @@ module Ferret::Index
     #                        use the create_if_missing option.
     # default_field::        This specifies the default field that will be
     #                        used when you add a simple string to the index
-    #                        using #add_document. This will also be used for
-    #                        default_search_field unless you set it
+    #                        using #add_document or <<. This will also be used
+    #                        for default_search_field unless you set it
     #                        explicitly. The default for this value is the
-    #                        empty string "".
+    #                        string "id".
+    # id_field:              This field is as the field to search when doing
+    #                        searches on a term. For example, if you do a
+    #                        lookup by term "cat", ie index["cat"], this will
+    #                        be the field that is searched. This will default
+    #                        to default_field if not set.
     # default_search_field:: This specifies the field or fields that will be 
     #                        searched by the query parser. You can use a
     #                        string to specify one field, eg, "title". Or you
@@ -54,7 +59,7 @@ module Ferret::Index
     #                        in-memory index which you'd like to read with
     #                        this class. If you want to create a new index,
     #                        you are better off passing in a path.
-    # close_dir::            This specifies whether you would this class to
+    # close_dir::            This specifies whether you want this class to
     #                        close the index directory when this class is
     #                        closed. This only has any meaning when you pass
     #                        in a directory object in the *dir* option, in
@@ -76,7 +81,9 @@ module Ferret::Index
     #                        as an existing document, the existing document will
     #                        be replaced by the new object. This will slow
     #                        down indexing so it should not be used if
-    #                        performance is a concern.
+    #                        performance is a concern. You must make sure that
+    #                        your key/keys are either untokenized or that they
+    #                        are not broken up by the analyzer.
     # use_compound_file::    Uses a compound file to store the index. This
     #                        prevents an error being raised for having too
     #                        many files open at the same time. The default is
@@ -117,13 +124,16 @@ module Ferret::Index
     def initialize(options = {})
       super()
 
-      options[:default_search_field] &&= options[:default_search_field].to_s
       options[:default_field] &&= options[:default_field].to_s
       options[:create_if_missing] = true if options[:create_if_missing].nil? 
       @key = [options[:key]].flatten if options[:key]
 
       if options[:path]
-        @dir = FSDirectory.new(options[:path], options[:create])
+        begin
+          @dir = FSDirectory.new(options[:path], options[:create])
+        rescue IOError => io
+          @dir = FSDirectory.new(options[:path], options[:create_if_missing])
+        end
         options[:close_dir] = true
       elsif options[:dir]
         @dir = options[:dir]
@@ -145,7 +155,8 @@ module Ferret::Index
         @auto_flush = @options[:auto_flush] || false
         @default_search_field = (@options[:default_search_field] || \
                                  @options[:default_field] || "*")
-        @default_field = @options[:default_field] || ""
+        @default_field = (@options[:default_field] || @options[:id_field] || "id").to_s
+        @id_field = (@options[:id_field] || @options[:default_field] || "id").to_s
         @options[:handle_parse_errors] = true if @options[:handle_parse_errors].nil?
         @open = true
         @qp = nil
@@ -261,7 +272,10 @@ module Ferret::Index
 
         # delete existing documents with the same key
         if @key
-          query = @key.map {|field| "+#{field}:#{fdoc[field]}" }.join(" ")
+          query = @key.inject(BooleanQuery.new()) do |bq, field|
+            bq.add_query(TermQuery.new(Term.new(field, fdoc[field])), 
+                         BooleanClause::Occur::MUST)
+          end
           query_delete(query)
         end
 
@@ -317,7 +331,7 @@ module Ferret::Index
       @dir.synchronize do
         ensure_reader_open()
         if id.is_a?(String)
-          t = Term.new("id", id.to_s)
+          t = Term.new(@id_field, id.to_s)
           return @reader.get_document_with_term(t)
         elsif id.is_a?(Term)
           return @reader.get_document_with_term(id)
@@ -338,7 +352,7 @@ module Ferret::Index
         cnt = 0
         ensure_reader_open()
         if id.is_a?(String)
-          t = Term.new("id", id.to_s)
+          t = Term.new(@id_field, id.to_s)
           cnt = @reader.delete_docs_with_term(t)
         elsif id.is_a?(Term)
           cnt = @reader.delete_docs_with_term(id)
@@ -398,7 +412,7 @@ module Ferret::Index
           document = doc(id)
           if new_val.is_a?(Hash)
             new_val.each_pair {|name, content| document[name] = content.to_s}
-          elsif new_val.is_a?(Document)
+          elsif new_val.is_a?(Ferret::Document::Document)
             document = new_val
           else
             document[@options[:default_field]] = new_val.to_s
@@ -478,6 +492,7 @@ module Ferret::Index
         @reader = nil
         @writer = nil
         @searcher = nil
+        @has_writes = false
       end
     end
 
@@ -575,27 +590,28 @@ module Ferret::Index
         @writer = IndexWriter.new(@dir, @options)
       end
 
+      # returns the new reader if one is opened
       def ensure_reader_open()
         raise "tried to use a closed index" if not @open
         if @reader
           if not @reader.latest?
-            @reader = IndexReader.open(@dir, false)
+            return @reader = IndexReader.open(@dir, false)
           end
-          return
-        end
-
-        if @writer
-          @writer.close
-          @writer = nil
+        else
+          if @writer
+            @writer.close
+            @writer = nil
+          end
+          return @reader = IndexReader.open(@dir, false)
         end
-        @reader = IndexReader.open(@dir, false)
+        return false
       end
 
       def ensure_searcher_open()
         raise "tried to use a closed index" if not @open
-        return if @searcher
-        ensure_reader_open()
-        @searcher = IndexSearcher.new(@reader)
+        if ensure_reader_open() or not @searcher
+          @searcher = IndexSearcher.new(@reader)
+        end
       end
 
     private

data/lib/ferret/index/index_writer.rb

@@ -28,8 +28,8 @@ module Index
 
     WRITE_LOCK_TIMEOUT = 1
     COMMIT_LOCK_TIMEOUT = 10
-    WRITE_LOCK_NAME = "write.lock"
-    COMMIT_LOCK_NAME = "commit.lock"
+    WRITE_LOCK_NAME = "write"
+    COMMIT_LOCK_NAME = "commit"
     DEFAULT_MERGE_FACTOR = 10 
     DEFAULT_MIN_MERGE_DOCS = 10
     DEFAULT_MAX_MERGE_DOCS = 0x7fffffff

data/lib/ferret/index/multiple_term_doc_pos_enum.rb

@@ -4,6 +4,7 @@ module Ferret::Index
   # @author Anders Nielsen
   class MultipleTermDocPosEnum < TermDocEnum 
 
+    attr_accessor :doc, :freq
     class TermPositionsQueue < Ferret::Utils::PriorityQueue 
       def initialize(term_positions)
         super(term_positions.size)
@@ -76,14 +77,6 @@ module Ferret::Index
       return next?
     end
 
-    def doc() 
-      return @doc
-    end
-
-    def freq() 
-      return @freq
-    end
-
     def close()
       while (tps = @tps_queue.pop())
         tps.close()