ferret 0.2.2 → 0.3.0

data/lib/ferret/search/phrase_scorer.rb

@@ -69,7 +69,6 @@ module Ferret::Search
     end
 
     def score()
-      #puts("scoring #{@first.doc}")
       raw = similarity().tf(@freq) * @value      # raw score
       return raw * Similarity.decode_norm(@norms[@first.doc])  # normalize
     end

data/lib/ferret/search/similarity.rb

@@ -39,13 +39,13 @@ module Ferret::Search
       end
       mantissa = b & 0x07           # 0x07 =  7 = 0b00000111
       exponent = (b >> 3) & 0x1F    # 0x1f = 31 = 0b00011111
-      return [0,0,(mantissa << 5),(exponent+48)].pack("cccc").unpack("f")[0]
+      return [0,0,(mantissa << 5),(exponent+48)].pack("cccc").unpack("e")[0]
     end
 
     def Similarity.float_to_byte(f) 
       if (f <= 0.0) then return 0 end
 
-      bits = [f].pack("f").unpack("cccc")
+      bits = [f].pack("e").unpack("cccc")
       mantissa = (bits[2] & 0xEf) >> 5 
       exponent = (bits[3] - 48)
 

data/lib/ferret/search/term_scorer.rb

@@ -17,8 +17,8 @@ module Ferret::Search
       super(similarity)
 
       @doc = 0
-      @docs = Array.new(32, 0) # buffered doc numbers
-      @freqs = Array.new(32, 0) # buffered term freqs
+      @docs = Array.new(SCORE_CACHE_SIZE, 0) # buffered doc numbers
+      @freqs = Array.new(SCORE_CACHE_SIZE, 0) # buffered term freqs
       @pointer = @pointer_max = 0;
       @score_cache = Array.new(SCORE_CACHE_SIZE)
 

data/lib/ferret/store/directory.rb

@@ -14,6 +14,8 @@ module Ferret::Store
   # called _open_input_ If there is a risk of simultaneous modifications of
   # the files then locks should be used. See Lock to find out how.
   class Directory
+    LOCK_PREFIX = "ferret-"
+
     # returns an array of strings, one for each file in the directory
     def each # :yeilds: file_name
       raise NotImplementedError

data/lib/ferret/store/fs_store.rb

@@ -192,6 +192,7 @@ module Ferret::Store
         if (@ref_count <= 0) then
           @@Directories.synchronize do
             @@Directories.delete(@dir.path)
+            close_internal
           end
         end
       end
@@ -206,6 +207,12 @@ module Ferret::Store
       # pass the name of the file that we are going to lock
       def initialize(lock_file)
         @lock_file = lock_file
+        #@clean = FSLock.make_finalizer(lock_file)
+        @clean = lambda { File.delete(lock_file) rescue nil}
+      end
+
+      def FSLock.make_finalizer(lock_file)
+        lambda { File.delete(lock_file) rescue nil}
       end
 
       # obtain the lock on the data source 
@@ -216,6 +223,7 @@ module Ferret::Store
             # create a file if none exists. If one already exists
             # then someone beat us to the lock so return false
             File.open(@lock_file, File::WRONLY|File::EXCL|File::CREAT) {|f|}
+            ObjectSpace.define_finalizer(self, @clean)
             return true
           rescue SystemCallError
             # lock was not obtained so sleep for timeout then try again.
@@ -223,7 +231,7 @@ module Ferret::Store
           end
         end
         # lock could not be obtained so raise an exception
-        raise "could not obtain lock: " + @lock_file.to_s
+        raise "could not obtain lock: #{@lock_file}"
       end 
 
       # Release the lock on the data source. Returns true if successful.
@@ -231,6 +239,7 @@ module Ferret::Store
         return if FSDirectory.locks_disabled?
         begin
           File.delete(@lock_file)
+          ObjectSpace.undefine_finalizer(self)
         rescue SystemCallError
           # maybe we tried to release a lock that wasn't locked. This
           # isn't critical so just return false
@@ -246,7 +255,8 @@ module Ferret::Store
       end
     end
 
-    # A file system output stream extending OutputStream to read from the file system
+    # A file system output stream extending OutputStream to read from the file
+    # system
     class FSIndexOutput < BufferedIndexOutput
       def initialize(path)
         super()
@@ -330,7 +340,7 @@ module Ferret::Store
 
       # returns the lock prefix for this directory
       def lock_prefix
-        'ferret-' + Digest::MD5.hexdigest(@dir.path)
+        LOCK_PREFIX + Digest::MD5.hexdigest(@dir.path)
       end
 
       # Unfortunately, on Windows, Dir does not refresh when rewind is called
@@ -347,6 +357,9 @@ module Ferret::Store
         @lock_dir = tmp
       end
 
+      # This method is only used by the c extension to free the directory
+      def close_internal
+      end
     #end private
   end
 end

data/lib/ferret/store/ram_store.rb

@@ -25,7 +25,7 @@ module Ferret::Store
     # returns an array of strings, one for each file in the directory
     def each()
       @files.each do |path, file|
-        next if file =~ Regexp.new('^rubylock-')
+        next if file =~ /#{LOCK_PREFIX}/
         yield file
       end
     end
@@ -89,7 +89,7 @@ module Ferret::Store
 
     # Construct a Lock.
     def make_lock(name) 
-      RAMLock.new("rubylock-" + name, self)
+      RAMLock.new(LOCK_PREFIX + name, self)
     end
 
 

data/test/unit/document/tc_field.rb

@@ -15,6 +15,7 @@ class FieldTest < Test::Unit::TestCase
     assert_equal("TOKENIZED", Field::Index::TOKENIZED.to_s)
     assert_equal("UNTOKENIZED", Field::Index::UNTOKENIZED.to_s)
     assert_equal("NO", Field::Index::NO.to_s)
+    assert_equal("NO_NORMS", Field::Index::NO_NORMS.to_s)
   end
 
   def test_term_vector()
@@ -36,6 +37,7 @@ class FieldTest < Test::Unit::TestCase
     assert_equal(false, f.store_term_vector?)
     assert_equal(false, f.store_offsets?)
     assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
     assert_equal(false, f.binary?)
     assert_equal("stored/compressed,indexed,tokenized,<name:value>", f.to_s)
   end
@@ -53,7 +55,13 @@ class FieldTest < Test::Unit::TestCase
     f.index = Field::Index::NO
     assert_equal(false, f.indexed?)
     assert_equal(false, f.tokenized?)
+    assert_equal(false, f.omit_norms?)
     assert_equal("stored/compressed,<name:value>", f.to_s)
+    f.index = Field::Index::NO_NORMS
+    assert_equal(true, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(true, f.omit_norms?)
+    assert_equal("stored/compressed,indexed,omit_norms,<name:value>", f.to_s)
   end
 
   def test_set_term_vector()
@@ -79,6 +87,7 @@ class FieldTest < Test::Unit::TestCase
     assert_equal(false, f.store_term_vector?)
     assert_equal(false, f.store_offsets?)
     assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
     assert_equal(true, f.binary?)
     assert_equal("stored/uncompressed,binary,<name:#{bin}>", f.to_s)
   end

data/test/unit/index/tc_field_infos.rb

@@ -4,13 +4,14 @@ class FieldInfosTest < Test::Unit::TestCase
   include Ferret::Index
 
   def test_field_info()
-    fi = FieldInfo.new("name", true, 1, true, true, true)
+    fi = FieldInfo.new("name", true, 1, true, true, true, true)
     assert_equal(fi.name, "name")
     assert_equal(fi.number, 1)
     assert(fi.indexed?)
     assert(fi.store_term_vector?)
     assert(fi.store_offsets?)
     assert(fi.store_positions?)
+    assert(fi.omit_norms?)
 
     fi.name = "hello"
     fi.indexed = false
@@ -18,6 +19,7 @@ class FieldInfosTest < Test::Unit::TestCase
     fi.store_term_vector = false
     fi.store_offset = false
     fi.store_position = false
+    fi.omit_norms = false
 
     assert_equal(fi.name, "hello")
     assert_equal(fi.number, 2)
@@ -25,49 +27,53 @@ class FieldInfosTest < Test::Unit::TestCase
     assert(!fi.store_term_vector?)
     assert(!fi.store_offsets?)
     assert(!fi.store_positions?)
+    assert(!fi.omit_norms?)
 
-    fi.set!(true, true, true, true)
+    fi.set!(true, true, true, true, true)
     assert(fi.indexed?)
     assert(fi.store_term_vector?)
     assert(fi.store_offsets?)
     assert(fi.store_positions?)
+    assert(fi.omit_norms?)
 
     fi = FieldInfo.new("name", true, 1, true)
     assert(!fi.store_offsets?)
     assert(!fi.store_positions?)
+    assert(!fi.omit_norms?)
   end
 
-  def fi_test_attr(fi, name, number, indexed, store_tv, store_pos, store_off)
+  def fi_test_attr(fi, name, number, indexed, store_tv, store_pos, store_off, omit_norms)
     assert_equal(name, fi.name)
     assert_equal(number, fi.number)
     assert_equal(indexed, fi.indexed?)
     assert_equal(store_tv, fi.store_term_vector?)
     assert_equal(store_pos, fi.store_positions?)
     assert_equal(store_off, fi.store_offsets?)
+    assert_equal(omit_norms, fi.omit_norms?)
   end
 
   def test_fis_add()
     fis = FieldInfos.new()
     fi = fis.add("field1", false)
-    fi_test_attr(fi, "field1", 0, false, false, false, false)
+    fi_test_attr(fi, "field1", 0, false, false, false, false, false)
     assert_equal(1, fis.size)
 
     fi = fis.add("field1", true, true)
-    fi_test_attr(fi, "field1", 0, true, true, false, false)
+    fi_test_attr(fi, "field1", 0, true, true, false, false, false)
     assert_equal(1, fis.size)
 
     fi = fis.add("field2", false)
-    fi_test_attr(fi, "field2", 1, false, false, false, false)
+    fi_test_attr(fi, "field2", 1, false, false, false, false, false)
     assert_equal(2, fis.size)
 
-    fi = fis.add("field1", true, true, true, true)
+    fi = fis.add("field1", true, true, true, true, true)
     assert_equal(fi, fis[fi.number])
     assert_equal(fi, fis["field1"])
     assert_equal(0, fis.field_number("field1"))
     assert_equal(1, fis.field_number("field2"))
     assert_equal(FieldInfos::NOT_A_FIELD, fis.field_number("field3"))
     assert_equal(nil, fis["field3"])
-    fi_test_attr(fi, "field1", 0, true, true, true, true)
+    fi_test_attr(fi, "field1", 0, true, true, true, true, false)
     assert_equal(2, fis.size)
   end
 
@@ -90,7 +96,7 @@ class FieldInfosTest < Test::Unit::TestCase
     assert(! fis.has_vectors?)
     fis.add("random_field")
     assert(! fis.has_vectors?)
-    fis.add("store_term_vector_field", true, true, false, false)
+    fis.add("store_term_vector_field", true, true, false, false, false)
     assert(fis.has_vectors?)
   end
 
@@ -98,22 +104,24 @@ class FieldInfosTest < Test::Unit::TestCase
   def test_fis_rw()
     fis = FieldInfos.new()
     dir = Ferret::Store::RAMDirectory.new()
-    fis.add("field1", false, false, false, false)
-    fis.add("field2", true, false, false, false)
-    fis.add("field3", true, true, false, false)
-    fis.add("field4", true, true, true, false)
-    fis.add("field5", true, true, true, true)
+    fis.add("field1", false, false, false, false, true)
+    fis.add("field2", true, false, false, false, true)
+    fis.add("field3", true, true, false, false, true)
+    fis.add("field4", true, true, true, false, true)
+    fis.add("field5", true, true, true, true, true)
+    fis.add("field6", true, true, true, true, false)
     fis.write_to_dir(dir, "fis_rw.test")
     fis = nil
 
     fis = FieldInfos.new(dir, "fis_rw.test")
-    fi_test_attr(fis[0], "field1", 0, false, false, false, false)
-    fi_test_attr(fis[1], "field2", 1, true, false, false, false)
-    fi_test_attr(fis[2], "field3", 2, true, true, false, false)
-    fi_test_attr(fis[3], "field4", 3, true, true, true, false)
-    fi_test_attr(fis[4], "field5", 4, true, true, true, true)
-
-    assert_equal(5, fis.size)
+    fi_test_attr(fis[0], "field1", 0, false, false, false, false, true)
+    fi_test_attr(fis[1], "field2", 1, true, false, false, false, true)
+    fi_test_attr(fis[2], "field3", 2, true, true, false, false, true)
+    fi_test_attr(fis[3], "field4", 3, true, true, true, false, true)
+    fi_test_attr(fis[4], "field5", 4, true, true, true, true, true)
+    fi_test_attr(fis[5], "field6", 5, true, true, true, true, false)
+
+    assert_equal(6, fis.size)
   end
 
 end

data/test/unit/index/tc_index.rb

@@ -8,7 +8,6 @@ class IndexTest < Test::Unit::TestCase
   include Ferret::Document
 
   def setup()
-    @qp = Ferret::QueryParser.new()
   end
 
   def tear_down()
@@ -81,7 +80,7 @@ class IndexTest < Test::Unit::TestCase
       {"def_field" => "one two"},
       {"def_field" => "two", :field2 => "three", "field3" => "four"},
       {"def_field" => "one multi2"},
-      {"def_field" => "two", :field2 => "three multi2", "field3" => "five multi"}
+      {"def_field" => "two", :field2 => "this three multi2", "field3" => "five multi"}
     ]
     data.each {|doc| index << doc }
     q = "one AND two"
@@ -110,6 +109,7 @@ class IndexTest < Test::Unit::TestCase
     doc["field2"] = "dave"
     index << doc
     check_results(index, q, [6, 7])
+    check_results(index, "*:this", [])
     assert_equal(8, index.size)
     assert_equal("dave", index[7]["field2"])
     index.optimize
@@ -129,28 +129,39 @@ class IndexTest < Test::Unit::TestCase
   def test_ram_index
     index = Index.new(:default_field => "def_field")
     do_test_index_with_array(index)
+    index.close
+
     index = Index.new(:default_field => "def_field")
     do_test_index_with_hash(index)
+    index.close
+
     index = Index.new(:default_field => "def_field")
     do_test_index_with_doc_array(index)
+    index.close
   end
 
   def test_fs_index
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
-    `rm -rf #{File.join(fs_path, "*")}`
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     assert_raise(Errno::ENOENT) {Index.new(:path => fs_path, :create_if_missing => false, :default_field => "def_field")}
     index = Index.new(:path => fs_path, :default_field => "def_field")
     do_test_index_with_array(index)
-    `rm -rf #{File.join(fs_path, "*")}`
-    index = Index.new(:path => fs_path, :create => true, :default_field => "def_field")
+    index.close
+
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
+    index = Index.new(:path => fs_path, :default_field => "def_field")
     do_test_index_with_hash(index)
-    index = Index.new(:path => fs_path, :create => true, :default_field => "def_field")
+    index.close
+
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
+    index = Index.new(:path => fs_path, :default_field => "def_field")
     do_test_index_with_doc_array(index)
+    index.close
   end
 
   def test_fs_index_is_persistant
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
-    `rm -rf #{File.join(fs_path, "*")}`
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     data = [
       {"def_field" => "one two", :id => "me"},
       {"def_field" => "one", :field2 => "three"},
@@ -165,9 +176,11 @@ class IndexTest < Test::Unit::TestCase
     data.each {|doc| index << doc }
     assert_equal(8, index.size)
     index.close
+
     index = Index.new(:path => fs_path, :create_if_missing => false)
     assert_equal(8, index.size)
     assert_equal("four", index[5]["field3"])
+    index.close
   end
 
   def test_merging_indexes
@@ -246,6 +259,7 @@ class IndexTest < Test::Unit::TestCase
     index3.close
     dir3.close
     assert_equal("golf", index[15]["f"])
+    index.close
   end
 
   def test_persist_index
@@ -310,6 +324,7 @@ class IndexTest < Test::Unit::TestCase
     iw.close()
     assert_equal(3, index.size)
     assert_equal("content3", index[2]["f"])
+    index.close
   end
 
   def test_delete
@@ -340,6 +355,7 @@ class IndexTest < Test::Unit::TestCase
     index.query_delete("cat:/cat1*")
     assert_equal(3, index.size)
     assert_equal(0, index.search("cat:/cat1*").size)
+    index.close
   end
 
   def test_update
@@ -390,6 +406,7 @@ class IndexTest < Test::Unit::TestCase
     assert_equal("cool", index["3"][:tag])
     assert_equal("cool", index["4"][:tag])
     assert_equal(4, index.search("tag:cool").size)
+    index.close
   end
 
   def test_index_key
@@ -405,6 +422,7 @@ class IndexTest < Test::Unit::TestCase
     assert_equal(2, index.size)
     assert_equal("two", index[0][:val])
     assert_equal("four", index[1][:val])
+    index.close
   end
 
   def test_index_multi_key
@@ -427,5 +445,24 @@ class IndexTest < Test::Unit::TestCase
     assert_equal("second floor", index[1][:location])
     assert_equal("backpack", index[3][:product])
     assert_equal("first floor", index[2][:location])
+    index.close
+  end
+
+  def test_auto_flush
+    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
+    data = %q(one two three four five six seven eight nine ten eleven twelve)
+    index1 = Index.new(:path => fs_path, :auto_flush => true)
+    index2 = Index.new(:path => fs_path, :auto_flush => true)
+    begin
+      data.each do |doc|
+        index1 << doc
+        index2 << doc
+      end
+    rescue Exception => e
+      assert(false, "This should not cause an error when auto flush has been set")
+    end
+    index1.close
+    index2.close
   end
 end