sdsykes-ferret 0.11.6.19

data/test/unit/store/tc_fs_store.rb

@@ -0,0 +1,77 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.dirname(__FILE__) + "/tm_store"
+require File.dirname(__FILE__) + "/tm_store_lock"
+
+class FSStoreTest < Test::Unit::TestCase
+  include Ferret::Store
+  include StoreTest
+  include StoreLockTest
+  def setup
+    @dpath = File.expand_path(File.join(File.dirname(__FILE__),
+                       '../../temp/fsdir'))
+    @dir = FSDirectory.new(@dpath, true)
+  end
+
+  def teardown
+    @dir.refresh()
+    @dir.close()
+  end
+
+  def test_fslock
+    lock_name = "lfile"
+    lock_file_path = make_lock_file_path(lock_name)
+    assert(! File.exists?(lock_file_path), "There should be no lock file")
+    lock = @dir.make_lock(lock_name)
+    assert(! File.exists?(lock_file_path), "There should still be no lock file")
+    assert(! lock.locked?,                 "lock shouldn't be locked yet")
+
+    lock.obtain
+
+    assert(lock.locked?,                   "lock should now be locked")
+
+    assert(File.exists?(lock_file_path),   "A lock file should have been created")
+
+    assert(@dir.exists?(lfname(lock_name)),"The lock should exist")
+
+    lock.release
+
+    assert(! lock.locked?,                 "lock should be freed again")
+    assert(! File.exists?(lock_file_path), "The lock file should have been deleted")
+  end
+
+#  def make_and_loose_lock
+#    lock = @dir.make_lock("finalizer_lock")
+#    lock.obtain
+#    lock = nil
+#  end
+#
+#  def test_fslock_finalizer
+#    lock_name = "finalizer_lock"
+#    lock_file_path = make_lock_file_path(lock_name)
+#    assert(! File.exists?(lock_file_path), "There should be no lock file")
+#
+#    make_and_loose_lock
+#
+#    #assert(File.exists?(lock_file_path), "There should now be a lock file")
+#
+#    lock = @dir.make_lock(lock_name)
+#    assert(lock.locked?, "lock should now be locked")
+#
+#    GC.start
+#
+#    assert(! lock.locked?, "lock should be freed again")
+#    assert(! File.exists?(lock_file_path), "The lock file should have been deleted")
+#  end
+#
+  def make_lock_file_path(name)
+    lock_file_path = File.join(@dpath, lfname(name))
+    if File.exists?(lock_file_path) then
+      File.delete(lock_file_path)
+    end
+    return lock_file_path
+  end
+
+  def lfname(name)
+    "ferret-#{name}.lck"
+  end
+end

data/test/unit/store/tc_ram_store.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.dirname(__FILE__) + "/tm_store"
+require File.dirname(__FILE__) + "/tm_store_lock"
+
+class RAMStoreTest < Test::Unit::TestCase
+  include StoreTest
+  include StoreLockTest
+  def setup
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def teardown
+    @dir.close()
+  end
+
+  def test_ramlock
+    name = "lfile"
+    lfile = Ferret::Store::Directory::LOCK_PREFIX + name + ".lck"
+    assert(! @dir.exists?(lfile),
+           "There should be no lock file")
+    lock = @dir.make_lock(name)
+    assert(! @dir.exists?(lfile),
+           "There should still be no lock file")
+    assert(! @dir.exists?(lfile),
+           "The lock should be hidden by the FSDirectories directory scan")
+    assert(! lock.locked?, "lock shouldn't be locked yet")
+    lock.obtain
+    assert(lock.locked?, "lock should now be locked")
+    assert(@dir.exists?(lfile), "A lock file should have been created")
+    lock.release
+    assert(! lock.locked?, "lock should be freed again")
+    assert(! @dir.exists?(lfile),
+           "The lock file should have been deleted")
+  end
+end

data/test/unit/store/tm_store.rb

@@ -0,0 +1,34 @@
+module StoreTest
+  # declare dir so inheritors can access it.
+  attr_accessor :dir
+
+  # test the basic file manipulation methods;
+  # - exists?
+  # - touch
+  # - delete
+  # - file_count
+  def test_basic_file_ops
+    assert_equal(0, @dir.file_count(), "directory should be empty")
+    assert(! @dir.exists?('filename'), "File should not exist")
+    @dir.touch('tmpfile1')
+    assert_equal(1, @dir.file_count(), "directory should have one file")
+    @dir.touch('tmpfile2')
+    assert_equal(2, @dir.file_count(), "directory should have two files")
+    assert(@dir.exists?('tmpfile1'), "'tmpfile1' should exist")
+    @dir.delete('tmpfile1')
+    assert(! @dir.exists?('tmpfile1'), "'tmpfile1' should no longer exist")
+    assert_equal(1, @dir.file_count(), "directory should have one file")
+  end
+  
+  def test_rename
+    @dir.touch("from")
+    assert(@dir.exists?('from'), "File should exist")
+    assert(! @dir.exists?('to'), "File should not exist")
+    cnt_before = @dir.file_count()
+    @dir.rename('from', 'to')
+    cnt_after = @dir.file_count()
+    assert_equal(cnt_before, cnt_after, "the number of files shouldn't have changed")
+    assert(@dir.exists?('to'), "File should now exist")
+    assert(! @dir.exists?('from'), "File should no longer exist")
+  end
+end

data/test/unit/store/tm_store_lock.rb

@@ -0,0 +1,68 @@
+module StoreLockTest
+  class Switch
+    @@counter = 0
+    def Switch.counter() return @@counter end
+    def Switch.counter=(counter) @@counter = counter end
+  end
+
+  def test_locking()
+    lock_time_out = 0.001 # we want this test to run quickly
+    lock1 = @dir.make_lock("l.lck")
+    lock2 = @dir.make_lock("l.lck")
+
+    assert(!lock2.locked?)
+    assert(lock1.obtain(lock_time_out))
+    assert(lock2.locked?)
+
+    assert(! can_obtain_lock?(lock2))
+
+    exception_thrown = false
+    begin
+      lock2.while_locked(lock_time_out) do
+        assert(false, "lock should not have been obtained")
+      end
+    rescue
+      exception_thrown = true
+    ensure
+      assert(exception_thrown)
+    end
+
+    lock1.release()
+    assert(lock2.obtain(lock_time_out))
+    lock2.release()
+
+    t = Thread.new() do
+      lock1.while_locked(lock_time_out) do
+        Switch.counter = 1
+        # make sure lock2 obtain test was run
+        while Switch.counter < 2
+        end
+        Switch.counter = 3
+      end
+    end
+    t.run()
+
+    #make sure thread has started and lock been obtained
+    while Switch.counter < 1
+    end
+    
+    assert(! can_obtain_lock?(lock2))
+
+    Switch.counter = 2
+    while Switch.counter < 3
+    end
+    
+    assert(lock2.obtain(lock_time_out))
+    lock2.release()
+  end
+
+  def can_obtain_lock?(lock)
+    lock_time_out = 0.001 # we want this test to run quickly
+    begin
+      lock.obtain(lock_time_out)
+      return true
+    rescue
+    end
+    return false
+  end
+end

data/test/unit/tc_document.rb

@@ -0,0 +1,81 @@
+require File.dirname(__FILE__) + "/../test_helper"
+
+class DocumentTest < Test::Unit::TestCase
+  def test_field
+    f = Ferret::Field.new
+    assert_equal(0, f.size)
+    assert_equal(1.0, f.boost)
+
+    f2 = Ferret::Field.new
+    assert_equal(f, f2)
+
+    f << "section0"
+    assert_equal(1, f.size)
+    assert_equal(1.0, f.boost)
+    assert_equal("section0", f[0])
+    assert_not_equal(f, f2)
+
+    f << "section1"
+    assert_equal(2, f.size)
+    assert_equal(1.0, f.boost)
+    assert_equal("section0", f[0])
+    assert_equal("section1", f[1])
+    assert_equal('["section0", "section1"]', f.to_s)
+    assert_not_equal(f, f2)
+    f2 += f
+    assert_equal(f, f2)
+
+    f.boost = 4.0
+    assert_not_equal(f, f2)
+    assert_equal('["section0", "section1"]^4.0', f.to_s)
+
+    f2.boost = 4.0
+    assert_equal(f, f2)
+
+    f3 = Ferret::Field.new(["section0", "section1"], 4.0)
+    assert_equal(f, f3)
+  end
+
+  def test_document
+    d = Ferret::Document.new
+
+    d[:name] = Ferret::Field.new
+    d[:name] << "section0"
+    d[:name] << "section1"
+
+    assert_equal(1, d.size)
+    assert_equal(1.0, d.boost)
+    assert_equal(%(
+Document {
+  :name => ["section0", "section1"]
+}).strip, d.to_s)
+
+
+    d.boost = 123.0
+    d[:name] << "section2"
+    d[:name].boost = 321.0
+    assert_equal(123.0, d.boost)
+    assert_equal(321.0, d[:name].boost)
+    assert_equal(%(
+Document {
+  :name => ["section0", "section1", "section2"]^321.0
+}^123.0).strip, d.to_s)
+
+    d[:title] = "Shawshank Redemption"
+    d[:actors] = ["Tim Robbins", "Morgan Freeman"]
+
+    assert_equal(3, d.size)
+    assert_equal(%(
+Document {
+  :actors => ["Tim Robbins", "Morgan Freeman"]
+  :name => ["section0", "section1", "section2"]^321.0
+  :title => "Shawshank Redemption"
+}^123.0).strip, d.to_s)
+
+    d2 = Ferret::Document.new(123.0)
+    d2[:name] = Ferret::Field.new(["section0", "section1", "section2"], 321.0)
+    d2[:title] = "Shawshank Redemption"
+    d2[:actors] = ["Tim Robbins", "Morgan Freeman"]
+    assert_equal(d, d2)
+  end
+end

data/test/unit/ts_analysis.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/analysis')

data/test/unit/ts_index.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/index')

data/test/unit/ts_largefile.rb

@@ -0,0 +1,4 @@
+if ENV['FERRET_DEV']
+  require File.join(File.dirname(__FILE__), "../test_helper.rb")
+  load_test_dir('unit/largefile')
+end

data/test/unit/ts_query_parser.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/query_parser')

data/test/unit/ts_search.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/search')

data/test/unit/ts_store.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/store')

data/test/unit/ts_utils.rb

@@ -0,0 +1,2 @@
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+load_test_dir('unit/utils')

data/test/unit/utils/tc_bit_vector.rb

@@ -0,0 +1,295 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class BitVectorTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_bv_get_set()
+    bv = BitVector.new
+    assert_equal 0, bv.count 
+
+    bv.set 10 
+    assert bv.get(10) 
+    assert bv[10] 
+    assert_equal 1, bv.count 
+
+    bv[10] = false
+    assert ! bv[10] 
+
+    bv[10] = true
+    assert bv[10] 
+
+    bv[10] = nil
+    assert ! bv[10] 
+
+    bv[10] = true
+    assert bv[10] 
+
+    bv.unset 10 
+    assert ! bv[10] 
+
+    bv[10] = true
+    assert bv[10] 
+  end
+
+  def test_bv_count()
+    bv = BitVector.new
+    bv.set 10 
+    assert_equal 1, bv.count 
+
+    bv.set 20 
+    assert bv.get(20) 
+    assert_equal 2, bv.count 
+
+    bv.set 21 
+    assert bv.get(21) 
+    assert_equal 3, bv.count 
+
+    bv.unset 21 
+    assert ! bv.get(21) 
+    assert_equal 2, bv.count 
+
+    bv[20] = nil
+    assert ! bv.get(20) 
+    assert_equal 1, bv.count 
+
+    (50..100).each {|i| bv.set i }
+    (50..100).each {|i| assert bv[i] }
+    assert bv.get(10) 
+    assert_equal 52, bv.count 
+
+    bv.clear
+    assert_equal 0, bv.count
+    (50..100).each {|i| assert ! bv[i] }
+    assert ! bv.get(10) 
+  end
+
+  def test_bv_eql_hash
+    bv1 = BitVector.new
+    bv2 = BitVector.new
+    assert_equal(bv1, bv2)
+    assert_equal(bv1.hash, bv2.hash)
+
+    bv1.set(10)
+    assert_not_equal(bv1, bv2)
+    assert_not_equal(bv1.hash, bv2.hash)
+
+    bv2.set(10)
+    assert_equal(bv1, bv2)
+    assert_equal(bv1.hash, bv2.hash)
+
+    10.times {|i| bv1.set(i * 31)}
+    assert_not_equal(bv1, bv2)
+    assert_not_equal(bv1.hash, bv2.hash)
+
+    10.times {|i| bv2.set(i * 31)}
+    assert_equal(bv1, bv2)
+    assert_equal(bv1.hash, bv2.hash)
+
+    bv1.clear
+    assert_not_equal(bv1, bv2)
+    assert_not_equal(bv1.hash, bv2.hash)
+
+    bv2.clear
+    assert_equal(bv1, bv2)
+    assert_equal(bv1.hash, bv2.hash)
+  end
+
+  BV_COUNT = 500
+  BV_SIZE = 1000
+
+  def test_bv_and
+    bv1 = BitVector.new
+    bv2 = BitVector.new
+    set1 = set2 = count = 0
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv1.set(bit)
+      set1 |= (1 << bit)
+    end
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv2.set(bit)
+      bitmask = (1 << bit)
+      if ((set1 & bitmask) > 0) && ((set2 & bitmask) == 0)
+        set2 |= (1 << bit)
+        count += 1
+      end
+    end
+
+    and_bv = bv1 & bv2
+    assert_equal(count, and_bv.count)
+    BV_SIZE.times do |i|
+      assert_equal(((set2 & (1 << i)) > 0), and_bv[i])
+    end
+
+    bv2.and! bv1
+    assert_equal(bv2, and_bv)
+
+    bv2 = BitVector.new
+    and_bv = bv1 & bv2
+
+    assert_equal(bv2, and_bv, "and_bv should be empty")
+    assert_equal(0, and_bv.count)
+
+    bv1 = BitVector.new
+    bv2 = BitVector.new.not!
+    bv1.set(10)
+    bv1.set(11)
+    bv1.set(20)
+    assert_equal(bv1, bv1 & bv2, "bv anded with empty not bv should be same")
+  end
+  
+  def test_bv_or
+    bv1 = BitVector.new
+    bv2 = BitVector.new
+    set = count = 0
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv1.set(bit)
+      bitmask = (1 << bit)
+      if (set & bitmask) == 0
+        count += 1
+        set |= bitmask
+      end
+    end
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv2.set(bit)
+      bitmask = (1 << bit)
+      if (set & bitmask) == 0
+        count += 1
+        set |= bitmask
+      end
+    end
+
+    or_bv = bv1 | bv2
+    assert_equal(count, or_bv.count)
+    BV_SIZE.times do |i|
+      assert_equal(((set & (1 << i)) > 0), or_bv[i])
+    end
+
+    bv2.or! bv1
+    assert_equal(bv2, or_bv)
+
+    bv2 = BitVector.new
+    or_bv = bv1 | bv2
+
+    assert_equal(bv1, or_bv)
+  end
+  
+  def test_bv_xor
+    bv1 = BitVector.new
+    bv2 = BitVector.new
+    set1 = set2 = count = 0
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv1.set(bit)
+      set1 |= (1 << bit)
+    end
+
+    BV_COUNT.times do |i|
+      bit = rand(BV_SIZE)
+      bv2.set(bit)
+      set2 |= (1 << bit)
+    end
+
+    bitmask = 1
+    set1 ^= set2
+    BV_SIZE.times do |i|
+      count += 1 if (set1 & bitmask) > 0
+      bitmask <<= 1
+    end
+
+    xor_bv = bv1 ^ bv2
+    BV_SIZE.times do |i|
+      assert_equal(((set1 & (1 << i)) > 0), xor_bv[i])
+    end
+    assert_equal(count, xor_bv.count)
+
+    bv2.xor! bv1
+    assert_equal(bv2, xor_bv)
+
+    bv2 = BitVector.new
+    xor_bv = bv1 ^ bv2
+
+    assert_equal(bv1, xor_bv)
+  end
+
+  def test_bv_not
+    bv = BitVector.new
+    [1, 5, 25, 41, 97, 185].each {|i| bv.set(i)}
+    not_bv = ~bv
+    assert_equal(bv.count, not_bv.count)
+    200.times {|i| assert(bv[i] != not_bv[i])}
+
+    not_bv.not!
+    assert_equal(bv, not_bv)
+  end
+
+
+  SCAN_SIZE = 200
+  SCAN_INC = 97
+
+  def test_scan
+    bv = BitVector.new
+
+    SCAN_SIZE.times {|i| bv.set(i * SCAN_INC)}
+    not_bv = ~bv
+
+    SCAN_SIZE.times do |i|
+      assert_equal(i * SCAN_INC, bv.next_from((i - 1) * SCAN_INC + 1))
+      assert_equal(i * SCAN_INC, not_bv.next_unset_from((i - 1) * SCAN_INC + 1))
+    end
+    assert_equal(-1, bv.next_from((SCAN_SIZE - 1) * SCAN_INC + 1))
+    assert_equal(-1, not_bv.next_unset_from((SCAN_SIZE - 1) * SCAN_INC + 1))
+
+    bit = 0
+    bv.each {|i| assert_equal(bit, i); bit += SCAN_INC }
+    assert_equal(bit, SCAN_SIZE * SCAN_INC)
+
+    bit = 0
+    not_bv.each {|i| assert_equal(bit, i); bit += SCAN_INC }
+    assert_equal(bit, SCAN_SIZE * SCAN_INC)
+
+    bv.reset_scan
+    not_bv.reset_scan
+    SCAN_SIZE.times do |i|
+      assert_equal(i * SCAN_INC, bv.next)
+      assert_equal(i * SCAN_INC, not_bv.next_unset)
+    end
+    assert_equal(-1, bv.next)
+    assert_equal(-1, not_bv.next_unset)
+
+    bv.clear
+    SCAN_SIZE.times {|i| bv.set(i)}
+    not_bv = ~bv
+
+    SCAN_SIZE.times do |i|
+      assert_equal(i, bv.next)
+      assert_equal(i, not_bv.next_unset)
+    end
+    assert_equal(-1, bv.next)
+    assert_equal(-1, not_bv.next_unset)
+
+    bit = 0
+    bv.each {|i| assert_equal(bit, i); bit += 1 }
+    assert_equal(bit, SCAN_SIZE)
+
+    bit = 0
+    not_bv.each {|i| assert_equal(bit, i); bit += 1 }
+    assert_equal(bit, SCAN_SIZE)
+  end
+
+  def test_to_a
+    bv = BitVector.new
+    ary = (1..100).collect { rand(1000) }.sort.uniq
+    ary.each {|i| bv.set(i)}
+    assert_equal(ary, bv.to_a)
+  end
+end