ferret 0.1.0

data/test/unit/store/tc_ram_store.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.dirname(__FILE__) + "/tm_store"
+require File.dirname(__FILE__) + "/tm_store_lock"
+
+class RAMStoreTest < Test::Unit::TestCase
+  include StoreTest
+  include StoreLockTest
+  def setup
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def teardown
+    @dir.close()
+  end
+
+  def test_ramlock
+    name = "lfile"
+    lfile = "rubylock-" + name
+    assert(! @dir.exists?(lfile),
+           "There should be no lock file")
+    lock = @dir.make_lock(name)
+    assert(! @dir.exists?(lfile),
+           "There should still be no lock file")
+    assert(! @dir.exists?(lfile),
+           "The lock should be hidden by the FSDirectories directory scan")
+    assert(! lock.locked?, "lock shouldn't be locked yet")
+    lock.obtain
+    assert(lock.locked?, "lock should now be locked")
+    assert(@dir.exists?(lfile), "A lock file should have been created")
+    lock.release
+    assert(! lock.locked?, "lock should be freed again")
+    assert(! @dir.exists?(lfile),
+           "The lock file should have been deleted")
+  end
+end

data/test/unit/store/tm_store.rb

@@ -0,0 +1,180 @@
+module StoreTest
+  # declare dir so inheritors can access it.
+  @dir = nil
+
+  # test the basic file manipulation methods;
+  # - exists?
+  # - touch
+  # - delete
+  # - file_count
+  def test_basic_file_ops
+    assert_equal(0, @dir.file_count(), "directory should be empty")
+    assert(! @dir.exists?('filename'), "File should not exist")
+    @dir.touch('tmpfile1')
+    assert_equal(1, @dir.file_count(), "directory should have one file")
+    @dir.touch('tmpfile2')
+    assert_equal(2, @dir.file_count(), "directory should have two files")
+    assert(@dir.exists?('tmpfile1'), "'tmpfile1' should exist")
+    @dir.delete('tmpfile1')
+    assert(! @dir.exists?('tmpfile1'), "'tmpfile1' should no longer exist")
+    assert_equal(1, @dir.file_count(), "directory should have one file")
+  end
+  
+  def test_rename
+    @dir.touch("from")
+    assert(@dir.exists?('from'), "File should exist")
+    assert(! @dir.exists?('to'), "File should not exist")
+    cnt_before = @dir.file_count()
+    @dir.rename('from', 'to')
+    cnt_after = @dir.file_count()
+    assert_equal(cnt_before, cnt_after, "the number of files shouldn't have changed")
+    assert(@dir.exists?('to'), "File should now exist")
+    assert(! @dir.exists?('from'), "File should no longer exist")
+  end
+
+  def test_modified
+    # difficult to test this one but as file mtime is only stored to the nearest second.
+    # we can assume this test will happen in less than a few seconds. (I hope)
+    time = Time.new.to_i
+    @dir.touch('mtime.test')
+    time_before = @dir.modified('mtime.test').to_i
+    assert(time_before - time <= 3,
+           "test that mtime is approximately equal to the system time when the file was touched")
+  end
+
+  def test_rw_bytes
+    bytes = [0x34, 0x87, 0xF9, 0xEA, 0x00, 0xFF]
+    rw_test(bytes, "byte", 6)
+  end
+
+  def test_rw_ints
+    ints = [-2147483648, 2147483647, -1, 0]
+    rw_test(ints, "int", 16)
+  end
+
+  def test_rw_longs
+    longs = [-9223372036854775808, 9223372036854775807, -1, 0]
+    rw_test(longs, "long", 32)
+  end
+
+  def test_rw_uints
+    uints = [0xffffffff, 100000, 0]
+    rw_test(uints, "uint", 12)
+  end
+
+  def test_rw_ulongs
+    ulongs = [0xffffffffffffffff, 100000000000000, 0]
+    rw_test(ulongs, "ulong", 24)
+  end
+
+  def test_rw_vints
+    vints = [ 9223372036854775807,
+              0x00,
+              0xFFFFFFFFFFFFFFFF]
+    rw_test(vints, "vint", 20)
+  end
+
+  def test_rw_vlongs
+    vlongs = [ 9223372036854775807,
+               0x00,
+               0xFFFFFFFFFFFFFFFF]
+    rw_test(vlongs, "vlong", 20)
+  end
+
+  def test_rw_strings
+    text = 'This is a ruby ferret test string ~!@#$%^&*()`123456790-=\)_+|'
+    ostream = @dir.create_output("rw_strings.test")
+    ostream.write_string(text)
+    ostream.write_string(text*100)
+    ostream.close
+    istream = @dir.open_input("rw_strings.test")
+    assert_equal(text, istream.read_string, "Short string test failed")
+    assert_equal(text*100, istream.read_string, "Short string test failed")
+    istream.close
+    assert_equal(6265, @dir.length('rw_strings.test'))
+  end
+
+  def test_rw_utf8_strings
+    text = '�� ������'
+    ostream = @dir.create_output("rw_utf8_strings.test")
+    ostream.write_string(text)
+    ostream.write_string(text*100)
+    ostream.close
+    istream = @dir.open_input("rw_utf8_strings.test")
+    assert_equal(text, istream.read_string, "Short string test failed")
+    assert_equal(text*100, istream.read_string, "Short string test failed")
+    istream.close
+  end
+
+  # this test fills up the output stream so that the buffer will have to be
+  # written a few times. It then uses seek to make sure that it works
+  # correctly
+  def test_buffer_seek
+    ostream = @dir.create_output("rw_seek.test")
+    text = 'This is another long test string !@#$%#$%&%$*%^&*()(_'
+    1000.times {|i| ostream.write_long(i); ostream.write_string(text) }
+    ostream.seek(987)
+    assert_equal(987, ostream.pos)
+    ostream.write_vint(555)
+    ostream.seek(56)
+    assert_equal(56, ostream.pos)
+    ostream.write_vint(1234567890)
+    ostream.seek(4000)
+    assert_equal(4000, ostream.pos)
+    ostream.write_vint(9876543210)
+    ostream.close()
+    istream = @dir.open_input("rw_seek.test")
+    istream.seek(56)
+    assert_equal(56, istream.pos)
+    assert_equal(1234567890, istream.read_vint())
+    istream.seek(4000)
+    assert_equal(4000, istream.pos)
+    assert_equal(9876543210, istream.read_vint())
+    istream.seek(987)
+    assert_equal(987, istream.pos)
+    assert_equal(555, istream.read_vint())
+    istream.close()
+  end
+
+  def test_clone
+    ostream = @dir.create_output("clone_test")
+    10.times {|i| ostream.write_long(i) }
+    ostream.close
+    istream = @dir.open_input("clone_test")
+    istream.seek(24)
+    alt_istream = istream.clone
+    assert_equal(istream.pos, alt_istream.pos)
+    (3...10).each {|i| assert_equal(i, alt_istream.read_long) }
+    assert_equal(80, alt_istream.pos)
+    assert_equal(24, istream.pos)
+    alt_istream.close
+    (3...10).each {|i| assert_equal(i, istream.read_long) }
+    istream.close
+  end
+
+  def test_read_bytes
+    str = "0000000000"
+    ostream = @dir.create_output("rw_read_bytes")
+    ostream.write_bytes("how are you doing?", 18)
+    ostream.close
+    istream = @dir.open_input("rw_read_bytes")
+    istream.read_bytes(str, 2, 4)
+    assert_equal("00how 0000", str)
+    istream.read_bytes(str, 1, 8)
+    assert_equal("0are you 0", str)
+    istream.close
+  end
+
+  private
+
+  def rw_test(values, type, expected_length)
+    ostream = @dir.create_output("rw_#{type}.test")
+    values.each { |b| ostream.__send__("write_" + type, b) }
+    ostream.close
+    istream = @dir.open_input("rw_#{type}.test")
+    values.each { |b| assert_equal(b, istream.__send__("read_" + type), "#{type} should be equal") }
+    istream.close
+    assert_equal(expected_length, @dir.length("rw_#{type}.test"))
+  end
+
+end

data/test/unit/store/tm_store_lock.rb

@@ -0,0 +1,68 @@
+module StoreLockTest
+  class Switch
+    @@counter = 0
+    def Switch.counter() return @@counter end
+    def Switch.counter=(counter) @@counter = counter end
+  end
+
+  def test_locking()
+    lock_time_out = 0.001 # we want this test to run quickly
+    lock1 = @dir.make_lock("l.lck")
+    lock2 = @dir.make_lock("l.lck")
+
+    assert(!lock2.locked?)
+    assert(lock1.obtain(lock_time_out))
+    assert(lock2.locked?)
+
+    assert(! obtain_lock_true_false(lock2))
+
+    exception_thrown = false
+    begin
+      lock2.while_locked(lock_time_out) do
+        assert(false, "lock should not have been obtained")
+      end
+    rescue
+      exception_thrown = true
+    ensure
+      assert(exception_thrown)
+    end
+
+    lock1.release()
+    assert(lock2.obtain(lock_time_out))
+    lock2.release()
+
+    t = Thread.new() do
+      lock1.while_locked(lock_time_out) do
+        Switch.counter = 1
+        # make sure lock2 obtain test was run
+        while Switch.counter < 2
+        end
+        Switch.counter = 3
+      end
+    end
+    t.run()
+
+    #make sure thread has started and lock been obtained
+    while Switch.counter < 1
+    end
+    
+    assert(! obtain_lock_true_false(lock2))
+
+    Switch.counter = 2
+    while Switch.counter < 3
+    end
+    
+    assert(lock2.obtain(lock_time_out))
+    lock2.release()
+  end
+
+  def obtain_lock_true_false(lock)
+    lock_time_out = 0.001 # we want this test to run quickly
+    begin
+      lock.obtain(lock_time_out)
+      return true
+    rescue
+    end
+    return false
+  end
+end

data/test/unit/ts_analysis.rb

@@ -0,0 +1,16 @@
+$:.unshift File.dirname(__FILE__)
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+
+require 'analysis/tc_letter_tokenizer'
+require 'analysis/tc_white_space_tokenizer'
+require 'analysis/tc_lower_case_tokenizer'
+require 'analysis/tc_word_list_loader'
+require 'analysis/tc_lower_case_filter'
+require 'analysis/tc_stop_filter'
+require 'analysis/tc_porter_stem_filter'
+require 'analysis/tc_analyzer'
+require 'analysis/tc_stop_analyzer'
+require 'analysis/tc_white_space_analyzer'
+require 'analysis/tc_per_field_analyzer_wrapper'
+require 'analysis/tc_standard_tokenizer'
+require 'analysis/tc_standard_analyzer'

data/test/unit/ts_document.rb

@@ -0,0 +1,4 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'document/tc_field.rb'
+require 'document/tc_document.rb'

data/test/unit/ts_index.rb

@@ -0,0 +1,18 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'index/tc_compound_file_io.rb'
+require 'index/tc_field_infos.rb'
+require 'index/tc_fields_io.rb'
+require 'index/tc_index_writer.rb'
+require 'index/tc_index_reader.rb'
+require 'index/tc_segment_infos.rb'
+require 'index/tc_segment_term_docs.rb'
+require 'index/tc_segment_term_enum.rb'
+require 'index/tc_segment_term_vector.rb'
+require 'index/tc_term.rb'
+require 'index/tc_term_buffer.rb'
+require 'index/tc_term_info.rb'
+require 'index/tc_term_infos_io.rb'
+require 'index/tc_term_vector_offset_info.rb'
+require 'index/tc_term_vectors_io.rb'
+require 'index/tc_index.rb'

data/test/unit/ts_query_parser.rb

@@ -0,0 +1,3 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'query_parser/tc_query_parser.rb'

data/test/unit/ts_search.rb

@@ -0,0 +1,10 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'search/tc_similarity.rb'
+require 'search/tc_index_searcher.rb'
+require 'search/tc_fuzzy_query.rb'
+require 'search/tc_sort_field.rb'
+require 'search/tc_sort.rb'
+require 'search/tc_search_and_sort.rb'
+require 'search/tc_filter.rb'
+require 'search/tc_spans.rb'

data/test/unit/ts_store.rb

@@ -0,0 +1,6 @@
+$:.unshift File.dirname(__FILE__)
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+
+
+require 'store/tc_fs_store'
+require 'store/tc_ram_store'

data/test/unit/ts_utils.rb

@@ -0,0 +1,10 @@
+$:.unshift File.dirname(__FILE__)
+require File.join(File.dirname(__FILE__), "../test_helper.rb")
+
+require 'utils/tc_string_helper'
+require 'utils/tc_priority_queue'
+require 'utils/tc_bit_vector'
+require 'utils/tc_date_tools.rb'
+require 'utils/tc_number_tools.rb'
+require 'utils/tc_parameter.rb'
+require 'utils/tc_weak_key_hash.rb'

data/test/unit/utils/tc_bit_vector.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class BitVectorTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_bv()
+    bv = BitVector.new
+    assert_equal(0, bv.count)
+    bv.set(10)
+    assert(bv.get(10))
+    assert_equal(1, bv.count)
+    bv.set(10)
+    assert(bv.get(10))
+    assert_equal(1, bv.count)
+    bv.set(20)
+    assert(bv.get(20))
+    assert_equal(2, bv.count)
+    bv.set(21)
+    assert(bv.get(21))
+    assert_equal(3, bv.count)
+    bv.clear(21)
+    assert(!bv.get(21))
+    assert_equal(2, bv.count)
+    bv.clear(20)
+    assert(!bv.get(20))
+    assert_equal(1, bv.count)
+    assert(bv.get(10))
+  end
+
+  def test_bv_rw()
+    dir = Ferret::Store::RAMDirectory.new
+    bv = BitVector.new
+    assert_equal(0, bv.count)
+    bv.set(5)
+    assert_equal(1, bv.count)
+    bv.set(8)
+    assert_equal(2, bv.count)
+    bv.set(13)
+    assert_equal(3, bv.count)
+    bv.set(21)
+    assert_equal(4, bv.count)
+    bv.set(34)
+    assert_equal(5, bv.count)
+    bv.write(dir, "bv.test")
+    bv = nil
+    bv = BitVector.read(dir, "bv.test")
+    assert(!bv.get(4))
+    assert(bv.get(5))
+    assert(!bv.get(6))
+    assert(!bv.get(7))
+    assert(bv.get(8))
+    assert(!bv.get(9))
+    assert(!bv.get(12))
+    assert(bv.get(13))
+    assert(!bv.get(14))
+    assert(!bv.get(20))
+    assert(bv.get(21))
+    assert(!bv.get(22))
+    assert(!bv.get(33))
+    assert(bv.get(34))
+    assert(!bv.get(35))
+    assert_equal(5, bv.count)
+  end
+end

data/test/unit/utils/tc_date_tools.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class DateToolsTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_serialization()
+    # grab time to the nearest millisecond
+    t = Time.at((Time.now().to_i*1000).floor()/1000)
+
+    s = DateTools.serialize_time(t)
+
+    t_after = DateTools.deserialize_time(s)
+    assert_equal(t, t_after, "date changed after serialization")
+  end
+  
+  def test_serialization_constants()
+    # assert existance of these constants
+    assert(DateTools::MAX_SERIALIZED_DATE_STRING)
+    assert(DateTools::MIN_SERIALIZED_DATE_STRING)
+  end
+
+  def test_time_to_s()
+    t = Time.mktime(2004, 9, 5, 22, 33, 44, 555000)
+
+    assert_equal("2004", DateTools.time_to_s(t, DateTools::Resolution::YEAR))
+    assert_equal("200409", DateTools.time_to_s(t, DateTools::Resolution::MONTH))
+    assert_equal("20040905", DateTools.time_to_s(t, DateTools::Resolution::DAY))
+    assert_equal("2004090522", DateTools.time_to_s(t, DateTools::Resolution::HOUR))
+    assert_equal("200409052233", DateTools.time_to_s(t, DateTools::Resolution::MINUTE))
+    assert_equal("20040905223344", DateTools.time_to_s(t, DateTools::Resolution::SECOND))
+    assert_equal("20040905223344555", DateTools.time_to_s(t, DateTools::Resolution::MILLISECOND))
+  end
+
+  def test_s_to_time()
+    assert_equal(Time.mktime(2004), DateTools.s_to_time("2004"))
+    assert_equal(Time.mktime(2004, 9), DateTools.s_to_time("200409"))
+    assert_equal(Time.mktime(2004, 9, 5), DateTools.s_to_time("20040905"))
+    assert_equal(Time.mktime(2004, 9, 5, 22), DateTools.s_to_time("2004090522"))
+    assert_equal(Time.mktime(2004, 9, 5, 22, 33), DateTools.s_to_time("200409052233"))
+    assert_equal(Time.mktime(2004, 9, 5, 22, 33, 44), DateTools.s_to_time("20040905223344"))
+    assert_equal(Time.mktime(2004, 9, 5, 22, 33, 44, 555000), DateTools.s_to_time("20040905223344555"))
+  end
+
+  def test_round()
+    t = Time.mktime(2004, 9, 5, 22, 33, 44, 555000)
+    assert_equal(Time.mktime(2004, 9, 5), DateTools.round(t, DateTools::Resolution::DAY))
+  end
+
+end