ferret 0.1.0

data/test/unit/analysis/tc_white_space_tokenizer.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class WhiteSpaceTokenizerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_whitespacetokenizer()
+    input = StringReader.new('DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$')
+    t = WhiteSpaceTokenizer.new(input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
+    assert_equal(Token.new('is', 19, 21), t.next())
+    assert_equal(Token.new('My', 22, 24), t.next())
+    assert_equal(Token.new('e-mail', 25, 31), t.next())
+    assert_equal(Token.new('52', 32, 34), t.next())
+    assert_equal(Token.new('#$', 37, 39), t.next())
+    assert_equal(Token.new('address.', 40, 48), t.next())
+    assert_equal(Token.new('23#@$', 49, 54), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_word_list_loader.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class WordListLoaderTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  def test_word_set_from_file()
+    wl = WordListLoader.word_set_from_file(File.dirname(__FILE__) + '/data/wordfile')
+    assert_equal(6, wl.size())
+    assert(wl.member?('and'))
+    assert(wl.member?('to'))
+    assert(wl.member?('it'))
+    assert(wl.member?('the'))
+    assert(wl.member?('there'))
+    assert(wl.member?('their'))
+    assert(!wl.member?('horse'))
+    assert(!wl.member?('judo'))
+    assert(!wl.member?('dairy'))
+  end
+
+  def test_word_set_from_array()
+    wl = WordListLoader.word_set_from_array(['and','to','it','the','there','their'])
+    assert_equal(6, wl.size())
+    assert(wl.member?('and'))
+    assert(wl.member?('to'))
+    assert(wl.member?('it'))
+    assert(wl.member?('the'))
+    assert(wl.member?('there'))
+    assert(wl.member?('their'))
+    assert(!wl.member?('horse'))
+    assert(!wl.member?('judo'))
+    assert(!wl.member?('dairy'))
+  end
+end

data/test/unit/document/tc_document.rb

@@ -0,0 +1,47 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class DocumentTest < Test::Unit::TestCase
+  include Ferret::Document
+  def test_document()
+    doc = Document.new()
+    f11 = Field.new("field1", "value1", Field::Store::YES, Field::Index::NO)
+    f12 = Field.new("field1", "value2", Field::Store::YES, Field::Index::NO)
+    f13 = Field.new("field1", "value3", Field::Store::YES, Field::Index::NO)
+    f21 = Field.new("field2", "value1", Field::Store::YES, Field::Index::NO)
+    doc.add_field(f11)
+    doc.add_field(f12)
+    doc.add_field(f13)
+    doc.add_field(f21)
+    assert_equal(3, doc.fields("field1").size)
+    assert_equal(1, doc.fields("field2").size)
+    field = doc.remove_field("field1")
+    assert_equal(2, doc.fields("field1").size)
+    assert_equal(f11, field)
+    assert_equal("value2 value3", doc.values("field1"))
+    doc.remove_fields("field1")
+    assert_equal(nil, doc.field("field1"))
+  end
+
+  def test_binary_string()
+    tmp = []
+    256.times {|i| tmp[i] = i}
+    bin1 = tmp.pack("c*")
+    tmp = []
+    56.times {|i| tmp[i] = i}
+    bin2 = tmp.pack("c*")
+    doc = Document.new()
+    fs1 = Field.new("field1", "value1", Field::Store::YES, Field::Index::NO)
+    fs2 = Field.new("field1", "value2", Field::Store::YES, Field::Index::NO)
+    fb1 = Field.new_binary_field("field1", bin1, Field::Store::YES)
+    fb2 = Field.new_binary_field("field1", bin2, Field::Store::YES)
+
+    doc.add_field(fs1)
+    doc.add_field(fs2)
+    doc.add_field(fb1)
+    doc.add_field(fb2)
+
+    assert_equal(4, doc.fields("field1").size)
+    assert_equal("value1 value2", doc.values("field1").strip)
+    assert_equal([bin1, bin2], doc.binaries("field1"))
+  end
+end

data/test/unit/document/tc_field.rb

@@ -0,0 +1,80 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class FieldTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Utils
+
+  def test_store()
+    assert_equal("COMPRESS", Field::Store::COMPRESS.to_s)
+    assert_equal("YES", Field::Store::YES.to_s)
+    assert_equal("NO", Field::Store::NO.to_s)
+  end
+
+  def test_index()
+    assert_equal("TOKENIZED", Field::Index::TOKENIZED.to_s)
+    assert_equal("UNTOKENIZED", Field::Index::UNTOKENIZED.to_s)
+    assert_equal("NO", Field::Index::NO.to_s)
+  end
+
+  def test_term_vector()
+    assert_equal("YES", Field::TermVector::YES.to_s)
+    assert_equal("NO", Field::TermVector::NO.to_s)
+    assert_equal("WITH_POSITIONS", Field::TermVector::WITH_POSITIONS.to_s)
+    assert_equal("WITH_OFFSETS", Field::TermVector::WITH_OFFSETS.to_s)
+    assert_equal("WITH_POSITIONS_OFFSETS", Field::TermVector::WITH_POSITIONS_OFFSETS.to_s)
+  end
+
+  def test_standard_field()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    assert_equal("name", f.name)
+    assert_equal("value", f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(true, f.compressed?)
+    assert_equal(true, f.indexed?)
+    assert_equal(true, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(false, f.binary?)
+  end
+
+  def test_set_store()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.stored = Field::Store::NO
+    assert_equal(false, f.stored?)
+    assert_equal(false, f.compressed?)
+  end
+
+  def test_set_index()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.index = Field::Index::NO
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+  end
+
+  def test_set_term_vector()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.store_term_vector = Field::TermVector::WITH_POSITIONS_OFFSETS
+    assert_equal(true, f.store_term_vector?)
+    assert_equal(true, f.store_offsets?)
+    assert_equal(true, f.store_positions?)
+  end
+
+  def test_new_binary_field()
+    tmp = []
+    256.times {|i| tmp[i] = i}
+    bin = tmp.pack("c*")
+    f = Field.new_binary_field("name", bin, Field::Store::YES)
+    assert_equal("name", f.name)
+    assert_equal(bin, f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(false, f.compressed?)
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(true, f.binary?)
+  end
+end

data/test/unit/index/tc_compound_file_io.rb

@@ -0,0 +1,107 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class CompoundFileWriterTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_writer
+    file1 = @dir.create_output("file1")
+    file2 = @dir.create_output("file2")
+    file1.write_int(20)
+    file2.write_string('this is file2')
+    file1.close()
+    file2.close()
+    cfile_writer = CompoundFileWriter.new(@dir, "cfile")
+    cfile_writer.add_file("file1")
+    cfile_writer.add_file("file2")
+    cfile_writer.close()
+    
+    cfile = @dir.open_input("cfile")
+    assert_equal(2, cfile.read_vint())
+    assert_equal(29, cfile.read_long(), "Offset is incorrect")
+    assert_equal("file1", cfile.read_string(), "Filename is incorrect")
+    assert_equal(33, cfile.read_long(), "Offset is incorrect")
+    assert_equal("file2", cfile.read_string(), "Filename is incorrect")
+    assert_equal(20, cfile.read_int(), "Content is incorrect")
+    assert_equal('this is file2', cfile.read_string(), "Content is incorrect")
+  end
+end
+
+class CompoundFileReaderTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_reader
+    cfile = @dir.create_output("cfile")
+    cfile.write_vint(2)
+    cfile.write_long(29)
+    cfile.write_string('file1')
+    cfile.write_long(33)
+    cfile.write_string('file2')
+    cfile.write_int(20)
+    cfile.write_string("this is file 2")
+    cfile.close()
+    
+    cfile_reader = CompoundFileReader.new(@dir, "cfile")
+    assert_equal(4, cfile_reader.file_length('file1'))
+    assert_equal(15, cfile_reader.file_length('file2'))
+    file1 = cfile_reader.open_input('file1')
+    file2 = cfile_reader.open_input('file2')
+    assert_equal(20, file1.read_int())
+    assert_equal('this is file 2', file2.read_string())
+    file1.close()
+    file2.close()
+  end
+end
+
+class CompoundFileIOTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_buffer
+    file1 = @dir.create_output("file1")
+    file2 = @dir.create_output("file2")
+    file3 = @dir.create_output("file3")
+    20.times { file1.write_int(rand(10000)) }
+    file2.write_string('this is file2' * 1000)
+    file3.write_string('this is file2')
+    file1.close()
+    file2.close()
+    file3.close()
+    cfile_writer = CompoundFileWriter.new(@dir, "cfile")
+    cfile_writer.add_file("file1")
+    cfile_writer.add_file("file2")
+    cfile_writer.add_file("file3")
+    cfile_writer.close()
+    
+    cfile_reader = CompoundFileReader.new(@dir, "cfile")
+    file2 = cfile_reader.open_input('file2')
+    assert_equal('this is file2' * 1000, file2.read_string)
+    file2.close
+  end
+end

data/test/unit/index/tc_field_infos.rb

@@ -0,0 +1,119 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class FieldInfosTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  def test_field_info()
+    fi = FieldInfo.new("name", true, 1, true, true, true)
+    assert_equal(fi.name, "name")
+    assert_equal(fi.number, 1)
+    assert(fi.indexed?)
+    assert(fi.store_term_vector?)
+    assert(fi.store_offsets?)
+    assert(fi.store_positions?)
+
+    fi.name = "hello"
+    fi.indexed = false
+    fi.number = 2
+    fi.store_term_vector = false
+    fi.store_offset = false
+    fi.store_position = false
+
+    assert_equal(fi.name, "hello")
+    assert_equal(fi.number, 2)
+    assert(!fi.indexed?)
+    assert(!fi.store_term_vector?)
+    assert(!fi.store_offsets?)
+    assert(!fi.store_positions?)
+
+    fi.set!(true, true, true, true)
+    assert(fi.indexed?)
+    assert(fi.store_term_vector?)
+    assert(fi.store_offsets?)
+    assert(fi.store_positions?)
+
+    fi = FieldInfo.new("name", true, 1, true)
+    assert(!fi.store_offsets?)
+    assert(!fi.store_positions?)
+  end
+
+  def fi_test_attr(fi, name, number, indexed, store_tv, store_pos, store_off)
+    assert_equal(name, fi.name)
+    assert_equal(number, fi.number)
+    assert_equal(indexed, fi.indexed?)
+    assert_equal(store_tv, fi.store_term_vector?)
+    assert_equal(store_pos, fi.store_positions?)
+    assert_equal(store_off, fi.store_offsets?)
+  end
+
+  def test_fis_add()
+    fis = FieldInfos.new()
+    fi = fis.add("field1", false)
+    fi_test_attr(fi, "field1", 0, false, false, false, false)
+    assert_equal(1, fis.size)
+
+    fi = fis.add("field1", true, true)
+    fi_test_attr(fi, "field1", 0, true, true, false, false)
+    assert_equal(1, fis.size)
+
+    fi = fis.add("field2", false)
+    fi_test_attr(fi, "field2", 1, false, false, false, false)
+    assert_equal(2, fis.size)
+
+    fi = fis.add("field1", true, true, true, true)
+    assert_equal(fi, fis[fi.number])
+    assert_equal(fi, fis["field1"])
+    assert_equal(0, fis.field_number("field1"))
+    assert_equal(1, fis.field_number("field2"))
+    assert_equal(FieldInfos::NOT_A_FIELD, fis.field_number("field3"))
+    assert_equal(nil, fis["field3"])
+    fi_test_attr(fi, "field1", 0, true, true, true, true)
+    assert_equal(2, fis.size)
+  end
+
+  def test_add_doc_fields
+    doc = IndexTestHelper.prepare_document
+    fis = FieldInfos.new()
+    fis << doc
+    dir = Ferret::Store::RAMDirectory.new
+    fis.write_to_dir(dir, "_test")
+    fis2 = FieldInfos.new(dir, "_test")
+    assert_equal("text_field1", fis2["text_field1"].name)
+    fn = fis2.field_number("text_field2")
+    assert_equal("text_field2", fis2[fn].name)
+    assert_equal(9, fis2.size)
+    assert(fis.has_vectors?)
+  end
+
+  def test_fis_has_vectors
+    fis = FieldInfos.new()
+    assert(! fis.has_vectors?)
+    fis.add("random_field")
+    assert(! fis.has_vectors?)
+    fis.add("store_term_vector_field", true, true, false, false)
+    assert(fis.has_vectors?)
+  end
+
+
+  def test_fis_rw()
+    fis = FieldInfos.new()
+    dir = Ferret::Store::RAMDirectory.new()
+    fis.add("field1", false, false, false, false)
+    fis.add("field2", true, false, false, false)
+    fis.add("field3", true, true, false, false)
+    fis.add("field4", true, true, true, false)
+    fis.add("field5", true, true, true, true)
+    fis.write_to_dir(dir, "fis_rw.test")
+    fis = nil
+
+    fis = FieldInfos.new(dir, "fis_rw.test")
+    fi_test_attr(fis[0], "field1", 0, false, false, false, false)
+    fi_test_attr(fis[1], "field2", 1, true, false, false, false)
+    fi_test_attr(fis[2], "field3", 2, true, true, false, false)
+    fi_test_attr(fis[3], "field4", 3, true, true, true, false)
+    fi_test_attr(fis[4], "field5", 4, true, true, true, true)
+
+    assert_equal(5, fis.size)
+  end
+
+end

data/test/unit/index/tc_fields_io.rb

@@ -0,0 +1,167 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class FieldsWriterTest < Test::Unit::TestCase
+
+  include Ferret::Index
+  include Ferret::Document
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_writer
+    doc = Document.new
+    doc << Field.new("name", "daily news", Field::Store::YES)
+    doc << Field.new("content", "Nothing happened today.", Field::Store::YES)
+    
+    infos = FieldInfos.new
+    infos << doc
+    
+    writer = FieldsWriter.new(@dir, "fieldswritertest", infos)
+    writer << doc
+    writer.close
+
+    fstream = @dir.open_input("fieldswritertest.fdt")
+    istream = @dir.open_input("fieldswritertest.fdx")
+
+    stored = fstream.read_vint
+		field_num1 = fstream.read_vint
+		byte1 = fstream.read_byte
+		data1 = fstream.read_string
+		assert( stored == 2 )
+		assert( (byte1 |= FieldsWriter::FIELD_IS_TOKENIZED) != 0 )
+		assert( data1 == "daily news" )
+		
+		field_num2 = fstream.read_vint
+		byte2 = fstream.read_byte
+		data2 = fstream.read_string
+		assert( (byte2 |= FieldsWriter::FIELD_IS_TOKENIZED) != 0 )
+		assert( data2 == "Nothing happened today." )
+
+  end
+end
+
+class FieldsReaderTest < Test::Unit::TestCase
+
+  include Ferret::Index
+  include Ferret::Document
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_doc
+    doc = Document.new
+    doc << Field.new("name", "daily news")
+    doc << Field.new("content", "Nothing happened today.")
+    
+    infos = FieldInfos.new
+    infos << doc
+
+    fstream = @dir.create_output("fieldsreadertest.fdt")
+    istream = @dir.create_output("fieldsreadertest.fdx")
+
+    istream.write_long(0)
+    istream.close
+    fstream.write_vint(2)
+    fstream.write_vint(0)
+    fstream.write_byte(0)
+    fstream.write_string("daily news")
+    fstream.write_vint(1)
+    fstream.write_byte(0)
+    fstream.write_string("Nothing happened today.")
+    fstream.close
+
+    reader = FieldsReader.new(@dir, "fieldsreadertest", infos)
+    docres = reader.doc(0)
+    
+    assert_equal(docres.field("name").data, "daily news")
+    assert_equal(docres.field("content").data, "Nothing happened today.")
+  end
+end
+
+class FieldsIOTest < Test::Unit::TestCase
+
+  include Ferret::Index
+  include Ferret::Document
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+    doc = IndexTestHelper.prepare_document()
+    infos = FieldInfos.new
+    infos << doc
+
+    writer = FieldsWriter.new(@dir, "field_types", infos)
+    writer << doc
+    writer.close
+
+    reader = FieldsReader.new(@dir, "field_types", infos)
+    @docres = reader.doc(0)
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_text_field_no_term_vector
+    field = @docres.field("text_field1")
+    check_field_values(field, "field one text", true, true, true, false, false)
+  end
+
+  def test_text_field_term_vector
+    field = @docres.field("text_field2")
+    check_field_values(field, "field field field two text", true, true, true, true, false)
+  end
+
+  def test_key_field
+    field = @docres.field("key_field")
+    check_field_values(field, "keyword", true, true, false, false, false)
+  end
+
+  def test_unindexed_field
+    field = @docres.field("unindexed_field")
+    check_field_values(field, "unindexed field text", true, false, false, false, false)
+  end
+
+  def test_unstored_field_no_term_vector
+    field = @docres.field("unstored_field1")
+    assert_equal(nil, field)
+  end
+
+  def test_compressed_field
+    field = @docres.field("compressed_field")
+    check_field_values(field, "compressed text", true, true, true, true, false)
+  end
+
+  def test_binary_field
+    bin = IndexTestHelper::BINARY_DATA
+    field = @docres.field("binary_field")
+    check_field_values(field, bin, true, false, false, false, true)
+  end
+
+  def test_compressed_binary_field
+    cbin = IndexTestHelper::COMPRESSED_BINARY_DATA
+    field = @docres.field("compressed_binary_field")
+    check_field_values(field, cbin, true, false, false, false, true)
+  end
+
+
+  private
+  
+  def check_field_values(field, value, stored, indexed, tokenized, term_vector, binary)
+    assert_equal(value, field.data)
+    assert_equal(stored, field.stored?)
+    assert_equal(indexed, field.indexed?)
+    assert_equal(tokenized, field.tokenized?)
+    assert_equal(term_vector, field.store_term_vector?)
+    assert_equal(binary, field.binary?)
+  end
+end