ferret 0.1.0

data/test/unit/index/tc_index_writer.rb

@@ -0,0 +1,57 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class IndexWriterTest < Test::Unit::TestCase
+  include Ferret::Index
+  include Ferret::Analysis
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_initialize
+    wlock = @dir.make_lock(IndexWriter::WRITE_LOCK_NAME)
+    clock = @dir.make_lock(IndexWriter::COMMIT_LOCK_NAME)
+    assert(! wlock.locked?)
+    assert(! clock.locked?)
+    iw = IndexWriter.new(@dir, :create => true)
+    assert(@dir.exists?("segments"))
+    assert(wlock.locked?)
+    iw.close()
+    assert(@dir.exists?("segments"))
+    assert(! wlock.locked?)
+    assert(! clock.locked?)
+  end
+
+  def test_add_document
+    iw = IndexWriter.new(@dir, :analyzer => StandardAnalyzer.new(), :create => true)
+    doc = IndexTestHelper.prepare_document()
+    infos = FieldInfos.new
+    infos << doc
+    iw.add_document(doc)
+    assert_equal(1, iw.doc_count)
+    iw.close()
+  end
+
+  def test_add_documents
+    iw = IndexWriter.new(@dir, :analyzer => StandardAnalyzer.new(), :create => true)
+    # uncomment the following line to see logging
+    #iw.info_stream = $stdout
+    iw.merge_factor = 3
+    iw.min_merge_docs = 3
+    docs = IndexTestHelper.prepare_book_list()
+    infos = FieldInfos.new
+    infos << docs[0]
+    docs.each_with_index do |doc, i|
+      #puts "Index doc " + i.to_s
+      iw.add_document(doc)
+    end
+    assert_equal(37, iw.doc_count)
+    iw.close()
+  end
+
+end

data/test/unit/index/tc_multiple_term_doc_pos_enum.rb

@@ -0,0 +1,80 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class MultipleTermDocPosEnumTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Analysis
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new()
+    iw = IndexWriter.new(@dir, WhiteSpaceAnalyzer.new(), true, false)
+    @documents = IndexTestHelper.prepare_search_docs()
+    @documents.each { |doc| iw << doc }
+    iw.close()
+    @ir = IndexReader.open(@dir, true)
+  end
+
+  def tear_down()
+    @ir.close
+  end
+
+  def test_mtdpe()
+    t1 = Term.new("field", "red")
+    t2 = Term.new("field", "brown")
+    t3 = Term.new("field", "hairy")
+    mtdpe = MultipleTermDocPosEnum.new(@ir, [t1, t2, t3])
+    assert(mtdpe.next?)
+    assert_equal(1, mtdpe.doc)
+    assert_equal(1, mtdpe.freq)
+    assert_equal(4, mtdpe.next_position)
+    
+    assert(mtdpe.next?)
+    assert_equal(8, mtdpe.doc)
+    assert_equal(1, mtdpe.freq)
+    assert_equal(5, mtdpe.next_position)
+    
+    assert(mtdpe.next?)
+    assert_equal(11, mtdpe.doc)
+    assert_equal(1, mtdpe.freq)
+    assert_equal(4, mtdpe.next_position)
+    
+    assert(mtdpe.next?)
+    assert_equal(14, mtdpe.doc)
+    assert_equal(1, mtdpe.freq)
+    assert_equal(4, mtdpe.next_position)
+    
+    assert(mtdpe.next?)
+    assert_equal(16, mtdpe.doc)
+    assert_equal(3, mtdpe.freq)
+    assert_equal(5, mtdpe.next_position)
+    assert_equal(7, mtdpe.next_position)
+    assert_equal(11, mtdpe.next_position)
+    
+    assert(mtdpe.next?)
+    assert_equal(17, mtdpe.doc)
+    assert_equal(2, mtdpe.freq)
+    assert_equal(2, mtdpe.next_position)
+    assert_equal(7, mtdpe.next_position)
+    
+    assert(!mtdpe.next?)
+    mtdpe.close()
+  end
+
+  def test_tp
+    tp = @ir.term_positions_for(Term.new("field", "red"))
+    assert(tp.next?)
+    assert_equal(11, tp.doc)
+    assert_equal(1, tp.freq)
+    assert_equal(4, tp.next_position)
+
+    assert(tp.next?)
+    assert_equal(16, tp.doc)
+    assert_equal(1, tp.freq)
+    assert_equal(11, tp.next_position)
+
+    assert(tp.next?)
+    assert_equal(17, tp.doc)
+    assert_equal(1, tp.freq)
+    assert_equal(7, tp.next_position)
+    tp.close()
+  end
+end

data/test/unit/index/tc_segment_infos.rb

@@ -0,0 +1,74 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class SegmentInfosTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_read_write
+    assert_equal(0, SegmentInfos.read_current_version(@dir))
+    sis = SegmentInfos.new()
+    seg0 = SegmentInfo.new('seg0', 5, @dir)
+    seg1 = SegmentInfo.new('seg1', 5, @dir)
+    seg2 = SegmentInfo.new('seg2', 5, @dir)
+    seg3 = SegmentInfo.new('seg3', 5, @dir)
+    sis << seg0
+    sis << seg1
+    sis << seg2
+    assert_equal(sis.size(), 3)
+    assert_equal(sis[0], seg0)
+    assert_equal(sis[2], seg2)
+    sis.write(@dir)
+    version = SegmentInfos.read_current_version(@dir)
+    assert(@dir.exists?('segments'))
+    sis2 = SegmentInfos.new()
+    sis2.read(@dir)
+    assert_equal(sis2.size(), 3)
+    assert_equal(sis2[0], seg0)
+    assert_equal(sis2[2], seg2)
+    sis2 << seg3
+    sis2.write(@dir)
+    assert_equal(version + 1, SegmentInfos.read_current_version(@dir))
+    sis3 = SegmentInfos.new()
+    sis3.read(@dir)
+    assert_equal(sis3.size(), 4)
+    assert_equal(sis2[0], seg0)
+    assert_equal(sis2[3], seg3)
+  end
+end
+
+class SegmentInfoTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  # just test getters and setters. Nothing else.
+  def test_segment_info
+    si = SegmentInfo.new("seg1", 0, @dir)
+    assert_equal(si.directory, @dir)
+    assert_equal(si.doc_count, 0)
+    assert_equal(si.name, "seg1")
+    @dir.close()
+    @dpath = File.dirname(__FILE__) + '/../../temp/fsdir'
+    @dir = Ferret::Store::FSDirectory.get_directory(@dpath, true)
+    si.name = "seg2"
+    si.doc_count += 2
+    si.directory = @dir
+    assert_equal(si.directory, @dir)
+    assert_equal(si.doc_count, 2)
+    assert_equal(si.name, "seg2")
+  end
+end

data/test/unit/index/tc_segment_term_docs.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SegmentTermDocEnumTest < Test::Unit::TestCase
+
+  include Ferret::Index
+  include Ferret::Analysis
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+    @doc = IndexTestHelper.prepare_document()
+    IndexTestHelper.write_document(@dir, @doc)
+  end
+
+  def test_something()
+    assert true
+  end
+end

data/test/unit/index/tc_segment_term_enum.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class SegmentTermEnumTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  TEST_SEGMENT = "_test"
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def test_initialize()
+    fis = FieldInfos.new
+    fis.add("author", true, true)
+    fis.add("title", true, true)
+    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
+    terms = [ Term.new("author", "Martel"),
+              Term.new("title", "Life of Pi"),
+              Term.new("author", "Martin"),
+              Term.new("title", "Life on the edge") ].sort
+    term_infos = []
+    4.times {|i| term_infos << TermInfo.new(i,i,i,0)}
+    4.times {|i| tiw.add(terms[i], term_infos[i]) }
+    tiw.close()
+
+    tis_file = @dir.open_input(TEST_SEGMENT + ".tis")
+    
+    ste = SegmentTermEnum.new(tis_file, fis, false)
+    assert_equal(128, ste.index_interval)
+    assert_equal(16, ste.skip_interval)
+    assert_equal(4, ste.size)
+    assert(ste.next?)
+    assert_equal(terms[0], ste.term)
+    assert_equal(term_infos[0], ste.term_info)
+    ti = TermInfo.new
+    ste.term_info = ti
+    assert_equal(term_infos[0], ti)
+    assert(ste.next?)
+    assert_equal(terms[0], ste.prev)
+    assert_equal(terms[1], ste.term)
+    assert_equal(term_infos[1], ste.term_info)
+    assert(ste.next?)
+    assert_equal(terms[2], ste.term)
+    assert_equal(term_infos[2], ste.term_info)
+    assert(ste.next?)
+    assert_equal(terms[3], ste.term)
+    assert_equal(term_infos[3], ste.term_info)
+    ste.close()
+
+    tii_file = @dir.open_input(TEST_SEGMENT + ".tii")
+
+    ste = SegmentTermEnum.new(tii_file, fis, false)
+    assert_equal(128, ste.index_interval)
+    assert_equal(16, ste.skip_interval)
+    assert_equal(1, ste.size)
+    assert(ste.next?)
+    assert(Term.new("", ""), ste.term)
+  end
+end

data/test/unit/index/tc_segment_term_vector.rb

@@ -0,0 +1,71 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class SegmentTermVectorTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @terms = ["Apples", "Oranges", "Bananas", "Kiwis", "Mandarins"]
+    term_freqs = [4,2,1,12,4]
+    @stv = SegmentTermVector.new("Fruits", @terms, term_freqs)
+  end
+
+  def test_size()
+    assert_equal(@terms.size(), @stv.size())
+  end
+
+  def test_index_of()
+    assert_equal(0, @stv.index_of("Apples"))
+    assert_equal(4, @stv.term_frequencies[@stv.index_of("Apples")])
+  end
+
+  def test_indexes_of()
+    assert_equal([2, 0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 0, 3))
+    assert_equal([0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 1, 2))
+  end
+end
+
+class SegmentTermVectorWithPosOffsetsTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @terms = ["Apples", "Oranges", "Bananas", "Kiwis", "Mandarins"]
+    term_freqs = [4,2,1,12,4]
+    term_positions = [
+      [1,3,5,7],
+      [2,4],
+      [6],
+      [8,9,10,12,13,14,16,17,18,20,21,22],
+      [11,15,19,23]
+    ]
+    term_offsets = [
+      [[1,4],[10,14],[20,24],[30,34]],
+      [[5,9],[15,19]],
+      [[25,29]],
+      [[35,39],[40,44],[45,49],[55,59],[60,64],[65,69],[75,79],[80,84],[85,89],[95,99],[100,104],[105,109]],
+      [[50,54],[70,74],[90,94],[110,114]]
+    ]
+    @stv = SegmentTermVector.new("Fruits", @terms, term_freqs, term_positions, term_offsets)
+  end
+
+  def test_size()
+    assert_equal(@terms.size(), @stv.size())
+  end
+
+  def test_index_of()
+    assert_equal(0, @stv.index_of("Apples"))
+    assert_equal(4, @stv.term_frequencies[@stv.index_of("Apples")])
+  end
+
+  def test_indexes_of()
+    assert_equal([2, 0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 0, 3))
+    assert_equal([0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 1, 2))
+  end
+
+  def test_positions_offsets()
+    assert_equal([1,3,5,7], @stv.positions[@stv.index_of("Apples")])
+    assert_equal([[35,39],[40,44],[45,49],[55,59],[60,64],[65,69],[75,79],[80,84],[85,89],[95,99],[100,104],[105,109]], @stv.offsets[@stv.index_of("Kiwis")])
+  end
+end

data/test/unit/index/tc_term.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermTest < Test::Unit::TestCase
+  include Ferret::Index
+  def test_term()
+    term1 = Term.new("bfield1", "athis is text1")
+    assert_equal(term1.field, "bfield1")
+    assert_equal(term1.text, "athis is text1")
+    term2 = Term.new("afield2", "athis is text1")
+    term3 = Term.new("bfield1", "bthis is text2")
+    term4 = Term.new("bfield1", "athis is text1")
+    assert(term1 > term2)
+    assert(term1 < term3)
+    assert(term1.between?(term2, term3))
+    assert(term1 == term4)
+    assert(term1.eql?(term4))
+    term4.set!("field3", "text3")
+    assert(term1 != term4)
+  end
+
+end

data/test/unit/index/tc_term_buffer.rb

@@ -0,0 +1,57 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermBufferTest < Test::Unit::TestCase
+  include Ferret::Index
+  def test_term_set()
+    t = Term.new("title", "Ferret Tutorial")
+    tb = TermBuffer.new
+    tb.term = t
+    assert_equal(t.field, tb.field)
+    assert_equal("Ferret Tutorial", tb.text_str)
+    assert_equal("Ferret Tutorial".length, tb.text_length)
+    assert_equal(t, tb.term)
+  end
+
+  def test_set()
+    tb = TermBuffer.new
+    tb.term = Term.new("title", "Ferret Tutorial")
+    tb2 = TermBuffer.new
+    tb2.set!(tb)
+    assert_equal(tb.field, tb2.field)
+    assert_equal("Ferret Tutorial", tb2.text_str)
+    assert_equal("Ferret Tutorial".length, tb2.text_length)
+    assert_equal(tb.term, tb2.term)
+  end
+
+  def test_compare()
+    tb1 = TermBuffer.new
+    tb2 = TermBuffer.new
+    tb1.term = Term.new("alpha", "text")
+    tb2.term = Term.new("bravo", "text")
+    assert(tb1 < tb2)
+    tb2.term = Term.new("alpha", "text")
+    assert(tb1 == tb2)
+    tb2.term = Term.new("alpha", "tex")
+    assert(tb1 > tb2)
+  end
+
+  def test_read()
+    dir = Ferret::Store::RAMDirectory.new
+    fi = FieldInfos.new
+    tb = TermBuffer.new
+    tb.term = Term.new("Author", "Dave")
+    fi.add("Writer", true)
+    output = dir.create_output("term_buffer_read_test")
+    output.write_vint(4)
+    output.write_vint(8)
+    output.write_chars(" Balmain", 0, 8)
+    output.write_vint(fi.field_number("Writer"))
+    output.close
+    input = dir.open_input("term_buffer_read_test")
+    tb.read(input, fi)
+    assert_equal("Dave Balmain", tb.text_str)
+    assert_equal("Dave Balmain", tb.term.text)
+    assert_equal("Writer", tb.field)
+  end
+end