ferret 0.9.6 → 0.10.0

data/test/unit/index/rtc_term_vectors_io.rb

@@ -1,108 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class TermVectorsIOTest < Test::Unit::TestCase
-
-  include Ferret::Index
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-    @fis = FieldInfos.new
-    @fis.add("field1", true, true, true, true)
-    @fis.add("field2", true, true)
-  end
-
-  def tear_down()
-    @dir.close()
-  end
-
-  def test_tv_io_add_fields()
-    tv_w = TermVectorsWriter.new(@dir, "_test", @fis)
-    tv_w.open_document
-    assert(tv_w.document_open?)
-    tv_w.open_field("field1")
-    tv_w.add_term("text1", 1, [1], [t(0,4)])
-    tv_w.add_term("text2", 2, [3,4], [t(5,10), t(11,16)])
-    tv_w.close_field()
-    tv_w.close_document()
-    tv_w.close()
-
-    tv_r = TermVectorsReader.new(@dir, "_test", @fis)
-    assert_equal(1, tv_r.size)
-    tv = tv_r.get_field_tv(0, "field1")
-
-    assert_equal(2, tv.size)
-    assert_equal("text1", tv.terms[0])
-    assert_equal(1, tv.freqs[0])
-    assert_equal(1, tv.positions[0][0])
-    assert_equal(t(0,4), tv.offsets[0][0])
-
-    assert_equal("text2", tv.terms[1])
-    assert_equal(2, tv.freqs[1])
-    assert_equal(3, tv.positions[1][0])
-    assert_equal(t(5,10), tv.offsets[1][0])
-    assert_equal(4, tv.positions[1][1])
-    assert_equal(t(11,16), tv.offsets[1][1])
-    tv_r.close
-  end
-
-  def test_tv_io_add_documents()
-    tvs1 = [] 
-    tvs2 = [] 
-    tv = SegmentTermVector.new("field1",
-           ["word1", "word2"],
-           [3, 2],
-           [[1, 5, 8], [2, 9]],
-           [[t(0,5), t(34,39), t(45,50)],[t(6,11), t(51,56)]])
-    tvs1 << tv
-    tv = SegmentTermVector.new("field2",
-           ["word3", "word4"],
-           [1, 5],
-           [[8], [2, 9, 11, 34, 56]],
-           [[t(45,50)], [t(6,10), t(51,56), t(64,69), t(103,108), t(183,188)]])
-    tvs1 << tv
-    tv_w = TermVectorsWriter.new(@dir, "_test", @fis)
-    tv = SegmentTermVector.new("field1",
-           ["word1", "word2"],
-           [3, 2],
-           [[1, 5, 8], [2, 9]],
-           [[t(0,5), t(34,39), t(45,50)],[t(6,11), t(51,56)]])
-    tvs2 << tv
-    tv_w.add_all_doc_vectors(tvs1)
-    tv_w.add_all_doc_vectors(tvs2)
-    tv_w.close
-    tv_r = TermVectorsReader.new(@dir, "_test", @fis)
-    assert_equal(2, tv_r.size)
-    tv = tv_r.get_field_tv(0, "field1")
-
-    assert_equal(2, tv.size)
-    assert_equal("word1", tv.terms[0])
-    assert_equal(3, tv.freqs[0])
-    assert_equal(1, tv.positions[0][0])
-    assert_equal(5, tv.positions[0][1])
-    assert_equal(8, tv.positions[0][2])
-    assert_equal(t(0,5), tv.offsets[0][0])
-    assert_equal(t(34,39), tv.offsets[0][1])
-    assert_equal(t(45,50), tv.offsets[0][2])
-
-    assert_equal("word2", tv.terms[1])
-    assert_equal(2, tv.freqs[1])
-    assert_equal(2, tv.positions[1][0])
-    assert_equal(9, tv.positions[1][1])
-    assert_equal(t(6,11), tv.offsets[1][0])
-    assert_equal(t(51,56), tv.offsets[1][1])
-
-    tv = tv_r.get_field_tv(0, "field2")
-    assert_equal(2, tv.size)
-    assert_equal("word3", tv.terms[0])
-
-    tv = tv_r.get_field_tv(1, "field1")
-    assert_equal(2, tv.size)
-    assert_equal("word1", tv.terms[0])
-  end
-
-  private
-    def t(start, finish)
-      return TermVectorOffsetInfo.new(start, finish)
-    end
-end

data/test/unit/index/tc_term.rb

@@ -1,27 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class TermTest < Test::Unit::TestCase
-  include Ferret::Index
-  def test_term()
-    term1 = Term.new("bfield1", "athis is text1")
-    assert_equal(term1.field, "bfield1")
-    assert_equal(term1.text, "athis is text1")
-    term2 = Term.new("afield2", "athis is text1")
-    term3 = Term.new("bfield1", "bthis is text2")
-    term4 = Term.new("bfield1", "athis is text1")
-    assert(term1 > term2)
-    assert(term1 < term3)
-    assert(term1.between?(term2, term3))
-    assert(term1 == term4)
-    assert(term1.eql?(term4))
-    term4.set!("field3", "text3")
-    assert_not_equal(term1, term4)
-  end
-
-  def test_non_strings()
-    t = Term.new(2345, 3)
-    t = Term.new(:symbol, :symbol)
-    t.set!(:symbol, :symbol)
-    t.set!(234, 23462346)
-  end
-end

data/test/unit/index/tc_term_voi.rb

@@ -1,18 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class TermVectorOffsetInfoTest < Test::Unit::TestCase
-  include Ferret::Index
-  def test_tvoi()
-    t1 = TermVectorOffsetInfo.new(1, 3)
-    assert_equal(t1.start, 1)
-    assert_equal(t1.end, 3)
-    t2 = TermVectorOffsetInfo.new(1, 3)
-    assert(t1 == t2)
-    t2.start = 2
-    assert(t1 != t2)
-    t2.start = 1
-    t2.end = 1
-    assert(t1 != t2)
-  end
-end

data/test/unit/search/rtc_similarity.rb

@@ -1,37 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class SimilarityTest < Test::Unit::TestCase
-  include Ferret::Search
-  include Ferret::Index
-
-  def test_byte_float_conversion()
-    256.times do |i|
-      assert_equal(i, Similarity.float_to_byte(Similarity.byte_to_float(i)))
-      assert_equal(Similarity.byte_to_float(i), Similarity::NORM_TABLE[i])
-      assert_equal(i, Similarity.encode_norm(Similarity.decode_norm(i)))
-    end
-  end
-
-  def test_default_similarity
-    dsim = DefaultSimilarity.new()
-    assert_equal(1.0/4, dsim.length_norm("field", 16))
-    assert_equal(1.0/4, dsim.query_norm(16))
-    assert_equal(3.0, dsim.tf(9))
-    assert_equal(1.0/10, dsim.sloppy_freq(9))
-    assert_equal(1.0, dsim.idf(9, 10))
-    assert_equal(4.0, dsim.coord(12, 3))
-    searcher = Object.new
-    def searcher.doc_freq(term) 9 end
-    def searcher.max_doc() 10 end
-    term = Term.new("field", "text")
-    assert_equal(1.0, dsim.idf_term(term, searcher))
-    terms = [
-      Term.new("field1", "text1"),
-      Term.new("field1", "text2"),
-      Term.new("field2", "text3"),
-      Term.new("field2", "text4")
-    ]
-    assert_equal(4.0, dsim.idf_phrase(terms, searcher))
-  end
-end

data/test/unit/search/rtc_sort_field.rb

@@ -1,14 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class SortFieldTest < Test::Unit::TestCase
-  include Ferret::Search
-
-  def test_params()
-    assert_equal("SCORE",   SortField::SortType::SCORE.to_s)
-    assert_equal("DOC",     SortField::SortType::DOC.to_s)
-    assert_equal("auto",    SortField::SortType::AUTO.to_s)
-    assert_equal("string",  SortField::SortType::STRING.to_s)
-    assert_equal("integer", SortField::SortType::INTEGER.to_s)
-    assert_equal("float",   SortField::SortType::FLOAT.to_s)
-  end
-end

data/test/unit/search/tc_multi_searcher2.rb

@@ -1,126 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-# Tests the multisearcher by comparing it's results
-# with those returned by an IndexSearcher.
-# Taken from TestMultiSearcherRanking.java of Lucene
-class MultiSearcher2Test < Test::Unit::TestCase
-  include Ferret::Document
-  include Ferret::Search
-  include Ferret::Store
-  include Ferret::Analysis
-  include Ferret::Index
-
-  FIELD_NAME = 'body'
-
-  def test_one_Term_query
-    check_query 'three'
-  end
-
-  def test_two_term_query
-    check_query 'three foo'
-    # as of 2006/03/11 these fail in Java Lucene as 
-    # well, hits are returned in slightly different order. 
-    #check_query '+pizza +blue*', :body
-    #check_query '+pizza blue*', :body
-    #check_query 'pizza blue*', :body
-  end
-
-  def test_prefix_query
-    check_query 'multi*'
-  end
-
-  def test_fuzzy_query
-    check_query 'multiThree~'
-  end
-
-  def test_range_query
-    check_query '{multiA multiP}'
-  end
-  
-  # fails (query parse error)
-  #def test_multi_phrase_query
-  #  check_query '"blueberry pi*"'
-  #end
-  
-  def test_nomatch_query
-    check_query '+three +nomatch'
-  end
-  
-  # this yields differing scores, but doesn't work in 
-  # Java Lucene either
-  #def test_term_repeated_query
-  #  check_query 'multi* multi* foo'
-  #end
-  
-  
-  def check_query(query_str, debug_field=nil)
-    @parser ||= Ferret::QueryParser.new(FIELD_NAME, :analyzer => @analyzer)
-    query = @parser.parse(query_str)
-    puts "Query: #{query}" if debug_field
-    IndexTestHelper.explain(query, @multi, debug_field) if debug_field
-    IndexTestHelper.explain(query, @single, debug_field) if debug_field
-    multi_hits = @multi.search(query)
-    single_hits = @single.search(query)
-    assert_equal single_hits.size, multi_hits.size, "hit count differs"
-    multi_hits.score_docs.each_with_index { |multi_sd, i|
-      single_sd = single_hits.score_docs[i]
-      doc_multi = @multi.doc(multi_sd.doc)
-      doc_single = @single.doc(single_sd.doc)
-      assert_equal single_sd.score, multi_sd.score, "score differs in result #{i}"
-      assert_equal doc_single[FIELD_NAME], doc_multi[FIELD_NAME], "field values differ in result #{i}"
-    }
-  end
-
-  def setup()
-    @analyzer = WhiteSpaceAnalyzer.new()
-    # create MultiSearcher from two seperate searchers
-    d1 = RAMDirectory.new()
-    iw1 = IndexWriter.new(d1, :analyzer => @analyzer, :create => true)
-    add_collection1(iw1)
-    iw1.close()
-    
-    d2 = RAMDirectory.new()
-    iw2 = IndexWriter.new(d2, :analyzer => @analyzer, :create => true)
-    add_collection2(iw2)
-    iw2.close()
-    @multi = MultiSearcher.new([IndexSearcher.new(d1), IndexSearcher.new(d2)])
-
-    # create IndexSearcher which contains all documents
-    d = RAMDirectory.new()
-    iw = IndexWriter.new(d, :analyzer => @analyzer, :create => true)
-    add_collection1(iw)
-    add_collection2(iw)
-    iw.close()
-    @single = IndexSearcher.new(d)
-  end
-
-  def tear_down()
-    @multi.close
-    @single.close
-  end
-
-  def add(value, iw)
-    d = Document.new
-    d << Field.new(FIELD_NAME, value, Field::Store::YES, Field::Index::TOKENIZED)
-    iw << d
-  end
-
-  def add_collection1(iw)
-    add("one blah three", iw)
-    add("one foo three multiOne", iw)
-    add("one foobar three multiThree", iw)
-    add("blueberry pie", iw)
-    add("blueberry strudel", iw)
-    add("blueberry pizza", iw)
-  end
-  def add_collection2(iw)
-    add("two blah three", iw)
-    add("two foo xxx multiTwo", iw)
-    add("two foobar xxx multiThreee", iw)
-    add("blueberry chewing gum", iw)
-    add("bluebird pizza", iw)
-    add("bluebird foobar pizza", iw)
-    add("piccadilly circus", iw)
-  end
-
-end

data/test/unit/store/rtc_fs_store.rb

@@ -1,62 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-require File.dirname(__FILE__) + "/rtm_store"
-require File.dirname(__FILE__) + "/rtm_store_lock"
-
-module Ferret::Store
-
-  class FSDirectory
-    def FSDirectory.directory_cache
-      @@Directories
-    end
-
-    def ref_count
-      @ref_count
-    end
-
-    def get_lock_prefix
-      lock_prefix
-    end
-  end
-end
-
-class FSStoreTest < Test::Unit::TestCase
-  include Ferret::Store
-  include StoreTest
-  include StoreLockTest
-  def setup
-    @dpath = File.join(File.dirname(__FILE__),
-                       '../../temp/fsdir')
-    @dir = FSDirectory.new(@dpath, true)
-  end
-
-  def teardown
-    @dir.refresh()
-    @dir.close()
-  end
-
-  def test_cache
-    dir_path = File.join(File.dirname(__FILE__),
-                         '/../../temp/cachetest')
-    assert(! FSDirectory.directory_cache[dir_path],
-           "this directory should not be cached yet") 
-    @dir1 = FSDirectory.new(dir_path, true)
-    assert(FSDirectory.directory_cache[dir_path],
-           "this directory should now be cached") 
-    assert_equal(@dir1.ref_count, 1,
-                 "There is one reference so the refcount should now be 1")
-    @dir2 = FSDirectory.new(dir_path, true)
-    assert(@dir1 === @dir2,
-           "The directory should be cached so the same directory object should have been returned")
-    assert_equal(@dir1.ref_count, 2,
-                 "There are two references so the refcount should now be 2")
-    @dir1.close
-    assert(FSDirectory.directory_cache[dir_path],
-           "this directory shouldn't have been removed yet") 
-    assert_equal(@dir2.ref_count, 1,
-                 "There is one reference so the refcount should now be 1")
-    @dir2.close
-    assert(! FSDirectory.directory_cache[dir_path],
-           "this directory should have been removed from the cache") 
-  end
-
-end

data/test/unit/store/rtc_ram_store.rb

@@ -1,15 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-require File.dirname(__FILE__) + "/rtm_store"
-require File.dirname(__FILE__) + "/rtm_store_lock"
-
-class RAMStoreTest < Test::Unit::TestCase
-  include StoreTest
-  include StoreLockTest
-  def setup
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def teardown
-    @dir.close()
-  end
-end

data/test/unit/store/rtm_store.rb

@@ -1,150 +0,0 @@
-module StoreTest
-  # declare dir so inheritors can access it.
-  @dir = nil
-
-  def test_modified
-    # difficult to test this one but as file mtime is only stored to the nearest second.
-    # we can assume this test will happen in less than a few seconds. (I hope)
-    time = Time.new.to_i
-    @dir.touch('mtime.test')
-    time_before = @dir.modified('mtime.test').to_i
-    assert(time_before - time <= 3,
-           "test that mtime is approximately equal to the system time when the file was touched")
-  end
-
-  def test_rw_bytes
-    bytes = [0x34, 0x87, 0xF9, 0xEA, 0x00, 0xFF]
-    rw_test(bytes, "byte", 6)
-  end
-
-  def test_rw_ints
-    ints = [-2147483648, 2147483647, -1, 0]
-    rw_test(ints, "int", 16)
-  end
-
-  def test_rw_longs
-    longs = [-9223372036854775808, 9223372036854775807, -1, 0]
-    rw_test(longs, "long", 32)
-  end
-
-  def test_rw_uints
-    uints = [0xffffffff, 100000, 0]
-    rw_test(uints, "uint", 12)
-  end
-
-  def test_rw_ulongs
-    ulongs = [0xffffffffffffffff, 100000000000000, 0]
-    rw_test(ulongs, "ulong", 24)
-  end
-
-  def test_rw_vints
-    vints = [ 9223372036854775807,
-              0x00,
-              0xFFFFFFFFFFFFFFFF]
-    rw_test(vints, "vint", 20)
-  end
-
-  def test_rw_vlongs
-    vlongs = [ 9223372036854775807,
-               0x00,
-               0xFFFFFFFFFFFFFFFF]
-    rw_test(vlongs, "vlong", 20)
-  end
-
-  def test_rw_strings
-    text = 'This is a ruby ferret test string ~!@#$%^&*()`123456790-=\)_+|'
-    ostream = @dir.create_output("rw_strings.test")
-    ostream.write_string(text)
-    ostream.write_string(text*100)
-    ostream.close
-    istream = @dir.open_input("rw_strings.test")
-    assert_equal(text, istream.read_string, "Short string test failed")
-    assert_equal(text*100, istream.read_string, "Short string test failed")
-    istream.close
-    assert_equal(6265, @dir.length('rw_strings.test'))
-  end
-
-  def test_rw_utf8_strings
-    text = '³³ ÄÄÄÄÄÄ 道德經'
-    ostream = @dir.create_output("rw_utf8_strings.test")
-    ostream.write_string(text)
-    ostream.write_string(text*100)
-    ostream.close
-    istream = @dir.open_input("rw_utf8_strings.test")
-    assert_equal(text, x = istream.read_string, "Short string test failed")
-    assert_equal(text*100, istream.read_string, "Short string test failed")
-    istream.close
-  end
-
-  # this test fills up the output stream so that the buffer will have to be
-  # written a few times. It then uses seek to make sure that it works
-  # correctly
-  def test_buffer_seek
-    ostream = @dir.create_output("rw_seek.test")
-    text = 'This is another long test string !@#$%#$%&%$*%^&*()(_'
-    1000.times {|i| ostream.write_long(i); ostream.write_string(text) }
-    ostream.seek(987)
-    assert_equal(987, ostream.pos)
-    ostream.write_vint(555)
-    ostream.seek(56)
-    assert_equal(56, ostream.pos)
-    ostream.write_vint(1234567890)
-    ostream.seek(4000)
-    assert_equal(4000, ostream.pos)
-    ostream.write_vint(9876543210)
-    ostream.close()
-    istream = @dir.open_input("rw_seek.test")
-    istream.seek(56)
-    assert_equal(56, istream.pos)
-    assert_equal(1234567890, istream.read_vint())
-    istream.seek(4000)
-    assert_equal(4000, istream.pos)
-    assert_equal(9876543210, istream.read_vint())
-    istream.seek(987)
-    assert_equal(987, istream.pos)
-    assert_equal(555, istream.read_vint())
-    istream.close()
-  end
-
-  def test_clone
-    ostream = @dir.create_output("clone_test")
-    10.times {|i| ostream.write_long(i) }
-    ostream.close
-    istream = @dir.open_input("clone_test")
-    istream.seek(24)
-    alt_istream = istream.clone
-    assert_equal(istream.pos, alt_istream.pos)
-    (3...10).each {|i| assert_equal(i, alt_istream.read_long) }
-    assert_equal(80, alt_istream.pos)
-    assert_equal(24, istream.pos)
-    alt_istream.close
-    (3...10).each {|i| assert_equal(i, istream.read_long) }
-    istream.close
-  end
-
-  def test_read_bytes
-    str = "0000000000"
-    ostream = @dir.create_output("rw_read_bytes")
-    ostream.write_bytes("how are you doing?", 18)
-    ostream.close
-    istream = @dir.open_input("rw_read_bytes")
-    istream.read_bytes(str, 2, 4)
-    assert_equal("00how 0000", str)
-    istream.read_bytes(str, 1, 8)
-    assert_equal("0are you 0", str)
-    istream.close
-  end
-
-  private
-
-  def rw_test(values, type, expected_length)
-    ostream = @dir.create_output("rw_#{type}.test")
-    values.each { |b| ostream.__send__("write_" + type, b) }
-    ostream.close
-    istream = @dir.open_input("rw_#{type}.test")
-    values.each { |b| assert_equal(b, istream.__send__("read_" + type), "#{type} should be equal") }
-    istream.close
-    assert_equal(expected_length, @dir.length("rw_#{type}.test"))
-  end
-
-end