ferret 0.9.6 → 0.10.0

data/test/unit/index/rtc_segment_infos.rb

@@ -1,74 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class SegmentInfosTest < Test::Unit::TestCase
-  include Ferret::Index
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def tear_down()
-    @dir.close()
-  end
-
-  def test_read_write
-    assert_equal(0, SegmentInfos.read_current_version(@dir))
-    sis = SegmentInfos.new()
-    seg0 = SegmentInfo.new('seg0', 5, @dir)
-    seg1 = SegmentInfo.new('seg1', 5, @dir)
-    seg2 = SegmentInfo.new('seg2', 5, @dir)
-    seg3 = SegmentInfo.new('seg3', 5, @dir)
-    sis << seg0
-    sis << seg1
-    sis << seg2
-    assert_equal(sis.size(), 3)
-    assert_equal(sis[0], seg0)
-    assert_equal(sis[2], seg2)
-    sis.write(@dir)
-    version = SegmentInfos.read_current_version(@dir)
-    assert(@dir.exists?('segments'))
-    sis2 = SegmentInfos.new()
-    sis2.read(@dir)
-    assert_equal(sis2.size(), 3)
-    assert_equal(sis2[0], seg0)
-    assert_equal(sis2[2], seg2)
-    sis2 << seg3
-    sis2.write(@dir)
-    assert_equal(version + 1, SegmentInfos.read_current_version(@dir))
-    sis3 = SegmentInfos.new()
-    sis3.read(@dir)
-    assert_equal(sis3.size(), 4)
-    assert_equal(sis2[0], seg0)
-    assert_equal(sis2[3], seg3)
-  end
-end
-
-class SegmentInfoTest < Test::Unit::TestCase
-  include Ferret::Index
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def tear_down()
-    @dir.close()
-  end
-
-  # just test getters and setters. Nothing else.
-  def test_segment_info
-    si = SegmentInfo.new("seg1", 0, @dir)
-    assert_equal(si.directory, @dir)
-    assert_equal(si.doc_count, 0)
-    assert_equal(si.name, "seg1")
-    @dir.close()
-    @dpath = File.dirname(__FILE__) + '/../../temp/fsdir'
-    @dir = Ferret::Store::FSDirectory.new(@dpath, true)
-    si.name = "seg2"
-    si.doc_count += 2
-    si.directory = @dir
-    assert_equal(si.directory, @dir)
-    assert_equal(si.doc_count, 2)
-    assert_equal(si.name, "seg2")
-  end
-end

data/test/unit/index/rtc_segment_term_docs.rb

@@ -1,17 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class SegmentTermDocEnumTest < Test::Unit::TestCase
-
-  include Ferret::Index
-  include Ferret::Analysis
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-    @doc = IndexTestHelper.prepare_document()
-    IndexTestHelper.write_document(@dir, @doc)
-  end
-
-  def test_something()
-    assert true
-  end
-end

data/test/unit/index/rtc_segment_term_enum.rb

@@ -1,60 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class SegmentTermEnumTest < Test::Unit::TestCase
-  include Ferret::Index
-
-  TEST_SEGMENT = "_test"
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def test_initialize()
-    fis = FieldInfos.new
-    fis.add("author", true, true)
-    fis.add("title", true, true)
-    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
-    terms = [ Term.new("author", "Martel"),
-              Term.new("title", "Life of Pi"),
-              Term.new("author", "Martin"),
-              Term.new("title", "Life on the edge") ].sort
-    term_infos = []
-    4.times {|i| term_infos << TermInfo.new(i,i,i,0)}
-    4.times {|i| tiw.add(terms[i], term_infos[i]) }
-    tiw.close()
-
-    tis_file = @dir.open_input(TEST_SEGMENT + ".tis")
-    
-    ste = SegmentTermEnum.new(tis_file, fis, false)
-    assert_equal(128, ste.index_interval)
-    assert_equal(16, ste.skip_interval)
-    assert_equal(4, ste.size)
-    assert(ste.next?)
-    assert_equal(terms[0], ste.term)
-    assert_equal(term_infos[0], ste.term_info)
-    ti = TermInfo.new
-    ste.term_info = ti
-    assert_equal(term_infos[0], ti)
-    assert(ste.next?)
-    assert_equal(terms[0], ste.prev)
-    assert_equal(terms[1], ste.term)
-    assert_equal(term_infos[1], ste.term_info)
-    assert(ste.next?)
-    assert_equal(terms[2], ste.term)
-    assert_equal(term_infos[2], ste.term_info)
-    assert(ste.next?)
-    assert_equal(terms[3], ste.term)
-    assert_equal(term_infos[3], ste.term_info)
-    ste.close()
-
-    tii_file = @dir.open_input(TEST_SEGMENT + ".tii")
-
-    ste = SegmentTermEnum.new(tii_file, fis, false)
-    assert_equal(128, ste.index_interval)
-    assert_equal(16, ste.skip_interval)
-    assert_equal(1, ste.size)
-    assert(ste.next?)
-    assert(Term.new("", ""), ste.term)
-  end
-end

data/test/unit/index/rtc_segment_term_vector.rb

@@ -1,71 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class SegmentTermVectorTest < Test::Unit::TestCase
-
-  include Ferret::Index
-
-  def setup()
-    @terms = ["Apples", "Oranges", "Bananas", "Kiwis", "Mandarins"]
-    term_freqs = [4,2,1,12,4]
-    @stv = SegmentTermVector.new("Fruits", @terms, term_freqs)
-  end
-
-  def test_size()
-    assert_equal(@terms.size(), @stv.size())
-  end
-
-  def test_index_of()
-    assert_equal(0, @stv.index_of("Apples"))
-    assert_equal(4, @stv.freqs[@stv.index_of("Apples")])
-  end
-
-  def test_indexes_of()
-    assert_equal([2, 0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 0, 3))
-    assert_equal([0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 1, 2))
-  end
-end
-
-class SegmentTermVectorWithPosOffsetsTest < Test::Unit::TestCase
-
-  include Ferret::Index
-
-  def setup()
-    @terms = ["Apples", "Oranges", "Bananas", "Kiwis", "Mandarins"]
-    term_freqs = [4,2,1,12,4]
-    term_positions = [
-      [1,3,5,7],
-      [2,4],
-      [6],
-      [8,9,10,12,13,14,16,17,18,20,21,22],
-      [11,15,19,23]
-    ]
-    term_offsets = [
-      [[1,4],[10,14],[20,24],[30,34]],
-      [[5,9],[15,19]],
-      [[25,29]],
-      [[35,39],[40,44],[45,49],[55,59],[60,64],[65,69],[75,79],[80,84],[85,89],[95,99],[100,104],[105,109]],
-      [[50,54],[70,74],[90,94],[110,114]]
-    ]
-    @stv = SegmentTermVector.new("Fruits", @terms, term_freqs, term_positions, term_offsets)
-  end
-
-  def test_size()
-    assert_equal(@terms.size(), @stv.size())
-  end
-
-  def test_index_of()
-    assert_equal(0, @stv.index_of("Apples"))
-    assert_equal(4, @stv.freqs[@stv.index_of("Apples")])
-  end
-
-  def test_indexes_of()
-    assert_equal([2, 0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 0, 3))
-    assert_equal([0, 3], @stv.indexes_of(["Bananas", "Apples", "Kiwis"], 1, 2))
-  end
-
-  def test_positions_offsets()
-    assert_equal([1,3,5,7], @stv.positions[@stv.index_of("Apples")])
-    assert_equal([[35,39],[40,44],[45,49],[55,59],[60,64],[65,69],[75,79],[80,84],[85,89],[95,99],[100,104],[105,109]], @stv.offsets[@stv.index_of("Kiwis")])
-  end
-end

data/test/unit/index/rtc_term_buffer.rb

@@ -1,57 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class TermBufferTest < Test::Unit::TestCase
-  include Ferret::Index
-  def test_term_set()
-    t = Term.new("title", "Ferret Tutorial")
-    tb = TermBuffer.new
-    tb.term = t
-    assert_equal(t.field, tb.field)
-    assert_equal("Ferret Tutorial", tb.text)
-    assert_equal("Ferret Tutorial".length, tb.text_length)
-    assert_equal(t, tb.term)
-  end
-
-  def test_set()
-    tb = TermBuffer.new
-    tb.term = Term.new("title", "Ferret Tutorial")
-    tb2 = TermBuffer.new
-    tb2.set!(tb)
-    assert_equal(tb.field, tb2.field)
-    assert_equal("Ferret Tutorial", tb2.text)
-    assert_equal("Ferret Tutorial".length, tb2.text_length)
-    assert_equal(tb.term, tb2.term)
-  end
-
-  def test_compare()
-    tb1 = TermBuffer.new
-    tb2 = TermBuffer.new
-    tb1.term = Term.new("alpha", "text")
-    tb2.term = Term.new("bravo", "text")
-    assert(tb1 < tb2)
-    tb2.term = Term.new("alpha", "text")
-    assert(tb1 == tb2)
-    tb2.term = Term.new("alpha", "tex")
-    assert(tb1 > tb2)
-  end
-
-  def test_read()
-    dir = Ferret::Store::RAMDirectory.new
-    fi = FieldInfos.new
-    tb = TermBuffer.new
-    tb.term = Term.new("Author", "Dave")
-    fi.add("Writer", true)
-    output = dir.create_output("term_buffer_read_test")
-    output.write_vint(4)
-    output.write_vint(8)
-    output.write_chars(" Balmain", 0, 8)
-    output.write_vint(fi.field_number("Writer"))
-    output.close
-    input = dir.open_input("term_buffer_read_test")
-    tb.read(input, fi)
-    assert_equal("Dave Balmain", tb.text)
-    assert_equal("Dave Balmain", tb.term.text)
-    assert_equal("Writer", tb.field)
-  end
-end

data/test/unit/index/rtc_term_info.rb

@@ -1,19 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class TermInfoTest < Test::Unit::TestCase
-  include Ferret::Index
-  def test_term()
-    ti1 = TermInfo.new(1, 2, 3, 1)
-    assert_equal(ti1.doc_freq, 1)
-    assert_equal(ti1.freq_pointer, 2)
-    assert_equal(ti1.prox_pointer, 3)
-    assert_equal(ti1.skip_offset, 1)
-    ti2 = ti1.clone()
-    assert(ti1 == ti2)
-    ti2 = TermInfo.new(10, 9, 8)
-    assert(ti1 != ti2)
-    ti2.set!(ti1)
-    assert(ti1 == ti2)
-  end
-end

data/test/unit/index/rtc_term_infos_io.rb

@@ -1,192 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class TermInfosIOTest < Test::Unit::TestCase
-  include Ferret::Index
-
-  DICT = [ "duad", "dual", "dualism", "dualist", "duality", "dualize", "duan",
-      "duarchy", "dub", "dubber", "dubbin", "dubbing", "dubiety", "dubiosity",
-      "dubious", "dubiously", "dubiousness", "dubitate", "dubitation", "dubnium",
-      "dubonnet", "ducal", "ducat", "ducatoon", "duce", "duchess", "duchesse",
-      "duchy", "duck", "duckbill", "duckboard", "ducker", "duckie", "ducking",
-      "duckling", "duckpin", "duckshove", "duckshover", "ducktail", "duckwalk",
-      "duckweed", "ducky", "duct", "ductile", "ductileness", "ductility",
-      "ducting", "ductless", "ductule", "ductulus", "ductwork", "dud", "dudder",
-      "duddery", "duddie", "duddy", "dude", "dudeen", "dudgeon", "due",
-      "duecento", "duel", "dueler", "dueling", "duelist", "dueller", "duelling",
-      "duellist", "duello", "duende", "dueness", "duenna", "duennaship", "duet",
-      "duette", "duettino", "duettist", "duetto", "duff", "duffel", "duffer",
-      "duffle", "dufus", "dug", "dugong", "dugout", "duiker", "duit", "duke",
-      "dukedom", "dukeling", "dukery", "dukeship", "dulcamara", "dulcet",
-      "dulcian", "dulciana", "dulcification", "dulcify", "dulcimer", "dulcimore",
-      "dulcinea", "dulcitone", "dulcorate", "dule", "dulfer", "dulia", "dull",
-      "dullard", "dullness", "dullsville", "dully", "dulness", "dulocracy",
-      "dulosis", "dulse", "duly", "duma", "dumaist", "dumb", "dumbass",
-      "dumbbell", "dumbcane", "dumbfound", "dumbfounder", "dumbhead",
-      "dumbledore", "dumbly", "dumbness", "dumbo", "dumbstruck", "dumbwaiter",
-      "dumdum", "dumfound", "dummerer", "dummkopf", "dummy", "dumortierite",
-      "dump", "dumpbin", "dumpcart", "dumper", "dumpiness", "dumping",
-      "dumpling", "dumplings", "dumpsite", "dumpster", "dumpy", "dun", "dunam",
-      "dunce", "dunch", "dunder", "dunderhead", "dunderheadedness", "dunderpate",
-      "dune", "duneland", "dunfish", "dung", "dungaree", "dungeon", "dungeoner",
-      "dungheap", "dunghill", "dungy", "dunite", "duniwassal", "dunk", "dunker",
-      "dunlin", "dunnage", "dunnakin", "dunness", "dunnite", "dunnock", "dunny",
-      "dunt", "duo", "duodecillion", "duodecimal", "duodecimo", "duodenectomy",
-      "duodenum", "duolog", "duologue", "duomo", "duopoly", "duopsony",
-      "duotone", "dup", "dupability", "dupatta", "dupe", "duper", "dupery",
-      "dupion", "duple", "duplet", "duplex", "duplexer", "duplexity",
-      "duplicability", "duplicand", "duplicate", "duplication", "duplicator",
-      "duplicature", "duplicitousness", "duplicity", "dupondius", "duppy",
-      "dura", "durability", "durable", "durableness", "durably", "dural",
-      "duralumin", "duramen", "durance", "duration", "durative", "durbar",
-      "dure", "dures", "duress", "durgan", "durian", "durion", "durmast",
-      "durn", "durned", "duro", "duroc", "durometer", "durr", "durra", "durrie",
-      "durukuli", "durum", "durzi", "dusk", "duskiness", "dusky", "dust",
-      "dustbin", "dustcart", "dustcloth", "dustcover", "duster", "dustheap",
-      "dustiness", "dusting", "dustless", "dustman", "dustmop", "dustoff",
-      "dustpan", "dustpanful", "dustrag", "dustsheet", "dustup", "dusty",
-      "dutch", "dutchman", "duteous", "duteously", "duteousness", "dutiability",
-      "dutiable", "dutifulness", "duty", "duumvir", "duumvirate", "duvet",
-      "duvetine", "duvetyn", "duvetyne", "dux", "duyker"]
-
-  TEST_SEGMENT = "_test"
-
-  def setup()
-    @dir = Ferret::Store::RAMDirectory.new
-  end
-
-  def tear_down()
-    @dir.close()
-  end
-
-  def test_two_field_io
-    term_dumbly = Term.new("word", "dumbly")
-    term_dualize = Term.new("word", "dualize")
-    term_rev_dualize = Term.new("reverse", "ezilaud")
-
-    fis = FieldInfos.new
-    fis.add("word", true, true)
-    fis.add("reverse", true, true)
-    terms = []
-    term_infos = []
-    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT+"G", fis, 128)
-
-    reverse_words = []
-    DICT.each { |word| reverse_words << word.reverse }
-    reverse_words.sort!
-    reverse_words.each_with_index do |word, i|
-      tiw.add(Term.new("reverse", word), TermInfo.new(1, i, i, 0))
-    end
-    DICT.each_with_index do |word, i|
-      tiw.add(Term.new("word", word), TermInfo.new(1, 1000 + i, 1000 + i, 0))
-    end
-
-    tiw.close()
-    tir = TermInfosReader.new(@dir, TEST_SEGMENT+"G", fis)
-    assert_equal(564, tir.size)
-    assert_equal(16, tir.skip_interval)
-    assert_equal(561, tir.get_terms_position(Term.new("word", "duvetyne")))
-    assert_equal(TermInfo.new(1, 1005, 1005, 0), tir.get_term_info(term_dualize))
-    assert_equal(TermInfo.new(1, 70, 70, 0), tir.get_term_info(term_rev_dualize))
-  end
-
-  def test_io
-    term_dumbly = Term.new("word", "dumbly")
-    term_dualize = Term.new("word", "dualize")
-
-    fis = FieldInfos.new
-    fis.add("word", true, true)
-    terms = []
-    term_infos = []
-    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
-    DICT.each_with_index do |word, i|
-      terms << Term.new("word", word)
-      term_infos << TermInfo.new(1, i, i, 0)
-      tiw.add(terms[i], term_infos[i])
-    end
-    tiw.close()
-    tir = TermInfosReader.new(@dir, TEST_SEGMENT, fis)
-    assert_equal(282, tir.size)
-    assert_equal(16, tir.skip_interval)
-    assert_equal(281, tir.get_terms_position(Term.new("word", "duyker")))
-    assert_equal(279, tir.get_terms_position(Term.new("word", "duvetyne")))
-    assert_equal(254, tir.get_terms_position(Term.new("word", "dusting")))
-    assert_equal(255, tir.get_terms_position(Term.new("word", "dustless")))
-    assert_equal(256, tir.get_terms_position(Term.new("word", "dustman")))
-    assert_equal(257, tir.get_terms_position(Term.new("word", "dustmop")))
-    assert_equal(TermInfo.new(1, 5, 5, 0), tir.get_term_info(term_dualize))
-    assert_equal(term_dumbly, tir.get_term(127))
-    terms = tir.terms_from(term_dumbly)
-    assert_equal(term_dumbly, terms.term)
-    assert(terms.next?)
-    assert_equal(Term.new("word", "dumbness"), terms.term)
-    assert(terms.next?)
-    assert_equal(Term.new("word", "dumbo"), terms.term)
-  end
-
-  def test_small_writer
-    fis = FieldInfos.new
-    fis.add("author", true, true)
-    fis.add("title", true, true)
-    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
-    terms = [ Term.new("author", "Martel"),
-              Term.new("title", "Life of Pi"),
-              Term.new("author", "Martin"),
-              Term.new("title", "Life on the edge") ].sort
-    term_infos = []
-    4.times {|i| term_infos << TermInfo.new(i,i,i,i)}
-    4.times {|i| tiw.add(terms[i], term_infos[i]) }
-    tiw.close()
-
-    tis_file = @dir.open_input(TEST_SEGMENT + ".tis")
-    tii_file = @dir.open_input(TEST_SEGMENT + ".tii")
-    assert_equal(TermInfosWriter::FORMAT, tis_file.read_int())
-    assert_equal(4, tis_file.read_long())  # term count
-    assert_equal(128, tis_file.read_int()) # @index_interval
-    assert_equal(16, tis_file.read_int())  # @skip_interval
-    assert_equal(0, tis_file.read_vint())  # string_equal length
-    assert_equal(6, tis_file.read_vint())  # rest of string length
-    tis_file.read_chars(author = "", 0, 6) # the difference string
-    assert_equal("Martel", author.to_s)
-    assert_equal(0, tis_file.read_vint())  # field number
-    assert_equal(0, tis_file.read_vint())  # doc_freq
-    assert_equal(0, tis_file.read_vlong()) # freq pointer difference
-    assert_equal(0, tis_file.read_vlong()) # prox pointer difference
-    assert_equal(4, tis_file.read_vint())  # string_equal length
-    assert_equal(2, tis_file.read_vint())  # rest of string length
-    tis_file.read_chars(author = "", 0, 2) # the difference string
-    assert_equal("in", author.to_s)                                  
-    assert_equal(0, tis_file.read_vint())  # field number
-    assert_equal(1, tis_file.read_vint())  # doc_freq
-    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
-    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
-    assert_equal(0, tis_file.read_vint())  # string_equal length
-    assert_equal(10, tis_file.read_vint()) # rest of string length
-    tis_file.read_chars(title = "", 0, 10) # the difference string
-    assert_equal("Life of Pi", title.to_s)                           
-    assert_equal(1, tis_file.read_vint())  # field number
-    assert_equal(2, tis_file.read_vint())  # doc_freq
-    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
-    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
-    assert_equal(6, tis_file.read_vint())  # string_equal length
-    assert_equal(10, tis_file.read_vint()) # rest of string length
-    tis_file.read_chars(title = "", 0, 10) # the difference string
-    assert_equal("n the edge", title.to_s)                           
-    assert_equal(1, tis_file.read_vint())  # field number
-    assert_equal(3, tis_file.read_vint())  # doc_freq
-    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
-    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
-
-    assert_equal(TermInfosWriter::FORMAT, tii_file.read_int())
-    assert_equal(1, tii_file.read_long())
-    assert_equal(128, tii_file.read_int())
-    assert_equal(16, tii_file.read_int())
-    assert_equal(0, tii_file.read_vint())  # string_equal length
-    assert_equal(0, tii_file.read_vint())  # rest of string length
-    assert_equal(0xFFFFFFFF, tii_file.read_vint())  # field number
-    assert_equal(0, tii_file.read_vint())  # doc_freq
-    assert_equal(0, tii_file.read_vlong()) # freq pointer difference
-    assert_equal(0, tii_file.read_vlong()) # prox pointer difference
-    assert_equal(20, tii_file.read_vlong()) # pointer to first element in other
-  end
-end