ferret 0.1.0

data/test/unit/index/tc_term_info.rb

@@ -0,0 +1,19 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermInfoTest < Test::Unit::TestCase
+  include Ferret::Index
+  def test_term()
+    ti1 = TermInfo.new(1, 2, 3, 1)
+    assert_equal(ti1.doc_freq, 1)
+    assert_equal(ti1.freq_pointer, 2)
+    assert_equal(ti1.prox_pointer, 3)
+    assert_equal(ti1.skip_offset, 1)
+    ti2 = ti1.copy_of()
+    assert(ti1 == ti2)
+    ti2 = TermInfo.new(10, 9, 8)
+    assert(ti1 != ti2)
+    ti2.set!(ti1)
+    assert(ti1 == ti2)
+  end
+end

data/test/unit/index/tc_term_infos_io.rb

@@ -0,0 +1,192 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermInfosIOTest < Test::Unit::TestCase
+  include Ferret::Index
+
+  DICT = [ "duad", "dual", "dualism", "dualist", "duality", "dualize", "duan",
+      "duarchy", "dub", "dubber", "dubbin", "dubbing", "dubiety", "dubiosity",
+      "dubious", "dubiously", "dubiousness", "dubitate", "dubitation", "dubnium",
+      "dubonnet", "ducal", "ducat", "ducatoon", "duce", "duchess", "duchesse",
+      "duchy", "duck", "duckbill", "duckboard", "ducker", "duckie", "ducking",
+      "duckling", "duckpin", "duckshove", "duckshover", "ducktail", "duckwalk",
+      "duckweed", "ducky", "duct", "ductile", "ductileness", "ductility",
+      "ducting", "ductless", "ductule", "ductulus", "ductwork", "dud", "dudder",
+      "duddery", "duddie", "duddy", "dude", "dudeen", "dudgeon", "due",
+      "duecento", "duel", "dueler", "dueling", "duelist", "dueller", "duelling",
+      "duellist", "duello", "duende", "dueness", "duenna", "duennaship", "duet",
+      "duette", "duettino", "duettist", "duetto", "duff", "duffel", "duffer",
+      "duffle", "dufus", "dug", "dugong", "dugout", "duiker", "duit", "duke",
+      "dukedom", "dukeling", "dukery", "dukeship", "dulcamara", "dulcet",
+      "dulcian", "dulciana", "dulcification", "dulcify", "dulcimer", "dulcimore",
+      "dulcinea", "dulcitone", "dulcorate", "dule", "dulfer", "dulia", "dull",
+      "dullard", "dullness", "dullsville", "dully", "dulness", "dulocracy",
+      "dulosis", "dulse", "duly", "duma", "dumaist", "dumb", "dumbass",
+      "dumbbell", "dumbcane", "dumbfound", "dumbfounder", "dumbhead",
+      "dumbledore", "dumbly", "dumbness", "dumbo", "dumbstruck", "dumbwaiter",
+      "dumdum", "dumfound", "dummerer", "dummkopf", "dummy", "dumortierite",
+      "dump", "dumpbin", "dumpcart", "dumper", "dumpiness", "dumping",
+      "dumpling", "dumplings", "dumpsite", "dumpster", "dumpy", "dun", "dunam",
+      "dunce", "dunch", "dunder", "dunderhead", "dunderheadedness", "dunderpate",
+      "dune", "duneland", "dunfish", "dung", "dungaree", "dungeon", "dungeoner",
+      "dungheap", "dunghill", "dungy", "dunite", "duniwassal", "dunk", "dunker",
+      "dunlin", "dunnage", "dunnakin", "dunness", "dunnite", "dunnock", "dunny",
+      "dunt", "duo", "duodecillion", "duodecimal", "duodecimo", "duodenectomy",
+      "duodenum", "duolog", "duologue", "duomo", "duopoly", "duopsony",
+      "duotone", "dup", "dupability", "dupatta", "dupe", "duper", "dupery",
+      "dupion", "duple", "duplet", "duplex", "duplexer", "duplexity",
+      "duplicability", "duplicand", "duplicate", "duplication", "duplicator",
+      "duplicature", "duplicitousness", "duplicity", "dupondius", "duppy",
+      "dura", "durability", "durable", "durableness", "durably", "dural",
+      "duralumin", "duramen", "durance", "duration", "durative", "durbar",
+      "dure", "dures", "duress", "durgan", "durian", "durion", "durmast",
+      "durn", "durned", "duro", "duroc", "durometer", "durr", "durra", "durrie",
+      "durukuli", "durum", "durzi", "dusk", "duskiness", "dusky", "dust",
+      "dustbin", "dustcart", "dustcloth", "dustcover", "duster", "dustheap",
+      "dustiness", "dusting", "dustless", "dustman", "dustmop", "dustoff",
+      "dustpan", "dustpanful", "dustrag", "dustsheet", "dustup", "dusty",
+      "dutch", "dutchman", "duteous", "duteously", "duteousness", "dutiability",
+      "dutiable", "dutifulness", "duty", "duumvir", "duumvirate", "duvet",
+      "duvetine", "duvetyn", "duvetyne", "dux", "duyker"]
+
+  TEST_SEGMENT = "_test"
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_two_field_io
+    term_dumbly = Term.new("word", "dumbly")
+    term_dualize = Term.new("word", "dualize")
+    term_rev_dualize = Term.new("reverse", "ezilaud")
+
+    fis = FieldInfos.new
+    fis.add("word", true, true)
+    fis.add("reverse", true, true)
+    terms = []
+    term_infos = []
+    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT+"G", fis, 128)
+
+    reverse_words = []
+    DICT.each { |word| reverse_words << word.reverse }
+    reverse_words.sort!
+    reverse_words.each_with_index do |word, i|
+      tiw.add(Term.new("reverse", word), TermInfo.new(1, i, i, 0))
+    end
+    DICT.each_with_index do |word, i|
+      tiw.add(Term.new("word", word), TermInfo.new(1, 1000 + i, 1000 + i, 0))
+    end
+
+    tiw.close()
+    tir = TermInfosReader.new(@dir, TEST_SEGMENT+"G", fis)
+    assert_equal(564, tir.size)
+    assert_equal(16, tir.skip_interval)
+    assert_equal(561, tir.get_terms_position(Term.new("word", "duvetyne")))
+    assert_equal(TermInfo.new(1, 1005, 1005, 0), tir.get_term_info(term_dualize))
+    assert_equal(TermInfo.new(1, 70, 70, 0), tir.get_term_info(term_rev_dualize))
+  end
+
+  def test_io
+    term_dumbly = Term.new("word", "dumbly")
+    term_dualize = Term.new("word", "dualize")
+
+    fis = FieldInfos.new
+    fis.add("word", true, true)
+    terms = []
+    term_infos = []
+    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
+    DICT.each_with_index do |word, i|
+      terms << Term.new("word", word)
+      term_infos << TermInfo.new(1, i, i, 0)
+      tiw.add(terms[i], term_infos[i])
+    end
+    tiw.close()
+    tir = TermInfosReader.new(@dir, TEST_SEGMENT, fis)
+    assert_equal(282, tir.size)
+    assert_equal(16, tir.skip_interval)
+    assert_equal(281, tir.get_terms_position(Term.new("word", "duyker")))
+    assert_equal(279, tir.get_terms_position(Term.new("word", "duvetyne")))
+    assert_equal(254, tir.get_terms_position(Term.new("word", "dusting")))
+    assert_equal(255, tir.get_terms_position(Term.new("word", "dustless")))
+    assert_equal(256, tir.get_terms_position(Term.new("word", "dustman")))
+    assert_equal(257, tir.get_terms_position(Term.new("word", "dustmop")))
+    assert_equal(TermInfo.new(1, 5, 5, 0), tir.get_term_info(term_dualize))
+    assert_equal(term_dumbly, tir.get_term(127))
+    terms = tir.terms_from(term_dumbly)
+    assert_equal(term_dumbly, terms.term)
+    assert(terms.next?)
+    assert_equal(Term.new("word", "dumbness"), terms.term)
+    assert(terms.next?)
+    assert_equal(Term.new("word", "dumbo"), terms.term)
+  end
+
+  def test_small_writer
+    fis = FieldInfos.new
+    fis.add("author", true, true)
+    fis.add("title", true, true)
+    tiw = TermInfosWriter.new(@dir, TEST_SEGMENT, fis, 128)
+    terms = [ Term.new("author", "Martel"),
+              Term.new("title", "Life of Pi"),
+              Term.new("author", "Martin"),
+              Term.new("title", "Life on the edge") ].sort
+    term_infos = []
+    4.times {|i| term_infos << TermInfo.new(i,i,i,i)}
+    4.times {|i| tiw.add(terms[i], term_infos[i]) }
+    tiw.close()
+
+    tis_file = @dir.open_input(TEST_SEGMENT + ".tis")
+    tii_file = @dir.open_input(TEST_SEGMENT + ".tii")
+    assert_equal(TermInfosWriter::FORMAT, tis_file.read_int())
+    assert_equal(4, tis_file.read_long())  # term count
+    assert_equal(128, tis_file.read_int()) # @index_interval
+    assert_equal(16, tis_file.read_int())  # @skip_interval
+    assert_equal(0, tis_file.read_vint())  # string_equal length
+    assert_equal(6, tis_file.read_vint())  # rest of string length
+    tis_file.read_chars(author = "", 0, 6) # the difference string
+    assert_equal("Martel", author.to_s)
+    assert_equal(0, tis_file.read_vint())  # field number
+    assert_equal(0, tis_file.read_vint())  # doc_freq
+    assert_equal(0, tis_file.read_vlong()) # freq pointer difference
+    assert_equal(0, tis_file.read_vlong()) # prox pointer difference
+    assert_equal(4, tis_file.read_vint())  # string_equal length
+    assert_equal(2, tis_file.read_vint())  # rest of string length
+    tis_file.read_chars(author = "", 0, 2) # the difference string
+    assert_equal("in", author.to_s)                                  
+    assert_equal(0, tis_file.read_vint())  # field number
+    assert_equal(1, tis_file.read_vint())  # doc_freq
+    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
+    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
+    assert_equal(0, tis_file.read_vint())  # string_equal length
+    assert_equal(10, tis_file.read_vint()) # rest of string length
+    tis_file.read_chars(title = "", 0, 10) # the difference string
+    assert_equal("Life of Pi", title.to_s)                           
+    assert_equal(1, tis_file.read_vint())  # field number
+    assert_equal(2, tis_file.read_vint())  # doc_freq
+    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
+    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
+    assert_equal(6, tis_file.read_vint())  # string_equal length
+    assert_equal(10, tis_file.read_vint()) # rest of string length
+    tis_file.read_chars(title = "", 0, 10) # the difference string
+    assert_equal("n the edge", title.to_s)                           
+    assert_equal(1, tis_file.read_vint())  # field number
+    assert_equal(3, tis_file.read_vint())  # doc_freq
+    assert_equal(1, tis_file.read_vlong()) # freq pointer difference
+    assert_equal(1, tis_file.read_vlong()) # prox pointer difference
+
+    assert_equal(TermInfosWriter::FORMAT, tii_file.read_int())
+    assert_equal(1, tii_file.read_long())
+    assert_equal(128, tii_file.read_int())
+    assert_equal(16, tii_file.read_int())
+    assert_equal(0, tii_file.read_vint())  # string_equal length
+    assert_equal(0, tii_file.read_vint())  # rest of string length
+    assert_equal(0xFFFFFFFF, tii_file.read_vint())  # field number
+    assert_equal(0, tii_file.read_vint())  # doc_freq
+    assert_equal(0, tii_file.read_vlong()) # freq pointer difference
+    assert_equal(0, tii_file.read_vlong()) # prox pointer difference
+    assert_equal(20, tii_file.read_vlong()) # pointer to first element in other
+  end
+end

data/test/unit/index/tc_term_vector_offset_info.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermVectorOffsetInfoTest < Test::Unit::TestCase
+  include Ferret::Index
+  def test_tvoi()
+    t1 = TermVectorOffsetInfo.new(1, 3)
+    assert_equal(t1.start_offset, 1)
+    assert_equal(t1.end_offset, 3)
+    t2 = TermVectorOffsetInfo.new(1, 3)
+    assert(t1 == t2)
+    t2.start_offset = 2
+    assert(t1 != t2)
+    t2.start_offset = 1
+    t2.end_offset = 1
+    assert(t1 != t2)
+  end
+end

data/test/unit/index/tc_term_vectors_io.rb

@@ -0,0 +1,108 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class TermVectorsIOTest < Test::Unit::TestCase
+
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new
+    @fis = FieldInfos.new
+    @fis.add("field1", true, true, true, true)
+    @fis.add("field2", true, true)
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def test_tv_io_add_fields()
+    tv_w = TermVectorsWriter.new(@dir, "_test", @fis)
+    tv_w.open_document
+    assert(tv_w.document_open?)
+    tv_w.open_field("field1")
+    tv_w.add_term("text1", 1, [1], [t(0,4)])
+    tv_w.add_term("text2", 2, [3,4], [t(5,10), t(11,16)])
+    tv_w.close_field()
+    tv_w.close_document()
+    tv_w.close()
+
+    tv_r = TermVectorsReader.new(@dir, "_test", @fis)
+    assert_equal(1, tv_r.size)
+    tv = tv_r.get_field_tv(0, "field1")
+
+    assert_equal(2, tv.size)
+    assert_equal("text1", tv.terms[0])
+    assert_equal(1, tv.term_frequencies[0])
+    assert_equal(1, tv.positions[0][0])
+    assert_equal(t(0,4), tv.offsets[0][0])
+
+    assert_equal("text2", tv.terms[1])
+    assert_equal(2, tv.term_frequencies[1])
+    assert_equal(3, tv.positions[1][0])
+    assert_equal(t(5,10), tv.offsets[1][0])
+    assert_equal(4, tv.positions[1][1])
+    assert_equal(t(11,16), tv.offsets[1][1])
+    tv_r.close
+  end
+
+  def test_tv_io_add_documents()
+    tvs1 = [] 
+    tvs2 = [] 
+    tv = SegmentTermVector.new("field1",
+           ["word1", "word2"],
+           [3, 2],
+           [[1, 5, 8], [2, 9]],
+           [[t(0,5), t(34,39), t(45,50)],[t(6,11), t(51,56)]])
+    tvs1 << tv
+    tv = SegmentTermVector.new("field2",
+           ["word3", "word4"],
+           [1, 5],
+           [[8], [2, 9, 11, 34, 56]],
+           [[t(45,50)], [t(6,10), t(51,56), t(64,69), t(103,108), t(183,188)]])
+    tvs1 << tv
+    tv_w = TermVectorsWriter.new(@dir, "_test", @fis)
+    tv = SegmentTermVector.new("field1",
+           ["word1", "word2"],
+           [3, 2],
+           [[1, 5, 8], [2, 9]],
+           [[t(0,5), t(34,39), t(45,50)],[t(6,11), t(51,56)]])
+    tvs2 << tv
+    tv_w.add_all_doc_vectors(tvs1)
+    tv_w.add_all_doc_vectors(tvs2)
+    tv_w.close
+    tv_r = TermVectorsReader.new(@dir, "_test", @fis)
+    assert_equal(2, tv_r.size)
+    tv = tv_r.get_field_tv(0, "field1")
+
+    assert_equal(2, tv.size)
+    assert_equal("word1", tv.terms[0])
+    assert_equal(3, tv.term_frequencies[0])
+    assert_equal(1, tv.positions[0][0])
+    assert_equal(5, tv.positions[0][1])
+    assert_equal(8, tv.positions[0][2])
+    assert_equal(t(0,5), tv.offsets[0][0])
+    assert_equal(t(34,39), tv.offsets[0][1])
+    assert_equal(t(45,50), tv.offsets[0][2])
+
+    assert_equal("word2", tv.terms[1])
+    assert_equal(2, tv.term_frequencies[1])
+    assert_equal(2, tv.positions[1][0])
+    assert_equal(9, tv.positions[1][1])
+    assert_equal(t(6,11), tv.offsets[1][0])
+    assert_equal(t(51,56), tv.offsets[1][1])
+
+    tv = tv_r.get_field_tv(0, "field2")
+    assert_equal(2, tv.size)
+    assert_equal("word3", tv.terms[0])
+
+    tv = tv_r.get_field_tv(1, "field1")
+    assert_equal(2, tv.size)
+    assert_equal("word1", tv.terms[0])
+  end
+
+  private
+    def t(start, finish)
+      return TermVectorOffsetInfo.new(start, finish)
+    end
+end

data/test/unit/index/th_doc.rb

@@ -0,0 +1,244 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+module IndexTestHelper
+  include Ferret::Document
+  include Ferret::Index
+  include Ferret::Analysis
+  include Ferret::Search
+
+  def IndexTestHelper.make_binary(size)
+    tmp = Array.new(size)
+    size.times {|i| tmp[i] = i%256 }
+    return tmp.pack("c*")
+  end
+
+  BINARY_DATA = IndexTestHelper.make_binary(256)
+  COMPRESSED_BINARY_DATA = IndexTestHelper.make_binary(56)
+
+  def IndexTestHelper.prepare_document
+    doc = Document.new()
+
+    doc << Field.new("text_field1", "field one text", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::NO)
+    doc << Field.new("text_field2", "field field field two text", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    doc << Field.new("key_field", "keyword", Field::Store::YES, Field::Index::UNTOKENIZED)
+    doc << Field.new("unindexed_field", "unindexed field text", Field::Store::YES, Field::Index::NO)
+    doc << Field.new("unstored_field1", "unstored field text one", Field::Store::NO, Field::Index::TOKENIZED, Field::TermVector::NO)
+    doc << Field.new("unstored_field2", "unstored field text two", Field::Store::NO, Field::Index::TOKENIZED, Field::TermVector::YES)
+    doc << Field.new("compressed_field", "compressed text", Field::Store::COMPRESS, Field::Index::TOKENIZED, Field::TermVector::YES)
+    doc << Field.new_binary_field("binary_field", BINARY_DATA, Field::Store::YES)
+    doc << Field.new_binary_field("compressed_binary_field", COMPRESSED_BINARY_DATA, Field::Store::COMPRESS)
+    return doc
+  end
+
+  def IndexTestHelper.prepare_documents
+    data = [
+      ["apple", "green"],
+      ["apple", "red"],
+      ["orange", "orange"],
+      ["grape", "green"],
+      ["grape", "purple"],
+      ["mandarin", "orange"],
+      ["peach", "orange"],
+      ["apricot", "orange"]
+    ]
+
+    docs = []
+
+    data.each do |food|
+      doc = Document.new()
+      doc << Field.new("name", food[0], Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+      doc << Field.new("colour", food[1], Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+      docs << doc
+    end
+    return docs
+  end
+
+  def IndexTestHelper.write_document(dir, doc, segment="test", analyzer = WhiteSpaceAnalyzer.new(), similarity = Similarity.default())
+    writer = DocumentWriter.new(dir, analyzer, similarity, 50)
+    writer.add_document(segment, doc)
+  end
+
+  def IndexTestHelper.prepare_book_list
+    books = [
+      {"author" => "P.H. Newby", "title" => "Something To Answer For", "year" => "1969"},
+      {"author" => "Bernice Rubens", "title" => "The Elected Member", "year" => "1970"},
+      {"author" => "V. S. Naipaul", "title" => "In a Free State", "year" => "1971"},
+      {"author" => "John Berger", "title" => "G", "year" => "1972"},
+      {"author" => "J. G. Farrell", "title" => "The Siege of Krishnapur", "year" => "1973"},
+      {"author" => "Stanley Middleton", "title" => "Holiday", "year" => "1974"},
+      {"author" => "Nadine Gordimer", "title" => "The Conservationist", "year" => "1974"},
+      {"author" => "Ruth Prawer Jhabvala", "title" => "Heat and Dust", "year" => "1975"},
+      {"author" => "David Storey", "title" => "Saville", "year" => "1976"},
+      {"author" => "Paul Scott", "title" => "Staying On", "year" => "1977"},
+      {"author" => "Iris Murdoch", "title" => "The Sea", "year" => "1978"},
+      {"author" => "Penelope Fitzgerald", "title" => "Offshore", "year" => "1979"},
+      {"author" => "William Golding", "title" => "Rites of Passage", "year" => "1980"},
+      {"author" => "Salman Rushdie", "title" => "Midnight's Children", "year" => "1981"},
+      {"author" => "Thomas Keneally", "title" => "Schindler's Ark", "year" => "1982"},
+      {"author" => "J. M. Coetzee", "title" => "Life and Times of Michael K", "year" => "1983"},
+      {"author" => "Anita Brookner", "title" => "Hotel du Lac", "year" => "1984"},
+      {"author" => "Keri Hulme", "title" => "The Bone People", "year" => "1985"},
+      {"author" => "Kingsley Amis", "title" => "The Old Devils", "year" => "1986"},
+      {"author" => "Penelope Lively", "title" => "Moon Tiger", "year" => "1987"},
+      {"author" => "Peter Carey", "title" => "Oscar and Lucinda", "year" => "1988"},
+      {"author" => "Kazuo Ishiguro", "title" => "The Remains of the Day", "year" => "1989"},
+      {"author" => "A. S. Byatt", "title" => "Possession", "year" => "1990"},
+      {"author" => "Ben Okri", "title" => "The Famished Road", "year" => "1991"},
+      {"author" => "Michael Ondaatje", "title" => "The English Patient", "year" => "1992"},
+      {"author" => "Barry Unsworth", "title" => "Sacred Hunger", "year" => "1992"},
+      {"author" => "Roddy Doyle", "title" => "Paddy Clarke Ha Ha Ha", "year" => "1993"},
+      {"author" => "James Kelman", "title" => "How Late It Was, How Late", "year" => "1994"},
+      {"author" => "Pat Barker", "title" => "The Ghost Road", "year" => "1995"},
+      {"author" => "Graham Swift", "title" => "Last Orders", "year" => "1996"},
+      {"author" => "Arundati Roy", "title" => "The God of Small Things", "year" => "1997"},
+      {"author" => "Ian McEwan", "title" => "Amsterdam", "year" => "1998"},
+      {"author" => "J. M. Coetzee", "title" => "Disgrace", "year" => "1999"},
+      {"author" => "Margaret Atwood", "title" => "The Blind Assassin", "year" => "2000"},
+      {"author" => "Peter Carey", "title" => "True History of the Kelly Gang", "year" => "2001"},
+      {"author" => "Yann Martel", "title" => "The Life of Pi", "year" => "2002"},
+      {"author" => "DBC Pierre", "title" => "Vernon God Little", "year" => "2003"}
+    ]
+    docs = []
+
+    books.each do |book|
+      doc = Document.new()
+      doc << Field.new("author", book["author"], Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+      doc << Field.new("title", book["title"], Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+      doc << Field.new("year", book["year"], Field::Store::YES, Field::Index::NO, Field::TermVector::NO)
+      docs << doc
+    end
+    return docs
+  end
+
+  IR_TEST_DOC_CNT = 64
+
+  def IndexTestHelper.prepare_ir_test_docs()
+    body = "body"
+    title = "title"
+    author = "author"
+    text = "text"
+    year = "year"
+    changing_field = "changing_field"
+
+    docs = Array.new(IR_TEST_DOC_CNT)
+    docs[0] = Document.new()
+    docs[0] << Field.new(body, "Where is Wally", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[0] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::NO)
+    docs[1] = Document.new()
+    docs[1] << Field.new(body, "Some Random Sentence read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[2] = Document.new()
+    docs[2] << Field.new(body, "Some read Random Sentence read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[3] = Document.new()
+    docs[3] << Field.new(title, "War And Peace", Field::Store::YES, Field::Index::UNTOKENIZED, Field::TermVector::WITH_OFFSETS)
+    docs[3] << Field.new(body, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[3] << Field.new(author, "Leo Tolstoy", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS)
+    docs[3] << Field.new(year, "1865", Field::Store::YES, Field::Index::NO, Field::TermVector::NO)
+    docs[3] << Field.new(text, "more text which is not stored", Field::Store::NO, Field::Index::TOKENIZED, Field::TermVector::NO)
+    docs[4] = Document.new()
+    docs[4] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[5] = Document.new()
+    docs[5] << Field.new(body, "Here's Wally", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[6] = Document.new()
+    docs[6] << Field.new(body, "Some Random Sentence read read read read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[7] = Document.new()
+    docs[7] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[8] = Document.new()
+    docs[8] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[9] = Document.new()
+    docs[9] << Field.new(body, "read Some Random Sentence read this will be used after unfinished next position read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[10] = Document.new()
+    docs[10] << Field.new(body, "Some read Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[10] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::YES)
+    docs[11] = Document.new()
+    docs[11] << Field.new(body, "And here too. Well, maybe Not", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[12] = Document.new()
+    docs[12] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[13] = Document.new()
+    docs[13] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[14] = Document.new()
+    docs[14] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[15] = Document.new()
+    docs[15] << Field.new(body, "Some read Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[16] = Document.new()
+    docs[16] << Field.new(body, "Some Random read read Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[17] = Document.new()
+    docs[17] << Field.new(body, "Some Random read Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[17] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS)
+    docs[18] = Document.new()
+    docs[18] << Field.new(body, "Wally Wally Wally", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[19] = Document.new()
+    docs[19] << Field.new(body, "Some Random Sentence", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[19] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_OFFSETS)
+    docs[20] = Document.new()
+    docs[20] << Field.new(body, "Wally is where Wally usually likes to go. Wally Mart! Wally likes shopping there for Where's Wally books. Wally likes to read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[20] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[21] = Document.new()
+    docs[21] << Field.new(body, "Some Random Sentence read read read and more read read read", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    docs[21] << Field.new(changing_field, "word3 word4 word1 word2 word1 word3 word4 word1 word3 word3", Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::NO)
+
+    buf = ""
+    21.times { buf << "skip " }
+    22.upto(IR_TEST_DOC_CNT) do |i|
+      buf << "skip "
+      docs[i] = Document.new()
+      docs[i] << Field.new(text, buf.clone, Field::Store::NO, Field::Index::TOKENIZED, Field::TermVector::WITH_POSITIONS_OFFSETS)
+    end
+    return docs
+  end
+
+  def IndexTestHelper.prepare_search_docs
+    data = [
+      {"date" => "20050930", "field" => "word1",
+        "cat" => "cat1/"},
+      {"date" => "20051001", "field" => "word1 word2 the quick brown fox",
+        "cat" => "cat1/sub1"},
+      {"date" => "20051002", "field" => "word1 word3",
+        "cat" => "cat1/sub1/subsub1"},
+      {"date" => "20051003", "field" => "word1 word3",
+        "cat" => "cat1/sub2"},
+      {"date" => "20051004", "field" => "word1 word2",
+        "cat" => "cat1/sub2/subsub2"},
+      {"date" => "20051005", "field" => "word1",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051006", "field" => "word1 word3",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051007", "field" => "word1",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051008", "field" => "word1 word2 word3 the fast brown fox",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051009", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051010", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051011", "field" => "word1 word3 the quick red fox",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051012", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051013", "field" => "word1",
+        "cat" => "cat1/sub2"},
+      {"date" => "20051014", "field" => "word1 word3 the quick hairy fox",
+        "cat" => "cat1/sub1"},
+      {"date" => "20051015", "field" => "word1",
+        "cat" => "cat1/sub2/subsub1"},
+      {"date" => "20051016",
+        "field" => "word1 the quick fox is brown and hairy and a little red",
+        "cat" => "cat1/sub1/subsub2"},
+      {"date" => "20051017", "field" => "word1 the brown fox is quick and red",
+        "cat" => "cat1/"}
+    ]
+
+    docs = []
+    data.each_with_index do |fields, i|
+      doc = Document.new()
+      doc.boost = i+1
+
+      fields.each_pair do |field, text|
+        doc << Field.new(field, text, Field::Store::NO, Field::Index::TOKENIZED, Field::TermVector::NO, i+1)
+      end
+      docs << doc
+    end
+    return docs
+  end
+end
+