ferret 0.9.6 → 0.10.0

data/test/unit/analysis/{ctc_tokenstream.rb → tc_token_stream.rb}

@@ -1,5 +1,27 @@
 require File.dirname(__FILE__) + "/../../test_helper"
 
+class TokenTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  def test_token
+    t = Token.new("text", 1, 2, 3)
+    assert_equal("text", t.text)
+    assert_equal(1, t.start)
+    assert_equal(2, t.end)
+    assert_equal(3, t.pos_inc)
+    t.text    = "yada yada yada"
+    t.start   = 11
+    t.end     = 12
+    t.pos_inc = 13
+    assert_equal("yada yada yada", t.text)
+    assert_equal(11, t.start)
+    assert_equal(12, t.end)
+    assert_equal(13, t.pos_inc)
+
+    t = Token.new("text", 1, 2)
+    assert_equal(1, t.pos_inc)
+  end
+end
+
 class AsciiLetterTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
@@ -454,6 +476,7 @@ class CustomTokenizerTest < Test::Unit::TestCase
     assert(! t.next())
     t = AsciiLowerCaseFilter.new(MyCSVTokenizer.new(input))
     assert_equal(Token.new("first field", 0, 11), t.next)
+    return
     assert_equal(Token.new("2nd field", 12, 21), t.next)
     assert_equal(Token.new("  p a d d e d  f i e l d  ", 22, 48), t.next)
     assert(! t.next())

data/test/unit/index/tc_index.rb

@@ -5,7 +5,6 @@ class IndexTest < Test::Unit::TestCase
   include Ferret::Search
   include Ferret::Analysis
   include Ferret::Store
-  include Ferret::Document
 
   def setup()
   end
@@ -26,7 +25,7 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def do_test_index_with_array(index)
-    data = [
+    [
       ["one two"],
       ["one", "three"],
       ["two"],
@@ -35,8 +34,7 @@ class IndexTest < Test::Unit::TestCase
       ["two", "three", "four"],
       ["one"],
       ["two", "three", "four", "five"]
-    ]
-    data.each {|doc| index << doc }
+    ].each {|doc| index << doc }
     assert_equal(8, index.size)
     q = "one"
     check_results(index, q, [0, 1, 3, 4, 6])
@@ -44,19 +42,19 @@ class IndexTest < Test::Unit::TestCase
     check_results(index, q, [0, 4])
     q = "one OR five"
     check_results(index, q, [0, 1, 3, 4, 6, 7])
-    assert_equal("two three four five", index.doc(7)["def_field"])
+    assert_equal(%w{two three four five}, index.doc(7)[:xxx])
   end
 
   def do_test_index_with_hash(index)
     data = [
-      {"def_field" => "one two"},
-      {"def_field" => "one", "field2" => "three"},
-      {"def_field" => "two"},
-      {"def_field" => "one", "field2" => "four"},
-      {"def_field" => "one two"},
-      {"def_field" => "two", "field2" => "three", "field3" => "four"},
-      {"def_field" => "one"},
-      {"def_field" => "two", "field2" => "three", "field3" => "five"}
+      {:xxx => "one two"},
+      {:xxx => "one", :field2 => "three"},
+      {:xxx => "two"},
+      {:xxx => "one", :field2 => "four"},
+      {:xxx => "one two"},
+      {:xxx => "two", :field2 => "three", :field3 => "four"},
+      {:xxx => "one"},
+      {:xxx => "two", :field2 => "three", :field3 => "five"}
     ]
     data.each {|doc| index << doc }
     q = "one AND two"
@@ -71,19 +69,19 @@ class IndexTest < Test::Unit::TestCase
     q = "two AND field3:f*"
     check_results(index, q, [5, 7])
     assert_equal("five", index.doc(7)["field3"])
-    assert_equal("two", index.doc(7)["def_field"])
+    assert_equal("two", index.doc(7)[:xxx])
   end
 
   def do_test_index_with_doc_array(index)
     data = [
-      {"def_field" => "one two multi", :id => "myid"},
-      {"def_field" => "one", :field2 => "three multi"},
-      {"def_field" => "two"},
-      {"def_field" => "one", :field2 => "four"},
-      {"def_field" => "one two"},
-      {"def_field" => "two", :field2 => "three", "field3" => "four"},
-      {"def_field" => "one multi2"},
-      {"def_field" => "two", :field2 => "this three multi2", "field3" => "five multi"}
+      {:xxx => "one two multi", :id => "myid"},
+      {:xxx => "one", :field2 => "three multi"},
+      {:xxx => "two"},
+      {:xxx => "one", :field2 => "four"},
+      {:xxx => "one two"},
+      {:xxx => "two", :field2 => "three", :field3 => "four"},
+      {:xxx => "one multi2", :id => "hello"},
+      {:xxx => "two", :field2 => "this three multi2", :field3 => "five multi"}
     ]
     data.each {|doc| index << doc }
     q = "one AND two"
@@ -109,36 +107,37 @@ class IndexTest < Test::Unit::TestCase
     assert_equal(7, index.size)
     q = "two AND (field3:f*)"
     check_results(index, q, [7])
-    doc["field2"] = "dave"
+    
+    doc.load
+    doc[:field2] = "dave"
     index << doc
-    check_results(index, q, [6, 7])
+    check_results(index, q, [7, 8])
     check_results(index, "*:this", [])
     assert_equal(8, index.size)
-    assert_equal("dave", index[7]["field2"])
+    assert_equal("dave", index[8][:field2])
     index.optimize
     check_results(index, q, [6, 7])
-    t = Term.new("field2", "three")
-    index.delete(t)
+    assert_equal("dave", index[7][:field2])
+    index.query_delete("field2:three")
     assert(index.deleted?(1))
     assert(index.deleted?(6))
     assert(! index.deleted?(7))
-    t = Term.new("field2", "four")
-    assert_equal("one", index[t]["def_field"])
-    assert_equal("one two multi", index["myid"]["def_field"])
+    assert_equal("one multi2", index["hello"][:xxx])
+    assert_equal("one two multi", index["myid"][:xxx])
     index.delete("myid")
     assert(index.deleted?(0))
   end
 
   def test_ram_index
-    index = Index.new(:default_field => "def_field")
+    index = Index.new(:default_input_field => :xxx)
     do_test_index_with_array(index)
     index.close
 
-    index = Index.new(:default_field => "def_field")
+    index = Index.new(:default_field => :xxx)
     do_test_index_with_hash(index)
     index.close
 
-    index = Index.new(:default_field => "def_field", :id_field => "id")
+    index = Index.new(:default_field => :xxx, :id_field => "id")
     do_test_index_with_doc_array(index)
     index.close
   end
@@ -147,23 +146,24 @@ class IndexTest < Test::Unit::TestCase
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
 
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
-    assert_raise(StandardError) do
+    assert_raise(IOError) do
       Index.new(:path => fs_path,
                 :create_if_missing => false,
-                :default_field => "def_field")
+                :default_field => :xxx)
     end
-    index = Index.new(:path => fs_path, :default_field => "def_field")
+
+    index = Index.new(:path => fs_path, :default_input_field => :xxx)
     do_test_index_with_array(index)
     index.close
 
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
-    index = Index.new(:path => fs_path, :default_field => "def_field")
+    index = Index.new(:path => fs_path, :default_field => :xxx)
     do_test_index_with_hash(index)
     index.close
 
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     index = Index.new(:path => fs_path,
-                      :default_field => "def_field",
+                      :default_field => :xxx,
                       :id_field => "id")
     do_test_index_with_doc_array(index)
     index.close
@@ -171,20 +171,18 @@ class IndexTest < Test::Unit::TestCase
 
   def test_fs_index_is_persistant
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
-
-    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
-    data = [
-      {"def_field" => "one two", :id => "me"},
-      {"def_field" => "one", :field2 => "three"},
-      {"def_field" => "two"},
-      {"def_field" => "one", :field2 => "four"},
-      {"def_field" => "one two"},
-      {"def_field" => "two", :field2 => "three", "field3" => "four"},
-      {"def_field" => "one"},
-      {"def_field" => "two", :field2 => "three", "field3" => "five"}
-    ]
-    index = Index.new(:path => fs_path, :default_field => "def_field")
-    data.each {|doc| index << doc }
+    index = Index.new(:path => fs_path, :default_field => :xxx, :create => true)
+
+    [
+      {:xxx => "one two", :id => "me"},
+      {:xxx => "one", :field2 => "three"},
+      {:xxx => "two"},
+      {:xxx => "one", :field2 => "four"},
+      {:xxx => "one two"},
+      {:xxx => "two", :field2 => "three", :field3 => "four"},
+      {:xxx => "one"},
+      {:xxx => "two", :field2 => "three", :field3 => "five"}
+    ].each {|doc| index << doc }
     assert_equal(8, index.size)
     index.close
 
@@ -197,174 +195,171 @@ class IndexTest < Test::Unit::TestCase
   def test_key_used_for_id_field
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
 
-    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
-    data = [
-      {:my_id => "one two", :id => "me"},
+    index = Index.new(:path => fs_path, :key => :my_id, :create => true)
+    [
+      {:my_id => "three", :id => "me"},
       {:my_id => "one", :field2 => "three"},
       {:my_id => "two"},
       {:my_id => "one", :field2 => "four"},
-      {:my_id => "one two"},
-      {:my_id => "two", :field2 => "three", "field3" => "four"},
+      {:my_id => "three"},
+      {:my_id => "two", :field2 => "three", :field3 => "four"},
       {:my_id => "one"},
-      {:my_id => "two", :field2 => "three", "field3" => "five"}
-    ]
-    index = Index.new(:path => fs_path, :key => :my_id)
-    data.each {|doc| index << doc }
+      {:my_id => "two", :field2 => "three", :field3 => "five"}
+    ].each {|doc| index << doc }
+    index.optimize
     assert_equal(3, index.size)
-    assert_equal("three", index[:two][:field2])
+    assert_equal("three", index["two"][:field2])
     index.close
   end
 
   def test_merging_indexes
-    data = [
-      {"f" => "zero"},
-      {"f" => "one"},
-      {"f" => "two"}
-    ]
-    index1 = Index.new(:default_field => "f")
-    data.each {|doc| index1 << doc }
-    data = [
-      {"f" => "three"},
-      {"f" => "four"},
-      {"f" => "five"}
-    ]
-    index2 = Index.new(:default_field => "f")
-    data.each {|doc| index2 << doc }
-    data = [
-      {"f" => "six"},
-      {"f" => "seven"},
-      {"f" => "eight"}
-    ]
-    index3 = Index.new(:default_field => "f")
-    data.each {|doc| index3 << doc }
-
-    index = Index.new(:default_field => "f")
+    index1 = Index.new(:default_field => :f)
+    index2 = Index.new(:default_field => :f)
+    index3 = Index.new(:default_field => :f)
+
+    [
+      {:f => "zero"},
+      {:f => "one"},
+      {:f => "two"}
+    ].each {|doc| index1 << doc }
+    [
+      {:f => "three"},
+      {:f => "four"},
+      {:f => "five"}
+    ].each {|doc| index2 << doc }
+    [
+      {:f => "six"},
+      {:f => "seven"},
+      {:f => "eight"}
+    ].each {|doc| index3 << doc }
+
+    index = Index.new(:default_field => :f)
     index.add_indexes(index1)
     assert_equal(3, index.size)
-    assert_equal("zero", index[0]["f"])
+    assert_equal("zero", index[0][:f])
     index.add_indexes([index2, index3])
     assert_equal(9, index.size)
-    assert_equal("zero", index[0]["f"])
-    assert_equal("eight", index[8]["f"])
+    assert_equal("zero", index[0][:f])
+    assert_equal("eight", index[8][:f])
     index1.close
     index2.close
     index3.close
-    assert_equal("seven", index[7]["f"])
+    assert_equal("seven", index[7][:f])
     data = [
-      {"f" => "alpha"},
-      {"f" => "beta"},
-      {"f" => "charlie"}
+      {:f => "alpha"},
+      {:f => "beta"},
+      {:f => "charlie"}
     ]
     dir1 = RAMDirectory.new
-    index1 = Index.new(:dir => dir1, :default_field => "f")
+    index1 = Index.new(:dir => dir1, :default_field => :f)
     data.each {|doc| index1 << doc }
     index1.flush
     data = [
-      {"f" => "delta"},
-      {"f" => "echo"},
-      {"f" => "foxtrot"}
+      {:f => "delta"},
+      {:f => "echo"},
+      {:f => "foxtrot"}
     ]
     dir2 = RAMDirectory.new
-    index2 = Index.new(:dir => dir2, :default_field => "f")
+    index2 = Index.new(:dir => dir2, :default_field => :f)
     data.each {|doc| index2 << doc }
     index2.flush
     data = [
-      {"f" => "golf"},
-      {"f" => "india"},
-      {"f" => "juliet"}
+      {:f => "golf"},
+      {:f => "india"},
+      {:f => "juliet"}
     ]
     dir3 = RAMDirectory.new
-    index3 = Index.new(:dir => dir3, :default_field => "f")
+    index3 = Index.new(:dir => dir3, :default_field => :f)
     data.each {|doc| index3 << doc }
     index3.flush
 
     index.add_indexes(dir1)
     assert_equal(12, index.size)
-    assert_equal("alpha", index[9]["f"])
+    assert_equal("alpha", index[9][:f])
     index.add_indexes([dir2, dir3])
     assert_equal(18, index.size)
-    assert_equal("juliet", index[17]["f"])
+    assert_equal("juliet", index[17][:f])
     index1.close
     dir1.close
     index2.close
     dir2.close
     index3.close
     dir3.close
-    assert_equal("golf", index[15]["f"])
+    assert_equal("golf", index[15][:f])
     index.close
   end
 
   def test_persist_index
     data = [
-      {"f" => "zero"},
-      {"f" => "one"},
-      {"f" => "two"}
+      {:f => "zero"},
+      {:f => "one"},
+      {:f => "two"}
     ]
-    index = Index.new(:default_field => "f")
+    index = Index.new(:default_field => :f)
     data.each {|doc| index << doc }
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
 
     index.persist(fs_path, true)
     assert_equal(3, index.size)
-    assert_equal("zero", index[0]["f"])
+    assert_equal("zero", index[0][:f])
     index.close
 
     index = Index.new(:path => fs_path)
     assert_equal(3, index.size)
-    assert_equal("zero", index[0]["f"])
+    assert_equal("zero", index[0][:f])
     index.close
 
 
     data = [
-      {"f" => "romeo"},
-      {"f" => "sierra"},
-      {"f" => "tango"}
+      {:f => "romeo"},
+      {:f => "sierra"},
+      {:f => "tango"}
     ]
-    index = Index.new(:default_field => "f")
+    index = Index.new(:default_field => :f)
     data.each {|doc| index << doc }
     assert_equal(3, index.size)
-    assert_equal("romeo", index[0]["f"])
+    assert_equal("romeo", index[0][:f])
     dir = FSDirectory.new(fs_path, false)
     index.persist(dir)
     assert_equal(6, index.size)
-    assert_equal("zero", index[0]["f"])
-    assert_equal("romeo", index[3]["f"])
+    assert_equal("zero", index[0][:f])
+    assert_equal("romeo", index[3][:f])
     index.close
 
     index = Index.new(:path => fs_path)
     assert_equal(6, index.size)
-    assert_equal("zero", index[0]["f"])
-    assert_equal("romeo", index[3]["f"])
+    assert_equal("zero", index[0][:f])
+    assert_equal("romeo", index[3][:f])
     index.close
   end
 
   def test_auto_update_when_externally_modified()
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
-    index = Index.new(:path => fs_path, :default_field => "f", :create => true)
+    index = Index.new(:path => fs_path, :default_field => :f, :create => true)
     index << "document 1"
     assert_equal(1, index.size)
 
-    index2 = Index.new(:path => fs_path, :default_field => "f")
+    index2 = Index.new(:path => fs_path, :default_field => :f)
     assert_equal(1, index2.size)
     index2 << "document 2"
     assert_equal(2, index2.size)
     assert_equal(2, index.size)
     top_docs = index.search("content3")
-    assert_equal(0, top_docs.size)
+    assert_equal(0, top_docs.hits.size)
 
-    iw = IndexWriter.new(fs_path, :analyzer => WhiteSpaceAnalyzer.new())
-    doc = Document.new
-    doc << Field.new("f", "content3", Field::Store::YES, Field::Index::TOKENIZED)
-    iw << doc
+    iw = IndexWriter.new(:path => fs_path, :analyzer => WhiteSpaceAnalyzer.new())
+    iw << {:f, "content3"}
     iw.close()
+
     top_docs = index.search("content3")
-    assert_equal(1, top_docs.size)
+    assert_equal(1, top_docs.hits.size)
     assert_equal(3, index.size)
-    assert_equal("content3", index[2]["f"])
+    assert_equal("content3", index[2][:f])
     index.close
   end
 
   def test_delete
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
     data = [
       {:id => 0, :cat => "/cat1/subcat1"},
       {:id => 1, :cat => "/cat1/subcat2"},
@@ -376,26 +371,27 @@ class IndexTest < Test::Unit::TestCase
       {:id => 7, :cat => "/cat2/subcat3"},
       {:id => 8, :cat => "/cat2/subcat4"},
       {:id => 9, :cat => "/cat2/subcat5"},
-    ]
-    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
-    data.each {|doc| index << doc }
+    ].each {|doc| index << doc }
     assert_equal(10, index.size)
-    assert_equal(1, index.search("id:9").size)
+    assert_equal(1, index.search("id:9").total_hits)
     index.delete(9)
     assert_equal(9, index.size)
-    assert_equal(0, index.search("id:9").size)
-    assert_equal(1, index.search("id:8").size)
+    assert_equal(0, index.search("id:9").total_hits)
+    assert_equal(1, index.search("id:8").total_hits)
     index.delete("8")
     assert_equal(8, index.size)
-    assert_equal(0, index.search("id:8").size)
-    assert_equal(5, index.search("cat:/cat1*").size)
+    assert_equal(0, index.search("id:8").total_hits)
+    assert_equal(5, index.search("cat:/cat1*").total_hits)
     index.query_delete("cat:/cat1*")
     assert_equal(3, index.size)
-    assert_equal(0, index.search("cat:/cat1*").size)
+    assert_equal(0, index.search("cat:/cat1*").total_hits)
     index.close
   end
 
   def test_update
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
+                      :default_input_field => :content,
+                      :id_field => :id)
     data = [
       {:id => 0, :cat => "/cat1/subcat1", :content => "content0"},
       {:id => 1, :cat => "/cat1/subcat2", :content => "content1"},
@@ -407,28 +403,25 @@ class IndexTest < Test::Unit::TestCase
       {:id => 7, :cat => "/cat2/subcat3", :content => "content7"},
       {:id => 8, :cat => "/cat2/subcat4", :content => "content8"},
       {:id => 9, :cat => "/cat2/subcat5", :content => "content9"},
-    ]
-    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
-                      :default_field => :content,
-                      :id_field => :id)
-    data.each { |doc| index << doc }
+    ].each { |doc| index << doc }
     assert_equal(10, index.size)
     assert_equal("content5", index["5"][:content])
-    index.update(5, "content five")
+    index.query_update("id:5", {:content => "content five"})
     assert_equal("content five", index["5"][:content])
     assert_equal(nil, index["5"][:extra_content])
-    index.update("5", {:cat => "/cat1/subcat6",
+    index.update("5", {:id => "5",
+                       :cat => "/cat1/subcat6",
                        :content => "high five",
                        :extra_content => "hello"})
     assert_equal("hello", index["5"][:extra_content])
     assert_equal("high five", index["5"][:content])
     assert_equal("/cat1/subcat6", index["5"][:cat])
     assert_equal("content9", index["9"][:content])
-    index.update(Term.new("content", "content9"), {:content => "content nine"})
+    index.query_update("content:content9", {:content => "content nine"})
     assert_equal("content nine", index["9"][:content])
     assert_equal("content0", index["0"][:content])
     assert_equal(nil, index["0"][:extra_content])
-    document = index[0]
+    document = index[0].load
     document[:content] = "content zero"
     document[:extra_content] = "extra content"
     index.update(0, document)
@@ -443,7 +436,7 @@ class IndexTest < Test::Unit::TestCase
     assert_equal("cool", index["2"][:tag])
     assert_equal("cool", index["3"][:tag])
     assert_equal("cool", index["4"][:tag])
-    assert_equal(4, index.search("tag:cool").size)
+    assert_equal(4, index.search("tag:cool").total_hits)
     index.close
   end
 
@@ -458,12 +451,14 @@ class IndexTest < Test::Unit::TestCase
                       :key => :id)
     data.each { |doc| index << doc }
     assert_equal(2, index.size)
-    assert_equal("two", index[0][:val])
-    assert_equal("four", index[1][:val])
+    assert_equal("two", index["0"][:val])
+    assert_equal("four", index["1"][:val])
     index.close
   end
 
   def test_index_multi_key
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
+                      :key => [:id, :table])
     data = [
       {:id => 0, :table => "product", :product => "tent"},
       {:id => 0, :table => "location", :location => "first floor"},
@@ -474,10 +469,8 @@ class IndexTest < Test::Unit::TestCase
       {:id => 1, :table => "location", :location => "first floor"},
       {:id => 1, :table => "product", :product => "rucksack"},
       {:id => 1, :table => "product", :product => "backpack"}
-    ]
-    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
-                      :key => [:id, :table])
-    data.each { |doc| index << doc }
+    ].each { |doc| index << doc }
+    index.optimize
     assert_equal(4, index.size)
     assert_equal("super tent", index[0][:product])
     assert_equal("second floor", index[1][:location])
@@ -487,6 +480,13 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def test_index_multi_key_untokenized
+    field_infos = FieldInfos.new(:term_vector => :no)
+    field_infos.add_field(:id, :index => :untokenized)
+    field_infos.add_field(:table, :index => :untokenized)
+
+    index = Index.new(:analyzer => Analyzer.new,
+                      :key => [:id, :table],
+                      :field_infos => field_infos)
     data = [
       {:id => 0, :table => "Product", :product => "tent"},
       {:id => 0, :table => "location", :location => "first floor"},
@@ -497,21 +497,10 @@ class IndexTest < Test::Unit::TestCase
       {:id => 1, :table => "location", :location => "first floor"},
       {:id => 1, :table => "Product", :product => "rucksack"},
       {:id => 1, :table => "Product", :product => "backpack"}
-    ]
-    index = Index.new(:analyzer => Analyzer.new,
-                      :key => [:id, :table])
-    data.each do |dat|
-      doc = Document.new
-      dat.each_pair do |key, value|
-        if ([:id, :table].include?(key))
-          doc << Field.new(key, value, Field::Store::YES, Field::Index::UNTOKENIZED)
-        else
-          doc << Field.new(key, value, Field::Store::YES, Field::Index::TOKENIZED)
-        end
-      end
-      index << doc 
-    end
+    ].each {|doc| index << doc}
+
     assert_equal(4, index.size)
+    index.optimize
     assert_equal("super tent", index[0][:product])
     assert_equal("second floor", index[1][:location])
     assert_equal("backpack", index[3][:product])
@@ -520,6 +509,8 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def test_sortby_date
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
+
     data = [
       {:content => "one", :date => "20051023"},
       {:content => "two", :date => "19530315"},
@@ -530,37 +521,31 @@ class IndexTest < Test::Unit::TestCase
       {:content => "one", :date => "19770725"},
       {:content => "two", :date => "19751226"},
       {:content => "four", :date => "19390912"}
-    ]
-    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
-    data.each { |doc|
-      document = Document.new
-      doc.each_pair do |key, value|
-        document << Field.new(key.to_s, value, Field::Store::YES, Field::Index::TOKENIZED)
-      end
-      index << document
-    }
-    sf_date = SortField.new("date", {:sort_type => SortField::SortType::INTEGER})
-    #top_docs = index.search("one", :sort => [sf_date, SortField::FIELD_SCORE])
+    ].each {|doc| index << doc}
+
+    sf_date = SortField.new("date", {:sort_type => :integer})
+    #top_docs = index.search("one", :sort => [sf_date, SortField::SCORE])
     top_docs = index.search("one", :sort => Sort.new("date"))
-    assert_equal(3, top_docs.size)
-    assert_equal("19770725", index[top_docs.score_docs[0].doc][:date])
-    assert_equal("19770905", index[top_docs.score_docs[1].doc][:date])
-    assert_equal("20051023", index[top_docs.score_docs[2].doc][:date])
+    assert_equal(3, top_docs.total_hits)
+    assert_equal("19770725", index[top_docs.hits[0].doc][:date])
+    assert_equal("19770905", index[top_docs.hits[1].doc][:date])
+    assert_equal("20051023", index[top_docs.hits[2].doc][:date])
     top_docs = index.search("one two three four",
-                            :sort => [sf_date, SortField::FIELD_SCORE])
-    assert_equal("19390912", index[top_docs.score_docs[0].doc][:date])
-    assert_equal("three four", index[top_docs.score_docs[0].doc][:content])
-    assert_equal("19390912", index[top_docs.score_docs[1].doc][:date])
-    assert_equal("four", index[top_docs.score_docs[1].doc][:content])
-    assert_equal("19530315", index[top_docs.score_docs[2].doc][:date])
+                            :sort => [sf_date, SortField::SCORE])
+    return
+    assert_equal("19390912", index[top_docs.hits[0].doc][:date])
+    assert_equal("three four", index[top_docs.hits[0].doc][:content])
+    assert_equal("19390912", index[top_docs.hits[1].doc][:date])
+    assert_equal("four", index[top_docs.hits[1].doc][:content])
+    assert_equal("19530315", index[top_docs.hits[2].doc][:date])
 
     top_docs = index.search("one two three four",
                             :sort => [:date, :content])
-    assert_equal("19390912", index[top_docs.score_docs[0].doc][:date])
-    assert_equal("four", index[top_docs.score_docs[0].doc][:content])
-    assert_equal("19390912", index[top_docs.score_docs[1].doc][:date])
-    assert_equal("three four", index[top_docs.score_docs[1].doc][:content])
-    assert_equal("19530315", index[top_docs.score_docs[2].doc][:date])
+    assert_equal("19390912", index[top_docs.hits[0].doc][:date])
+    assert_equal("four", index[top_docs.hits[0].doc][:content])
+    assert_equal("19390912", index[top_docs.hits[1].doc][:date])
+    assert_equal("three four", index[top_docs.hits[1].doc][:content])
+    assert_equal("19530315", index[top_docs.hits[2].doc][:date])
 
     index.close
   end
@@ -568,6 +553,7 @@ class IndexTest < Test::Unit::TestCase
   def test_auto_flush
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
+
     data = %q(one two three four five six seven eight nine ten eleven twelve)
     index1 = Index.new(:path => fs_path, :auto_flush => true)
     index2 = Index.new(:path => fs_path, :auto_flush => true)
@@ -595,59 +581,16 @@ class IndexTest < Test::Unit::TestCase
     assert_equal(1, index.size)
   end
 
-
   def test_adding_empty_term_vectors
-    index = Index.new()
-    doc = Document.new
+    index = Index.new(:field_infos => FieldInfos.new(:term_vector => :no))
 
     # Note: Adding keywords to either field1 or field2 gets rid of the error
 
-    doc << Field.new('field1', '',
-            Field::Store::NO,
-            Field::Index::TOKENIZED,
-            Field::TermVector::YES)
-
-    doc << Field.new('field2', '',
-            Field::Store::NO,
-            Field::Index::TOKENIZED,
-            Field::TermVector::YES)
-
-    # Note: keywords in this un-term-vector-stored field don't help the situation
-
-    doc << Field.new('field3', 'foo bar baz',
-            Field::Store::YES,
-            Field::Index::TOKENIZED,
-            Field::TermVector::NO)
-
-    index << doc
+    index << {:field1, ''}
+    index << {:field2, ''}
+    index << {:field3, 'foo bar baz'}
 
     index.flush
     index.close
   end
-
-  def test_stopwords
-    i = Ferret::Index::Index.new(
-            :occur_default => Ferret::Search::BooleanClause::Occur::MUST,
-            :default_search_field => '*')
-    d = Ferret::Document::Document.new
- 
-    # adding this additional field to the document leads to failure below
-    # comment out this statement and all tests pass:
-    d << Ferret::Document::Field.new('id', '1',
-                                     Ferret::Document::Field::Store::YES,
-                                     Ferret::Document::Field::Index::UNTOKENIZED)
- 
-    d << Ferret::Document::Field.new('content', 'Move or shake',
-                                     Ferret::Document::Field::Store::NO,
-                                     Ferret::Document::Field::Index::TOKENIZED,
-                                     Ferret::Document::Field::TermVector::NO,
-                                     false, 1.0)
-    i << d
-    hits = i.search 'move nothere shake'
-    assert_equal 0, hits.size
-    hits = i.search 'move shake'
-    assert_equal 1, hits.size
-    hits = i.search 'move or shake'
-    assert_equal 1, hits.size # fails when id field is present
-  end
 end