ferret 0.9.6 → 0.10.0

data/test/functional/thread_safety_test.rb

@@ -1,137 +0,0 @@
-require File.dirname(__FILE__) + "/../test_helper"
-require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
-require 'thread'
-
-class ThreadSafetyTest
-  include Ferret::Index
-  include Ferret::Search
-  include Ferret::Store
-  include Ferret::Document
-
-  def initialize(options)
-    @options = options
-  end
-
-  INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
-  ANALYZER = Ferret::Analysis::Analyzer.new()
-  ITERATIONS = 19
-  @@searcher = nil
-  
-  def run_index_thread(writer)
-    reopen_interval = 30 + rand(60)
-
-    use_compound_file = false
-    
-    (400*ITERATIONS).times do |i|
-      d = Document.new()
-      n = rand(0xFFFFFFFF)
-      d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
-      d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
-      puts("Adding #{n}")
-      
-      # Switch between single and multiple file segments
-      use_compound_file = (rand < 0.5)
-      writer.use_compound_file = use_compound_file
-      
-      writer << d
-
-      if (i % reopen_interval == 0) 
-        writer.close()
-        writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER)
-      end
-    end
-    
-    writer.close()
-  rescue => e
-    puts e
-    puts e.backtrace
-    raise e
-  end
-
-  def run_search_thread(use_global)
-    reopen_interval = 10 + rand(20)
-
-    unless use_global
-      searcher = IndexSearcher.new(INDEX_DIR)
-    end
-
-    (50*ITERATIONS).times do |i|
-      search_for(rand(0xFFFFFFFF), (searcher.nil? ? @@searcher : searcher))
-      if (i%reopen_interval == 0) 
-        if (searcher == nil) 
-          @@searcher = IndexSearcher.new(INDEX_DIR)
-        else 
-          searcher.close()
-          searcher = IndexSearcher.new(INDEX_DIR)
-        end
-      end
-    end
-  rescue => e
-    puts e
-    puts e.backtrace
-    raise e
-  end
-
-  def search_for(n, searcher)
-    puts("Searching for #{n}")
-    hits =
-      searcher.search(Ferret::QueryParser.parse(n.to_spoken, "contents", :analyzer => ANALYZER),
-                      :num_docs => 3)
-    puts("Search for #{n}: total = #{hits.size}")
-    hits.each do |d, s|
-      puts "Hit for #{n}: #{searcher.reader.get_document(d)["id"]} - #{s}"
-    end
-  end
-
-  def run_test_threads
-
-    threads = []
-    unless @options[:read_only]
-      writer = IndexWriter.new(INDEX_DIR, :analyzer => ANALYZER,
-                               :create => !@options[:add])
-      
-      threads << Thread.new { run_index_thread(writer) }
-      
-      sleep(1)
-    end
-      
-    threads << Thread.new { run_search_thread(false)}
-
-    @@searcher = IndexSearcher.new(INDEX_DIR) 
-    threads << Thread.new { run_search_thread(true)}
-
-    threads << Thread.new { run_search_thread(true)}
-
-    threads.each {|t| t.join}
-  end
-end
-
-
-if $0 == __FILE__
-  require 'optparse'
-
-  OPTIONS = {
-    :all        => false,
-    :read_only  => false,
-  }
-
-  ARGV.options do |opts|
-    script_name = File.basename($0)
-    opts.banner = "Usage: ruby #{script_name} [options]"
-
-    opts.separator ""
-
-    opts.on("-r", "--read-only", "Read Only.") { OPTIONS[:all] = true }
-    opts.on("-a", "--all", "All.") { OPTIONS[:read_only] = true }
-
-    opts.separator ""
-
-    opts.on("-h", "--help",
-            "Show this help message.") { puts opts; exit }
-
-    opts.parse!
-  end
-
-  tst = ThreadSafetyTest.new(OPTIONS)
-  tst.run_test_threads
-end

data/test/longrunning/tc_numbertools.rb

@@ -1,60 +0,0 @@
-require File.dirname(__FILE__) + "/../../../test_helper"
-
-
-class NumberToolsTest < Test::Unit::TestCase
-  include Lucene::Document
-  def test_near_zero()
-    10.times() do |i|
-      10.times() { |j| subtest_two_longs(i, j) }
-    end
-  end
-
-  def test_max()
-    # make sure the constants convert to their equivelents
-    assert_equal(NumberTools::LONG_MAX_VALUE, NumberTools.s_to_long(NumberTools::MAX_STRING_VALUE))
-    assert_equal(NumberTools::MAX_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MAX_VALUE))
-    # test near MAX, too
-
-    NumberTools::LONG_MAX_VALUE.downto(NumberTools::LONG_MAX_VALUE - 100) do |l|
-      subtest_two_longs(l, l - 1)
-    end
-  end
-
-  def test_min()
-    # make sure the constants convert to their equivelents
-    assert_equal(NumberTools::LONG_MIN_VALUE, NumberTools.s_to_long(NumberTools::MIN_STRING_VALUE))
-    assert_equal(NumberTools::MIN_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MIN_VALUE))
-
-    # test near MIN, too
-    NumberTools::LONG_MIN_VALUE.upto(NumberTools::LONG_MIN_VALUE + 100) do |l|
-      subtest_two_longs(l, l + 1)
-    end
-  end
-
-  def subtest_two_longs(i, j)
-    # convert to strings
-    a = NumberTools.long_to_s(i)
-    b = NumberTools.long_to_s(j)
-
-    # are they the right length?
-    assert_equal(NumberTools::STR_SIZE, a.length())
-    assert_equal(NumberTools::STR_SIZE, b.length())
-
-    # are they the right order?
-    if (i < j)
-        assert(a < b)
-    elsif (i > j)
-        assert(a > b)
-    else
-        assert_equal(a, b)
-    end
-
-    # can we convert them back to longs?
-    i2 = NumberTools.s_to_long(a)
-    j2 = NumberTools.s_to_long(b)
-
-    assert_equal(i, i2)
-    assert_equal(j, j2)
-  end
-
-end

data/test/longrunning/tm_store.rb

@@ -1,19 +0,0 @@
-module StoreTest
-  # declare dir so inheritors can access it.
-  def test_modified_full
-    # difficult to test this one but as file mtime is only stored to the
-    # nearest second. We can assume this test will happen in less than one
-    # second. (I hope)
-    time = Time.new.to_i
-    @dir.touch('mtime_test')
-    time_before = @dir.modified('mtime_test').to_i
-    assert(time_before - time <= 2, "test that mtime is approximately equal to the system time when the file was touched")
-    # wait until the time ticks over one second.
-    time = Time.new while (time.to_i == time_before)
-    time_before_again = @dir.modified('mtime_test').to_i
-    assert_equal(time_before, time_before_again, "the modified time shouldn't change")
-    @dir.touch('mtime_test')
-    time_after = @dir.modified('mtime_test').to_i
-    assert(time_before < time_after, "the modified time should now be greater")
-  end
-end

data/test/unit/analysis/ctc_analyzer.rb

@@ -1,532 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class AnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
-    a = Analyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new("dbalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("my", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("address", 39, 46), t.next())
-    assert(! t.next())
-    assert_equal(Token.new("dbalmain", 0, 8), t2.next())
-    assert_equal(Token.new("gmail", 9, 14), t2.next())
-    assert_equal(Token.new("com", 15, 18), t2.next())
-    assert_equal(Token.new("is", 19, 21), t2.next())
-    assert_equal(Token.new("my", 22, 24), t2.next())
-    assert_equal(Token.new("e", 25, 26), t2.next())
-    assert_equal(Token.new("mail", 27, 31), t2.next())
-    assert_equal(Token.new("address", 39, 46), t2.next())
-    assert(! t2.next())
-    a = Analyzer.new(false)
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new("DBalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("My", 22, 24), t.next())
-    assert_equal(Token.new("E", 25, 26), t.next())
-    assert_equal(Token.new("Mail", 27, 31), t.next())
-    assert_equal(Token.new("ADDRESS", 39, 46), t.next())
-    assert(! t.next())
-  end
-end
-
-class AsciiLetterAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_letter_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
-    a = AsciiLetterAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new("dbalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("my", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("address", 39, 46), t.next())
-    assert(! t.next())
-    assert_equal(Token.new("dbalmain", 0, 8), t2.next())
-    assert_equal(Token.new("gmail", 9, 14), t2.next())
-    assert_equal(Token.new("com", 15, 18), t2.next())
-    assert_equal(Token.new("is", 19, 21), t2.next())
-    assert_equal(Token.new("my", 22, 24), t2.next())
-    assert_equal(Token.new("e", 25, 26), t2.next())
-    assert_equal(Token.new("mail", 27, 31), t2.next())
-    assert_equal(Token.new("address", 39, 46), t2.next())
-    assert(! t2.next())
-    a = AsciiLetterAnalyzer.new(false)
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new("DBalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("My", 22, 24), t.next())
-    assert_equal(Token.new("E", 25, 26), t.next())
-    assert_equal(Token.new("Mail", 27, 31), t.next())
-    assert_equal(Token.new("ADDRESS", 39, 46), t.next())
-    assert(! t.next())
-  end
-end
-
-class LetterAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_letter_analyzer()
-    Ferret.locale = ""
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
-    a = LetterAnalyzer.new(false)
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new("DBalmän", 0, 8), t.next)
-    assert_equal(Token.new("gmail", 9, 14), t.next)
-    assert_equal(Token.new("com", 15, 18), t.next)
-    assert_equal(Token.new("is", 19, 21), t.next)
-    assert_equal(Token.new("My", 22, 24), t.next)
-    assert_equal(Token.new("e", 25, 26), t.next)
-    assert_equal(Token.new("mail", 27, 31), t.next)
-    assert_equal(Token.new("address", 40, 47), t.next)
-    assert_equal(Token.new("ÁÄGÇ", 55, 62), t.next)
-    assert_equal(Token.new("ÊËÌ", 64, 70), t.next)
-    assert_equal(Token.new("ÚØÃ", 72, 78), t.next)
-    assert_equal(Token.new("ÖÎÍ", 80, 86), t.next)
-    assert(! t.next())
-    assert_equal(Token.new("DBalmän", 0, 8), t2.next)
-    assert_equal(Token.new("gmail", 9, 14), t2.next)
-    assert_equal(Token.new("com", 15, 18), t2.next)
-    assert_equal(Token.new("is", 19, 21), t2.next)
-    assert_equal(Token.new("My", 22, 24), t2.next)
-    assert_equal(Token.new("e", 25, 26), t2.next)
-    assert_equal(Token.new("mail", 27, 31), t2.next)
-    assert_equal(Token.new("address", 40, 47), t2.next)
-    assert_equal(Token.new("ÁÄGÇ", 55, 62), t2.next)
-    assert_equal(Token.new("ÊËÌ", 64, 70), t2.next)
-    assert_equal(Token.new("ÚØÃ", 72, 78), t2.next)
-    assert_equal(Token.new("ÖÎÍ", 80, 86), t2.next)
-    assert(! t2.next())
-    a = LetterAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new("dbalmän", 0, 8), t.next)
-    assert_equal(Token.new("gmail", 9, 14), t.next)
-    assert_equal(Token.new("com", 15, 18), t.next)
-    assert_equal(Token.new("is", 19, 21), t.next)
-    assert_equal(Token.new("my", 22, 24), t.next)
-    assert_equal(Token.new("e", 25, 26), t.next)
-    assert_equal(Token.new("mail", 27, 31), t.next)
-    assert_equal(Token.new("address", 40, 47), t.next)
-    assert_equal(Token.new("áägç", 55, 62), t.next)
-    assert_equal(Token.new("êëì", 64, 70), t.next)
-    assert_equal(Token.new("úøã", 72, 78), t.next)
-    assert_equal(Token.new("öîí", 80, 86), t.next)
-    assert(! t.next())
-  end
-end
-
-class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_white_space_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#@$'
-    a = AsciiWhiteSpaceAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('E-Mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('ADDRESS.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert(! t.next())
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t2.next)
-    assert_equal(Token.new('is', 19, 21), t2.next)
-    assert_equal(Token.new('My', 22, 24), t2.next)
-    assert_equal(Token.new('E-Mail', 25, 31), t2.next)
-    assert_equal(Token.new('52', 32, 34), t2.next)
-    assert_equal(Token.new('#$', 37, 39), t2.next)
-    assert_equal(Token.new('ADDRESS.', 40, 48), t2.next)
-    assert_equal(Token.new('23#@$', 49, 54), t2.next)
-    assert(! t2.next())
-    a = AsciiWhiteSpaceAnalyzer.new(true)
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert(! t.next())
-  end
-end
-
-class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_white_space_analyzer()
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
-    a = WhiteSpaceAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t.next)
-    assert(! t.next())
-    assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t2.next)
-    assert_equal(Token.new('is', 19, 21), t2.next)
-    assert_equal(Token.new('My', 22, 24), t2.next)
-    assert_equal(Token.new('e-mail', 25, 31), t2.next)
-    assert_equal(Token.new('52', 32, 34), t2.next)
-    assert_equal(Token.new('#$', 37, 39), t2.next)
-    assert_equal(Token.new('address.', 40, 48), t2.next)
-    assert_equal(Token.new('23#@$', 49, 54), t2.next)
-    assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t2.next)
-    assert(! t2.next())
-    a = WhiteSpaceAnalyzer.new(true)
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmän@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
-    assert(! t.next())
-  end
-end
-
-class AsciiStandardAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_standard_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
-    a = AsciiStandardAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
-    assert_equal(Token.new('tnt', 86, 91), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
-    assert(! t.next())
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t2.next)
-    assert_equal(Token.new('e-mail', 25, 31), t2.next)
-    assert_equal(Token.new('52', 32, 34), t2.next)
-    assert_equal(Token.new('address', 40, 47), t2.next)
-    assert_equal(Token.new('23', 49, 51), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
-    assert_equal(Token.new('tnt', 86, 91), t2.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
-    assert(! t2.next())
-    a = AsciiStandardAnalyzer.new(false)
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('Address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
-    assert_equal(Token.new('TNT', 86, 91), t.next)
-    assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
-    assert(! t.next())
-  end
-end
-
-class StandardAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_c_standard_analyzer()
-    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
-    a = StandardAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
-    assert_equal(Token.new('tnt', 86, 91), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
-    assert_equal(Token.new('23', 111, 113), t.next)
-    assert_equal(Token.new('áägç', 117, 124), t.next)
-    assert_equal(Token.new('êëì', 126, 132), t.next)
-    assert_equal(Token.new('úøã', 134, 140), t.next)
-    assert_equal(Token.new('öîí', 142, 148), t.next)
-    assert(! t.next())
-    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t2.next)
-    assert_equal(Token.new('e-mail', 25, 31), t2.next)
-    assert_equal(Token.new('address', 40, 47), t2.next)
-    assert_equal(Token.new('23', 49, 51), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
-    assert_equal(Token.new('tnt', 86, 91), t2.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
-    assert_equal(Token.new('23', 111, 113), t2.next)
-    assert_equal(Token.new('áägç', 117, 124), t2.next)
-    assert_equal(Token.new('êëì', 126, 132), t2.next)
-    assert_equal(Token.new('úøã', 134, 140), t2.next)
-    assert_equal(Token.new('öîí', 142, 148), t2.next)
-    assert(! t2.next())
-    a = StandardAnalyzer.new(nil, false)
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new('DBalmán@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('Address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
-    assert_equal(Token.new('TNT', 86, 91), t.next)
-    assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
-    assert_equal(Token.new('23', 111, 113), t.next)
-    assert_equal(Token.new('ÁÄGÇ', 117, 124), t.next)
-    assert_equal(Token.new('ÊËÌ', 126, 132), t.next)
-    assert_equal(Token.new('ÚØÃ', 134, 140), t.next)
-    assert_equal(Token.new('ÖÎÍ', 142, 148), t.next)
-    assert(! t.next())
-    a = StandardAnalyzer.new(["e-mail", "23", "tnt"])
-    t = a.token_stream("fieldname", input)
-    t2 = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('and', 32, 35), t.next)
-    assert_equal(Token.new('the', 36, 39), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
-    assert_equal(Token.new('áägç', 117, 124), t.next)
-    assert_equal(Token.new('êëì', 126, 132), t.next)
-    assert_equal(Token.new('úøã', 134, 140), t.next)
-    assert_equal(Token.new('öîí', 142, 148), t.next)
-    assert(! t.next())
-    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t2.next)
-    assert_equal(Token.new('is', 19, 21), t2.next)
-    assert_equal(Token.new('my', 22, 24), t2.next)
-    assert_equal(Token.new('and', 32, 35), t2.next)
-    assert_equal(Token.new('the', 36, 39), t2.next)
-    assert_equal(Token.new('address', 40, 47), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
-    assert_equal(Token.new('áägç', 117, 124), t2.next)
-    assert_equal(Token.new('êëì', 126, 132), t2.next)
-    assert_equal(Token.new('úøã', 134, 140), t2.next)
-    assert_equal(Token.new('öîí', 142, 148), t2.next)
-    assert(! t2.next())
-  end
-end
-
-class PerFieldAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  def test_c_per_field_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$'
-    pfa = PerFieldAnalyzer.new(StandardAnalyzer.new())
-    pfa['white'] = WhiteSpaceAnalyzer.new(false)
-    pfa['white_l'] = WhiteSpaceAnalyzer.new(true)
-    pfa['letter'] = LetterAnalyzer.new(false)
-    pfa.add_field('letter', LetterAnalyzer.new(true))
-    pfa.add_field('letter_u', LetterAnalyzer.new(false))
-    t = pfa.token_stream('white', input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert(! t.next())
-    t = pfa.token_stream('white_l', input)
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('#$', 37, 39), t.next)
-    assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
-    assert(! t.next())
-    t = pfa.token_stream('letter_u', input)
-    assert_equal(Token.new('DBalmain', 0, 8), t.next)
-    assert_equal(Token.new('gmail', 9, 14), t.next)
-    assert_equal(Token.new('com', 15, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('e', 25, 26), t.next)
-    assert_equal(Token.new('mail', 27, 31), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert(! t.next())
-    t = pfa.token_stream('letter', input)
-    assert_equal(Token.new('dbalmain', 0, 8), t.next)
-    assert_equal(Token.new('gmail', 9, 14), t.next)
-    assert_equal(Token.new('com', 15, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e', 25, 26), t.next)
-    assert_equal(Token.new('mail', 27, 31), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert(! t.next())
-    t = pfa.token_stream('XXX', input) # should use default StandardAnalzyer
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert(! t.next())
-  end
-end
-
-class RegExpAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_reg_exp_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
-    a = RegExpAnalyzer.new()
-    t = a.token_stream('XXX', input)
-    t2 = a.token_stream('XXX', "one_Two three")
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
-    assert_equal(Token.new('t.n.t.', 91, 97), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
-    assert_equal(Token.new('23', 116, 118), t.next)
-    assert_equal(Token.new('rob\'s', 119, 124), t.next)
-    assert(! t.next())
-    t = t2
-    assert_equal(Token.new("one_two", 0, 7), t.next())
-    assert_equal(Token.new("three", 8, 13), t.next())
-    assert(! t.next())
-    a = RegExpAnalyzer.new(/\w{2,}/, false)
-    t = a.token_stream('XXX', input)
-    t2 = a.token_stream('XXX', "one Two three")
-    assert_equal(Token.new('DBalmain', 0, 8), t.next)
-    assert_equal(Token.new('gmail', 9, 14), t.next)
-    assert_equal(Token.new('com', 15, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('My', 22, 24), t.next)
-    assert_equal(Token.new('mail', 27, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('Address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('http', 55, 59), t.next)
-    assert_equal(Token.new('www', 62, 65), t.next)
-    assert_equal(Token.new('google', 66, 72), t.next)
-    assert_equal(Token.new('com', 73, 76), t.next)
-    assert_equal(Token.new('RESULT_3', 77, 85), t.next)
-    assert_equal(Token.new('html', 86, 90), t.next)
-    assert_equal(Token.new('123', 98, 101), t.next)
-    assert_equal(Token.new('1235', 102, 106), t.next)
-    assert_equal(Token.new('ASD', 107, 110), t.next)
-    assert_equal(Token.new('1234', 111, 115), t.next)
-    assert_equal(Token.new('23', 116, 118), t.next)
-    assert_equal(Token.new('Rob', 119, 122), t.next)
-    assert(! t.next())
-    assert_equal(Token.new("one", 0, 3), t2.next())
-    assert_equal(Token.new("Two", 4, 7), t2.next())
-    assert_equal(Token.new("three", 8, 13), t2.next())
-    assert(! t2.next())
-    a = RegExpAnalyzer.new() do |str|
-      if str =~ /^[[:alpha:]]\.([[:alpha:]]\.)+$/
-        str.gsub!(/\./, '')
-      elsif str =~ /'[sS]$/
-        str.gsub!(/'[sS]$/, '')
-      end
-      str
-    end
-    t = a.token_stream('XXX', input)
-    t2 = a.token_stream('XXX', "one's don't T.N.T.")
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('is', 19, 21), t.next)
-    assert_equal(Token.new('my', 22, 24), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('52', 32, 34), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
-    assert_equal(Token.new('tnt', 91, 97), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
-    assert_equal(Token.new('23', 116, 118), t.next)
-    assert_equal(Token.new('rob', 119, 124), t.next)
-    assert(! t.next())
-    assert_equal(Token.new("one", 0, 5), t2.next())
-    assert_equal(Token.new("don't", 6, 11), t2.next())
-    assert_equal(Token.new("tnt", 12, 18), t2.next())
-    assert(! t2.next())
-  end
-end
-
-module Ferret::Analysis
-  class StemmingStandardAnalyzer < StandardAnalyzer
-    def token_stream(field, text)
-      StemFilter.new(super)
-    end
-  end
-end
-
-class CustomAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_custom_filter()
-    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
-    a = StemmingStandardAnalyzer.new()
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
-    assert_equal(Token.new('e-mail', 25, 31), t.next)
-    assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/result', 55, 84), t.next)
-    assert_equal(Token.new('tnt', 86, 91), t.next)
-    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
-    assert_equal(Token.new('23', 111, 113), t.next)
-    assert_equal(Token.new('áägç', 117, 124), t.next)
-    assert_equal(Token.new('êëì', 126, 132), t.next)
-    assert_equal(Token.new('úøã', 134, 140), t.next)
-    assert_equal(Token.new('öîí', 142, 148), t.next)
-    assert(! t.next())
-    input = "Debate Debates DEBATED DEBating Debater";
-    t = a.token_stream("fieldname", input)
-    assert_equal(Token.new("debat", 0, 6), t.next)
-    assert_equal(Token.new("debat", 7, 14), t.next)
-    assert_equal(Token.new("debat", 15, 22), t.next)
-    assert_equal(Token.new("debat", 23, 31), t.next)
-    assert_equal(Token.new("debat", 32, 39), t.next)
-    assert(! t.next())
-    input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
-    t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
-    assert_equal(Token.new("dêbate", 0, 7), t.next)
-    assert_equal(Token.new("dêbate", 8, 16), t.next)
-    assert_equal(Token.new("dêbate", 17, 25), t.next)
-    assert_equal(Token.new("dêbate", 26, 35), t.next)
-    assert_equal(Token.new("dêbater", 36, 44), t.next)
-    assert(! t.next())
-  end
-end