ferret 0.11.6 → 0.11.8.4

data/lib/ferret/number_tools.rb

@@ -145,12 +145,12 @@ class String
   
   def get_lex_format(len)
     case len
-    when  0.. 3: ""
-    when  4.. 5: "%Y"
-    when  6.. 7: "%Y%m"
-    when  8.. 9: "%Y%m%d"
-    when 10..11: "%Y%m%d%H"
-    when 12..13: "%Y%m%d%H%M"
+    when  0.. 3 then ""
+    when  4.. 5 then "%Y"
+    when  6.. 7 then "%Y%m"
+    when  8.. 9 then "%Y%m%d"
+    when 10..11 then "%Y%m%d%H"
+    when 12..13 then "%Y%m%d%H%M"
     else "%Y%m%d%H%M%S"
     end
   end

data/lib/ferret/version.rb

@@ -0,0 +1,3 @@
+module Ferret
+  VERSION = '0.11.8.4'
+end

data/test/{unit → long_running}/largefile/tc_largefile.rb

@@ -25,7 +25,7 @@ class SampleLargeTest < Test::Unit::TestCase
   end
   
   def test_read_file_after_two_gigs
-    assert @index.reader[RECORDS - 5].load.is_a?Hash
+    assert @index.reader[RECORDS - 5].load.is_a?(Hash)
   end
   
   def create_index!

data/test/test_helper.rb

@@ -1,6 +1,11 @@
 $:.unshift File.dirname(__FILE__)
-$:.unshift File.join(File.dirname(__FILE__), '../lib')
-$:.unshift File.join(File.dirname(__FILE__), '../ext')
+if $test_installed_gem
+  require 'rubygems'
+  require 'ferret'
+else
+  $:.unshift File.join(File.dirname(__FILE__), '../lib')
+  $:.unshift File.join(File.dirname(__FILE__), '../ext')
+end
 
 ENV['LANG'] = "en_US.UTF-8"
 ENV['LC_CTYPE'] = "en_US.UTF-8"

data/test/test_installed.rb

@@ -0,0 +1 @@
+$test_installed_gem = true

data/test/threading/thread_safety_index_test.rb

@@ -1,3 +1,5 @@
+$:.unshift('.')
+require 'monitor'
 require File.dirname(__FILE__) + "/../test_helper"
 require File.dirname(__FILE__) + "/number_to_spoken.rb"
 require 'thread'
@@ -21,6 +23,7 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
   def indexing_thread()
     index = Index.new(:path => INDEX_DIR,
                       :analyzer => ANALYZER,
+                      :auto_flush => true,
                       :default_field => :content)
 
     ITERATIONS.times do
@@ -37,6 +40,10 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
       end
       index.commit
     end
+  rescue Exception => e
+    puts e
+    puts e.backtrace
+    raise 'hell'
   end 
 
   def do_optimize(index)
@@ -74,6 +81,8 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
       threads << Thread.new { indexing_thread }
     end
 
-    threads.each {|t| t.join}
+    threads.each {|t| 
+      t.join
+    }
   end
 end

data/test/threading/thread_safety_read_write_test.rb

@@ -1,20 +1,19 @@
 require File.dirname(__FILE__) + "/../test_helper"
-require File.dirname(__FILE__) + "/../utils/number_to_spoken.rb"
+require File.dirname(__FILE__) + "/number_to_spoken.rb"
 require 'thread'
 
 class IndexThreadSafetyReadWriteTest < Test::Unit::TestCase
   include Ferret::Index
-  include Ferret::Document
 
   INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
   ITERATIONS = 10000
   ANALYZER = Ferret::Analysis::Analyzer.new()
 
   def setup
-    @index = Index.new(:path => 'index2',
+    @index = Index.new(:path => INDEX_DIR,
                        :create => true,
                        :analyzer => ANALYZER,
-                       :default_field => 'contents')
+                       :default_field => :content)
   end
 
   def search_thread()
@@ -42,10 +41,8 @@ class IndexThreadSafetyReadWriteTest < Test::Unit::TestCase
   end 
 
   def do_add_doc
-    d = Document.new()
     n = rand(0xFFFFFFFF)
-    d << Field.new("id", n.to_s, Field::Store::YES, Field::Index::UNTOKENIZED)
-    d << Field.new("contents", n.to_spoken, Field::Store::NO, Field::Index::TOKENIZED)
+    d = {:id => n.to_s, :content => n.to_spoken}
     puts("Adding #{n}")
     begin
       @index << d

data/test/unit/analysis/tc_analyzer.rb

@@ -1,10 +1,12 @@
+# encoding: utf-8
+
 require File.dirname(__FILE__) + "/../../test_helper"
 
 class AnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
+    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#!$'
     a = Analyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -44,7 +46,7 @@ class AsciiLetterAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_letter_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
+    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#!$'
     a = AsciiLetterAnalyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -85,7 +87,7 @@ class LetterAnalyzerTest < Test::Unit::TestCase
 
   def test_letter_analyzer()
     Ferret.locale = ""
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     a = LetterAnalyzer.new(false)
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -137,7 +139,7 @@ class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_white_space_analyzer()
-    input = 'DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#@$'
+    input = 'DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#!$'
     a = AsciiWhiteSpaceAnalyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -148,7 +150,7 @@ class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('ADDRESS.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t2.next)
     assert_equal(Token.new('is', 19, 21), t2.next)
@@ -157,7 +159,7 @@ class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t2.next)
     assert_equal(Token.new('#$', 37, 39), t2.next)
     assert_equal(Token.new('ADDRESS.', 40, 48), t2.next)
-    assert_equal(Token.new('23#@$', 49, 54), t2.next)
+    assert_equal(Token.new('23#!$', 49, 54), t2.next)
     assert(! t2.next())
     a = AsciiWhiteSpaceAnalyzer.new(true)
     t = a.token_stream("fieldname", input)
@@ -168,7 +170,7 @@ class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
   end
 end
@@ -177,7 +179,7 @@ class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_white_space_analyzer()
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     a = WhiteSpaceAnalyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -188,7 +190,7 @@ class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t.next)
     assert(! t.next())
     assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t2.next)
@@ -198,7 +200,7 @@ class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t2.next)
     assert_equal(Token.new('#$', 37, 39), t2.next)
     assert_equal(Token.new('address.', 40, 48), t2.next)
-    assert_equal(Token.new('23#@$', 49, 54), t2.next)
+    assert_equal(Token.new('23#!$', 49, 54), t2.next)
     assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t2.next)
     assert(! t2.next())
     a = WhiteSpaceAnalyzer.new(true)
@@ -210,7 +212,7 @@ class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
     assert(! t.next())
   end
@@ -220,7 +222,7 @@ class AsciiStandardAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_standard_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#!$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
     a = AsciiStandardAnalyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -231,7 +233,7 @@ class AsciiStandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert(! t.next())
@@ -242,7 +244,7 @@ class AsciiStandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t2.next)
     assert_equal(Token.new('address', 40, 47), t2.next)
     assert_equal(Token.new('23', 49, 51), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t2.next)
     assert_equal(Token.new('tnt', 86, 91), t2.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
     assert(! t2.next())
@@ -257,7 +259,7 @@ class AsciiStandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('Address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('TNT', 86, 91), t.next)
     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
     assert(! t.next())
@@ -268,7 +270,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_standard_analyzer()
-    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#!$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     a = StandardAnalyzer.new()
     t = a.token_stream("fieldname", input)
     t2 = a.token_stream("fieldname", input)
@@ -278,7 +280,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('mail', 27, 31), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -293,7 +295,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('mail', 27, 31), t2.next)
     assert_equal(Token.new('address', 40, 47), t2.next)
     assert_equal(Token.new('23', 49, 51), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t2.next)
     assert_equal(Token.new('tnt', 86, 91), t2.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
     assert_equal(Token.new('23', 111, 113), t2.next)
@@ -311,7 +313,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('mail', 27, 31), t.next)
     assert_equal(Token.new('Address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('TNT', 86, 91), t.next)
     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -329,7 +331,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('and', 32, 35), t.next)
     assert_equal(Token.new('the', 36, 39), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert_equal(Token.new('áägç', 117, 124), t.next)
     assert_equal(Token.new('êëì', 126, 132), t.next)
@@ -342,7 +344,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('and', 32, 35), t2.next)
     assert_equal(Token.new('the', 36, 39), t2.next)
     assert_equal(Token.new('address', 40, 47), t2.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t2.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
     assert_equal(Token.new('áägç', 117, 124), t2.next)
     assert_equal(Token.new('êëì', 126, 132), t2.next)
@@ -355,7 +357,7 @@ end if (/utf-8/i =~ Ferret.locale)
 class PerFieldAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
   def test_per_field_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ address. 23#!$'
     pfa = PerFieldAnalyzer.new(StandardAnalyzer.new())
     pfa['white'] = WhiteSpaceAnalyzer.new(false)
     pfa['white_l'] = WhiteSpaceAnalyzer.new(true)
@@ -370,7 +372,7 @@ class PerFieldAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
     t = pfa.token_stream('white_l', input)
     assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
@@ -380,7 +382,7 @@ class PerFieldAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
     t = pfa.token_stream('letter_u', input)
     assert_equal(Token.new('DBalmain', 0, 8), t.next)
@@ -418,7 +420,7 @@ class RegExpAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_reg_exp_analyzer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#!$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
     a = RegExpAnalyzer.new()
     t = a.token_stream('XXX', input)
     t2 = a.token_stream('XXX', "one_Two three")
@@ -510,7 +512,7 @@ class CustomAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_custom_filter()
-    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#!$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     a = StemmingStandardAnalyzer.new()
     t = a.token_stream("fieldname", input)
     assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
@@ -519,7 +521,7 @@ class CustomAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('mail', 27, 31), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/result', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/result', 55, 85), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)

data/test/unit/analysis/tc_token_stream.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.dirname(__FILE__) + "/../../test_helper"
 
 puts "Loading once"
@@ -27,7 +29,7 @@ class AsciiLetterTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_letter_tokenizer()
-    input = 'DBalmain@gmail.com is My e-mail 523@#$ ADDRESS. 23#@$'
+    input = 'DBalmain@gmail.com is My e-mail 523@#$ ADDRESS. 23#!$'
     t = AsciiLetterTokenizer.new(input)
     assert_equal(Token.new("DBalmain", 0, 8), t.next())
     assert_equal(Token.new("gmail", 9, 14), t.next())
@@ -60,7 +62,7 @@ class LetterTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_letter_tokenizer()
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     t = LetterTokenizer.new(input)
     assert_equal(Token.new('DBalmän', 0, 8), t.next)
     assert_equal(Token.new('gmail', 9, 14), t.next)
@@ -115,7 +117,7 @@ class AsciiWhiteSpaceTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_whitespace_tokenizer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ ADDRESS. 23#@$'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ ADDRESS. 23#!$'
     t = AsciiWhiteSpaceTokenizer.new(input)
     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -124,7 +126,7 @@ class AsciiWhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('ADDRESS.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
     t.text = "one_two three"
     assert_equal(Token.new("one_two", 0, 7), t.next())
@@ -138,7 +140,7 @@ class AsciiWhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert(! t.next())
   end
 end
@@ -147,7 +149,7 @@ class WhiteSpaceTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_whitespace_tokenizer()
-    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     t = WhiteSpaceTokenizer.new(input)
     assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -156,7 +158,7 @@ class WhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t.next)
     assert(! t.next())
     t.text = "one_two three"
@@ -171,7 +173,7 @@ class WhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
     assert(! t.next())
     t = WhiteSpaceTokenizer.new(input, true)
@@ -182,7 +184,7 @@ class WhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('#$', 37, 39), t.next)
     assert_equal(Token.new('address.', 40, 48), t.next)
-    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('23#!$', 49, 54), t.next)
     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
     assert(! t.next())
   end
@@ -192,7 +194,7 @@ class AsciiStandardTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_standard_tokenizer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#!$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
     t = AsciiStandardTokenizer.new(input)
     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -201,7 +203,7 @@ class AsciiStandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('Address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('TNT', 86, 91), t.next)
     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
     assert(! t.next())
@@ -217,7 +219,7 @@ class AsciiStandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 85), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert(! t.next())
@@ -228,7 +230,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_standard_tokenizer()
-    input = 'DBalmán@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/res_345/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmán@gmail.com is My e-mail 52   #$ Address. 23#!$ http://www.google.com/res_345/ T.N.T. 123-1235-ASD-1234 23#!$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     t = StandardTokenizer.new(input)
     assert_equal(Token.new('DBalmán@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -237,7 +239,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('Address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/res_345', 55, 85), t.next)
     assert_equal(Token.new('TNT', 86, 91), t.next)
     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -258,7 +260,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/res_345', 55, 85), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -287,7 +289,7 @@ class RegExpTokenizerTest < Test::Unit::TestCase
   APOSTROPHE_WORD = /^#{APOSTROPHE}$/
 
   def test_reg_exp_tokenizer()
-    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#!$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
     t = RegExpTokenizer.new(input)
     assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -483,6 +485,11 @@ class StemFilterTest < Test::Unit::TestCase
       assert_equal(Token.new("dêbater", 36, 44), t.next)
       assert(! t.next())
     end
+
+    tz = AsciiLetterTokenizer.new(input)
+    assert_not_nil(StemFilter.new(tz,'HunGarIaN', 'Utf-8'))
+    assert_not_nil(StemFilter.new(tz,'romanIAN', 'iso-8859-2'))
+    assert_raises(ArgumentError) {StemFilter.new(tz, 'Jibberish', 'UTF-8')}
   end
 end