ferret 0.9.6 → 0.10.0

data/lib/ferret/number_tools.rb

@@ -0,0 +1,157 @@
+require 'date'
+require 'time'
+
+class Float
+  # Return true if the float is within +precision+ of the other value +o+. This
+  # is used to accomodate for floating point errors.
+  #
+  # o::         value to compare with
+  # precision:: the precision to use in the comparison.
+  # return::    true if the match is within +precision+
+  def =~(o, precision = 0.0000000001)
+    return (1 - self/o).abs < precision
+  end
+end
+
+# Provides support for converting integers to Strings, and back again. The
+# strings are structured so that lexicographic sorting order is preserved.
+# 
+# That is, if integer1 is less than integer2 for any two integers integer1 and
+# integer2, then integer1.to_s_lex is lexicographically less than
+# integer2.to_s_lex. (Similarly for "greater than" and "equals".)
+#
+# This class handles numbers between - 10 ** 10,000 and 10 ** 10,000
+# which should cover all practical numbers. If you need bigger numbers,
+# increase Integer::LEN_STR_SIZE.
+class Integer
+  # LEN_SIZE of 4 should handle most numbers that can practically be held in
+  # memory.
+  LEN_STR_SIZE = 4
+  NEG_LEN_MASK = 10 ** LEN_STR_SIZE
+  LEN_STR_TEMPLATE = "%0#{LEN_STR_SIZE}d"
+
+  # Convert the number to a lexicographically sortable string. This string will
+  # use printable characters only but will not be human readable.
+  def to_s_lex
+    if (self >= 0)
+      num_str = self.to_s
+      len_str = LEN_STR_TEMPLATE % num_str.size
+      return len_str + num_str
+    else
+      num = self * -1
+      num_str = num.to_s
+      num_len = num_str.size
+      len_str = LEN_STR_TEMPLATE % (NEG_LEN_MASK - num_len)
+      num = (10 ** num_str.size) - num
+      return "-#{len_str}%0#{num_len}d" % num
+    end
+  end
+
+  # Convert the number to a lexicographically sortable string by padding with
+  # 0s. You should make sure that you set the width to a number large enough to
+  # accomodate all possible values. Also note that this method will not work
+  # with negative numbers. That is negative numbers will sort in the opposite
+  # direction as positive numbers. If you have very large numbers or a mix of
+  # positive and negative numbers you should use the Integer#to_s_lex method
+  #
+  # width::  number of characters in the string returned. Default is 10. So
+  #          123.to_s_pad(5) => 00123 and -123.to_s_pad(5) => -0123
+  # return:: padding string representation of the number.
+  def to_s_pad(width = 10)
+    "%#{width}d" % self
+  end
+end
+
+class Date
+  # Convert the Date to a lexicographically sortable string with the required
+  # precision. The format used is %Y%m%d
+  #
+  # precision:: the precision required in the string version of the date. The
+  #             options are :year, :month and :day
+  # return::    a lexicographically sortable string representing the date
+  def to_s_lex(precision = :day)
+    self.strftime(Time::LEX_FORMAT[precision])
+  end
+end
+
+class DateTime
+  # Convert the DateTime to a lexicographically sortable string with the
+  # required precision. The format used is %Y%m%d %H:%M:%S.
+  #
+  # precision:: the precision required in the string version of the date. The
+  #             options are :year, :month, :day, :hour, :minute and :second
+  # return::    a lexicographically sortable string representing the date
+  def to_s_lex(precision = :day)
+    self.strftime(Time::LEX_FORMAT[precision])
+  end
+end
+
+class Time
+  LEX_FORMAT = {
+      :year        => "%Y",
+      :month       => "%Y-%m",
+      :day         => "%Y-%m-%d",
+      :hour        => "%Y-%m-%d %H",
+      :minute      => "%Y-%m-%d %H:%M",
+      :second      => "%Y-%m-%d %H:%M:%S",
+      :millisecond => "%Y-%m-%d %H:%M:%S"
+  }
+
+  # Convert the Time to a lexicographically sortable string with the required
+  # precision. The format used is %Y%m%d %H:%M:%S.
+  #
+  # precision:: the precision required in the string version of the time. The
+  #             options are :year, :month, :day, :hour, :minute and :second
+  # return::    a lexicographically sortable string representing the date
+  def to_s_lex(precision = :day)
+    self.strftime(LEX_FORMAT[precision])
+  end
+end
+
+class String
+  # Convert a string to an integer. This method will only work on strings that
+  # were previously created with Integer#to_s_lex, otherwise the result will be
+  # unpredictable.
+  def to_i_lex
+    if (self[0] == ?-)
+      return self[(Integer::LEN_STR_SIZE + 1)..-1].to_i -
+        10 ** (self.size - Integer::LEN_STR_SIZE - 1)
+    else
+      return self[Integer::LEN_STR_SIZE..-1].to_i
+    end
+  end
+
+  # Convert a string to a Time. This method will only work on strings that
+  # match the format %Y%m%d %H%M%S, otherwise the result will be unpredictable.
+  def to_time_lex
+    vals = []
+    self.gsub(/(?:^|[- :])(\d+)/) {vals << $1.to_i; $&}
+    Time.mktime(*vals)
+  end
+
+  # Convert a string to a Date. This method will only work on strings that
+  # match the format %Y%m%d %H%M%S, otherwise the result will be unpredictable.
+  def to_date_lex
+    return Date.strptime(self + "-02-01", "%Y-%m-%d")
+  end
+
+  # Convert a string to a DateTime. This method will only work on strings that
+  # match the format %Y%m%d %H%M%S, otherwise the result will be unpredictable.
+  def to_date_time_lex
+    return DateTime.strptime(self + "-01-01", "%Y-%m-%d %H:%M:%S")
+  end
+
+  private
+  
+  def get_lex_format(len)
+    case len
+    when  0.. 3: ""
+    when  4.. 5: "%Y"
+    when  6.. 7: "%Y%m"
+    when  8.. 9: "%Y%m%d"
+    when 10..11: "%Y%m%d%H"
+    when 12..13: "%Y%m%d%H%M"
+    else "%Y%m%d%H%M%S"
+    end
+  end
+end

data/lib/ferret_version.rb

@@ -0,0 +1,3 @@
+module Ferret
+  VERSION = '0.9.4'
+end

data/test/test_helper.rb

@@ -9,20 +9,12 @@ class Float
 end
 
 require 'test/unit'
-require 'unit/index/th_doc'
-if $ferret_pure_ruby
-  require 'rferret'
-else
-  require 'ferret'
-end
+require 'ferret'
+require 'unit/index/th_doc' if (defined?(IndexTestHelper).nil?)
+
 
 def load_test_dir(dir)
-  dir = File.join(File.dirname(__FILE__), dir)
-  Dir.foreach(dir) do |file|
-    if $ferret_pure_ruby
-      require File.join(dir, file) if file =~ /^r?t[mcs]_.*\.rb$/
-    else
-      require File.join(dir, file) if file =~ /^c?t[mcs]_.*\.rb$/
-    end
+  Dir[File.join(File.dirname(__FILE__), dir, "t[scm]*.rb")].each do |file|
+    require file
   end
 end

data/test/unit/analysis/tc_analyzer.rb

@@ -3,10 +3,11 @@ require File.dirname(__FILE__) + "/../../test_helper"
 class AnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
-  def test_analyzer()
+  def test_c_analyzer()
     input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
     a = Analyzer.new()
     t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
     assert_equal(Token.new("dbalmain", 0, 8), t.next())
     assert_equal(Token.new("gmail", 9, 14), t.next())
     assert_equal(Token.new("com", 15, 18), t.next())
@@ -16,5 +17,516 @@ class AnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new("mail", 27, 31), t.next())
     assert_equal(Token.new("address", 39, 46), t.next())
     assert(! t.next())
+    assert_equal(Token.new("dbalmain", 0, 8), t2.next())
+    assert_equal(Token.new("gmail", 9, 14), t2.next())
+    assert_equal(Token.new("com", 15, 18), t2.next())
+    assert_equal(Token.new("is", 19, 21), t2.next())
+    assert_equal(Token.new("my", 22, 24), t2.next())
+    assert_equal(Token.new("e", 25, 26), t2.next())
+    assert_equal(Token.new("mail", 27, 31), t2.next())
+    assert_equal(Token.new("address", 39, 46), t2.next())
+    assert(! t2.next())
+    a = Analyzer.new(false)
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("DBalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("My", 22, 24), t.next())
+    assert_equal(Token.new("E", 25, 26), t.next())
+    assert_equal(Token.new("Mail", 27, 31), t.next())
+    assert_equal(Token.new("ADDRESS", 39, 46), t.next())
+    assert(! t.next())
+  end
+end
+
+class AsciiLetterAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_letter_analyzer()
+    input = 'DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$'
+    a = AsciiLetterAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new("dbalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("my", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("address", 39, 46), t.next())
+    assert(! t.next())
+    assert_equal(Token.new("dbalmain", 0, 8), t2.next())
+    assert_equal(Token.new("gmail", 9, 14), t2.next())
+    assert_equal(Token.new("com", 15, 18), t2.next())
+    assert_equal(Token.new("is", 19, 21), t2.next())
+    assert_equal(Token.new("my", 22, 24), t2.next())
+    assert_equal(Token.new("e", 25, 26), t2.next())
+    assert_equal(Token.new("mail", 27, 31), t2.next())
+    assert_equal(Token.new("address", 39, 46), t2.next())
+    assert(! t2.next())
+    a = AsciiLetterAnalyzer.new(false)
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("DBalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("My", 22, 24), t.next())
+    assert_equal(Token.new("E", 25, 26), t.next())
+    assert_equal(Token.new("Mail", 27, 31), t.next())
+    assert_equal(Token.new("ADDRESS", 39, 46), t.next())
+    assert(! t.next())
+  end
+end
+
+class LetterAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_letter_analyzer()
+    Ferret.locale = ""
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    a = LetterAnalyzer.new(false)
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new("DBalmän", 0, 8), t.next)
+    assert_equal(Token.new("gmail", 9, 14), t.next)
+    assert_equal(Token.new("com", 15, 18), t.next)
+    assert_equal(Token.new("is", 19, 21), t.next)
+    assert_equal(Token.new("My", 22, 24), t.next)
+    assert_equal(Token.new("e", 25, 26), t.next)
+    assert_equal(Token.new("mail", 27, 31), t.next)
+    assert_equal(Token.new("address", 40, 47), t.next)
+    assert_equal(Token.new("ÁÄGÇ", 55, 62), t.next)
+    assert_equal(Token.new("ÊËÌ", 64, 70), t.next)
+    assert_equal(Token.new("ÚØÃ", 72, 78), t.next)
+    assert_equal(Token.new("ÖÎÍ", 80, 86), t.next)
+    assert(! t.next())
+    assert_equal(Token.new("DBalmän", 0, 8), t2.next)
+    assert_equal(Token.new("gmail", 9, 14), t2.next)
+    assert_equal(Token.new("com", 15, 18), t2.next)
+    assert_equal(Token.new("is", 19, 21), t2.next)
+    assert_equal(Token.new("My", 22, 24), t2.next)
+    assert_equal(Token.new("e", 25, 26), t2.next)
+    assert_equal(Token.new("mail", 27, 31), t2.next)
+    assert_equal(Token.new("address", 40, 47), t2.next)
+    assert_equal(Token.new("ÁÄGÇ", 55, 62), t2.next)
+    assert_equal(Token.new("ÊËÌ", 64, 70), t2.next)
+    assert_equal(Token.new("ÚØÃ", 72, 78), t2.next)
+    assert_equal(Token.new("ÖÎÍ", 80, 86), t2.next)
+    assert(! t2.next())
+    a = LetterAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("dbalmän", 0, 8), t.next)
+    assert_equal(Token.new("gmail", 9, 14), t.next)
+    assert_equal(Token.new("com", 15, 18), t.next)
+    assert_equal(Token.new("is", 19, 21), t.next)
+    assert_equal(Token.new("my", 22, 24), t.next)
+    assert_equal(Token.new("e", 25, 26), t.next)
+    assert_equal(Token.new("mail", 27, 31), t.next)
+    assert_equal(Token.new("address", 40, 47), t.next)
+    assert_equal(Token.new("áägç", 55, 62), t.next)
+    assert_equal(Token.new("êëì", 64, 70), t.next)
+    assert_equal(Token.new("úøã", 72, 78), t.next)
+    assert_equal(Token.new("öîí", 80, 86), t.next)
+    assert(! t.next())
+  end
+end
+
+class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_white_space_analyzer()
+    input = 'DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#@$'
+    a = AsciiWhiteSpaceAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('E-Mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('ADDRESS.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t2.next)
+    assert_equal(Token.new('is', 19, 21), t2.next)
+    assert_equal(Token.new('My', 22, 24), t2.next)
+    assert_equal(Token.new('E-Mail', 25, 31), t2.next)
+    assert_equal(Token.new('52', 32, 34), t2.next)
+    assert_equal(Token.new('#$', 37, 39), t2.next)
+    assert_equal(Token.new('ADDRESS.', 40, 48), t2.next)
+    assert_equal(Token.new('23#@$', 49, 54), t2.next)
+    assert(! t2.next())
+    a = AsciiWhiteSpaceAnalyzer.new(true)
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('address.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert(! t.next())
+  end
+end
+
+class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_white_space_analyzer()
+    input = 'DBalmän@gmail.com is My e-mail 52   #$ address. 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    a = WhiteSpaceAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('address.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('DBalmän@gmail.com', 0, 18), t2.next)
+    assert_equal(Token.new('is', 19, 21), t2.next)
+    assert_equal(Token.new('My', 22, 24), t2.next)
+    assert_equal(Token.new('e-mail', 25, 31), t2.next)
+    assert_equal(Token.new('52', 32, 34), t2.next)
+    assert_equal(Token.new('#$', 37, 39), t2.next)
+    assert_equal(Token.new('address.', 40, 48), t2.next)
+    assert_equal(Token.new('23#@$', 49, 54), t2.next)
+    assert_equal(Token.new('ÁÄGÇ®ÊË̯ÚØìÖÎÍ', 55, 86), t2.next)
+    assert(! t2.next())
+    a = WhiteSpaceAnalyzer.new(true)
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmän@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('address.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
+    assert(! t.next())
+  end
+end
+
+class AsciiStandardAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_standard_analyzer()
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234'
+    a = AsciiStandardAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('tnt', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t2.next)
+    assert_equal(Token.new('e-mail', 25, 31), t2.next)
+    assert_equal(Token.new('52', 32, 34), t2.next)
+    assert_equal(Token.new('address', 40, 47), t2.next)
+    assert_equal(Token.new('23', 49, 51), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('tnt', 86, 91), t2.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
+    assert(! t2.next())
+    a = AsciiStandardAnalyzer.new(false)
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('Address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('TNT', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
+    assert(! t.next())
+  end
+end
+
+class StandardAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_c_standard_analyzer()
+    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    a = StandardAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('tnt', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
+    assert_equal(Token.new('23', 111, 113), t.next)
+    assert_equal(Token.new('áägç', 117, 124), t.next)
+    assert_equal(Token.new('êëì', 126, 132), t.next)
+    assert_equal(Token.new('úøã', 134, 140), t.next)
+    assert_equal(Token.new('öîí', 142, 148), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t2.next)
+    assert_equal(Token.new('e-mail', 25, 31), t2.next)
+    assert_equal(Token.new('address', 40, 47), t2.next)
+    assert_equal(Token.new('23', 49, 51), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('tnt', 86, 91), t2.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
+    assert_equal(Token.new('23', 111, 113), t2.next)
+    assert_equal(Token.new('áägç', 117, 124), t2.next)
+    assert_equal(Token.new('êëì', 126, 132), t2.next)
+    assert_equal(Token.new('úøã', 134, 140), t2.next)
+    assert_equal(Token.new('öîí', 142, 148), t2.next)
+    assert(! t2.next())
+    a = StandardAnalyzer.new(nil, false)
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new('DBalmán@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('Address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('TNT', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
+    assert_equal(Token.new('23', 111, 113), t.next)
+    assert_equal(Token.new('ÁÄGÇ', 117, 124), t.next)
+    assert_equal(Token.new('ÊËÌ', 126, 132), t.next)
+    assert_equal(Token.new('ÚØÃ', 134, 140), t.next)
+    assert_equal(Token.new('ÖÎÍ', 142, 148), t.next)
+    assert(! t.next())
+    a = StandardAnalyzer.new(["e-mail", "23", "tnt"])
+    t = a.token_stream("fieldname", input)
+    t2 = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('and', 32, 35), t.next)
+    assert_equal(Token.new('the', 36, 39), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
+    assert_equal(Token.new('áägç', 117, 124), t.next)
+    assert_equal(Token.new('êëì', 126, 132), t.next)
+    assert_equal(Token.new('úøã', 134, 140), t.next)
+    assert_equal(Token.new('öîí', 142, 148), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t2.next)
+    assert_equal(Token.new('is', 19, 21), t2.next)
+    assert_equal(Token.new('my', 22, 24), t2.next)
+    assert_equal(Token.new('and', 32, 35), t2.next)
+    assert_equal(Token.new('the', 36, 39), t2.next)
+    assert_equal(Token.new('address', 40, 47), t2.next)
+    assert_equal(Token.new('www.google.com/results', 55, 84), t2.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t2.next)
+    assert_equal(Token.new('áägç', 117, 124), t2.next)
+    assert_equal(Token.new('êëì', 126, 132), t2.next)
+    assert_equal(Token.new('úøã', 134, 140), t2.next)
+    assert_equal(Token.new('öîí', 142, 148), t2.next)
+    assert(! t2.next())
+  end
+end
+
+class PerFieldAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  def test_c_per_field_analyzer()
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$'
+    pfa = PerFieldAnalyzer.new(StandardAnalyzer.new())
+    pfa['white'] = WhiteSpaceAnalyzer.new(false)
+    pfa['white_l'] = WhiteSpaceAnalyzer.new(true)
+    pfa['letter'] = LetterAnalyzer.new(false)
+    pfa.add_field('letter', LetterAnalyzer.new(true))
+    pfa.add_field('letter_u', LetterAnalyzer.new(false))
+    t = pfa.token_stream('white', input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('address.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert(! t.next())
+    t = pfa.token_stream('white_l', input)
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('#$', 37, 39), t.next)
+    assert_equal(Token.new('address.', 40, 48), t.next)
+    assert_equal(Token.new('23#@$', 49, 54), t.next)
+    assert(! t.next())
+    t = pfa.token_stream('letter_u', input)
+    assert_equal(Token.new('DBalmain', 0, 8), t.next)
+    assert_equal(Token.new('gmail', 9, 14), t.next)
+    assert_equal(Token.new('com', 15, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e', 25, 26), t.next)
+    assert_equal(Token.new('mail', 27, 31), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert(! t.next())
+    t = pfa.token_stream('letter', input)
+    assert_equal(Token.new('dbalmain', 0, 8), t.next)
+    assert_equal(Token.new('gmail', 9, 14), t.next)
+    assert_equal(Token.new('com', 15, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e', 25, 26), t.next)
+    assert_equal(Token.new('mail', 27, 31), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert(! t.next())
+    t = pfa.token_stream('XXX', input) # should use default StandardAnalzyer
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert(! t.next())
+  end
+end
+
+class RegExpAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_reg_exp_analyzer()
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
+    a = RegExpAnalyzer.new()
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one_Two three")
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('t.n.t.', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob\'s', 119, 124), t.next)
+    assert(! t.next())
+    t = t2
+    assert_equal(Token.new("one_two", 0, 7), t.next())
+    assert_equal(Token.new("three", 8, 13), t.next())
+    assert(! t.next())
+    a = RegExpAnalyzer.new(/\w{2,}/, false)
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one Two three")
+    assert_equal(Token.new('DBalmain', 0, 8), t.next)
+    assert_equal(Token.new('gmail', 9, 14), t.next)
+    assert_equal(Token.new('com', 15, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('mail', 27, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('Address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http', 55, 59), t.next)
+    assert_equal(Token.new('www', 62, 65), t.next)
+    assert_equal(Token.new('google', 66, 72), t.next)
+    assert_equal(Token.new('com', 73, 76), t.next)
+    assert_equal(Token.new('RESULT_3', 77, 85), t.next)
+    assert_equal(Token.new('html', 86, 90), t.next)
+    assert_equal(Token.new('123', 98, 101), t.next)
+    assert_equal(Token.new('1235', 102, 106), t.next)
+    assert_equal(Token.new('ASD', 107, 110), t.next)
+    assert_equal(Token.new('1234', 111, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('Rob', 119, 122), t.next)
+    assert(! t.next())
+    assert_equal(Token.new("one", 0, 3), t2.next())
+    assert_equal(Token.new("Two", 4, 7), t2.next())
+    assert_equal(Token.new("three", 8, 13), t2.next())
+    assert(! t2.next())
+    a = RegExpAnalyzer.new() do |str|
+      if str =~ /^[[:alpha:]]\.([[:alpha:]]\.)+$/
+        str.gsub!(/\./, '')
+      elsif str =~ /'[sS]$/
+        str.gsub!(/'[sS]$/, '')
+      end
+      str
+    end
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one's don't T.N.T.")
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('tnt', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob', 119, 124), t.next)
+    assert(! t.next())
+    assert_equal(Token.new("one", 0, 5), t2.next())
+    assert_equal(Token.new("don't", 6, 11), t2.next())
+    assert_equal(Token.new("tnt", 12, 18), t2.next())
+    assert(! t2.next())
+  end
+end
+
+module Ferret::Analysis
+  class StemmingStandardAnalyzer < StandardAnalyzer
+    def token_stream(field, text)
+      StemFilter.new(super)
+    end
+  end
+end
+
+class CustomAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_custom_filter()
+    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    a = StemmingStandardAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/result', 55, 84), t.next)
+    assert_equal(Token.new('tnt', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
+    assert_equal(Token.new('23', 111, 113), t.next)
+    assert_equal(Token.new('áägç', 117, 124), t.next)
+    assert_equal(Token.new('êëì', 126, 132), t.next)
+    assert_equal(Token.new('úøã', 134, 140), t.next)
+    assert_equal(Token.new('öîí', 142, 148), t.next)
+    assert(! t.next())
+    input = "Debate Debates DEBATED DEBating Debater";
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("debat", 0, 6), t.next)
+    assert_equal(Token.new("debat", 7, 14), t.next)
+    assert_equal(Token.new("debat", 15, 22), t.next)
+    assert_equal(Token.new("debat", 23, 31), t.next)
+    assert_equal(Token.new("debat", 32, 39), t.next)
+    assert(! t.next())
+    input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
+    t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
+    assert_equal(Token.new("dêbate", 0, 7), t.next)
+    assert_equal(Token.new("dêbate", 8, 16), t.next)
+    assert_equal(Token.new("dêbate", 17, 25), t.next)
+    assert_equal(Token.new("dêbate", 26, 35), t.next)
+    assert_equal(Token.new("dêbater", 36, 44), t.next)
+    assert(! t.next())
   end
 end