ferret 0.9.1 → 0.9.2

data/lib/ferret/search/phrase_query.rb

@@ -173,7 +173,7 @@ module Ferret::Search
 
     # See Query#extract_terms()
     def extract_terms(query_terms) 
-      query_terms.add_all(@terms)
+      query_terms.merge(@terms)
     end
 
     # Prints a user-readable version of this query. 

data/lib/ferret/search/query.rb

@@ -62,15 +62,44 @@ module Ferret::Search
 
     # Expert: called when re-writing queries under MultiSearcher.
     # 
-    # Only implemented by derived queries, with no #create_weight()
-    # implementatation.
+    # Create a single query suitable for use by all subsearchers (in 1-1
+    # correspondence with queries). This is an optimization of the OR of
+    # all queries. We handle the common optimization cases of equal
+    # queries and overlapping clauses of boolean OR queries (as generated
+    # by MultiTermQuery.rewrite() and RangeQuery.rewrite()).
+    # Be careful overriding this method as queries[0] determines which
+    # method will be called and is not necessarily of the same type as
+    # the other queries.
     def combine(queries) 
+      uniques = Set.new
       queries.each do |query|
-        if self != query
-          raise ArgumentError
+        clauses = []
+        # check if we can split the query into clauses
+        splittable = query.respond_to? :clauses
+        if splittable
+          splittable = query.coord_disabled?
+          clauses = query.clauses
+          clauses.each do |clause|
+            splittable = clause.occur == BooleanClause::Occur::SHOULD
+            break unless splittable
+          end
+        end
+        if splittable
+          clauses.each { |clause| uniques << clause.query }
+        else
+          uniques << query
         end
       end
-      return self
+      # optimization: if we have just one query, just return it
+      if uniques.size == 1
+        uniques.each { |query| return query }
+      end
+      
+      result = BooleanQuery.new(true)
+      uniques.each do |query|
+        result.add_query(query, BooleanClause::Occur::SHOULD)
+      end
+      return result
     end
 
     # Expert: adds all terms occuring in this query to the terms set

data/lib/ferret/search/sort.rb

@@ -84,8 +84,12 @@ module Ferret::Search
       fields = fields.map {|field| field.is_a?(Symbol) ? field.to_s : field}
       if fields[0].is_a?(String)
         @fields = fields.map do |field|
-          SortField.new(field, {:sort_type => SortField::SortType::AUTO,
-                                :reverse => reverse})
+          if (field.is_a?(String))
+            next SortField.new(field, {:sort_type => SortField::SortType::AUTO,
+                                       :reverse => reverse})
+          else
+            next field
+          end
         end
       end
       doc_sort_added = false
@@ -102,7 +106,7 @@ module Ferret::Search
     INDEX_ORDER = Sort.new(SortField::FIELD_DOC)
 
     def to_s() 
-      return @fields.map {|field| "#{field}"}.join(", ")
+      return "Sort[" + @fields.map {|field| "#{field}"}.join(", ") + "]"
     end
   end
 end

data/lib/ferret/search/sort_field.rb

@@ -20,11 +20,11 @@ module Ferret::Search
 
       # Sort by document score (relevancy).  Sort values are Float and higher
       # values are at the front. 
-      SCORE = SortType.new("score")
+      SCORE = SortType.new("SCORE")
 
       # Sort by document number (order).  Sort values are Integer and lower
       # values are at the front. 
-      DOC = SortType.new("doc")
+      DOC = SortType.new("DOC")
 
       # Guess sort type of sort based on field contents. We try parsing the
       # field as an integer and then as a floating point number. If we are
@@ -37,7 +37,7 @@ module Ferret::Search
 
       # Sort using term values as encoded Integers.  Sort values are Integer
       # and lower values are at the front. 
-      INTEGER = SortType.new("int", lambda{|str| str.to_i})
+      INTEGER = SortType.new("integer", lambda{|str| str.to_i})
 
       # Sort using term values as encoded Floats.  Sort values are Float and
       # lower values are at the front. 
@@ -79,7 +79,11 @@ module Ferret::Search
     FIELD_DOC = SortField.new(nil, {:sort_type => SortType::DOC})
 
     def to_s() 
-      buffer = '"' + (@name||"<#{@sort_type}>") + '"'
+      if @name
+        buffer = "#@name:<#@sort_type>"
+      else
+        buffer = "<#{@sort_type}>"
+      end
       buffer << '!' if @reverse
       return buffer
     end

data/lib/ferret/store/fs_store.rb

@@ -102,13 +102,13 @@ module Ferret::Store
         # delete all the files
         refresh_dir
         each do |fname|
-          File.delete(dir_path(fname))
+          FileUtils.rm_rf(dir_path(fname))
         end
         # clear all the locks
         refresh_lock_dir
         @lock_dir.each do |lock_fname|
           next if lock_fname == '.' or lock_fname == '..'
-          File.delete(@lock_dir.path + '/' + lock_fname)
+          FileUtils.rm_rf(@lock_dir.path + '/' + lock_fname)
         end
       end
     end
@@ -159,7 +159,13 @@ module Ferret::Store
     # This replacement should be atomic.
     def rename(from, to)
       synchronize do
-        File.rename(dir_path(from), dir_path(to))
+        begin
+          File.rename(dir_path(from), dir_path(to))
+        rescue
+          # try again, this time forcing the delete
+          FileUtils.rm_rf(dir_path(to))
+          FileUtils.cp(dir_path(from), dir_path(to))
+        end
       end
     end
 
@@ -208,11 +214,11 @@ module Ferret::Store
       def initialize(lock_file)
         @lock_file = lock_file
         #@clean = FSLock.make_finalizer(lock_file)
-        @clean = lambda { File.delete(lock_file) rescue nil}
+        @clean = lambda { FileUtils.rm_rf(lock_file)}
       end
 
       def FSLock.make_finalizer(lock_file)
-        lambda { File.delete(lock_file) rescue nil}
+        lambda { FileUtils.rm_rf(lock_file)}
       end
 
       # obtain the lock on the data source 
@@ -238,7 +244,7 @@ module Ferret::Store
       def release 
         return if FSDirectory.locks_disabled?
         begin
-          File.delete(@lock_file)
+          FileUtils.rm_rf(@lock_file)
           ObjectSpace.undefine_finalizer(self)
         rescue SystemCallError
           # maybe we tried to release a lock that wasn't locked. This
@@ -364,6 +370,7 @@ module Ferret::Store
       # This method is only used by the c extension to free the directory
       def close_internal
       end
+
     #end private
   end
 end

data/lib/ferret/store/index_io.rb

@@ -213,20 +213,6 @@ module Ferret::Store
       last = start + length
       (start ... last).each do |i|
         write_byte(buf[i])
-#          code = buf[i]
-#          if code >= 0x01 and code <= 0x7F
-#            write_byte(code)
-#          else
-#            # We need to write unicode characters. ToDo: test that this works.
-#            if code > 0x80 and code <= 0x7FF or code == 0
-#              write_byte(0xC0 | code >> 6)
-#              write_byte(0x80 | code & 0x3F)
-#            else
-#              write_byte(0xE0 | (code >> 12))
-#              write_byte(0x80 | ((code >> 6) & 0x3F))
-#              write_byte(0x80 | (code & 0x3F))
-#            end
-#          end
       end
     end
 

data/lib/ferret/store/ram_store.rb

@@ -159,9 +159,10 @@ module Ferret::Store
         flush()
         last_buffer_number = (@file.length / BUFFER_SIZE).to_i
         last_buffer_offset = @file.length % BUFFER_SIZE
-        @file.buffers.each_with_index do |buffer, i|
+
+        (0..last_buffer_number).each do |i|
           len = (i == last_buffer_number ? last_buffer_offset : BUFFER_SIZE)
-          output.write_bytes(buffer, len)
+          output.write_bytes(@file.buffers[i], len)
         end
       end
       

data/lib/rferret.rb

@@ -23,7 +23,7 @@ $: << File.dirname(__FILE__)
 #++
 # :include: ../TUTORIAL
 module Ferret
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
 end
 
 $ferret_pure_ruby = true

data/test/unit/analysis/ctc_analyzer.rb

@@ -399,3 +399,134 @@ class PerFieldAnalyzerTest < Test::Unit::TestCase
     assert(! t.next())
   end
 end
+
+class RegExpAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_reg_exp_analyzer()
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
+    a = RegExpAnalyzer.new()
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one_Two three")
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('t.n.t.', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob\'s', 119, 124), t.next)
+    assert(! t.next())
+    t = t2
+    assert_equal(Token.new("one_two", 0, 7), t.next())
+    assert_equal(Token.new("three", 8, 13), t.next())
+    assert(! t.next())
+    a = RegExpAnalyzer.new(/\w{2,}/, false)
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one Two three")
+    assert_equal(Token.new('DBalmain', 0, 8), t.next)
+    assert_equal(Token.new('gmail', 9, 14), t.next)
+    assert_equal(Token.new('com', 15, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('mail', 27, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('Address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http', 55, 59), t.next)
+    assert_equal(Token.new('www', 62, 65), t.next)
+    assert_equal(Token.new('google', 66, 72), t.next)
+    assert_equal(Token.new('com', 73, 76), t.next)
+    assert_equal(Token.new('RESULT_3', 77, 85), t.next)
+    assert_equal(Token.new('html', 86, 90), t.next)
+    assert_equal(Token.new('123', 98, 101), t.next)
+    assert_equal(Token.new('1235', 102, 106), t.next)
+    assert_equal(Token.new('ASD', 107, 110), t.next)
+    assert_equal(Token.new('1234', 111, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('Rob', 119, 122), t.next)
+    assert(! t.next())
+    assert_equal(Token.new("one", 0, 3), t2.next())
+    assert_equal(Token.new("Two", 4, 7), t2.next())
+    assert_equal(Token.new("three", 8, 13), t2.next())
+    assert(! t2.next())
+    a = RegExpAnalyzer.new() do |str|
+      if str =~ /^[[:alpha:]]\.([[:alpha:]]\.)+$/
+        str.gsub!(/\./, '')
+      elsif str =~ /'[sS]$/
+        str.gsub!(/'[sS]$/, '')
+      end
+      str
+    end
+    t = a.token_stream('XXX', input)
+    t2 = a.token_stream('XXX', "one's don't T.N.T.")
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('tnt', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob', 119, 124), t.next)
+    assert(! t.next())
+    assert_equal(Token.new("one", 0, 5), t2.next())
+    assert_equal(Token.new("don't", 6, 11), t2.next())
+    assert_equal(Token.new("tnt", 12, 18), t2.next())
+    assert(! t2.next())
+  end
+end
+
+module Ferret::Analysis
+  class StemmingStandardAnalyzer < StandardAnalyzer
+    def token_stream(field, text)
+      StemFilter.new(super)
+    end
+  end
+end
+
+class CustomAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_custom_filter()
+    input = 'DBalmán@gmail.com is My e-mail and the Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    a = StemmingStandardAnalyzer.new()
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new('dbalmán@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('www.google.com/result', 55, 84), t.next)
+    assert_equal(Token.new('tnt', 86, 91), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
+    assert_equal(Token.new('23', 111, 113), t.next)
+    assert_equal(Token.new('áägç', 117, 124), t.next)
+    assert_equal(Token.new('êëì', 126, 132), t.next)
+    assert_equal(Token.new('úøã', 134, 140), t.next)
+    assert_equal(Token.new('öîí', 142, 148), t.next)
+    assert(! t.next())
+    input = "Debate Debates DEBATED DEBating Debater";
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("debat", 0, 6), t.next)
+    assert_equal(Token.new("debat", 7, 14), t.next)
+    assert_equal(Token.new("debat", 15, 22), t.next)
+    assert_equal(Token.new("debat", 23, 31), t.next)
+    assert_equal(Token.new("debat", 32, 39), t.next)
+    assert(! t.next())
+    input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
+    t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
+    assert_equal(Token.new("dêbate", 0, 7), t.next)
+    assert_equal(Token.new("dêbate", 8, 16), t.next)
+    assert_equal(Token.new("dêbate", 17, 25), t.next)
+    assert_equal(Token.new("dêbate", 26, 35), t.next)
+    assert_equal(Token.new("dêbater", 36, 44), t.next)
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/ctc_tokenstream.rb

@@ -205,7 +205,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
 
   def test_standard_tokenizer()
-    input = 'DBalmán@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/results/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
+    input = 'DBalmán@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/res_345/ T.N.T. 123-1235-ASD-1234 23#@$ ÁÄGÇ®ÊË̯ÚØìÖÎÍ'
     t = StandardTokenizer.new(input)
     assert_equal(Token.new('DBalmán@gmail.com', 0, 18), t.next)
     assert_equal(Token.new('is', 19, 21), t.next)
@@ -214,7 +214,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('Address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
     assert_equal(Token.new('TNT', 86, 91), t.next)
     assert_equal(Token.new('123-1235-ASD-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -235,7 +235,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('52', 32, 34), t.next)
     assert_equal(Token.new('address', 40, 47), t.next)
     assert_equal(Token.new('23', 49, 51), t.next)
-    assert_equal(Token.new('www.google.com/results', 55, 84), t.next)
+    assert_equal(Token.new('www.google.com/res_345', 55, 84), t.next)
     assert_equal(Token.new('tnt', 86, 91), t.next)
     assert_equal(Token.new('123-1235-asd-1234', 93, 110), t.next)
     assert_equal(Token.new('23', 111, 113), t.next)
@@ -247,6 +247,97 @@ class StandardTokenizerTest < Test::Unit::TestCase
   end
 end
 
+class RegExpTokenizerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  ALPHA      = /[[:alpha:]_-]+/
+  APOSTROPHE = /#{ALPHA}('#{ALPHA})+/
+  ACRONYM    = /#{ALPHA}\.(#{ALPHA}\.)+/
+  ACRONYM_WORD    = /^#{ACRONYM}$/
+  APOSTROPHE_WORD = /^#{APOSTROPHE}$/
+
+  def test_reg_exp_tokenizer()
+    input = 'DBalmain@gmail.com is My e-mail 52   #$ Address. 23#@$ http://www.google.com/RESULT_3.html T.N.T. 123-1235-ASD-1234 23 Rob\'s'
+    t = RegExpTokenizer.new(input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('My', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('Address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/RESULT_3.html', 55, 90), t.next)
+    assert_equal(Token.new('T.N.T.', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-ASD-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('Rob\'s', 119, 124), t.next)
+    assert(! t.next())
+    t.text = "one_two three"
+    assert_equal(Token.new("one_two", 0, 7), t.next())
+    assert_equal(Token.new("three", 8, 13), t.next())
+    assert(! t.next())
+    t = LowerCaseFilter.new(RegExpTokenizer.new(input))
+    t2 = LowerCaseFilter.new(RegExpTokenizer.new(input, /\w{2,}/))
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('t.n.t.', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob\'s', 119, 124), t.next)
+    assert(! t.next())
+    assert_equal(Token.new('dbalmain', 0, 8), t2.next)
+    assert_equal(Token.new('gmail', 9, 14), t2.next)
+    assert_equal(Token.new('com', 15, 18), t2.next)
+    assert_equal(Token.new('is', 19, 21), t2.next)
+    assert_equal(Token.new('my', 22, 24), t2.next)
+    assert_equal(Token.new('mail', 27, 31), t2.next)
+    assert_equal(Token.new('52', 32, 34), t2.next)
+    assert_equal(Token.new('address', 40, 47), t2.next)
+    assert_equal(Token.new('23', 49, 51), t2.next)
+    assert_equal(Token.new('http', 55, 59), t2.next)
+    assert_equal(Token.new('www', 62, 65), t2.next)
+    assert_equal(Token.new('google', 66, 72), t2.next)
+    assert_equal(Token.new('com', 73, 76), t2.next)
+    assert_equal(Token.new('result_3', 77, 85), t2.next)
+    assert_equal(Token.new('html', 86, 90), t2.next)
+    assert_equal(Token.new('123', 98, 101), t2.next)
+    assert_equal(Token.new('1235', 102, 106), t2.next)
+    assert_equal(Token.new('asd', 107, 110), t2.next)
+    assert_equal(Token.new('1234', 111, 115), t2.next)
+    assert_equal(Token.new('23', 116, 118), t2.next)
+    assert_equal(Token.new('rob', 119, 122), t2.next)
+    assert(! t2.next())
+    t = RegExpTokenizer.new(input) do |str|
+      if str =~ ACRONYM_WORD
+        str.gsub!(/\./, '')
+      elsif str =~ APOSTROPHE_WORD
+        str.gsub!(/'[sS]$/, '')
+      end
+      str
+    end
+    t = LowerCaseFilter.new(t)
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next)
+    assert_equal(Token.new('is', 19, 21), t.next)
+    assert_equal(Token.new('my', 22, 24), t.next)
+    assert_equal(Token.new('e-mail', 25, 31), t.next)
+    assert_equal(Token.new('52', 32, 34), t.next)
+    assert_equal(Token.new('address', 40, 47), t.next)
+    assert_equal(Token.new('23', 49, 51), t.next)
+    assert_equal(Token.new('http://www.google.com/result_3.html', 55, 90), t.next)
+    assert_equal(Token.new('tnt', 91, 97), t.next)
+    assert_equal(Token.new('123-1235-asd-1234', 98, 115), t.next)
+    assert_equal(Token.new('23', 116, 118), t.next)
+    assert_equal(Token.new('rob', 119, 124), t.next)
+    assert(! t.next())
+  end
+end
+
 class StopFilterTest < Test::Unit::TestCase
   include Ferret::Analysis
 
@@ -383,11 +474,9 @@ module Ferret::Analysis
     def next()
       t = @input.next()
 
-      if (t == nil)
-        return nil
-      end
+      return nil if (t.nil?)
 
-      t.text = t.text[0,1].upcase + t.text[1..-1]
+      t.text = t.text.capitalize
 
       return t
     end
@@ -402,7 +491,7 @@ class CustomFilterTest < Test::Unit::TestCase
     t = CapitalizeFilter.new(AsciiLetterTokenizer.new(input))
     assert_equal(Token.new("This", 0, 4), t.next)
     assert_equal(Token.new("Text", 5, 9), t.next)
-    assert_equal(Token.new("SHOULD", 10, 16), t.next)
+    assert_equal(Token.new("Should", 10, 16), t.next)
     assert_equal(Token.new("Be", 17, 19), t.next)
     assert_equal(Token.new("Capitalized", 20, 31), t.next)
     assert_equal(Token.new("I", 36, 37), t.next)
@@ -412,7 +501,7 @@ class CustomFilterTest < Test::Unit::TestCase
     t = StemFilter.new(CapitalizeFilter.new(AsciiLetterTokenizer.new(input)))
     assert_equal(Token.new("This", 0, 4), t.next)
     assert_equal(Token.new("Text", 5, 9), t.next)
-    assert_equal(Token.new("SHOULD", 10, 16), t.next)
+    assert_equal(Token.new("Should", 10, 16), t.next)
     assert_equal(Token.new("Be", 17, 19), t.next)
     assert_equal(Token.new("Capit", 20, 31), t.next)
     assert_equal(Token.new("I", 36, 37), t.next)