wirb 1.0.3 → 2.0.0

data/lib/wirb/version.rb

@@ -1,3 +1,3 @@
 module Wirb
-  VERSION = '1.0.3'
+  VERSION = '2.0.0'.freeze
 end

data/lib/wirb/wp.rb

@@ -1,13 +1,9 @@
-require File.dirname(__FILE__) + '/../wirb' unless defined? Wirb
+require_relative '../wirb'
 
 module Kernel
   private
 
-  def wp(object, color = nil)
-    if color
-      puts Wirb.colorize_string object.inspect, color
-    else
-      puts Wirb.colorize_result object.inspect
-    end
+  def wp(object)
+    puts Wirb.colorize_result object.inspect
   end
 end

data/spec/spec_helper.rb

@@ -1,19 +1,16 @@
 require 'wirb'
 require 'wirb/wp'
-require 'zucker/engine'
-require 'zucker/ruby_version'
-
-Wirb.start
+require 'ruby_engine'
 
 # TOKENIZER
 
 def tokenizer(filename)
   filename =~ /tokenizer_(.*)_spec\.rb/
-  "Wirb.tokenize" + ($1 ? "(#{$1})" : "")
+  "Wirb::Tokenizer.run" + ($1 ? "(#{$1})" : "")
 end
 
 def check_value
-  Wirb.tokenize(@a).map{|_,c|c}.join.should == @a
+  Wirb::Tokenizer.run(@a).map{|_,c|c}.join.should == @a
 end
 
 def please(*args, &block)
@@ -31,63 +28,11 @@ def check_inspected(str)
 end
 
 def tokens
-  Wirb.tokenize(@a).to_a
+  Wirb::Tokenizer.run(@a).to_a
 end
 
 # COLORIZER
 
-shared_examples_for "a Wirb0 colorizer" do
-  it "creates good color codes based on original Wirb color symbols" do
-    # Wirb.get_color(:black).should match_colors(30)
-    Wirb.get_color(:light_purple).should match_colors(1,35)
-    Wirb.get_color(:brown_underline).should match_colors(4,33)
-    Wirb.get_color(:green_background).should match_colors(42)
-  end
-  
-  it "colorizes strings based on original Wirb color symbols" do
-    Wirb.colorize_string('Mos Eisley', :blue).should match_colored_string('Mos Eisley', 34)
-    Wirb.colorize_string('Tatooine', :light_red).should match_colored_string('Tatooine', 1, 31)
-    Wirb.colorize_string('Bespin', :white).should match_colored_string('Bespin', 1, 37)
-    Wirb.colorize_string('Coruscant', :cyan_underline).should match_colored_string('Coruscant', 4, 36)
-  end
-end
-
-def extract_color_codes(string)
-  res = string.scan(/\e\[(?:\d+(?:;\d+)*)?m/).map do |code_match|
-    codes = []
-    code_match.scan(/((?:38|48);5;\d+)|((\d+);(\d+))|(\d+)|^$/) do |code_parts|
-      rgb_code, color_with_style, style, color, color_with_no_style = code_parts
-      codes << if rgb_code
-        rgb_code
-      elsif color_with_style
-        case style
-        when '0'
-          color
-        when '7'
-          (color.to_i+10).to_s
-        else
-          [style,color]
-        end
-      elsif color_with_no_style
-        color_with_no_style
-      else
-        '0'
-      end
-    end
-    codes
-  end.flatten
-  standardize_color_codes(res)
-end
-
-def standardize_color_codes(codes)
-  codes = codes.map{|code| code.to_s}
-  codes = codes.sort.uniq                               # Order of codes doesn't matter, nor does repetition
-  if codes != ['0']
-    codes -= ['0']                                      # Code '0' doesn't matter, unless it's the only one
-  end
-  codes
-end
-
 # MATCHER
 
 # match regex in arrays (e.g. for  object ids)
@@ -128,72 +73,10 @@ RSpec::Matchers.define :be_sorted_like do |should_array|
   end
 end
 
-# Match the color codes from a string with an array of codes, e.g.
-#  "\e\[1m\e\[34m".should match_colors(1,34)
-# Order of the codes does not matter, and repeated codes are ignored
-# Codes like "\e[0;32" are equivalent to "\e[0m\e[32m"
-# Codes like 0,34 and 34 are equivalent
-# Codes like 7,34 and 44 are equivalent
-RSpec::Matchers.define :match_colors do |*should_codes|
-  match do |got_string|
-    got_codes = extract_color_codes(got_string)
-    should_codes = standardize_color_codes(should_codes.flatten)
-    
-    should_codes == got_codes
-  end
-  
-  failure_message_for_should do |actual|
-    "expected that #{actual.inspect} would match the color codes #{should_codes.inspect}"
-  end
-
-  failure_message_for_should_not do |actual|
-    "expected that #{actual.inspect} would match the color codes #{should_codes.inspect}"
-  end
-  
-end
-
-# Match an encoded string:
-# Usage "\e[4m\e[33mStuff\e[0m".should match_colored_string('Stuff', 4, 33)
-RSpec::Matchers.define :match_colored_string do |*should_spec|
-  match do |got_string|
-    should_string = should_spec.first
-    should_codes = should_spec[1..-1]
-    
-    if matches=(got_string=~/^((?:\e\[\d+(?:;\d+)*m)+)(.*)(\e\[\d+(;\d+)*m)$/)
-      got_codes  = $1
-      got_string = $2
-      got_terminating_code = $3
-    end
-    
-    
-    matches && 
-    should_string == got_string && 
-    got_codes.should(match_colors(should_codes)) && 
-    got_terminating_code.should(match_colors(0))
-  end
-
-  failure_message_for_should do |actual|
-    "expected that #{actual.inspect} would match the string #{should_spec.first.inspect}, colored with color codes #{should_spec[1..-1].inspect}"
-  end
-
-  failure_message_for_should_not do |actual|
-    "expected that #{actual.inspect} would not match the string #{should_spec.first}, colored with color codes #{should_spec[1..-1].inspect}"
-  end
-end
 
 # GENERAL
 
-def only19
-  yield if RubyVersion == 1.9
-end
-
-def only18
-  yield if RubyVersion == 1.8
-end
-
 # common regex patterns
 OBJECT_ID = /0x[0-9a-f]+/
 
-=begin helper method for getting tokens
-def ws(obj); puts Wirb.tokenize(obj.inspect).map{|*x| x.inspect + ','}*$/; end
-=end
+def ws(obj) puts Wirb::Tokenizer.run(obj.inspect).map{|*x| x.inspect + ','}*$/ end

data/spec/tokenizer_enumerator_spec.rb

@@ -1,7 +1,7 @@
 describe tokenizer(__FILE__) do
   after :each do check_value end
 
-  only19 do
+  if !RubyEngine.rbx?
     please do check [2,3,4].each
       tokens.should == [
         [:open_object, "#<"],
@@ -116,14 +116,16 @@ describe tokenizer(__FILE__) do
       ]
     end
 
-    please do check Wirb.tokenize('[2,3,4]')
+    please do check Wirb::Tokenizer.run('[2,3,4]')
       tokens.should == [
         [:open_object, "#<"],
         [:object_class, "Enumerator"],
         [:object_description_prefix, ":"],
         [:whitespace, " "],
         [:class, "Wirb"],
-        [:object_description, ":tokenize("],
+        [:class_separator, "::"],
+        [:class, "Tokenizer"],
+        [:object_description, ":run("],
         [:open_string, "\""],
         [:string, "[2,3,4]"],
         [:close_string, "\""],
@@ -132,53 +134,546 @@ describe tokenizer(__FILE__) do
       ]
     end
 
-    require 'prime'
-    please do check Prime.each
-      tokens.should be_like [
+    please do check [{1=>2},Wirb::Tokenizer.run('2'),Set[2,3],[3,4],[5,6].each].map
+      tokens.should == [
         [:open_object, "#<"],
-        [:object_class, "Prime"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_array, "["],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "2"],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:class, "Wirb"],
         [:class_separator, "::"],
-        [:object_class, "EratosthenesGenerator"],
+        [:class, "Tokenizer"],
+        [:object_description, ":run("],
+        [:open_string, "\""],
+        [:string, "2"],
+        [:close_string, "\""],
+        [:object_description, ")"],
+        [:close_object, ">"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_set, "{"],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:close_set, "}"],
+        [:close_object, ">"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_array, "["],
+        [:number, "3"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_array, "]"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_array, "["],
+        [:number, "5"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "6"],
+        [:close_array, "]"],
+        [:object_description, ":each"],
+        [:close_object, ">"],
+        [:close_array, "]"],
+        [:object_description, ":map"],
+        [:close_object, ">"],
+      ]
+    end
+  else # is rbx
+    please do check [2,3,4].each
+      tokens.sort.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
         [:object_description_prefix, ":"],
         [:object_address, /#{OBJECT_ID}/],
-        [:object_description, ' '],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "each"],
+        [:object_description, " "],
         [:object_variable_prefix, "@"],
-        [:object_variable, "last_prime"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
         [:object_description, "="],
         [:nil, "nil"],
-        [:object_description, ", "],
+        [:object_description, " "],
         [:object_variable_prefix, "@"],
-        [:object_variable, "ubound"],
+        [:object_variable, "feedvalue"],
         [:object_description, "="],
         [:nil, "nil"],
         [:close_object, ">"],
-      ]
+      ].sort
     end
 
-    please do check [{1=>2},Wirb.tokenize('2'),Set[2,3],[3,4],[5,6].each].map
-      tokens.should == [
+    please do check Set[2,3,4].map
+      tokens.sort.sort.should be_like [
         [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
         [:object_class, "Enumerator"],
         [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "collect"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
         [:whitespace, " "],
+        [:open_set, "{"],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_set, "}"],
+        [:close_object, ">"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+      ].sort
+    end
+
+    please do check({1=>3}.each)
+      tokens.sort.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "each"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "3"],
+        [:close_hash, "}"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+      ].sort
+    end
+
+    please do check({1=>3}.each.map)
+      tokens.sort.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "collect"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "each"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "3"],
+        [:close_hash, "}"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
         [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+      ].sort
+    end
+
+    please do check [2,Set[{1=>2}],4].map
+      tokens.sort.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "map"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_set, "{"],
         [:open_hash, "{"],
         [:number, "1"],
         [:refers, "=>"],
         [:number, "2"],
         [:close_hash, "}"],
+        [:close_set, "}"],
+        [:close_object, ">"],
         [:comma, ","],
         [:whitespace, " "],
+        [:number, "4"],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+      ].sort
+    end
+
+    please do check Wirb::Tokenizer.run('[2,3,4]')
+      tokens.sort.should be_like [
         [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
         [:object_class, "Enumerator"],
         [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "run"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:class, "Wirb"],
+        [:class_separator, "::"],
+        [:class, "Tokenizer"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:open_string, "\""],
+        [:string, "[2,3,4]"],
+        [:close_string, "\""],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:close_object, ">"],
+      ].sort
+    end
+
+    please do check [{1=>2},Wirb::Tokenizer.run('2'),Set[2,3],[3,4],[5,6].each].map
+      tokens.sort.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "map"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "2"],
+        [:close_hash, "}"],
+        [:comma, ","],
         [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "run"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
         [:class, "Wirb"],
-        [:object_description, ":tokenize("],
+        [:class_separator, "::"],
+        [:class, "Tokenizer"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
         [:open_string, "\""],
         [:string, "2"],
         [:close_string, "\""],
-        [:object_description, ")"],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
         [:close_object, ">"],
         [:comma, ","],
         [:whitespace, " "],
@@ -204,37 +699,85 @@ describe tokenizer(__FILE__) do
         [:comma, ","],
         [:whitespace, " "],
         [:open_object, "#<"],
+        [:object_class, "Enumerable"],
+        [:class_separator, "::"],
         [:object_class, "Enumerator"],
         [:object_description_prefix, ":"],
-        [:whitespace, " "],
+        [:object_address, /#{OBJECT_ID}/],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "iter"],
+        [:object_description, "="],
+        [:symbol_prefix, ":"],
+        [:symbol, "each"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "object"],
+        [:object_description, "="],
         [:open_array, "["],
         [:number, "5"],
         [:comma, ","],
         [:whitespace, " "],
         [:number, "6"],
         [:close_array, "]"],
-        [:object_description, ":each"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
         [:close_object, ">"],
         [:close_array, "]"],
-        [:object_description, ":map"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "generator"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "lookahead"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "args"],
+        [:object_description, "="],
+        [:open_array, "["],
+        [:close_array, "]"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "size"],
+        [:object_description, "="],
+        [:nil, "nil"],
+        [:object_description, " "],
+        [:object_variable_prefix, "@"],
+        [:object_variable, "feedvalue"],
+        [:object_description, "="],
+        [:nil, "nil"],
         [:close_object, ">"],
-      ]
-    end
-  end
-
-  only18 do
-    if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby' # TODO tests for jruby + rbx
-      please do check [3,4,5].each
-        tokens.should be_like [
-          [:open_object, "#<"],
-          [:object_class, "Enumerable"],
-          [:class_separator, "::"],
-          [:object_class, "Enumerator"],
-          [:object_description_prefix, ":"],
-          [:object_address, /#{OBJECT_ID}/],
-          [:close_object, ">"],
-        ]
-      end
+      ].sort
     end
   end
 end