wirb 0.2.3 → 0.2.4

data/CHANGELOG.rdoc

@@ -1,4 +1,4 @@
-== 0.2.3
+== 0.2.3 / 0.2.4
 * Support for rubygems-test <gem-testers.org>
 
 == 0.2.2

data/lib/wirb/version.rb

@@ -1,3 +1,3 @@
 module Wirb
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end

data/spec/spec_helper.rb

@@ -0,0 +1,88 @@
+#require File.dirname(__FILE__) + '/../lib/wirb' unless defined? Wirb
+#require File.dirname(__FILE__) + '/../lib/wirb/wp'
+require 'wirb'
+require 'wirb/wp'
+require 'zucker/engine'
+require 'zucker/version'
+
+def tokenizer(filename)
+  filename =~ /tokenizer_(.*)_spec\.rb/
+  "Wirb.tokenize" + ($1 ? "(#{$1})" : "")
+end
+
+def check_value
+  Wirb.tokenize(@a).map{|_,c|c}.join.should == @a
+end
+
+def please(*args, &block)
+  it *args, &block
+end
+
+def check(obj)
+  @a = obj.inspect
+  wp obj
+end
+
+def check_inspected(str)
+  @a = str
+  puts Wirb.colorize_result str
+end
+
+def tokens
+  Wirb.tokenize(@a).to_a
+end
+
+def only19
+  yield if RubyVersion == 1.9
+end
+
+def only18
+  yield if RubyVersion == 1.8
+end
+
+# match regex in arrays (e.g. for  object ids)
+RSpec::Matchers.define :be_like do |should_array|
+  match do |got_array|
+    if got_array.size != should_array.size
+      false
+    else
+      got_array.zip(should_array).all? do |(plain_kind, plain_string),(regex_kind, regex_string)|
+        regex_kind   ==  plain_kind &&
+        regex_string === plain_string
+      end
+    end
+  end
+end
+
+RSpec::Matchers.define :be_sorted do |should_array|
+  match do |got_array|
+    should_array = should_array.sort_by {|x,y|[x.to_s,y]}
+    got_array    = got_array.sort_by    {|x,y|[x.to_s,y]}
+    should_array == got_array
+  end
+end
+
+RSpec::Matchers.define :be_sorted_like do |should_array|
+  match do |got_array|
+    should_array = should_array.sort_by {|x,y|[x.to_s,y]}
+    got_array    = got_array.sort_by    {|x,y|[x.to_s,y]}
+    if got_array.size != should_array.size
+      false
+    else
+      got_array.zip(should_array).all? do
+          |(plain_kind, plain_string),(regex_kind, regex_string)|
+        regex_kind   ==  plain_kind &&
+        regex_string === plain_string
+      end
+    end
+  end
+end
+
+# common regex patterns
+OBJECT_ID = /0x[0-9a-f]+/
+
+=begin helper methods for getting tokens
+def ws(obj)
+  puts Wirb.tokenize(obj.inspect).map{|*x| x.inspect + ','}*$/
+end
+=end

data/spec/tokenizer_array_spec.rb

@@ -0,0 +1,31 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check [1,2,3,4]
+    tokens.should == [
+      [:open_array, "["],
+      [:number, "1"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "2"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "3"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "4"],
+      [:close_array, "]"]
+    ]
+  end
+
+  please do check [[[]]]
+    tokens.should == [
+      [:open_array, "["],
+      [:open_array, "["],
+      [:open_array, "["],
+      [:close_array, "]"],
+      [:close_array, "]"],
+      [:close_array, "]"],
+    ]
+  end
+end

data/spec/tokenizer_enumerator_spec.rb

@@ -0,0 +1,148 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  only19 do
+    please do check [2,3,4].each
+      tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " "],
+        [:open_enumerator, "["],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_enumerator, "]"],
+        [:object_description, ":each"],
+        [:close_object, ">"],
+      ]
+    end
+
+    please do check [2,Set[{1=>2}],4].map
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " "],
+        [:open_enumerator, "["],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_set, "{"],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "2"],
+        [:close_hash, "}"],
+        [:close_set, "}"],
+        [:close_object, ">"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_enumerator, "]"],
+        [:object_description, ":map"],
+        [:close_object, ">"],
+      ]
+    end
+
+    please do check Wirb.tokenize('[2,3,4]')
+      tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " Wirb:tokenize("],
+        [:open_string, "\""],
+        [:string, "[2,3,4]"],
+        [:close_string, "\""],
+        [:object_description, ")"],
+        [:close_object, ">"],
+      ]
+    end
+
+    require 'prime'
+    please do check Prime.each
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Prime"],
+        [:class_separator, "::"],
+        [:object_class, "EratosthenesGenerator"],
+        [:object_description_prefix, ":"],
+        [:object_addr, /#{OBJECT_ID}/],
+        [:object_description, ' @last_prime=nil, @ubound=nil'],
+        [:close_object, ">"],
+      ]
+    end
+
+    please do check [{1=>2},Wirb.tokenize('2'),Set[2,3],[3,4],[5,6].each].map
+      tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " "],
+        [:open_enumerator, "["],
+        [:open_hash, "{"],
+        [:number, "1"],
+        [:refers, "=>"],
+        [:number, "2"],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " Wirb:tokenize("],
+        [:open_string, "\""],
+        [:string, "2"],
+        [:close_string, "\""],
+        [:object_description, ")"],
+        [:close_object, ">"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_set, "{"],
+        [:number, "2"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:close_set, "}"],
+        [:close_object, ">"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_array, "["],
+        [:number, "3"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "4"],
+        [:close_array, "]"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_object, "#<"],
+        [:object_class, "Enumerator"],
+        [:object_description_prefix, ":"],
+        [:object_description, " "],
+        [:open_enumerator, "["],
+        [:number, "5"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, "6"],
+        [:close_enumerator, "]"],
+        [:object_description, ":each"],
+        [:close_object, ">"],
+        [:close_enumerator, "]"],
+        [:object_description, ":map"],
+        [:close_object, ">"],
+      ]
+    end
+  end
+end

data/spec/tokenizer_hash_spec.rb

@@ -0,0 +1,65 @@
+require 'set'
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check( {1=>2} )
+    tokens.should == [
+      [:open_hash, "{"],
+      [:number, "1"],
+      [:refers, "=>"],
+      [:number, "2"],
+      [:close_hash, "}"],
+    ]
+  end
+
+  please do check({:hallo => {1=>Set[2,3,4]}})
+    tokens.should be_like [
+      [:open_hash, "{"],
+      [:symbol_prefix, ":"],
+      [:symbol, "hallo"],
+      [:refers, "=>"],
+      [:open_hash, "{"],
+      [:number, "1"],
+      [:refers, "=>"],
+      [:open_object, "#<"],
+      [:object_class, "Set"],
+      [:object_description_prefix, ":"],
+      [:whitespace, " "],
+      [:open_set, "{"],
+      [:number, /\d+/],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, /\d+/],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, /\d+/],
+      [:close_set, "}"],
+      [:close_object, ">"],
+      [:close_hash, "}"],
+      [:close_hash, "}"],
+    ]
+  end
+
+  please do check( {1=>2, 3=>8, {} => {}} )
+    tokens.should be_sorted [
+      [:close_hash, "}"],
+      [:close_hash, "}"],
+      [:close_hash, "}"],
+      [:comma, ","],
+      [:comma, ","],
+      [:number, "1"],
+      [:number, "2"],
+      [:number, "3"],
+      [:number, "8"],
+      [:open_hash, "{"],
+      [:open_hash, "{"],
+      [:open_hash, "{"],
+      [:refers, "=>"],
+      [:refers, "=>"],
+      [:refers, "=>"],
+      [:whitespace, " "],
+      [:whitespace, " "],
+    ]
+  end
+end

data/spec/tokenizer_misc_spec.rb

@@ -0,0 +1,70 @@
+# encoding: US-ASCII
+require 'stringio'
+require 'date'
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check StringIO.new 'wirb'
+    if RubyEngine.rbx?
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "StringIO"],
+        [:object_description_prefix, ":"],
+        [:object_addr, OBJECT_ID],
+        [:object_description, /.*/],
+        [:open_string, "\""],
+        [:string, "wirb"],
+        [:close_string, "\""],
+        [:object_description, /.*/],
+        [:close_object, ">"],
+      ]
+   else
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "StringIO"],
+        [:object_description_prefix, ":"],
+        [:object_addr, OBJECT_ID],
+        [:close_object, ">"],
+      ]
+    end
+  end
+  
+  please do check Date.today
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "Date"],
+      [:object_description_prefix, ":"],
+      [:object_description, /.*/],
+      [:close_object, ">"],
+    ]
+  end
+
+  please do check "communication".match /.*(.)\1.*/
+    tokens.should == [
+      [:open_object, "#<"],
+      [:object_class, "MatchData"],
+      [:object_description_prefix, " "],
+      [:open_string, "\""],
+      [:string, "communication"],
+      [:close_string, "\""],
+      [:object_description, " 1:"],
+      [:open_string, "\""],
+      [:string, "m"],
+      [:close_string, "\""],
+      [:close_object, ">"],
+    ]
+  end
+
+  only19 do
+    please do check __ENCODING__
+      tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "Encoding"],
+        [:object_description_prefix, ":"],
+        [:object_description, "US-ASCII"],
+        [:close_object, ">"],
+      ]
+    end
+  end
+end

data/spec/tokenizer_nested_spec.rb

@@ -0,0 +1,64 @@
+require 'set'
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check [2,'sdf',:hallosfe, /34jf3/, [[[[]]]], {{5=> "4"} => {Set.new => binding}}]
+    tokens.should be_like [
+      [:open_array, "["],
+      [:number, "2"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_string, "\""],
+      [:string, "sdf"],
+      [:close_string, "\""],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "hallosfe"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_regexp, "/"],
+      [:regexp, "34jf3"],
+      [:close_regexp, "/"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_array, "["],
+      [:open_array, "["],
+      [:open_array, "["],
+      [:open_array, "["],
+      [:close_array, "]"],
+      [:close_array, "]"],
+      [:close_array, "]"],
+      [:close_array, "]"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_hash, "{"],
+      [:open_hash, "{"],
+      [:number, "5"],
+      [:refers, "=>"],
+      [:open_string, "\""],
+      [:string, "4"],
+      [:close_string, "\""],
+      [:close_hash, "}"],
+      [:refers, "=>"],
+      [:open_hash, "{"],
+      [:open_object, "#<"],
+      [:object_class, "Set"],
+      [:object_description_prefix, ":"],
+      [:whitespace, " "],
+      [:open_set, "{"],
+      [:close_set, "}"],
+      [:close_object, ">"],
+      [:refers, "=>"],
+      [:open_object, "#<"],
+      [:object_class, "Binding"],
+      [:object_description_prefix, ":"],
+      [:object_addr, OBJECT_ID],
+      [:close_object, ">"],
+      [:close_hash, "}"],
+      [:close_hash, "}"],
+      [:close_array, "]"],
+    ]
+  end
+end

data/spec/tokenizer_nil_false_true_spec.rb

@@ -0,0 +1,15 @@
+describe 'Wirb.tokenize' do
+  after :each do check_value end
+
+  please do check true
+    tokens.should == [[:true, "true"]]
+  end
+
+  please do check false
+    tokens.should == [[:false, "false"]]
+  end
+
+  please do check nil
+    tokens.should == [[:nil, "nil"]]
+  end
+end

data/spec/tokenizer_number_spec.rb

@@ -0,0 +1,104 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check 42
+    tokens.should == [[:number, "42"]]
+  end
+
+  please do check [1..2, 3...4, 'a'..'z', 'a'...'z']
+    tokens.should == [
+      [:open_array, "["],
+      [:number, "1"],
+      [:range, ".."],
+      [:number, "2"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "3"],
+      [:range, "..."],
+      [:number, "4"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_string, "\""],
+      [:string, "a"],
+      [:close_string, "\""],
+      [:range, ".."],
+      [:open_string, "\""],
+      [:string, "z"],
+      [:close_string, "\""],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_string, "\""],
+      [:string, "a"],
+      [:close_string, "\""],
+      [:range, "..."],
+      [:open_string, "\""],
+      [:string, "z"],
+      [:close_string, "\""],
+      [:close_array, "]"],
+    ]
+  end
+ 
+  please do check [2, 3, 4.0, -3, 99e99, 1e-99, 2_3_4, 4E8]
+    tokens.should == [
+      [:open_array, "["],
+      [:number, "2"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "3"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "4.0"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "-3"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "9.9e+100"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "1.0e-99"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "234"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, "400000000.0"],
+      [:close_array, "]"],
+    ]
+  end
+
+  only19 do
+    please do check Rational(2,3)
+      tokens.should == [
+        [:open_rational, "("],
+        [:number, "2"],
+        [:rational_separator, "/"],
+        [:number, "3"],
+        [:close_rational, ")"],
+      ]
+    end
+  end
+
+  only18 do
+    please do check Rational(2,3)
+      tokens.should == [
+        [:class, "Rational"],
+        [:open_rational, "("],
+        [:number, "2"],
+        [:rational_separator, ","],
+        [:whitespace, " "],
+        [:number, "3"],
+        [:close_rational, ")"],
+      ]
+
+      # modifying the world, see https://github.com/janlelis/wirb/issues#issue/1
+      require 'mathn'
+      check Rational(2,3)
+      tokens.should == [
+        [:number, "2"],
+        [:rational_separator, "/"],
+        [:number, "3"],
+      ]
+    end
+  end
+end

data/spec/tokenizer_object_spec.rb

@@ -0,0 +1,108 @@
+require 'ostruct'
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check Object.new
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "Object"],
+      [:object_description_prefix, ":"],
+      [:object_addr, OBJECT_ID],
+      [:close_object, ">"],
+    ]
+ end
+
+  please do check proc{}
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "Proc"],
+      [:object_description_prefix, ":"],
+      [:object_addr, OBJECT_ID],
+      [:object_line_prefix, "@"],
+      [:object_line, /.*tokenizer_object_spec.rb:/],
+      [:object_line_number, /\d+/],
+      [:close_object, ">"],
+    ]
+  end
+
+  please do check lambda{}
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "Proc"],
+      [:object_description_prefix, ":"],
+      [:object_addr, OBJECT_ID],
+      [:object_line_prefix, "@"],
+      [:object_line, /.*tokenizer_object_spec.rb:/],
+      [:object_line_number, /\d+/],
+      RubyVersion == 1.9 ? [:object_description, " (lambda)"] : nil,
+      [:close_object, ">"],
+    ].compact
+  end
+
+  unless RubyEngine.rbx? # rubinius provides more information
+    please do check binding
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Binding"],
+        [:object_description_prefix, ":"],
+        [:object_addr, OBJECT_ID],
+        [:close_object, ">"],
+      ]
+    end
+  end
+
+  only19 do
+    please do check STDOUT
+      tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "IO"],
+        [:object_description_prefix, ":"],
+        [:object_description, "<STDOUT>"],
+        [:close_object, ">"],
+      ]
+    end
+
+    please do check STDIN
+       tokens.should == [
+        [:open_object, "#<"],
+        [:object_class, "IO"],
+        [:object_description_prefix, ":"],
+        [:object_description, "<STDIN>"],
+        [:close_object, ">"],
+      ]
+    end
+     
+    please do check Module.new.singleton_class.singleton_class
+      tokens.should be_like [
+        [:open_object, "#<"],
+        [:object_class, "Class"],
+        [:object_description_prefix, ":"],
+        [:object_description, '#<Class:#<Module:'],
+        [:object_addr, OBJECT_ID],
+        [:object_description, '>>'],
+        [:close_object, ">"]
+      ]
+    end
+  end
+
+  please do
+    SomeStruct=Struct.new :favourite, :language
+    check SomeStruct.new :ruby
+    tokens.should == [
+      [:open_object, "#<"],
+      [:object_class, "struct"],
+      [:object_description_prefix, " "],
+      [:object_description, "SomeStruct favourite=:ruby, language=nil"],
+      [:close_object, ">"],
+    ]
+  end
+
+  please do check OpenStruct.new
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "OpenStruct"],
+      [:close_object, ">"],
+    ]
+  end
+end

data/spec/tokenizer_rails_spec.rb

@@ -0,0 +1,19 @@
+# only very basic support so far.. open issues ;)
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check_inspected "User(Table doesn't exist)"
+    tokens.should == [
+      [:class, "User"],
+      [:object_description, "(Table doesn't exist)"],
+    ]
+  end
+ 
+  please do check_inspected "User(id: integer, name: string, created_at: datetime, updated_at: datetime)"
+    tokens.should == [
+      [:class, "User"],
+      [:object_description, "(id: integer, name: string, created_at: datetime, updated_at: datetime)"],
+    ]
+  end
+end

data/spec/tokenizer_regexp_spec.rb

@@ -0,0 +1,24 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check /music/
+    tokens.should == [[:open_regexp, "/"], [:regexp, "music"], [:close_regexp, "/"]]
+  end
+
+  please do check /music/mix
+    tokens.should == [[:open_regexp, "/"], [:regexp, "music"], [:close_regexp, "/"], [:regexp_flags, "mix"]]
+  end
+
+  please do check /mus\/ic/
+    tokens.should == [[:open_regexp, "/"], [:regexp, "mus\\/ic"], [:close_regexp, "/"]]
+  end
+
+  please do check /\b([A-Z0-9._%+-]+)@([A-Z0-9.-]+\.[A-Z]{2,4})\b/i
+    tokens.should == [
+      [:open_regexp, "/"],
+      [:regexp, "\\b([A-Z0-9._%+-]+)@([A-Z0-9.-]+\\.[A-Z]{2,4})\\b"],
+      [:close_regexp, "/"],
+      [:regexp_flags, "i"],
+    ]
+  end
+end

data/spec/tokenizer_set_spec.rb

@@ -0,0 +1,53 @@
+require 'set'
+
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check Set[2,3,4]
+    tokens.should be_like [
+      [:open_object, "#<"],
+      [:object_class, "Set"],
+      [:object_description_prefix, ":"],
+      [:whitespace, " "],
+      [:open_set, "{"],
+      [:number, /\d/],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, /\d/],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:number, /\d/],
+      [:close_set, "}"],
+      [:close_object, ">"],
+    ]
+  end
+
+   please do check Set[[2,3], {1=>{2=>3}}]
+     tokens.should be_sorted [
+       [:close_array, "]"],
+       [:close_hash, "}"],
+       [:close_hash, "}"],
+       [:close_object, ">"],
+       [:close_set, "}"],
+       [:comma, ","],
+       [:comma, ","],
+       [:number, "1"],
+       [:number, "2"],
+       [:number, "2"],
+       [:number, "3"],
+       [:number, "3"],
+       [:object_class, "Set"],
+       [:object_description_prefix, ":"],
+       [:open_array, "["],
+       [:open_hash, "{"],
+       [:open_hash, "{"],
+       [:open_object, "#<"],
+       [:open_set, "{"],
+       [:refers, "=>"],
+       [:refers, "=>"],
+       [:whitespace, " "],
+       [:whitespace, " "],
+       [:whitespace, " "],
+     ]
+   end
+end

data/spec/tokenizer_string_spec.rb

@@ -0,0 +1,11 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check "always, again."
+    tokens.should == [[:open_string, '"'], [:string, "always, again."], [:close_string, '"']]
+  end
+
+  please do check '"quote"'
+    tokens.should == [[:open_string, '"'], [:string, "\\\"quote\\\""], [:close_string, '"']]
+  end
+end

data/spec/tokenizer_symbol_spec.rb

@@ -0,0 +1,112 @@
+describe tokenizer(__FILE__) do
+  after :each do check_value end
+
+  please do check %w|a ab ab0 ab= < > * + <=> <= >= == =~ __send__ is_a? [] []= |.map(&:to_sym)
+    tokens.should == [
+      [:open_array, "["],
+      [:symbol_prefix, ":"],
+      [:symbol, "a"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "ab"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "ab0"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "ab="],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "<"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, ">"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "*"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "+"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "<=>"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "<="],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, ">="],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "=="],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "=~"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "__send__"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "is_a?"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "[]"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:symbol_prefix, ":"],
+      [:symbol, "[]="],
+      [:close_array, "]"],
+    ]
+  end
+
+  please do check :'hello"word'
+    tokens.should == [
+      [:symbol_prefix, ":"], [:open_symbol_string, "\""],
+      [:symbol_string, "hello\\\"word"], [:close_symbol_string, "\""]
+    ]
+  end
+
+  please do check [[:hallo],{:'hallo'=>:'test'}, {:'hallo='=>:'test='}]
+    tokens.should == [
+      [:open_array, "["],
+      [:open_array, "["],
+      [:symbol_prefix, ":"],
+      [:symbol, "hallo"],
+      [:close_array, "]"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_hash, "{"],
+      [:symbol_prefix, ":"],
+      [:symbol, "hallo"],
+      [:refers, "=>"],
+      [:symbol_prefix, ":"],
+      [:symbol, "test"],
+      [:close_hash, "}"],
+      [:comma, ","],
+      [:whitespace, " "],
+      [:open_hash, "{"],
+      [:symbol_prefix, ":"],
+      [:symbol, "hallo="],
+      [:refers, "=>"],
+      [:symbol_prefix, ":"],
+      [:symbol, "test="],
+      [:close_hash, "}"],
+      [:close_array, "]"],
+    ]
+  end
+end

data/wirb.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.description =  "Colorizes your irb results. It's based on Wirble but only provides result highlighting. Just call Wirb.start and enjoy the colors ;)."
   s.required_rubygems_version = '>= 1.3.6'
   s.required_ruby_version     = '>= 1.8.7'
-  s.files = Dir.glob(%w[{lib,test}/**/*.rb bin/* [A-Z]*.{txt,rdoc} ext/**/*.{rb,c} **/deps.rip]) + %w{Rakefile wirb.gemspec .gemtest}
+  s.files = Dir.glob(%w[{lib,test,spec}/**/*.rb bin/* [A-Z]*.{txt,rdoc} ext/**/*.{rb,c} **/deps.rip]) + %w{Rakefile wirb.gemspec .gemtest}
   s.extra_rdoc_files = ["README.rdoc", "COPYING"]
   s.license = 'MIT'
   s.add_development_dependency 'rspec'

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Jan Lelis
@@ -73,6 +73,20 @@ files:
 - lib/wirb/colors.rb
 - lib/wirb/version.rb
 - lib/wirb/irb.rb
+- spec/tokenizer_rails_spec.rb
+- spec/tokenizer_hash_spec.rb
+- spec/tokenizer_number_spec.rb
+- spec/tokenizer_regexp_spec.rb
+- spec/tokenizer_array_spec.rb
+- spec/tokenizer_enumerator_spec.rb
+- spec/spec_helper.rb
+- spec/tokenizer_misc_spec.rb
+- spec/tokenizer_nil_false_true_spec.rb
+- spec/tokenizer_symbol_spec.rb
+- spec/tokenizer_set_spec.rb
+- spec/tokenizer_object_spec.rb
+- spec/tokenizer_nested_spec.rb
+- spec/tokenizer_string_spec.rb
 - README.rdoc
 - CHANGELOG.rdoc
 - Rakefile