rubylexer 0.7.6 → 0.7.7

data/test/code/dumptokens.rb

@@ -30,18 +30,19 @@ class Token
   def strify
     [self.class.name[/[^:]+$/],": ",instance_variables.sort.collect{|v| 
       [v,"=",instance_variable_get(v).inspect," "]
-    }].to_s
+    }].join
   end
 end
 end
 
-macros=silent=file=nil
-
+name=macros=silent=file=nil
+options={}
 #allow -e
 opts=GetoptLong.new(
   ["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT],
   ["--silent", "-s", GetoptLong::NO_ARGUMENT],
-  ["--macro", "-m", GetoptLong::NO_ARGUMENT]
+  ["--macro", "-m", GetoptLong::NO_ARGUMENT],
+  ["--ruby19", "-9", GetoptLong::NO_ARGUMENT]
 )
 opts.each{|opt,arg|
   case opt
@@ -52,6 +53,8 @@ opts.each{|opt,arg|
     silent=true
   when '--macro'
     macros=true
+  when '--ruby19'
+    options[:rubyversion]=1.9
   end
 }
      
@@ -63,7 +66,9 @@ file||=if name=ARGV.first
     $stdin.read
   end
 
-lexer=RubyLexer.new(name, file) 
+args=name, file
+args.push 1,0,options unless options.empty?
+lexer=RubyLexer.new(*args) 
 lexer.enable_macros! if macros
 if silent
   until RubyLexer::EoiToken===(tok=lexer.get1token)

data/test/code/regression.rb

@@ -3,6 +3,8 @@ require 'open3'
 require 'rubygems'
 require 'rubylexer/test/testcases'
 require 'test/code/rubylexervsruby'
+require 'test/code/test_1.9'
+
   SEP='';
   'caleb clausen'.each_byte{|ch| SEP<<ch.to_s(2).gsub('0','-').gsub('1','+')}
   SEP<<'(0)'
@@ -68,17 +70,23 @@ end
          define_method '#{name}' do 
            difflines=[]
            begin
-             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
-             difflines.empty? or raise DifferencesFromMRILex, difflines
+             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
+             unless difflines.empty?
+               puts '#{esctc}'
+               puts difflines.join
+               raise DifferencesFromMRILex
+             end
              res or raise LexerTestFailure, ''
-           rescue Interrupt: exit
-           rescue Exception=>e: 
-             e.message<<"\n"+'while lexing: #{esctc[0...1000]}'
-             raise e
+           rescue Interrupt; exit
+           rescue Exception=>e 
+             message=e.message.dup<<"\n"+'while lexing: #{esctc}'
+             e2=e.class.new(message)
+             e2.set_backtrace(e.backtrace)
+             raise e2
            end
          end  
       ]
-    }.to_s
+    }
     
     illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
       i+=1
@@ -89,20 +97,26 @@ end
            difflines=[]
            begin
              res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
-             difflines.empty? or raise DifferencesFromMRILex, difflines
+             unless difflines.empty?
+               puts '#{esctc}'
+               puts difflines.join
+               raise DifferencesFromMRILex
+             end
              res or raise LexerTestFailure, ''
-           rescue LexerTestFailure: 
+           rescue LexerTestFailure
              puts 'warning: test failure lexing "#{esctc}"' 
-           rescue Interrupt: exit
-           rescue Exception=>e: 
-             e.message<<"\n"+'while lexing: #{esctc}'
-             raise e
+           rescue Interrupt; exit
+           rescue Exception=>e;
+             message=e.message.dup<<"\n"+'while lexing: #{esctc}'
+             e2=e.class.new(message)
+             e2.set_backtrace(e.backtrace)
+             raise e2
            end
          end  
       ]
-    }.to_s
-
+    }
 
-#    puts test_code+illegal_test_code
-    eval test_code+illegal_test_code
+    src=(test_code+illegal_test_code).join
+#   puts src
+    eval src
   end

data/test/code/rubylexervsruby.rb

@@ -194,7 +194,7 @@ end
 return false
 =end
 
-rescue Interrupt: raise
+rescue Interrupt; raise
 rescue Exception
   File.exist? nopfile and
     system "ruby -c #{nopfile} >/dev/null 2>&1" or expected="(expected) " unless stringdata

data/test/code/test_1.9.rb

@@ -0,0 +1,31 @@
+require 'test/unit'
+require "rubylexer"
+require "rubylexer/test/oneliners_1.9"
+
+class Ruby1_9Tests < Test::Unit::TestCase
+  include Ruby1_9OneLiners
+  def test_1_9_roughly
+    EXPECT_NO_METHODS.each{|snippet| 
+      begin
+        tokens=RubyLexer.new('string',snippet,1,0,:rubyversion=>1.9).to_a
+        assert_equal [],tokens.grep(RubyLexer::MethNameToken)
+        assert_equal [],tokens.grep(RubyLexer::ErrorToken)
+      rescue Exception=>e
+        e2=e.class.new(e.message+" while testing '#{snippet}'")
+        e2.set_backtrace(e.backtrace)
+        raise e2
+      end
+    }
+    EXPECT_1_METHOD.each{|snippet| 
+      begin
+        tokens=RubyLexer.new('string',snippet,1,0,:rubyversion=>1.9).to_a
+        assert_equal 1,tokens.grep(RubyLexer::MethNameToken).size
+        assert_equal [],tokens.grep(RubyLexer::ErrorToken)
+      rescue Exception=>e
+        e2=e.class.new(e.message+" while testing '#{snippet}'")
+        e2.set_backtrace(e.backtrace)
+        raise e2
+      end
+    }
+  end
+end

data/test/code/tokentest.rb

@@ -66,15 +66,15 @@ class SymbolToken
   def verify_offset(fd)
     la=fd.read(2)
     case la
-      when '%s': 
+      when '%s'
         quote=fd.read(1)
         ender=RubyLexer::PAIRS[quote] || quote
         body=@ident[2...-1]
-      when /^:(['"])/: 
+      when /^:(['"])/
         #stay right here
         quote=ender=$1
         body=@ident[2...-1]
-      when /^:/: 
+      when /^:/
         fd.pos-=1
         body=@ident[1..-1]
       else raise 'unrecognized symbol type'
@@ -85,7 +85,7 @@ class SymbolToken
     #punt if its too hard
     if quote
       bs="\\"
-      hardstuff= /[#{bs}#{quote}#{bs}#{ender}\#\\]/
+      hardstuff= Regexp.new %/[#{bs}#{quote}#{bs}#{ender}\#\\\\]/
       return true if (body+bodyread).match(hardstuff)
     end
 
@@ -154,7 +154,7 @@ class StringToken
     #verify offsets of subtokens
     @elems.each{|elem|
       case elem
-      when String: 
+      when String
         #get string data to compare against,
         #translating dos newlines to unix.
         #(buffer mgt is a PITA)
@@ -245,7 +245,7 @@ def check_offset(tok,file=nil,endpos=nil)
     when RubyLexer::StringToken,RubyLexer::NumberToken,
          RubyLexer::HereBodyToken,RubyLexer::SymbolToken,
          RubyLexer::HerePlaceholderToken,
-         RubyLexer::FileAndLineToken: #do nothing
+         RubyLexer::FileAndLineToken #do nothing
     else 
       file.pos==endpos or allow_ooo or 
         $stderr.puts "positions don't line up, expected #{endpos}, got #{file.pos}, token: #{tok.to_s.gsub("\n","\n  ") }"

data/test/test_all.rb

@@ -0,0 +1,2 @@
+require 'test/code/regression'
+require 'test/code/test_1.9'

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: rubylexer
 version: !ruby/object:Gem::Version 
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors: 
 - Caleb Clausen
 autorequire: 
-bindir: ""
+bindir: bin
 cert_chain: []
 
-date: 2009-07-06 00:00:00 -07:00
+date: 2010-01-03 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,153 +20,149 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.2.0
+        version: 0.2.3
     version: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.12.2
-    version: 
-description: RubyLexer is a lexer library for Ruby, written in Ruby. Rubylexer is meant as a lexer for Ruby that's complete and correct; all legal Ruby  code should be lexed correctly by RubyLexer as well. Just enough parsing  capability is included to give RubyLexer enough context to tokenize correctly in all cases. (This turned out to be more parsing than I had thought or  wanted to take on at first.) RubyLexer handles the hard things like  complicated strings, the ambiguous nature of some punctuation characters and  keywords in ruby, and distinguishing methods and local variables.
-email: rubylexer-owner @at@ inforadical .dot. net
+description: RubyLexer is a lexer library for Ruby, written in Ruby. Rubylexer is meant as a lexer for Ruby that's complete and correct; all legal Ruby  code should be lexed correctly by RubyLexer as well. Just enough parsing  capability is included to give RubyLexer enough context to tokenize correctly in all cases. (This turned out to be more parsing than I had thought or  wanted to take on at first.) RubyLexer handles the hard things like  complicated strings, the ambiguous nature of some punctuation characters and  keywords in ruby, and distinguishing methods and local variables.  RubyLexer is not particularly clean code. As I progressed in writing this,  I've learned a little about how these things are supposed to be done; the  lexer is not supposed to have any state of it's own, instead it gets whatever  it needs to know from the parser. As a stand-alone lexer, Rubylexer maintains  quite a lot of state. Every instance variable in the RubyLexer class is some  sort of lexer state. Most of the complication and ugly code in RubyLexer is  in maintaining or using this state.
+email: caleb (at) inforadical (dot) net
 executables: []
 
 extensions: []
 
 extra_rdoc_files: 
 - README.txt
-- Manifest.txt
-- howtouse.txt
-- History.txt
-- testing.txt
+- COPYING
 files: 
+- .document
 - COPYING
-- README.txt
+- History.txt
+- Makefile
 - Manifest.txt
-- Rakefile
+- README.txt
 - howtouse.txt
-- History.txt
-- testing.txt
-- lib/rubylexer/rubycode.rb
-- lib/rubylexer/context.rb
-- lib/rubylexer/token.rb
-- lib/rubylexer/0.6.rb
+- lib/.document
+- lib/assert.rb
+- lib/rubylexer.rb
+- lib/rubylexer/.document
 - lib/rubylexer/0.6.2.rb
+- lib/rubylexer/0.6.rb
 - lib/rubylexer/0.7.0.rb
-- lib/rubylexer/version.rb
-- lib/rubylexer/rulexer.rb
-- lib/rubylexer/tokenprinter.rb
+- lib/rubylexer/0.7.1.rb
+- lib/rubylexer/charhandler.rb
 - lib/rubylexer/charset.rb
+- lib/rubylexer/context.rb
 - lib/rubylexer/lextable.rb
+- lib/rubylexer/rubycode.rb
+- lib/rubylexer/rulexer.rb
 - lib/rubylexer/symboltable.rb
-- lib/rubylexer/charhandler.rb
-- lib/assert.rb
-- lib/rubylexer.rb
-- test/data/blockassigntest.rb
-- test/data/for.rb
-- test/data/chunky_bacon.rb
-- test/data/and.rb
-- test/data/pre.unix.rb
-- test/data/untermed_string.rb.broken
-- test/data/__end__2.rb
-- test/data/w.rb
-- test/data/if.rb
-- test/data/pre.rb
-- test/data/jarh.rb
-- test/data/regtest.rb
-- test/data/chunky_bacon4.rb
-- test/data/__end__.rb
-- test/data/strinc.rb
-- test/data/lbrace.rb
-- test/data/p.rb
-- test/data/chunky.plain.rb
-- test/data/noeolatend.rb
-- test/data/g.rb
-- test/data/23.rb
-- test/data/lbrack.rb
-- test/data/untitled1.rb
-- test/data/rescue.rb
-- test/data/pleac.rb
-- test/data/pleac.rb.broken
-- test/data/heart.rb
-- test/data/s.rb
-- test/data/wsdlDriver.rb
-- test/data/p-op.rb
-- test/data/1.rb
-- test/data/1.rb.broken
-- test/data/untermed_here.rb.broken
-- test/data/newsyntax.rb
-- test/data/chunky_bacon3.rb
-- test/data/chunky_bacon2.rb
-- test/data/format.rb
-- test/code/locatetest.rb
-- test/code/rubylexervsruby.rb
-- test/code/tokentest.rb
-- test/code/dumptokens.rb
-- test/code/torment
-- test/code/locatetest
-- test/code/deletewarns.rb
-- lib/rubylexer/0.7.1.rb
+- lib/rubylexer/test/illegal_oneliners.rb
+- lib/rubylexer/test/illegal_stanzas.rb
+- lib/rubylexer/test/oneliners.rb
+- lib/rubylexer/test/oneliners_1.9.rb
+- lib/rubylexer/test/stanzas.rb
+- lib/rubylexer/test/testcases.rb
+- lib/rubylexer/token.rb
+- lib/rubylexer/tokenprinter.rb
+- lib/rubylexer/version.rb
+- rubylexer.gemspec
 - test/code/all_the_gems.rb
 - test/code/all_the_raas.rb
 - test/code/all_the_rubies.rb
+- test/code/coloruby.rb
+- test/code/deletewarns.rb
+- test/code/dumptokens.rb
 - test/code/errscan
 - test/code/isolate_error.rb
 - test/code/lexloop
+- test/code/locatetest
+- test/code/locatetest.rb
 - test/code/regression.rb
+- test/code/rubylexervsruby.rb
 - test/code/strgen.rb
 - test/code/tarball.rb
-- lib/rubylexer/test/testcases.rb
-- test/data/cvtesc.rb
+- test/code/test_1.9.rb
+- test/code/tokentest.rb
+- test/code/torment
+- test/data/1.rb
+- test/data/1.rb.broken
+- test/data/23.rb
+- test/data/__end__.rb
+- test/data/__end__2.rb
 - test/data/__eof2.rb
 - test/data/__eof5.rb
 - test/data/__eof6.rb
+- test/data/and.rb
+- test/data/blockassigntest.rb
+- test/data/chunky.plain.rb
+- test/data/chunky_bacon.rb
+- test/data/chunky_bacon2.rb
+- test/data/chunky_bacon3.rb
+- test/data/chunky_bacon4.rb
+- test/data/cvtesc.rb
+- test/data/for.rb
+- test/data/format.rb
+- test/data/g.rb
 - test/data/hd0.rb
 - test/data/hdateof.rb
 - test/data/hdempty.rb
-- test/data/hdr_dos2.rb
-- test/data/hdr_dos.rb
 - test/data/hdr.rb
-- test/data/here_escnl_dos.rb
+- test/data/hdr_dos.rb
+- test/data/hdr_dos2.rb.broken
+- test/data/heart.rb
 - test/data/here_escnl.rb
-- test/data/heremonsters_dos.rb
-- test/data/heremonsters_dos.rb.broken
-- test/data/heremonsters.rb
-- test/data/heremonsters.rb.broken
+- test/data/here_escnl_dos.rb
 - test/data/here_squote.rb
-- lib/rubylexer/test/illegal_oneliners.rb
-- lib/rubylexer/test/illegal_stanzas.rb
+- test/data/heremonsters.rb
+- test/data/heremonsters_broken.rb
+- test/data/heremonsters_dos.rb
+- test/data/heremonsters_dos_broken.rb
+- test/data/if.rb
+- test/data/jarh.rb
+- test/data/lbrace.rb
+- test/data/lbrack.rb
 - test/data/make_ws_strdelim.rb
 - test/data/maven2_builer_test.rb
 - test/data/migration.rb
+- test/data/modl.rb
 - test/data/modl_dos.rb
 - test/data/modl_fails.rb
-- test/data/modl.rb
 - test/data/multilinestring.rb
-- lib/rubylexer/test/oneliners.rb
-- test/data/simple_dos.rb
+- test/data/newsyntax.rb
+- test/data/noeolatend.rb
+- test/data/p-op.rb
+- test/data/p.rb
+- test/data/pleac.rb
+- test/data/pleac.rb.broken
+- test/data/pre.rb
+- test/data/pre.unix.rb
+- test/data/regtest.rb
+- test/data/rescue.rb
+- test/data/s.rb
 - test/data/simple.rb
-- lib/rubylexer/test/stanzas.rb
+- test/data/simple_dos.rb
 - test/data/strdelim_crlf.rb
+- test/data/strinc.rb
+- test/data/stuff.rb
 - test/data/stuff2.rb
 - test/data/stuff3.rb
 - test/data/stuff4.rb
-- test/data/stuff.rb
 - test/data/tkweird.rb
 - test/data/unending_stuff.rb
+- test/data/untermed_here.rb.broken
+- test/data/untermed_string.rb.broken
+- test/data/untitled1.rb
+- test/data/w.rb
 - test/data/whatnot.rb
 - test/data/ws_strdelim.rb
+- test/data/wsdlDriver.rb
 - test/test.sh
+- test/test_all.rb
+- testing.txt
 has_rdoc: true
-homepage: http://github.com/coatl/rubylexer/
+homepage: http://github.com/coatl/rubylexer
 post_install_message: 
 rdoc_options: 
-- -x lib/rubylexer/test/oneliners.rb
+- --main
+- README.txt
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -189,4 +185,4 @@ signing_key:
 specification_version: 2
 summary: RubyLexer is a lexer library for Ruby, written in Ruby.
 test_files: 
-- test/code/regression.rb
+- test/test_all.rb