rubylexer 0.7.7 → 0.8.0

data/lib/rubylexer/tokenprinter.rb

@@ -1,6 +1,6 @@
-=begin legal crap
+=begin legalia
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -34,7 +34,7 @@ class SimpleTokenPrinter
    TOKENSPERLINE=8
    TOKENSMAGICMAP="\n"+' '*(TOKENSPERLINE-1)
 
-   def pprint(tok,output=$stdout) output.print(sprint(tok)) end
+   def pprint(tok,output=$stdout) output<<(sprint(tok)) end
 
    def sprint(tok)
       case tok
@@ -73,9 +73,9 @@ end
          else
            ''
          end
-         if ?= == @ident.to_s[0]
+         #if ?= == @ident.to_s[0]
            result+="\\\n"*@ident.to_s.scan(/\r\n?|\n\r?/).size
-         end
+         #end
          
          return result
    end end
@@ -85,6 +85,9 @@ end
    class OutlinedHereBodyToken; def ws_munge(tp)
      nil
    end end
+   class EncodingDeclToken; def ws_munge(tp)
+     nil
+   end end
    class ZwToken; def ws_munge(tp)
       case tp.showzw 
       when 2; explicit_form_all
@@ -117,7 +120,7 @@ class KeepWsTokenPrinter
    def pprint(tok,output=$stdout)
       @accum<<aprint(tok).to_s
       if (@accum.size>ACCUMSIZE and NewlineToken===tok) or EoiToken===tok
-         output.print(@accum.join)
+         output<<(@accum.join)
          @accum=[]
       end
    end

data/lib/rubylexer/version.rb

@@ -1,3 +1,3 @@
 class RubyLexer
-  VERSION='0.7.7'
+  VERSION='0.8.0'
 end

data/rubylexer.gemspec

@@ -1,9 +1,12 @@
 # -*- encoding: utf-8 -*-
+dir=File.dirname(__FILE__)
+require "#{dir}/lib/rubylexer/version"
+RubyLexer::Description=open("#{dir}/README.txt"){|f|
+  f.read[/^==+ ?description[^\n]*?\n *\n?((?:(?!\n *\n).)*)\n *\n/im,1]
+}
+RubyLexer::Latest_changes="###"+open("#{dir}/History.txt"){|f| f.read[/\A===(.*?)(?====)/m,1] }
 
-require "#{File.dirname(__FILE__)}/lib/rubylexer/version"
-RubyLexer::Description=open("README.txt"){|f| f.read[/^==+ ?description[^\n]*?\n *\n?(.*?\n *\n.*?)\n *\n/im,1] }
-RubyLexer::Latest_changes="###"+open("History.txt"){|f| f.read[/\A===(.*?)(?====)/m,1] }
-
+@@the_gem=
 Gem::Specification.new do |s|
   s.name = "rubylexer"
   s.version = RubyLexer::VERSION
@@ -15,9 +18,10 @@ Gem::Specification.new do |s|
   s.homepage = %{http://github.com/coatl/rubylexer}
   s.rubyforge_project = %q{rubylexer}
 
-  s.files = `git ls-files`.split
+  s.files = `git ls-files`.split - ['.gitignore']
   s.test_files = %w[test/test_all.rb]
   s.require_paths = ["lib"]
+  s.bindir = "bin"
   s.extra_rdoc_files = ["README.txt", "COPYING"]
   s.has_rdoc = true
   s.rdoc_options = %w[--main README.txt]
@@ -30,11 +34,11 @@ Gem::Specification.new do |s|
     s.specification_version = 2
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency("sequence", [">= 0.2.3"])
+      s.add_runtime_dependency("sequence", [">= 0.2.4"])
     else
-      s.add_dependency("sequence", [">= 0.2.3"])
+      s.add_dependency("sequence", [">= 0.2.4"])
     end
   else
-    s.add_dependency("sequence", [">= 0.2.3"])
+    s.add_dependency("sequence", [">= 0.2.4"])
   end
 end

data/test/bad/ruby_lexer.rb

@@ -0,0 +1,7 @@
+class RubyLexer
+  def initialize(*args)
+    p :the_wrong_rubylexer!
+    $the_wrong_rubylexer=1
+  end
+end
+$the_wrong_rubylexer=0

data/test/code/deletewarns.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 =begin legal crap
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public

data/test/code/dumptokens.rb

@@ -1,81 +1 @@
-#!/usr/bin/env ruby
-=begin legal crap
-    rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Lesser General Public
-    License as published by the Free Software Foundation; either
-    version 2.1 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-=end
-
-$Debug=true
-require 'rubylexer'
-require 'getoptlong'
-
-#def puts(x) end
-
-#a Token#inspect that omits the object id
-class RubyLexer
-class Token
-  def strify
-    [self.class.name[/[^:]+$/],": ",instance_variables.sort.collect{|v| 
-      [v,"=",instance_variable_get(v).inspect," "]
-    }].join
-  end
-end
-end
-
-name=macros=silent=file=nil
-options={}
-#allow -e
-opts=GetoptLong.new(
-  ["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT],
-  ["--silent", "-s", GetoptLong::NO_ARGUMENT],
-  ["--macro", "-m", GetoptLong::NO_ARGUMENT],
-  ["--ruby19", "-9", GetoptLong::NO_ARGUMENT]
-)
-opts.each{|opt,arg|
-  case opt
-  when '--eval'
-    file=arg
-    name='-e'
-  when '--silent'
-    silent=true
-  when '--macro'
-    macros=true
-  when '--ruby19'
-    options[:rubyversion]=1.9
-  end
-}
-     
-#determine input file and its name if not already known
-file||=if name=ARGV.first
-    File.open(name)
-  else 
-    name='-'
-    $stdin.read
-  end
-
-args=name, file
-args.push 1,0,options unless options.empty?
-lexer=RubyLexer.new(*args) 
-lexer.enable_macros! if macros
-if silent
-  until RubyLexer::EoiToken===(tok=lexer.get1token)
-  end
-else
-  until RubyLexer::EoiToken===(tok=lexer.get1token)
-    puts tok.strify
-  end
-end
-puts tok.strify #print eoi token
+load File.join(Dir.pwd,"bin/rubylexer")

data/test/code/heredoc_blast_test.rb

@@ -0,0 +1,112 @@
+require 'open3'
+
+require 'rubygems'
+require 'rubylexer/test/testcases'
+require 'test/code/rubylexervsruby'
+#require 'test/code/test_1.9'
+
+  SEP='';
+  'caleb clausen'.each_byte{|ch| SEP<<ch.to_s(2).gsub('0','-').gsub('1','+')}
+  SEP<<'(0)'
+
+#require 'test/code/rubyoracle'
+class<<RubyLexerVsRuby
+
+  def progress ruby,input; end 
+=begin oracular version was just a bad idea....
+  def ruby_parsedump(input,output,ruby)
+    #todo: use ruby's md5 lib
+    #recursive ruby call here is unavoidable because -y flag has to be set
+
+    @oracle||= Open3.popen3("#{ruby} -w -y -c")
+    @oracle[0].write IO.read(input)
+    @oracle[0].flush
+    #timeout=Time.now+0.1
+    data=''
+    while data.empty? #  and Time.now<timeout
+      begin 
+        data<< chunk=@oracle[2].read_nonblock(1024) 
+        timeout=Time.now+0.02
+      rescue EOFError
+#        data<<"\nError: premature eof...\n"
+        break
+      rescue Errno::EAGAIN
+        break if data[/^Reading a token: \Z/] and Time.now>=timeout
+      end while chunk
+    end
+
+    status=0
+    lines=data.split("\n")
+    File.open(output,"w") { |outfd|
+      lines.each{|line| 
+        outfd.puts(line) if /^Shifting/===line
+        if /^#{DeleteWarns::WARNERRREX}|^Error|^(Now at end of input\.)/o===line
+          outfd.puts(line) 
+          if status.zero? and $2!="warning"
+            status=1 unless $4 #unless end of input seen
+            @oracle.each{|fd| fd.close} if @oracle
+            @oracle=nil
+          end
+        end
+      }
+    }
+    return status
+  end
+=end
+end
+
+  require 'test/unit'
+  
+  class LexerTests<Test::Unit::TestCase
+    class LexerTestFailure<RuntimeError; end
+    class DifferencesFromMRILex<LexerTestFailure; end
+    
+    i=-1
+    test_code= TestCases::TESTCASES.grep(/insane/).map{|tc|
+      i+=1
+      name="testcase_#{i}__"
+      esctc=tc.gsub(/['\\]/){"\\"+$&}
+      %[  
+         define_method '#{name}' do 
+           10_000.times do
+             lexer=RubyLexer.new('__#{name}','#{esctc}')
+             until RubyLexer::EoiToken===(token=lexer.get1token)
+               #...do stuff w/ token...
+             end
+           end
+         end  
+      ]
+    }
+    
+    illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
+      i+=1
+      name="testcase_#{i}__"
+      esctc=tc.gsub(/['\\]/){"\\"+$&}
+      %[  
+         define_method '#{name}' do 
+           difflines=[]
+           begin
+             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
+             unless difflines.empty?
+               puts '#{esctc}'
+               puts difflines.join
+               raise DifferencesFromMRILex
+             end
+             res or raise LexerTestFailure, ''
+           rescue LexerTestFailure
+             puts 'warning: test failure lexing "#{esctc}"' 
+           rescue Interrupt; exit
+           rescue Exception=>e;
+             message=e.message.dup<<"\n"+'while lexing: #{esctc}'
+             e2=e.class.new(message)
+             e2.set_backtrace(e.backtrace)
+             raise e2
+           end
+         end  
+      ]
+    }
+
+    src=(test_code+illegal_test_code).join
+#   puts src
+    eval src
+  end

data/test/code/locatetest.rb

@@ -1,6 +1,6 @@
 =begin legal crap
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public

data/test/code/regression.rb

@@ -60,32 +60,34 @@ end
   class LexerTests<Test::Unit::TestCase
     class LexerTestFailure<RuntimeError; end
     class DifferencesFromMRILex<LexerTestFailure; end
-    
-    i=-1
+
+    i=0
+     
     test_code= TestCases::TESTCASES.map{|tc|
       i+=1
-      name="testcase_#{i}__"
       esctc=tc.gsub(/['\\]/){"\\"+$&}
+      shorttc=esctc[0..200]
+      shorttc.chop! while shorttc[-1]==?\\
+      shorttc.gsub!("\0",'\\\\0') 
+      name="test_lexing_of_#{shorttc}"
       %[  
          define_method '#{name}' do 
            difflines=[]
            begin
-             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
+             res=RubyLexerVsRuby.rubylexervsruby('__testcase_#{i}','#{esctc}',difflines)
              unless difflines.empty?
-               puts '#{esctc}'
-               puts difflines.join
-               raise DifferencesFromMRILex
+               raise DifferencesFromMRILex, difflines.join
              end
              res or raise LexerTestFailure, ''
-           rescue Interrupt; exit
-           rescue Exception=>e 
-             message=e.message.dup<<"\n"+'while lexing: #{esctc}'
-             e2=e.class.new(message)
-             e2.set_backtrace(e.backtrace)
-             raise e2
            end
          end  
       ]
+           #rescue Interrupt; exit
+           #rescue Exception=>e 
+           #  message=e.message.dup<<"\n"+'while lexing: \#{esctc}'
+           #  e2=e.class.new(message)
+           #  e2.set_backtrace(e.backtrace)
+           #  raise e2
     }
     
     illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
@@ -98,25 +100,23 @@ end
            begin
              res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
              unless difflines.empty?
-               puts '#{esctc}'
-               puts difflines.join
-               raise DifferencesFromMRILex
+               raise DifferencesFromMRILex, difflines.join
              end
              res or raise LexerTestFailure, ''
            rescue LexerTestFailure
              puts 'warning: test failure lexing "#{esctc}"' 
-           rescue Interrupt; exit
-           rescue Exception=>e;
-             message=e.message.dup<<"\n"+'while lexing: #{esctc}'
-             e2=e.class.new(message)
-             e2.set_backtrace(e.backtrace)
-             raise e2
            end
          end  
       ]
+           #rescue Interrupt; exit
+           #rescue Exception=>e;
+           #  message=e.message.dup<<"\n"+'while lexing: \#{esctc}'
+           #  e2=e.class.new(message)
+           #  e2.set_backtrace(e.backtrace)
+           #  raise e2
     }
 
     src=(test_code+illegal_test_code).join
-#   puts src
+    #puts src
     eval src
   end

data/test/code/rubylexervsruby.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/ruby
-=begin legal crap
+=begin legal
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -29,12 +29,43 @@ class<<RubyLexerVsRuby
 ENABLEMD5=false
 def nop_ruby(cmd,input,output,stringdata)
 #   system %[echo "BEGIN{exit};">#{output}]
-   File.open(output,'w'){|f| f.write "BEGIN{exit};\n" }
-   if stringdata 
-     File.open(output,'a'){|f| f.write stringdata }     
-   else
-     system [cmd,'"'+input+'"','>>',output].join(' ')
-   end
+   File.open(output,'w'){|f|
+     if stringdata 
+       stringdata=stringdata.dup
+       first=stringdata.slice! /\A.*\n?/
+       second=stringdata.slice! /\A.*\n?/
+     else
+       input=IO.popen %{#{cmd} "#{input}"}
+       first=input.readline
+       second=input.readline
+       stringdata=input.read
+     end
+     if first[0,2]=="#!"
+       if /\A\s*#.*coding/o===second
+         f.write first
+         f.write second
+         f.write "BEGIN{exit};\n"
+         f.write stringdata
+       else
+         f.write first
+         f.write "BEGIN{exit};\n"
+         f.write second
+         f.write stringdata
+       end
+     else
+       if /\A\s*#.*coding/o===first
+         f.write first
+         f.write "BEGIN{exit};\n"
+         f.write second
+         f.write stringdata
+       else
+         f.write "BEGIN{exit};\n"
+         f.write first
+         f.write second
+         f.write stringdata
+       end
+     end
+   }
 end
 
 def ruby_parsedump(input,output,ruby)
@@ -45,17 +76,28 @@ def ruby_parsedump(input,output,ruby)
   ENABLEMD5 and system "md5sum -c #{input}.md5 2>/dev/null" and return
 
   status=0
+#lexing_of_class.String
+  if RUBY_VERSION>"1.8" #workaround for 1.9 crash ???!!!
+    warn "warning: dodging segfault in mri 1.9 by pushing work off to egrep(?!)" unless defined? $parsedump_19_crash_warned
+    $parsedump_19_crash_warned=true
+    system "#{ruby} -w -y < #{input} 2>&1 | egrep '^Shifting|^((Reading a token: )?-:([0-9]+): (warning|(syntax )?error)(: (.+))?)' > #{output}"
+  else
+
   IO.popen("#{ruby} -w -y < #{input} 2>&1"){ |pipe| 
     File.open(output,"w") { |outfd|
       pipe.each{ |line|
         outfd.print(line) \
           if /^Shifting|^#{DeleteWarns::WARNERRREX}/o===line
+        #WARNERRREX='(?:Reading a token: )?-:(\d+): (warning|(?:syntax )?error)(?:: ([^\n]+))?'
         #elsif /(warning|error)/i===line
         #  raise("a warning or error, appearently, not caught by rex above: "+line)
       }
-      pid,status=Process.waitpid2 pipe.pid #get err status of subprocess
+      #pid,status=Process.waitpid2 pipe.pid #get err status of subprocess
     } 
   }
+  end
+
+  status=$?
   ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
   return status>>8
 end
@@ -92,7 +134,8 @@ expected_failures=Dir.getwd+"/test/code/"+File.basename(input)+".expected_failur
 nop_ruby "#{input[/\.gz$/]&&'z'}cat", input, nopfile, stringdata
 
 
-legal=ruby_parsedump nopfile, origfile, ruby
+ruby_parsedump nopfile, origfile, ruby
+`#{ruby} -c #{nopfile} >/dev/null 2>/dev/null`; legal=$?.to_i
 if legal.nonzero?
   puts "skipping #{input}; not legal"
   return true
@@ -106,11 +149,14 @@ rescue Exception=>rl_oops
 end
 
 begin
-  raise unless 0==ruby_parsedump( _ttfile, p_ttfile, ruby )
-  raise unless 0==ruby_parsedump( mttfile, pmttfile, ruby )
+  p_tt_fail=true unless 0==ruby_parsedump( _ttfile, p_ttfile, ruby )
+  pmtt_fail=true unless 0==ruby_parsedump( mttfile, pmttfile, ruby )
 rescue Exception=>ru_oops
 end
 
+warn "syntax error parsing #{pmttfile}" if pmtt_fail
+warn "syntax error parsing #{p_ttfile}" if p_tt_fail
+
 if rl_oops
   if ru_oops
     #good, ignore it
@@ -123,6 +169,7 @@ elsif ru_oops
   return true
 end
 
+
 if File.exists?(p_ttfile)
   IO.popen("diff --unified=1 -b #{origfile} #{p_ttfile}"){ |pipe|
   File.open(p_ttdiff,"w") { |diff|