rubylexer 0.6.2 → 0.7.0

data/test/data/heart.rb

@@ -0,0 +1,7 @@
+#-Ku required on command line...
+def ♥(name)
+  puts "I ♥ you, #{name}!"
+end
+
+♥ ARGV.shift
+#♥

data/test/data/if.rb

@@ -0,0 +1,6 @@
+if true
+  puts 1234
+end
+
+
+puts "1234" if true

data/test/data/jarh.rb

@@ -0,0 +1,369 @@
+print ',rekcah ybuR rehtona tsuJ'.reverse
+
+print %q/92G5S="!A;F]T:&5R(%)U8GD@:&%C:V5R+```/.unpack('u')
+
+print %q/SnVzdCBhbm90aGVyIFJ1YnkgaGFja2VyLA==/.unpack('m')
+
+print 'Whfg nabgure Ehol unpxre,'.tr('A-Za-z', 'N-ZA-Mn-za-m')
+
+print({'ruby' => 'Just another Ruby hacker,'}['ruby'])
+
+print 'jUsT aNoThEr '.capitalize, 'RuBy HaCkEr,'.capitalize
+
+print ['Just another Ruby hacker,'][0]
+
+print ['hacker,', 'Ruby ', 'another ', 'Just '].reverse
+
+proc {print 'Just another Ruby hacker,'}.call
+
+def pri() print 'Just another Ruby hacker,' end; pri
+
+for i in 'Just another Ruby hacker,' do print i end
+
+print `echo -n Just another Ruby hacker,`
+
+print 'Just another Ruby hacker,'.split('')
+
+print 'JQust aQnotQher RQuby hQackQer,'.delete('Q')
+
+'Just another Ruby hacker,'.scan(/(.)/) {print $1}
+
+print 'Just ' + 'another ' + 'Ruby ' + 'hacker,'
+
+print ['4a75737420616e6f746865722052756279206861636b65722c'].pack('H*')
+
+print 'Just another Ruby hacker,'.split(/(.)/)
+
+print %w(Just another Ruby hacker,).join(' ')
+
+print({'another '=>'Just ', 'hacker,'=>'Ruby '}.invert.sort)
+
+print eval(%-eval %: %|Just another Ruby hacker,| :-)
+
+#;eval %qPprint eval %-eval %:'Just another Ruby hacker,' :-P
+
+print Hash.new('Just another Ruby hacker,')[:Ruby]
+
+print ['Just ', ['another ', ['Ruby ', ['hacker,']]]].flatten
+
+print Struct.new(*%w|Ruby Just another Ruby hacker,|)[*[0]*4].members.join(' ')
+
+print Time.now.strftime('Just another Ruby hacker,')
+
+print "   Just another Ruby hacker,    \r\n".strip
+
+print ['Just ', nil, 'another ', nil, 'Ruby ', nil, 'hacker,'].compact
+
+['hacker,', 'Ruby ', 'another ', 'Just '].reverse_each do |x| print x end
+
+/Just another Ruby hacker,/.source.display
+
+send :print, 'Just another Ruby hacker,'
+
+print ['Just ', 'another ', 'Ruby '] | ['another ', 'Ruby ', 'hacker,']
+
+print ['Just ', 'another ', 'Perl', 'Ruby ', 'hacker,'] - ['Perl']
+
+print [['', 'Just another Ruby hacker,']].assoc('')
+
+print ['Just another Ruby hacker,'].pack('a*')
+
+print 'Just another Ruby hacker,'.unpack('a*')
+
+print [['Just another Ruby hacker,'].pop].sort.shift
+
+print [['Just another Ruby hacker,', '']].rassoc('')
+
+{'Just another ' => 'Ruby hacker,'}.to_a.display
+
+print 'Just another Ruby hackeq,'.succ
+
+print 'jUST ANOTHER rUBY HACKER,'.swapcase
+
+print ['Just another Ruby hacker,'].find {|x| x}
+
+print %w(Just another Ruby hacker,).find_all {|x| x}.join(' ')
+
+print %w(Just another Ruby hacker,).grep(/./).join(' ')
+
+Ruby = 'Just another Ruby hacker,' and print Object.const_get(:Ruby)
+
+Proc.new {print 'Just another Ruby hacker,'}[]
+
+print 'JJuusstt  aannootthheerr  RRuubbyy  hhaacckkeerr,,'.squeeze
+
+print [1249211252, 543256175, 1952998770, 542274914, 2032167009, 1667982706].pack('N*'), ','
+
+$> << 'Just ' << 'another ' << 'Ruby ' << 'hacker,'
+
+5.times do|i|'Jaebcunrykso  ettRhr hua,'.scan(/.{5}/) do print $&[i,1] end end
+
+a=',rekcah ybuR rehtona tsuJ'.split('');print a.pop while not a.empty?
+
+print ',Pr-eok,c"a$h! kypbquuRA %roeqhbtaoon;ab rtesbujJ&'.gsub(/(.)./, '\1').reverse
+
+print /.*/.match('Just another Ruby hacker,')
+
+print $& if /.*/ === 'Just another Ruby hacker,'
+
+case 'Just another Ruby hacker,' when /.*/; print $~[0] end
+
+print Marshal.load(Marshal.dump("Just another Ruby hacker,"))
+
+'Just another Ruby hacker,'.each_byte do |x| print x.chr end
+
+print '/hacker,/another /Ruby /Just '.split('/').sort.values_at(1,3,2,4)
+
+print 'Just another Perl hacker,'.sub('Perl', 'Ruby')
+
+#$><<'RbyckJust another,'.split('').values_at(5..16,9,0,6,1,2,9,14,10,3,4,15..17)
+
+('a'..'y').to_a.join.tr('a-y','Just another Ruby hacker,').display
+
+begin print 'Just another ' ensure print 'Ruby hacker,' end
+
+'J,ursetk caanho tyhbeurR  Rruebhyt ohnaac ktesru,J'.scan(/.(.)/).reverse.display
+
+'J,ursetk caanho tyhbeurR  Rruebhyt ohnaac ktesru,J'.gsub(/(.)./,'\1').display
+
+a='J,ursetk caanho tyhbeurR ';0.step(24,2){|i|print a[i,1]};23.step(1,-2){|i|print a[i,1]}
+
+a='Js nte uyhce,rka bRrhoatu'.split('');12.times{print a.shift,a.pop};print a
+
+a='Js nte uyhce,rka bRrhoatu';begin a.sub!(/^(.)(.*)(.)$/){print $1,$3;$2}end while$1;print a
+
+a='J Ruby hacker,ust another';11.times{a.sub!(/^(.)(.*)(.)$/,'\1\3\2')};print a
+
+print 'Just. another! -Ruby? $hacker,'.tr_s('-.!?$ ', ' ')
+
+' e m r,yJaRnafatbpcuekewhjhskvngohrlrxsctdtiubuoyq'.scan(/(.)(.)/).collect!{|x|[x[1],x[0]]}.sort.collect!{|x|x[1]}.display
+
+'e m r y,aJnRfatapbuckewejhshvkgnholrxrcsdtitbuouqy'.scan(/../).sort.collect!{|x|x[1,1]}.display
+
+'4a75737420616e6f746865722052756279206861636b65722c'.scan(/../){print $&.hex.chr}
+
+print ['Just another Perl hacker,'].fill('Just another Ruby hacker,')
+
+$><<{1=>'Just ',2=>'another ',3=>'Ruby ',4=>'hacker,'}.sort.to_s.delete('1-4')
+
+Kernel.print %q<Just another Ruby hacker,>
+
+class Just_another_Ruby_hackerq;end;print Just_another_Ruby_hackerq.to_s.tr('_q',' ,')
+
+Kust_another_Ruby_hacker=0;print Module.constants.grep(/Kust/)[0].tr('K_','J '), ','
+
+module Lust_another_Ruby_hackerq; print name.tr('L_q', 'J ,') end
+
+class Bar; print 'Just another Ruby hacker,' end
+
+class Baz; def initialize() print 'Just another Ruby hacker,' end end; Baz.new
+
+eval "A<')I;G0@)TIU<W0@86YO=&AE<B!2=6)Y(&AA8VME<BPG".unpack('u')[0]
+
+eval "cHJpbnQgJ0p1c3QgYW5vdGhlciBSdWJ5IGhhY2tlciwn".unpack('m')[0]
+
+$stdout.write 'Just another Ruby hacker,'
+
+"Just \nanother \nRuby \nhacker,\n".each do |x| print x.chomp end
+
+print $' if /\n/x === "\nJust another Ruby hacker,"
+
+raise 'Just another Ruby hacker,' rescue print $!
+
+print File.basename('~/Just another Ruby hacker,.rb','.*')
+
+proc{|x|x['Just another Ruby hacker,']}[proc{|x|print x}]
+
+method(:print)['Just another Ruby hacker,']
+
+'Just another Ruby hacker,'.method(:display)[]
+
+print ''.replace('Just another Ruby hacker,')
+
+instance_eval do print 'Just another Ruby hacker,' end
+
+Kernel.module_eval do print 'Just another Ruby hacker,' end
+
+print %%\%s%%%%Just another Ruby hacker,%%%%%%%%%
+
+print !?????:???%?Just another Ruby hacker,?:??
+
+#({}.default='Just another Ruby hacker,')[0].display
+
+print Hash['Just another ', 'Ruby hacker,']
+
+print 'Just ' 'another ' 'Ruby ' 'hacker,'
+
+print File.dirname('Just another Ruby hacker,/Just another Ruby porter,')
+
+def $_.singleton_method_added(*) print 'Just another Ruby hacker,' end
+
+print ['Just another Ruby hacker,'].delete_at(0)
+
+print '%s %s %s %s' % %w[Just another Ruby hacker,]
+
+'cker,by haer RuanothJust '.scan(/.{5}/).reverse.display
+
+(97..121).to_a.pack('c*').tr('a-y','Just another Ruby hacker,').display
+
+'abcdefghijklmnopqrstuvwxy'.tr('a-y','Just another Ruby hacker,').display
+
+',rekcah ybuR rehtona tsuJ'.scan(/.{1}/).reverse.display
+
+print 'Just another Ruby hacker,'%%; (^_^;;;
+
+print('Just another Ruby hacker,'% %- ;-)
+
+'r,keac hbyRur heotant usJ'.scan(/.{1,2}/).reverse.display
+
+print 'Just another Ruby hacker,' * ('Just another Ruby hacker,' =~ /u/)
+
+print 'uJtsa onhtreR bu yahkcre,'.unpack('v12c').pack('n12c')
+
+print 'uJtsa onhtreR bu yahkcre,'.gsub(/(.)(.?)/, '\2\1')
+
+'uJtsa onhtreR bu yahkcre,'.scan(/(.)(.?)/){|x,y| print y+x}
+
+print 'Just another Ruby hacker,'['Just another Ruby hacker,']
+
+print 'Just another Ruby hacker,'[/.*/]
+
+print [].push('Just ', 'another ', 'Ruby ', 'hacker,')
+
+print 'Just another Ruby hacker,'[0..-1]
+
+print 'Just another Ruby hacker,'.instance_eval{self}
+
+class String; def +@() print self end; +'Just another Ruby hacker,' end
+
+print '1J2u3s4t5 6a7n8o9t0h1e2r3 4R5u6b7y8 9h0a1c2k3e4r5,'.delete('0-9')
+
+'Jaubsctd eafngohtihjekrl mRnuobpyq rhsatcukvewrx,y'.gsub(/(.)./, '\1').display
+
+#$><<{'Just another Ruby hacker,'=>''}.default=0
+
+print eval('Just another Ruby hacker,'.dump)
+
+print 'Just another Ruby hacker'.concat(?,)
+
+print 'Just another Ruby hacker,'.intern.to_s
+
+print 'Just another Ruby hacker,'.dump[1..-2]
+
+'Just another Ruby hacker,'.each_line do |x| print x end
+
+['JUST ANOTHER ', 'RUBY HACKER,'].each do |x| print x.capitalize end
+
+printf '%.25s', 'Just another Ruby hacker,JARH'
+
+print 'Just another Ruby hacker,'.split(/Z/)[0]
+
+print 'Just another Ruby hacker,'.split(//)
+
+print %w.J u s t \ a n o t h e r \ R u b y \ h a c k e r ,.
+
+print String([%[Just another Ruby hacker,]])
+
+print catch(:x){throw :x, 'Just another Ruby hacker,'}
+
+r,w = IO::pipe;w.print 'Just another Ruby hacker,';w.close;print r.gets
+
+print sprintf('%s', format('%s' % '%s', 'Just another Ruby hacker,'))
+
+print eval('Just another Ruby hacker,'.inspect)
+
+/#{print 'Just another Ruby hacker,'}/
+
+print 'Just another Ruby hacker,'.scan(/./).collect{|x|x[0]}.pack('U*')
+
+print 'Just another Ruby hacker,'.scan(/./).pack('a'*25)
+
+'Just another Ruby hacker,'.send 'yalpsid'.reverse.intern
+
+print %w.Just another Ruby Ruby Ruby hacker,..uniq.join(' ')
+
+{}.fetch(:x){'Just another Ruby hacker,'}.display
+
+{}.delete(:x){'Just another Ruby hacker,'}.display
+
+print [:x].map{'Just another Ruby hacker,'}
+
+{'Ruby hacker,'=>'Just another'}.each{|*x|print x.reverse.join(' ')}
+
+print [].unshift('Ruby hacker,').unshift('Just another').join(' ')
+
+{3=>'Ruby ',2=>'another ',4=>'hacker,',1=>'Just '}.values_at(1,2,3,4).display
+
+print [[0,'Just another Ruby hacker,'],1].first.last
+
+{''=>'JARH'}.update(''=>'Just another Ruby hacker,').display
+
+{:JARH => 'Just another Ruby hacker,'}.values.display
+
+{'Just another Ruby hacker,'=>:JARH}.invert[:JARH].display
+
+print ['Just another Ruby hacker,'=>:JARH][0].keys
+
+print [{}].slice(0..0)[0].store('', 'Just another Ruby hacker,')
+
+print 'Just another Ruby hacker,'.split.join(' ')
+
+begin print 'Just another Ruby hacker,' end while false
+
+begin rescue then else print 'Just another Ruby hacker,' end
+
+print //.match('Just another Ruby hacker,').string
+
+callcc{print 'Just another Ruby hacker,'}
+
+Thread.start{print'Just another Ruby hacker,'}.join
+
+('JARH'..'Just another Ruby hacker,').end.display
+
+('Just another'..'Ruby hacker,').to_s.tr_s('.', ' ').display
+
+unless true then else 'Just another Ruby hacker,' end.display
+
+loop do print 'Just another Ruby hacker,'; break end 
+
+1.times do print 'Just another Ruby hacker,' end
+
+0.upto(0) do print 'Just another Ruby hacker,' end
+
+0.downto(0) do print 'Just another Ruby hacker,' end
+
+print Struct.new('Just_another_Ruby_hacker').to_s.split(':')[2].tr('_',' '),','
+
+def (x='Just another Ruby hacker,').foo() print self end; x.foo
+
+begin print 'Just another Ruby hacker,' end until true
+
+for i in %w[Just\  another\  Ruby\  hacker,] do print i end
+
+'Just another Ruby hacker,'.each_byte do |x| putc x end
+
+',rekcah ybuR rehtona tsuJ'.scan(/./).reverse_each do |x| putc x end
+
+print Regexp.quote('Just another Ruby hacker,').delete('\\')
+
+print true ? "Just another Ruby hacker," :-P
+
+print true ? "Just another Ruby hacker," :-D
+
+print *("Just another Ruby hacker,".."Just another Ruby hacker,").to_a
+
+print eval(%<%%Just another Ruby hacker,%>)
+
+print case when true then "Just another Ruby hacker," end
+
+print %w[Just another Ruby hacker,] * " "
+
+begin end if print "Just another Ruby hacker,"
+
+print begin "Just another Ruby hacker," end
+
+print "Just another Ruby hacker,".center(10000).strip
+
+print "Just " or print "another " or print "Ruby " or print "hacker,"
+
+print :"Just another Ruby hacker," 

data/test/data/lbrace.rb

@@ -0,0 +1,4 @@
+p({1=>2,3=>4}) { foo bar baz }
+
+
+

data/test/data/lbrack.rb

@@ -0,0 +1,4 @@
+p [1,2,3][1]
+begin p[1]
+rescue
+end

data/test/data/p-op.rb

@@ -0,0 +1,8 @@
+if false
+  p+1
+  p&1
+  p|1
+  p^1
+  p%1
+  p*1
+end

data/{testdata → test/data}/p.rb

@@ -1,13 +1,551 @@
+if false
+  return {}.size
+  1.return {}.size
+
+  break {}.size
+  1.break {}.size
+
+  next {}.size
+  1.next {}.size
+
+  raise {}.to_s+"****"
+  1.raise {}.to_s+"****"
+
+  throw {}.to_s+"****"
+  1.throw {}.to_s+"****"
+end
+
+module SR
+def SelfReferencing 
+#old name alias
+end
+
+def SelfReferencing #old name alias
+end
+
+def SelfReferencing#old name alias
+end
+
+def SelfReferencing;end#old name alias
+end
+
+P ::Class 
+
+p <<Class
+  zzzz
+Class
+
+p(String <<Class) 
+sgsdfgf
+  Class
+Class
+
+proc{q=1;def q.foo; end}  #q should be varnametoken, both times
+module Defined_p_syntax_tests
+  def self.defined?(foo) :baz end  #should be methname
+  def defined?(foo) :bar end  #should be methname
+  def ameth
+    p(defined? 44)  #should be keyword
+    p(self.defined? 44) #should be methname
+  end
+end
+
+p 2**3[4] #should be nowstoken before bracket
+
+proc{
+p /$/  #regexp
+p /=$/ #operator
+p /$/  #operator
+}
+
+proc{
+p %=b=
+2
+p %(1)
+}
+
+proc{
+p *=5
+p %(1)
+}
+
+proc{
+p &nil
+p &=1
+p %(1)
+}
+
+p %(1)
+
+p %\hah, backslash as string delimiter\
+def foo(bar=5,tt=6) end
+wwww,eeee=1,2
+proc{a.b,c.d=1,2}
+proc{|a.b,c.d|}
+
+
+
+
+
+
+p foo
+
+if false
+p % foo 
+p % foo 
+p %
+foo
+
+p(% foo )
+p(% foo )
+p(%
+foo
+
+)
+end
+
+p eval "%\sfoo\s"
+
+p eval "%\tfoo\t"
+p eval "%\vfoo\v"
+p eval "%\rfoo\r"
+p eval "%\nfoo\n"
+p eval "%\0foo\0"
+
+p eval "%\r\nfoo\r\n"
+
+#foo
+
+=begin
+
+=end
+
+p <<p
+\
+p
+p
+
+p <<'p'
+\
+p
+p
+
+p <<p
+\n\t\r\v\\
+p
+
+p <<'p'
+\n\t\r\v\\
+p
+
+p <<p
+\
+
+p
+
+
+p %W[\  
+] #ruby interprets this one as a single (totally unescaped) newline,
+  #but i suspect that might be a bug.
+
+p "\
+"
+
+p '\
+'
+
+p /\
+/
+
+p `\
+`
+
+p %w[\
+]
+
+p %[\
+]
+
+p "
+
+"
+
+p %{
+
+}
+
+p <<END
+
+END
+
+p <<'END'
+
+END
+
+
+p
+p(1,2)     #these 2 lines should tokenize identically
+p (1,2)    #except for the extra space on this one
+
+
+
+
+p File
+module Y19  #limit lvar scope
+  a,b,c,(d,e)=1,2,3,[4,5]
+  p a %(4)
+  p c %(4)
+  p d %(4)
+  p e %(4)
+end
+
+def printem___1 a
+   a
+end
+
+class F0000; end
+def F0000; end
+
+f0000=1
+def f0000; end
+
+def printem__1 a
+   p(a +77)
+end
+def printem_1 a,b
+   p(a +77)
+   p(b +77)
+end
+def printem1 a,b,c
+   p(a +77)
+   p(b +77)
+   p(c +77)
+end
+def printem10 a,b,c,d,e,f
+   p(a +77)
+   p(b +77)
+   p(c +77)
+   p(d +77)
+   p(e +77)
+   p(f +77)
+end
+def printem100 a,b,c,d,e,f,*g,&h
+   p(a +77)
+   p(b +77)
+   p(c +77)
+   p(d +77)
+   p(e +77)
+   p(f +77)
+   p(g +77)
+   p(h +77)
+end
+#<<'abc123def'
+ddd="ddd"
+
+#@@ddd=@ddd=$ddd=nil
+def DDD;end
+def ddd2; "asdAds" end
+def (DDD()).foofarendle;end
+def Integer.foofarendle;end
+def @@ddd.foofarendle;  33;end
+def @ddd.foofarendle;  33;end
+def $ddd.foofarendle;  33;end
+def ddd.foofarendle;  33;end
+def ddd2.foofarendle;  33;end
+def (ddd).foofarendle2; end
+def (ddd()).foofarendle2; end
+
+
+def (ddd2).foofarendle;end
+def ddd2.foofarendle;end
+
+
+def (is_a?(File::Stat)).foofarendle;end
+
+p()
+
+p((4\
++5))
+p((4\
+
++5))
+p((4+\
+
++5))
+p((4+\
+
+5))
+
+p(<<-jjj \
+ dsfgdf
+ jjj
+ )
+
+p(<<-jjj 
+ dsfgdf
+ jjj
+ )
+
+p(<<-jjj \
+ dsfgdf
+ jjj
+ +"dfsdfs"
+ )
+
+p(<<-jjj +
+ dsfgdf
+ jjj
+ "dfsdfs"
+ )
+
+
+p( {:rest=>99})
+p %{{{{#{"}}}"}}}}}
+case 1
+when *[2]: p 1
+else p 2
+end
+
+p :"jim";
+p :'jim';
+p %s/jim/;
+p %s"jim";
+p %s'jim';
+p %s`jim`;
+p %s[jim];
+p %s{jim};
+p %s(jim);
+p %s<jim>;
+p %s jim ;
+for bob in [100] do p(bob %(22)) end
+
+
+proc{
+  foo bar=>baz
+  bar %(1)
+}
+
+module Y2534534
+def foo1
+  p (1..10).method(:each)    #implicit parens around the whole thing
+end
+
+def foo2()
+  p((1..10).method(:each))  #explicitly parenthesized... no implicit parens needed
+end
+
+def foo3()
+  p (1..10).method(:each)   #implicit parens around the whole thing
+  #matz ruby parses this differently than identical line in foo1!?
+end
+
+end
+
+
+module Y668
+p [1,2,3,]
+#p {1=>2,3=>4,}  #syntax error
+#p aaa,bbb,ccc,=1,2,3  #syntax error
+p aaa
+p bbb
+p ccc
+proc{|a,b,c,| p a,b,c }.call(1,2,3)
+p proc{|
+a,b,(c,d,),e| 
+        p a %(1)
+        p b %(2)
+        p c %(3)
+        p d %(4)
+        p e %(5)
+[ a,b,c,d,e]}.call(1,2,[3,4],5)
+p (Module.instance_methods - Object.instance_methods).sort  # an outer set of implicit parens is needed
+p(Module.instance_methods - Object.instance_methods).sort  #no outer implicit parens
+"foo".slice (1-2).nil?   #no outer implicit parens
+p (1-2).nil?     #outer implicit parens needed
+p(1-2).nil?     #no outer implicit parens needed
+
+def self.z(*); end
+def self.zz(*,&x); end
+def self.zzz(&x); end
+
+z z z z z z {}
+z z z z z z do end
+
+(/ 1/)
+ p(/ 1/)
+# (p / 1/)  #ruby no like
+
+def ggg(x=nil) p x;9 end
+ (ggg / 1)
+
+false and( true ? f : g )
+false and( f .. g )
+false and( f ... g )
+ p 1&2 #should be OperatorToken, not KeywordToken
+ p 1|2 #ditto
+ p 1**2
+ p 1*2
+ p 1%(2)
+ p 1^2
+ p 1+2
+ p 1-2
+ p 1/2
+
+ p 1?2:3  #keyword
+
+ p 1 ?2:3  #keyword
+
+ p 1? 2:3  #keyword
+
+ p 1?2 :3  #keyword
+
+ p 1?2: 3  #keyword
+
+ p 1 ? 2:3  #keyword
+
+ p 1 ?2 :3  #keyword
+
+ p 1 ?2: 3  #keyword
+
+ p 1? 2 :3  #keyword
+
+ p 1? 2: 3  #keyword
+
+ p 1?2 : 3  #keyword
+
+ p 1 ? 2 : 3  #keyword
+
+ p 1==2
+ p 1===2
+ p 1[2]  #keyword
+ p 1;2  #keyword
+ p 1,2  #keyword
+ p 1.2
+ p 1.class  #keyword
+ p 1..2  #keyword
+ p 1...2  #keyword
+ p 1<2
+ p 1>2
+ p 1<=2
+ p 1>=2
+ p 1<<2
+ p 1>>2
+ p 1!=2  #this is keyword
+ p 1=~2
+ p 1!~2  #this is keyword
+ p 1&&2  #...
+ p 1||2  #...
+ p 1<=>2
+ 
+      define_method(:foo, &proc{:bar}) 
+      define_method :foo, &proc{:bar}
+      define_method(:foo) &proc{:bar}
+      define_method :foo &proc{:bar} 
+      p :k, *nil
+      p :k, *nil, &nil
+      p  *nil
+      p  *nil, &nil
+end
+
+
+module Y19  #limit lvar scope
+  a,b,c,(d,e)=1,2,3,[4,5]
+  p a %(4)
+  p c %(4)
+  p d %(4)
+  p e %(4)
+
+  
+  
+  a=[1,2,3,4,5]
+  
+  def self.g(x=nil); 3 end
+  def a.g=(x) p x end
+  
+  g = 5
+  
+  self.g = 55
+  
+
+  class<<a
+    def bb=(x) p :bb=, x end
+  end
+
+  A=a
+  class<<self
+  def aa; :aa end
+  def bb(arg=nil); p :bb; A end
+  alias bbb bb
+  
+  def m; self end
+  def n; self end
+  
+  def +(other) self end
+  def kk; nil end
+  def kk=(foo); nil end
+  end
+  
+
+  proc{|a[4]|}.call 6
+  proc{|a[b]|}.call 7
+  proc{|a.bb| bb %(9) }.call 9
+  proc{|a[f]| f %(9)  }.call 8
+  proc{|bb(aa).bb| aa %(10) }.call 10
+  proc{|bbb(aa).bb| bbb %(11) }.call 11
+  proc{|t,u,(v,w,x),(y,),z|
+    t %(12)
+    u %(12)
+    v %(12)
+    w %(12)
+    x %(12)
+    y %(12)
+    z %(12)
+  }.call(1,2,[3,4,5],[6],7)
+  proc{|(m).kk,(m+n).kk|
+    m %(13)
+    n %(13)
+    kk %(13)
+  }.call(13,14)
+  
+  proc{|a.g| g %(9)} 
+  
+  p a
+
+end
+
+
+module R34634534
+#how do I avoid getting into trouble with this test case?
+#seems like I should be seeing local vars in the param list of
+#the 2nd call to p....
+  class<<self
+   alias q p
+   alias r p
+   alias s p
+  end
+
+  p(q,r,s)
+  
+  q %(1)
+  r %(1)
+  s %(1)
+
+  p (q,r,s)
+  
+  q %(1)
+  r %(1)
+  s %(1)
+end
+
+
+
+p p :c, :d
 p true ?
   1 : 2
-#breakpoint
 module M555
  def r;4 end
-#breakpoint
  r=r.nil?  #2nd r seems like it should be MethNameToken, but is VarNameToken (for matz too)
-#breakpoint
 end
-#breakpoint
 
 p ?e.+?y
 p ?e.+ ?y
@@ -19,7 +557,6 @@ p ?e.%?y
 p ?e.**?y
 p ?e.&?y
 
-p %s/jim/;
 
 p "#{<<foobar0.each('|'){|s| '\nthort: '+s} }"
 jbvd|g4543ghb|!@G$dfsd|fafr|e
@@ -61,16 +598,30 @@ p eval "%\nfoo\n"
 p eval "%\0foo\0"
 
 p eval "%\r\nfoo\r\n"
+
+#given that \n\r is not considered a valid eol 
+#sequence in ruby (unlike \r\n), the next 2 are ok
 p eval "%\n\rfoo\n\r"  #ruby get this wrong
 p eval "%\n\rfoo\n"    #this shouldn't work but does
+
+#seems like just \r isn't valid newline either, which is
+#puzzling but the following are are correct.
+#does ruby do these all correctly? seems kinda inconsistant
+#either both of these should fail, or both should succeed
 p eval "%\r\nfoo\r"    #this does not work, 
 p eval "%\r\nfoo\n"    #this shouldn't work but does
 
+p eval "%\rfoo\r"    #this works 
+p eval "%\rfoo\n"    #this does not work,
+p eval "%\rfoo\r\n"    #this does not work,
+
+p eval "%\nfoo\r"    #this does not work 
+p eval "%\nfoo\n"    #this works
+p eval "%\nfoo\r\n"    #this works,
 
 p "Hi, my name is #{"Slim #{(4)>2?"Whitman":"Shady"} "}."
 p "Hi, my name is #{"Slim #{(4)<2?"Whitman":"Shady"} "}."
 
-#breakpoint
 
 p(String *Class)
 class String
@@ -82,6 +633,7 @@ def String.*(right) [self,right] end
 def String.<<(right) [self,:<<,right] end
 def String./(right) [self,:/,right] end
 def String.[](right) [self,:[],right] end
+def Class.-@; [:-@, self] end
 p(String::Class)
 p(String:: Class)
 p(String ::Class)
@@ -90,8 +642,32 @@ p(String<<Class)
 p(String<< Class)
 p(String <<Class) 
 sgsdfgf
+  Class
 Class
 p(String << Class)
+p(String<<-Class)
+p(String<< -Class)
+p(String <<-Class) 
+sgsdfgf
+  Class
+Class
+p(String <<-Class) 
+sgsdfgf
+Class
+  Class
+Class
+p(String << -Class)
+
+p(String<<- Class)
+p(String<< - Class)
+p(String <<- Class) 
+  Class
+Class
+p(String <<- Class) 
+Class
+  Class
+Class
+p(String << - Class)
 p(String/Class)
 p(String/ Class)
 p(String /Class/) 
@@ -114,16 +690,16 @@ p(false::to_s)
 p(false ::to_s)
 p(false:: to_s)
 p(false :: to_s)
-#breakpoint
+
 
 class C2
-class <<self
+ class <<self
   def self.p8; p 8 end
   alias p? p
   alias P? p
   alias [] p
   alias <=> p
-end
+ end
 
 q=9
 Q=99
@@ -178,7 +754,6 @@ self <=>:p8
 self <=> :p8
 #false ? self <=> : p8  #gives ruby indigestion
 end
-#breakpoint
 
 p <<stuff+'foobar'.tr('j-l','d-f')
 "more stuff"
@@ -192,12 +767,9 @@ sdfsfdf^^^^@@@
 p
 mix=nil
 p / 5/mix
-#breakpoint
 
 module M33
-#breakpoint
   p="var:"
-#breakpoint
   Q="func:"
   def Q.method_missing(name,*args)
     self+name.to_s+(args.join' ')
@@ -330,10 +902,9 @@ module M33
   p(p> 2)
   p(p > 2)
  
-#breakpoint
 end
 
-#breakpoint
+
 module M34
   p(p~6)
   p(p ~6)
@@ -489,9 +1060,11 @@ p :`
 
 p{}
 p {}
+module Y7284
 a=5
 p p +5
 p a +5
+end
 
 def nil.+(x) ~x end
 def nil.[](*x) [x] end
@@ -507,6 +1080,9 @@ class Foou
  public
  def [] x=-100,&y; p x; 100 end
 end
+
+
+module Y345636
 a0=8
 p Foou.new.[]!false  #value
 p Foou.new.[] !false #value
@@ -537,7 +1113,9 @@ p({:foo=>:bar})   #why does this work? i'd think that ':foo=' would be 1 token
 p   EMPTY = 0
 p   BLACK = 1
 p   WHITE = - BLACK
+end
 
+module Y8576456
  a=b=c=0
   a ? b:c
   a ?b:c
@@ -545,7 +1123,6 @@ p   WHITE = - BLACK
   p(a ? b:c)
   p(a ?b:c)
 
-
 p~4
 p:f
 p(~4){}
@@ -557,7 +1134,10 @@ p def (h="foobar").default= v; p @v=v;v end
 p h
 
 p h.default=:b
+end
+
 
+module Y6543423
 x, (*y) = [:x, :y, :z]
 p x
 p y
@@ -568,7 +1148,7 @@ p y
 
 x, * = [:x, :y, :z]
 p x
-
+end
 
 
 p Array("foo\nbar")
@@ -607,7 +1187,6 @@ without_creating=widgetname=nil
 
 
 
-=begin disable for now
 
 #class, module, and def should temporarily hide local variables
 def mopsdfjskdf arg; arg*2 end
@@ -627,9 +1206,8 @@ p d
 p mopsdfjskdf %(6)     #reads variable
 p proc{mopsdfjskdf %(7)}[] #reads variable
 
-#fancy symbols not supported yet
+#fancy symbols 
 p %s{symbol}
-=end
 
 #multiple assignment test
 proc {
@@ -640,8 +1218,9 @@ proc {
   p(p %(c))
 }.call
 
-
 =begin disable for now
+
+
 p "#{<<kekerz}#{"foob"
 zimpler
 kekerz
@@ -664,9 +1243,26 @@ proc {
   p "jentawz: #{baz=200}"
   p( baz %(9))
 }.call
-=end
+
+#scope of local variables always includes the here document
+#body if it includes the head
+p %w[well, whaddaya know].map{|j| <<-END }
+#{j #previous j should be local var, not method
+}45634543  
+END
+
+
+
+=end disable for now
+
 
 =begin ought to work...ruby doesn't like
+p :$1
+p :$98349576875974523789734582394578
+
+ (p / 1/)  #ruby no like
+
+class F34534334
 class A
 class B
 class C
@@ -677,9 +1273,6 @@ def A::B::C::d() :abcd end
 def A::B::d() :abd end   #this used to work as well... i think
 
 
-p :$1
-p :$98349576875974523789734582394578
-
 def nil.d=; end #this works
 def (;).d=; end
 def ().d=; end
@@ -690,22 +1283,37 @@ p~4{}
 p:f{}
 p ~4{}
 p :f{}
+p ~4 {}
+p :f {}
+p ~ 4{}
 
 def g a=:g; [a] end
 g g g  #this works
 g *g   #this works
 g *g g #this doesn't
+g+g 5  #this doesn't
 
 [nil,p 5]
 "foo"+[1].join' '
+if x=p :a,:b then end
+{:nil?=> 1}
+
+1+[1,2,3][1..-1].index 3
+
+_=@cu.fixup_match_result 'md'  #works
+_=md=@cu.fixup_match_result 'md'  #don't work
 
-class Fook
+3+"".slice 1
+
+""+p ""
+
+#given:
 def foo; end
 
-file.===/#/  #syntax error
+foo.===/#/  #syntax error
 
-file.=== /#/ #no error
-file===//    #no error
+foo.=== /#/ #no error
+foo===/#/    #no error
 
 #these work:
 (not true)
@@ -723,9 +1331,21 @@ p(true and false) #etc
 p(undef foo)
 p(alias bar foo)
 p(BEGIN{p :yyy})
+ p 1{2}
+ p 1 {2}
+p(false ? Q :p8 )
+p(false ? Q:p8 )
 end
 
-=end
+
+if false
+      equal?(p) or Position===p && equal? p.data #syntax error unless add parens, but why?
+end
+
+    raise LexerError, "expected >=#{tok.offset}, got #{endpos}, "\ 
+                      "token #{tok}:#{tok.class}"  #there's a space after the backslash
+
+=end disable for now
 
 proc {
 }
@@ -753,54 +1373,16 @@ end
 func
 
 
+
 p( %r{\/$})
 p( %r~<!include:([\/\w\.\-]+)>~m)
 
-p <<end
-#{compile_body}\
-#{outvar}
-end
-
-
-
-
-
 proc {
   h={:a=>(foo=100)}
   p( foo %(5))
 }.call
 
 
-p "#{<<foobar3}"
-bim
-baz
-bof
-foobar3
-
-p "#{<<foobar2
-bim
-baz
-bof
-foobar2
-}"
-
-
-
-p <<one ; p "#{<<two}"
-1111111111111111
-fdsgdfgdsfffff
-one
-2222222222222222
-sdfasdfasdfads
-two
-
-p "#{<<foobar1.each('|'){|s| '\nthort: '+s}
-jbvd|g4543ghb|!@G$dfsd|fafr|e
-|s4e5rrwware|BBBBB|*&^(*&^>"PMK:njs;d|
-
-foobar1
-}"
-
 
 def foo(a=<<a,b=<<b,c=<<c)
 jfksdkjf
@@ -897,6 +1479,8 @@ def ==(o) 444 end
 def String.ffff4() self.to_s+"ffff" end
 
 p *[]
+#abc123def
+module Y23423423
 for i in \
 [44,55,66,77,88] do p i**Math.sqrt(i) end
 
@@ -905,6 +1489,7 @@ for i in \
 for i in if
 false then foob12345; else [44,55,66,77,88] end do p i**Math.sqrt(i) end
 #)
+#end
 (
 for i in if false then
 foob12345; else [44,55,66,77,88] end do p i**Math.sqrt(i) end
@@ -937,6 +1522,7 @@ for i in (
 [44,55,66,77,88]) do p i**Math.sqrt(i) end
 
 )
+end
 
 
 
@@ -980,7 +1566,8 @@ d e f
 
 p "\""
 
-a=a.to_s
+module Y89567567999
+a='a'
 class <<a
   def foobar
      self*101
@@ -989,11 +1576,13 @@ class <<a
 end
 
 p a.foobar
-
+end
 
 p(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
 p(:%)
+module Y45234234234223567675
 p( { :class => class_=0})
+end
 p cls_name = {}[:class]
 
 
@@ -1026,8 +1615,10 @@ p %w[\]]
 p :+
 p 99 / 3
 
+module Y66
 a=99;b=3
 p 1+(a / b)
+end
 p %Q[\"]
 p %Q[ some [nested] text]
 
@@ -1093,13 +1684,11 @@ p <<stuff+'foobar'.tr('j-l','d-f')
 the quick brown fox jumped over the lazy dog
 stuff
 
-=begin doesn't work
 p <<stuff+'foobar'.tr('j-l','d-f')\
 +"more stuff"
 12345678
 the quick brown fox jumped over the lazy dog
 stuff
-=end
 
 p ENV["AmritaCacheDir"]
 p <<-BEGIN + <<-END
@@ -1118,13 +1707,16 @@ koomblatz!() p 'jdkfsk' end
 
 koomblatz!
 
+module Y3423454
 p f = 3.7517675036461267e+17
+end
 p $10
 p $1001
 p( <<end )
 nine time nine men have stood untold.
 end
 
+
 def jd_to_wday(jd) (jd + 1) % 7 end
 p jd_to_wday(98)
 
@@ -1162,6 +1754,7 @@ clist.each do |str|
 end
 
 S
+
 def printem1 a,b,c
    p(a +77)
    p(b +77)
@@ -1175,11 +1768,6 @@ end
 def printem0(a)
    p(a +77)
 end
-def printem0(a,b,c)
-   p(a +77)
-   p(b +77)
-   p(c +77)
-end
 def printem2 a,b,c; p(a +77); p(b +77); p(c +77) end
 def three() (1+2) end
 
@@ -1194,11 +1782,12 @@ def printem a,b,c
 end
 printem 1,2,3
 
+module Y24345678756
 a=1
 p(a +77)
 
 def hhh(a=(1+2)) a end
-
+end
 
 
 END {
@@ -1213,11 +1802,14 @@ p <<-what
      ? that's
   what
 p proc{||}
+
+module Y5566342346
 for i in if false
 foob12345; else [44,55,66,77,88] end do p i**Math.sqrt(i) end
 p "\v"
 c=0
       while c == /[ \t\f\r\13]/; end
+end