rubylexer 0.7.2 → 0.7.3

data/History.txt

@@ -1,3 +1,28 @@
+=== 0.7.3/4-19-2009
+* 9 Bugfixes:
+  * remember whether comma was seen in paren context
+  * reducing the warning load
+  * remember whether we're in a method def when starting a recursive lexer
+  * (so that class vars can be marked as being in a method if in #{})
+  * make sure Token#to_s always outputs _something_ halfway sensible
+  * make sure funclike keywords come out as a keyword
+  * break/next/return method after . was not detected properly
+  * need a NoWsToken before :: in names of compound modules
+  * abort implicit paren contexts when elsif seen
+  * all files should be world-readable now
+
+* 9 Minor Enhancements:
+  * move test data into lib/ dir so I can get to it better from redparse
+  * split parse_keywords into separate methods for each keyword
+  * (I had hoped to speed up keywords, but keyword_def is still slow)
+  * more of the lists of keywords are now available as arrays as well
+  * (parenthesis) KeywordToken needs to know if it's for grouping or call
+  * minor speedups in newline and ident
+  * don't panic if rubygems not available
+  * make linenum publicly available in RubyCode
+  * various other little helper methods needed by redparse in Tokens
+  * hack Rakefile so 'rake test' will stay in 1 process (keeps netbeans happy)
+
 === 0.7.2/10-12-2008
 * 12 Minor Enhancements:
   * a new context for then kw expected

data/Manifest.txt

@@ -43,7 +43,6 @@ test/data/23.rb
 test/data/lbrack.rb
 test/data/untitled1.rb
 test/data/rescue.rb
-test/data/tokentest.assert.rb.can
 test/data/pleac.rb.broken
 test/data/heart.rb
 test/data/s.rb
@@ -63,7 +62,6 @@ test/code/torment
 test/code/locatetest
 test/code/deletewarns.rb
 lib/rubylexer/0.7.1.rb
-rubylexer.vpj
 test/code/all_the_gems.rb
 test/code/all_the_raas.rb
 test/code/all_the_rubies.rb
@@ -73,7 +71,7 @@ test/code/lexloop
 test/code/regression.rb
 test/code/strgen.rb
 test/code/tarball.rb
-test/code/testcases.rb
+lib/rubylexer/test/testcases.rb
 test/data/chunky.plain.rb
 test/data/cvtesc.rb
 test/data/__eof2.rb
@@ -91,10 +89,9 @@ test/data/heremonsters_dos.rb
 test/data/heremonsters_dos.rb.broken
 test/data/heremonsters.rb
 test/data/heremonsters.rb.broken
-test/data/heremonsters.rb.broken.save
 test/data/here_squote.rb
-test/data/illegal_oneliners.rb
-test/data/illegal_stanzas.rb
+lib/rubylexer/test/illegal_oneliners.rb
+lib/rubylexer/test/illegal_stanzas.rb
 test/data/make_ws_strdelim.rb
 test/data/maven2_builer_test.rb
 test/data/migration.rb
@@ -102,10 +99,10 @@ test/data/modl_dos.rb
 test/data/modl_fails.rb
 test/data/modl.rb
 test/data/multilinestring.rb
-test/data/oneliners.rb
+lib/rubylexer/test/oneliners.rb
 test/data/simple_dos.rb
 test/data/simple.rb
-test/data/stanzas.rb
+lib/rubylexer/test/stanzas.rb
 test/data/strdelim_crlf.rb
 test/data/stuff2.rb
 test/data/stuff3.rb

data/Rakefile

@@ -2,9 +2,21 @@
 # Distributed under the terms of Ruby's license.
 require 'rubygems'
 require 'hoe'
+
+if $*==["test"]
+  #hack to get 'rake test' to stay in one process
+  #which keeps netbeans happy
+#  Object.send :remove_const, :RubyLexer
+  $:<<"lib"
+  require 'rubylexer.rb'
+  require "test/unit"
+  require "test/code/regression.rb"
+  Test::Unit::AutoRunner.run
+  exit
+end
+
 require 'lib/rubylexer/version.rb'
  
- 
    readme=open("README.txt")
    readme.readline("\n=== DESCRIPTION:")
    readme.readline("\n\n")
@@ -18,8 +30,8 @@ require 'lib/rubylexer/version.rb'
      _.test_globs=["test/code/regression.rb"]
      _.description=desc
      _.summary=desc[/\A[^.]+\./]
-     _.spec_extras={:bindir=>''}
-     _.rdoc_pattern=/\A(howtouse\.txt|testing\.txt|README\.txt|lib\/.*\.rb)\Z/
+     _.spec_extras={:bindir=>'',:rdoc_options=>'-x lib/rubylexer/test'}
+     _.rdoc_pattern=/\A(howtouse\.txt|testing\.txt|README\.txt|lib\/[^\/]*\.rb|lib\/rubylexer\/[^\d][^\/]*\.rb)\Z/
    end
 
 

data/lib/rubylexer.rb

@@ -52,10 +52,14 @@ class RubyLexer
    BEGINWORDLIST=%w(def class module begin for case do)+OPORBEGINWORDLIST
    OPORBEGINWORDS="(#{OPORBEGINWORDLIST.join '|'})"
    BEGINWORDS=/^(#{BEGINWORDLIST.join '|'})$/o
-   FUNCLIKE_KEYWORDS=/^(break|next|redo|return|yield|retry|super|BEGIN|END)$/
-   VARLIKE_KEYWORDS=/^(__FILE__|__LINE__|false|nil|self|true)$/
-   INNERBOUNDINGWORDS="(else|elsif|ensure|in|then|rescue|when)"
-   BINOPWORDS="(and|or)"
+   FUNCLIKE_KEYWORDLIST=%w/break next redo return yield retry super BEGIN END/
+   FUNCLIKE_KEYWORDS=/^(#{FUNCLIKE_KEYWORDLIST.join '|'})$/
+   VARLIKE_KEYWORDLIST=%w/__FILE__ __LINE__ false nil self true/
+   VARLIKE_KEYWORDS=/^(#{VARLIKE_KEYWORDLIST.join '|'})$/
+   INNERBOUNDINGWORDLIST=%w"else elsif ensure in then rescue when"
+   INNERBOUNDINGWORDS="(#{INNERBOUNDINGWORDLIST.join '|'})"
+   BINOPWORDLIST=%w"and or"
+   BINOPWORDS="(#{BINOPWORDLIST.join '|'})"
    NEVERSTARTPARAMLISTWORDS=/\A(#{OPORBEGINWORDS}|#{INNERBOUNDINGWORDS}|#{BINOPWORDS}|end)([^a-zA-Z0-9_!?=]|\Z)/o
    NEVERSTARTPARAMLISTFIRST=CharSet['aoeitrwu']  #chars that begin NEVERSTARTPARAMLIST
    NEVERSTARTPARAMLISTMAXLEN=7     #max len of a NEVERSTARTPARAMLIST
@@ -155,6 +159,7 @@ class RubyLexer
      @localvars_stack.last
    end
 
+   attr_accessor :in_def
    attr :localvars_stack	
    attr :offset_adjust
    attr_writer :pending_here_bodies
@@ -258,6 +263,7 @@ class RubyLexer
 private
    #-----------------------------------
    def inside_method_def?
+     return true if (defined? @in_def) and @in_def
      @parsestack.reverse_each{|ctx|
        ctx.starter=='def' and ctx.state!=:saw_def and return true
      }
@@ -376,12 +382,14 @@ private
       @moretokens.unshift(*parse_keywords(str,oldpos) do |tok|
         #if not a keyword,
         case str
-          when FUNCLIKE_KEYWORDS; #do nothing
+          when FUNCLIKE_KEYWORDS; except=tok
           when VARLIKE_KEYWORDS,RUBYKEYWORDS; raise "shouldnt see keywords here, now"
         end
         was_last=@last_operative_token
         @last_operative_token=tok if tok
-        safe_recurse { |a| var_or_meth_name(str,was_last,oldpos,after_nonid_op?{true}) }
+        normally=safe_recurse { |a| var_or_meth_name(str,was_last,oldpos,after_nonid_op?{true}) }
+        (Array===normally ? normally[0]=except : normally=except) if except
+        normally
       end)
       return @moretokens.shift
    end
@@ -390,6 +398,7 @@ private
    IDENTREX={}
    def identifier_as_string(context)
       #must begin w/ letter or underscore
+      #char class needs changing here for utf8 support
       /[_a-z]/i===nextchar.chr or return
 
       #equals, question mark, and exclamation mark
@@ -409,7 +418,7 @@ private
         end      
       @in_def_name||context==?: and trailers<<"|=(?![=~>])"
 
-      @file.scan(IDENTREX[trailers]||=/^[_a-z][a-z0-9_]*(?:#{trailers})?/i)
+      @file.scan(IDENTREX[trailers]||=/^(?>[_a-z][a-z0-9_]*(?:#{trailers})?)/i)
    end
 
   #-----------------------------------
@@ -450,7 +459,7 @@ private
      @defining_lvar or case ctx=@parsestack.last
        #when ForSMContext; ctx.state==:for
        when RescueSMContext
-         lasttok.ident=="=>" and @file.match? /\A[\s\v]*([:;#\n]|then[^a-zA-Z0-9_])/m
+         lasttok.ident=="=>" and @file.match?( /\A[\s\v]*([:;#\n]|then[^a-zA-Z0-9_])/m )
        #when BlockParamListLhsContext; true
      end 
    end
@@ -589,20 +598,20 @@ private
 #                     )
 
          #look ahead for closing paren (after some whitespace...)
-         want_parens=false if @file.match? /\A.(?:\s|\v|\#.*\n)*\)/
+         want_parens=false if @file.match?( /\A.(?:\s|\v|\#.*\n)*\)/ )
 #         afterparen=@file.pos
 #         getchar
 #         ignored_tokens(true)
 #         want_parens=false if nextchar==?)
 #         @file.pos=afterparen
          want_parens=true if /^(return|break|next)$/===@last_operative_token.ident and not(
-              KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
+              KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident
             )
          want_parens ? 1 : 0
        when ?},?],?),?;,(?^ unless @enable_macro), ?|, ?>, ?,, ?., ?=; 2
        when ?+, ?-, ?%, ?/, (?^ if @enable_macro)
          if /^(return|break|next)$/===@last_operative_token.ident and not(
-              KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
+              KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident
             )
            1 
          else
@@ -611,7 +620,7 @@ private
        when ?*, ?&
  #        lasttok=@last_operative_token
          if /^(return|break|next)$/===@last_operative_token.ident and not(
-              KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
+              KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident
             )
            1 
          else
@@ -665,7 +674,7 @@ private
          last=result.last
          last.set_callsite! false if last.respond_to? :callsite? and last.callsite? #KeywordToken===last and last.ident==')'
          if /^(break|next|return)$/===name and 
-            !(KeywordToken===lasttok and /^(.|::)$/===lasttok.ident)
+            !(KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident)
            ty=KWParamListContextNoParen 
          else
            ty=ParamListContextNoParen
@@ -802,6 +811,9 @@ private
    #-----------------------------------
    def enable_macros!
      @enable_macro="macro"
+     class <<self
+       alias keyword_macro keyword_def
+     end
    end
    public :enable_macros!
 
@@ -836,13 +848,16 @@ private
    #parse keywords now, to prevent confusion over bare symbols
    #and match end with corresponding preceding def or class or whatever.
    #if arg is not a keyword, the block is called
-   def parse_keywords(str,offset)
+   def parse_keywords(str,offset,&block)
       assert @moretokens.empty?
-      assert !(KeywordToken===@last_operative_token and /A(.|::|def)\Z/===@last_operative_token.ident)
+      assert !(KeywordToken===@last_operative_token and /A(\.|::|def)\Z/===@last_operative_token.ident)
       result=[KeywordToken.new(str,offset)]
 
-      case str
-      when "end"
+      m="keyword_#{str}"
+      respond_to?(m) ? (send m,str,offset,result,&block) : block[MethNameToken.new(str)]
+   end
+   public #these have to be public so respond_to? can see them (sigh)
+   def keyword_end(str,offset,result)
          result.unshift(*abort_noparens!(str))
          @parsestack.last.see self,:semi #sorta hacky... should make an :end event instead?
 
@@ -859,8 +874,10 @@ private
          BEGINWORDS===start or lexerror result.last, "end does not match #{start or "nil"}"
          /^(do)$/===start and localvars.end_block
          /^(class|module|def)$/===start and @localvars_stack.pop
+         return result
+   end
 
-      when "module"
+   def keyword_module(str,offset,result) 
          result.first.has_end!
          @parsestack.push WantsEndContext.new(str,@linenum)
          @localvars_stack.push SymbolTable.new 
@@ -885,17 +902,22 @@ private
            end
            @moretokens.push VarNameToken.new(name,offset+incr)
            break unless dc
+           @moretokens.push NoWsToken.new(offset+md.end(0)-2)
            @moretokens.push KeywordToken.new('::',offset+md.end(0)-2)
          end
          @moretokens.push EndHeaderToken.new(input_position)
-           
+         return result
+   end        
         
 
-      when "class"
+   def keyword_class(str,offset,result)
          result.first.has_end!
          @parsestack.push ClassContext.new(str,@linenum)
+         return result
+   end
+
          
-      when "if","unless" #could be infix form without end
+   def keyword_if(str,offset,result)  #could be infix form without end
          if after_nonid_op?{false} #prefix form
             result.first.has_end!
             @parsestack.push WantsEndContext.new(str,@linenum)
@@ -903,12 +925,23 @@ private
          else #infix form
            result.unshift(*abort_noparens!(str))
          end
-      when "elsif"
+         return result
+   end
+   alias keyword_unless keyword_if
+
+   def keyword_elsif(str,offset,result) 
+         result.unshift(*abort_noparens!(str))
          @parsestack.push ExpectThenOrNlContext.new(str,@linenum)
-      when "begin","case"
+         return result
+   end
+   def keyword_begin(str,offset,result)   
          result.first.has_end!
          @parsestack.push WantsEndContext.new(str,@linenum)
-      when "while","until" #could be infix form without end
+         return result
+   end
+
+   alias keyword_case keyword_begin
+   def keyword_while(str,offset,result) #could be infix form without end
          if after_nonid_op?{false} #prefix form
            result.first.has_end!
            @parsestack.push WantsEndContext.new(str,@linenum)
@@ -917,14 +950,21 @@ private
          else #infix form
            result.unshift(*abort_noparens!(str))
          end
-      when "for"
+         return result
+   end
+
+   alias keyword_until keyword_while
+
+   def keyword_for(str,offset,result)
          result.first.has_end!
          result.push KwParamListStartToken.new(offset+str.length)
          # corresponding EndToken emitted leaving ForContext ("in" branch, below)
          @parsestack.push WantsEndContext.new(str,@linenum)
          #expect_do_or_end_or_nl! str #handled by ForSMContext now
          @parsestack.push ForSMContext.new(@linenum)
-      when "do"
+         return result
+   end
+   def keyword_do(str,offset,result)
          result.unshift(*abort_noparens_for_do!(str))
          if ExpectDoOrNlContext===@parsestack.last
             @parsestack.pop
@@ -936,12 +976,17 @@ private
             localvars.start_block
             block_param_list_lookahead
          end
-      when "def",@enable_macro
+         return result
+   end
+   def keyword_def(str,offset,result)         #macros too, if enabled
          result.first.has_end!
          @parsestack.push ctx=DefContext.new(@linenum)
          ctx.state=:saw_def
-         safe_recurse { |aa|
-            set_last_token KeywordToken.new str #hack
+      old_moretokens=@moretokens
+      @moretokens=[]
+      aa=@moretokens
+         #safe_recurse { |aa|
+            set_last_token KeywordToken.new(str) #hack
             result.concat ignored_tokens
 
             #read an expr like a.b.c or a::b::c
@@ -960,7 +1005,7 @@ private
               end until  parencount==0 #@parsestack.size==old_size
               @localvars_stack.push SymbolTable.new
            else #no parentheses, all tail
-              set_last_token KeywordToken.new "." #hack hack
+             set_last_token KeywordToken.new(".") #hack hack
               tokindex=result.size
               result << tok=symbol(false,false)
               name=tok.to_s
@@ -1006,11 +1051,11 @@ private
               #a could even be a keyword (eg self or block_given?).
             end
             #read tail: .b.c.d etc
-            result.reverse_each{|res| break set_last_token res unless StillIgnoreToken===res}
+            result.reverse_each{|res| break set_last_token( res ) unless StillIgnoreToken===res}
             assert !(IgnoreToken===@last_operative_token)
             state=:expect_op
             @in_def_name=true
-            loop do
+            while true
 
                #look for start of parameter list
                nc=(@moretokens.empty? ? nextchar.chr : @moretokens.first.to_s[0,1])
@@ -1041,7 +1086,7 @@ private
                when /^(\.|::)$/.token_pat
                   lexerror tok,'expected ident' unless state==:expect_op
                   if endofs
-                    result.insert -2, ImplicitParamListEndToken.new(endofs)
+                    result.insert( -2, ImplicitParamListEndToken.new(endofs) )
                     endofs=nil
                   end
                   state=:expect_name
@@ -1049,9 +1094,9 @@ private
                   ctx.state=:def_body
                   state==:expect_op or lexerror tok,'expected identifier'
                   if endofs
-                    result.insert -2,ImplicitParamListEndToken.new(tok.offset)
+                    result.insert( -2,ImplicitParamListEndToken.new(tok.offset) )
                   end
-                  result.insert -2, EndHeaderToken.new(tok.offset)
+                  result.insert( -2, EndHeaderToken.new(tok.offset) )
                   break
                else
                   lexerror(tok, "bizarre token in def name: " +
@@ -1059,10 +1104,13 @@ private
                end
             end
             @in_def_name=false
-         }
-      when "alias"
+         #}
+      @moretokens= old_moretokens.concat @moretokens
+         return result
+   end
+   def keyword_alias(str,offset,result)
          safe_recurse { |a|
-            set_last_token KeywordToken.new "alias" #hack
+            set_last_token KeywordToken.new( "alias" )#hack
             result.concat ignored_tokens
             res=symbol(eat_next_if(?:),false) 
             unless res
@@ -1070,7 +1118,7 @@ private
             else
               res.ident[0]==?$ and res=VarNameToken.new(res.ident,res.offset)
               result<< res
-              set_last_token KeywordToken.new "alias" #hack
+              set_last_token KeywordToken.new( "alias" )#hack
               result.concat ignored_tokens
               res=symbol(eat_next_if(?:),false) 
               unless res
@@ -1081,10 +1129,12 @@ private
               end
             end
          }
-      when "undef"
+         return result
+   end
+   def keyword_undef(str,offset,result)
          safe_recurse { |a|
             loop do
-               set_last_token KeywordToken.new "," #hack
+               set_last_token KeywordToken.new( "," )#hack
                result.concat ignored_tokens
                tok=symbol(eat_next_if(?:),false)
                tok or lexerror(result.first,"bad symbol in undef")
@@ -1101,18 +1151,22 @@ private
             end
          }
          
+         return result
+   end
 #      when "defined?"
          #defined? might have a baresymbol following it
          #does it need to be handled specially?
          #it would seem not.....
 
-      when "when"
+   def keyword_when(str,offset,result)
          #abort_noparens! emits EndToken on leaving context
          result.unshift(*abort_noparens!(str))
          result.push KwParamListStartToken.new( offset+str.length)
          @parsestack.push WhenParamListContext.new(str,@linenum)
+         return result
+   end
 
-      when "rescue"
+   def keyword_rescue(str,offset,result)
          unless after_nonid_op? {false}
            #rescue needs to be treated differently when in operator context... 
            #i think no RescueSMContext should be pushed on the stack...
@@ -1124,8 +1178,10 @@ private
            @parsestack.push RescueSMContext.new(@linenum)
            result.unshift(*abort_noparens!(str))  
          end
+         return result
+   end
 
-      when "then"
+   def keyword_then(str,offset,result)
          result.unshift(*abort_noparens!(str))
          @parsestack.last.see self,:then
 
@@ -1133,24 +1189,38 @@ private
            @parsestack.pop
          else #error... does anyone care?
          end
+         return result
+   end
          
-      when "in"
+   def keyword_in(str,offset,result)
          result.unshift KwParamListEndToken.new( offset)
          result.unshift(*abort_noparens!(str))
          @parsestack.last.see self,:in
+         return result
+   end
          
-      when /\A(#{BINOPWORDS}|#{INNERBOUNDINGWORDS})\Z/o
+   def _keyword_innerbounding(str,offset,result)
          result.unshift(*abort_noparens!(str))
+         return result
+   end
+   for kw in BINOPWORDLIST+INNERBOUNDINGWORDLIST-["in","then","rescue","when","elsif"]
+     alias_method "keyword_#{kw}".to_sym, :_keyword_innerbounding
+   end
 
-      when /\A(return|break|next)\Z/
+   def keyword_return(str,offset,result)     
          fail if KeywordToken===@last_operative_token and @last_operative_token===/\A(\.|::)\Z/
          tok=KeywordToken.new(str,offset)
          result=yield tok
          result[0]=tok
          tok.has_no_block!
+         return result
+   end
+
+   alias keyword_break keyword_return
+   alias keyword_next keyword_return
     
      
-      when 'END'
+   def keyword_END(str,offset,result)
          #END could be treated, lexically, just as if it is an
          #ordinary method, except that local vars created in
          #END blocks are visible to subsequent code. (Why??) 
@@ -1161,7 +1231,7 @@ private
            safe_recurse{
              old=result.first
              result=[
-               MethNameToken.new(old.ident,old.offset),
+               KeywordToken.new(old.ident,old.offset),
                ImplicitParamListStartToken.new(input_position),
                ImplicitParamListEndToken.new(input_position),
                *ignored_tokens
@@ -1173,20 +1243,31 @@ private
              @parsestack.push BeginEndContext.new(str,offset)
            }
          end
+         return result
+   end
 
-      when FUNCLIKE_KEYWORDS
-         result=yield MethNameToken.new(str) #should be a keyword token?
-
-      when RUBYKEYWORDS
-        #do nothing
-        
-      else result=yield  MethNameToken.new(str)
-
-      end
 
-      return result
+   def _keyword_funclike(str,offset,result)
+         if @last_operative_token===/^(\.|::)$/
+           result=yield MethNameToken.new(str) #should pass a methname token here
+         else
+           result=yield KeywordToken.new(str)
+         end
+         return result
+   end
+   for kw in FUNCLIKE_KEYWORDLIST-["END","return","break","next"] do
+     alias_method "keyword_#{kw}".to_sym, :_keyword_funclike
+   end
+ 
+   def _keyword_varlike(str,offset,result)
+         #do nothing
+         return result
+   end
+   for kw in VARLIKE_KEYWORDLIST+["defined?", "not"] do
+     alias_method "keyword_#{kw}".to_sym, :_keyword_varlike
    end
 
+   private
 
    #-----------------------------------
    def parsestack_lastnonassign_is?(obj)
@@ -1221,7 +1302,7 @@ private
    #-----------------------------------
    def block_param_list_lookahead
       safe_recurse{ |la|
-         set_last_token KeywordToken.new  ';'
+         set_last_token KeywordToken.new(  ';' )
          a=ignored_tokens
 
          if eat_next_if(?|)
@@ -1267,7 +1348,7 @@ end
            end
          end
 
-         set_last_token KeywordToken.new ';'
+         set_last_token KeywordToken.new( ';' )
          #a.concat ignored_tokens
 
          #assert @last_operative_token===';'
@@ -1313,7 +1394,7 @@ end
             alias === call
          end
 
-         set_last_token KeywordToken.new ',' #hack
+         set_last_token KeywordToken.new( ',' )#hack
          #read local parameter names
          nextvar=nil
          loop do
@@ -1348,7 +1429,7 @@ end
                 when /^[&*]$/.token_pat #unary form...
                   #a NoWsToken is also expected... read it now
                   result.concat maybe_no_ws_token #not needed?
-                  set_last_token KeywordToken.new ','
+                  set_last_token KeywordToken.new( ',' )
                 else 
                   lexerror tok,"unfamiliar var name '#{tok}'"
               end
@@ -1382,7 +1463,7 @@ end
 #             !(NewlineToken===@last_operative_token) and
 #             !(/^(end|;)$/===@last_operative_token)
            #result<<EndHeaderToken.new(result.last.offset+result.last.to_s.size)
-           set_last_token KeywordToken.new ';'
+           set_last_token KeywordToken.new( ';' )
            result<< get1token
 #         end
       }
@@ -1767,7 +1848,7 @@ end
       end
       
       @offset_adjust=@min_offset_adjust
-      @moretokens.push *optional_here_bodies
+      @moretokens.push( *optional_here_bodies )
       ln=@linenum
       @moretokens.push lexerror(EscNlToken.new(@filename,ln-1,result,input_position-result.size), error),
                        FileAndLineToken.new(@filename,ln,input_position)
@@ -1854,7 +1935,7 @@ end
               back1char  #-1 to make newline char the next to read
               @linenum-=1
   
-              assert /[\r\n]/===nextchar.chr
+              assert( /[\r\n]/===nextchar.chr )
 
               #retr evrything til next nl
 if FASTER_STRING_ESCAPES
@@ -2253,9 +2334,9 @@ end
         @parsestack.last.see self,:arrow
       when '': #plain assignment: record local variable definitions
         last_context_not_implicit.lhs=false
-        @moretokens.push *ignored_tokens(true).map{|x| 
+        @moretokens.push( *ignored_tokens(true).map{|x| 
           NewlineToken===x ? EscNlToken.new(@filename,@linenum,x.ident,x.offset) : x 
-        }
+        } )
         @parsestack.push AssignmentRhsContext.new(@linenum)
         if eat_next_if ?* 
           tok=OperatorToken.new('*', input_position-1)
@@ -2401,7 +2482,7 @@ end
         # 'need to find matching callsite context and end it if implicit'
         lasttok=last_operative_token
         if !(lasttok===')' and lasttok.callsite?) #or ParamListContextNoParen===parsestack.last
-          @moretokens.push *(abort_1_noparen!(1).push tokch)
+          @moretokens.push( *(abort_1_noparen!(1).push tokch) )
           tokch=@moretokens.shift
         end
 #=end
@@ -2475,6 +2556,14 @@ end
   #-----------------------------------
   def comma(ch)
     @moretokens.push token=single_char_token(ch)
+
+    #if assignment rhs seen inside method param list, when param list, array or hash literal,
+    #       rescue where comma is expected, or method def param list
+    #          then end the assignment rhs now
+       #+[OBS,ParamListContext|ParamListContextNoParen|WhenParamListContext|ListImmedContext|
+       #      (RescueSMContext&-{:state=>:rescue})|(DefContext&-{:in_body=>FalseClass|nil}),
+       #  AssignmentRhsContext
+       #]===@parsestack
     if AssignmentRhsContext===@parsestack[-1] and
        ParamListContext===@parsestack[-2] || 
        ParamListContextNoParen===@parsestack[-2] ||
@@ -2487,9 +2576,9 @@ end
     end
     token.comma_type=
     case @parsestack[-1]
-    when AssignmentRhsContext: :rhs
-    when ParamListContext,ParamListContextNoParen: :call
-    when ListImmedContext: :array
+    when AssignmentRhsContext; :rhs
+    when ParamListContext,ParamListContextNoParen; :call
+    when ListImmedContext; :array
     else
       :lhs if comma_in_lvalue_list? 
     end