ruby_parser 3.12.0 → 3.13.0

data/lib/ruby_lexer.rex

@@ -27,9 +27,9 @@ start
 
   return process_string if lex_strterm
 
-  self.command_state = self.command_start
+  self.cmd_state = self.command_start
   self.command_start = false
-  self.space_seen    = false
+  self.space_seen    = false # TODO: rename token_seen?
   self.last_state    = lex_state
 
 rule
@@ -41,7 +41,7 @@ rule
 
                 /\n|\#/                 process_newline_or_comment
 
-                /[\]\)\}]/              process_bracing
+                /[\]\)\}]/              process_brace_close
 
 : /\!/
 | in_arg_state? /\!\@/                  { result :expr_arg, :tUBANG, "!@" }
@@ -50,10 +50,11 @@ rule
 : /\./
 |               /\.\.\.?/               { result :expr_beg, TOKENS[text], text }
 |               /\.\d/                  { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
-|               /\./                    { result :expr_dot, :tDOT, "." }
+|               /\./                    { self.lex_state = :expr_beg; result :expr_dot, :tDOT, "." }
 
                 /\(/                    process_paren
 
+# TODO: :expr_beg|:expr_label
                 /\,/                    { result :expr_beg, TOKENS[text], text }
 
 : /=/
@@ -101,7 +102,7 @@ was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 |               /\|\=/                  { result :expr_beg, :tOP_ASGN, "|" }
 |               /\|/                    { result :arg_state, :tPIPE,    "|" }
 
-                /\{/                    process_curly_brace
+                /\{/                    process_brace_open
 
 : /\*/
 |               /\*\*=/                 { result :expr_beg, :tOP_ASGN, "**" }
@@ -109,22 +110,23 @@ was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 |               /\*\=/                  { result(:expr_beg, :tOP_ASGN, "*") }
 |               /\*/                    { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
 
+# TODO: fix result+process_lchevron to set command_start = true
 : /</
 |               /\<\=\>/                { result :arg_state, :tCMP, "<=>"    }
 |               /\<\=/                  { result :arg_state, :tLEQ, "<="     }
-|               /\<\<\=/                { result :arg_state, :tOP_ASGN, "<<" }
+|               /\<\<\=/                { result :expr_beg,  :tOP_ASGN, "<<" }
 |               /\<\</                  process_lchevron
 |               /\</                    { result :arg_state, :tLT, "<"       }
 
 : />/
 |               /\>\=/                  { result :arg_state, :tGEQ, ">="     }
-|               /\>\>=/                 { result :arg_state, :tOP_ASGN, ">>" }
+|               /\>\>=/                 { result :expr_beg,  :tOP_ASGN, ">>" }
 |               /\>\>/                  { result :arg_state, :tRSHFT, ">>"   }
 |               /\>/                    { result :arg_state, :tGT, ">"       }
 
 : /\`/
 | expr_fname?   /\`/                   { result(:expr_end, :tBACK_REF2, "`") }
-| expr_dot?     /\`/                   { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
+| expr_dot?     /\`/                   { result((cmd_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
 |               /\`/                   { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
 
                 /\?/                    process_questionmark

data/lib/ruby_lexer.rex.rb

@@ -70,9 +70,9 @@ class RubyLexer
 
   def next_token
     return process_string if lex_strterm
-    self.command_state = self.command_start
+    self.cmd_state = self.command_start
     self.command_start = false
-    self.space_seen    = false
+    self.space_seen    = false # TODO: rename token_seen?
     self.last_state    = lex_state
 
     token = nil
@@ -87,7 +87,7 @@ class RubyLexer
           when text = ss.scan(/\n|\#/) then
             process_newline_or_comment text
           when text = ss.scan(/[\]\)\}]/) then
-            process_bracing text
+            process_brace_close text
           when ss.match?(/\!/) then
             case
             when in_arg_state? && (ss.skip(/\!\@/)) then
@@ -102,7 +102,7 @@ class RubyLexer
             when ss.skip(/\.\d/) then
               action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
             when ss.skip(/\./) then
-              action { result :expr_dot, :tDOT, "." }
+              action { self.lex_state = :expr_beg; result :expr_dot, :tDOT, "." }
             end # group /\./
           when text = ss.scan(/\(/) then
             process_paren text
@@ -183,7 +183,7 @@ class RubyLexer
               action { result :arg_state, :tPIPE,    "|" }
             end # group /\|/
           when text = ss.scan(/\{/) then
-            process_curly_brace text
+            process_brace_open text
           when ss.match?(/\*/) then
             case
             when ss.skip(/\*\*=/) then
@@ -202,7 +202,7 @@ class RubyLexer
             when ss.skip(/\<\=/) then
               action { result :arg_state, :tLEQ, "<="     }
             when ss.skip(/\<\<\=/) then
-              action { result :arg_state, :tOP_ASGN, "<<" }
+              action { result :expr_beg,  :tOP_ASGN, "<<" }
             when text = ss.scan(/\<\</) then
               process_lchevron text
             when ss.skip(/\</) then
@@ -213,7 +213,7 @@ class RubyLexer
             when ss.skip(/\>\=/) then
               action { result :arg_state, :tGEQ, ">="     }
             when ss.skip(/\>\>=/) then
-              action { result :arg_state, :tOP_ASGN, ">>" }
+              action { result :expr_beg,  :tOP_ASGN, ">>" }
             when ss.skip(/\>\>/) then
               action { result :arg_state, :tRSHFT, ">>"   }
             when ss.skip(/\>/) then
@@ -224,7 +224,7 @@ class RubyLexer
             when expr_fname? && (ss.skip(/\`/)) then
               action { result(:expr_end, :tBACK_REF2, "`") }
             when expr_dot? && (ss.skip(/\`/)) then
-              action { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
+              action { result((cmd_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
             when ss.skip(/\`/) then
               action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
             end # group /\`/

data/lib/ruby_parser.rb

@@ -34,7 +34,7 @@ class RubyParser
       begin
         return parser.process s, f, t
       rescue Racc::ParseError, RubyParser::SyntaxError => exc
-        e = exc
+        e ||= exc
       end
     end
     raise e
@@ -66,27 +66,25 @@ end
 
 ##
 # Unfortunately a problem with racc is that it won't let me namespace
-# properly, so instead of RubyParser::V18, I still have to generate
-# the old Ruby23Parser and shove it in as V23.
+# properly, so instead of RubyParser::V25, I still have to generate
+# the old Ruby25Parser and shove it in as V25.
 
-require "ruby18_parser"
-require "ruby19_parser"
 require "ruby20_parser"
 require "ruby21_parser"
 require "ruby22_parser"
 require "ruby23_parser"
 require "ruby24_parser"
 require "ruby25_parser"
+require "ruby26_parser"
 
 class RubyParser # HACK
   VERSIONS.clear # also a HACK caused by racc namespace issues
 
+  class V26 < ::Ruby26Parser; end
   class V25 < ::Ruby25Parser; end
   class V24 < ::Ruby24Parser; end
   class V23 < ::Ruby23Parser; end
   class V22 < ::Ruby22Parser; end
   class V21 < ::Ruby21Parser; end
   class V20 < ::Ruby20Parser; end
-  class V19 < ::Ruby19Parser; end
-  class V18 < ::Ruby18Parser; end
 end

data/lib/ruby_parser.yy

@@ -12,6 +12,8 @@ class Ruby23Parser
 class Ruby24Parser
 #elif V == 25
 class Ruby25Parser
+#elif V == 26
+class Ruby26Parser
 #else
 fail "version not specified or supported on code generation"
 #endif
@@ -108,14 +110,35 @@ rule
                       end
                       self.env.extend
                     }
-                  tLCURLY top_compstmt tRCURLY
+                    begin_block
                     {
-                      result = new_iter s(:preexe), nil, val[3]
+                      _, _, block = val
+                      result = block
+                    }
+
+     begin_block: tLCURLY top_compstmt tRCURLY
+                    {
+                      _, stmt, _ = val
+                      result = new_iter s(:preexe), 0, stmt
+                    }
+
+        bodystmt: compstmt opt_rescue k_else
+                    {
+                      res = _values[-2]
+                      yyerror "else without rescue is useless" unless res
                     }
+                    compstmt
+                    opt_ensure
+                    {
+                      body, resc, _, _, els, ens = val
 
-        bodystmt: compstmt opt_rescue opt_else opt_ensure
+                      result = new_body [body, resc, els, ens]
+                    }
+                | compstmt opt_rescue opt_ensure
                     {
-                      result = new_body val
+                      body, resc, ens = val
+
+                      result = new_body [body, resc, nil, ens]
                     }
 
         compstmt: stmts opt_terms
@@ -124,8 +147,8 @@ rule
                     }
 
            stmts: none
-                | stmt
-                | stmts terms stmt
+                | stmt_or_begin # TODO: newline_node ?
+                | stmts terms stmt_or_begin
                     {
                       result = self.block_append val[0], val[2]
                     }
@@ -135,6 +158,21 @@ rule
                       debug20 2, val, result
                     }
 
+   stmt_or_begin: stmt
+                | klBEGIN
+                    {
+                      if (self.in_def || self.in_single > 0) then
+                        debug20 1
+                        yyerror "BEGIN in method"
+                      end
+                      self.env.extend
+                    }
+                    begin_block
+                    {
+                      _, _, stmt = val
+                      result = stmt
+                    }
+
             stmt: kALIAS fitem
                     {
                       lexer.lex_state = :expr_fname
@@ -178,7 +216,8 @@ rule
                     }
                 | stmt kRESCUE_MOD stmt
                     {
-                      result = s(:rescue, val[0], new_resbody(s(:array), val[2]))
+                      body, _, resbody = val
+                      result = new_rescue body, new_resbody(s(:array), resbody)
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
@@ -193,15 +232,41 @@ rule
                     {
                       result = new_masgn val[0], val[2], :wrap
                     }
-                | var_lhs tOP_ASGN command_call
+                | lhs tEQL mrhs
+                    {
+                      result = new_assign val[0], s(:svalue, val[2])
+                    }
+#if V == 20
+                | mlhs tEQL arg_value
+                    {
+                      result = new_masgn val[0], val[2], :wrap
+                    }
+                | mlhs tEQL mrhs
+#else
+                | mlhs tEQL mrhs_arg
+#endif
+                    {
+                      result = new_masgn val[0], val[2]
+                    }
+                | expr
+
+    command_asgn: lhs tEQL command_rhs
+                    {
+                      result = new_assign val[0], val[2]
+                    }
+                # | lhs tEQL command_asgn
+                #     {
+                #       result = new_assign val[0], val[2]
+                #     }
+                | var_lhs tOP_ASGN command_rhs
                     {
                       result = new_op_asgn val
                     }
-                | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN command_call
+                | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN command_rhs
                     {
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
                     }
-                | primary_value call_op tIDENTIFIER tOP_ASGN command_call
+                | primary_value call_op tIDENTIFIER tOP_ASGN command_rhs
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
@@ -209,7 +274,7 @@ rule
                       end
                       result.line = val[0].line
                     }
-                | primary_value call_op tCONSTANT tOP_ASGN command_call
+                | primary_value call_op tCONSTANT tOP_ASGN command_rhs
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
@@ -217,46 +282,33 @@ rule
                       end
                       result.line = val[0].line
                     }
-                | primary_value tCOLON2 tCONSTANT tOP_ASGN command_call
+                | primary_value tCOLON2 tCONSTANT tOP_ASGN command_rhs
                     {
                       result = s(:op_asgn, val[0], val[4], val[2], val[3])
                       debug20 4, val, result
                     }
-                | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_call
+                | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_rhs
                     {
                       result = s(:op_asgn, val[0], val[4], val[2], val[3])
                       debug20 5, val, result
                     }
-                | backref tOP_ASGN command_call
+                | backref tOP_ASGN command_rhs
                     {
                       self.backref_assign_error val[0]
                     }
-                | lhs tEQL mrhs
-                    {
-                      result = new_assign val[0], s(:svalue, val[2])
-                    }
-#if V == 20
-                | mlhs tEQL arg_value
-                    {
-                      result = new_masgn val[0], val[2], :wrap
-                    }
-                | mlhs tEQL mrhs
-#else
-                | mlhs tEQL mrhs_arg
-#endif
-                    {
-                      result = new_masgn val[0], val[2]
-                    }
-                | expr
 
-    command_asgn: lhs tEQL command_call
+     command_rhs: command_call                =tOP_ASGN
                     {
-                      result = new_assign val[0], val[2]
+                      expr, = val
+                      result = value_expr expr
                     }
-                | lhs tEQL command_asgn
+                | command_call kRESCUE_MOD stmt
                     {
-                      result = new_assign val[0], val[2]
+                      expr, _, resbody = val
+                      expr = value_expr expr
+                      result = new_rescue(expr, new_resbody(s(:array), resbody))
                     }
+                | command_asgn
 
             expr: command_call
                 | expr kAND expr
@@ -282,30 +334,40 @@ rule
                       result = value_expr(val[0])
                     }
 
+   expr_value_do:   {
+                      lexer.cond.push true
+                    }
+                    expr_value do
+                    {
+                      lexer.cond.pop
+                    }
+                    {
+                      _, expr, _, _ = val
+                      result = expr
+                    }
+
     command_call: command
                 | block_command
 
    block_command: block_call
-                | block_call dot_or_colon operation2 command_args
+                | block_call call_op2 operation2 command_args
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
                     }
 
  cmd_brace_block: tLBRACE_ARG
                     {
-                      self.env.extend(:dynamic)
+                      # self.env.extend(:dynamic)
                       result = self.lexer.lineno
                     }
-                    opt_block_param
-                    {
-                      result = nil # self.env.dynamic.keys
-                    }
-                    compstmt tRCURLY
+                    brace_body tRCURLY
                     {
-                      result = new_iter nil, val[2], val[4]
-                      result.line = val[1]
+                      _, line, body, _ = val
 
-                      self.env.unextend
+                      result = body
+                      result.line = line
+
+                      # self.env.unextend
                     }
 
            fcall: operation
@@ -363,7 +425,7 @@ rule
                     {
                       result = new_yield val[1]
                     }
-                | kRETURN call_args
+                | k_return call_args
                     {
                       line = val[0].last
                       result = s(:return, ret_args(val[1])).line(line)
@@ -587,8 +649,9 @@ rule
 
                 | reswords
                     {
+                      (sym, _line), = val
                       lexer.lex_state = :expr_end
-                      result = val[0]
+                      result = sym
                     }
 
             fsym: fname | symbol
@@ -618,7 +681,7 @@ rule
                 |   tNEQ     | tLSHFT  | tRSHFT   | tPLUS | tMINUS | tSTAR2
                 |   tSTAR    | tDIVIDE | tPERCENT | tPOW  | tDSTAR | tBANG   | tTILDE
                 |   tUPLUS   | tUMINUS | tAREF    | tASET | tBACK_REF2
-#if V == 20
+#if V >= 20
                 |   tUBANG
 #endif
 
@@ -632,51 +695,46 @@ rule
                 | kWHEN     | kYIELD    | kIF           | kUNLESS | kWHILE
                 | kUNTIL
 
-             arg: lhs tEQL arg
+             arg: lhs tEQL arg_rhs
                     {
                       result = new_assign val[0], val[2]
                     }
-                | lhs tEQL arg kRESCUE_MOD arg
-                    {
-                      result = new_assign val[0], s(:rescue, val[2], new_resbody(s(:array), val[4]))
-                    }
-                | var_lhs tOP_ASGN arg
-                    {
-                      result = new_op_asgn val
-                    }
-                | var_lhs tOP_ASGN arg kRESCUE_MOD arg
+                | var_lhs tOP_ASGN arg_rhs
                     {
                       result = new_op_asgn val
-                      result = s(:rescue, result, new_resbody(s(:array), val[4]))
                     }
-                | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg
+                | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg_rhs
                     {
                       val[2].sexp_type = :arglist if val[2]
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
                     }
-                | primary_value call_op tIDENTIFIER tOP_ASGN arg
+                | primary_value call_op tIDENTIFIER tOP_ASGN arg_rhs
                     {
                       result = new_op_asgn2 val
                     }
-                | primary_value call_op tCONSTANT tOP_ASGN arg
+                | primary_value call_op tCONSTANT tOP_ASGN arg_rhs
                     {
                       result = new_op_asgn2 val
                     }
-                | primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg
+                | primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg_rhs
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     }
-                | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
+                | primary_value tCOLON2 tCONSTANT tOP_ASGN arg_rhs
                     {
-                      yyerror "constant re-assignment"
+                      # TODO: assignment
+                      raise "not yet: %p" % [val]
                     }
-                | tCOLON3 tCONSTANT tOP_ASGN arg
+                | tCOLON3 tCONSTANT tOP_ASGN arg_rhs
                     {
-                      yyerror "constant re-assignment"
+                      # TODO: assignment
+                      raise "not yet: %p" % [val]
                     }
-                | backref tOP_ASGN arg
+                | backref tOP_ASGN arg_rhs
                     {
-                      self.backref_assign_error val[0]
+                      # TODO: lhs = var_field val[0]
+                      asgn = new_op_asgn val
+                      result = self.backref_assign_error asgn
                     }
                 | arg tDOT2 arg
                     {
@@ -696,6 +754,20 @@ rule
                         result = s(:dot3, v1, v2)
                       end
                     }
+#if V >= 26
+                | arg tDOT2
+                    {
+                      v1, v2 = val[0], nil
+
+                      result = s(:dot2, v1, v2)
+                    }
+                | arg tDOT3
+                    {
+                      v1, v2 = val[0], nil
+
+                      result = s(:dot3, v1, v2)
+                    }
+#endif
                 | arg tPLUS arg
                     {
                       result = new_call val[0], :+, argl(val[2])
@@ -760,22 +832,7 @@ rule
                     {
                       result = new_call val[0], :"<=>", argl(val[2])
                     }
-                | arg tGT arg
-                    {
-                      result = new_call val[0], :">", argl(val[2])
-                    }
-                | arg tGEQ arg
-                    {
-                      result = new_call val[0], :">=", argl(val[2])
-                    }
-                | arg tLT arg
-                    {
-                      result = new_call val[0], :"<", argl(val[2])
-                    }
-                | arg tLEQ arg
-                    {
-                      result = new_call val[0], :"<=", argl(val[2])
-                    }
+                | rel_expr                      =tCMP
                 | arg tEQ arg
                     {
                       result = new_call val[0], :"==", argl(val[2])
@@ -834,6 +891,23 @@ rule
                     }
                 | primary
 
+           relop: tGT
+                | tLT
+                | tGEQ
+                | tLEQ
+
+        rel_expr: arg      relop arg                    =tGT
+                    {
+                      lhs, op, rhs = val
+                      result = new_call lhs, op.to_sym, argl(rhs)
+                    }
+                | rel_expr relop arg                    =tGT
+                    {
+                      lhs, op, rhs = val
+                      warn "comparison '%s' after comparison", op
+                      result = new_call lhs, op.to_sym, argl(rhs)
+                    }
+
        arg_value: arg
                     {
                       result = value_expr(val[0])
@@ -853,6 +927,15 @@ rule
                       result = args [array_to_hash(val[0])]
                     }
 
+         arg_rhs: arg                   =tOP_ASGN
+                | arg kRESCUE_MOD arg
+                    {
+                      body, _, resbody = val
+                      body    = value_expr body
+                      resbody = remove_begin resbody
+                      result  = new_rescue(body, new_resbody(s(:array), resbody))
+                    }
+
       paren_args: tLPAREN2 opt_call_args rparen
                     {
                       result = val[1]
@@ -908,12 +991,11 @@ rule
                     }
 
     command_args:   {
-                      result = lexer.cmdarg.stack.dup # TODO: smell?
-                      lexer.cmdarg.push true
+                      result = lexer.cmdarg.store true
                     }
                       call_args
                     {
-                      lexer.cmdarg.stack.replace val[0]
+                      lexer.cmdarg.restore val[0]
                       result = val[1]
                     }
 
@@ -983,12 +1065,16 @@ rule
                     {
                       result = new_call nil, val[0].to_sym
                     }
-                | kBEGIN
+                | k_begin
                     {
                       result = self.lexer.lineno
+                      # TODO:
+                      # $<val>1 = cmdarg_stack;
+                      # CMDARG_SET(0);
                     }
-                    bodystmt kEND
+                    bodystmt k_end
                     {
+                      # TODO: CMDARG_SET($<val>1);
                       unless val[2] then
                         result = s(:nil)
                       else
@@ -999,22 +1085,25 @@ rule
                     }
                 | tLPAREN_ARG rparen
                     {
+                      # TODO: lex_state = :expr_endarg in between
                       debug20 13, val, result
                     }
                 | tLPAREN_ARG
                     {
-                      result = self.lexer.cmdarg.stack.dup
-                      lexer.cmdarg.stack.replace [false] # TODO add api for these
+                      result = lexer.cmdarg.store false
+                      # result = self.lexer.cmdarg.stack.dup
+                      # lexer.cmdarg.stack.replace [false] # TODO add api for these
                     }
-                    expr
+                    stmt
                     {
                       lexer.lex_state = :expr_endarg
                     }
                     rparen
                     {
+                      _, cmdarg, stmt, _, _, = val
                       warning "(...) interpreted as grouped expression"
-                      lexer.cmdarg.stack.replace val[1]
-                      result = val[2]
+                      lexer.cmdarg.restore cmdarg
+                      result = stmt
                     }
                 | tLPAREN compstmt tRPAREN
                     {
@@ -1042,7 +1131,7 @@ rule
                     {
                       result = new_hash val
                     }
-                | kRETURN
+                | k_return
                     {
                       result = s(:return)
                     }
@@ -1090,61 +1179,42 @@ rule
                     {
                       result = val[1] # TODO: fix lineno
                     }
-                | kIF expr_value then compstmt if_tail kEND
-                    {
-                      result = new_if val[1], val[3], val[4]
-                    }
-                | kUNLESS expr_value then compstmt opt_else kEND
-                    {
-                      result = new_if val[1], val[4], val[3]
-                    }
-                | kWHILE
-                    {
-                      lexer.cond.push true
-                    }
-                    expr_value do
-                    {
-                      lexer.cond.pop
-                    }
-                    compstmt kEND
+                | k_if expr_value then compstmt if_tail k_end
                     {
-                      result = new_while val[5], val[2], true
+                      _, c, _, t, f, _ = val
+                      result = new_if c, t, f
                     }
-                | kUNTIL
+                | k_unless expr_value then compstmt opt_else k_end
                     {
-                      lexer.cond.push true
+                      _, c, _, t, f, _ = val
+                      result = new_if c, f, t
                     }
-                    expr_value do
+                | k_while expr_value_do compstmt k_end
                     {
-                      lexer.cond.pop
+                      _, cond, body, _ = val
+                      result = new_while body, cond, true
                     }
-                    compstmt kEND
+                | k_until expr_value_do compstmt k_end
                     {
-                      result = new_until val[5], val[2], true
+                      _, cond, body, _ = val
+                      result = new_until body, cond, true
                     }
-                | kCASE expr_value opt_terms case_body kEND
+                | k_case expr_value opt_terms case_body k_end
                     {
                       (_, line), expr, _, body, _ = val
                       result = new_case expr, body, line
                     }
-                | kCASE            opt_terms case_body kEND
+                | k_case            opt_terms case_body k_end
                     {
                       (_, line), _, body, _ = val
                       result = new_case nil, body, line
                     }
-                | kFOR for_var kIN
-                    {
-                      lexer.cond.push true
-                    }
-                    expr_value do
-                    {
-                      lexer.cond.pop
-                    }
-                    compstmt kEND
+                | k_for for_var kIN expr_value_do compstmt k_end
                     {
-                      result = new_for val[4], val[1], val[7]
+                      _, var, _, iter, body, _ = val
+                      result = new_for iter, var, body
                     }
-                | kCLASS
+                | k_class
                     {
                       result = self.lexer.lineno
                     }
@@ -1156,13 +1226,13 @@ rule
                       end
                       self.env.extend
                     }
-                    bodystmt kEND
+                    bodystmt k_end
                     {
                       result = new_class val
                       self.env.unextend
                       self.lexer.comments # we don't care about comments in the body
                     }
-                | kCLASS tLSHFT
+                | k_class tLSHFT
                     {
                       result = self.lexer.lineno
                     }
@@ -1177,13 +1247,13 @@ rule
                       self.in_single = 0
                       self.env.extend
                     }
-                    bodystmt kEND
+                    bodystmt k_end
                     {
                       result = new_sclass val
                       self.env.unextend
                       self.lexer.comments # we don't care about comments in the body
                     }
-                | kMODULE
+                | k_module
                     {
                       result = self.lexer.lineno
                     }
@@ -1195,13 +1265,13 @@ rule
 
                       self.env.extend
                     }
-                    bodystmt kEND
+                    bodystmt k_end
                     {
                       result = new_module val
                       self.env.unextend
                       self.lexer.comments # we don't care about comments in the body
                     }
-                | kDEF fname
+                | k_def fname
                     {
                       result = [self.in_def, self.lexer.cmdarg.stack.dup]
 
@@ -1212,7 +1282,7 @@ rule
                       # TODO: port local_push_gen and local_pop_gen
                       lexer.cmdarg.stack.replace [false]
                     }
-                    f_arglist bodystmt kEND
+                    f_arglist bodystmt k_end
                     {
                       in_def, cmdarg = val[2]
 
@@ -1223,7 +1293,7 @@ rule
                       self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     }
-                | kDEF singleton dot_or_colon
+                | k_def singleton dot_or_colon
                     {
                       self.comments.push self.lexer.comments
                       lexer.lex_state = :expr_fname
@@ -1236,7 +1306,7 @@ rule
                       result = [lexer.lineno, self.lexer.cmdarg.stack.dup]
                       lexer.cmdarg.stack.replace [false]
                     }
-                    f_arglist bodystmt kEND
+                    f_arglist bodystmt k_end
                     {
                       line, cmdarg = val[5]
                       result = new_defs val
@@ -1281,7 +1351,15 @@ rule
          k_class: kCLASS
         k_module: kMODULE
            k_def: kDEF
+            k_do: kDO
+      k_do_block: kDO_BLOCK
+        k_rescue: kRESCUE
+        k_ensure: kENSURE
+          k_when: kWHEN
+          k_else: kELSE
+         k_elsif: kELSIF
            k_end: kEND
+        k_return: kRETURN
 
             then: term
                 | kTHEN
@@ -1291,7 +1369,7 @@ rule
                 | kDO_COND
 
          if_tail: opt_else
-                | kELSIF expr_value then compstmt if_tail
+                | k_elsif expr_value then compstmt if_tail
                     {
                       result = s(:if, val[1], val[3], val[4])
                     }
@@ -1462,18 +1540,22 @@ opt_block_args_tail: tCOMMA block_args_tail
 
  opt_block_param: none { result = 0 }
                 | block_param_def
+                    {
+                      self.lexer.command_start = true
+                    }
 
  block_param_def: tPIPE opt_bv_decl tPIPE
                     {
+                      # TODO: current_arg = 0
                       result = args val
                     }
                 | tOROP
                     {
-                      self.lexer.command_start = true
                       result = s(:args)
                     }
                 | tPIPE block_param opt_bv_decl tPIPE
                     {
+                      # TODO: current_arg = 0
                       result = args val
                     }
 
@@ -1506,13 +1588,21 @@ opt_block_args_tail: tCOMMA block_args_tail
                       lexer.paren_nest += 1
                       lexer.lpar_beg = lexer.paren_nest
                     }
-                    f_larglist lambda_body
+                    f_larglist
+                    {
+                      result = [lexer.cmdarg.store(false), self.lexer.lineno]
+                    }
+                    lambda_body
                     {
-                      lpar, args, body = val
+                      lpar, args, (cmdarg, lineno), body = val
                       lexer.lpar_beg = lpar
 
+                      lexer.cmdarg.restore cmdarg
+                      lexer.cmdarg.lexpop
+
                       call = new_call nil, :lambda
                       result = new_iter call, args, body
+                      result.line = lineno
                       self.env.unextend
                     }
 
@@ -1530,28 +1620,15 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       result = val[1]
                     }
-                | kDO_LAMBDA compstmt kEND
+                | kDO_LAMBDA bodystmt kEND
                     {
                       result = val[1]
                     }
 
-        do_block: kDO_BLOCK
-                    {
-                      self.env.extend :dynamic
-                      result = self.lexer.lineno
-                    }
-                    opt_block_param
-                    {
-                      result = nil # self.env.dynamic.keys
-                    }
-                    compstmt kEND
+        do_block: k_do_block do_body kEND
                     {
-                      args   = val[2]
-                      body   = val[4]
-                      result = new_iter nil, args, body
-                      result.line = val[1]
-
-                      self.env.unextend
+                      # TODO: maybe fix lineno to kDO's lineno?
+                      result = val[1]
                     }
 
       block_call: command do_block
@@ -1568,11 +1645,11 @@ opt_block_args_tail: tCOMMA block_args_tail
                       result = val[1]
                       result.insert 1, val[0]
                     }
-                | block_call dot_or_colon operation2 opt_paren_args
+                | block_call call_op2 operation2 opt_paren_args
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
                     }
-                | block_call dot_or_colon operation2 opt_paren_args brace_block
+                | block_call call_op2 operation2 opt_paren_args brace_block
                     {
                       iter1, _, name, args, iter2 = val
 
@@ -1581,7 +1658,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
                       result = iter2
                     }
-                | block_call dot_or_colon operation2 command_args do_block
+                | block_call call_op2 operation2 command_args do_block
                     {
                       iter1, _, name, args, iter2 = val
 
@@ -1638,43 +1715,59 @@ opt_block_args_tail: tCOMMA block_args_tail
                       self.env.extend :dynamic
                       result = self.lexer.lineno
                     }
-                    opt_block_param
-                    {
-                      result = nil # self.env.dynamic.keys
-                    }
-                    compstmt tRCURLY
+                    brace_body tRCURLY
                     {
-                      _, line, args, _, body, _ = val
+                      _, line, body, _ = val
 
-                      result = new_iter nil, args, body
+                      result = body
                       result.line = line
 
                       self.env.unextend
                     }
-                | kDO
+                | k_do
                     {
                       self.env.extend :dynamic
                       result = self.lexer.lineno
                     }
-                    opt_block_param
+                    do_body kEND
+                    {
+                      _, line, body, _ = val
+
+                      result = body
+                      result.line = line
+
+                      self.env.unextend
+                    }
+
+      brace_body:   { self.env.extend :dynamic; result = self.lexer.lineno }
+                    { result = lexer.cmdarg.store(false) }
+                    opt_block_param compstmt
                     {
-                      result = nil # self.env.dynamic.keys
+                      line, cmdarg, param, cmpstmt = val
+
+                      result = new_brace_body param, cmpstmt, line
+                      self.env.unextend
+                      lexer.cmdarg.restore cmdarg
+                      lexer.cmdarg.pop # because of: cmdarg_stack >> 1 ?
                     }
+
+         do_body:   { self.env.extend :dynamic; result = self.lexer.lineno }
+                    { result = lexer.cmdarg.store(false) }
+                    opt_block_param
 #if V >= 25
-                    bodystmt kEND
+                    bodystmt
 #else
-                    compstmt kEND
+                    compstmt
 #endif
                     {
-                      _, line, args, _, body, _ = val
-
-                      result = new_iter nil, args, body
-                      result.line = line
+                      line, cmdarg, param, cmpstmt = val
 
+                      result = new_do_body param, cmpstmt, line
                       self.env.unextend
+                      lexer.cmdarg.restore cmdarg
                     }
 
-       case_body: kWHEN
+       case_body: k_when
                     {
                       result = self.lexer.lineno
                     }
@@ -1687,7 +1780,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
            cases: opt_else | case_body
 
-      opt_rescue: kRESCUE exc_list exc_var then compstmt opt_rescue
+      opt_rescue: k_rescue exc_list exc_var then compstmt opt_rescue
                     {
                       (_, line), klasses, var, _, body, rest = val
 
@@ -1716,7 +1809,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
                 | none
 
-      opt_ensure: kENSURE compstmt
+      opt_ensure: k_ensure compstmt
                     {
                       _, body = val
 
@@ -1883,7 +1976,7 @@ regexp_contents: none
                       result = lexer.lex_strterm
 
                       lexer.lex_strterm = nil
-                      lexer.lex_state = :expr_beg # TODO: expr_value ?
+                      lexer.lex_state = :expr_beg
                     }
                     string_dvar
                     {
@@ -1904,15 +1997,11 @@ regexp_contents: none
                       lexer.brace_nest  = 0
                       lexer.string_nest = 0
 
-                      lexer.lex_state   = :expr_value
+                      lexer.lex_state   = :expr_beg
                     }
-                    compstmt tRCURLY
+                    compstmt
+                    tSTRING_DEND
                     {
-#if V == 20
-                      # TODO: tRCURLY -> tSTRING_DEND
-#else
-                      # TODO: tRCURLY -> tSTRING_END
-#endif
                       _, memo, stmt, _ = val
 
                       lex_strterm, brace_nest, string_nest, oldcond, oldcmdarg, oldlex_state = memo
@@ -2049,23 +2138,18 @@ keyword_variable: kNIL      { result = s(:nil)   }
          backref: tNTH_REF  { result = s(:nth_ref,  val[0]) }
                 | tBACK_REF { result = s(:back_ref, val[0]) }
 
-      superclass: term
-                    {
-                      result = nil
-                    }
-                | tLT
+      superclass: tLT
                     {
                       lexer.lex_state = :expr_beg
+                      lexer.command_start = true
                     }
                     expr_value term
                     {
                       result = val[2]
                     }
-                | error term
+                | none
                     {
-                      yyerrok
                       result = nil
-                      debug20 30, val, result
                     }
 
        f_arglist: tLPAREN2 f_args rparen
@@ -2073,16 +2157,20 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = val[1]
                       self.lexer.lex_state = :expr_beg
                       self.lexer.command_start = true
-                      # TODO:
-                      # $<num>$ = parser->parser_in_kwarg;
-                      # parser->parser_in_kwarg = 1;
                     }
-                | f_args term
+                |   {
+                      result = self.in_kwarg
+                      self.in_kwarg = true
+                      # TODO: self.lexer.lex_state |= :expr_label
+                    }
+                    f_args term
                     {
-                      # TODO: parser->parser_in_kwarg = $<num>1;
-                      result = val[0]
-                      self.lexer.lex_state = :expr_beg
-                      self.lexer.command_start = true
+                      kwarg, args, _ = val
+
+                      self.in_kwarg = kwarg
+                      result = args
+                      lexer.lex_state     = :expr_beg
+                      lexer.command_start = true
                     }
 
        args_tail: f_kwarg tCOMMA f_kwrest opt_f_block_arg
@@ -2305,7 +2393,9 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
         f_kwrest: kwrest_mark tIDENTIFIER
                     {
-                      result = :"**#{val[1]}"
+                      name = val[1].to_sym
+                      self.assignable name
+                      result = :"**#{name}"
                     }
                 | kwrest_mark
                     {
@@ -2424,9 +2514,10 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = s(:array, val[0], val[2])
                     }
-                | tLABEL opt_nl arg_value
+                | tLABEL arg_value
                     {
-                      result = s(:array, s(:lit, val[0][0].to_sym), val.last)
+                      (label, _), arg = val
+                      result = s(:array, s(:lit, label.to_sym), arg)
                     }
 #if V >= 22
                 | tSTRING_BEG string_contents tLABEL_END arg_value
@@ -2435,11 +2526,6 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       sym.sexp_type = :dsym
                       result = s(:array, sym, value)
                     }
-                | tSYMBOL arg_value
-                    {
-                      raise "not yet: #{val.inspect}"
-                      # result = s(:array, s(:lit, val[1].to_sym), val[1])
-                    }
 #endif
                 | tDSTAR arg_value
                     {
@@ -2452,8 +2538,12 @@ keyword_variable: kNIL      { result = s(:nil)   }
     dot_or_colon: tDOT | tCOLON2
          call_op: tDOT
 #if V >= 23
-                | tLONELY
+                | tLONELY # TODO: rename tANDDOT?
 #endif
+
+        call_op2: call_op
+                | tCOLON2
+
        opt_terms:  | terms
           opt_nl:  | tNL
           rparen: opt_nl tRPAREN