ruby_parser 3.14.0 → 3.16.0

data/lib/ruby_parser.yy

@@ -14,6 +14,10 @@ class Ruby24Parser
 class Ruby25Parser
 #elif V == 26
 class Ruby26Parser
+#elif V == 27
+class Ruby27Parser
+#elif V == 30
+class Ruby30Parser
 #else
 fail "version not specified or supported on code generation"
 #endif
@@ -44,6 +48,9 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
 #if V >= 23
        tLONELY
 #endif
+#if V >= 26
+       tBDOT2 tBDOT3
+#endif
 
 preclow
   nonassoc tLOWEST
@@ -55,7 +62,7 @@ preclow
   right    tEQL tOP_ASGN
   left     kRESCUE_MOD
   right    tEH tCOLON
-  nonassoc tDOT2 tDOT3
+  nonassoc tDOT2 tDOT3 tBDOT2 tBDOT3
   left     tOROP
   left     tANDOP
   nonassoc tCMP tEQ tEQQ tNEQ tMATCH tNMATCH
@@ -78,6 +85,9 @@ rule
                     top_compstmt
                     {
                       result = new_compstmt val
+
+                      lexer.cond.pop # local_pop
+                      lexer.cmdarg.pop
                     }
 
     top_compstmt: top_stmts opt_terms
@@ -286,13 +296,15 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN command_rhs
                     {
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
-                      debug20 4, val, result
+                      lhs1, _, lhs2, op, rhs = val
+
+                      result = s(:op_asgn, lhs1, rhs, lhs2.to_sym, op.to_sym)
                     }
                 | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_rhs
                     {
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
-                      debug20 5, val, result
+                      lhs1, _, lhs2, op, rhs = val
+
+                      result = s(:op_asgn, lhs1, rhs, lhs2.to_sym, op.to_sym)
                     }
                 | backref tOP_ASGN command_rhs
                     {
@@ -852,6 +864,24 @@ rule
                       result = s(:dot3, v1, v2).line v1.line
                     }
 #endif
+
+#if V >= 27
+                | tBDOT2 arg
+                    {
+                      _, v2, = val
+                      v1 = nil
+
+                      result = s(:dot2, v1, v2).line v2.line
+                    }
+                | tBDOT3 arg
+                    {
+                      _, v2 = val
+                      v1 = nil
+
+                      result = s(:dot3, v1, v2).line v2.line
+                    }
+#endif
+
                 | arg tPLUS arg
                     {
                       result = new_call val[0], :+, argl(val[2])
@@ -1063,17 +1093,14 @@ rule
                 | args opt_block_arg
                     {
                       result = call_args val
-                      result = self.arg_blk_pass val[0], val[1]
                     }
                 | assocs opt_block_arg
                     {
-                      result = call_args [array_to_hash(val[0])]
-                      result = self.arg_blk_pass result, val[1]
+                      result = call_args [array_to_hash(val[0]), val[1]]
                     }
                 | args tCOMMA assocs opt_block_arg
                     {
-                      result = call_args [val[0], array_to_hash(val[2])]
-                      result = self.arg_blk_pass result, val[3]
+                      result = call_args [val[0], array_to_hash(val[2]), val[3]]
                     }
                 | block_arg
                     {
@@ -1081,12 +1108,39 @@ rule
                     }
 
     command_args:   {
-                      result = lexer.cmdarg.store true
+                      # parse26.y line 2200
+
+                      # If call_args starts with a open paren '(' or
+                      # '[', look-ahead reading of the letters calls
+                      # CMDARG_PUSH(0), but the push must be done
+                      # after CMDARG_PUSH(1). So this code makes them
+                      # consistent by first cancelling the premature
+                      # CMDARG_PUSH(0), doing CMDARG_PUSH(1), and
+                      # finally redoing CMDARG_PUSH(0).
+
+                      result = yychar = self.last_token_type.first
+                      lookahead = [:tLPAREN, :tLPAREN_ARG, :tLPAREN2, :tLBRACK, :tLBRACK2].include?(yychar)
+                      lexer.cmdarg.pop if lookahead
+                      lexer.cmdarg.push true
+                      lexer.cmdarg.push false if lookahead
                     }
                       call_args
                     {
-                      lexer.cmdarg.restore val[0]
-                      result = val[1]
+                      yychar, args = val
+
+                      # call_args can be followed by tLBRACE_ARG (that
+                      # does CMDARG_PUSH(0) in the lexer) but the push
+                      # must be done after CMDARG_POP() in the parser.
+                      # So this code does CMDARG_POP() to pop 0 pushed
+                      # by tLBRACE_ARG, CMDARG_POP() to pop 1 pushed
+                      # by command_args, and CMDARG_PUSH(0) to restore
+                      # back the flag set by tLBRACE_ARG.
+
+                      lookahead = [:tLBRACE_ARG].include?(yychar)
+                      lexer.cmdarg.pop if lookahead
+                      lexer.cmdarg.pop
+                      lexer.cmdarg.push false if lookahead
+                      result = args
                     }
 
        block_arg: tAMPER arg_value
@@ -1104,8 +1158,9 @@ rule
             args: arg_value
                     {
                       arg, = val
+                      lineno = arg.line || lexer.lineno # HACK
 
-                      result = s(:array, arg).line arg.line
+                      result = s(:array, arg).line lineno
                     }
                 | tSTAR arg_value
                     {
@@ -1117,9 +1172,11 @@ rule
                       args, _, id = val
                       result = self.list_append args, id
                     }
-                | args tCOMMA tSTAR { result = lexer.lineno } arg_value
+                | args tCOMMA tSTAR arg_value
                     {
-                      args, _, _, line, id = val
+                      # TODO: the line number from tSTAR has been dropped
+                      args, _, _, id = val
+                      line = lexer.lineno
                       result = self.list_append args, s(:splat, id).line(line)
                     }
 
@@ -1140,7 +1197,6 @@ rule
                     }
                 | args tCOMMA tSTAR arg_value
                     {
-                      # FIX: bad shift/reduce conflict with rhs' comma star prod
                       # TODO: make all tXXXX terminals include lexer.lineno
                       arg, _, _, splat = val
                       result = self.arg_concat arg, splat
@@ -1168,21 +1224,13 @@ rule
                     }
                 | k_begin
                     {
+                      lexer.cmdarg.push false
                       result = self.lexer.lineno
-                      # TODO:
-                      # $<val>1 = cmdarg_stack;
-                      # CMDARG_SET(0);
                     }
                     bodystmt k_end
                     {
-                      # TODO: CMDARG_SET($<val>1);
-                      unless val[2] then
-                        result = s(:nil)
-                      else
-                        result = s(:begin, val[2])
-                      end
-
-                      result.line = val[1]
+                      lexer.cmdarg.pop
+                      result = new_begin val
                     }
                 | tLPAREN_ARG
                     {
@@ -1195,18 +1243,14 @@ rule
                       result = s(:begin).line line
                     }
                 | tLPAREN_ARG
-                    {
-                      result = lexer.cmdarg.store false
-                    }
                     stmt
                     {
                       lexer.lex_state = EXPR_ENDARG
                     }
                     rparen
                     {
-                      _, cmdarg, stmt, _, _, = val
-                      warning "(...) interpreted as grouped expression"
-                      lexer.cmdarg.restore cmdarg
+                      _, stmt, _, _, = val
+                      # warning "(...) interpreted as grouped expression"
                       result = stmt
                     }
                 | tLPAREN compstmt tRPAREN
@@ -1389,48 +1433,61 @@ rule
                     }
                 | k_def fname
                     {
-                      result = [self.in_def, self.lexer.cmdarg.stack.dup]
+                      result = self.in_def
 
-                      self.comments.push self.lexer.comments
-                      self.in_def = true
+                      self.in_def = true # group = local_push
                       self.env.extend
-                      # TODO: local->cmdargs = cmdarg_stack;
-                      # TODO: port local_push_gen and local_pop_gen
-                      lexer.cmdarg.stack.replace [false]
+                      lexer.cmdarg.push false
+                      lexer.cond.push false
+
+                      self.comments.push self.lexer.comments
                     }
                     f_arglist bodystmt { result = lexer.lineno } k_end
                     {
-                      in_def, cmdarg = val[2]
+                      in_def = val[2]
 
                       result = new_defn val
 
-                      lexer.cmdarg.stack.replace cmdarg
+                      lexer.cond.pop # group = local_pop
+                      lexer.cmdarg.pop
                       self.env.unextend
                       self.in_def = in_def
+
                       self.lexer.comments # we don't care about comments in the body
                     }
                 | k_def singleton dot_or_colon
                     {
-                      self.comments.push self.lexer.comments
                       lexer.lex_state = EXPR_FNAME
                     }
                     fname
                     {
-                      self.in_single += 1
+                      result = [self.in_def, lexer.lineno]
+
+                      self.in_single += 1 # TODO: remove?
+
+                      self.in_def = true # local_push
                       self.env.extend
-                      lexer.lex_state = EXPR_ENDFN # force for args
-                      result = [lexer.lineno, self.lexer.cmdarg.stack.dup]
-                      lexer.cmdarg.stack.replace [false]
+                      lexer.cmdarg.push false
+                      lexer.cond.push false
+
+                      lexer.lex_state = EXPR_ENDFN|EXPR_LABEL
+                      self.comments.push self.lexer.comments
                     }
                     f_arglist bodystmt k_end
                     {
-                      _, cmdarg = val[5]
-                      result = new_defs val
+                      _, _recv, _, _, _name, (in_def, _lineno), _args, _body, _ = val
 
-                      lexer.cmdarg.stack.replace cmdarg
+                      result = new_defs val
 
+                      lexer.cond.pop # group = local_pop
+                      lexer.cmdarg.pop
                       self.env.unextend
+                      self.in_def = in_def
+
                       self.in_single -= 1
+
+                      # TODO: restore cur_arg ? what's cur_arg?
+
                       self.lexer.comments # we don't care about comments in the body
                     }
                 | kBREAK
@@ -1715,20 +1772,19 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
                     f_larglist
                     {
-                      result = lexer.cmdarg.store(false)
+                      lexer.cmdarg.push false
                     }
                     lambda_body
                     {
-                      (line, lpar), args, cmdarg, body = val
+                      (line, lpar), args, _cmdarg, body = val
                       lexer.lpar_beg = lpar
 
-                      lexer.cmdarg.restore cmdarg
-                      lexer.cmdarg.lexpop
+                      lexer.cmdarg.pop
 
                       call = s(:lambda).line line
                       result = new_iter call, args, body
                       result.line = line
-                      self.env.unextend
+                      self.env.unextend # TODO: dynapush & dynapop
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl rparen
@@ -1881,7 +1937,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
 
          do_body:   { self.env.extend :dynamic; result = self.lexer.lineno }
-                    { result = lexer.cmdarg.store(false) }
+                    { lexer.cmdarg.push false }
                     opt_block_param
 #if V >= 25
                     bodystmt
@@ -1889,11 +1945,11 @@ opt_block_args_tail: tCOMMA block_args_tail
                     compstmt
 #endif
                     {
-                      line, cmdarg, param, cmpstmt = val
+                      line, _cmdarg, param, cmpstmt = val
 
                       result = new_do_body param, cmpstmt, line
+                      lexer.cmdarg.pop
                       self.env.unextend
-                      lexer.cmdarg.restore cmdarg
                     }
 
        case_body: k_when
@@ -1993,7 +2049,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
          xstring: tXSTRING_BEG xstring_contents tSTRING_END
                     {
-                      result = new_xstring val[1]
+                      result = new_xstring val
                       # TODO: dedent?!?! SERIOUSLY?!?
                     }
 
@@ -2135,12 +2191,13 @@ regexp_contents: none
                       result = [lexer.lex_strterm,
                                 lexer.brace_nest,
                                 lexer.string_nest, # TODO: remove
-                                lexer.cond.store,
-                                lexer.cmdarg.store,
                                 lexer.lex_state,
                                 lexer.lineno,
                                ]
 
+                      lexer.cmdarg.push false
+                      lexer.cond.push false
+
                       lexer.lex_strterm = nil
                       lexer.brace_nest  = 0
                       lexer.string_nest = 0
@@ -2152,14 +2209,15 @@ regexp_contents: none
                     {
                       _, memo, stmt, _ = val
 
-                      lex_strterm, brace_nest, string_nest, oldcond, oldcmdarg, oldlex_state, line = memo
+                      lex_strterm, brace_nest, string_nest, oldlex_state, line = memo
+                      # TODO: heredoc_indent
 
                       lexer.lex_strterm = lex_strterm
                       lexer.brace_nest  = brace_nest
                       lexer.string_nest = string_nest
 
-                      lexer.cond.restore oldcond
-                      lexer.cmdarg.restore oldcmdarg
+                      lexer.cmdarg.pop
+                      lexer.cond.pop
 
                       lexer.lex_state = oldlex_state
 

data/lib/ruby_parser_extras.rb

@@ -1,4 +1,5 @@
 # encoding: ASCII-8BIT
+# TODO: remove
 
 require "sexp"
 require "ruby_lexer"
@@ -28,7 +29,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.14.0"
+  VERSION = "3.16.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -45,6 +46,11 @@ module RubyParserStuff
 
   attr_accessor :canonicalize_conditions
 
+  ##
+  # The last token type returned from #next_token
+
+  attr_accessor :last_token_type
+
   $good20 = []
 
   %w[
@@ -109,7 +115,7 @@ module RubyParserStuff
   def initialize(options = {})
     super()
 
-    v = self.class.name[/2\d/]
+    v = self.class.name[/[23]\d/]
     raise "Bad Class name #{self.class}" unless v
 
     self.lexer = RubyLexer.new v && v.to_i
@@ -124,12 +130,6 @@ module RubyParserStuff
     self.reset
   end
 
-  def arg_blk_pass node1, node2 # TODO: nuke
-    node1 = s(:arglist, node1) unless ARG_TYPES[node1.sexp_type]
-    node1 << node2 if node2
-    node1
-  end
-
   def arg_concat node1, node2 # TODO: nuke
     raise "huh" unless node2
 
@@ -499,6 +499,8 @@ module RubyParserStuff
           str.encode! Encoding::UTF_8
           break
         end
+      rescue ArgumentError # unknown encoding name
+        # do nothing
       rescue Encoding::InvalidByteSequenceError
         # do nothing
       rescue Encoding::UndefinedConversionError
@@ -538,7 +540,7 @@ module RubyParserStuff
     header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
 
     first = header.first || ""
-    encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
+    encoding, str = "utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
 
     encoding = $1.strip if header.find { |s|
       s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
@@ -598,7 +600,9 @@ module RubyParserStuff
     case ttype
     when :str then
       if htype == :str
-        head.last << tail.last
+        a, b = head.last, tail.last
+        b = b.dup.force_encoding a.encoding unless Encoding.compatible?(a, b)
+        a << b
       elsif htype == :dstr and head.size == 2 then
         head.last << tail.last
       else
@@ -702,6 +706,15 @@ module RubyParserStuff
     result
   end
 
+  def new_begin val
+    _, lineno, body, _ = val
+
+    result = body ? s(:begin, body) : s(:nil)
+    result.line lineno
+
+    result
+  end
+
   def new_body val
     body, resbody, elsebody, ensurebody = val
 
@@ -729,7 +742,10 @@ module RubyParserStuff
       result = block_append(result, elsebody)
     end
 
-    result = s(:ensure, result, ensurebody).compact.line result.line if ensurebody
+    if ensurebody
+      lineno = (result || ensurebody).line
+      result = s(:ensure, result, ensurebody).compact.line lineno
+    end
 
     result
   end
@@ -831,6 +847,8 @@ module RubyParserStuff
     (_, line), name, _, args, body, nil_body_line, * = val
     body ||= s(:nil).line nil_body_line
 
+    args.line line
+
     result = s(:defn, name.to_sym, args).line line
 
     if body then
@@ -847,14 +865,17 @@ module RubyParserStuff
   end
 
   def new_defs val
-    recv, (name, _line), args, body = val[1], val[4], val[6], val[7]
-    line, _ = val[5]
+    _, recv, _, _, name, (_in_def, line), args, body, _ = val
+
     body ||= s(:nil).line line
 
     args.line line
 
     result = s(:defs, recv, name.to_sym, args)
 
+    # TODO: remove_begin
+    # TODO: reduce_nodes
+
     if body then
       if body.sexp_type == :block then
         result.push(*body.sexp_body)
@@ -879,7 +900,9 @@ module RubyParserStuff
   end
 
   def new_hash val
-    s(:hash, *val[2].values).line(val[1])
+    _, line, assocs = val
+
+    s(:hash).line(line).concat assocs.values
   end
 
   def new_if c, t, f
@@ -1138,6 +1161,7 @@ module RubyParserStuff
   def new_string val
     str, = val
     str.force_encoding("UTF-8")
+    # TODO: remove:
     str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
     result = s(:str, str).line lexer.lineno
     self.lexer.fixup_lineno str.count("\n")
@@ -1232,20 +1256,23 @@ module RubyParserStuff
     result
   end
 
-  def new_xstring str
-    if str then
-      case str.sexp_type
+  def new_xstring val
+    _, node = val
+
+    node ||= s(:str, "").line lexer.lineno
+
+    if node then
+      case node.sexp_type
       when :str
-        str.sexp_type = :xstr
+        node.sexp_type = :xstr
       when :dstr
-        str.sexp_type = :dxstr
+        node.sexp_type = :dxstr
       else
-        str = s(:dxstr, "", str)
+        node = s(:dxstr, "", node).line node.line
       end
-      str
-    else
-      s(:xstr, "")
     end
+
+    node
   end
 
   def new_yield args = nil
@@ -1266,6 +1293,7 @@ module RubyParserStuff
     token = self.lexer.next_token
 
     if token and token.first != RubyLexer::EOF then
+      self.last_token_type = token
       return token
     else
       return [false, false]
@@ -1324,6 +1352,7 @@ module RubyParserStuff
     self.in_single = 0
     self.env.reset
     self.comments.clear
+    self.last_token_type = nil
   end
 
   def ret_args node