ruby_parser 3.9.0 → 3.10.0

data/lib/ruby24_parser.y

@@ -185,7 +185,7 @@ rule
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
-                        result[0] = :safe_op_asgn
+                        result.sexp_type = :safe_op_asgn
                       end
                       result.line = val[0].line
                     }
@@ -193,7 +193,7 @@ rule
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
-                        result[0] = :safe_op_asgn
+                        result.sexp_type = :safe_op_asgn
                       end
                       result.line = val[0].line
                     }
@@ -287,11 +287,11 @@ rule
 
          command: fcall command_args =tLOWEST
                     {
-                      result = val[0].concat val[1][1..-1] # REFACTOR pattern
+                      result = val[0].concat val[1].sexp_body # REFACTOR pattern
                     }
                 | fcall command_args cmd_brace_block
                     {
-                      result = val[0].concat val[1][1..-1]
+                      result = val[0].concat val[1].sexp_body
                       if val[2] then
                         block_dup_check result, val[2]
 
@@ -380,7 +380,7 @@ rule
                       ary1, _, splat, _, ary2 = val
 
                       result = list_append ary1, s(:splat, splat)
-                      result.concat ary2[1..-1]
+                      result.concat ary2.sexp_body
                       result = s(:masgn, result)
                     }
                 | mlhs_head tSTAR
@@ -390,7 +390,7 @@ rule
                 | mlhs_head tSTAR tCOMMA mlhs_post
                     {
                       ary = list_append val[0], s(:splat)
-                      ary.concat val[3][1..-1]
+                      ary.concat val[3].sexp_body
                       result = s(:masgn, ary)
                     }
                 | tSTAR mlhs_node
@@ -400,7 +400,7 @@ rule
                 | tSTAR mlhs_node tCOMMA mlhs_post
                     {
                       ary = s(:array, s(:splat, val[1]))
-                      ary.concat val[3][1..-1]
+                      ary.concat val[3].sexp_body
                       result = s(:masgn, ary)
                     }
                 | tSTAR
@@ -409,7 +409,7 @@ rule
                     }
                 | tSTAR tCOMMA mlhs_post
                     {
-                      result = s(:masgn, s(:array, s(:splat), *val[2][1..-1]))
+                      result = s(:masgn, s(:array, s(:splat), *val[2].sexp_body))
                     }
 
        mlhs_item: mlhs_node
@@ -620,7 +620,7 @@ rule
                     }
                 | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg
                     {
-                      val[2][0] = :arglist if val[2]
+                      val[2].sexp_type = :arglist if val[2]
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
                     }
                 | primary_value call_op tIDENTIFIER tOP_ASGN arg
@@ -956,14 +956,20 @@ rule
                     {
                       debug20 13, val, result
                     }
-                | tLPAREN_ARG expr
+                | tLPAREN_ARG
+                    {
+                      result = self.lexer.cmdarg.stack.dup
+                      lexer.cmdarg.stack.replace [false] # TODO add api for these
+                    }
+                    expr
                     {
                       lexer.lex_state = :expr_endarg
                     }
                     rparen
                     {
                       warning "(...) interpreted as grouped expression"
-                      result = val[1]
+                      lexer.cmdarg.stack.replace val[1]
+                      result = val[2]
                     }
                 | tLPAREN compstmt tRPAREN
                     {
@@ -981,7 +987,7 @@ rule
                 | tLBRACK aref_args tRBRACK
                     {
                       result = val[1] || s(:array)
-                      result[0] = :array # aref_args is :args
+                      result.sexp_type = :array # aref_args is :args
                     }
                 | tLBRACE
                     {
@@ -1182,12 +1188,16 @@ rule
                       self.in_single += 1
                       self.env.extend
                       lexer.lex_state = :expr_endfn # force for args
-                      result = lexer.lineno
+                      result = [lexer.lineno, self.lexer.cmdarg.stack.dup]
+                      lexer.cmdarg.stack.replace [false]
                     }
                     f_arglist bodystmt kEND
                     {
+                      line, cmdarg = val[5]
                       result = new_defs val
-                      result[3].line val[5]
+                      result[3].line line
+
+                      lexer.cmdarg.stack.replace cmdarg
 
                       self.env.unextend
                       self.in_single -= 1
@@ -1543,7 +1553,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     paren_args
                     {
                       args = self.call_args val[2..-1]
-                      result = val[0].concat args[1..-1]
+                      result = val[0].concat args.sexp_body
                     }
                 | primary_value call_op operation2 opt_paren_args
                     {
@@ -1630,10 +1640,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       opt_rescue: kRESCUE exc_list exc_var then compstmt opt_rescue
                     {
-                      _, klasses, var, _, body, rest = val
+                      (_, line), klasses, var, _, body, rest = val
 
                       klasses ||= s(:array)
                       klasses << new_assign(var, s(:gvar, :"$!")) if var
+                      klasses.line line
 
                       result = new_resbody(klasses, body)
                       result << rest if rest # UGH, rewritten above
@@ -1676,7 +1687,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
          strings: string
                     {
-                      val[0] = s(:dstr, val[0].value) if val[0][0] == :evstr
+                      val[0] = s(:dstr, val[0].value) if val[0].sexp_type == :evstr
                       result = val[0]
                     }
 
@@ -1864,7 +1875,7 @@ regexp_contents: none
 
                       case stmt
                       when Sexp then
-                        case stmt[0]
+                        case stmt.sexp_type
                         when :str, :dstr, :evstr then
                           result = stmt
                         else
@@ -1902,9 +1913,9 @@ regexp_contents: none
 
                       result ||= s(:str, "")
 
-                      case result[0]
+                      case result.sexp_type
                       when :dstr then
-                        result[0] = :dsym
+                        result.sexp_type = :dsym
                       when :str then
                         result = s(:lit, result.last.to_sym)
                       when :evstr then
@@ -2288,7 +2299,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = val[2]
                       yyerror "Can't define single method for literals." if
-                        result[0] == :lit
+                        result.sexp_type == :lit
                     }
 
       assoc_list: none # [!nil]
@@ -2304,10 +2315,10 @@ keyword_variable: kNIL      { result = s(:nil)   }
                 | assocs tCOMMA assoc
                     {
                       list = val[0].dup
-                      more = val[2][1..-1]
+                      more = val[2].sexp_body
                       list.push(*more) unless more.empty?
                       result = list
-                      result[0] = :hash
+                      result.sexp_type = :hash
                     }
 
            assoc: arg_value tASSOC arg_value
@@ -2321,7 +2332,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                 | tSTRING_BEG string_contents tLABEL_END arg_value
                     {
                       _, sym, _, value = val
-                      sym[0] = :dsym
+                      sym.sexp_type = :dsym
                       result = s(:array, sym, value)
                     }
                 | tSYMBOL arg_value

data/lib/ruby_lexer.rb

@@ -22,6 +22,7 @@ class RubyLexer
   STR_FUNC_QWORDS = 0x08
   STR_FUNC_SYMBOL = 0x10
   STR_FUNC_INDENT = 0x20 # <<-HEREDOC
+  STR_FUNC_ICNTNT = 0x40 # <<~HEREDOC
 
   STR_SQUOTE = STR_FUNC_BORING
   STR_DQUOTE = STR_FUNC_BORING | STR_FUNC_EXPAND
@@ -62,6 +63,8 @@ class RubyLexer
     "->"  => :tLAMBDA,
   }
 
+  TAB_WIDTH = 8
+
   @@regexp_cache = Hash.new { |h,k| h[k] = Regexp.new(Regexp.escape(k)) }
   @@regexp_cache[nil] = nil
 
@@ -89,6 +92,17 @@ class RubyLexer
   attr_accessor :string_buffer
   attr_accessor :string_nest
 
+  if $DEBUG then
+    alias lex_state= lex_state=
+    def lex_state=o
+      return if @lex_state == o
+      c = caller.first
+      c = caller[1] if c =~ /\bresult\b/
+      warn "lex_state: %p -> %p from %s" % [@lex_state, o, c.clean_caller]
+      @lex_state = o
+    end
+  end
+
   # Last token read via next_token.
   attr_accessor :token
 
@@ -102,6 +116,10 @@ class RubyLexer
 
   def initialize v = 18
     self.version = v
+    @lex_state = :expr_none
+
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
+    self.cond   = RubyParserStuff::StackState.new(:cond, $DEBUG)
 
     reset
   end
@@ -146,10 +164,11 @@ class RubyLexer
   def heredoc here # TODO: rewrite / remove
     _, eos, func, last_line = here
 
-    indent  = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
-    expand  = (func & STR_FUNC_EXPAND) != 0
-    eos_re  = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
-    err_msg = "can't match #{eos_re.inspect} anywhere in "
+    indent         = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
+    content_indent = (func & STR_FUNC_ICNTNT) != 0
+    expand         = (func & STR_FUNC_EXPAND) != 0
+    eos_re         = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
+    err_msg        = "can't match #{eos_re.inspect} anywhere in "
 
     rb_compile_error err_msg if end_of_stream?
 
@@ -195,17 +214,64 @@ class RubyLexer
 
     self.lex_strterm = [:heredoc, eos, func, last_line]
 
-    return :tSTRING_CONTENT, string_buffer.join.delete("\r")
+    string_content = string_buffer.join.delete("\r")
+
+    string_content = heredoc_dedent(string_content) if content_indent && ruby23?
+
+    return :tSTRING_CONTENT, string_content
+  end
+
+  def heredoc_dedent(string_content)
+    width = string_content.scan(/^[ \t]*(?=\S)/).map do |whitespace|
+      heredoc_whitespace_indent_size whitespace
+    end.min || 0
+
+    string_content.split("\n", -1).map do |line|
+      dedent_string line, width
+    end.join "\n"
+  end
+
+  def dedent_string(string, width)
+    characters_skipped = 0
+    indentation_skipped = 0
+
+    string.chars.each do |char|
+      break if indentation_skipped >= width
+      if char == ' '
+        characters_skipped += 1
+        indentation_skipped += 1
+      elsif char == "\t"
+        proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1)
+        break if (proposed > width)
+        characters_skipped += 1
+        indentation_skipped = proposed
+      end
+    end
+    string[characters_skipped..-1]
+  end
+
+  def heredoc_whitespace_indent_size(whitespace)
+    whitespace.chars.inject 0 do |size, char|
+      if char == "\t"
+        size + TAB_WIDTH
+      else
+        size + 1
+      end
+    end
   end
 
   def heredoc_identifier # TODO: remove / rewrite
     term, func = nil, STR_FUNC_BORING
     self.string_buffer = []
 
+    heredoc_indent_mods = '-'
+    heredoc_indent_mods += '\~' if ruby23?
+
     case
-    when scan(/(-?)([\'\"\`])(.*?)\2/) then
+    when scan(/([#{heredoc_indent_mods}]?)([\'\"\`])(.*?)\2/) then
       term = ss[2]
       func |= STR_FUNC_INDENT unless ss[1].empty?
+      func |= STR_FUNC_ICNTNT if ss[1] == '~'
       func |= case term
               when "\'" then
                 STR_SQUOTE
@@ -215,13 +281,14 @@ class RubyLexer
                 STR_XQUOTE
               end
       string_buffer << ss[3]
-    when scan(/-?([\'\"\`])(?!\1*\Z)/) then
+    when scan(/[#{heredoc_indent_mods}]?([\'\"\`])(?!\1*\Z)/) then
       rb_compile_error "unterminated here document identifier"
-    when scan(/(-?)(#{IDENT_CHAR}+)/) then
+    when scan(/([#{heredoc_indent_mods}]?)(#{IDENT_CHAR}+)/) then
       term = '"'
       func |= STR_DQUOTE
       unless ss[1].empty? then
         func |= STR_FUNC_INDENT
+        func |= STR_FUNC_ICNTNT if ss[1] == '~'
       end
       string_buffer << ss[2]
     else
@@ -507,6 +574,7 @@ class RubyLexer
 
     self.paren_nest += 1
 
+    # TODO: add :expr_label to :expr_beg (set in expr_result below)
     return expr_result(token, "(")
   end
 
@@ -817,6 +885,7 @@ class RubyLexer
     when scan(/\\/) then                  # Backslash
       '\\'
     when scan(/n/) then                   # newline
+      self.extra_lineno -= 1
       "\n"
     when scan(/t/) then                   # horizontal tab
       "\t"
@@ -888,7 +957,7 @@ class RubyLexer
     self.brace_nest    = 0
     self.command_start = true
     self.comments      = []
-    self.lex_state     = nil
+    self.lex_state     = :expr_none
     self.lex_strterm   = nil
     self.lineno        = 1
     self.lpar_beg      = nil
@@ -898,8 +967,8 @@ class RubyLexer
     self.token         = nil
     self.extra_lineno  = 0
 
-    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
-    self.cond   = RubyParserStuff::StackState.new(:cond)
+    self.cmdarg.reset
+    self.cond.reset
   end
 
   def result lex_state, token, text # :nodoc:
@@ -1106,6 +1175,10 @@ class RubyLexer
     parser.class.version >= 22
   end
 
+  def ruby23?
+    Ruby23Parser === parser
+  end
+
   def process_string # TODO: rewrite / remove
     token = if lex_strterm[0] == :heredoc then
               self.heredoc lex_strterm

data/lib/ruby_parser.yy

@@ -203,7 +203,7 @@ rule
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
-                        result[0] = :safe_op_asgn
+                        result.sexp_type = :safe_op_asgn
                       end
                       result.line = val[0].line
                     }
@@ -211,7 +211,7 @@ rule
                     {
                       result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                       if val[1] == '&.'
-                        result[0] = :safe_op_asgn
+                        result.sexp_type = :safe_op_asgn
                       end
                       result.line = val[0].line
                     }
@@ -313,11 +313,11 @@ rule
 
          command: fcall command_args =tLOWEST
                     {
-                      result = val[0].concat val[1][1..-1] # REFACTOR pattern
+                      result = val[0].concat val[1].sexp_body # REFACTOR pattern
                     }
                 | fcall command_args cmd_brace_block
                     {
-                      result = val[0].concat val[1][1..-1]
+                      result = val[0].concat val[1].sexp_body
                       if val[2] then
                         block_dup_check result, val[2]
 
@@ -406,7 +406,7 @@ rule
                       ary1, _, splat, _, ary2 = val
 
                       result = list_append ary1, s(:splat, splat)
-                      result.concat ary2[1..-1]
+                      result.concat ary2.sexp_body
                       result = s(:masgn, result)
                     }
                 | mlhs_head tSTAR
@@ -416,7 +416,7 @@ rule
                 | mlhs_head tSTAR tCOMMA mlhs_post
                     {
                       ary = list_append val[0], s(:splat)
-                      ary.concat val[3][1..-1]
+                      ary.concat val[3].sexp_body
                       result = s(:masgn, ary)
                     }
                 | tSTAR mlhs_node
@@ -426,7 +426,7 @@ rule
                 | tSTAR mlhs_node tCOMMA mlhs_post
                     {
                       ary = s(:array, s(:splat, val[1]))
-                      ary.concat val[3][1..-1]
+                      ary.concat val[3].sexp_body
                       result = s(:masgn, ary)
                     }
                 | tSTAR
@@ -435,7 +435,7 @@ rule
                     }
                 | tSTAR tCOMMA mlhs_post
                     {
-                      result = s(:masgn, s(:array, s(:splat), *val[2][1..-1]))
+                      result = s(:masgn, s(:array, s(:splat), *val[2].sexp_body))
                     }
 
        mlhs_item: mlhs_node
@@ -649,7 +649,7 @@ rule
                     }
                 | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg
                     {
-                      val[2][0] = :arglist if val[2]
+                      val[2].sexp_type = :arglist if val[2]
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
                     }
                 | primary_value call_op tIDENTIFIER tOP_ASGN arg
@@ -999,14 +999,20 @@ rule
                     {
                       debug20 13, val, result
                     }
-                | tLPAREN_ARG expr
+                | tLPAREN_ARG
+                    {
+                      result = self.lexer.cmdarg.stack.dup
+                      lexer.cmdarg.stack.replace [false] # TODO add api for these
+                    }
+                    expr
                     {
                       lexer.lex_state = :expr_endarg
                     }
                     rparen
                     {
                       warning "(...) interpreted as grouped expression"
-                      result = val[1]
+                      lexer.cmdarg.stack.replace val[1]
+                      result = val[2]
                     }
                 | tLPAREN compstmt tRPAREN
                     {
@@ -1024,7 +1030,7 @@ rule
                 | tLBRACK aref_args tRBRACK
                     {
                       result = val[1] || s(:array)
-                      result[0] = :array # aref_args is :args
+                      result.sexp_type = :array # aref_args is :args
                     }
                 | tLBRACE
                     {
@@ -1225,12 +1231,16 @@ rule
                       self.in_single += 1
                       self.env.extend
                       lexer.lex_state = :expr_endfn # force for args
-                      result = lexer.lineno
+                      result = [lexer.lineno, self.lexer.cmdarg.stack.dup]
+                      lexer.cmdarg.stack.replace [false]
                     }
                     f_arglist bodystmt kEND
                     {
+                      line, cmdarg = val[5]
                       result = new_defs val
-                      result[3].line val[5]
+                      result[3].line line
+
+                      lexer.cmdarg.stack.replace cmdarg
 
                       self.env.unextend
                       self.in_single -= 1
@@ -1586,7 +1596,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     paren_args
                     {
                       args = self.call_args val[2..-1]
-                      result = val[0].concat args[1..-1]
+                      result = val[0].concat args.sexp_body
                     }
                 | primary_value call_op operation2 opt_paren_args
                     {
@@ -1673,10 +1683,11 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       opt_rescue: kRESCUE exc_list exc_var then compstmt opt_rescue
                     {
-                      _, klasses, var, _, body, rest = val
+                      (_, line), klasses, var, _, body, rest = val
 
                       klasses ||= s(:array)
                       klasses << new_assign(var, s(:gvar, :"$!")) if var
+                      klasses.line line
 
                       result = new_resbody(klasses, body)
                       result << rest if rest # UGH, rewritten above
@@ -1719,7 +1730,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
          strings: string
                     {
-                      val[0] = s(:dstr, val[0].value) if val[0][0] == :evstr
+                      val[0] = s(:dstr, val[0].value) if val[0].sexp_type == :evstr
                       result = val[0]
                     }
 
@@ -1911,7 +1922,7 @@ regexp_contents: none
 
                       case stmt
                       when Sexp then
-                        case stmt[0]
+                        case stmt.sexp_type
                         when :str, :dstr, :evstr then
                           result = stmt
                         else
@@ -1949,9 +1960,9 @@ regexp_contents: none
 
                       result ||= s(:str, "")
 
-                      case result[0]
+                      case result.sexp_type
                       when :dstr then
-                        result[0] = :dsym
+                        result.sexp_type = :dsym
                       when :str then
                         result = s(:lit, result.last.to_sym)
                       when :evstr then
@@ -2381,7 +2392,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = val[2]
                       yyerror "Can't define single method for literals." if
-                        result[0] == :lit
+                        result.sexp_type == :lit
                     }
 
       assoc_list: none # [!nil]
@@ -2397,10 +2408,10 @@ keyword_variable: kNIL      { result = s(:nil)   }
                 | assocs tCOMMA assoc
                     {
                       list = val[0].dup
-                      more = val[2][1..-1]
+                      more = val[2].sexp_body
                       list.push(*more) unless more.empty?
                       result = list
-                      result[0] = :hash
+                      result.sexp_type = :hash
                     }
 
            assoc: arg_value tASSOC arg_value
@@ -2415,7 +2426,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                 | tSTRING_BEG string_contents tLABEL_END arg_value
                     {
                       _, sym, _, value = val
-                      sym[0] = :dsym
+                      sym.sexp_type = :dsym
                       result = s(:array, sym, value)
                     }
                 | tSYMBOL arg_value