ruby_parser 3.15.0 → 3.18.0

data/lib/ruby_parser_extras.rb

@@ -1,5 +1,6 @@
 # encoding: ASCII-8BIT
-# TODO: remove
+# frozen_string_literal: true
+# TODO: remove encoding comment
 
 require "sexp"
 require "ruby_lexer"
@@ -29,7 +30,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.15.0"
+  VERSION = "3.18.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -115,7 +116,7 @@ module RubyParserStuff
   def initialize(options = {})
     super()
 
-    v = self.class.name[/2\d/]
+    v = self.class.name[/[23]\d/]
     raise "Bad Class name #{self.class}" unless v
 
     self.lexer = RubyLexer.new v && v.to_i
@@ -155,11 +156,31 @@ module RubyParserStuff
     end
 
     args.each do |arg|
+      if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
+        arg = arg.first
+      end
+
       case arg
       when Sexp then
         case arg.sexp_type
         when :args, :block, :array, :call_args then # HACK call_args mismatch
-          result.concat arg.sexp_body
+          rest = arg.sexp_body
+
+          rest.map! { |x|
+            if x.instance_of? Array and x.size == 2 and Numeric === x.last then
+              x.first
+            else
+              x
+            end
+          }
+
+          result.concat rest
+        when :forward_args then
+          self.env[:*]  = :lvar # TODO: arg_var(p, idFWD_REST) ?
+          self.env[:**] = :lvar
+          self.env[:&]  = :lvar
+
+          result << arg
         when :block_arg then
           result << :"&#{arg.last}"
         when :shadow then
@@ -179,6 +200,8 @@ module RubyParserStuff
         name = arg.to_s.delete("&*")
         self.env[name.to_sym] = :lvar unless name.empty?
         result << arg
+      when true, false then
+        self.in_kwarg = arg
       when ",", "|", ";", "(", ")", nil then
         # ignore
       else
@@ -189,6 +212,23 @@ module RubyParserStuff
     result
   end
 
+  def end_args args
+    lexer.lex_state     = RubyLexer::State::Values::EXPR_BEG
+    lexer.command_start = true
+    self.args args
+  end
+
+  def endless_method_name defn_or_defs
+    name = defn_or_defs[1]
+    name = defn_or_defs[2] unless Symbol === name
+
+    if name.end_with? "=" then
+      yyerror "setter method cannot be defined in an endless method definition"
+    end
+
+    # TODO? token_info_drop(p, "def", loc->beg_pos);
+  end
+
   def array_to_hash array
     case array.sexp_type
     when :kwsplat then
@@ -208,17 +248,10 @@ module RubyParserStuff
   end
 
   def assignable(lhs, value = nil)
-    id = lhs.to_sym unless Sexp === lhs
-
-    raise "WTF" if Sexp === id
-    id = id.to_sym if Sexp === id
-
-    raise "write a test 1" if id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
+    id, line = lhs
+    id = id.to_sym
 
-    raise SyntaxError, "Can't change the value of #{id}" if
-      id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
-
-    result = case id.to_s
+    result = case id
              when /^@@/ then
                asgn = in_def || in_single > 0
                s((asgn ? :cvasgn : :cvdecl), id)
@@ -239,17 +272,9 @@ module RubyParserStuff
 
     self.env[id] ||= :lvar if result.sexp_type == :lasgn
 
-    line = case lhs
-           when Sexp then
-             lhs.line
-           else
-             value && value.line || lexer.lineno
-           end
-
     result << value if value
-    result.line = line
-
-    return result
+    result.line line
+    result
   end
 
   def backref_assign_error ref
@@ -273,9 +298,9 @@ module RubyParserStuff
     line = [head.line, tail.line].compact.min
 
     head = remove_begin(head)
-    head = s(:block, head) unless head.node_type == :block
+    head = s(:block, head).line(line) unless head.sexp_type == :block
 
-    head.line = line
+    # head.line = line
     head << tail
   end
 
@@ -301,6 +326,10 @@ module RubyParserStuff
     end
 
     args.each do |arg|
+      if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
+        arg = arg.first
+      end
+
       case arg
       when Sexp then
         case arg.sexp_type
@@ -311,7 +340,10 @@ module RubyParserStuff
         end
       when Symbol then
         result << arg
-      when ",", nil then
+      when Array then
+        id, _line = arg
+        result << id
+      when ",", nil, "(" then
         # ignore
       else
         raise "unhandled: #{arg.inspect} in #{args.inspect}"
@@ -435,7 +467,7 @@ module RubyParserStuff
         end
       else
         warn "unprocessed: %p" % [s]
-      end.map { |l| whitespace_width l[/^[ \t]*/] }
+      end.map { |l| whitespace_width l.chomp }
     }.compact.min
   end
 
@@ -459,7 +491,6 @@ module RubyParserStuff
   end
 
   def gettable(id)
-    lineno = id.lineno if id.respond_to? :lineno
     id = id.to_sym if String === id
 
     result = case id.to_s
@@ -480,8 +511,6 @@ module RubyParserStuff
                end
              end
 
-    result.line lineno if lineno
-
     raise "identifier #{id.inspect} is not valid" unless result
 
     result
@@ -540,7 +569,7 @@ module RubyParserStuff
     header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
 
     first = header.first || ""
-    encoding, str = "utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
+    encoding, str = +"utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
 
     encoding = $1.strip if header.find { |s|
       s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
@@ -624,7 +653,7 @@ module RubyParserStuff
     when :evstr then
       if htype == :str then
         f, l = head.file, head.line
-        head = s(:dstr, *head.sexp_body).line head.line
+        head = s(:dstr, *head.sexp_body)
         head.file = f
         head.line = l
       end
@@ -643,6 +672,13 @@ module RubyParserStuff
     return head
   end
 
+  def local_pop in_def
+    lexer.cond.pop # group = local_pop
+    lexer.cmdarg.pop
+    self.env.unextend
+    self.in_def = in_def
+  end
+
   def logical_op type, left, right
     left = value_expr left
 
@@ -672,6 +708,73 @@ module RubyParserStuff
     new_call val[0], :"[]", val[2]
   end
 
+  def new_arg val
+    arg, = val
+
+    case arg
+    when Symbol then
+      result = s(:args, arg).line line
+    when Sexp then
+      result = arg
+    when Array then
+      (arg, line), = val
+      result = s(:args, arg).line line
+    else
+      debug20 32
+      raise "Unknown f_arg type: #{val.inspect}"
+    end
+
+    result
+  end
+
+  def new_array_pattern const, pre_arg, arypat, loc
+    result = s(:array_pat, const).line loc
+    result << pre_arg if pre_arg
+
+    if arypat && arypat.sexp_type == :array_TAIL then
+      result.concat arypat.sexp_body
+    else
+      raise "NO?: %p" % [arypat]
+    end
+
+    result
+  end
+
+  def array_pat_concat lhs, rhs
+    case lhs.sexp_type
+    when :PATTERN then
+      lhs.sexp_type = :array_pat
+    end
+
+    if rhs then
+      case rhs.sexp_type
+      when :array_pat, :array_TAIL, :PATTERN then
+        lhs.concat rhs.sexp_body
+      else
+        lhs << rhs
+      end
+    end
+  end
+
+  def new_array_pattern_tail pre_args, has_rest, rest_arg, post_args
+    # TODO: remove has_rest once all tests pass !!!
+    rest_arg = if has_rest then
+                 :"*#{rest_arg}"
+               else
+                 nil
+               end
+
+    result = s(:array_TAIL).line 666
+
+    array_pat_concat result, pre_args
+
+    result << rest_arg if rest_arg
+
+    array_pat_concat result, post_args
+
+    result
+  end
+
   def new_assign lhs, rhs
     return nil unless lhs
 
@@ -691,6 +794,8 @@ module RubyParserStuff
   end
 
   def new_attrasgn recv, meth, call_op = :"."
+    call_op = call_op.first if Array === call_op
+
     meth = :"#{meth}="
 
     result = case call_op.to_sym
@@ -755,6 +860,8 @@ module RubyParserStuff
   end
 
   def new_call recv, meth, args = nil, call_op = :"."
+    call_op = call_op.first if Array === call_op
+
     result = case call_op.to_sym
              when :"."
                s(:call, recv, meth)
@@ -782,10 +889,14 @@ module RubyParserStuff
     result
   end
 
+  def new_in pat, body, cases, line
+    s(:in, pat, body, cases).line line
+  end
+
   def new_case expr, body, line
     result = s(:case, expr)
 
-    while body and body.node_type == :when
+    while body and [:when, :in].include? body.sexp_type
       result << body
       body = body.delete_at 3
     end
@@ -804,8 +915,11 @@ module RubyParserStuff
   end
 
   def new_class val
+    # TODO: get line from class keyword
     line, path, superclass, body = val[1], val[2], val[3], val[5]
 
+    path = path.first if path.instance_of? Array
+
     result = s(:class, path, superclass)
 
     if body then
@@ -828,7 +942,8 @@ module RubyParserStuff
   end
 
   def new_const_op_asgn val
-    lhs, asgn_op, rhs = val[0], val[1].to_sym, val[2]
+    lhs, (asgn_op, _), rhs = val
+    asgn_op = asgn_op.to_sym
 
     result = case asgn_op
              when :"||" then
@@ -844,55 +959,71 @@ module RubyParserStuff
   end
 
   def new_defn val
-    (_, line), name, _, args, body, nil_body_line, * = val
-    body ||= s(:nil).line nil_body_line
+    _, (name, line), in_def, args, body, _ = val
+
+    body ||= s(:nil).line line
 
     args.line line
 
     result = s(:defn, name.to_sym, args).line line
 
-    if body then
-      if body.sexp_type == :block then
-        result.push(*body.sexp_body)
-      else
-        result.push body
-      end
+    if body.sexp_type == :block then
+      result.push(*body.sexp_body)
+    else
+      result.push body
     end
 
     result.comments = self.comments.pop
 
-    result
+    [result, in_def]
   end
 
   def new_defs val
-    _, recv, _, _, name, (_in_def, line), args, body, _ = val
+    _, recv, (name, line), in_def, args, body, _ = val
 
     body ||= s(:nil).line line
 
     args.line line
 
-    result = s(:defs, recv, name.to_sym, args)
+    result = s(:defs, recv, name.to_sym, args).line line
 
     # TODO: remove_begin
     # TODO: reduce_nodes
 
-    if body then
-      if body.sexp_type == :block then
-        result.push(*body.sexp_body)
-      else
-        result.push body
-      end
+    if body.sexp_type == :block then
+      result.push(*body.sexp_body)
+    else
+      result.push body
     end
 
-    result.line = recv.line
     result.comments = self.comments.pop
-    result
+
+    [result, in_def]
   end
 
   def new_do_body args, body, lineno
     new_iter(nil, args, body).line(lineno)
   end
 
+  def new_find_pattern const, pat
+    pat.sexp_type = :find_pat
+    pat.insert 1, const
+  end
+
+  def new_find_pattern_tail lhs, mid, rhs
+    lhs_id, line = lhs
+    rhs_id, _line = rhs
+
+    # TODO: fpinfo->pre_rest_arg = pre_rest_arg ? assignable(p, pre_rest_arg, 0, loc) : NODE_SPECIAL_NO_NAME_REST;
+
+    lhs_id = "*#{lhs_id}".to_sym
+    rhs_id = "*#{rhs_id}".to_sym
+
+    mid.sexp_type = :array_pat # HACK?
+
+    s(:find_pat_TAIL, lhs_id, mid, rhs_id).line line
+  end
+
   def new_for expr, var, body
     result = s(:for, expr, var).line(var.line)
     result << body if body
@@ -902,7 +1033,47 @@ module RubyParserStuff
   def new_hash val
     _, line, assocs = val
 
-    s(:hash).line(line).concat assocs.values
+    s(:hash).line(line).concat assocs.sexp_body
+  end
+
+  def new_hash_pattern const, hash_pat, loc
+    _, pat, kw_args, kw_rest_arg = hash_pat
+
+    line = (const||hash_pat).line
+
+    result = s(:hash_pat, const).line line
+    result.concat pat.sexp_body if pat
+    result << kw_args     if kw_args
+    result << kw_rest_arg if kw_rest_arg
+    result
+  end
+
+  def new_hash_pattern_tail kw_args, kw_rest_arg, line # TODO: remove line arg
+    # kw_rest_arg = assignable(kw_rest_arg, nil).line line if kw_rest_arg
+
+    result = s(:hash_pat).line line
+    result << kw_args
+
+    if kw_rest_arg then
+      name = kw_rest_arg.value
+      # TODO: I _hate_ this:
+      assignable [name, kw_rest_arg.line] if name != :**
+      result << kw_rest_arg
+    end
+
+    result
+  end
+
+  def push_pktbl
+  end
+
+  def pop_pktbl
+  end
+
+  def push_pvtbl
+  end
+
+  def pop_pvtbl
   end
 
   def new_if c, t, f
@@ -979,9 +1150,12 @@ module RubyParserStuff
   end
 
   def new_module val
+    # TODO: get line from module keyword
     line, path, body = val[1], val[2], val[4]
 
-    result = s(:module, path)
+    path = path.first if path.instance_of? Array
+
+    result = s(:module, path).line line
 
     if body then # REFACTOR?
       if body.sexp_type == :block then
@@ -991,32 +1165,33 @@ module RubyParserStuff
       end
     end
 
-    result.line = line
     result.comments = self.comments.pop
     result
   end
 
   def new_op_asgn val
-    lhs, asgn_op, arg = val[0], val[1].to_sym, val[2]
-    name = gettable(lhs.value).line lhs.line
-    arg = remove_begin(arg)
-    result = case asgn_op # REFACTOR
+    lhs, (op, _line), rhs = val
+    op = op.to_sym
+
+    name = gettable(lhs.last).line lhs.line
+    arg = remove_begin rhs
+    result = case op # REFACTOR
              when :"||" then
                lhs << arg
-               s(:op_asgn_or, name, lhs)
+               s(:op_asgn_or, name, lhs).line lhs.line
              when :"&&" then
                lhs << arg
-               s(:op_asgn_and, name, lhs)
+               s(:op_asgn_and, name, lhs).line lhs.line
              else
-               lhs << new_call(name, asgn_op, argl(arg))
+               lhs << new_call(name, op, argl(arg))
                lhs
              end
-    result.line = lhs.line
+
     result
   end
 
   def new_op_asgn1 val
-    lhs, _, args, _, op, rhs = val
+    lhs, _, args, _, (op, _), rhs = val
 
     args.sexp_type = :arglist if args
 
@@ -1026,7 +1201,7 @@ module RubyParserStuff
   end
 
   def new_op_asgn2 val
-    recv, call_op, meth, op, arg = val
+    recv, (call_op, _), (meth, _), (op, _), arg = val
     meth = :"#{meth}="
 
     result = case call_op.to_sym
@@ -1043,36 +1218,28 @@ module RubyParserStuff
   end
 
   def new_qsym_list
-    result = s(:array).line lexer.lineno
-    self.lexer.fixup_lineno
-    result
+    s(:array).line lexer.lineno
   end
 
   def new_qsym_list_entry val
-    _, str, _ = val
-    result = s(:lit, str.to_sym).line lexer.lineno
-    self.lexer.fixup_lineno
-    result
+    _, (str, line), _ = val
+    s(:lit, str.to_sym).line line
   end
 
   def new_qword_list
-    result = s(:array).line lexer.lineno
-    self.lexer.fixup_lineno
-    result
+    s(:array).line lexer.lineno
   end
 
   def new_qword_list_entry val
-    _, str, _ = val
+    _, (str, line), _ = val
     str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
-    result = s(:str, str).line lexer.lineno # TODO: problematic? grab from parser
-    self.lexer.fixup_lineno
-    result
+    s(:str, str).line line
   end
 
   def new_regexp val
-    _, node, options = val
+    (_, line), node, (options, _) = val
 
-    node ||= s(:str, "").line lexer.lineno
+    node ||= s(:str, "").line line
 
     o, k = 0, nil
     options.split(//).uniq.each do |c| # FIX: this has a better home
@@ -1099,12 +1266,12 @@ module RubyParserStuff
                   begin
                     Regexp.new(node[1], o)
                   rescue RegexpError => e
-                    warn "WA\RNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
+                    warn "WARNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
                     begin
-                      warn "WA\RNING: trying to recover with ENC_UTF8"
+                      warn "WARNING: trying to recover with ENC_UTF8"
                       Regexp.new(node[1], Regexp::ENC_UTF8)
                     rescue RegexpError => e
-                      warn "WA\RNING: trying to recover with ENC_NONE"
+                      warn "WARNING: trying to recover with ENC_NONE"
                       Regexp.new(node[1], Regexp::ENC_NONE)
                     end
                   end
@@ -1117,7 +1284,7 @@ module RubyParserStuff
       end
       node << o if o and o != 0
     else
-      node = s(:dregx, "", node).line node.line
+      node = s(:dregx, "", node).line line
       node.sexp_type = :dregx_once if options =~ /o/
       node << o if o and o != 0
     end
@@ -1159,17 +1326,16 @@ module RubyParserStuff
   end
 
   def new_string val
-    str, = val
+    (str, line), = val
+
     str.force_encoding("UTF-8")
     # TODO: remove:
     str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
-    result = s(:str, str).line lexer.lineno
-    self.lexer.fixup_lineno str.count("\n")
-    result
+    s(:str, str).line line
   end
 
   def new_super args
-    if args && args.node_type == :block_pass then
+    if args && args.sexp_type == :block_pass then
       s(:super, args).line args.line
     else
       args ||= s(:arglist).line lexer.lineno
@@ -1177,32 +1343,30 @@ module RubyParserStuff
     end
   end
 
+  def new_symbol val
+    name = val.last
+    s(:lit, name.to_sym).line lexer.lineno
+  end
+
   def new_symbol_list
-    result = s(:array).line lexer.lineno
-    self.lexer.fixup_lineno
-    result
+    # TODO: hunt down and try to remove ALL lexer.lineno usage!
+    s(:array).line lexer.lineno
   end
 
   def new_symbol_list_entry val
     _, sym, _ = val
 
-    sym ||= s(:str, "")
-
-    line = lexer.lineno
+    sym ||= s(:str, "").line lexer.lineno
 
     case sym.sexp_type
     when :dstr then
       sym.sexp_type = :dsym
     when :str then
-      sym = s(:lit, sym.last.to_sym)
+      sym = s(:lit, sym.last.to_sym).line sym.line
     else
-      sym = s(:dsym, "", sym || s(:str, "").line(line))
+      sym = s(:dsym, "", sym).line sym.line
     end
 
-    sym.line line
-
-    self.lexer.fixup_lineno
-
     sym
   end
 
@@ -1244,16 +1408,12 @@ module RubyParserStuff
   end
 
   def new_word_list
-    result = s(:array).line lexer.lineno
-    self.lexer.fixup_lineno
-    result
+    s(:array).line lexer.lineno
   end
 
   def new_word_list_entry val
     _, word, _ = val
-    result = word.sexp_type == :evstr ? s(:dstr, "", word).line(word.line) : word
-    self.lexer.fixup_lineno
-    result
+    word.sexp_type == :evstr ? s(:dstr, "", word).line(word.line) : word
   end
 
   def new_xstring val
@@ -1277,9 +1437,9 @@ module RubyParserStuff
 
   def new_yield args = nil
     # TODO: raise args.inspect unless [:arglist].include? args.first # HACK
-    raise "write a test 4" if args && args.node_type == :block_pass
+    raise "write a test 4" if args && args.sexp_type == :block_pass
     raise SyntaxError, "Block argument should not be given." if
-      args && args.node_type == :block_pass
+      args && args.sexp_type == :block_pass
 
     args ||= s(:arglist).line lexer.lineno
 
@@ -1289,18 +1449,30 @@ module RubyParserStuff
     s(:yield, *args.sexp_body).line args.line
   end
 
+  def prev_value_to_lineno v
+    s, n = v
+    if String === s then
+      n
+    else
+      lexer.lineno
+    end
+  end
+
   def next_token
     token = self.lexer.next_token
 
     if token and token.first != RubyLexer::EOF then
       self.last_token_type = token
       return token
+    elsif !token
+      return self.lexer.next_token
     else
       return [false, false]
     end
   end
 
   def on_error(et, ev, values)
+    ev = ev.first if ev.instance_of?(Array) && ev.size == 2 && ev.last.is_a?(Integer)
     super
   rescue Racc::ParseError => e
     # I don't like how the exception obscures the error message
@@ -1314,18 +1486,17 @@ module RubyParserStuff
   # Timeout::Error if it runs for more than +time+ seconds.
 
   def process(str, file = "(string)", time = 10)
+    str.freeze
+
     Timeout.timeout time do
       raise "bad val: #{str.inspect}" unless String === str
 
-      str = handle_encoding str
+      self.lexer.string = handle_encoding str
 
       self.file = file.dup
 
       @yydebug = ENV.has_key? "DEBUG"
 
-      # HACK -- need to get tests passing more than have graceful code
-      self.lexer.ss = RPStringScanner.new str
-
       do_parse
     end
   end
@@ -1381,6 +1552,14 @@ module RubyParserStuff
     result
   end
 
+  def debug n
+    if ENV["PRY"] then
+      require "pry"; binding.pry
+    end
+
+    raise RubyParser::SyntaxError, "debug #{n}"
+  end
+
   def syntax_error msg
     raise RubyParser::SyntaxError, msg
   end
@@ -1425,6 +1604,8 @@ module RubyParserStuff
 
     if remove_width then
       line[idx..-1]
+    elsif line[idx].nil?
+      nil
     else
       col
     end