ruby_parser 3.12.0 → 3.13.0

data/lib/ruby_lexer.rb

@@ -34,6 +34,10 @@ class RubyLexer
   STR_SSYM   = STR_FUNC_SYMBOL
   STR_DSYM   = STR_FUNC_SYMBOL | STR_FUNC_EXPAND
 
+  EXPR_BEG_ANY =  [:expr_beg, :expr_mid,  :expr_class  ]
+  EXPR_ARG_ANY =  [:expr_arg, :expr_cmdarg,            ]
+  EXPR_END_ANY =  [:expr_end, :expr_endarg, :expr_endfn]
+
   ESCAPES = {
     "a"    => "\007",
     "b"    => "\010",
@@ -77,7 +81,7 @@ class RubyLexer
   attr_accessor :brace_nest
   attr_accessor :cmdarg
   attr_accessor :command_start
-  attr_accessor :command_state
+  attr_accessor :cmd_state # temporary--ivar to avoid passing everywhere
   attr_accessor :last_state
   attr_accessor :cond
   attr_accessor :extra_lineno
@@ -109,20 +113,13 @@ class RubyLexer
   # Last token read via next_token.
   attr_accessor :token
 
-  ##
-  # What version of ruby to parse. 18 and 19 are the only valid values
-  # currently supported.
-
-  attr_accessor :version
-
   attr_writer :comments
 
-  def initialize v = 18
-    self.version = v
+  def initialize _ = nil
     @lex_state = :expr_none
 
-    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
     self.cond   = RubyParserStuff::StackState.new(:cond, $DEBUG)
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
 
     reset
   end
@@ -343,23 +340,30 @@ class RubyLexer
   end
 
   def is_arg?
-    in_lex_state? :expr_arg, :expr_cmdarg
+    in_lex_state?(*EXPR_ARG_ANY)
   end
 
   def is_beg?
-    in_lex_state? :expr_beg, :expr_value, :expr_mid, :expr_class, :expr_labelarg
+    # TODO: in_lex_state?(*EXPR_BEG_ANY) || lex_state == [:expr_arg, :expr_labeled]
+    in_lex_state?(*EXPR_BEG_ANY, :expr_value, :expr_labeled)
   end
 
   def is_end?
-    in_lex_state? :expr_end, :expr_endarg, :expr_endfn
+    in_lex_state?(*EXPR_END_ANY)
   end
 
+  def lvar_defined? id
+    # TODO: (dyna_in_block? && dvar_defined?(id)) || local_id?(id)
+    self.parser.env[id.to_sym] == :lvar
+  end
+
+
   def ruby22_label?
     ruby22plus? and is_label_possible?
   end
 
   def is_label_possible?
-    (in_lex_state?(:expr_beg, :expr_endfn) && !command_state) || is_arg?
+    (in_lex_state?(:expr_beg, :expr_endfn) && !cmd_state) || is_arg?
   end
 
   def is_label_suffix?
@@ -370,6 +374,10 @@ class RubyLexer
     is_arg? and space_seen and c !~ /\s/
   end
 
+  def lambda_beginning?
+    lpar_beg && lpar_beg == paren_nest
+  end
+
   def matched
     ss.matched
   end
@@ -411,20 +419,17 @@ class RubyLexer
     nil # TODO
   end
 
-  def process_bracing text
+  def process_brace_close text
+    # matching compare/parse23.y:8561
     cond.lexpop
     cmdarg.lexpop
 
     case matched
     when "}" then
       self.brace_nest -= 1
-      self.lex_state   = :expr_endarg
-
-      # TODO
-      # if (c == '}') {
-      #     if (!brace_nest--) c = tSTRING_DEND;
-      # }
+      self.lex_state   = :expr_endarg # TODO: :expr_end ? Look at 2.6
 
+      return :tSTRING_DEND, matched if brace_nest < 0
       return :tRCURLY, matched
     when "]" then
       self.paren_nest -= 1
@@ -463,24 +468,30 @@ class RubyLexer
     end
   end
 
-  def process_curly_brace text
+  def process_brace_open text
+    # matching compare/parse23.y:8694
     self.brace_nest += 1
-    if lpar_beg && lpar_beg == paren_nest then
+
+    if lambda_beginning? then
       self.lpar_beg = nil
-      self.paren_nest -= 1
+      self.paren_nest -= 1 # close arg list when lambda opens body
 
       return expr_result(:tLAMBEG, "{")
     end
 
-    token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
-               :tLCURLY      #  block (primary)
-             elsif in_lex_state?(:expr_endarg) then
-               :tLBRACE_ARG  #  block (expr)
-             else
-               :tLBRACE      #  hash
-             end
+    token = case lex_state
+            when :expr_labeled then
+              :tLBRACE     # hash
+            when *EXPR_ARG_ANY, :expr_end, :expr_endfn then
+              :tLCURLY     # block (primary)
+            when :expr_endarg
+              :tLBRACE_ARG # block (expr)
+            else
+              :tLBRACE     # hash
+            end
 
-    self.command_start = true unless token == :tLBRACE
+    # TODO: self.lex_state |= :expr_label if token != :tLBRACE_ARG
+    self.command_start = true if token != :tLBRACE
 
     return expr_result(token, "{")
   end
@@ -519,12 +530,19 @@ class RubyLexer
   def process_lchevron text
     if (!in_lex_state?(:expr_dot, :expr_class) &&
         !is_end? &&
-        (!is_arg? || space_seen)) then
+        (!is_arg? || space_seen)) then # TODO: || in_state(:expr_labeled)
       tok = self.heredoc_identifier
       return tok if tok
     end
 
-    return result(:arg_state, :tLSHFT, "\<\<")
+    if in_arg_state? then
+      self.lex_state = :expr_arg
+    else
+      self.command_start = true if lex_state == :expr_class
+      self.lex_state = :expr_beg
+    end
+
+    return result(lex_state, :tLSHFT, "\<\<")
   end
 
   def process_newline_or_comment text
@@ -534,6 +552,7 @@ class RubyLexer
     if c == '#' then
       ss.pos -= 1
 
+      # TODO: handle magic comments
       while scan(/\s*\#.*(\n+|\z)/) do
         hit = true
         self.lineno += matched.lines.to_a.size
@@ -548,8 +567,21 @@ class RubyLexer
     # Replace a string of newlines with a single one
     self.lineno += matched.lines.to_a.size if scan(/\n+/)
 
-    return if in_lex_state?(:expr_beg, :expr_value, :expr_class,
-                            :expr_fname, :expr_dot)
+    # TODO: remove :expr_value -- audit all uses of it
+    c = in_lex_state?(:expr_beg, :expr_value, :expr_class,
+                      :expr_fname, :expr_dot) && !in_lex_state?(:expr_labeled)
+
+    # TODO: figure out what token_seen is for
+    # TODO: if c || self.lex_state == [:expr_beg, :expr_labeled] then
+    if c || self.lex_state == :expr_labeled then
+      # ignore if !fallthrough?
+      if !c && parser.in_kwarg then
+        # normal newline
+        return result(:expr_beg, :tNL, nil)
+      else
+        return # skip
+      end
+    end
 
     if scan(/([\ \t\r\f\v]*)(\.|&)/) then
       self.space_seen = true unless ss[1].empty?
@@ -569,11 +601,7 @@ class RubyLexer
   end
 
   def process_paren text
-    token = if ruby18 then
-              process_paren18
-            else
-              process_paren19
-            end
+    token = process_paren19
 
     self.paren_nest += 1
 
@@ -581,25 +609,6 @@ class RubyLexer
     return expr_result(token, "(")
   end
 
-  def process_paren18
-    self.command_start = true
-    token = :tLPAREN2
-
-    if in_lex_state? :expr_beg, :expr_mid then
-      token = :tLPAREN
-    elsif space_seen then
-      if in_lex_state? :expr_cmdarg then
-        token = :tLPAREN_ARG
-      elsif in_lex_state? :expr_arg then
-        warning "don't put space before argument parentheses"
-      end
-    else
-      # not a ternary -- do nothing?
-    end
-
-    token
-  end
-
   def process_paren19
     if is_beg? then
       :tLPAREN
@@ -654,8 +663,7 @@ class RubyLexer
 
   def process_questionmark text
     if is_end? then
-      state = ruby18 ? :expr_beg : :expr_value # HACK?
-      return result(state, :tEH, "?")
+      return result(:expr_value, :tEH, "?")
     end
 
     if end_of_stream? then
@@ -677,8 +685,7 @@ class RubyLexer
       end
 
       # ternary
-      state = ruby18 ? :expr_beg : :expr_value # HACK?
-      return result(state, :tEH, "?")
+      return result(:expr_value, :tEH, "?")
     elsif check(/\w(?=\w)/) then # ternary, also
       return result(:expr_beg, :tEH, "?")
     end
@@ -689,11 +696,7 @@ class RubyLexer
           ss.getch
         end
 
-    if version == 18 then
-      return result(:expr_end, :tINTEGER, c[0].ord & 0xff)
-    else
-      return result(:expr_end, :tSTRING, c)
-    end
+    return result(:expr_end, :tSTRING, c)
   end
 
   def process_slash text
@@ -742,7 +745,9 @@ class RubyLexer
       token = :tLBRACK2
     end
 
-    return expr_result(token, "[")
+    # TODO: this is done by expr_result except "|EXPR_LABEL")
+    # SET_LEX_STATE(EXPR_BEG|EXPR_LABEL);
+    expr_result token, "["
   end
 
   def possibly_escape_string text, check
@@ -758,9 +763,6 @@ class RubyLexer
   def process_symbol text
     symbol = possibly_escape_string text, /^:"/
 
-    rb_compile_error "symbol cannot contain '\\0'" if
-      ruby18 && symbol =~ /\0/
-
     return result(:expr_end, :tSYMBOL, symbol)
   end
 
@@ -784,11 +786,14 @@ class RubyLexer
   def process_label text
     symbol = possibly_escape_string text, /^"/
 
-    result(:expr_labelarg, :tLABEL, [symbol, self.lineno])
+    result(:expr_labeled, :tLABEL, [symbol, self.lineno]) # TODO: expr_arg|expr_labeled
   end
 
   def process_token text
+    # matching: parse_ident in compare/parse23.y:7989
     # TODO: make this always return [token, lineno]
+    self.last_state = lex_state
+
     token = self.token = text
     token << matched if scan(/[\!\?](?!=)/)
 
@@ -807,36 +812,33 @@ class RubyLexer
         :tIDENTIFIER
       end
 
-    if !ruby18 and is_label_possible? and is_label_suffix? then
+    if is_label_possible? and is_label_suffix? then
       scan(/:/)
-      return result(:expr_labelarg, :tLABEL, [token, self.lineno])
+      # TODO: :expr_arg|:expr_labeled
+      return result :expr_labeled, :tLABEL, [token, self.lineno]
     end
 
+    # TODO: mb == ENC_CODERANGE_7BIT && !in_lex_state?(:expr_dot)
     unless in_lex_state? :expr_dot then
       # See if it is a reserved word.
-      keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
-                  RubyParserStuff::Keyword.keyword18 token
-                else
-                  RubyParserStuff::Keyword.keyword19 token
-                end
+      keyword = RubyParserStuff::Keyword.keyword token
 
       return process_token_keyword keyword if keyword
     end # unless in_lex_state? :expr_dot
 
-    # TODO:
-    # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {
-
+    # matching: compare/parse23.y:8079
     state = if is_beg? or is_arg? or in_lex_state? :expr_dot then
-              command_state ? :expr_cmdarg : :expr_arg
-            elsif not ruby18 and in_lex_state? :expr_fname then
+              cmd_state ? :expr_cmdarg : :expr_arg
+            elsif in_lex_state? :expr_fname then
               :expr_endfn
             else
               :expr_end
             end
 
     if not [:expr_dot, :expr_fname].include? last_state and
-        self.parser.env[token.to_sym] == :lvar then
-      state = :expr_end
+        (tok_id == :tIDENTIFIER) and # not :expr_fname, not attrasgn
+        lvar_defined?(token) then
+      state = :expr_end # TODO: EXPR_END|EXPR_LABEL
     end
 
     token.lineno = self.lineno # yes, on a string. I know... I know...
@@ -845,38 +847,38 @@ class RubyLexer
   end
 
   def process_token_keyword keyword
-    state = keyword.state
+    # matching MIDDLE of parse_ident in compare/parse23.y:8046
+    state = lex_state
+    self.lex_state = keyword.state
 
     value = [token, self.lineno]
 
-    self.command_start = true if state == :expr_beg and lex_state != :expr_fname
+    return result(lex_state, keyword.id0, value) if state == :expr_fname
+
+    self.command_start = true if lex_state == :expr_beg
 
     case
-    when lex_state == :expr_fname then
-      result(state, keyword.id0, keyword.name)
     when keyword.id0 == :kDO then
       case
-      when lpar_beg && lpar_beg == paren_nest then
-        self.lpar_beg = nil
+      when lambda_beginning? then
+        self.lpar_beg = nil # lambda_beginning? == FALSE in the body of "-> do ... end"
         self.paren_nest -= 1
-        expr_result(:kDO_LAMBDA, value)
+        result(lex_state, :kDO_LAMBDA, value)
       when cond.is_in_state then
-        result(state, :kDO_COND, value)
-      when cmdarg.is_in_state && lex_state != :expr_cmdarg then
-        result(state, :kDO_BLOCK, value)
-      when in_lex_state?(:expr_beg, :expr_endarg) then
-        result(state, :kDO_BLOCK, value)
-      when lex_state == :expr_end # eg: a -> do end do end
-        result(state, :kDO_BLOCK, value)
+        result(lex_state, :kDO_COND, value)
+      when cmdarg.is_in_state && state != :expr_cmdarg then
+        result(lex_state, :kDO_BLOCK, value)
+      when [:expr_beg, :expr_endarg].include?(state) then
+        result(lex_state, :kDO_BLOCK, value)
       else
-        result(state, :kDO, value)
+        result(lex_state, :kDO, value)
       end
-    when in_lex_state?(:expr_beg, :expr_value, :expr_labelarg) then
-      result(state, keyword.id0, value)
+    when [:expr_beg, :expr_labeled].include?(state) then
+      result(lex_state, keyword.id0, value)
     when keyword.id0 != keyword.id1 then
-      result(:expr_beg, keyword.id1, value)
+      result(:expr_beg, keyword.id1, value) # TODO: :expr_beg|:expr_label
     else
-      result(state, keyword.id1, value)
+      result(lex_state, keyword.id1, value)
     end
   end
 
@@ -982,8 +984,8 @@ class RubyLexer
     self.token         = nil
     self.extra_lineno  = 0
 
-    self.cmdarg.reset
     self.cond.reset
+    self.cmdarg.reset
   end
 
   def result lex_state, token, text # :nodoc:
@@ -992,10 +994,6 @@ class RubyLexer
     [token, text]
   end
 
-  def ruby18
-    RubyParser::V18 === parser
-  end
-
   def scan re
     ss.scan re
   end
@@ -1133,11 +1131,7 @@ class RubyLexer
         t = Regexp.escape term
         x = Regexp.escape(paren) if paren && paren != "\000"
         re = if qwords then
-               if HAS_ENC then
-                 /[^#{t}#{x}\#\0\\\s]+|./ # |. to pick up whatever
-               else
-                 /[^#{t}#{x}\#\0\\\s\v]+|./ # argh. 1.8's \s doesn't pick up \v
-               end
+               /[^#{t}#{x}\#\0\\\s]+|./ # |. to pick up whatever
              else
                /[^#{t}#{x}\#\0\\]+|./
              end
@@ -1201,6 +1195,7 @@ class RubyLexer
   end
 
   def process_string # TODO: rewrite / remove
+    # matches top of parser_yylex in compare/parse23.y:8113
     token = if lex_strterm[0] == :heredoc then
               self.heredoc lex_strterm
             else
@@ -1209,6 +1204,7 @@ class RubyLexer
 
     token_type, c = token
 
+    # matches parser_string_term
     if ruby22plus? && token_type == :tSTRING_END && ["'", '"'].include?(c) then
       if (([:expr_beg, :expr_endfn].include?(lex_state) &&
            !cond.is_in_state) || is_arg?) &&
@@ -1220,7 +1216,8 @@ class RubyLexer
 
     if [:tSTRING_END, :tREGEXP_END, :tLABEL_END].include? token_type then
       self.lex_strterm = nil
-      self.lex_state   = (token_type == :tLABEL_END) ? :expr_labelarg : :expr_end
+      # TODO: :expr_beg|:expr_label
+      self.lex_state   = (token_type == :tLABEL_END) ? :expr_label : :expr_end
     end
 
     return token
@@ -1326,6 +1323,7 @@ class RubyLexer
         # TODO: !ISASCII
         return :tSTRING_DVAR, nil
       when scan(/#[{]/) then
+        self.command_start = true
         return :tSTRING_DBEG, nil
       when scan(/#/) then
         string_buffer << '#'