ruby-next-parser 3.0.1.0 → 3.1.1.1

data/lib/parser/ruby-next/lexer.rl

@@ -96,7 +96,7 @@ class Next
   attr_accessor :static_env
   attr_accessor :force_utf32
 
-  attr_accessor :cond, :cmdarg, :in_kwarg, :context, :command_start
+  attr_accessor :cond, :cmdarg, :context, :command_start
 
   attr_accessor :tokens, :comments
 
@@ -519,7 +519,8 @@ class Next
   c_nl_zlen  = c_nl | zlen;
   c_line     = any - c_nl_zlen;
 
-  c_unicode  = c_any - 0x00..0x7f;
+  c_ascii    = 0x00..0x7f;
+  c_unicode  = c_any - c_ascii;
   c_upper    = [A-Z];
   c_lower    = [a-z_]  | c_unicode;
   c_alpha    = c_lower | c_upper;
@@ -706,6 +707,11 @@ class Next
 
   action unescape_char {
     codepoint = @source_pts[p - 1]
+
+    if @version >= 30 && (codepoint == 117 || codepoint == 85) # 'u' or 'U'
+      diagnostic :fatal, :invalid_escape
+    end
+
     if (@escape = ESCAPES[codepoint]).nil?
       @escape = encode_escape(@source_buffer.slice(p - 1))
     end
@@ -733,12 +739,14 @@ class Next
 
   maybe_escaped_char = (
         '\\' c_any      %unescape_char
+    |   '\\x' xdigit{1,2} % { @escape = encode_escape(tok(p - 2, p).to_i(16)) } %slash_c_char
     | ( c_any - [\\] )  %read_post_meta_or_ctrl_char
   );
 
   maybe_escaped_ctrl_char = ( # why?!
         '\\' c_any      %unescape_char %slash_c_char
     |   '?'             % { @escape = "\x7f" }
+    |   '\\x' xdigit{1,2} % { @escape = encode_escape(tok(p - 2, p).to_i(16)) } %slash_c_char
     | ( c_any - [\\?] ) %read_post_meta_or_ctrl_char %slash_c_char
   );
 
@@ -930,6 +938,10 @@ class Next
         #   b"
         # must be parsed as "ab"
         current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te)
+      elsif current_literal.regexp? && @version >= 31 && %w[c C m M].include?(escaped_char)
+        # Ruby >= 3.1 escapes \c- and \m chars, that's the only escape sequence
+        # supported by regexes so far, so it needs a separate branch.
+        current_literal.extend_string(@escape, @ts, @te)
       elsif current_literal.regexp?
         # Regular expressions should include escape sequences in their
         # escaped form. On the other hand, escaped newlines are removed (in cases like "\\C-\\\n\\M-x")
@@ -1402,7 +1414,7 @@ class Next
       ':'
       => { fhold; fgoto expr_beg; };
 
-      '%s' c_any
+      '%s' (c_ascii - [A-Za-z0-9])
       => {
         if version?(23)
           type, delimiter = tok[0..-2], tok[-1].chr
@@ -1431,6 +1443,18 @@ class Next
       => { emit(:tLABEL, tok(@ts, @te - 2), @ts, @te - 1)
            fhold; fnext expr_labelarg; fbreak; };
 
+      '...'
+      => {
+        if @version >= 31 && @context.in_argdef
+          emit(:tBDOT3, '...'.freeze)
+          # emit(:tNL, "\n".freeze, @te - 1, @te)
+          fnext expr_end; fbreak;
+        else
+          p -= 3;
+          fgoto expr_end;
+        end
+      };
+
       w_space_comment;
 
       c_any
@@ -1754,14 +1778,14 @@ class Next
       };
 
       # %<string>
-      '%' ( any - [A-Za-z] )
+      '%' ( c_ascii - [A-Za-z0-9] )
       => {
         type, delimiter = @source_buffer.slice(@ts).chr, tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts);
       };
 
       # %w(we are the people)
-      '%' [A-Za-z]+ c_any
+      '%' [A-Za-z] (c_ascii - [A-Za-z0-9])
       => {
         type, delimiter = tok[0..-2], tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts);
@@ -2035,19 +2059,38 @@ class Next
         fnext expr_beg; fbreak;
       };
 
-      '...'
+      '...' c_nl?
       => {
+        # Here we scan and conditionally emit "\n":
+        # + if it's there
+        #   + and emitted we do nothing
+        #   + and not emitted we return `p` to "\n" to process it on the next scan
+        # + if it's not there we do nothing
+        followed_by_nl = @te - 1 == @newline_s
+        nl_emitted = false
+        dots_te = followed_by_nl ? @te - 1 : @te
+
         if @version >= 30
           if @lambda_stack.any? && @lambda_stack.last + 1 == @paren_nest
             # To reject `->(...)` like `->...`
-            emit(:tDOT3)
+            emit(:tDOT3, '...'.freeze, @ts, dots_te)
           else
-            emit(:tBDOT3)
+            emit(:tBDOT3, '...'.freeze, @ts, dots_te)
+
+            if @version >= 31 && followed_by_nl && @context.in_argdef
+              emit(:tNL, @te - 1, @te)
+              nl_emitted = true
+            end
           end
         elsif @version >= 27
-          emit(:tBDOT3)
+          emit(:tBDOT3, '...'.freeze, @ts, dots_te)
         else
-          emit(:tDOT3)
+          emit(:tDOT3, '...'.freeze, @ts, dots_te)
+        end
+
+        if followed_by_nl && !nl_emitted
+          # return "\n" to process it on the next scan
+          fhold;
         end
 
         fnext expr_beg; fbreak;
@@ -2119,7 +2162,7 @@ class Next
 
     w_newline
     => {
-      if @in_kwarg
+      if @context.in_kwarg
         fhold; fgoto expr_end;
       else
         fgoto line_begin;
@@ -2526,6 +2569,28 @@ class Next
         end
       };
 
+      c_space* '..'
+      => {
+        emit(:tNL, nil, @newline_s, @newline_s + 1)
+        if @version < 27
+          fhold; fnext line_begin; fbreak;
+        else
+          emit(:tBDOT2)
+          fnext expr_beg; fbreak;
+        end
+      };
+
+      c_space* '...'
+      => {
+        emit(:tNL, nil, @newline_s, @newline_s + 1)
+        if @version < 27
+          fhold; fnext line_begin; fbreak;
+        else
+          emit(:tBDOT3)
+          fnext expr_beg; fbreak;
+        end
+      };
+
       c_space* %{ tm = p } ('.' | '&.')
       => { p = tm - 1; fgoto expr_end; };
 

data/lib/parser/ruby-next/meta.rb

@@ -5,7 +5,7 @@ require "parser/meta"
 module Parser
   # Parser metadata
   module Meta
-    NEXT_NODE_TYPES = (NODE_TYPES + %i[meth_ref ipair]).to_set.freeze
+    NEXT_NODE_TYPES = (NODE_TYPES + %i[meth_ref]).to_set.freeze
 
     remove_const(:NODE_TYPES)
     const_set(:NODE_TYPES, NEXT_NODE_TYPES)

data/lib/parser/ruby-next/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parser
-  NEXT_VERSION = "3.0.1.0"
+  NEXT_VERSION = "3.1.1.1"
 end