c_lexer 2.6.3.0.0 → 2.6.4.0.0

data/ext/lexer/lexer.h

@@ -50,6 +50,8 @@ struct Lexer {
   VALUE escape;
 
   int cs_before_block_comment;
+
+  VALUE max_numparam_stack;
 };
 
 static void lexer_mark(void*);
@@ -229,6 +231,7 @@ VALUE tNEQ;
 VALUE tNL;
 VALUE tNMATCH;
 VALUE tNTH_REF;
+VALUE tNUMPARAM;
 VALUE tOP_ASGN;
 VALUE tOROP;
 VALUE tPERCENT;
@@ -268,6 +271,7 @@ VALUE tXSTRING_BEG;
 VALUE comment_klass;
 VALUE diagnostic_klass;
 VALUE range_klass;
+VALUE max_numparam_stack_klass;
 
 VALUE severity_error;
 VALUE fatal;
@@ -281,6 +285,8 @@ VALUE cvar_name;
 VALUE embedded_document;
 VALUE empty_numeric;
 VALUE escape_eof;
+VALUE heredoc_id_ends_with_nl;
+VALUE heredoc_id_has_newline;
 VALUE incomplete_escape;
 VALUE invalid_escape;
 VALUE invalid_escape_use;
@@ -288,22 +294,26 @@ VALUE invalid_hex_escape;
 VALUE invalid_octal;
 VALUE invalid_unicode_escape;
 VALUE ivar_name;
-VALUE heredoc_id_ends_with_nl;
-VALUE heredoc_id_has_newline;
+VALUE leading_zero_in_numparam;
 VALUE no_dot_digit_literal;
+VALUE numparam_outside_block;
+VALUE ordinary_param_defined;
 VALUE prefix;
 VALUE regexp_options;
 VALUE string_eof;
+VALUE too_large_numparam;
 VALUE trailing_in_number;
 VALUE unexpected;
 VALUE unexpected_percent_str;
 VALUE unicode_point_too_large;
+VALUE unterminated_heredoc_id;
 VALUE unterminated_unicode;
 
 VALUE empty_array;
 VALUE blank_string;
 VALUE newline;
 VALUE escaped_newline;
+VALUE slash_r;
 VALUE utf8_encoding;
 VALUE cr_then_anything_to_eol;
 VALUE crs_to_eol;

data/ext/lexer/lexer.rl

@@ -12,6 +12,7 @@
 #define GET_LEXER(self) Data_Get_Struct(self, Lexer, lexer)
 #define STATIC_ENV_DECLARED(name) \
   lexer->static_env != Qnil && RTEST(rb_funcall(lexer->static_env, rb_intern("declared?"), 1, name))
+#define NUMPARAM_MAX 100
 
 #include "stack_state/cmdarg.h"
 #include "stack_state/cond.h"
@@ -44,6 +45,7 @@ static VALUE lexer_alloc(VALUE klass)
   lexer->comments      = Qnil;
   lexer->encoding      = Qnil;
   lexer->escape        = Qnil;
+  lexer->max_numparam_stack = Qnil;
 
   ss_stack_init(&lexer->cond_stack);
   ss_stack_init(&lexer->cmdarg_stack);
@@ -66,6 +68,7 @@ static void lexer_mark(void *ptr)
   rb_gc_mark(lexer->comments);
   rb_gc_mark(lexer->encoding);
   rb_gc_mark(lexer->escape);
+  rb_gc_mark(lexer->max_numparam_stack);
 
   for (literal *lit = lexer->literal_stack.bottom; lit < lexer->literal_stack.top; lit++) {
     rb_gc_mark(lit->buffer);
@@ -157,6 +160,8 @@ static VALUE lexer_reset(int argc, VALUE *argv, VALUE self)
 
   lexer->cs_before_block_comment = lex_en_line_begin;
 
+  lexer->max_numparam_stack = rb_class_new_instance(0, NULL, max_numparam_stack_klass) ;
+
   return self;
 }
 
@@ -306,6 +311,18 @@ static VALUE lexer_set_in_kwarg(VALUE self, VALUE val)
   return val;
 }
 
+static VALUE lexer_max_numparam_stack(VALUE self)
+{
+  Lexer* lexer = GET_LEXER(self);
+  return lexer->max_numparam_stack;
+}
+
+static VALUE lexer_max_numparam(VALUE self)
+{
+  Lexer* lexer = GET_LEXER(self);
+  return rb_funcall(lexer->max_numparam_stack, rb_intern("top"), 0);
+}
+
 static VALUE lexer_get_dedent_level(VALUE self)
 {
   Lexer* lexer = GET_LEXER(self);
@@ -335,6 +352,7 @@ static VALUE lexer_advance(VALUE self)
   long ident_ts = 0, ident_te = 0;
   long numeric_s = 0;
   Data_Get_Struct(self, Lexer, lexer);
+  VALUE diag_msg;
 
   if (RARRAY_LEN(lexer->token_queue) > 0)
     return rb_ary_shift(lexer->token_queue);
@@ -788,9 +806,9 @@ void Init_lexer()
   init_symbol(tASSOC);
   init_symbol(tBACK_REF);
   init_symbol(tBACK_REF2);
+  init_symbol(tBANG);
   init_symbol(tBDOT2);
   init_symbol(tBDOT3);
-  init_symbol(tBANG);
   init_symbol(tCARET);
   init_symbol(tCHARACTER);
   init_symbol(tCMP);
@@ -841,6 +859,7 @@ void Init_lexer()
   init_symbol(tNL);
   init_symbol(tNMATCH);
   init_symbol(tNTH_REF);
+  init_symbol(tNUMPARAM);
   init_symbol(tOP_ASGN);
   init_symbol(tOROP);
   init_symbol(tPERCENT);
@@ -890,6 +909,8 @@ void Init_lexer()
   init_symbol(embedded_document);
   init_symbol(empty_numeric);
   init_symbol(escape_eof);
+  init_symbol(heredoc_id_ends_with_nl);
+  init_symbol(heredoc_id_has_newline);
   init_symbol(incomplete_escape);
   init_symbol(invalid_escape);
   init_symbol(invalid_escape_use);
@@ -897,16 +918,19 @@ void Init_lexer()
   init_symbol(invalid_octal);
   init_symbol(invalid_unicode_escape);
   init_symbol(ivar_name);
-  init_symbol(heredoc_id_ends_with_nl);
-  init_symbol(heredoc_id_has_newline);
+  init_symbol(leading_zero_in_numparam);
   init_symbol(no_dot_digit_literal);
+  init_symbol(numparam_outside_block);
+  init_symbol(ordinary_param_defined);
   init_symbol(prefix);
   init_symbol(regexp_options);
   init_symbol(string_eof);
+  init_symbol(too_large_numparam);
   init_symbol(trailing_in_number);
   init_symbol(unexpected);
   init_symbol(unexpected_percent_str);
   init_symbol(unicode_point_too_large);
+  init_symbol(unterminated_heredoc_id);
   init_symbol(unterminated_unicode);
 
   VALUE m_Parser = rb_define_module("Parser");
@@ -958,12 +982,17 @@ void Init_lexer()
   rb_define_method(c_Lexer, "source_buffer=", lexer_set_source_buffer, 1);
   rb_define_method(c_Lexer, "force_utf32=",   lexer_set_force_utf32,   1);
 
+  rb_define_method(c_Lexer, "max_numparam_stack", lexer_max_numparam_stack, 0);
+  rb_define_method(c_Lexer, "max_numparam",       lexer_max_numparam,    0);
+
   rb_define_attr(c_Lexer, "context", 1, 1);
 
   VALUE m_Source   = rb_const_get(m_Parser, rb_intern("Source"));
   comment_klass    = rb_const_get(m_Source, rb_intern("Comment"));
   diagnostic_klass = rb_const_get(m_Parser, rb_intern("Diagnostic"));
   range_klass      = rb_const_get(m_Source, rb_intern("Range"));
+  VALUE lexer_class = rb_const_get(m_Parser, rb_intern("Lexer"));
+  max_numparam_stack_klass = rb_const_get(lexer_class, rb_intern("MaxNumparamStack"));
 
   empty_array = rb_obj_freeze(rb_ary_new2(0));
   rb_gc_register_address(&empty_array);
@@ -973,6 +1002,8 @@ void Init_lexer()
   rb_gc_register_address(&newline);
   escaped_newline = rb_obj_freeze(rb_str_new2("\\\n"));
   rb_gc_register_address(&escaped_newline);
+  slash_r = rb_obj_freeze(rb_str_new2("\r"));
+  rb_gc_register_address(&slash_r);
 
   if (rb_const_defined(rb_cObject, rb_intern("Encoding"))) {
     VALUE encoding = rb_const_get(rb_cObject, rb_intern("Encoding"));
@@ -1170,6 +1201,21 @@ void Init_lexer()
                empty_array);
   }
 
+  action read_post_meta_or_ctrl_char {
+    VALUE codepoint = rb_funcall(lexer->source_buffer, rb_intern("slice"), 1, INT2NUM(p - 1));
+    lexer->escape = rb_funcall(codepoint, rb_intern("chr"), 0);
+    int codepoint_i = FIX2INT(rb_funcall(codepoint, rb_intern("ord"), 0));
+
+    if (
+      lexer->version >= 27 && (
+        (codepoint_i >= 0 && codepoint_i <= 8) ||
+        (codepoint_i >= 14 && codepoint_i <= 31)
+      )
+    ) {
+      diagnostic(lexer, fatal, invalid_escape, Qnil, range(lexer, ts, te), empty_array);
+    }
+  }
+
   action slash_c_char {
     char c = *RSTRING_PTR(lexer->escape) & 0x9f;
     lexer->escape = rb_str_new(&c, 1);
@@ -1184,13 +1230,13 @@ void Init_lexer()
 
   maybe_escaped_char = (
         '\\' c_any      %unescape_char
-    | ( c_any - [\\] )  % { lexer->escape = rb_str_substr(lexer->source, p - 1, 1); }
+    | ( c_any - [\\] )  %read_post_meta_or_ctrl_char
   );
 
   maybe_escaped_ctrl_char = (
         '\\' c_any      %unescape_char %slash_c_char
     |   '?'             % { lexer->escape = rb_str_new2("\x7f"); }
-    | ( c_any - [\\?] ) % { lexer->escape = rb_str_substr(lexer->source, p - 1, 1); } %slash_c_char
+    | ( c_any - [\\?] ) %read_post_meta_or_ctrl_char %slash_c_char
   );
 
   escape = (
@@ -1636,6 +1682,47 @@ void Init_lexer()
         fnext *stack[--top]; fbreak;
       };
 
+      '@' [0-9]+
+      => {
+        VALUE token = tok(lexer, ts, te);
+
+        if (lexer->version < 27) {
+          VALUE hash = rb_hash_new();
+          rb_hash_aset(hash, ID2SYM(rb_intern("name")), token);
+          diagnostic(lexer, severity_error, ivar_name, hash, range(lexer, ts, te), empty_array);
+        }
+
+        VALUE value = rb_funcall(token, rb_intern("[]"), 1, rb_range_new(INT2NUM(1), INT2NUM(-1), 0));
+        VALUE int_value = rb_funcall(value, rb_intern("to_i"), 0);
+
+        if (*RSTRING_PTR(value) == '0') {
+          diagnostic(lexer, severity_error, leading_zero_in_numparam, Qnil, range(lexer, ts, te), empty_array);
+        }
+
+        if (FIX2INT(int_value) > NUMPARAM_MAX) {
+          diagnostic(lexer, severity_error, too_large_numparam, Qnil, range(lexer, ts, te), empty_array);
+        }
+
+        VALUE context = rb_iv_get(self, "@context");
+        int in_block  = RTEST(rb_funcall(context, rb_intern("in_block?"), 0));
+        int in_lambda = RTEST(rb_funcall(context, rb_intern("in_lambda?"), 0));
+
+        if (!in_block && !in_lambda) {
+          diagnostic(lexer, severity_error, numparam_outside_block, Qnil, range(lexer, ts, te), empty_array);
+        }
+
+        VALUE max_numparam_stack = lexer->max_numparam_stack;
+        int can_have_numparams = RTEST(rb_funcall(max_numparam_stack, rb_intern("can_have_numparams?"), 0));
+        if (!can_have_numparams) {
+          diagnostic(lexer, severity_error, ordinary_param_defined, Qnil, range(lexer, ts, te), empty_array);
+        }
+
+        rb_funcall(max_numparam_stack, rb_intern("register"), 1, int_value);
+
+        emit_token(lexer, tNUMPARAM, tok(lexer, ts + 1, te), ts, te);
+        fnext *stack[--top]; fbreak;
+      };
+
       instance_var_v => {
         VALUE str = tok(lexer, ts, te);
 
@@ -1960,7 +2047,15 @@ void Init_lexer()
 
         VALUE delimiter = tok(lexer, rng_s, rng_e);
 
-        if (lexer->version >= 24) {
+        if (lexer->version >= 27) {
+          int newlines_count = NUM2INT(rb_funcall(delimiter, rb_intern("count"), 1, newline));
+          int slash_r_count = NUM2INT(rb_funcall(delimiter, rb_intern("count"), 1, slash_r));
+
+          if (newlines_count > 0 || slash_r_count > 0) {
+            diagnostic(lexer, severity_error, unterminated_heredoc_id, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
+          }
+        } else if (lexer->version >= 24) {
           if (NUM2INT(rb_funcall(delimiter, rb_intern("count"), 1, newline)) > 0) {
             if (str_end_with_p(delimiter, "\n")) {
               diagnostic(lexer, warning, heredoc_id_ends_with_nl, Qnil,
@@ -1985,6 +2080,21 @@ void Init_lexer()
         }
       };
 
+      # Escaped unterminated heredoc start
+      # <<'END  | <<"END  | <<`END  |
+      # <<-'END | <<-"END | <<-`END |
+      # <<~'END | <<~"END | <<~`END
+      #
+      # If the heredoc is terminated the rule above should handle it
+      '<<' [~\-]?
+        ('"' (any - c_nl - '"')*
+        |"'" (any - c_nl - "'")*
+        |"`" (any - c_nl - "`")
+        )
+      => {
+        diagnostic(lexer, severity_error, unterminated_heredoc_id, Qnil, range(lexer, ts, ts + 1), empty_array);
+      };
+
       ':' ('&&' | '||') => {
         fhold; fhold;
         emit_token(lexer, tSYMBEG, tok(lexer, ts, ts + 1), ts, ts + 1);
@@ -2015,6 +2125,23 @@ void Init_lexer()
         fnext expr_end; fbreak;
       };
 
+      ':' ( '@'  %{ tm = p - 1; diag_msg = ivar_name; }
+          | '@@' %{ tm = p - 2; diag_msg = cvar_name; }
+          ) [0-9]*
+      => {
+        if (lexer->version >= 27) {
+          VALUE hash = rb_hash_new();
+          rb_hash_aset(hash, ID2SYM(rb_intern("name")), tok(lexer, tm, te));
+          diagnostic(lexer, severity_error, diag_msg, hash, range(lexer, tm, te), empty_array);
+        } else {
+          emit_token(lexer, tCOLON, tok(lexer, ts, ts + 1), ts, ts + 1);
+          p = ts;
+        }
+
+        fnext expr_end; fbreak;
+      };
+
+
       '?' ( e_bs ( escape - ( '\u{' (xdigit+ [ \t]+)+ xdigit+ '}' ))
           | (c_any - c_space_nl - e_bs) % { lexer->escape = Qnil; }
           ) => {

data/lib/c_lexer.rb

@@ -74,6 +74,7 @@ module Parser
       @lexer = ::Parser::CLexer.new(version)
       @lexer.diagnostics = @diagnostics
       @lexer.static_env  = @static_env
+      @lexer.context     = @context
     end
   end
 
@@ -83,6 +84,7 @@ module Parser
       @lexer = ::Parser::CLexer.new(version)
       @lexer.diagnostics = @diagnostics
       @lexer.static_env  = @static_env
+      @lexer.context     = @context
     end
   end
 
@@ -92,6 +94,7 @@ module Parser
       @lexer = ::Parser::CLexer.new(version)
       @lexer.diagnostics = @diagnostics
       @lexer.static_env  = @static_env
+      @lexer.context     = @context
     end
   end
 end

data/lib/c_lexer/version.rb

@@ -1,3 +1,3 @@
 module CLexer
-  VERSION = '2.6.3.0.0'
+  VERSION = '2.6.4.0.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: c_lexer
 version: !ruby/object:Gem::Version
-  version: 2.6.3.0.0
+  version: 2.6.4.0.0
 platform: ruby
 authors:
 - Ilya Bylich
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.6.3.0
+        version: 2.6.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.6.3.0
+        version: 2.6.4.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement