natalie_parser 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/Dockerfile +1 -1
- data/Gemfile +1 -1
- data/README.md +1 -0
- data/Rakefile +2 -1
- data/ext/natalie_parser/mri_creator.hpp +5 -5
- data/ext/natalie_parser/natalie_parser.cpp +5 -4
- data/include/natalie_parser/lexer.hpp +17 -4
- data/include/natalie_parser/node/begin_rescue_node.hpp +4 -5
- data/include/natalie_parser/node/for_node.hpp +1 -16
- data/include/natalie_parser/parser.hpp +15 -5
- data/include/natalie_parser/token.hpp +21 -0
- data/lib/natalie_parser/version.rb +1 -1
- data/src/lexer/interpolated_string_lexer.cpp +1 -1
- data/src/lexer/regexp_lexer.cpp +1 -1
- data/src/lexer/word_array_lexer.cpp +1 -1
- data/src/lexer.cpp +189 -248
- data/src/node/begin_rescue_node.cpp +4 -3
- data/src/node/for_node.cpp +13 -0
- data/src/parser.cpp +143 -106
- metadata +3 -2
data/src/lexer.cpp
CHANGED
@@ -80,7 +80,47 @@ Token Lexer::next_token() {
|
|
80
80
|
m_whitespace_precedes = skip_whitespace();
|
81
81
|
m_token_line = m_cursor_line;
|
82
82
|
m_token_column = m_cursor_column;
|
83
|
-
|
83
|
+
Token token = build_next_token();
|
84
|
+
switch (token.type()) {
|
85
|
+
case Token::Type::AliasKeyword:
|
86
|
+
m_remaining_method_names = 2;
|
87
|
+
break;
|
88
|
+
case Token::Type::ConstantResolution:
|
89
|
+
case Token::Type::DefKeyword:
|
90
|
+
m_remaining_method_names = 1;
|
91
|
+
m_allow_assignment_method = true;
|
92
|
+
break;
|
93
|
+
case Token::Type::Dot:
|
94
|
+
m_remaining_method_names = 1;
|
95
|
+
break;
|
96
|
+
case Token::Type::UndefKeyword:
|
97
|
+
m_remaining_method_names = std::numeric_limits<size_t>::max();
|
98
|
+
m_method_name_separator = Token::Type::Comma;
|
99
|
+
break;
|
100
|
+
default:
|
101
|
+
if (m_method_name_separator != Token::Type::Invalid) {
|
102
|
+
if (m_last_method_name) {
|
103
|
+
m_last_method_name = {};
|
104
|
+
if (token.type() != m_method_name_separator) {
|
105
|
+
m_remaining_method_names = 0;
|
106
|
+
m_method_name_separator = Token::Type::Invalid;
|
107
|
+
}
|
108
|
+
} else {
|
109
|
+
m_last_method_name = token;
|
110
|
+
}
|
111
|
+
} else if (m_remaining_method_names > 0) {
|
112
|
+
m_remaining_method_names--;
|
113
|
+
} else {
|
114
|
+
m_allow_assignment_method = false;
|
115
|
+
}
|
116
|
+
break;
|
117
|
+
}
|
118
|
+
return token;
|
119
|
+
}
|
120
|
+
|
121
|
+
bool is_name_start_char(char c) {
|
122
|
+
if (!c) return false;
|
123
|
+
return (c >= 'a' && c <= 'z') || c == '_' || (unsigned int)c >= 128;
|
84
124
|
}
|
85
125
|
|
86
126
|
bool is_identifier_char(char c) {
|
@@ -204,10 +244,10 @@ Token Lexer::build_next_token() {
|
|
204
244
|
advance();
|
205
245
|
return Token { Token::Type::PlusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
206
246
|
case '@':
|
207
|
-
if (
|
247
|
+
if (m_remaining_method_names > 0) {
|
208
248
|
advance();
|
209
249
|
SharedPtr<String> lit = new String("+@");
|
210
|
-
return Token { Token::Type::
|
250
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
211
251
|
} else {
|
212
252
|
return Token { Token::Type::Plus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
213
253
|
}
|
@@ -224,10 +264,10 @@ Token Lexer::build_next_token() {
|
|
224
264
|
advance();
|
225
265
|
return Token { Token::Type::MinusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
226
266
|
case '@':
|
227
|
-
if (
|
267
|
+
if (m_remaining_method_names > 0) {
|
228
268
|
advance();
|
229
269
|
SharedPtr<String> lit = new String("-@");
|
230
|
-
return Token { Token::Type::
|
270
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
231
271
|
} else {
|
232
272
|
return Token { Token::Type::Minus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
233
273
|
}
|
@@ -256,17 +296,20 @@ Token Lexer::build_next_token() {
|
|
256
296
|
advance();
|
257
297
|
if (!m_last_token)
|
258
298
|
return consume_regexp('/', '/');
|
299
|
+
if (m_remaining_method_names > 0)
|
300
|
+
return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
259
301
|
switch (m_last_token.type()) {
|
260
302
|
case Token::Type::Comma:
|
261
303
|
case Token::Type::Doc:
|
304
|
+
case Token::Type::Equal:
|
262
305
|
case Token::Type::LBracket:
|
263
306
|
case Token::Type::LCurlyBrace:
|
264
307
|
case Token::Type::LParen:
|
265
308
|
case Token::Type::Match:
|
266
309
|
case Token::Type::Newline:
|
310
|
+
case Token::Type::Not:
|
311
|
+
case Token::Type::Pipe:
|
267
312
|
return consume_regexp('/', '/');
|
268
|
-
case Token::Type::DefKeyword:
|
269
|
-
return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
270
313
|
default: {
|
271
314
|
switch (current_char()) {
|
272
315
|
case ' ':
|
@@ -295,216 +338,26 @@ Token Lexer::build_next_token() {
|
|
295
338
|
advance();
|
296
339
|
return Token { Token::Type::PercentEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
297
340
|
case 'q':
|
298
|
-
|
299
|
-
case '[':
|
300
|
-
advance(2);
|
301
|
-
return consume_single_quoted_string('[', ']');
|
302
|
-
case '{':
|
303
|
-
advance(2);
|
304
|
-
return consume_single_quoted_string('{', '}');
|
305
|
-
case '<':
|
306
|
-
advance(2);
|
307
|
-
return consume_single_quoted_string('<', '>');
|
308
|
-
case '(':
|
309
|
-
advance(2);
|
310
|
-
return consume_single_quoted_string('(', ')');
|
311
|
-
default: {
|
312
|
-
char c = peek();
|
313
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
314
|
-
advance(2);
|
315
|
-
return consume_single_quoted_string(c, c);
|
316
|
-
} else {
|
317
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
318
|
-
}
|
319
|
-
}
|
320
|
-
}
|
341
|
+
return consume_percent_string(&Lexer::consume_single_quoted_string);
|
321
342
|
case 'Q':
|
322
|
-
|
323
|
-
case '[':
|
324
|
-
advance(2);
|
325
|
-
return consume_double_quoted_string('[', ']');
|
326
|
-
case '{':
|
327
|
-
advance(2);
|
328
|
-
return consume_double_quoted_string('{', '}');
|
329
|
-
case '<':
|
330
|
-
advance(2);
|
331
|
-
return consume_double_quoted_string('<', '>');
|
332
|
-
case '(':
|
333
|
-
advance(2);
|
334
|
-
return consume_double_quoted_string('(', ')');
|
335
|
-
default: {
|
336
|
-
char c = peek();
|
337
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
338
|
-
advance(2);
|
339
|
-
return consume_double_quoted_string(c, c);
|
340
|
-
} else {
|
341
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
342
|
-
}
|
343
|
-
}
|
344
|
-
}
|
343
|
+
return consume_percent_string(&Lexer::consume_interpolated_string);
|
345
344
|
case 'r':
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
return consume_regexp('[', ']');
|
350
|
-
case '{':
|
351
|
-
advance(2);
|
352
|
-
return consume_regexp('{', '}');
|
353
|
-
case '(':
|
354
|
-
advance(2);
|
355
|
-
return consume_regexp('(', ')');
|
356
|
-
case '<':
|
357
|
-
advance(2);
|
358
|
-
return consume_regexp('<', '>');
|
359
|
-
default: {
|
360
|
-
char c = peek();
|
361
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
362
|
-
advance(2);
|
363
|
-
return consume_regexp(c, c);
|
364
|
-
} else {
|
365
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
366
|
-
}
|
367
|
-
}
|
368
|
-
}
|
345
|
+
return consume_percent_string(&Lexer::consume_regexp);
|
346
|
+
case 's':
|
347
|
+
return consume_percent_string(&Lexer::consume_percent_symbol);
|
369
348
|
case 'x':
|
370
|
-
|
371
|
-
case '/': {
|
372
|
-
advance(2);
|
373
|
-
return consume_double_quoted_string('/', '/', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
374
|
-
}
|
375
|
-
case '[': {
|
376
|
-
advance(2);
|
377
|
-
return consume_double_quoted_string('[', ']', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
378
|
-
}
|
379
|
-
case '{': {
|
380
|
-
advance(2);
|
381
|
-
return consume_double_quoted_string('{', '}', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
382
|
-
}
|
383
|
-
case '(': {
|
384
|
-
advance(2);
|
385
|
-
return consume_double_quoted_string('(', ')', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
386
|
-
}
|
387
|
-
default:
|
388
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
389
|
-
}
|
349
|
+
return consume_percent_string(&Lexer::consume_interpolated_shell);
|
390
350
|
case 'w':
|
391
|
-
|
392
|
-
case '/':
|
393
|
-
case '|': {
|
394
|
-
char c = next();
|
395
|
-
advance();
|
396
|
-
return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerW);
|
397
|
-
}
|
398
|
-
case '[':
|
399
|
-
advance(2);
|
400
|
-
return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerW);
|
401
|
-
case '{':
|
402
|
-
advance(2);
|
403
|
-
return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerW);
|
404
|
-
case '<':
|
405
|
-
advance(2);
|
406
|
-
return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerW);
|
407
|
-
case '(':
|
408
|
-
advance(2);
|
409
|
-
return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerW);
|
410
|
-
default:
|
411
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
412
|
-
}
|
351
|
+
return consume_percent_string(&Lexer::consume_percent_lower_w);
|
413
352
|
case 'W':
|
414
|
-
|
415
|
-
case '/':
|
416
|
-
case '|': {
|
417
|
-
char c = next();
|
418
|
-
advance();
|
419
|
-
return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperW);
|
420
|
-
}
|
421
|
-
case '[':
|
422
|
-
advance(2);
|
423
|
-
return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperW);
|
424
|
-
case '{':
|
425
|
-
advance(2);
|
426
|
-
return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperW);
|
427
|
-
case '<':
|
428
|
-
advance(2);
|
429
|
-
return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperW);
|
430
|
-
case '(':
|
431
|
-
advance(2);
|
432
|
-
return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperW);
|
433
|
-
default:
|
434
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
435
|
-
}
|
353
|
+
return consume_percent_string(&Lexer::consume_percent_upper_w);
|
436
354
|
case 'i':
|
437
|
-
|
438
|
-
case '|':
|
439
|
-
case '/': {
|
440
|
-
char c = next();
|
441
|
-
advance();
|
442
|
-
return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerI);
|
443
|
-
}
|
444
|
-
case '[':
|
445
|
-
advance(2);
|
446
|
-
return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerI);
|
447
|
-
case '{':
|
448
|
-
advance(2);
|
449
|
-
return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerI);
|
450
|
-
case '<':
|
451
|
-
advance(2);
|
452
|
-
return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerI);
|
453
|
-
case '(':
|
454
|
-
advance(2);
|
455
|
-
return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerI);
|
456
|
-
default:
|
457
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
458
|
-
}
|
355
|
+
return consume_percent_string(&Lexer::consume_percent_lower_i);
|
459
356
|
case 'I':
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
char c = next();
|
464
|
-
advance();
|
465
|
-
return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperI);
|
466
|
-
}
|
467
|
-
case '[':
|
468
|
-
advance(2);
|
469
|
-
return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperI);
|
470
|
-
case '{':
|
471
|
-
advance(2);
|
472
|
-
return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperI);
|
473
|
-
case '<':
|
474
|
-
advance(2);
|
475
|
-
return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperI);
|
476
|
-
case '(':
|
477
|
-
advance(2);
|
478
|
-
return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperI);
|
479
|
-
default:
|
480
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
481
|
-
}
|
482
|
-
case '[':
|
483
|
-
advance();
|
484
|
-
return consume_double_quoted_string('[', ']');
|
485
|
-
case '{':
|
486
|
-
advance();
|
487
|
-
return consume_double_quoted_string('{', '}');
|
488
|
-
case '<':
|
489
|
-
advance();
|
490
|
-
return consume_double_quoted_string('<', '>');
|
491
|
-
case '(':
|
492
|
-
if (m_last_token.type() == Token::Type::DefKeyword || m_last_token.type() == Token::Type::Dot) {
|
493
|
-
// It's a trap! This looks like a %(string) but it's a method def/call!
|
494
|
-
break;
|
495
|
-
}
|
496
|
-
advance();
|
497
|
-
return consume_double_quoted_string('(', ')');
|
498
|
-
default: {
|
499
|
-
auto c = current_char();
|
500
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
501
|
-
advance();
|
502
|
-
return consume_double_quoted_string(c, c);
|
503
|
-
}
|
504
|
-
break;
|
505
|
-
}
|
357
|
+
return consume_percent_string(&Lexer::consume_percent_upper_i);
|
358
|
+
default:
|
359
|
+
return consume_percent_string(&Lexer::consume_interpolated_string, false);
|
506
360
|
}
|
507
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
508
361
|
case '!':
|
509
362
|
advance();
|
510
363
|
switch (current_char()) {
|
@@ -515,10 +368,10 @@ Token Lexer::build_next_token() {
|
|
515
368
|
advance();
|
516
369
|
return Token { Token::Type::NotMatch, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
517
370
|
case '@':
|
518
|
-
if (
|
371
|
+
if (m_remaining_method_names > 0) {
|
519
372
|
advance();
|
520
373
|
SharedPtr<String> lit = new String("!@");
|
521
|
-
return Token { Token::Type::
|
374
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
522
375
|
} else {
|
523
376
|
return Token { Token::Type::Not, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
524
377
|
}
|
@@ -653,10 +506,10 @@ Token Lexer::build_next_token() {
|
|
653
506
|
advance();
|
654
507
|
switch (current_char()) {
|
655
508
|
case '@':
|
656
|
-
if (
|
509
|
+
if (m_remaining_method_names > 0) {
|
657
510
|
advance();
|
658
511
|
SharedPtr<String> lit = new String("~@");
|
659
|
-
return Token { Token::Type::
|
512
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
660
513
|
} else {
|
661
514
|
return Token { Token::Type::Tilde, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
662
515
|
}
|
@@ -665,7 +518,7 @@ Token Lexer::build_next_token() {
|
|
665
518
|
}
|
666
519
|
case '?': {
|
667
520
|
auto c = next();
|
668
|
-
if (isspace(c)) {
|
521
|
+
if (isspace(c) || c == 0) {
|
669
522
|
m_open_ternary = true;
|
670
523
|
return Token { Token::Type::TernaryQuestion, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
671
524
|
} else {
|
@@ -695,7 +548,7 @@ Token Lexer::build_next_token() {
|
|
695
548
|
advance();
|
696
549
|
auto string = consume_single_quoted_string('\'', '\'');
|
697
550
|
return Token { Token::Type::Symbol, string.literal(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
698
|
-
} else if (isspace(c)) {
|
551
|
+
} else if (isspace(c) || c == 0) {
|
699
552
|
m_open_ternary = false;
|
700
553
|
auto token = Token { Token::Type::TernaryColon, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
701
554
|
return token;
|
@@ -793,13 +646,18 @@ Token Lexer::build_next_token() {
|
|
793
646
|
return Token { Token::Type::Comma, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
794
647
|
case '"':
|
795
648
|
advance();
|
796
|
-
return
|
649
|
+
return consume_interpolated_string('"', '"');
|
797
650
|
case '\'':
|
798
651
|
advance();
|
799
652
|
return consume_single_quoted_string('\'', '\'');
|
800
653
|
case '`': {
|
801
654
|
advance();
|
802
|
-
|
655
|
+
if (m_remaining_method_names > 0) {
|
656
|
+
SharedPtr<String> lit = new String("`");
|
657
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
658
|
+
} else {
|
659
|
+
return consume_interpolated_shell('`', '`');
|
660
|
+
}
|
803
661
|
}
|
804
662
|
case '#':
|
805
663
|
if (token_is_first_on_line()) {
|
@@ -862,14 +720,14 @@ Token Lexer::build_next_token() {
|
|
862
720
|
|
863
721
|
Token keyword_token;
|
864
722
|
|
865
|
-
if (!m_last_token.is_dot() && match(4, "self")) {
|
866
|
-
if (current_char() == '.')
|
723
|
+
if (!m_last_token.is_dot() && !m_last_token.is_constant_resolution() && match(4, "self")) {
|
724
|
+
if (current_char() == '.' || (current_char() == ':' && peek() == ':'))
|
867
725
|
keyword_token = { Token::Type::SelfKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
868
726
|
else
|
869
727
|
rewind(4);
|
870
728
|
}
|
871
729
|
|
872
|
-
if (
|
730
|
+
if (m_remaining_method_names == 0) {
|
873
731
|
if (match(12, "__ENCODING__"))
|
874
732
|
keyword_token = { Token::Type::ENCODINGKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
875
733
|
else if (match(8, "__LINE__"))
|
@@ -964,10 +822,10 @@ Token Lexer::build_next_token() {
|
|
964
822
|
}
|
965
823
|
|
966
824
|
auto c = current_char();
|
967
|
-
if ((c
|
968
|
-
return
|
825
|
+
if (is_name_start_char(c)) {
|
826
|
+
return consume_bare_name_or_constant(Token::Type::BareName);
|
969
827
|
} else if (c >= 'A' && c <= 'Z') {
|
970
|
-
return
|
828
|
+
return consume_bare_name_or_constant(Token::Type::Constant);
|
971
829
|
} else {
|
972
830
|
auto buf = consume_non_whitespace();
|
973
831
|
auto token = Token { Token::Type::Invalid, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
@@ -1097,45 +955,51 @@ Token Lexer::consume_symbol() {
|
|
1097
955
|
return Token { Token::Type::Symbol, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1098
956
|
}
|
1099
957
|
|
1100
|
-
|
958
|
+
SharedPtr<String> Lexer::consume_word() {
|
1101
959
|
char c = current_char();
|
1102
960
|
SharedPtr<String> buf = new String("");
|
1103
961
|
do {
|
1104
962
|
buf->append_char(c);
|
1105
963
|
c = next();
|
1106
964
|
} while (is_identifier_char(c));
|
965
|
+
return buf;
|
966
|
+
}
|
967
|
+
|
968
|
+
Token Lexer::consume_word(Token::Type type) {
|
969
|
+
return Token { type, consume_word(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
970
|
+
}
|
971
|
+
|
972
|
+
Token Lexer::consume_bare_name_or_constant(Token::Type type) {
|
973
|
+
auto buf = consume_word();
|
974
|
+
auto c = current_char();
|
1107
975
|
switch (c) {
|
1108
976
|
case '?':
|
1109
977
|
case '!':
|
978
|
+
if (peek() == ':' && m_last_token.can_precede_symbol_key()) {
|
979
|
+
advance();
|
980
|
+
type = Token::Type::SymbolKey;
|
981
|
+
}
|
1110
982
|
advance();
|
1111
983
|
buf->append_char(c);
|
1112
984
|
break;
|
985
|
+
case '=':
|
986
|
+
if (m_allow_assignment_method || (!m_last_token.is_dot() && m_remaining_method_names > 0)) {
|
987
|
+
advance();
|
988
|
+
buf->append_char(c);
|
989
|
+
}
|
990
|
+
break;
|
991
|
+
case ':':
|
992
|
+
if (peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
993
|
+
advance();
|
994
|
+
type = Token::Type::SymbolKey;
|
995
|
+
}
|
996
|
+
break;
|
1113
997
|
default:
|
1114
998
|
break;
|
1115
999
|
}
|
1116
1000
|
return Token { type, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1117
1001
|
}
|
1118
1002
|
|
1119
|
-
Token Lexer::consume_bare_name() {
|
1120
|
-
auto token = consume_word(Token::Type::BareName);
|
1121
|
-
auto c = current_char();
|
1122
|
-
if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
1123
|
-
advance();
|
1124
|
-
token.set_type(Token::Type::SymbolKey);
|
1125
|
-
}
|
1126
|
-
return token;
|
1127
|
-
}
|
1128
|
-
|
1129
|
-
Token Lexer::consume_constant() {
|
1130
|
-
auto token = consume_word(Token::Type::Constant);
|
1131
|
-
auto c = current_char();
|
1132
|
-
if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
1133
|
-
advance();
|
1134
|
-
token.set_type(Token::Type::SymbolKey);
|
1135
|
-
}
|
1136
|
-
return token;
|
1137
|
-
}
|
1138
|
-
|
1139
1003
|
Token Lexer::consume_global_variable() {
|
1140
1004
|
switch (peek()) {
|
1141
1005
|
case '?':
|
@@ -1157,7 +1021,6 @@ Token Lexer::consume_global_variable() {
|
|
1157
1021
|
case '.':
|
1158
1022
|
case ',':
|
1159
1023
|
case ':':
|
1160
|
-
case '_':
|
1161
1024
|
case '~': {
|
1162
1025
|
advance();
|
1163
1026
|
SharedPtr<String> buf = new String("$");
|
@@ -1281,7 +1144,7 @@ Token Lexer::consume_heredoc() {
|
|
1281
1144
|
}
|
1282
1145
|
advance();
|
1283
1146
|
} else {
|
1284
|
-
heredoc_name =
|
1147
|
+
heredoc_name = *consume_word();
|
1285
1148
|
}
|
1286
1149
|
|
1287
1150
|
SharedPtr<String> doc = new String("");
|
@@ -1425,7 +1288,26 @@ Token Lexer::consume_numeric() {
|
|
1425
1288
|
break;
|
1426
1289
|
}
|
1427
1290
|
default:
|
1428
|
-
|
1291
|
+
char c = peek();
|
1292
|
+
|
1293
|
+
if (isdigit(c)) {
|
1294
|
+
// bare octal case, e.g. 0777.
|
1295
|
+
// If starts with a 0 but next number is not 0..7 then that's an error.
|
1296
|
+
if (!(c >= '0' && c <= '7'))
|
1297
|
+
return Token { Token::Type::Invalid, c, m_file, m_cursor_line,
|
1298
|
+
m_cursor_column, m_whitespace_precedes };
|
1299
|
+
chars->append_char(c);
|
1300
|
+
c = next();
|
1301
|
+
do {
|
1302
|
+
chars->append_char(c);
|
1303
|
+
c = next();
|
1304
|
+
if (c == '_')
|
1305
|
+
c = next();
|
1306
|
+
} while (c >= '0' && c <= '7');
|
1307
|
+
token = chars_to_fixnum_or_bignum_token(chars, 8, 1);
|
1308
|
+
} else {
|
1309
|
+
token = consume_decimal_digits_and_build_token();
|
1310
|
+
}
|
1429
1311
|
}
|
1430
1312
|
} else {
|
1431
1313
|
token = consume_decimal_digits_and_build_token();
|
@@ -1677,7 +1559,7 @@ Token Lexer::consume_single_quoted_string(char start_char, char stop_char) {
|
|
1677
1559
|
SharedPtr<String> buf = new String("");
|
1678
1560
|
char c = current_char();
|
1679
1561
|
while (c) {
|
1680
|
-
if (c == '\\') {
|
1562
|
+
if (c == '\\' && stop_char != '\\') {
|
1681
1563
|
c = next();
|
1682
1564
|
if (c == stop_char || c == '\\') {
|
1683
1565
|
buf->append_char(c);
|
@@ -1724,6 +1606,65 @@ Token Lexer::consume_regexp(char start_char, char stop_char) {
|
|
1724
1606
|
return Token { Token::Type::InterpolatedRegexpBegin, start_char, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1725
1607
|
}
|
1726
1608
|
|
1609
|
+
Token Lexer::consume_percent_symbol(char start_char, char stop_char) {
|
1610
|
+
Token token = consume_single_quoted_string(start_char, stop_char);
|
1611
|
+
token.set_type(Token::Type::Symbol);
|
1612
|
+
return token;
|
1613
|
+
}
|
1614
|
+
|
1615
|
+
Token Lexer::consume_interpolated_string(char start_char, char stop_char) {
|
1616
|
+
return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedStringBegin, Token::Type::InterpolatedStringEnd);
|
1617
|
+
}
|
1618
|
+
|
1619
|
+
Token Lexer::consume_interpolated_shell(char start_char, char stop_char) {
|
1620
|
+
return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
1621
|
+
}
|
1622
|
+
|
1623
|
+
Token Lexer::consume_percent_lower_w(char start_char, char stop_char) {
|
1624
|
+
return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerW);
|
1625
|
+
}
|
1626
|
+
|
1627
|
+
Token Lexer::consume_percent_upper_w(char start_char, char stop_char) {
|
1628
|
+
return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperW);
|
1629
|
+
}
|
1630
|
+
|
1631
|
+
Token Lexer::consume_percent_lower_i(char start_char, char stop_char) {
|
1632
|
+
return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerI);
|
1633
|
+
}
|
1634
|
+
|
1635
|
+
Token Lexer::consume_percent_upper_i(char start_char, char stop_char) {
|
1636
|
+
return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperI);
|
1637
|
+
}
|
1638
|
+
|
1639
|
+
Token Lexer::consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered) {
|
1640
|
+
if (m_remaining_method_names > 0) {
|
1641
|
+
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1642
|
+
}
|
1643
|
+
char c = is_lettered ? peek() : current_char();
|
1644
|
+
size_t bytes = is_lettered ? 2 : 1;
|
1645
|
+
switch (c) {
|
1646
|
+
case '[':
|
1647
|
+
advance(bytes);
|
1648
|
+
return (this->*consumer)('[', ']');
|
1649
|
+
case '{':
|
1650
|
+
advance(bytes);
|
1651
|
+
return (this->*consumer)('{', '}');
|
1652
|
+
case '<':
|
1653
|
+
advance(bytes);
|
1654
|
+
return (this->*consumer)('<', '>');
|
1655
|
+
case '(':
|
1656
|
+
advance(bytes);
|
1657
|
+
return (this->*consumer)('(', ')');
|
1658
|
+
default:
|
1659
|
+
if (char_can_be_string_or_regexp_delimiter(c)) {
|
1660
|
+
advance(bytes);
|
1661
|
+
return (this->*consumer)(c, c);
|
1662
|
+
} else {
|
1663
|
+
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1664
|
+
}
|
1665
|
+
}
|
1666
|
+
}
|
1667
|
+
|
1727
1668
|
SharedPtr<String> Lexer::consume_non_whitespace() {
|
1728
1669
|
char c = current_char();
|
1729
1670
|
SharedPtr<String> buf = new String("");
|
@@ -1,14 +1,15 @@
|
|
1
1
|
#include "natalie_parser/node/begin_rescue_node.hpp"
|
2
2
|
#include "natalie_parser/node/array_node.hpp"
|
3
3
|
#include "natalie_parser/node/assignment_node.hpp"
|
4
|
+
#include "natalie_parser/node/identifier_node.hpp"
|
4
5
|
|
5
6
|
namespace NatalieParser {
|
6
7
|
|
7
|
-
SharedPtr<Node> BeginRescueNode::
|
8
|
+
SharedPtr<Node> BeginRescueNode::name_to_assignment() const {
|
8
9
|
assert(m_name);
|
9
10
|
return new AssignmentNode {
|
10
11
|
token(),
|
11
|
-
m_name
|
12
|
+
m_name,
|
12
13
|
new IdentifierNode {
|
13
14
|
Token { Token::Type::GlobalVariable, "$!", file(), line(), column(), false },
|
14
15
|
false },
|
@@ -21,7 +22,7 @@ void BeginRescueNode::transform(Creator *creator) const {
|
|
21
22
|
for (auto exception_node : m_exceptions)
|
22
23
|
array.add_node(exception_node);
|
23
24
|
if (m_name)
|
24
|
-
array.add_node(
|
25
|
+
array.add_node(name_to_assignment());
|
25
26
|
creator->append(array);
|
26
27
|
if (m_body->nodes().is_empty())
|
27
28
|
creator->append_nil();
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#include "natalie_parser/node/for_node.hpp"
|
2
|
+
|
3
|
+
namespace NatalieParser {
|
4
|
+
|
5
|
+
void ForNode::transform(Creator *creator) const {
|
6
|
+
creator->set_type("for");
|
7
|
+
creator->append(m_expr);
|
8
|
+
creator->with_assignment(true, [&]() { creator->append(*m_vars); });
|
9
|
+
if (!m_body->is_empty())
|
10
|
+
creator->append(m_body->without_unnecessary_nesting());
|
11
|
+
}
|
12
|
+
|
13
|
+
}
|