natalie_parser 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +43 -0
- data/ext/natalie_parser/natalie_parser.cpp +1 -0
- data/include/natalie_parser/lexer.hpp +17 -4
- data/include/natalie_parser/node/for_node.hpp +50 -0
- data/include/natalie_parser/node/match_node.hpp +6 -0
- data/include/natalie_parser/node/node.hpp +1 -0
- data/include/natalie_parser/node.hpp +1 -0
- data/include/natalie_parser/parser.hpp +16 -5
- data/include/natalie_parser/token.hpp +38 -0
- data/lib/natalie_parser/version.rb +1 -1
- data/src/lexer/interpolated_string_lexer.cpp +1 -1
- data/src/lexer/regexp_lexer.cpp +1 -1
- data/src/lexer/word_array_lexer.cpp +1 -1
- data/src/lexer.cpp +170 -248
- data/src/node/match_node.cpp +5 -0
- data/src/parser.cpp +195 -105
- metadata +3 -2
data/src/lexer.cpp
CHANGED
@@ -80,7 +80,47 @@ Token Lexer::next_token() {
|
|
80
80
|
m_whitespace_precedes = skip_whitespace();
|
81
81
|
m_token_line = m_cursor_line;
|
82
82
|
m_token_column = m_cursor_column;
|
83
|
-
|
83
|
+
Token token = build_next_token();
|
84
|
+
switch (token.type()) {
|
85
|
+
case Token::Type::AliasKeyword:
|
86
|
+
m_remaining_method_names = 2;
|
87
|
+
break;
|
88
|
+
case Token::Type::ConstantResolution:
|
89
|
+
case Token::Type::DefKeyword:
|
90
|
+
m_remaining_method_names = 1;
|
91
|
+
m_allow_assignment_method = true;
|
92
|
+
break;
|
93
|
+
case Token::Type::Dot:
|
94
|
+
m_remaining_method_names = 1;
|
95
|
+
break;
|
96
|
+
case Token::Type::UndefKeyword:
|
97
|
+
m_remaining_method_names = std::numeric_limits<size_t>::max();
|
98
|
+
m_method_name_separator = Token::Type::Comma;
|
99
|
+
break;
|
100
|
+
default:
|
101
|
+
if (m_method_name_separator != Token::Type::Invalid) {
|
102
|
+
if (m_last_method_name) {
|
103
|
+
m_last_method_name = {};
|
104
|
+
if (token.type() != m_method_name_separator) {
|
105
|
+
m_remaining_method_names = 0;
|
106
|
+
m_method_name_separator = Token::Type::Invalid;
|
107
|
+
}
|
108
|
+
} else {
|
109
|
+
m_last_method_name = token;
|
110
|
+
}
|
111
|
+
} else if (m_remaining_method_names > 0) {
|
112
|
+
m_remaining_method_names--;
|
113
|
+
} else {
|
114
|
+
m_allow_assignment_method = false;
|
115
|
+
}
|
116
|
+
break;
|
117
|
+
}
|
118
|
+
return token;
|
119
|
+
}
|
120
|
+
|
121
|
+
bool is_name_start_char(char c) {
|
122
|
+
if (!c) return false;
|
123
|
+
return (c >= 'a' && c <= 'z') || c == '_' || (unsigned int)c >= 128;
|
84
124
|
}
|
85
125
|
|
86
126
|
bool is_identifier_char(char c) {
|
@@ -204,10 +244,10 @@ Token Lexer::build_next_token() {
|
|
204
244
|
advance();
|
205
245
|
return Token { Token::Type::PlusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
206
246
|
case '@':
|
207
|
-
if (
|
247
|
+
if (m_remaining_method_names > 0) {
|
208
248
|
advance();
|
209
249
|
SharedPtr<String> lit = new String("+@");
|
210
|
-
return Token { Token::Type::
|
250
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
211
251
|
} else {
|
212
252
|
return Token { Token::Type::Plus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
213
253
|
}
|
@@ -224,10 +264,10 @@ Token Lexer::build_next_token() {
|
|
224
264
|
advance();
|
225
265
|
return Token { Token::Type::MinusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
226
266
|
case '@':
|
227
|
-
if (
|
267
|
+
if (m_remaining_method_names > 0) {
|
228
268
|
advance();
|
229
269
|
SharedPtr<String> lit = new String("-@");
|
230
|
-
return Token { Token::Type::
|
270
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
231
271
|
} else {
|
232
272
|
return Token { Token::Type::Minus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
233
273
|
}
|
@@ -256,21 +296,28 @@ Token Lexer::build_next_token() {
|
|
256
296
|
advance();
|
257
297
|
if (!m_last_token)
|
258
298
|
return consume_regexp('/', '/');
|
299
|
+
if (m_remaining_method_names > 0)
|
300
|
+
return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
259
301
|
switch (m_last_token.type()) {
|
260
302
|
case Token::Type::Comma:
|
261
303
|
case Token::Type::Doc:
|
304
|
+
case Token::Type::Equal:
|
262
305
|
case Token::Type::LBracket:
|
263
306
|
case Token::Type::LCurlyBrace:
|
264
307
|
case Token::Type::LParen:
|
265
308
|
case Token::Type::Match:
|
266
309
|
case Token::Type::Newline:
|
310
|
+
case Token::Type::Not:
|
311
|
+
case Token::Type::Pipe:
|
267
312
|
return consume_regexp('/', '/');
|
268
|
-
case Token::Type::DefKeyword:
|
269
|
-
return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
270
313
|
default: {
|
271
314
|
switch (current_char()) {
|
272
315
|
case ' ':
|
273
|
-
|
316
|
+
if (m_last_token.is_keyword() && m_last_token.can_precede_regexp_literal()) {
|
317
|
+
return consume_regexp('/', '/');
|
318
|
+
} else {
|
319
|
+
return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
320
|
+
}
|
274
321
|
case '=':
|
275
322
|
advance();
|
276
323
|
return Token { Token::Type::SlashEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
@@ -291,216 +338,26 @@ Token Lexer::build_next_token() {
|
|
291
338
|
advance();
|
292
339
|
return Token { Token::Type::PercentEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
293
340
|
case 'q':
|
294
|
-
|
295
|
-
case '[':
|
296
|
-
advance(2);
|
297
|
-
return consume_single_quoted_string('[', ']');
|
298
|
-
case '{':
|
299
|
-
advance(2);
|
300
|
-
return consume_single_quoted_string('{', '}');
|
301
|
-
case '<':
|
302
|
-
advance(2);
|
303
|
-
return consume_single_quoted_string('<', '>');
|
304
|
-
case '(':
|
305
|
-
advance(2);
|
306
|
-
return consume_single_quoted_string('(', ')');
|
307
|
-
default: {
|
308
|
-
char c = peek();
|
309
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
310
|
-
advance(2);
|
311
|
-
return consume_single_quoted_string(c, c);
|
312
|
-
} else {
|
313
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
314
|
-
}
|
315
|
-
}
|
316
|
-
}
|
341
|
+
return consume_percent_string(&Lexer::consume_single_quoted_string);
|
317
342
|
case 'Q':
|
318
|
-
|
319
|
-
case '[':
|
320
|
-
advance(2);
|
321
|
-
return consume_double_quoted_string('[', ']');
|
322
|
-
case '{':
|
323
|
-
advance(2);
|
324
|
-
return consume_double_quoted_string('{', '}');
|
325
|
-
case '<':
|
326
|
-
advance(2);
|
327
|
-
return consume_double_quoted_string('<', '>');
|
328
|
-
case '(':
|
329
|
-
advance(2);
|
330
|
-
return consume_double_quoted_string('(', ')');
|
331
|
-
default: {
|
332
|
-
char c = peek();
|
333
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
334
|
-
advance(2);
|
335
|
-
return consume_double_quoted_string(c, c);
|
336
|
-
} else {
|
337
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
338
|
-
}
|
339
|
-
}
|
340
|
-
}
|
343
|
+
return consume_percent_string(&Lexer::consume_interpolated_string);
|
341
344
|
case 'r':
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
return consume_regexp('[', ']');
|
346
|
-
case '{':
|
347
|
-
advance(2);
|
348
|
-
return consume_regexp('{', '}');
|
349
|
-
case '(':
|
350
|
-
advance(2);
|
351
|
-
return consume_regexp('(', ')');
|
352
|
-
case '<':
|
353
|
-
advance(2);
|
354
|
-
return consume_regexp('<', '>');
|
355
|
-
default: {
|
356
|
-
char c = peek();
|
357
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
358
|
-
advance(2);
|
359
|
-
return consume_regexp(c, c);
|
360
|
-
} else {
|
361
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
362
|
-
}
|
363
|
-
}
|
364
|
-
}
|
345
|
+
return consume_percent_string(&Lexer::consume_regexp);
|
346
|
+
case 's':
|
347
|
+
return consume_percent_string(&Lexer::consume_percent_symbol);
|
365
348
|
case 'x':
|
366
|
-
|
367
|
-
case '/': {
|
368
|
-
advance(2);
|
369
|
-
return consume_double_quoted_string('/', '/', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
370
|
-
}
|
371
|
-
case '[': {
|
372
|
-
advance(2);
|
373
|
-
return consume_double_quoted_string('[', ']', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
374
|
-
}
|
375
|
-
case '{': {
|
376
|
-
advance(2);
|
377
|
-
return consume_double_quoted_string('{', '}', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
378
|
-
}
|
379
|
-
case '(': {
|
380
|
-
advance(2);
|
381
|
-
return consume_double_quoted_string('(', ')', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
382
|
-
}
|
383
|
-
default:
|
384
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
385
|
-
}
|
349
|
+
return consume_percent_string(&Lexer::consume_interpolated_shell);
|
386
350
|
case 'w':
|
387
|
-
|
388
|
-
case '/':
|
389
|
-
case '|': {
|
390
|
-
char c = next();
|
391
|
-
advance();
|
392
|
-
return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerW);
|
393
|
-
}
|
394
|
-
case '[':
|
395
|
-
advance(2);
|
396
|
-
return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerW);
|
397
|
-
case '{':
|
398
|
-
advance(2);
|
399
|
-
return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerW);
|
400
|
-
case '<':
|
401
|
-
advance(2);
|
402
|
-
return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerW);
|
403
|
-
case '(':
|
404
|
-
advance(2);
|
405
|
-
return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerW);
|
406
|
-
default:
|
407
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
408
|
-
}
|
351
|
+
return consume_percent_string(&Lexer::consume_percent_lower_w);
|
409
352
|
case 'W':
|
410
|
-
|
411
|
-
case '/':
|
412
|
-
case '|': {
|
413
|
-
char c = next();
|
414
|
-
advance();
|
415
|
-
return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperW);
|
416
|
-
}
|
417
|
-
case '[':
|
418
|
-
advance(2);
|
419
|
-
return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperW);
|
420
|
-
case '{':
|
421
|
-
advance(2);
|
422
|
-
return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperW);
|
423
|
-
case '<':
|
424
|
-
advance(2);
|
425
|
-
return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperW);
|
426
|
-
case '(':
|
427
|
-
advance(2);
|
428
|
-
return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperW);
|
429
|
-
default:
|
430
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
431
|
-
}
|
353
|
+
return consume_percent_string(&Lexer::consume_percent_upper_w);
|
432
354
|
case 'i':
|
433
|
-
|
434
|
-
case '|':
|
435
|
-
case '/': {
|
436
|
-
char c = next();
|
437
|
-
advance();
|
438
|
-
return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerI);
|
439
|
-
}
|
440
|
-
case '[':
|
441
|
-
advance(2);
|
442
|
-
return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerI);
|
443
|
-
case '{':
|
444
|
-
advance(2);
|
445
|
-
return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerI);
|
446
|
-
case '<':
|
447
|
-
advance(2);
|
448
|
-
return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerI);
|
449
|
-
case '(':
|
450
|
-
advance(2);
|
451
|
-
return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerI);
|
452
|
-
default:
|
453
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
454
|
-
}
|
355
|
+
return consume_percent_string(&Lexer::consume_percent_lower_i);
|
455
356
|
case 'I':
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
char c = next();
|
460
|
-
advance();
|
461
|
-
return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperI);
|
462
|
-
}
|
463
|
-
case '[':
|
464
|
-
advance(2);
|
465
|
-
return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperI);
|
466
|
-
case '{':
|
467
|
-
advance(2);
|
468
|
-
return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperI);
|
469
|
-
case '<':
|
470
|
-
advance(2);
|
471
|
-
return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperI);
|
472
|
-
case '(':
|
473
|
-
advance(2);
|
474
|
-
return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperI);
|
475
|
-
default:
|
476
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
477
|
-
}
|
478
|
-
case '[':
|
479
|
-
advance();
|
480
|
-
return consume_double_quoted_string('[', ']');
|
481
|
-
case '{':
|
482
|
-
advance();
|
483
|
-
return consume_double_quoted_string('{', '}');
|
484
|
-
case '<':
|
485
|
-
advance();
|
486
|
-
return consume_double_quoted_string('<', '>');
|
487
|
-
case '(':
|
488
|
-
if (m_last_token.type() == Token::Type::DefKeyword || m_last_token.type() == Token::Type::Dot) {
|
489
|
-
// It's a trap! This looks like a %(string) but it's a method def/call!
|
490
|
-
break;
|
491
|
-
}
|
492
|
-
advance();
|
493
|
-
return consume_double_quoted_string('(', ')');
|
494
|
-
default: {
|
495
|
-
auto c = current_char();
|
496
|
-
if (char_can_be_string_or_regexp_delimiter(c)) {
|
497
|
-
advance();
|
498
|
-
return consume_double_quoted_string(c, c);
|
499
|
-
}
|
500
|
-
break;
|
501
|
-
}
|
357
|
+
return consume_percent_string(&Lexer::consume_percent_upper_i);
|
358
|
+
default:
|
359
|
+
return consume_percent_string(&Lexer::consume_interpolated_string, false);
|
502
360
|
}
|
503
|
-
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
504
361
|
case '!':
|
505
362
|
advance();
|
506
363
|
switch (current_char()) {
|
@@ -511,10 +368,10 @@ Token Lexer::build_next_token() {
|
|
511
368
|
advance();
|
512
369
|
return Token { Token::Type::NotMatch, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
513
370
|
case '@':
|
514
|
-
if (
|
371
|
+
if (m_remaining_method_names > 0) {
|
515
372
|
advance();
|
516
373
|
SharedPtr<String> lit = new String("!@");
|
517
|
-
return Token { Token::Type::
|
374
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
518
375
|
} else {
|
519
376
|
return Token { Token::Type::Not, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
520
377
|
}
|
@@ -649,10 +506,10 @@ Token Lexer::build_next_token() {
|
|
649
506
|
advance();
|
650
507
|
switch (current_char()) {
|
651
508
|
case '@':
|
652
|
-
if (
|
509
|
+
if (m_remaining_method_names > 0) {
|
653
510
|
advance();
|
654
511
|
SharedPtr<String> lit = new String("~@");
|
655
|
-
return Token { Token::Type::
|
512
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
656
513
|
} else {
|
657
514
|
return Token { Token::Type::Tilde, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
658
515
|
}
|
@@ -661,7 +518,7 @@ Token Lexer::build_next_token() {
|
|
661
518
|
}
|
662
519
|
case '?': {
|
663
520
|
auto c = next();
|
664
|
-
if (isspace(c)) {
|
521
|
+
if (isspace(c) || c == 0) {
|
665
522
|
m_open_ternary = true;
|
666
523
|
return Token { Token::Type::TernaryQuestion, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
667
524
|
} else {
|
@@ -691,7 +548,7 @@ Token Lexer::build_next_token() {
|
|
691
548
|
advance();
|
692
549
|
auto string = consume_single_quoted_string('\'', '\'');
|
693
550
|
return Token { Token::Type::Symbol, string.literal(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
694
|
-
} else if (isspace(c)) {
|
551
|
+
} else if (isspace(c) || c == 0) {
|
695
552
|
m_open_ternary = false;
|
696
553
|
auto token = Token { Token::Type::TernaryColon, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
697
554
|
return token;
|
@@ -789,13 +646,18 @@ Token Lexer::build_next_token() {
|
|
789
646
|
return Token { Token::Type::Comma, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
790
647
|
case '"':
|
791
648
|
advance();
|
792
|
-
return
|
649
|
+
return consume_interpolated_string('"', '"');
|
793
650
|
case '\'':
|
794
651
|
advance();
|
795
652
|
return consume_single_quoted_string('\'', '\'');
|
796
653
|
case '`': {
|
797
654
|
advance();
|
798
|
-
|
655
|
+
if (m_remaining_method_names > 0) {
|
656
|
+
SharedPtr<String> lit = new String("`");
|
657
|
+
return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
658
|
+
} else {
|
659
|
+
return consume_interpolated_shell('`', '`');
|
660
|
+
}
|
799
661
|
}
|
800
662
|
case '#':
|
801
663
|
if (token_is_first_on_line()) {
|
@@ -858,14 +720,14 @@ Token Lexer::build_next_token() {
|
|
858
720
|
|
859
721
|
Token keyword_token;
|
860
722
|
|
861
|
-
if (!m_last_token.is_dot() && match(4, "self")) {
|
862
|
-
if (current_char() == '.')
|
723
|
+
if (!m_last_token.is_dot() && !m_last_token.is_constant_resolution() && match(4, "self")) {
|
724
|
+
if (current_char() == '.' || (current_char() == ':' && peek() == ':'))
|
863
725
|
keyword_token = { Token::Type::SelfKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
864
726
|
else
|
865
727
|
rewind(4);
|
866
728
|
}
|
867
729
|
|
868
|
-
if (
|
730
|
+
if (m_remaining_method_names == 0) {
|
869
731
|
if (match(12, "__ENCODING__"))
|
870
732
|
keyword_token = { Token::Type::ENCODINGKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
871
733
|
else if (match(8, "__LINE__"))
|
@@ -960,10 +822,10 @@ Token Lexer::build_next_token() {
|
|
960
822
|
}
|
961
823
|
|
962
824
|
auto c = current_char();
|
963
|
-
if ((c
|
964
|
-
return
|
825
|
+
if (is_name_start_char(c)) {
|
826
|
+
return consume_bare_name_or_constant(Token::Type::BareName);
|
965
827
|
} else if (c >= 'A' && c <= 'Z') {
|
966
|
-
return
|
828
|
+
return consume_bare_name_or_constant(Token::Type::Constant);
|
967
829
|
} else {
|
968
830
|
auto buf = consume_non_whitespace();
|
969
831
|
auto token = Token { Token::Type::Invalid, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
@@ -1093,45 +955,47 @@ Token Lexer::consume_symbol() {
|
|
1093
955
|
return Token { Token::Type::Symbol, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1094
956
|
}
|
1095
957
|
|
1096
|
-
|
958
|
+
SharedPtr<String> Lexer::consume_word() {
|
1097
959
|
char c = current_char();
|
1098
960
|
SharedPtr<String> buf = new String("");
|
1099
961
|
do {
|
1100
962
|
buf->append_char(c);
|
1101
963
|
c = next();
|
1102
964
|
} while (is_identifier_char(c));
|
965
|
+
return buf;
|
966
|
+
}
|
967
|
+
|
968
|
+
Token Lexer::consume_word(Token::Type type) {
|
969
|
+
return Token { type, consume_word(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
970
|
+
}
|
971
|
+
|
972
|
+
Token Lexer::consume_bare_name_or_constant(Token::Type type) {
|
973
|
+
auto buf = consume_word();
|
974
|
+
auto c = current_char();
|
1103
975
|
switch (c) {
|
1104
976
|
case '?':
|
1105
977
|
case '!':
|
1106
978
|
advance();
|
1107
979
|
buf->append_char(c);
|
1108
980
|
break;
|
981
|
+
case '=':
|
982
|
+
if (m_allow_assignment_method || (!m_last_token.is_dot() && m_remaining_method_names > 0)) {
|
983
|
+
advance();
|
984
|
+
buf->append_char(c);
|
985
|
+
}
|
986
|
+
break;
|
987
|
+
case ':':
|
988
|
+
if (peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
989
|
+
advance();
|
990
|
+
type = Token::Type::SymbolKey;
|
991
|
+
}
|
992
|
+
break;
|
1109
993
|
default:
|
1110
994
|
break;
|
1111
995
|
}
|
1112
996
|
return Token { type, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1113
997
|
}
|
1114
998
|
|
1115
|
-
Token Lexer::consume_bare_name() {
|
1116
|
-
auto token = consume_word(Token::Type::BareName);
|
1117
|
-
auto c = current_char();
|
1118
|
-
if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
1119
|
-
advance();
|
1120
|
-
token.set_type(Token::Type::SymbolKey);
|
1121
|
-
}
|
1122
|
-
return token;
|
1123
|
-
}
|
1124
|
-
|
1125
|
-
Token Lexer::consume_constant() {
|
1126
|
-
auto token = consume_word(Token::Type::Constant);
|
1127
|
-
auto c = current_char();
|
1128
|
-
if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
|
1129
|
-
advance();
|
1130
|
-
token.set_type(Token::Type::SymbolKey);
|
1131
|
-
}
|
1132
|
-
return token;
|
1133
|
-
}
|
1134
|
-
|
1135
999
|
Token Lexer::consume_global_variable() {
|
1136
1000
|
switch (peek()) {
|
1137
1001
|
case '?':
|
@@ -1153,7 +1017,6 @@ Token Lexer::consume_global_variable() {
|
|
1153
1017
|
case '.':
|
1154
1018
|
case ',':
|
1155
1019
|
case ':':
|
1156
|
-
case '_':
|
1157
1020
|
case '~': {
|
1158
1021
|
advance();
|
1159
1022
|
SharedPtr<String> buf = new String("$");
|
@@ -1277,7 +1140,7 @@ Token Lexer::consume_heredoc() {
|
|
1277
1140
|
}
|
1278
1141
|
advance();
|
1279
1142
|
} else {
|
1280
|
-
heredoc_name =
|
1143
|
+
heredoc_name = *consume_word();
|
1281
1144
|
}
|
1282
1145
|
|
1283
1146
|
SharedPtr<String> doc = new String("");
|
@@ -1673,7 +1536,7 @@ Token Lexer::consume_single_quoted_string(char start_char, char stop_char) {
|
|
1673
1536
|
SharedPtr<String> buf = new String("");
|
1674
1537
|
char c = current_char();
|
1675
1538
|
while (c) {
|
1676
|
-
if (c == '\\') {
|
1539
|
+
if (c == '\\' && stop_char != '\\') {
|
1677
1540
|
c = next();
|
1678
1541
|
if (c == stop_char || c == '\\') {
|
1679
1542
|
buf->append_char(c);
|
@@ -1720,6 +1583,65 @@ Token Lexer::consume_regexp(char start_char, char stop_char) {
|
|
1720
1583
|
return Token { Token::Type::InterpolatedRegexpBegin, start_char, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1721
1584
|
}
|
1722
1585
|
|
1586
|
+
Token Lexer::consume_percent_symbol(char start_char, char stop_char) {
|
1587
|
+
Token token = consume_single_quoted_string(start_char, stop_char);
|
1588
|
+
token.set_type(Token::Type::Symbol);
|
1589
|
+
return token;
|
1590
|
+
}
|
1591
|
+
|
1592
|
+
Token Lexer::consume_interpolated_string(char start_char, char stop_char) {
|
1593
|
+
return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedStringBegin, Token::Type::InterpolatedStringEnd);
|
1594
|
+
}
|
1595
|
+
|
1596
|
+
Token Lexer::consume_interpolated_shell(char start_char, char stop_char) {
|
1597
|
+
return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
|
1598
|
+
}
|
1599
|
+
|
1600
|
+
Token Lexer::consume_percent_lower_w(char start_char, char stop_char) {
|
1601
|
+
return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerW);
|
1602
|
+
}
|
1603
|
+
|
1604
|
+
Token Lexer::consume_percent_upper_w(char start_char, char stop_char) {
|
1605
|
+
return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperW);
|
1606
|
+
}
|
1607
|
+
|
1608
|
+
Token Lexer::consume_percent_lower_i(char start_char, char stop_char) {
|
1609
|
+
return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerI);
|
1610
|
+
}
|
1611
|
+
|
1612
|
+
Token Lexer::consume_percent_upper_i(char start_char, char stop_char) {
|
1613
|
+
return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperI);
|
1614
|
+
}
|
1615
|
+
|
1616
|
+
Token Lexer::consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered) {
|
1617
|
+
if (m_remaining_method_names > 0) {
|
1618
|
+
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1619
|
+
}
|
1620
|
+
char c = is_lettered ? peek() : current_char();
|
1621
|
+
size_t bytes = is_lettered ? 2 : 1;
|
1622
|
+
switch (c) {
|
1623
|
+
case '[':
|
1624
|
+
advance(bytes);
|
1625
|
+
return (this->*consumer)('[', ']');
|
1626
|
+
case '{':
|
1627
|
+
advance(bytes);
|
1628
|
+
return (this->*consumer)('{', '}');
|
1629
|
+
case '<':
|
1630
|
+
advance(bytes);
|
1631
|
+
return (this->*consumer)('<', '>');
|
1632
|
+
case '(':
|
1633
|
+
advance(bytes);
|
1634
|
+
return (this->*consumer)('(', ')');
|
1635
|
+
default:
|
1636
|
+
if (char_can_be_string_or_regexp_delimiter(c)) {
|
1637
|
+
advance(bytes);
|
1638
|
+
return (this->*consumer)(c, c);
|
1639
|
+
} else {
|
1640
|
+
return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
|
1641
|
+
}
|
1642
|
+
}
|
1643
|
+
}
|
1644
|
+
|
1723
1645
|
SharedPtr<String> Lexer::consume_non_whitespace() {
|
1724
1646
|
char c = current_char();
|
1725
1647
|
SharedPtr<String> buf = new String("");
|
data/src/node/match_node.cpp
CHANGED