natalie_parser 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/lexer.cpp CHANGED
@@ -80,7 +80,47 @@ Token Lexer::next_token() {
80
80
  m_whitespace_precedes = skip_whitespace();
81
81
  m_token_line = m_cursor_line;
82
82
  m_token_column = m_cursor_column;
83
- return build_next_token();
83
+ Token token = build_next_token();
84
+ switch (token.type()) {
85
+ case Token::Type::AliasKeyword:
86
+ m_remaining_method_names = 2;
87
+ break;
88
+ case Token::Type::ConstantResolution:
89
+ case Token::Type::DefKeyword:
90
+ m_remaining_method_names = 1;
91
+ m_allow_assignment_method = true;
92
+ break;
93
+ case Token::Type::Dot:
94
+ m_remaining_method_names = 1;
95
+ break;
96
+ case Token::Type::UndefKeyword:
97
+ m_remaining_method_names = std::numeric_limits<size_t>::max();
98
+ m_method_name_separator = Token::Type::Comma;
99
+ break;
100
+ default:
101
+ if (m_method_name_separator != Token::Type::Invalid) {
102
+ if (m_last_method_name) {
103
+ m_last_method_name = {};
104
+ if (token.type() != m_method_name_separator) {
105
+ m_remaining_method_names = 0;
106
+ m_method_name_separator = Token::Type::Invalid;
107
+ }
108
+ } else {
109
+ m_last_method_name = token;
110
+ }
111
+ } else if (m_remaining_method_names > 0) {
112
+ m_remaining_method_names--;
113
+ } else {
114
+ m_allow_assignment_method = false;
115
+ }
116
+ break;
117
+ }
118
+ return token;
119
+ }
120
+
121
+ bool is_name_start_char(char c) {
122
+ if (!c) return false;
123
+ return (c >= 'a' && c <= 'z') || c == '_' || (unsigned int)c >= 128;
84
124
  }
85
125
 
86
126
  bool is_identifier_char(char c) {
@@ -204,10 +244,10 @@ Token Lexer::build_next_token() {
204
244
  advance();
205
245
  return Token { Token::Type::PlusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
206
246
  case '@':
207
- if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
247
+ if (m_remaining_method_names > 0) {
208
248
  advance();
209
249
  SharedPtr<String> lit = new String("+@");
210
- return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
250
+ return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
211
251
  } else {
212
252
  return Token { Token::Type::Plus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
213
253
  }
@@ -224,10 +264,10 @@ Token Lexer::build_next_token() {
224
264
  advance();
225
265
  return Token { Token::Type::MinusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
226
266
  case '@':
227
- if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
267
+ if (m_remaining_method_names > 0) {
228
268
  advance();
229
269
  SharedPtr<String> lit = new String("-@");
230
- return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
270
+ return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
231
271
  } else {
232
272
  return Token { Token::Type::Minus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
233
273
  }
@@ -256,17 +296,20 @@ Token Lexer::build_next_token() {
256
296
  advance();
257
297
  if (!m_last_token)
258
298
  return consume_regexp('/', '/');
299
+ if (m_remaining_method_names > 0)
300
+ return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
259
301
  switch (m_last_token.type()) {
260
302
  case Token::Type::Comma:
261
303
  case Token::Type::Doc:
304
+ case Token::Type::Equal:
262
305
  case Token::Type::LBracket:
263
306
  case Token::Type::LCurlyBrace:
264
307
  case Token::Type::LParen:
265
308
  case Token::Type::Match:
266
309
  case Token::Type::Newline:
310
+ case Token::Type::Not:
311
+ case Token::Type::Pipe:
267
312
  return consume_regexp('/', '/');
268
- case Token::Type::DefKeyword:
269
- return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
270
313
  default: {
271
314
  switch (current_char()) {
272
315
  case ' ':
@@ -295,216 +338,26 @@ Token Lexer::build_next_token() {
295
338
  advance();
296
339
  return Token { Token::Type::PercentEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
297
340
  case 'q':
298
- switch (peek()) {
299
- case '[':
300
- advance(2);
301
- return consume_single_quoted_string('[', ']');
302
- case '{':
303
- advance(2);
304
- return consume_single_quoted_string('{', '}');
305
- case '<':
306
- advance(2);
307
- return consume_single_quoted_string('<', '>');
308
- case '(':
309
- advance(2);
310
- return consume_single_quoted_string('(', ')');
311
- default: {
312
- char c = peek();
313
- if (char_can_be_string_or_regexp_delimiter(c)) {
314
- advance(2);
315
- return consume_single_quoted_string(c, c);
316
- } else {
317
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
318
- }
319
- }
320
- }
341
+ return consume_percent_string(&Lexer::consume_single_quoted_string);
321
342
  case 'Q':
322
- switch (peek()) {
323
- case '[':
324
- advance(2);
325
- return consume_double_quoted_string('[', ']');
326
- case '{':
327
- advance(2);
328
- return consume_double_quoted_string('{', '}');
329
- case '<':
330
- advance(2);
331
- return consume_double_quoted_string('<', '>');
332
- case '(':
333
- advance(2);
334
- return consume_double_quoted_string('(', ')');
335
- default: {
336
- char c = peek();
337
- if (char_can_be_string_or_regexp_delimiter(c)) {
338
- advance(2);
339
- return consume_double_quoted_string(c, c);
340
- } else {
341
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
342
- }
343
- }
344
- }
343
+ return consume_percent_string(&Lexer::consume_interpolated_string);
345
344
  case 'r':
346
- switch (peek()) {
347
- case '[':
348
- advance(2);
349
- return consume_regexp('[', ']');
350
- case '{':
351
- advance(2);
352
- return consume_regexp('{', '}');
353
- case '(':
354
- advance(2);
355
- return consume_regexp('(', ')');
356
- case '<':
357
- advance(2);
358
- return consume_regexp('<', '>');
359
- default: {
360
- char c = peek();
361
- if (char_can_be_string_or_regexp_delimiter(c)) {
362
- advance(2);
363
- return consume_regexp(c, c);
364
- } else {
365
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
366
- }
367
- }
368
- }
345
+ return consume_percent_string(&Lexer::consume_regexp);
346
+ case 's':
347
+ return consume_percent_string(&Lexer::consume_percent_symbol);
369
348
  case 'x':
370
- switch (peek()) {
371
- case '/': {
372
- advance(2);
373
- return consume_double_quoted_string('/', '/', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
374
- }
375
- case '[': {
376
- advance(2);
377
- return consume_double_quoted_string('[', ']', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
378
- }
379
- case '{': {
380
- advance(2);
381
- return consume_double_quoted_string('{', '}', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
382
- }
383
- case '(': {
384
- advance(2);
385
- return consume_double_quoted_string('(', ')', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
386
- }
387
- default:
388
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
389
- }
349
+ return consume_percent_string(&Lexer::consume_interpolated_shell);
390
350
  case 'w':
391
- switch (peek()) {
392
- case '/':
393
- case '|': {
394
- char c = next();
395
- advance();
396
- return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerW);
397
- }
398
- case '[':
399
- advance(2);
400
- return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerW);
401
- case '{':
402
- advance(2);
403
- return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerW);
404
- case '<':
405
- advance(2);
406
- return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerW);
407
- case '(':
408
- advance(2);
409
- return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerW);
410
- default:
411
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
412
- }
351
+ return consume_percent_string(&Lexer::consume_percent_lower_w);
413
352
  case 'W':
414
- switch (peek()) {
415
- case '/':
416
- case '|': {
417
- char c = next();
418
- advance();
419
- return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperW);
420
- }
421
- case '[':
422
- advance(2);
423
- return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperW);
424
- case '{':
425
- advance(2);
426
- return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperW);
427
- case '<':
428
- advance(2);
429
- return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperW);
430
- case '(':
431
- advance(2);
432
- return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperW);
433
- default:
434
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
435
- }
353
+ return consume_percent_string(&Lexer::consume_percent_upper_w);
436
354
  case 'i':
437
- switch (peek()) {
438
- case '|':
439
- case '/': {
440
- char c = next();
441
- advance();
442
- return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerI);
443
- }
444
- case '[':
445
- advance(2);
446
- return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerI);
447
- case '{':
448
- advance(2);
449
- return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerI);
450
- case '<':
451
- advance(2);
452
- return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerI);
453
- case '(':
454
- advance(2);
455
- return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerI);
456
- default:
457
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
458
- }
355
+ return consume_percent_string(&Lexer::consume_percent_lower_i);
459
356
  case 'I':
460
- switch (peek()) {
461
- case '|':
462
- case '/': {
463
- char c = next();
464
- advance();
465
- return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperI);
466
- }
467
- case '[':
468
- advance(2);
469
- return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperI);
470
- case '{':
471
- advance(2);
472
- return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperI);
473
- case '<':
474
- advance(2);
475
- return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperI);
476
- case '(':
477
- advance(2);
478
- return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperI);
479
- default:
480
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
481
- }
482
- case '[':
483
- advance();
484
- return consume_double_quoted_string('[', ']');
485
- case '{':
486
- advance();
487
- return consume_double_quoted_string('{', '}');
488
- case '<':
489
- advance();
490
- return consume_double_quoted_string('<', '>');
491
- case '(':
492
- if (m_last_token.type() == Token::Type::DefKeyword || m_last_token.type() == Token::Type::Dot) {
493
- // It's a trap! This looks like a %(string) but it's a method def/call!
494
- break;
495
- }
496
- advance();
497
- return consume_double_quoted_string('(', ')');
498
- default: {
499
- auto c = current_char();
500
- if (char_can_be_string_or_regexp_delimiter(c)) {
501
- advance();
502
- return consume_double_quoted_string(c, c);
503
- }
504
- break;
505
- }
357
+ return consume_percent_string(&Lexer::consume_percent_upper_i);
358
+ default:
359
+ return consume_percent_string(&Lexer::consume_interpolated_string, false);
506
360
  }
507
- return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
508
361
  case '!':
509
362
  advance();
510
363
  switch (current_char()) {
@@ -515,10 +368,10 @@ Token Lexer::build_next_token() {
515
368
  advance();
516
369
  return Token { Token::Type::NotMatch, m_file, m_token_line, m_token_column, m_whitespace_precedes };
517
370
  case '@':
518
- if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
371
+ if (m_remaining_method_names > 0) {
519
372
  advance();
520
373
  SharedPtr<String> lit = new String("!@");
521
- return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
374
+ return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
522
375
  } else {
523
376
  return Token { Token::Type::Not, m_file, m_token_line, m_token_column, m_whitespace_precedes };
524
377
  }
@@ -653,10 +506,10 @@ Token Lexer::build_next_token() {
653
506
  advance();
654
507
  switch (current_char()) {
655
508
  case '@':
656
- if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
509
+ if (m_remaining_method_names > 0) {
657
510
  advance();
658
511
  SharedPtr<String> lit = new String("~@");
659
- return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
512
+ return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
660
513
  } else {
661
514
  return Token { Token::Type::Tilde, m_file, m_token_line, m_token_column, m_whitespace_precedes };
662
515
  }
@@ -665,7 +518,7 @@ Token Lexer::build_next_token() {
665
518
  }
666
519
  case '?': {
667
520
  auto c = next();
668
- if (isspace(c)) {
521
+ if (isspace(c) || c == 0) {
669
522
  m_open_ternary = true;
670
523
  return Token { Token::Type::TernaryQuestion, m_file, m_token_line, m_token_column, m_whitespace_precedes };
671
524
  } else {
@@ -695,7 +548,7 @@ Token Lexer::build_next_token() {
695
548
  advance();
696
549
  auto string = consume_single_quoted_string('\'', '\'');
697
550
  return Token { Token::Type::Symbol, string.literal(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
698
- } else if (isspace(c)) {
551
+ } else if (isspace(c) || c == 0) {
699
552
  m_open_ternary = false;
700
553
  auto token = Token { Token::Type::TernaryColon, m_file, m_token_line, m_token_column, m_whitespace_precedes };
701
554
  return token;
@@ -793,13 +646,18 @@ Token Lexer::build_next_token() {
793
646
  return Token { Token::Type::Comma, m_file, m_token_line, m_token_column, m_whitespace_precedes };
794
647
  case '"':
795
648
  advance();
796
- return consume_double_quoted_string('"', '"');
649
+ return consume_interpolated_string('"', '"');
797
650
  case '\'':
798
651
  advance();
799
652
  return consume_single_quoted_string('\'', '\'');
800
653
  case '`': {
801
654
  advance();
802
- return consume_double_quoted_string('`', '`', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
655
+ if (m_remaining_method_names > 0) {
656
+ SharedPtr<String> lit = new String("`");
657
+ return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
658
+ } else {
659
+ return consume_interpolated_shell('`', '`');
660
+ }
803
661
  }
804
662
  case '#':
805
663
  if (token_is_first_on_line()) {
@@ -862,14 +720,14 @@ Token Lexer::build_next_token() {
862
720
 
863
721
  Token keyword_token;
864
722
 
865
- if (!m_last_token.is_dot() && match(4, "self")) {
866
- if (current_char() == '.')
723
+ if (!m_last_token.is_dot() && !m_last_token.is_constant_resolution() && match(4, "self")) {
724
+ if (current_char() == '.' || (current_char() == ':' && peek() == ':'))
867
725
  keyword_token = { Token::Type::SelfKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
868
726
  else
869
727
  rewind(4);
870
728
  }
871
729
 
872
- if (!m_last_token.is_dot() && !m_last_token.is_def_keyword()) {
730
+ if (m_remaining_method_names == 0) {
873
731
  if (match(12, "__ENCODING__"))
874
732
  keyword_token = { Token::Type::ENCODINGKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
875
733
  else if (match(8, "__LINE__"))
@@ -964,10 +822,10 @@ Token Lexer::build_next_token() {
964
822
  }
965
823
 
966
824
  auto c = current_char();
967
- if ((c >= 'a' && c <= 'z') || c == '_') {
968
- return consume_bare_name();
825
+ if (is_name_start_char(c)) {
826
+ return consume_bare_name_or_constant(Token::Type::BareName);
969
827
  } else if (c >= 'A' && c <= 'Z') {
970
- return consume_constant();
828
+ return consume_bare_name_or_constant(Token::Type::Constant);
971
829
  } else {
972
830
  auto buf = consume_non_whitespace();
973
831
  auto token = Token { Token::Type::Invalid, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
@@ -1097,45 +955,51 @@ Token Lexer::consume_symbol() {
1097
955
  return Token { Token::Type::Symbol, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
1098
956
  }
1099
957
 
1100
- Token Lexer::consume_word(Token::Type type) {
958
+ SharedPtr<String> Lexer::consume_word() {
1101
959
  char c = current_char();
1102
960
  SharedPtr<String> buf = new String("");
1103
961
  do {
1104
962
  buf->append_char(c);
1105
963
  c = next();
1106
964
  } while (is_identifier_char(c));
965
+ return buf;
966
+ }
967
+
968
+ Token Lexer::consume_word(Token::Type type) {
969
+ return Token { type, consume_word(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
970
+ }
971
+
972
+ Token Lexer::consume_bare_name_or_constant(Token::Type type) {
973
+ auto buf = consume_word();
974
+ auto c = current_char();
1107
975
  switch (c) {
1108
976
  case '?':
1109
977
  case '!':
978
+ if (peek() == ':' && m_last_token.can_precede_symbol_key()) {
979
+ advance();
980
+ type = Token::Type::SymbolKey;
981
+ }
1110
982
  advance();
1111
983
  buf->append_char(c);
1112
984
  break;
985
+ case '=':
986
+ if (m_allow_assignment_method || (!m_last_token.is_dot() && m_remaining_method_names > 0)) {
987
+ advance();
988
+ buf->append_char(c);
989
+ }
990
+ break;
991
+ case ':':
992
+ if (peek() != ':' && m_last_token.can_precede_symbol_key()) {
993
+ advance();
994
+ type = Token::Type::SymbolKey;
995
+ }
996
+ break;
1113
997
  default:
1114
998
  break;
1115
999
  }
1116
1000
  return Token { type, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
1117
1001
  }
1118
1002
 
1119
- Token Lexer::consume_bare_name() {
1120
- auto token = consume_word(Token::Type::BareName);
1121
- auto c = current_char();
1122
- if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
1123
- advance();
1124
- token.set_type(Token::Type::SymbolKey);
1125
- }
1126
- return token;
1127
- }
1128
-
1129
- Token Lexer::consume_constant() {
1130
- auto token = consume_word(Token::Type::Constant);
1131
- auto c = current_char();
1132
- if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
1133
- advance();
1134
- token.set_type(Token::Type::SymbolKey);
1135
- }
1136
- return token;
1137
- }
1138
-
1139
1003
  Token Lexer::consume_global_variable() {
1140
1004
  switch (peek()) {
1141
1005
  case '?':
@@ -1157,7 +1021,6 @@ Token Lexer::consume_global_variable() {
1157
1021
  case '.':
1158
1022
  case ',':
1159
1023
  case ':':
1160
- case '_':
1161
1024
  case '~': {
1162
1025
  advance();
1163
1026
  SharedPtr<String> buf = new String("$");
@@ -1281,7 +1144,7 @@ Token Lexer::consume_heredoc() {
1281
1144
  }
1282
1145
  advance();
1283
1146
  } else {
1284
- heredoc_name = String(consume_word(Token::Type::BareName).literal());
1147
+ heredoc_name = *consume_word();
1285
1148
  }
1286
1149
 
1287
1150
  SharedPtr<String> doc = new String("");
@@ -1425,7 +1288,26 @@ Token Lexer::consume_numeric() {
1425
1288
  break;
1426
1289
  }
1427
1290
  default:
1428
- token = consume_decimal_digits_and_build_token();
1291
+ char c = peek();
1292
+
1293
+ if (isdigit(c)) {
1294
+ // bare octal case, e.g. 0777.
1295
+ // If starts with a 0 but next number is not 0..7 then that's an error.
1296
+ if (!(c >= '0' && c <= '7'))
1297
+ return Token { Token::Type::Invalid, c, m_file, m_cursor_line,
1298
+ m_cursor_column, m_whitespace_precedes };
1299
+ chars->append_char(c);
1300
+ c = next();
1301
+ do {
1302
+ chars->append_char(c);
1303
+ c = next();
1304
+ if (c == '_')
1305
+ c = next();
1306
+ } while (c >= '0' && c <= '7');
1307
+ token = chars_to_fixnum_or_bignum_token(chars, 8, 1);
1308
+ } else {
1309
+ token = consume_decimal_digits_and_build_token();
1310
+ }
1429
1311
  }
1430
1312
  } else {
1431
1313
  token = consume_decimal_digits_and_build_token();
@@ -1677,7 +1559,7 @@ Token Lexer::consume_single_quoted_string(char start_char, char stop_char) {
1677
1559
  SharedPtr<String> buf = new String("");
1678
1560
  char c = current_char();
1679
1561
  while (c) {
1680
- if (c == '\\') {
1562
+ if (c == '\\' && stop_char != '\\') {
1681
1563
  c = next();
1682
1564
  if (c == stop_char || c == '\\') {
1683
1565
  buf->append_char(c);
@@ -1724,6 +1606,65 @@ Token Lexer::consume_regexp(char start_char, char stop_char) {
1724
1606
  return Token { Token::Type::InterpolatedRegexpBegin, start_char, m_file, m_token_line, m_token_column, m_whitespace_precedes };
1725
1607
  }
1726
1608
 
1609
+ Token Lexer::consume_percent_symbol(char start_char, char stop_char) {
1610
+ Token token = consume_single_quoted_string(start_char, stop_char);
1611
+ token.set_type(Token::Type::Symbol);
1612
+ return token;
1613
+ }
1614
+
1615
+ Token Lexer::consume_interpolated_string(char start_char, char stop_char) {
1616
+ return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedStringBegin, Token::Type::InterpolatedStringEnd);
1617
+ }
1618
+
1619
+ Token Lexer::consume_interpolated_shell(char start_char, char stop_char) {
1620
+ return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
1621
+ }
1622
+
1623
+ Token Lexer::consume_percent_lower_w(char start_char, char stop_char) {
1624
+ return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerW);
1625
+ }
1626
+
1627
+ Token Lexer::consume_percent_upper_w(char start_char, char stop_char) {
1628
+ return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperW);
1629
+ }
1630
+
1631
+ Token Lexer::consume_percent_lower_i(char start_char, char stop_char) {
1632
+ return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerI);
1633
+ }
1634
+
1635
+ Token Lexer::consume_percent_upper_i(char start_char, char stop_char) {
1636
+ return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperI);
1637
+ }
1638
+
1639
+ Token Lexer::consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered) {
1640
+ if (m_remaining_method_names > 0) {
1641
+ return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
1642
+ }
1643
+ char c = is_lettered ? peek() : current_char();
1644
+ size_t bytes = is_lettered ? 2 : 1;
1645
+ switch (c) {
1646
+ case '[':
1647
+ advance(bytes);
1648
+ return (this->*consumer)('[', ']');
1649
+ case '{':
1650
+ advance(bytes);
1651
+ return (this->*consumer)('{', '}');
1652
+ case '<':
1653
+ advance(bytes);
1654
+ return (this->*consumer)('<', '>');
1655
+ case '(':
1656
+ advance(bytes);
1657
+ return (this->*consumer)('(', ')');
1658
+ default:
1659
+ if (char_can_be_string_or_regexp_delimiter(c)) {
1660
+ advance(bytes);
1661
+ return (this->*consumer)(c, c);
1662
+ } else {
1663
+ return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
1664
+ }
1665
+ }
1666
+ }
1667
+
1727
1668
  SharedPtr<String> Lexer::consume_non_whitespace() {
1728
1669
  char c = current_char();
1729
1670
  SharedPtr<String> buf = new String("");
@@ -1,14 +1,15 @@
1
1
  #include "natalie_parser/node/begin_rescue_node.hpp"
2
2
  #include "natalie_parser/node/array_node.hpp"
3
3
  #include "natalie_parser/node/assignment_node.hpp"
4
+ #include "natalie_parser/node/identifier_node.hpp"
4
5
 
5
6
  namespace NatalieParser {
6
7
 
7
- SharedPtr<Node> BeginRescueNode::name_to_node() const {
8
+ SharedPtr<Node> BeginRescueNode::name_to_assignment() const {
8
9
  assert(m_name);
9
10
  return new AssignmentNode {
10
11
  token(),
11
- m_name.static_cast_as<Node>(),
12
+ m_name,
12
13
  new IdentifierNode {
13
14
  Token { Token::Type::GlobalVariable, "$!", file(), line(), column(), false },
14
15
  false },
@@ -21,7 +22,7 @@ void BeginRescueNode::transform(Creator *creator) const {
21
22
  for (auto exception_node : m_exceptions)
22
23
  array.add_node(exception_node);
23
24
  if (m_name)
24
- array.add_node(name_to_node());
25
+ array.add_node(name_to_assignment());
25
26
  creator->append(array);
26
27
  if (m_body->nodes().is_empty())
27
28
  creator->append_nil();
@@ -0,0 +1,13 @@
1
+ #include "natalie_parser/node/for_node.hpp"
2
+
3
+ namespace NatalieParser {
4
+
5
+ void ForNode::transform(Creator *creator) const {
6
+ creator->set_type("for");
7
+ creator->append(m_expr);
8
+ creator->with_assignment(true, [&]() { creator->append(*m_vars); });
9
+ if (!m_body->is_empty())
10
+ creator->append(m_body->without_unnecessary_nesting());
11
+ }
12
+
13
+ }