parser 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/parser/lexer.rl CHANGED
@@ -265,14 +265,18 @@ class Parser::Lexer
265
265
  new_literal = Literal.new(self, *args)
266
266
  @literal_stack.push(new_literal)
267
267
 
268
- if new_literal.type == :tWORDS_BEG
269
- self.class.lex_en_interp_words
270
- elsif new_literal.type == :tQWORDS_BEG
271
- self.class.lex_en_plain_words
272
- elsif new_literal.interpolate?
273
- self.class.lex_en_interp_string
268
+ if new_literal.words?
269
+ if new_literal.interpolate?
270
+ self.class.lex_en_interp_words
271
+ else
272
+ self.class.lex_en_plain_words
273
+ end
274
274
  else
275
- self.class.lex_en_plain_string
275
+ if new_literal.interpolate?
276
+ self.class.lex_en_interp_string
277
+ else
278
+ self.class.lex_en_plain_string
279
+ end
276
280
  end
277
281
  end
278
282
 
@@ -313,9 +317,9 @@ class Parser::Lexer
313
317
  }
314
318
 
315
319
  PUNCTUATION_BEGIN = {
316
- '&' => :tAMPER, '*' => :tSTAR, '+' => :tUPLUS,
317
- '-' => :tUMINUS, '::' => :tCOLON3, '(' => :tLPAREN,
318
- '{' => :tLBRACE, '[' => :tLBRACK,
320
+ '&' => :tAMPER, '*' => :tSTAR, '**' => :tDSTAR,
321
+ '+' => :tUPLUS, '-' => :tUMINUS, '::' => :tCOLON3,
322
+ '(' => :tLPAREN, '{' => :tLBRACE, '[' => :tLBRACK,
319
323
  }
320
324
 
321
325
  KEYWORDS = {
@@ -410,7 +414,8 @@ class Parser::Lexer
410
414
 
411
415
  # A list of punctuation which has different meaning when used at the
412
416
  # beginning of expression.
413
- punctuation_begin = '-' | '+' | '::' | '(' | '[' | '*' | '&' ;
417
+ punctuation_begin = '-' | '+' | '::' | '(' | '[' |
418
+ '*' | '**' | '&' ;
414
419
 
415
420
  # A list of all punctuation except punctuation_begin.
416
421
  punctuation_end = ',' | '=' | '->' | '(' | '[' | ']' |
@@ -748,6 +753,10 @@ class Parser::Lexer
748
753
  literal.flush_string
749
754
  }
750
755
 
756
+ action extend_string_space {
757
+ literal.extend_space @ts, @te
758
+ }
759
+
751
760
  #
752
761
  # === INTERPOLATION PARSING ===
753
762
  #
@@ -759,6 +768,8 @@ class Parser::Lexer
759
768
 
760
769
  action extend_interp_var {
761
770
  literal.flush_string
771
+ literal.extend_content
772
+
762
773
  emit(:tSTRING_DVAR, nil, @ts, @ts + 1)
763
774
 
764
775
  p = @ts
@@ -790,10 +801,10 @@ class Parser::Lexer
790
801
  e_rbrace = '}' % {
791
802
  if literal
792
803
  if literal.end_interp_brace_and_try_closing
793
- emit(:tRCURLY, '}')
794
-
795
- if literal.words?
796
- emit(:tSPACE, nil)
804
+ if version?(18, 19)
805
+ emit(:tRCURLY, '}')
806
+ else
807
+ emit(:tSTRING_DEND, '}')
797
808
  end
798
809
 
799
810
  if literal.saved_herebody_s
@@ -809,6 +820,8 @@ class Parser::Lexer
809
820
 
810
821
  action extend_interp_code {
811
822
  literal.flush_string
823
+ literal.extend_content
824
+
812
825
  emit(:tSTRING_DBEG, '#{')
813
826
 
814
827
  literal.saved_herebody_s = @herebody_s
@@ -825,7 +838,7 @@ class Parser::Lexer
825
838
  interp_code => extend_interp_code;
826
839
  interp_var => extend_interp_var;
827
840
  e_bs escape => extend_string_escaped;
828
- c_space_nl => { literal.flush_string };
841
+ c_space_nl+ => extend_string_space;
829
842
  c_eol => extend_string_eol;
830
843
  c_any => extend_string;
831
844
  *|;
@@ -840,7 +853,7 @@ class Parser::Lexer
840
853
 
841
854
  plain_words := |*
842
855
  e_bs c_any => extend_string_escaped;
843
- c_space_nl => { literal.flush_string };
856
+ c_space_nl+ => extend_string_space;
844
857
  c_eol => extend_string_eol;
845
858
  c_any => extend_string;
846
859
  *|;
@@ -999,7 +1012,9 @@ class Parser::Lexer
999
1012
  => { p = @ts - 1
1000
1013
  fcall expr_variable; };
1001
1014
 
1002
- c_space_nl+;
1015
+ # TODO whitespace rule
1016
+ c_space;
1017
+ e_heredoc_nl;
1003
1018
 
1004
1019
  c_any
1005
1020
  => { fhold; fgoto expr_end; };
@@ -1030,9 +1045,11 @@ class Parser::Lexer
1030
1045
  => { emit_table(PUNCTUATION)
1031
1046
  fnext expr_arg; fbreak; };
1032
1047
 
1033
- c_space_nl+;
1034
-
1035
- '#' c_line* c_nl;
1048
+ # TODO whitespace rule
1049
+ c_space;
1050
+ e_heredoc_nl;
1051
+ '\\' e_heredoc_nl;
1052
+ '#' c_line*;
1036
1053
 
1037
1054
  c_any
1038
1055
  => { fhold; fgoto expr_end; };
@@ -1108,12 +1125,12 @@ class Parser::Lexer
1108
1125
  };
1109
1126
 
1110
1127
  # x *1
1111
- # Ambiguous splat or block-pass.
1112
- c_space+ [*&]
1128
+ # Ambiguous splat, kwsplat or block-pass.
1129
+ c_space+ %{ tm = p } ( '*' | '&' | '**' )
1113
1130
  => {
1114
- message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(@te - 1, @te) }
1131
+ message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(tm, @te) }
1115
1132
  diagnostic :warning, message,
1116
- range(@te - 1, @te)
1133
+ range(tm, @te)
1117
1134
 
1118
1135
  fhold; fgoto expr_beg;
1119
1136
  };
@@ -1182,7 +1199,7 @@ class Parser::Lexer
1182
1199
  => { emit(:kDO_BLOCK)
1183
1200
  fnext expr_value; };
1184
1201
 
1185
- c_space*;
1202
+ c_space;
1186
1203
 
1187
1204
  c_any
1188
1205
  => { fhold; fgoto expr_end; };
@@ -1200,11 +1217,11 @@ class Parser::Lexer
1200
1217
  => { emit_table(KEYWORDS)
1201
1218
  fnext expr_beg; fbreak; };
1202
1219
 
1203
- c_space+;
1204
-
1220
+ # TODO whitespace rule
1221
+ c_space;
1205
1222
  '#' c_line*;
1206
1223
 
1207
- c_nl
1224
+ e_heredoc_nl
1208
1225
  => { fhold; fgoto expr_end; };
1209
1226
 
1210
1227
  c_any
@@ -1430,7 +1447,8 @@ class Parser::Lexer
1430
1447
  => {
1431
1448
  emit(:tIDENTIFIER)
1432
1449
 
1433
- if @static_env && @static_env.declared?(tok)
1450
+ if !version?(18) &&
1451
+ !@static_env.nil? && @static_env.declared?(tok)
1434
1452
  fnext expr_end; fbreak;
1435
1453
  else
1436
1454
  fnext expr_arg; fbreak;
@@ -1441,15 +1459,16 @@ class Parser::Lexer
1441
1459
  # WHITESPACE
1442
1460
  #
1443
1461
 
1444
- c_space_nl;
1445
-
1446
- '\\\n';
1462
+ # TODO whitespace rule
1463
+ c_space;
1464
+ e_heredoc_nl;
1465
+ '\\' e_heredoc_nl;
1447
1466
 
1448
1467
  '#' c_line* c_eol
1449
1468
  => { @comments << tok
1450
1469
  fhold; };
1451
1470
 
1452
- c_nl '=begin' ( c_space | c_eol )
1471
+ e_heredoc_nl '=begin' ( c_space | c_eol )
1453
1472
  => { p = @ts - 1
1454
1473
  fgoto line_begin; };
1455
1474
 
@@ -1476,7 +1495,9 @@ class Parser::Lexer
1476
1495
  => { p = @ts - 1
1477
1496
  fgoto expr_end; };
1478
1497
 
1479
- c_space_nl+;
1498
+ # TODO whitespace rule
1499
+ c_space;
1500
+ e_heredoc_nl;
1480
1501
 
1481
1502
  c_any
1482
1503
  => { fhold; fgoto expr_beg; };
@@ -1489,9 +1510,9 @@ class Parser::Lexer
1489
1510
  # STABBY LAMBDA
1490
1511
  #
1491
1512
 
1492
- '->'
1513
+ '->' c_space*
1493
1514
  => {
1494
- emit_table(PUNCTUATION)
1515
+ emit_table(PUNCTUATION, @ts, @ts + 2)
1495
1516
 
1496
1517
  @lambda_stack.push @paren_nest
1497
1518
  fbreak;
@@ -1535,7 +1556,7 @@ class Parser::Lexer
1535
1556
  'class' c_space_nl* '<<'
1536
1557
  => { emit(:kCLASS, 'class', @ts, @ts + 5)
1537
1558
  emit(:tLSHFT, '<<', @te - 2, @te)
1538
- fnext expr_beg; fbreak; };
1559
+ fnext expr_value; fbreak; };
1539
1560
 
1540
1561
  # a if b:c: Syntax error.
1541
1562
  keyword_modifier
@@ -1717,6 +1738,7 @@ class Parser::Lexer
1717
1738
  # WHITESPACE
1718
1739
  #
1719
1740
 
1741
+ # TODO whitespace rule
1720
1742
  '\\' e_heredoc_nl;
1721
1743
 
1722
1744
  '\\' c_line {
@@ -1779,6 +1801,7 @@ class Parser::Lexer
1779
1801
  *|;
1780
1802
 
1781
1803
  line_begin := |*
1804
+ # TODO whitespace rule
1782
1805
  c_space_nl+;
1783
1806
 
1784
1807
  '#' c_line* c_eol