parser 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -7
- data/.gitignore +1 -0
- data/README.md +1 -1
- data/Rakefile +12 -2
- data/bin/benchmark +3 -3
- data/lib/parser/builders/default.rb +46 -3
- data/lib/parser/current.rb +34 -0
- data/lib/parser/lexer.rb +2760 -2667
- data/lib/parser/lexer.rl +38 -14
- data/lib/parser/ruby18.rb +1956 -1957
- data/lib/parser/ruby19.rb +1962 -1969
- data/lib/parser/ruby19.y +2 -0
- data/lib/parser/ruby20.rb +2412 -2407
- data/lib/parser/ruby20.y +16 -2
- data/lib/parser/ruby21.rb +7310 -0
- data/lib/parser/ruby21.y +2352 -0
- data/lib/parser/source/buffer.rb +52 -2
- data/lib/parser.rb +16 -5
- data/parser.gemspec +2 -1
- data/test/helper.rb +1 -1
- data/test/parse_helper.rb +12 -4
- data/test/test_current.rb +17 -0
- data/test/test_encoding.rb +31 -0
- data/test/test_lexer.rb +5 -7
- data/test/test_parse_helper.rb +9 -4
- data/test/test_parser.rb +172 -10
- metadata +93 -125
- data/TODO.md +0 -9
data/lib/parser/lexer.rl
CHANGED
@@ -158,14 +158,15 @@ class Parser::Lexer
|
|
158
158
|
|
159
159
|
LEX_STATES = {
|
160
160
|
:line_begin => lex_en_line_begin,
|
161
|
-
:expr_beg => lex_en_expr_beg,
|
162
|
-
:expr_value => lex_en_expr_value,
|
163
|
-
:expr_mid => lex_en_expr_mid,
|
164
161
|
:expr_dot => lex_en_expr_dot,
|
165
162
|
:expr_fname => lex_en_expr_fname,
|
166
|
-
:
|
163
|
+
:expr_value => lex_en_expr_value,
|
164
|
+
:expr_beg => lex_en_expr_beg,
|
165
|
+
:expr_mid => lex_en_expr_mid,
|
167
166
|
:expr_arg => lex_en_expr_arg,
|
167
|
+
:expr_end => lex_en_expr_end,
|
168
168
|
:expr_endarg => lex_en_expr_endarg,
|
169
|
+
:expr_endfn => lex_en_expr_endfn,
|
169
170
|
}
|
170
171
|
|
171
172
|
def state
|
@@ -371,8 +372,8 @@ class Parser::Lexer
|
|
371
372
|
c_line = c_any - c_nl;
|
372
373
|
|
373
374
|
c_unicode = c_any - 0x00..0x7f;
|
375
|
+
c_upper = [A-Z];
|
374
376
|
c_lower = [a-z_] | c_unicode;
|
375
|
-
c_upper = [A-Z] | c_unicode;
|
376
377
|
c_alpha = c_lower | c_upper;
|
377
378
|
c_alnum = c_alpha | [0-9];
|
378
379
|
|
@@ -451,7 +452,7 @@ class Parser::Lexer
|
|
451
452
|
keyword_with_end | keyword_with_arg |
|
452
453
|
keyword_with_fname | keyword_modifier ;
|
453
454
|
|
454
|
-
constant =
|
455
|
+
constant = c_upper c_alnum*;
|
455
456
|
bareword = c_alpha c_alnum*;
|
456
457
|
|
457
458
|
call_or_var = c_lower c_alnum*;
|
@@ -984,30 +985,33 @@ class Parser::Lexer
|
|
984
985
|
# statements `alias $a $b`. Symbols are returned verbatim; this
|
985
986
|
# is used in `alias :a :"b#{foo}"` and `undef :a`.
|
986
987
|
#
|
987
|
-
# Transitions to `
|
988
|
+
# Transitions to `expr_endfn` afterwards.
|
988
989
|
#
|
989
990
|
expr_fname := |*
|
990
991
|
keyword
|
991
992
|
=> { emit(KEYWORDS[tok]);
|
992
|
-
fnext
|
993
|
+
fnext expr_endfn; fbreak; };
|
993
994
|
|
994
995
|
bareword [?=!]?
|
995
996
|
=> { emit(:tIDENTIFIER)
|
996
|
-
fnext
|
997
|
+
fnext expr_endfn; fbreak; };
|
997
998
|
|
999
|
+
global_var
|
1000
|
+
=> { p = @ts - 1
|
1001
|
+
fcall expr_variable; };
|
1002
|
+
|
1003
|
+
# If the handling was to be delegated to expr_end,
|
1004
|
+
# these cases would transition to something else than
|
1005
|
+
# expr_end, which is undesirable.
|
998
1006
|
operator_fname |
|
999
1007
|
operator_arithmetic |
|
1000
1008
|
operator_rest
|
1001
1009
|
=> { emit_table(PUNCTUATION)
|
1002
|
-
fnext
|
1010
|
+
fnext expr_endfn; fbreak; };
|
1003
1011
|
|
1004
1012
|
':'
|
1005
1013
|
=> { fhold; fgoto expr_beg; };
|
1006
1014
|
|
1007
|
-
global_var
|
1008
|
-
=> { p = @ts - 1
|
1009
|
-
fcall expr_variable; };
|
1010
|
-
|
1011
1015
|
# TODO whitespace rule
|
1012
1016
|
c_space;
|
1013
1017
|
e_heredoc_nl;
|
@@ -1018,6 +1022,25 @@ class Parser::Lexer
|
|
1018
1022
|
c_eof => do_eof;
|
1019
1023
|
*|;
|
1020
1024
|
|
1025
|
+
# After literal function name in definition. Behaves like `expr_end`,
|
1026
|
+
# but allows a tLABEL.
|
1027
|
+
#
|
1028
|
+
# Transitions to `expr_end` afterwards.
|
1029
|
+
#
|
1030
|
+
expr_endfn := |*
|
1031
|
+
bareword ':'
|
1032
|
+
=> { emit(:tLABEL, tok(@ts, @te - 1))
|
1033
|
+
fnext expr_end; fbreak; };
|
1034
|
+
|
1035
|
+
# TODO whitespace rule
|
1036
|
+
c_space;
|
1037
|
+
|
1038
|
+
c_any
|
1039
|
+
=> { fhold; fgoto expr_end; };
|
1040
|
+
|
1041
|
+
c_eof => do_eof;
|
1042
|
+
*|;
|
1043
|
+
|
1021
1044
|
# Literal function name in method call (e.g. `a.class`).
|
1022
1045
|
#
|
1023
1046
|
# Transitions to `expr_arg` afterwards.
|
@@ -1035,6 +1058,7 @@ class Parser::Lexer
|
|
1035
1058
|
=> { emit(:tFID, tok(@ts, tm), @ts, tm)
|
1036
1059
|
fnext expr_arg; p = tm - 1; fbreak; };
|
1037
1060
|
|
1061
|
+
# See the comment in `expr_fname`.
|
1038
1062
|
operator_fname |
|
1039
1063
|
operator_arithmetic |
|
1040
1064
|
operator_rest
|