parser 2.3.0.pre.2 → 2.3.0.pre.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +1 -1
- data/CHANGELOG.md +34 -0
- data/README.md +4 -2
- data/Rakefile +1 -0
- data/doc/AST_FORMAT.md +45 -3
- data/lib/parser/all.rb +2 -0
- data/lib/parser/ast/processor.rb +27 -1
- data/lib/parser/builders/default.rb +54 -6
- data/lib/parser/current.rb +9 -0
- data/lib/parser/lexer.rl +87 -64
- data/lib/parser/lexer/literal.rb +1 -1
- data/lib/parser/macruby.y +1 -1
- data/lib/parser/ruby18.y +1 -1
- data/lib/parser/ruby19.y +1 -1
- data/lib/parser/ruby20.y +1 -1
- data/lib/parser/ruby21.y +2 -1
- data/lib/parser/ruby22.y +7 -1
- data/lib/parser/ruby23.y +2363 -0
- data/lib/parser/rubymotion.y +1 -1
- data/lib/parser/runner.rb +7 -0
- data/lib/parser/source/buffer.rb +10 -6
- data/lib/parser/version.rb +1 -1
- data/parser.gemspec +2 -2
- data/test/parse_helper.rb +2 -2
- data/test/test_current.rb +1 -1
- data/test/test_lexer.rb +34 -0
- data/test/test_parser.rb +91 -17
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 374ddf2bd2cca89254c973f71c865f131ef55f17
|
4
|
+
data.tar.gz: 212ddd9876be32894d1523b25d3415087a09ee42
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b8fa11379a34cce56319589a6e8159919edf5583d41a602e5fdbfe6e4ddb7ae58ff3292552e4dd247ecf535387e58dc0d80695e4cebc422506ffc3d8ec1378e8
|
7
|
+
data.tar.gz: 764f0680aa9fdd6cf7dcf53b47cccea3f847e7f40930a2ce7d8a8b1ae835a54140a28e5f33ec31b5802a612cac909c02ed8bd8182c28c1c26f74042e8d99504c
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,40 @@
|
|
1
1
|
Changelog
|
2
2
|
=========
|
3
3
|
|
4
|
+
v2.3.0.pre.3 (2015-11-25)
|
5
|
+
-------------------------
|
6
|
+
|
7
|
+
API modifications:
|
8
|
+
* builders/default: introduce a (lambda) node (fixes #212). (whitequark)
|
9
|
+
* lexer.rl: do not override argument variable. (Keiji, Yoshimi)
|
10
|
+
* lexer.rl: rename variable names from lit to current_literal. (Keiji, Yoshimi)
|
11
|
+
* lexer.rl: use Regexp to match string. (Keiji, Yoshimi)
|
12
|
+
* lib/parser/source/buffer: reduce respond_to?(:bsearch) (Keiji, Yoshimi)
|
13
|
+
* lexer.rl: reduce String literal allocations (Keiji, Yoshimi)
|
14
|
+
* lexer.rl: reduce respond_to?(:encode) method call on #advance (Keiji, Yoshimi)
|
15
|
+
* lexer.rl: reduce Stirng.length method call on #advance (Keiji, Yoshimi)
|
16
|
+
* lexer.rl: reduce .class method call on #advance (Keiji, Yoshimi)
|
17
|
+
|
18
|
+
Features implemented:
|
19
|
+
* lexer.rl, ruby23.y: "a&.b": implement "safe navigation operator" (fixes #209). (whitequark)
|
20
|
+
* ruby23.y: fork grammar. (whitequark)
|
21
|
+
|
22
|
+
Bugs fixed:
|
23
|
+
* lexer.rl: never let EOF token location point past EOF. (whitequark)
|
24
|
+
|
25
|
+
v2.2.3.0 (2015-10-08)
|
26
|
+
---------------------
|
27
|
+
|
28
|
+
Bugs fixed:
|
29
|
+
* lexer.rl: "-> a: {}": state after -> is ENDFN, not END (fixes #203). (whitequark)
|
30
|
+
* ruby{21,22}.y: "p -> { :hello }, a: 1 do end": lexpop cmdarg. (whitequark)
|
31
|
+
|
32
|
+
v2.2.2.6 (2015-06-30)
|
33
|
+
---------------------
|
34
|
+
|
35
|
+
API modifications:
|
36
|
+
* parser/current: link to README from syntax deviation warning. (whitequark)
|
37
|
+
|
4
38
|
v2.3.0.pre.2 (2015-06-15)
|
5
39
|
-------------------------
|
6
40
|
|
data/README.md
CHANGED
@@ -5,8 +5,9 @@
|
|
5
5
|
[![Code Climate](https://codeclimate.com/github/whitequark/parser.png)](https://codeclimate.com/github/whitequark/parser)
|
6
6
|
[![Coverage Status](https://coveralls.io/repos/whitequark/parser/badge.png?branch=master)](https://coveralls.io/r/whitequark/parser)
|
7
7
|
|
8
|
-
_Parser_ is a production-ready Ruby parser written in pure Ruby. It
|
9
|
-
|
8
|
+
_Parser_ is a production-ready Ruby parser written in pure Ruby. It recognizes as
|
9
|
+
much or more code than Ripper, Melbourne, JRubyParser or ruby\_parser, and
|
10
|
+
is vastly more convenient to use.
|
10
11
|
|
11
12
|
You can also use [unparser](https://github.com/mbj/unparser) to produce
|
12
13
|
equivalent source code from Parser's ASTs.
|
@@ -23,6 +24,7 @@ MacRuby and RubyMotion support sponsored by [CodeClimate](http://codeclimate.com
|
|
23
24
|
Parse a chunk of code:
|
24
25
|
|
25
26
|
require 'parser/current'
|
27
|
+
Parser::Builders::Default.emit_lambda = true # opt-in to most recent AST format
|
26
28
|
|
27
29
|
p Parser::CurrentRuby.parse("2 + 2")
|
28
30
|
# (send
|
data/Rakefile
CHANGED
data/doc/AST_FORMAT.md
CHANGED
@@ -540,6 +540,31 @@ Format:
|
|
540
540
|
~~~~~~~ expression
|
541
541
|
~~~
|
542
542
|
|
543
|
+
### To attribute
|
544
|
+
|
545
|
+
Format:
|
546
|
+
|
547
|
+
~~~
|
548
|
+
(send (self) :foo= (int 1))
|
549
|
+
"self.foo = 1"
|
550
|
+
^ dot
|
551
|
+
~~~ selector
|
552
|
+
^ operator
|
553
|
+
~~~~~~~~~~~~ expression
|
554
|
+
~~~
|
555
|
+
|
556
|
+
### To attribute, using "safe navigation operator"
|
557
|
+
|
558
|
+
Format:
|
559
|
+
|
560
|
+
~~~
|
561
|
+
(csend (self) :foo= (int 1))
|
562
|
+
"self&.foo = 1"
|
563
|
+
^^ dot
|
564
|
+
~~~ selector
|
565
|
+
^ operator
|
566
|
+
~~~~~~~~~~~~~ expression
|
567
|
+
~~~
|
543
568
|
|
544
569
|
### Multiple assignment
|
545
570
|
|
@@ -1011,9 +1036,9 @@ Format:
|
|
1011
1036
|
~~~
|
1012
1037
|
(objc-restarg (objc-kwarg :foo))
|
1013
1038
|
"(*a: b)"
|
1014
|
-
~
|
1015
|
-
~
|
1016
|
-
~
|
1039
|
+
~ objc-kwarg.keyword
|
1040
|
+
~ objc-kwarg.operator
|
1041
|
+
~ objc-kwarg.argument
|
1017
1042
|
~ operator
|
1018
1043
|
~~~~~ expression
|
1019
1044
|
~~~
|
@@ -1053,6 +1078,7 @@ Format:
|
|
1053
1078
|
~~~
|
1054
1079
|
(send (lvar :foo) :bar (int 1))
|
1055
1080
|
"foo.bar(1)"
|
1081
|
+
^ dot
|
1056
1082
|
~~~ selector
|
1057
1083
|
^ begin
|
1058
1084
|
^ end
|
@@ -1146,6 +1172,22 @@ Used when passing expression as block `foo(&bar)`
|
|
1146
1172
|
~~~~ expression
|
1147
1173
|
~~~
|
1148
1174
|
|
1175
|
+
### "Stabby lambda"
|
1176
|
+
|
1177
|
+
~~~
|
1178
|
+
(block (lambda) (args) nil)
|
1179
|
+
"-> {}"
|
1180
|
+
~~ lambda.expression
|
1181
|
+
~~~
|
1182
|
+
|
1183
|
+
### "Safe navigation operator"
|
1184
|
+
|
1185
|
+
~~~
|
1186
|
+
(csend (send nil :foo) :bar)
|
1187
|
+
"foo&.bar"
|
1188
|
+
~~ dot
|
1189
|
+
~~~
|
1190
|
+
|
1149
1191
|
### Objective-C variadic send
|
1150
1192
|
|
1151
1193
|
MacRuby allows to pass a variadic amount of arguments via the last
|
data/lib/parser/all.rb
CHANGED
data/lib/parser/ast/processor.rb
CHANGED
@@ -24,6 +24,7 @@ module Parser
|
|
24
24
|
node
|
25
25
|
end
|
26
26
|
|
27
|
+
# @private
|
27
28
|
def process_variable_node(node)
|
28
29
|
on_var(node)
|
29
30
|
end
|
@@ -43,6 +44,7 @@ module Parser
|
|
43
44
|
])
|
44
45
|
end
|
45
46
|
|
47
|
+
# @private
|
46
48
|
def process_var_asgn_node(node)
|
47
49
|
on_vasgn(node)
|
48
50
|
end
|
@@ -92,6 +94,7 @@ module Parser
|
|
92
94
|
])
|
93
95
|
end
|
94
96
|
|
97
|
+
# @private
|
95
98
|
def process_argument_node(node)
|
96
99
|
on_argument(node)
|
97
100
|
end
|
@@ -185,7 +188,30 @@ module Parser
|
|
185
188
|
|
186
189
|
alias on_preexe process_regular_node
|
187
190
|
alias on_postexe process_regular_node
|
188
|
-
end
|
189
191
|
|
192
|
+
# @private
|
193
|
+
def process_variable_node(node)
|
194
|
+
warn 'Parser::AST::Processor#process_variable_node is deprecated as a' \
|
195
|
+
' public API and will be removed. Please use ' \
|
196
|
+
'Parser::AST::Processor#on_var instead.'
|
197
|
+
on_var(node)
|
198
|
+
end
|
199
|
+
|
200
|
+
# @private
|
201
|
+
def process_var_asgn_node(node)
|
202
|
+
warn 'Parser::AST::Processor#process_var_asgn_node is deprecated as a' \
|
203
|
+
' public API and will be removed. Please use ' \
|
204
|
+
'Parser::AST::Processor#on_vasgn instead.'
|
205
|
+
on_vasgn(node)
|
206
|
+
end
|
207
|
+
|
208
|
+
# @private
|
209
|
+
def process_argument_node(node)
|
210
|
+
warn 'Parser::AST::Processor#process_argument_node is deprecated as a' \
|
211
|
+
' public API and will be removed. Please use ' \
|
212
|
+
'Parser::AST::Processor#on_argument instead.'
|
213
|
+
on_argument(node)
|
214
|
+
end
|
215
|
+
end
|
190
216
|
end
|
191
217
|
end
|
@@ -4,6 +4,32 @@ module Parser
|
|
4
4
|
# Default AST builder. Uses {AST::Node}s.
|
5
5
|
#
|
6
6
|
class Builders::Default
|
7
|
+
class << self
|
8
|
+
##
|
9
|
+
# AST compatibility attribute; since `-> {}` is not semantically
|
10
|
+
# equivalent to `lambda {}`, all new code should set this attribute
|
11
|
+
# to true.
|
12
|
+
#
|
13
|
+
# If set to false (the default), `-> {}` is emitted as
|
14
|
+
# `s(:block, s(:send, nil, :lambda), s(:args), nil)`.
|
15
|
+
#
|
16
|
+
# If set to true, `-> {}` is emitted as
|
17
|
+
# `s(:block, s(:lambda), s(:args), nil)`.
|
18
|
+
#
|
19
|
+
# @return [Boolean]
|
20
|
+
attr_accessor :emit_lambda
|
21
|
+
end
|
22
|
+
|
23
|
+
@emit_lambda = false
|
24
|
+
|
25
|
+
class << self
|
26
|
+
##
|
27
|
+
# @api private
|
28
|
+
def modernize
|
29
|
+
@emit_lambda = true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
7
33
|
##
|
8
34
|
# @api private
|
9
35
|
attr_accessor :parser
|
@@ -650,20 +676,41 @@ module Parser
|
|
650
676
|
# Method calls
|
651
677
|
#
|
652
678
|
|
679
|
+
def call_type_for_dot(dot_t)
|
680
|
+
if !dot_t.nil? && value(dot_t) == :anddot
|
681
|
+
:csend
|
682
|
+
else
|
683
|
+
# This case is a bit tricky. ruby23.y returns the token tDOT with
|
684
|
+
# the value :dot, and the token :tANDDOT with the value :anddot.
|
685
|
+
#
|
686
|
+
# But, ruby{18..22}.y (which unconditionally expect tDOT) just
|
687
|
+
# return "." there, since they are to be kept close to the corresponding
|
688
|
+
# Ruby MRI grammars.
|
689
|
+
#
|
690
|
+
# Thankfully, we don't have to care.
|
691
|
+
:send
|
692
|
+
end
|
693
|
+
end
|
694
|
+
|
653
695
|
def call_method(receiver, dot_t, selector_t,
|
654
696
|
lparen_t=nil, args=[], rparen_t=nil)
|
697
|
+
type = call_type_for_dot(dot_t)
|
655
698
|
if selector_t.nil?
|
656
|
-
n(
|
699
|
+
n(type, [ receiver, :call, *args ],
|
657
700
|
send_map(receiver, dot_t, nil, lparen_t, args, rparen_t))
|
658
701
|
else
|
659
|
-
n(
|
702
|
+
n(type, [ receiver, value(selector_t).to_sym, *args ],
|
660
703
|
send_map(receiver, dot_t, selector_t, lparen_t, args, rparen_t))
|
661
704
|
end
|
662
705
|
end
|
663
706
|
|
664
707
|
def call_lambda(lambda_t)
|
665
|
-
|
666
|
-
|
708
|
+
if self.class.emit_lambda
|
709
|
+
n0(:lambda, expr_map(loc(lambda_t)))
|
710
|
+
else
|
711
|
+
n(:send, [ nil, :lambda ],
|
712
|
+
send_map(nil, nil, lambda_t))
|
713
|
+
end
|
667
714
|
end
|
668
715
|
|
669
716
|
def block(method_call, begin_t, args, body, end_t)
|
@@ -678,7 +725,7 @@ module Parser
|
|
678
725
|
diagnostic :error, :block_and_blockarg, nil, last_arg.loc.expression, [loc(begin_t)]
|
679
726
|
end
|
680
727
|
|
681
|
-
if [:send, :super, :zsuper].include?(method_call.type)
|
728
|
+
if [:send, :super, :zsuper, :lambda].include?(method_call.type)
|
682
729
|
n(:block, [ method_call, args, body ],
|
683
730
|
block_map(method_call.loc.expression, begin_t, end_t))
|
684
731
|
else
|
@@ -710,9 +757,10 @@ module Parser
|
|
710
757
|
|
711
758
|
def attr_asgn(receiver, dot_t, selector_t)
|
712
759
|
method_name = (value(selector_t) + '=').to_sym
|
760
|
+
type = call_type_for_dot(dot_t)
|
713
761
|
|
714
762
|
# Incomplete method call.
|
715
|
-
n(
|
763
|
+
n(type, [ receiver, method_name ],
|
716
764
|
send_map(receiver, dot_t, selector_t))
|
717
765
|
end
|
718
766
|
|
data/lib/parser/current.rb
CHANGED
@@ -3,6 +3,7 @@ module Parser
|
|
3
3
|
def warn_syntax_deviation(feature, version)
|
4
4
|
warn "warning: parser/current is loading #{feature}, which recognizes"
|
5
5
|
warn "warning: #{version}-compliant syntax, but you are running #{RUBY_VERSION}."
|
6
|
+
warn "warning: please see https://github.com/whitequark/parser#compatibility-with-ruby-mri."
|
6
7
|
end
|
7
8
|
private :warn_syntax_deviation
|
8
9
|
end
|
@@ -48,6 +49,14 @@ module Parser
|
|
48
49
|
require 'parser/ruby22'
|
49
50
|
CurrentRuby = Ruby22
|
50
51
|
|
52
|
+
when /^2\.3\./
|
53
|
+
if RUBY_VERSION != '2.3.0'
|
54
|
+
warn_syntax_deviation 'parser/ruby23', '2.3.0'
|
55
|
+
end
|
56
|
+
|
57
|
+
require 'parser/ruby23'
|
58
|
+
CurrentRuby = Ruby23
|
59
|
+
|
51
60
|
else # :nocov:
|
52
61
|
# Keep this in sync with released Ruby.
|
53
62
|
warn_syntax_deviation 'parser/ruby22', '2.2.x'
|
data/lib/parser/lexer.rl
CHANGED
@@ -87,6 +87,13 @@ class Parser::Lexer
|
|
87
87
|
'v' => "\v", '\\' => "\\"
|
88
88
|
}
|
89
89
|
|
90
|
+
BLANK_STRING = ''.freeze
|
91
|
+
ESCAPED_NEXT_LINE = "\\\n".freeze
|
92
|
+
REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
|
93
|
+
UNDERSCORE_STRING = '_'.freeze
|
94
|
+
|
95
|
+
RBRACE_OR_RPAREN = %w"} ]".freeze
|
96
|
+
|
90
97
|
attr_reader :source_buffer
|
91
98
|
attr_reader :encoding
|
92
99
|
|
@@ -105,6 +112,8 @@ class Parser::Lexer
|
|
105
112
|
@tokens = nil
|
106
113
|
@comments = nil
|
107
114
|
|
115
|
+
@has_encode = ''.respond_to?(:encode)
|
116
|
+
|
108
117
|
reset
|
109
118
|
end
|
110
119
|
|
@@ -125,6 +134,7 @@ class Parser::Lexer
|
|
125
134
|
@source = nil # source string
|
126
135
|
@source_pts = nil # @source as a codepoint array
|
127
136
|
@encoding = nil # target encoding for output strings
|
137
|
+
@need_encode = nil
|
128
138
|
|
129
139
|
@p = 0 # stream position (saved manually in #advance)
|
130
140
|
@ts = nil # token start
|
@@ -187,11 +197,12 @@ class Parser::Lexer
|
|
187
197
|
|
188
198
|
if defined?(Encoding) && @source.encoding == Encoding::UTF_8
|
189
199
|
@source_pts = @source.unpack('U*')
|
200
|
+
@need_encode = @has_encode && @encoding != Encoding::UTF_8
|
190
201
|
else
|
191
202
|
@source_pts = @source.unpack('C*')
|
192
203
|
end
|
193
204
|
|
194
|
-
if (@source_pts.size > 1_000_000 && @
|
205
|
+
if (@source_pts.size > 1_000_000 && @has_encode) ||
|
195
206
|
@force_utf32
|
196
207
|
# A heuristic: if the buffer is larger than 1M, then
|
197
208
|
# store it in UTF-32 and convert the tokens as they're
|
@@ -205,6 +216,7 @@ class Parser::Lexer
|
|
205
216
|
#
|
206
217
|
# Patches accepted.
|
207
218
|
@source = @source.encode(Encoding::UTF_32LE)
|
219
|
+
@need_encode = @has_encode && @encoding != Encoding::UTF_32LE
|
208
220
|
end
|
209
221
|
|
210
222
|
if @source_pts[0] == 0xfeff
|
@@ -270,20 +282,22 @@ class Parser::Lexer
|
|
270
282
|
end
|
271
283
|
|
272
284
|
# Ugly, but dependent on Ragel output. Consider refactoring it somehow.
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
285
|
+
klass = self.class
|
286
|
+
_lex_trans_keys = klass.send :_lex_trans_keys
|
287
|
+
_lex_key_spans = klass.send :_lex_key_spans
|
288
|
+
_lex_index_offsets = klass.send :_lex_index_offsets
|
289
|
+
_lex_indicies = klass.send :_lex_indicies
|
290
|
+
_lex_trans_targs = klass.send :_lex_trans_targs
|
291
|
+
_lex_trans_actions = klass.send :_lex_trans_actions
|
292
|
+
_lex_to_state_actions = klass.send :_lex_to_state_actions
|
293
|
+
_lex_from_state_actions = klass.send :_lex_from_state_actions
|
294
|
+
_lex_eof_trans = klass.send :_lex_eof_trans
|
295
|
+
|
296
|
+
pe = @source.length + 1
|
297
|
+
p, eof = @p, pe
|
298
|
+
|
299
|
+
@command_state = (@cs == klass.lex_en_expr_value ||
|
300
|
+
@cs == klass.lex_en_line_begin)
|
287
301
|
|
288
302
|
%% write exec;
|
289
303
|
# %
|
@@ -292,10 +306,11 @@ class Parser::Lexer
|
|
292
306
|
|
293
307
|
if @token_queue.any?
|
294
308
|
@token_queue.shift
|
295
|
-
elsif @cs ==
|
309
|
+
elsif @cs == klass.lex_error
|
296
310
|
[ false, [ '$error', range(p - 1, p) ] ]
|
297
311
|
else
|
298
|
-
|
312
|
+
eof = @source.length
|
313
|
+
[ false, [ '$eof', range(eof, eof) ] ]
|
299
314
|
end
|
300
315
|
end
|
301
316
|
|
@@ -320,7 +335,9 @@ class Parser::Lexer
|
|
320
335
|
end
|
321
336
|
|
322
337
|
def tok(s = @ts, e = @te)
|
323
|
-
@source[s...e]
|
338
|
+
source = @source[s...e]
|
339
|
+
return source unless @need_encode
|
340
|
+
source.encode(@encoding)
|
324
341
|
end
|
325
342
|
else
|
326
343
|
def encode_escape(ord)
|
@@ -444,7 +461,7 @@ class Parser::Lexer
|
|
444
461
|
'=>' => :tASSOC, '::' => :tCOLON2, '===' => :tEQQ,
|
445
462
|
'<=>' => :tCMP, '[]' => :tAREF, '[]=' => :tASET,
|
446
463
|
'{' => :tLCURLY, '}' => :tRCURLY, '`' => :tBACK_REF2,
|
447
|
-
'!@' => :tBANG,
|
464
|
+
'!@' => :tBANG, '&.' => :tANDDOT,
|
448
465
|
}
|
449
466
|
|
450
467
|
PUNCTUATION_BEGIN = {
|
@@ -828,15 +845,16 @@ class Parser::Lexer
|
|
828
845
|
|
829
846
|
action extend_string {
|
830
847
|
string = @source[@ts...@te]
|
831
|
-
string = string.encode(@encoding) if
|
848
|
+
string = string.encode(@encoding) if @need_encode
|
832
849
|
|
833
850
|
# tLABEL_END is only possible in non-cond context on >= 2.2
|
834
851
|
if @version >= 22 && !@cond.active?
|
835
852
|
lookahead = @source[@te...@te+2]
|
836
|
-
lookahead = lookahead.encode(@encoding) if
|
853
|
+
lookahead = lookahead.encode(@encoding) if @need_encode
|
837
854
|
end
|
838
855
|
|
839
|
-
|
856
|
+
current_literal = literal
|
857
|
+
if !current_literal.heredoc? && (token = current_literal.nest_and_try_closing(string, @ts, @te, lookahead))
|
840
858
|
if token[0] == :tLABEL_END
|
841
859
|
p += 1
|
842
860
|
pop_literal
|
@@ -844,14 +862,15 @@ class Parser::Lexer
|
|
844
862
|
else
|
845
863
|
fnext *pop_literal;
|
846
864
|
end
|
847
|
-
|
865
|
+
fbreak;
|
848
866
|
else
|
849
|
-
|
867
|
+
current_literal.extend_string(string, @ts, @te)
|
850
868
|
end
|
851
869
|
}
|
852
870
|
|
853
871
|
action extend_string_escaped {
|
854
|
-
|
872
|
+
current_literal = literal
|
873
|
+
if current_literal.nest_and_try_closing('\\', @ts, @ts + 1)
|
855
874
|
# If the literal is actually closed by the backslash,
|
856
875
|
# rewind the input prior to consuming the escape sequence.
|
857
876
|
p = @escape_s - 1
|
@@ -860,12 +879,12 @@ class Parser::Lexer
|
|
860
879
|
# Get the first character after the backslash.
|
861
880
|
escaped_char = @source[@escape_s].chr
|
862
881
|
|
863
|
-
if
|
882
|
+
if current_literal.munge_escape? escaped_char
|
864
883
|
# If this particular literal uses this character as an opening
|
865
884
|
# or closing delimiter, it is an escape sequence for that
|
866
885
|
# particular character. Write it without the backslash.
|
867
886
|
|
868
|
-
if
|
887
|
+
if current_literal.regexp? && REGEXP_META_CHARACTERS.match(escaped_char)
|
869
888
|
# Regular expressions should include escaped delimiters in their
|
870
889
|
# escaped form, except when the escaped character is
|
871
890
|
# a closing delimiter but not a regexp metacharacter.
|
@@ -874,9 +893,9 @@ class Parser::Lexer
|
|
874
893
|
# at the same time as an escape symbol, but it is always munged,
|
875
894
|
# so this branch also executes for the non-closing-delimiter case
|
876
895
|
# for the backslash.
|
877
|
-
|
896
|
+
current_literal.extend_string(tok, @ts, @te)
|
878
897
|
else
|
879
|
-
|
898
|
+
current_literal.extend_string(escaped_char, @ts, @te)
|
880
899
|
end
|
881
900
|
else
|
882
901
|
# It does not. So this is an actual escape sequence, yay!
|
@@ -891,12 +910,12 @@ class Parser::Lexer
|
|
891
910
|
|
892
911
|
@escape.call if @escape.respond_to? :call
|
893
912
|
|
894
|
-
if
|
913
|
+
if current_literal.regexp?
|
895
914
|
# Regular expressions should include escape sequences in their
|
896
915
|
# escaped form. On the other hand, escaped newlines are removed.
|
897
|
-
|
916
|
+
current_literal.extend_string(tok.gsub(ESCAPED_NEXT_LINE, BLANK_STRING), @ts, @te)
|
898
917
|
else
|
899
|
-
|
918
|
+
current_literal.extend_string(@escape || tok, @ts, @te)
|
900
919
|
end
|
901
920
|
end
|
902
921
|
end
|
@@ -906,27 +925,28 @@ class Parser::Lexer
|
|
906
925
|
# As heredoc closing line can immediately precede EOF, this action
|
907
926
|
# has to handle such case specially.
|
908
927
|
action extend_string_eol {
|
928
|
+
current_literal = literal
|
909
929
|
if @te == pe
|
910
930
|
diagnostic :fatal, :string_eof, nil,
|
911
|
-
range(
|
931
|
+
range(current_literal.str_s, current_literal.str_s + 1)
|
912
932
|
end
|
913
933
|
|
914
|
-
if
|
915
|
-
line = tok(@herebody_s, @ts).gsub(/\r+$/,
|
934
|
+
if current_literal.heredoc?
|
935
|
+
line = tok(@herebody_s, @ts).gsub(/\r+$/, BLANK_STRING)
|
916
936
|
|
917
937
|
if version?(18, 19, 20)
|
918
938
|
# See ruby:c48b4209c
|
919
|
-
line = line.gsub(/\r.*$/,
|
939
|
+
line = line.gsub(/\r.*$/, BLANK_STRING)
|
920
940
|
end
|
921
941
|
|
922
942
|
# Try ending the heredoc with the complete most recently
|
923
943
|
# scanned line. @herebody_s always refers to the start of such line.
|
924
|
-
if
|
944
|
+
if current_literal.nest_and_try_closing(line, @herebody_s, @ts)
|
925
945
|
# Adjust @herebody_s to point to the next line.
|
926
946
|
@herebody_s = @te
|
927
947
|
|
928
948
|
# Continue regular lexing after the heredoc reference (<<END).
|
929
|
-
p =
|
949
|
+
p = current_literal.heredoc_e - 1
|
930
950
|
fnext *pop_literal; fbreak;
|
931
951
|
else
|
932
952
|
# Ditto.
|
@@ -934,7 +954,7 @@ class Parser::Lexer
|
|
934
954
|
end
|
935
955
|
else
|
936
956
|
# Try ending the literal with a newline.
|
937
|
-
if
|
957
|
+
if current_literal.nest_and_try_closing(tok, @ts, @te)
|
938
958
|
fnext *pop_literal; fbreak;
|
939
959
|
end
|
940
960
|
|
@@ -952,14 +972,14 @@ class Parser::Lexer
|
|
952
972
|
end
|
953
973
|
end
|
954
974
|
|
955
|
-
if
|
956
|
-
|
975
|
+
if current_literal.words? && !eof_codepoint?(@source_pts[p])
|
976
|
+
current_literal.extend_space @ts, @te
|
957
977
|
else
|
958
978
|
# A literal newline is appended if the heredoc was _not_ closed
|
959
979
|
# this time (see fbreak above). See also Literal#nest_and_try_closing
|
960
980
|
# for rationale of calling #flush_string here.
|
961
|
-
|
962
|
-
|
981
|
+
current_literal.extend_string tok, @ts, @te
|
982
|
+
current_literal.flush_string
|
963
983
|
end
|
964
984
|
}
|
965
985
|
|
@@ -977,8 +997,9 @@ class Parser::Lexer
|
|
977
997
|
interp_var = '#' ( global_var | class_var_v | instance_var_v );
|
978
998
|
|
979
999
|
action extend_interp_var {
|
980
|
-
literal
|
981
|
-
|
1000
|
+
current_literal = literal
|
1001
|
+
current_literal.flush_string
|
1002
|
+
current_literal.extend_content
|
982
1003
|
|
983
1004
|
emit(:tSTRING_DVAR, nil, @ts, @ts + 1)
|
984
1005
|
|
@@ -1003,22 +1024,24 @@ class Parser::Lexer
|
|
1003
1024
|
e_lbrace = '{' % {
|
1004
1025
|
@cond.push(false); @cmdarg.push(false)
|
1005
1026
|
|
1006
|
-
|
1007
|
-
|
1027
|
+
current_literal = literal
|
1028
|
+
if current_literal
|
1029
|
+
current_literal.start_interp_brace
|
1008
1030
|
end
|
1009
1031
|
};
|
1010
1032
|
|
1011
1033
|
e_rbrace = '}' % {
|
1012
|
-
|
1013
|
-
|
1034
|
+
current_literal = literal
|
1035
|
+
if current_literal
|
1036
|
+
if current_literal.end_interp_brace_and_try_closing
|
1014
1037
|
if version?(18, 19)
|
1015
1038
|
emit(:tRCURLY, '}', p - 1, p)
|
1016
1039
|
else
|
1017
1040
|
emit(:tSTRING_DEND, '}', p - 1, p)
|
1018
1041
|
end
|
1019
1042
|
|
1020
|
-
if
|
1021
|
-
@herebody_s =
|
1043
|
+
if current_literal.saved_herebody_s
|
1044
|
+
@herebody_s = current_literal.saved_herebody_s
|
1022
1045
|
end
|
1023
1046
|
|
1024
1047
|
fhold;
|
@@ -1029,17 +1052,18 @@ class Parser::Lexer
|
|
1029
1052
|
};
|
1030
1053
|
|
1031
1054
|
action extend_interp_code {
|
1032
|
-
literal
|
1033
|
-
|
1055
|
+
current_literal = literal
|
1056
|
+
current_literal.flush_string
|
1057
|
+
current_literal.extend_content
|
1034
1058
|
|
1035
1059
|
emit(:tSTRING_DBEG, '#{')
|
1036
1060
|
|
1037
|
-
if
|
1038
|
-
|
1061
|
+
if current_literal.heredoc?
|
1062
|
+
current_literal.saved_herebody_s = @herebody_s
|
1039
1063
|
@herebody_s = nil
|
1040
1064
|
end
|
1041
1065
|
|
1042
|
-
|
1066
|
+
current_literal.start_interp_brace
|
1043
1067
|
fcall expr_value;
|
1044
1068
|
}
|
1045
1069
|
|
@@ -1891,7 +1915,7 @@ class Parser::Lexer
|
|
1891
1915
|
emit_table(PUNCTUATION, @ts, @ts + 2)
|
1892
1916
|
|
1893
1917
|
@lambda_stack.push @paren_nest
|
1894
|
-
fbreak;
|
1918
|
+
fnext expr_endfn; fbreak;
|
1895
1919
|
};
|
1896
1920
|
|
1897
1921
|
e_lbrace | 'do'
|
@@ -1987,8 +2011,8 @@ class Parser::Lexer
|
|
1987
2011
|
=> {
|
1988
2012
|
digits = tok(@num_digits_s, @num_suffix_s)
|
1989
2013
|
|
1990
|
-
if digits.end_with?
|
1991
|
-
diagnostic :error, :trailing_in_number, { :character =>
|
2014
|
+
if digits.end_with? UNDERSCORE_STRING
|
2015
|
+
diagnostic :error, :trailing_in_number, { :character => UNDERSCORE_STRING },
|
1992
2016
|
range(@te - 1, @te)
|
1993
2017
|
elsif digits.empty? && @num_base == 8 && version?(18)
|
1994
2018
|
# 1.8 did not raise an error on 0o.
|
@@ -2085,7 +2109,7 @@ class Parser::Lexer
|
|
2085
2109
|
# METHOD CALLS
|
2086
2110
|
#
|
2087
2111
|
|
2088
|
-
'.' | '::'
|
2112
|
+
'.' | '&.' | '::'
|
2089
2113
|
=> { emit_table(PUNCTUATION)
|
2090
2114
|
fnext expr_dot; fbreak; };
|
2091
2115
|
|
@@ -2121,7 +2145,7 @@ class Parser::Lexer
|
|
2121
2145
|
emit_table(PUNCTUATION)
|
2122
2146
|
@cond.lexpop; @cmdarg.lexpop
|
2123
2147
|
|
2124
|
-
if
|
2148
|
+
if RBRACE_OR_RPAREN.include?(tok)
|
2125
2149
|
fnext expr_endarg;
|
2126
2150
|
else # )
|
2127
2151
|
# fnext expr_endfn; ?
|
@@ -2176,9 +2200,8 @@ class Parser::Lexer
|
|
2176
2200
|
# Insane leading dots:
|
2177
2201
|
# a #comment
|
2178
2202
|
# .b: a.b
|
2179
|
-
c_space*
|
2180
|
-
=> {
|
2181
|
-
fgoto expr_end; };
|
2203
|
+
c_space* %{ tm = p } ('.' | '&.')
|
2204
|
+
=> { p = tm - 1; fgoto expr_end; };
|
2182
2205
|
|
2183
2206
|
any
|
2184
2207
|
=> { emit(:tNL, nil, @newline_s, @newline_s + 1)
|