prism 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -1
- data/Makefile +2 -1
- data/README.md +1 -0
- data/config.yml +264 -37
- data/docs/parser_translation.md +8 -23
- data/docs/ripper_translation.md +1 -1
- data/ext/prism/api_node.c +2 -0
- data/ext/prism/extension.c +14 -1
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +275 -49
- data/include/prism/diagnostic.h +4 -0
- data/include/prism/options.h +43 -3
- data/include/prism/regexp.h +2 -2
- data/include/prism/util/pm_buffer.h +8 -0
- data/include/prism/util/pm_integer.h +4 -0
- data/include/prism/util/pm_list.h +6 -0
- data/include/prism/util/pm_string.h +12 -2
- data/include/prism/version.h +2 -2
- data/include/prism.h +39 -14
- data/lib/prism/compiler.rb +456 -151
- data/lib/prism/desugar_compiler.rb +1 -0
- data/lib/prism/dispatcher.rb +16 -0
- data/lib/prism/dot_visitor.rb +5 -1
- data/lib/prism/dsl.rb +3 -0
- data/lib/prism/ffi.rb +17 -7
- data/lib/prism/inspect_visitor.rb +3 -0
- data/lib/prism/lex_compat.rb +1 -0
- data/lib/prism/mutation_compiler.rb +3 -0
- data/lib/prism/node.rb +506 -335
- data/lib/prism/node_ext.rb +4 -1
- data/lib/prism/pack.rb +2 -0
- data/lib/prism/parse_result/comments.rb +1 -0
- data/lib/prism/parse_result/errors.rb +1 -0
- data/lib/prism/parse_result/newlines.rb +1 -0
- data/lib/prism/parse_result.rb +1 -0
- data/lib/prism/pattern.rb +1 -0
- data/lib/prism/polyfill/scan_byte.rb +14 -0
- data/lib/prism/polyfill/warn.rb +42 -0
- data/lib/prism/reflection.rb +3 -0
- data/lib/prism/relocation.rb +1 -0
- data/lib/prism/serialize.rb +24 -19
- data/lib/prism/string_query.rb +1 -0
- data/lib/prism/translation/parser/builder.rb +1 -0
- data/lib/prism/translation/parser/compiler.rb +47 -25
- data/lib/prism/translation/parser/lexer.rb +29 -21
- data/lib/prism/translation/parser.rb +13 -1
- data/lib/prism/translation/parser33.rb +1 -0
- data/lib/prism/translation/parser34.rb +1 -0
- data/lib/prism/translation/parser35.rb +1 -0
- data/lib/prism/translation/parser_current.rb +24 -0
- data/lib/prism/translation/ripper/sexp.rb +1 -0
- data/lib/prism/translation/ripper.rb +17 -1
- data/lib/prism/translation/ruby_parser.rb +286 -3
- data/lib/prism/translation.rb +2 -0
- data/lib/prism/visitor.rb +457 -152
- data/lib/prism.rb +2 -0
- data/prism.gemspec +5 -1
- data/rbi/prism/dsl.rbi +3 -3
- data/rbi/prism/node.rbi +21 -9
- data/sig/prism/dispatcher.rbs +3 -0
- data/sig/prism/dsl.rbs +3 -3
- data/sig/prism/node.rbs +444 -30
- data/sig/prism/node_ext.rbs +84 -17
- data/sig/prism/parse_result/comments.rbs +38 -0
- data/sig/prism/parse_result.rbs +4 -0
- data/sig/prism/reflection.rbs +1 -1
- data/src/diagnostic.c +7 -1
- data/src/node.c +2 -0
- data/src/options.c +2 -2
- data/src/prettyprint.c +2 -0
- data/src/prism.c +252 -130
- data/src/serialize.c +2 -0
- data/src/token_type.c +36 -34
- metadata +6 -2
data/lib/prism/node_ext.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
4
|
+
#--
|
3
5
|
# Here we are reopening the prism module to provide methods on nodes that aren't
|
4
6
|
# templated and are meant as convenience methods.
|
7
|
+
#++
|
5
8
|
module Prism
|
6
9
|
class Node
|
7
10
|
def deprecated(*replacements) # :nodoc:
|
@@ -9,7 +12,7 @@ module Prism
|
|
9
12
|
location = location[0].label if location
|
10
13
|
suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
|
11
14
|
|
12
|
-
warn(<<~MSG, category: :deprecated)
|
15
|
+
warn(<<~MSG, uplevel: 1, category: :deprecated)
|
13
16
|
[deprecation]: #{self.class}##{location} is deprecated and will be \
|
14
17
|
removed in the next major version. Use #{suggest.join("/")} instead.
|
15
18
|
#{(caller(1, 3) || []).join("\n")}
|
data/lib/prism/pack.rb
CHANGED
data/lib/prism/parse_result.rb
CHANGED
data/lib/prism/pattern.rb
CHANGED
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "strscan"
|
4
|
+
|
5
|
+
# Polyfill for StringScanner#scan_byte, which didn't exist until Ruby 3.4.
|
6
|
+
if !(StringScanner.instance_methods.include?(:scan_byte))
|
7
|
+
StringScanner.include(
|
8
|
+
Module.new {
|
9
|
+
def scan_byte # :nodoc:
|
10
|
+
get_byte&.b&.ord
|
11
|
+
end
|
12
|
+
}
|
13
|
+
)
|
14
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Polyfill for Kernel#warn with the category parameter. Not all Ruby engines
|
4
|
+
# have Method#parameters implemented, so we check the arity instead if
|
5
|
+
# necessary.
|
6
|
+
if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.parameters.none? { |_, name| name == :category } : (method.arity == -1)
|
7
|
+
Kernel.prepend(
|
8
|
+
Module.new {
|
9
|
+
def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
|
10
|
+
uplevel =
|
11
|
+
case uplevel
|
12
|
+
when nil
|
13
|
+
1
|
14
|
+
when Integer
|
15
|
+
uplevel + 1
|
16
|
+
else
|
17
|
+
uplevel.to_int + 1
|
18
|
+
end
|
19
|
+
|
20
|
+
super(*msgs, uplevel: uplevel)
|
21
|
+
end
|
22
|
+
}
|
23
|
+
)
|
24
|
+
|
25
|
+
Object.prepend(
|
26
|
+
Module.new {
|
27
|
+
def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
|
28
|
+
uplevel =
|
29
|
+
case uplevel
|
30
|
+
when nil
|
31
|
+
1
|
32
|
+
when Integer
|
33
|
+
uplevel + 1
|
34
|
+
else
|
35
|
+
uplevel.to_int + 1
|
36
|
+
end
|
37
|
+
|
38
|
+
super(*msgs, uplevel: uplevel)
|
39
|
+
end
|
40
|
+
}
|
41
|
+
)
|
42
|
+
end
|
data/lib/prism/reflection.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
3
4
|
=begin
|
5
|
+
--
|
4
6
|
This file is generated by the templates/template.rb script and should not be
|
5
7
|
modified manually. See templates/lib/prism/reflection.rb.erb
|
6
8
|
if you are looking to modify the template
|
9
|
+
++
|
7
10
|
=end
|
8
11
|
|
9
12
|
module Prism
|
data/lib/prism/relocation.rb
CHANGED
data/lib/prism/serialize.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
3
4
|
=begin
|
5
|
+
--
|
4
6
|
This file is generated by the templates/template.rb script and should not be
|
5
7
|
modified manually. See templates/lib/prism/serialize.rb.erb
|
6
8
|
if you are looking to modify the template
|
9
|
+
++
|
7
10
|
=end
|
8
11
|
|
9
12
|
require "stringio"
|
@@ -18,7 +21,7 @@ module Prism
|
|
18
21
|
|
19
22
|
# The minor version of prism that we are expecting to find in the serialized
|
20
23
|
# strings.
|
21
|
-
MINOR_VERSION =
|
24
|
+
MINOR_VERSION = 5
|
22
25
|
|
23
26
|
# The patch version of prism that we are expecting to find in the serialized
|
24
27
|
# strings.
|
@@ -435,6 +438,8 @@ module Prism
|
|
435
438
|
:expect_for_delimiter,
|
436
439
|
:expect_ident_req_parameter,
|
437
440
|
:expect_in_delimiter,
|
441
|
+
:expect_lparen_after_not_lparen,
|
442
|
+
:expect_lparen_after_not_other,
|
438
443
|
:expect_lparen_req_parameter,
|
439
444
|
:expect_message,
|
440
445
|
:expect_rbracket,
|
@@ -2219,8 +2224,21 @@ module Prism
|
|
2219
2224
|
TOKEN_TYPES = [
|
2220
2225
|
nil,
|
2221
2226
|
:EOF,
|
2222
|
-
:
|
2223
|
-
:
|
2227
|
+
:BRACE_RIGHT,
|
2228
|
+
:COMMA,
|
2229
|
+
:EMBEXPR_END,
|
2230
|
+
:KEYWORD_DO,
|
2231
|
+
:KEYWORD_ELSE,
|
2232
|
+
:KEYWORD_ELSIF,
|
2233
|
+
:KEYWORD_END,
|
2234
|
+
:KEYWORD_ENSURE,
|
2235
|
+
:KEYWORD_IN,
|
2236
|
+
:KEYWORD_RESCUE,
|
2237
|
+
:KEYWORD_THEN,
|
2238
|
+
:KEYWORD_WHEN,
|
2239
|
+
:NEWLINE,
|
2240
|
+
:PARENTHESIS_RIGHT,
|
2241
|
+
:SEMICOLON,
|
2224
2242
|
:AMPERSAND,
|
2225
2243
|
:AMPERSAND_AMPERSAND,
|
2226
2244
|
:AMPERSAND_AMPERSAND_EQUAL,
|
@@ -2232,7 +2250,6 @@ module Prism
|
|
2232
2250
|
:BANG_EQUAL,
|
2233
2251
|
:BANG_TILDE,
|
2234
2252
|
:BRACE_LEFT,
|
2235
|
-
:BRACE_RIGHT,
|
2236
2253
|
:BRACKET_LEFT,
|
2237
2254
|
:BRACKET_LEFT_ARRAY,
|
2238
2255
|
:BRACKET_LEFT_RIGHT,
|
@@ -2244,7 +2261,6 @@ module Prism
|
|
2244
2261
|
:CLASS_VARIABLE,
|
2245
2262
|
:COLON,
|
2246
2263
|
:COLON_COLON,
|
2247
|
-
:COMMA,
|
2248
2264
|
:COMMENT,
|
2249
2265
|
:CONSTANT,
|
2250
2266
|
:DOT,
|
@@ -2254,7 +2270,6 @@ module Prism
|
|
2254
2270
|
:EMBDOC_END,
|
2255
2271
|
:EMBDOC_LINE,
|
2256
2272
|
:EMBEXPR_BEGIN,
|
2257
|
-
:EMBEXPR_END,
|
2258
2273
|
:EMBVAR,
|
2259
2274
|
:EQUAL,
|
2260
2275
|
:EQUAL_EQUAL,
|
@@ -2288,38 +2303,29 @@ module Prism
|
|
2288
2303
|
:KEYWORD_CLASS,
|
2289
2304
|
:KEYWORD_DEF,
|
2290
2305
|
:KEYWORD_DEFINED,
|
2291
|
-
:KEYWORD_DO,
|
2292
2306
|
:KEYWORD_DO_LOOP,
|
2293
|
-
:KEYWORD_ELSE,
|
2294
|
-
:KEYWORD_ELSIF,
|
2295
|
-
:KEYWORD_END,
|
2296
2307
|
:KEYWORD_END_UPCASE,
|
2297
|
-
:KEYWORD_ENSURE,
|
2298
2308
|
:KEYWORD_FALSE,
|
2299
2309
|
:KEYWORD_FOR,
|
2300
2310
|
:KEYWORD_IF,
|
2301
2311
|
:KEYWORD_IF_MODIFIER,
|
2302
|
-
:KEYWORD_IN,
|
2303
2312
|
:KEYWORD_MODULE,
|
2304
2313
|
:KEYWORD_NEXT,
|
2305
2314
|
:KEYWORD_NIL,
|
2306
2315
|
:KEYWORD_NOT,
|
2307
2316
|
:KEYWORD_OR,
|
2308
2317
|
:KEYWORD_REDO,
|
2309
|
-
:KEYWORD_RESCUE,
|
2310
2318
|
:KEYWORD_RESCUE_MODIFIER,
|
2311
2319
|
:KEYWORD_RETRY,
|
2312
2320
|
:KEYWORD_RETURN,
|
2313
2321
|
:KEYWORD_SELF,
|
2314
2322
|
:KEYWORD_SUPER,
|
2315
|
-
:KEYWORD_THEN,
|
2316
2323
|
:KEYWORD_TRUE,
|
2317
2324
|
:KEYWORD_UNDEF,
|
2318
2325
|
:KEYWORD_UNLESS,
|
2319
2326
|
:KEYWORD_UNLESS_MODIFIER,
|
2320
2327
|
:KEYWORD_UNTIL,
|
2321
2328
|
:KEYWORD_UNTIL_MODIFIER,
|
2322
|
-
:KEYWORD_WHEN,
|
2323
2329
|
:KEYWORD_WHILE,
|
2324
2330
|
:KEYWORD_WHILE_MODIFIER,
|
2325
2331
|
:KEYWORD_YIELD,
|
@@ -2338,11 +2344,9 @@ module Prism
|
|
2338
2344
|
:MINUS,
|
2339
2345
|
:MINUS_EQUAL,
|
2340
2346
|
:MINUS_GREATER,
|
2341
|
-
:NEWLINE,
|
2342
2347
|
:NUMBERED_REFERENCE,
|
2343
2348
|
:PARENTHESIS_LEFT,
|
2344
2349
|
:PARENTHESIS_LEFT_PARENTHESES,
|
2345
|
-
:PARENTHESIS_RIGHT,
|
2346
2350
|
:PERCENT,
|
2347
2351
|
:PERCENT_EQUAL,
|
2348
2352
|
:PERCENT_LOWER_I,
|
@@ -2359,7 +2363,6 @@ module Prism
|
|
2359
2363
|
:QUESTION_MARK,
|
2360
2364
|
:REGEXP_BEGIN,
|
2361
2365
|
:REGEXP_END,
|
2362
|
-
:SEMICOLON,
|
2363
2366
|
:SLASH,
|
2364
2367
|
:SLASH_EQUAL,
|
2365
2368
|
:STAR,
|
@@ -2382,7 +2385,9 @@ module Prism
|
|
2382
2385
|
:USTAR_STAR,
|
2383
2386
|
:WORDS_SEP,
|
2384
2387
|
:__END__,
|
2385
|
-
|
2388
|
+
:MISSING,
|
2389
|
+
:NOT_PROVIDED,
|
2390
|
+
].freeze
|
2386
2391
|
|
2387
2392
|
private_constant :MAJOR_VERSION, :MINOR_VERSION, :PATCH_VERSION
|
2388
2393
|
private_constant :ConstantPool, :FastStringIO, :Loader, :TOKEN_TYPES
|
data/lib/prism/string_query.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
3
4
|
module Prism
|
4
5
|
module Translation
|
@@ -133,8 +134,8 @@ module Prism
|
|
133
134
|
def visit_assoc_node(node)
|
134
135
|
key = node.key
|
135
136
|
|
136
|
-
if
|
137
|
-
if
|
137
|
+
if node.value.is_a?(ImplicitNode)
|
138
|
+
if in_pattern
|
138
139
|
if key.is_a?(SymbolNode)
|
139
140
|
if key.opening.nil?
|
140
141
|
builder.match_hash_var([key.unescaped, srange(key.location)])
|
@@ -144,23 +145,19 @@ module Prism
|
|
144
145
|
else
|
145
146
|
builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
|
146
147
|
end
|
147
|
-
elsif key.opening.nil?
|
148
|
-
builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value))
|
149
148
|
else
|
150
|
-
|
151
|
-
end
|
152
|
-
elsif node.value.is_a?(ImplicitNode)
|
153
|
-
value = node.value.value
|
149
|
+
value = node.value.value
|
154
150
|
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
151
|
+
implicit_value = if value.is_a?(CallNode)
|
152
|
+
builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
|
153
|
+
elsif value.is_a?(ConstantReadNode)
|
154
|
+
builder.const([value.name, srange(key.value_loc)])
|
155
|
+
else
|
156
|
+
builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
|
157
|
+
end
|
162
158
|
|
163
|
-
|
159
|
+
builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
|
160
|
+
end
|
164
161
|
elsif node.operator_loc
|
165
162
|
builder.pair(visit(key), token(node.operator_loc), visit(node.value))
|
166
163
|
elsif key.is_a?(SymbolNode) && key.opening_loc.nil?
|
@@ -696,13 +693,37 @@ module Prism
|
|
696
693
|
# defined?(a)
|
697
694
|
# ^^^^^^^^^^^
|
698
695
|
def visit_defined_node(node)
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
696
|
+
# Very weird circumstances here where something like:
|
697
|
+
#
|
698
|
+
# defined?
|
699
|
+
# (1)
|
700
|
+
#
|
701
|
+
# gets parsed in Ruby as having only the `1` expression but in parser
|
702
|
+
# it gets parsed as having a begin. In this case we need to synthesize
|
703
|
+
# that begin to match parser's behavior.
|
704
|
+
if node.lparen_loc && node.keyword_loc.join(node.lparen_loc).slice.include?("\n")
|
705
|
+
builder.keyword_cmd(
|
706
|
+
:defined?,
|
707
|
+
token(node.keyword_loc),
|
708
|
+
nil,
|
709
|
+
[
|
710
|
+
builder.begin(
|
711
|
+
token(node.lparen_loc),
|
712
|
+
visit(node.value),
|
713
|
+
token(node.rparen_loc)
|
714
|
+
)
|
715
|
+
],
|
716
|
+
nil
|
717
|
+
)
|
718
|
+
else
|
719
|
+
builder.keyword_cmd(
|
720
|
+
:defined?,
|
721
|
+
token(node.keyword_loc),
|
722
|
+
token(node.lparen_loc),
|
723
|
+
[visit(node.value)],
|
724
|
+
token(node.rparen_loc)
|
725
|
+
)
|
726
|
+
end
|
706
727
|
end
|
707
728
|
|
708
729
|
# if foo then bar else baz end
|
@@ -1032,7 +1053,7 @@ module Prism
|
|
1032
1053
|
builder.index_asgn(
|
1033
1054
|
visit(node.receiver),
|
1034
1055
|
token(node.opening_loc),
|
1035
|
-
visit_all(node.arguments
|
1056
|
+
visit_all(node.arguments&.arguments || []),
|
1036
1057
|
token(node.closing_loc),
|
1037
1058
|
)
|
1038
1059
|
end
|
@@ -1461,7 +1482,8 @@ module Prism
|
|
1461
1482
|
# foo => ^(bar)
|
1462
1483
|
# ^^^^^^
|
1463
1484
|
def visit_pinned_expression_node(node)
|
1464
|
-
|
1485
|
+
parts = node.expression.accept(copy_compiler(in_pattern: false)) # Don't treat * and similar as match_rest
|
1486
|
+
expression = builder.begin(token(node.lparen_loc), parts, token(node.rparen_loc))
|
1465
1487
|
builder.pin(token(node.operator_loc), expression)
|
1466
1488
|
end
|
1467
1489
|
|
@@ -1,7 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
3
4
|
require "strscan"
|
4
5
|
require_relative "../../polyfill/append_as_bytes"
|
6
|
+
require_relative "../../polyfill/scan_byte"
|
5
7
|
|
6
8
|
module Prism
|
7
9
|
module Translation
|
@@ -200,8 +202,8 @@ module Prism
|
|
200
202
|
# The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
|
201
203
|
# The following token types are listed as those classified as `tLPAREN`.
|
202
204
|
LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
|
203
|
-
:kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
|
204
|
-
:tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
|
205
|
+
:kBREAK, :tCARET, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
|
206
|
+
:tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS, :tLCURLY
|
205
207
|
])
|
206
208
|
|
207
209
|
# Types of tokens that are allowed to continue a method call with comments in-between.
|
@@ -275,20 +277,20 @@ module Prism
|
|
275
277
|
when :tCOMMENT
|
276
278
|
if token.type == :EMBDOC_BEGIN
|
277
279
|
|
278
|
-
while !((next_token = lexed[index]
|
280
|
+
while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
|
279
281
|
value += next_token.value
|
280
282
|
index += 1
|
281
283
|
end
|
282
284
|
|
283
285
|
value += next_token.value
|
284
|
-
location = range(token.location.start_offset,
|
286
|
+
location = range(token.location.start_offset, next_token.location.end_offset)
|
285
287
|
index += 1
|
286
288
|
else
|
287
289
|
is_at_eol = value.chomp!.nil?
|
288
290
|
location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
|
289
291
|
|
290
|
-
prev_token = lexed[index - 2]
|
291
|
-
next_token = lexed[index]
|
292
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
293
|
+
next_token, _ = lexed[index]
|
292
294
|
|
293
295
|
is_inline_comment = prev_token&.location&.start_line == token.location.start_line
|
294
296
|
if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
|
@@ -307,7 +309,7 @@ module Prism
|
|
307
309
|
end
|
308
310
|
end
|
309
311
|
when :tNL
|
310
|
-
next_token
|
312
|
+
next_token, _ = lexed[index]
|
311
313
|
# Newlines after comments are emitted out of order.
|
312
314
|
if next_token&.type == :COMMENT
|
313
315
|
comment_newline_location = location
|
@@ -344,8 +346,8 @@ module Prism
|
|
344
346
|
location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
|
345
347
|
value = nil
|
346
348
|
when :tSTRING_BEG
|
347
|
-
next_token = lexed[index]
|
348
|
-
next_next_token = lexed[index + 1]
|
349
|
+
next_token, _ = lexed[index]
|
350
|
+
next_next_token, _ = lexed[index + 1]
|
349
351
|
basic_quotes = value == '"' || value == "'"
|
350
352
|
|
351
353
|
if basic_quotes && next_token&.type == :STRING_END
|
@@ -413,7 +415,8 @@ module Prism
|
|
413
415
|
while token.type == :STRING_CONTENT
|
414
416
|
current_length += token.value.bytesize
|
415
417
|
# Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
|
416
|
-
|
418
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
419
|
+
is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
|
417
420
|
# The parser gem only removes indentation when the heredoc is not nested
|
418
421
|
not_nested = heredoc_stack.size == 1
|
419
422
|
if is_percent_array
|
@@ -423,11 +426,16 @@ module Prism
|
|
423
426
|
end
|
424
427
|
|
425
428
|
current_string << unescape_string(value, quote_stack.last)
|
426
|
-
|
429
|
+
relevant_backslash_count = if quote_stack.last.start_with?("%W", "%I")
|
430
|
+
0 # the last backslash escapes the newline
|
431
|
+
else
|
432
|
+
token.value[/(\\{1,})\n/, 1]&.length || 0
|
433
|
+
end
|
434
|
+
if relevant_backslash_count.even? || !interpolation?(quote_stack.last)
|
427
435
|
tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
|
428
436
|
break
|
429
437
|
end
|
430
|
-
token = lexed[index]
|
438
|
+
token, _ = lexed[index]
|
431
439
|
index += 1
|
432
440
|
end
|
433
441
|
else
|
@@ -482,7 +490,7 @@ module Prism
|
|
482
490
|
end
|
483
491
|
|
484
492
|
if percent_array?(quote_stack.pop)
|
485
|
-
prev_token = lexed[index - 2]
|
493
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
486
494
|
empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
|
487
495
|
ends_with_whitespace = prev_token&.type == :WORDS_SEP
|
488
496
|
# parser always emits a space token after content in a percent array, even if no actual whitespace is present.
|
@@ -491,7 +499,7 @@ module Prism
|
|
491
499
|
end
|
492
500
|
end
|
493
501
|
when :tSYMBEG
|
494
|
-
if (next_token = lexed[index]
|
502
|
+
if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
|
495
503
|
next_location = token.location.join(next_token.location)
|
496
504
|
type = :tSYMBOL
|
497
505
|
value = next_token.value
|
@@ -506,13 +514,13 @@ module Prism
|
|
506
514
|
type = :tIDENTIFIER
|
507
515
|
end
|
508
516
|
when :tXSTRING_BEG
|
509
|
-
if (next_token = lexed[index]
|
517
|
+
if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
|
510
518
|
# self.`()
|
511
519
|
type = :tBACK_REF2
|
512
520
|
end
|
513
521
|
quote_stack.push(value)
|
514
522
|
when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
|
515
|
-
if (next_token = lexed[index]
|
523
|
+
if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
|
516
524
|
index += 1
|
517
525
|
end
|
518
526
|
|
@@ -588,9 +596,9 @@ module Prism
|
|
588
596
|
previous_line = -1
|
589
597
|
result = Float::MAX
|
590
598
|
|
591
|
-
while (
|
599
|
+
while (next_token = lexed[next_token_index]&.first)
|
592
600
|
next_token_index += 1
|
593
|
-
next_next_token = lexed[next_token_index]
|
601
|
+
next_next_token, _ = lexed[next_token_index]
|
594
602
|
first_token_on_line = next_token.location.start_column == 0
|
595
603
|
|
596
604
|
# String content inside nested heredocs and interpolation is ignored
|
@@ -761,12 +769,12 @@ module Prism
|
|
761
769
|
elsif (value = scanner.scan(/M-\\?(?=[[:print:]])/))
|
762
770
|
# \M-x where x is an ASCII printable character
|
763
771
|
escape_read(result, scanner, control, true)
|
764
|
-
elsif (byte = scanner.
|
772
|
+
elsif (byte = scanner.scan_byte)
|
765
773
|
# Something else after an escape.
|
766
|
-
if control && byte ==
|
774
|
+
if control && byte == 0x3f # ASCII '?'
|
767
775
|
result.append_as_bytes(escape_build(0x7f, false, meta))
|
768
776
|
else
|
769
|
-
result.append_as_bytes(escape_build(byte
|
777
|
+
result.append_as_bytes(escape_build(byte, control, meta))
|
770
778
|
end
|
771
779
|
end
|
772
780
|
end
|
@@ -1,9 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
2
3
|
|
3
4
|
begin
|
5
|
+
required_version = ">= 3.3.7.2"
|
6
|
+
gem "parser", required_version
|
4
7
|
require "parser"
|
5
8
|
rescue LoadError
|
6
|
-
warn(
|
9
|
+
warn(<<~MSG)
|
10
|
+
Error: Unable to load parser #{required_version}. \
|
11
|
+
Add `gem "parser"` to your Gemfile or run `bundle update parser`.
|
12
|
+
MSG
|
7
13
|
exit(1)
|
8
14
|
end
|
9
15
|
|
@@ -59,6 +65,12 @@ module Prism
|
|
59
65
|
# should be implemented as needed.
|
60
66
|
#
|
61
67
|
def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
|
68
|
+
if !builder.is_a?(Prism::Translation::Parser::Builder)
|
69
|
+
warn(<<~MSG, uplevel: 1, category: :deprecated)
|
70
|
+
[deprecation]: The builder passed to `Prism::Translation::Parser.new` is not a \
|
71
|
+
`Prism::Translation::Parser::Builder` subclass. This will raise in the next major version.
|
72
|
+
MSG
|
73
|
+
end
|
62
74
|
@parser = parser
|
63
75
|
|
64
76
|
super(builder)
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# :markup: markdown
|
3
|
+
# typed: ignore
|
4
|
+
|
5
|
+
#
|
6
|
+
module Prism
|
7
|
+
module Translation
|
8
|
+
case RUBY_VERSION
|
9
|
+
when /^3\.3\./
|
10
|
+
ParserCurrent = Parser33
|
11
|
+
when /^3\.4\./
|
12
|
+
ParserCurrent = Parser34
|
13
|
+
when /^3\.5\./
|
14
|
+
ParserCurrent = Parser35
|
15
|
+
else
|
16
|
+
# Keep this in sync with released Ruby.
|
17
|
+
parser = Parser34
|
18
|
+
major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
|
19
|
+
warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
|
20
|
+
"but you are running #{major}.#{minor}."
|
21
|
+
ParserCurrent = parser
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|