prism 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +51 -1
- data/Makefile +4 -2
- data/README.md +2 -0
- data/config.yml +266 -38
- data/docs/design.md +2 -2
- data/docs/parser_translation.md +8 -23
- data/docs/releasing.md +5 -24
- data/docs/ripper_translation.md +1 -1
- data/ext/prism/api_node.c +2 -0
- data/ext/prism/extension.c +25 -3
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +306 -50
- data/include/prism/diagnostic.h +5 -0
- data/include/prism/options.h +43 -3
- data/include/prism/regexp.h +2 -2
- data/include/prism/util/pm_buffer.h +8 -0
- data/include/prism/util/pm_integer.h +4 -0
- data/include/prism/util/pm_list.h +6 -0
- data/include/prism/util/pm_string.h +12 -2
- data/include/prism/version.h +2 -2
- data/include/prism.h +39 -14
- data/lib/prism/compiler.rb +456 -151
- data/lib/prism/desugar_compiler.rb +1 -0
- data/lib/prism/dispatcher.rb +16 -0
- data/lib/prism/dot_visitor.rb +5 -1
- data/lib/prism/dsl.rb +3 -0
- data/lib/prism/ffi.rb +25 -9
- data/lib/prism/inspect_visitor.rb +3 -0
- data/lib/prism/lex_compat.rb +1 -0
- data/lib/prism/mutation_compiler.rb +3 -0
- data/lib/prism/node.rb +507 -336
- data/lib/prism/node_ext.rb +4 -1
- data/lib/prism/pack.rb +2 -0
- data/lib/prism/parse_result/comments.rb +1 -0
- data/lib/prism/parse_result/errors.rb +1 -0
- data/lib/prism/parse_result/newlines.rb +1 -0
- data/lib/prism/parse_result.rb +1 -0
- data/lib/prism/pattern.rb +1 -0
- data/lib/prism/polyfill/scan_byte.rb +14 -0
- data/lib/prism/polyfill/warn.rb +36 -0
- data/lib/prism/reflection.rb +3 -0
- data/lib/prism/relocation.rb +1 -0
- data/lib/prism/serialize.rb +25 -19
- data/lib/prism/string_query.rb +1 -0
- data/lib/prism/translation/parser/builder.rb +1 -0
- data/lib/prism/translation/parser/compiler.rb +47 -25
- data/lib/prism/translation/parser/lexer.rb +29 -21
- data/lib/prism/translation/parser.rb +21 -2
- data/lib/prism/translation/parser33.rb +1 -0
- data/lib/prism/translation/parser34.rb +1 -0
- data/lib/prism/translation/parser35.rb +1 -0
- data/lib/prism/translation/parser_current.rb +24 -0
- data/lib/prism/translation/ripper/sexp.rb +1 -0
- data/lib/prism/translation/ripper.rb +17 -1
- data/lib/prism/translation/ruby_parser.rb +287 -4
- data/lib/prism/translation.rb +2 -0
- data/lib/prism/visitor.rb +457 -152
- data/lib/prism.rb +23 -0
- data/prism.gemspec +5 -1
- data/rbi/prism/dsl.rbi +3 -3
- data/rbi/prism/node.rbi +21 -9
- data/sig/prism/dispatcher.rbs +3 -0
- data/sig/prism/dsl.rbs +3 -3
- data/sig/prism/node.rbs +444 -30
- data/sig/prism/node_ext.rbs +84 -17
- data/sig/prism/parse_result/comments.rbs +38 -0
- data/sig/prism/parse_result.rbs +4 -0
- data/sig/prism/reflection.rbs +1 -1
- data/sig/prism.rbs +4 -0
- data/src/diagnostic.c +9 -1
- data/src/node.c +2 -0
- data/src/options.c +2 -2
- data/src/prettyprint.c +2 -0
- data/src/prism.c +324 -147
- data/src/serialize.c +2 -0
- data/src/token_type.c +36 -34
- data/src/util/pm_string.c +6 -8
- metadata +7 -3
data/lib/prism/node_ext.rb
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
4
|
+
#--
|
|
3
5
|
# Here we are reopening the prism module to provide methods on nodes that aren't
|
|
4
6
|
# templated and are meant as convenience methods.
|
|
7
|
+
#++
|
|
5
8
|
module Prism
|
|
6
9
|
class Node
|
|
7
10
|
def deprecated(*replacements) # :nodoc:
|
|
@@ -9,7 +12,7 @@ module Prism
|
|
|
9
12
|
location = location[0].label if location
|
|
10
13
|
suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
|
|
11
14
|
|
|
12
|
-
warn(<<~MSG, category: :deprecated)
|
|
15
|
+
warn(<<~MSG, uplevel: 1, category: :deprecated)
|
|
13
16
|
[deprecation]: #{self.class}##{location} is deprecated and will be \
|
|
14
17
|
removed in the next major version. Use #{suggest.join("/")} instead.
|
|
15
18
|
#{(caller(1, 3) || []).join("\n")}
|
data/lib/prism/pack.rb
CHANGED
data/lib/prism/parse_result.rb
CHANGED
data/lib/prism/pattern.rb
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "strscan"
|
|
4
|
+
|
|
5
|
+
# Polyfill for StringScanner#scan_byte, which didn't exist until Ruby 3.4.
|
|
6
|
+
if !(StringScanner.instance_methods.include?(:scan_byte))
|
|
7
|
+
StringScanner.include(
|
|
8
|
+
Module.new {
|
|
9
|
+
def scan_byte # :nodoc:
|
|
10
|
+
get_byte&.b&.ord
|
|
11
|
+
end
|
|
12
|
+
}
|
|
13
|
+
)
|
|
14
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Polyfill for Kernel#warn with the category parameter. Not all Ruby engines
|
|
4
|
+
# have Method#parameters implemented, so we check the arity instead if
|
|
5
|
+
# necessary.
|
|
6
|
+
if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.parameters.none? { |_, name| name == :category } : (method.arity == -1)
|
|
7
|
+
Kernel.prepend(
|
|
8
|
+
Module.new {
|
|
9
|
+
def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
|
|
10
|
+
case uplevel
|
|
11
|
+
when nil
|
|
12
|
+
super(*msgs)
|
|
13
|
+
when Integer
|
|
14
|
+
super(*msgs, uplevel: uplevel + 1)
|
|
15
|
+
else
|
|
16
|
+
super(*msgs, uplevel: uplevel.to_int + 1)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
Object.prepend(
|
|
23
|
+
Module.new {
|
|
24
|
+
def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
|
|
25
|
+
case uplevel
|
|
26
|
+
when nil
|
|
27
|
+
super(*msgs)
|
|
28
|
+
when Integer
|
|
29
|
+
super(*msgs, uplevel: uplevel + 1)
|
|
30
|
+
else
|
|
31
|
+
super(*msgs, uplevel: uplevel.to_int + 1)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
}
|
|
35
|
+
)
|
|
36
|
+
end
|
data/lib/prism/reflection.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
3
4
|
=begin
|
|
5
|
+
--
|
|
4
6
|
This file is generated by the templates/template.rb script and should not be
|
|
5
7
|
modified manually. See templates/lib/prism/reflection.rb.erb
|
|
6
8
|
if you are looking to modify the template
|
|
9
|
+
++
|
|
7
10
|
=end
|
|
8
11
|
|
|
9
12
|
module Prism
|
data/lib/prism/relocation.rb
CHANGED
data/lib/prism/serialize.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
3
4
|
=begin
|
|
5
|
+
--
|
|
4
6
|
This file is generated by the templates/template.rb script and should not be
|
|
5
7
|
modified manually. See templates/lib/prism/serialize.rb.erb
|
|
6
8
|
if you are looking to modify the template
|
|
9
|
+
++
|
|
7
10
|
=end
|
|
8
11
|
|
|
9
12
|
require "stringio"
|
|
@@ -18,7 +21,7 @@ module Prism
|
|
|
18
21
|
|
|
19
22
|
# The minor version of prism that we are expecting to find in the serialized
|
|
20
23
|
# strings.
|
|
21
|
-
MINOR_VERSION =
|
|
24
|
+
MINOR_VERSION = 6
|
|
22
25
|
|
|
23
26
|
# The patch version of prism that we are expecting to find in the serialized
|
|
24
27
|
# strings.
|
|
@@ -394,6 +397,7 @@ module Prism
|
|
|
394
397
|
:conditional_while_predicate,
|
|
395
398
|
:constant_path_colon_colon_constant,
|
|
396
399
|
:def_endless,
|
|
400
|
+
:def_endless_parameters,
|
|
397
401
|
:def_endless_setter,
|
|
398
402
|
:def_name,
|
|
399
403
|
:def_params_term,
|
|
@@ -435,6 +439,8 @@ module Prism
|
|
|
435
439
|
:expect_for_delimiter,
|
|
436
440
|
:expect_ident_req_parameter,
|
|
437
441
|
:expect_in_delimiter,
|
|
442
|
+
:expect_lparen_after_not_lparen,
|
|
443
|
+
:expect_lparen_after_not_other,
|
|
438
444
|
:expect_lparen_req_parameter,
|
|
439
445
|
:expect_message,
|
|
440
446
|
:expect_rbracket,
|
|
@@ -2219,8 +2225,21 @@ module Prism
|
|
|
2219
2225
|
TOKEN_TYPES = [
|
|
2220
2226
|
nil,
|
|
2221
2227
|
:EOF,
|
|
2222
|
-
:
|
|
2223
|
-
:
|
|
2228
|
+
:BRACE_RIGHT,
|
|
2229
|
+
:COMMA,
|
|
2230
|
+
:EMBEXPR_END,
|
|
2231
|
+
:KEYWORD_DO,
|
|
2232
|
+
:KEYWORD_ELSE,
|
|
2233
|
+
:KEYWORD_ELSIF,
|
|
2234
|
+
:KEYWORD_END,
|
|
2235
|
+
:KEYWORD_ENSURE,
|
|
2236
|
+
:KEYWORD_IN,
|
|
2237
|
+
:KEYWORD_RESCUE,
|
|
2238
|
+
:KEYWORD_THEN,
|
|
2239
|
+
:KEYWORD_WHEN,
|
|
2240
|
+
:NEWLINE,
|
|
2241
|
+
:PARENTHESIS_RIGHT,
|
|
2242
|
+
:SEMICOLON,
|
|
2224
2243
|
:AMPERSAND,
|
|
2225
2244
|
:AMPERSAND_AMPERSAND,
|
|
2226
2245
|
:AMPERSAND_AMPERSAND_EQUAL,
|
|
@@ -2232,7 +2251,6 @@ module Prism
|
|
|
2232
2251
|
:BANG_EQUAL,
|
|
2233
2252
|
:BANG_TILDE,
|
|
2234
2253
|
:BRACE_LEFT,
|
|
2235
|
-
:BRACE_RIGHT,
|
|
2236
2254
|
:BRACKET_LEFT,
|
|
2237
2255
|
:BRACKET_LEFT_ARRAY,
|
|
2238
2256
|
:BRACKET_LEFT_RIGHT,
|
|
@@ -2244,7 +2262,6 @@ module Prism
|
|
|
2244
2262
|
:CLASS_VARIABLE,
|
|
2245
2263
|
:COLON,
|
|
2246
2264
|
:COLON_COLON,
|
|
2247
|
-
:COMMA,
|
|
2248
2265
|
:COMMENT,
|
|
2249
2266
|
:CONSTANT,
|
|
2250
2267
|
:DOT,
|
|
@@ -2254,7 +2271,6 @@ module Prism
|
|
|
2254
2271
|
:EMBDOC_END,
|
|
2255
2272
|
:EMBDOC_LINE,
|
|
2256
2273
|
:EMBEXPR_BEGIN,
|
|
2257
|
-
:EMBEXPR_END,
|
|
2258
2274
|
:EMBVAR,
|
|
2259
2275
|
:EQUAL,
|
|
2260
2276
|
:EQUAL_EQUAL,
|
|
@@ -2288,38 +2304,29 @@ module Prism
|
|
|
2288
2304
|
:KEYWORD_CLASS,
|
|
2289
2305
|
:KEYWORD_DEF,
|
|
2290
2306
|
:KEYWORD_DEFINED,
|
|
2291
|
-
:KEYWORD_DO,
|
|
2292
2307
|
:KEYWORD_DO_LOOP,
|
|
2293
|
-
:KEYWORD_ELSE,
|
|
2294
|
-
:KEYWORD_ELSIF,
|
|
2295
|
-
:KEYWORD_END,
|
|
2296
2308
|
:KEYWORD_END_UPCASE,
|
|
2297
|
-
:KEYWORD_ENSURE,
|
|
2298
2309
|
:KEYWORD_FALSE,
|
|
2299
2310
|
:KEYWORD_FOR,
|
|
2300
2311
|
:KEYWORD_IF,
|
|
2301
2312
|
:KEYWORD_IF_MODIFIER,
|
|
2302
|
-
:KEYWORD_IN,
|
|
2303
2313
|
:KEYWORD_MODULE,
|
|
2304
2314
|
:KEYWORD_NEXT,
|
|
2305
2315
|
:KEYWORD_NIL,
|
|
2306
2316
|
:KEYWORD_NOT,
|
|
2307
2317
|
:KEYWORD_OR,
|
|
2308
2318
|
:KEYWORD_REDO,
|
|
2309
|
-
:KEYWORD_RESCUE,
|
|
2310
2319
|
:KEYWORD_RESCUE_MODIFIER,
|
|
2311
2320
|
:KEYWORD_RETRY,
|
|
2312
2321
|
:KEYWORD_RETURN,
|
|
2313
2322
|
:KEYWORD_SELF,
|
|
2314
2323
|
:KEYWORD_SUPER,
|
|
2315
|
-
:KEYWORD_THEN,
|
|
2316
2324
|
:KEYWORD_TRUE,
|
|
2317
2325
|
:KEYWORD_UNDEF,
|
|
2318
2326
|
:KEYWORD_UNLESS,
|
|
2319
2327
|
:KEYWORD_UNLESS_MODIFIER,
|
|
2320
2328
|
:KEYWORD_UNTIL,
|
|
2321
2329
|
:KEYWORD_UNTIL_MODIFIER,
|
|
2322
|
-
:KEYWORD_WHEN,
|
|
2323
2330
|
:KEYWORD_WHILE,
|
|
2324
2331
|
:KEYWORD_WHILE_MODIFIER,
|
|
2325
2332
|
:KEYWORD_YIELD,
|
|
@@ -2338,11 +2345,9 @@ module Prism
|
|
|
2338
2345
|
:MINUS,
|
|
2339
2346
|
:MINUS_EQUAL,
|
|
2340
2347
|
:MINUS_GREATER,
|
|
2341
|
-
:NEWLINE,
|
|
2342
2348
|
:NUMBERED_REFERENCE,
|
|
2343
2349
|
:PARENTHESIS_LEFT,
|
|
2344
2350
|
:PARENTHESIS_LEFT_PARENTHESES,
|
|
2345
|
-
:PARENTHESIS_RIGHT,
|
|
2346
2351
|
:PERCENT,
|
|
2347
2352
|
:PERCENT_EQUAL,
|
|
2348
2353
|
:PERCENT_LOWER_I,
|
|
@@ -2359,7 +2364,6 @@ module Prism
|
|
|
2359
2364
|
:QUESTION_MARK,
|
|
2360
2365
|
:REGEXP_BEGIN,
|
|
2361
2366
|
:REGEXP_END,
|
|
2362
|
-
:SEMICOLON,
|
|
2363
2367
|
:SLASH,
|
|
2364
2368
|
:SLASH_EQUAL,
|
|
2365
2369
|
:STAR,
|
|
@@ -2382,7 +2386,9 @@ module Prism
|
|
|
2382
2386
|
:USTAR_STAR,
|
|
2383
2387
|
:WORDS_SEP,
|
|
2384
2388
|
:__END__,
|
|
2385
|
-
|
|
2389
|
+
:MISSING,
|
|
2390
|
+
:NOT_PROVIDED,
|
|
2391
|
+
].freeze
|
|
2386
2392
|
|
|
2387
2393
|
private_constant :MAJOR_VERSION, :MINOR_VERSION, :PATCH_VERSION
|
|
2388
2394
|
private_constant :ConstantPool, :FastStringIO, :Loader, :TOKEN_TYPES
|
data/lib/prism/string_query.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
3
4
|
module Prism
|
|
4
5
|
module Translation
|
|
@@ -133,8 +134,8 @@ module Prism
|
|
|
133
134
|
def visit_assoc_node(node)
|
|
134
135
|
key = node.key
|
|
135
136
|
|
|
136
|
-
if
|
|
137
|
-
if
|
|
137
|
+
if node.value.is_a?(ImplicitNode)
|
|
138
|
+
if in_pattern
|
|
138
139
|
if key.is_a?(SymbolNode)
|
|
139
140
|
if key.opening.nil?
|
|
140
141
|
builder.match_hash_var([key.unescaped, srange(key.location)])
|
|
@@ -144,23 +145,19 @@ module Prism
|
|
|
144
145
|
else
|
|
145
146
|
builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
|
|
146
147
|
end
|
|
147
|
-
elsif key.opening.nil?
|
|
148
|
-
builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value))
|
|
149
148
|
else
|
|
150
|
-
|
|
151
|
-
end
|
|
152
|
-
elsif node.value.is_a?(ImplicitNode)
|
|
153
|
-
value = node.value.value
|
|
149
|
+
value = node.value.value
|
|
154
150
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
151
|
+
implicit_value = if value.is_a?(CallNode)
|
|
152
|
+
builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
|
|
153
|
+
elsif value.is_a?(ConstantReadNode)
|
|
154
|
+
builder.const([value.name, srange(key.value_loc)])
|
|
155
|
+
else
|
|
156
|
+
builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
|
|
157
|
+
end
|
|
162
158
|
|
|
163
|
-
|
|
159
|
+
builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
|
|
160
|
+
end
|
|
164
161
|
elsif node.operator_loc
|
|
165
162
|
builder.pair(visit(key), token(node.operator_loc), visit(node.value))
|
|
166
163
|
elsif key.is_a?(SymbolNode) && key.opening_loc.nil?
|
|
@@ -696,13 +693,37 @@ module Prism
|
|
|
696
693
|
# defined?(a)
|
|
697
694
|
# ^^^^^^^^^^^
|
|
698
695
|
def visit_defined_node(node)
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
696
|
+
# Very weird circumstances here where something like:
|
|
697
|
+
#
|
|
698
|
+
# defined?
|
|
699
|
+
# (1)
|
|
700
|
+
#
|
|
701
|
+
# gets parsed in Ruby as having only the `1` expression but in parser
|
|
702
|
+
# it gets parsed as having a begin. In this case we need to synthesize
|
|
703
|
+
# that begin to match parser's behavior.
|
|
704
|
+
if node.lparen_loc && node.keyword_loc.join(node.lparen_loc).slice.include?("\n")
|
|
705
|
+
builder.keyword_cmd(
|
|
706
|
+
:defined?,
|
|
707
|
+
token(node.keyword_loc),
|
|
708
|
+
nil,
|
|
709
|
+
[
|
|
710
|
+
builder.begin(
|
|
711
|
+
token(node.lparen_loc),
|
|
712
|
+
visit(node.value),
|
|
713
|
+
token(node.rparen_loc)
|
|
714
|
+
)
|
|
715
|
+
],
|
|
716
|
+
nil
|
|
717
|
+
)
|
|
718
|
+
else
|
|
719
|
+
builder.keyword_cmd(
|
|
720
|
+
:defined?,
|
|
721
|
+
token(node.keyword_loc),
|
|
722
|
+
token(node.lparen_loc),
|
|
723
|
+
[visit(node.value)],
|
|
724
|
+
token(node.rparen_loc)
|
|
725
|
+
)
|
|
726
|
+
end
|
|
706
727
|
end
|
|
707
728
|
|
|
708
729
|
# if foo then bar else baz end
|
|
@@ -1032,7 +1053,7 @@ module Prism
|
|
|
1032
1053
|
builder.index_asgn(
|
|
1033
1054
|
visit(node.receiver),
|
|
1034
1055
|
token(node.opening_loc),
|
|
1035
|
-
visit_all(node.arguments
|
|
1056
|
+
visit_all(node.arguments&.arguments || []),
|
|
1036
1057
|
token(node.closing_loc),
|
|
1037
1058
|
)
|
|
1038
1059
|
end
|
|
@@ -1461,7 +1482,8 @@ module Prism
|
|
|
1461
1482
|
# foo => ^(bar)
|
|
1462
1483
|
# ^^^^^^
|
|
1463
1484
|
def visit_pinned_expression_node(node)
|
|
1464
|
-
|
|
1485
|
+
parts = node.expression.accept(copy_compiler(in_pattern: false)) # Don't treat * and similar as match_rest
|
|
1486
|
+
expression = builder.begin(token(node.lparen_loc), parts, token(node.rparen_loc))
|
|
1465
1487
|
builder.pin(token(node.operator_loc), expression)
|
|
1466
1488
|
end
|
|
1467
1489
|
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
3
4
|
require "strscan"
|
|
4
5
|
require_relative "../../polyfill/append_as_bytes"
|
|
6
|
+
require_relative "../../polyfill/scan_byte"
|
|
5
7
|
|
|
6
8
|
module Prism
|
|
7
9
|
module Translation
|
|
@@ -200,8 +202,8 @@ module Prism
|
|
|
200
202
|
# The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
|
|
201
203
|
# The following token types are listed as those classified as `tLPAREN`.
|
|
202
204
|
LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
|
|
203
|
-
:kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
|
|
204
|
-
:tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
|
|
205
|
+
:kBREAK, :tCARET, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
|
|
206
|
+
:tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS, :tLCURLY
|
|
205
207
|
])
|
|
206
208
|
|
|
207
209
|
# Types of tokens that are allowed to continue a method call with comments in-between.
|
|
@@ -275,20 +277,20 @@ module Prism
|
|
|
275
277
|
when :tCOMMENT
|
|
276
278
|
if token.type == :EMBDOC_BEGIN
|
|
277
279
|
|
|
278
|
-
while !((next_token = lexed[index]
|
|
280
|
+
while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
|
|
279
281
|
value += next_token.value
|
|
280
282
|
index += 1
|
|
281
283
|
end
|
|
282
284
|
|
|
283
285
|
value += next_token.value
|
|
284
|
-
location = range(token.location.start_offset,
|
|
286
|
+
location = range(token.location.start_offset, next_token.location.end_offset)
|
|
285
287
|
index += 1
|
|
286
288
|
else
|
|
287
289
|
is_at_eol = value.chomp!.nil?
|
|
288
290
|
location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
|
|
289
291
|
|
|
290
|
-
prev_token = lexed[index - 2]
|
|
291
|
-
next_token = lexed[index]
|
|
292
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
|
293
|
+
next_token, _ = lexed[index]
|
|
292
294
|
|
|
293
295
|
is_inline_comment = prev_token&.location&.start_line == token.location.start_line
|
|
294
296
|
if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
|
|
@@ -307,7 +309,7 @@ module Prism
|
|
|
307
309
|
end
|
|
308
310
|
end
|
|
309
311
|
when :tNL
|
|
310
|
-
next_token
|
|
312
|
+
next_token, _ = lexed[index]
|
|
311
313
|
# Newlines after comments are emitted out of order.
|
|
312
314
|
if next_token&.type == :COMMENT
|
|
313
315
|
comment_newline_location = location
|
|
@@ -344,8 +346,8 @@ module Prism
|
|
|
344
346
|
location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
|
|
345
347
|
value = nil
|
|
346
348
|
when :tSTRING_BEG
|
|
347
|
-
next_token = lexed[index]
|
|
348
|
-
next_next_token = lexed[index + 1]
|
|
349
|
+
next_token, _ = lexed[index]
|
|
350
|
+
next_next_token, _ = lexed[index + 1]
|
|
349
351
|
basic_quotes = value == '"' || value == "'"
|
|
350
352
|
|
|
351
353
|
if basic_quotes && next_token&.type == :STRING_END
|
|
@@ -413,7 +415,8 @@ module Prism
|
|
|
413
415
|
while token.type == :STRING_CONTENT
|
|
414
416
|
current_length += token.value.bytesize
|
|
415
417
|
# Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
|
|
416
|
-
|
|
418
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
|
419
|
+
is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
|
|
417
420
|
# The parser gem only removes indentation when the heredoc is not nested
|
|
418
421
|
not_nested = heredoc_stack.size == 1
|
|
419
422
|
if is_percent_array
|
|
@@ -423,11 +426,16 @@ module Prism
|
|
|
423
426
|
end
|
|
424
427
|
|
|
425
428
|
current_string << unescape_string(value, quote_stack.last)
|
|
426
|
-
|
|
429
|
+
relevant_backslash_count = if quote_stack.last.start_with?("%W", "%I")
|
|
430
|
+
0 # the last backslash escapes the newline
|
|
431
|
+
else
|
|
432
|
+
token.value[/(\\{1,})\n/, 1]&.length || 0
|
|
433
|
+
end
|
|
434
|
+
if relevant_backslash_count.even? || !interpolation?(quote_stack.last)
|
|
427
435
|
tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
|
|
428
436
|
break
|
|
429
437
|
end
|
|
430
|
-
token = lexed[index]
|
|
438
|
+
token, _ = lexed[index]
|
|
431
439
|
index += 1
|
|
432
440
|
end
|
|
433
441
|
else
|
|
@@ -482,7 +490,7 @@ module Prism
|
|
|
482
490
|
end
|
|
483
491
|
|
|
484
492
|
if percent_array?(quote_stack.pop)
|
|
485
|
-
prev_token = lexed[index - 2]
|
|
493
|
+
prev_token, _ = lexed[index - 2] if index - 2 >= 0
|
|
486
494
|
empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
|
|
487
495
|
ends_with_whitespace = prev_token&.type == :WORDS_SEP
|
|
488
496
|
# parser always emits a space token after content in a percent array, even if no actual whitespace is present.
|
|
@@ -491,7 +499,7 @@ module Prism
|
|
|
491
499
|
end
|
|
492
500
|
end
|
|
493
501
|
when :tSYMBEG
|
|
494
|
-
if (next_token = lexed[index]
|
|
502
|
+
if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
|
|
495
503
|
next_location = token.location.join(next_token.location)
|
|
496
504
|
type = :tSYMBOL
|
|
497
505
|
value = next_token.value
|
|
@@ -506,13 +514,13 @@ module Prism
|
|
|
506
514
|
type = :tIDENTIFIER
|
|
507
515
|
end
|
|
508
516
|
when :tXSTRING_BEG
|
|
509
|
-
if (next_token = lexed[index]
|
|
517
|
+
if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
|
|
510
518
|
# self.`()
|
|
511
519
|
type = :tBACK_REF2
|
|
512
520
|
end
|
|
513
521
|
quote_stack.push(value)
|
|
514
522
|
when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
|
|
515
|
-
if (next_token = lexed[index]
|
|
523
|
+
if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
|
|
516
524
|
index += 1
|
|
517
525
|
end
|
|
518
526
|
|
|
@@ -588,9 +596,9 @@ module Prism
|
|
|
588
596
|
previous_line = -1
|
|
589
597
|
result = Float::MAX
|
|
590
598
|
|
|
591
|
-
while (
|
|
599
|
+
while (next_token = lexed[next_token_index]&.first)
|
|
592
600
|
next_token_index += 1
|
|
593
|
-
next_next_token = lexed[next_token_index]
|
|
601
|
+
next_next_token, _ = lexed[next_token_index]
|
|
594
602
|
first_token_on_line = next_token.location.start_column == 0
|
|
595
603
|
|
|
596
604
|
# String content inside nested heredocs and interpolation is ignored
|
|
@@ -761,12 +769,12 @@ module Prism
|
|
|
761
769
|
elsif (value = scanner.scan(/M-\\?(?=[[:print:]])/))
|
|
762
770
|
# \M-x where x is an ASCII printable character
|
|
763
771
|
escape_read(result, scanner, control, true)
|
|
764
|
-
elsif (byte = scanner.
|
|
772
|
+
elsif (byte = scanner.scan_byte)
|
|
765
773
|
# Something else after an escape.
|
|
766
|
-
if control && byte ==
|
|
774
|
+
if control && byte == 0x3f # ASCII '?'
|
|
767
775
|
result.append_as_bytes(escape_build(0x7f, false, meta))
|
|
768
776
|
else
|
|
769
|
-
result.append_as_bytes(escape_build(byte
|
|
777
|
+
result.append_as_bytes(escape_build(byte, control, meta))
|
|
770
778
|
end
|
|
771
779
|
end
|
|
772
780
|
end
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
2
3
|
|
|
3
4
|
begin
|
|
5
|
+
required_version = ">= 3.3.7.2"
|
|
6
|
+
gem "parser", required_version
|
|
4
7
|
require "parser"
|
|
5
8
|
rescue LoadError
|
|
6
|
-
warn(
|
|
9
|
+
warn(<<~MSG)
|
|
10
|
+
Error: Unable to load parser #{required_version}. \
|
|
11
|
+
Add `gem "parser"` to your Gemfile or run `bundle update parser`.
|
|
12
|
+
MSG
|
|
7
13
|
exit(1)
|
|
8
14
|
end
|
|
9
15
|
|
|
@@ -13,6 +19,13 @@ module Prism
|
|
|
13
19
|
# whitequark/parser gem's syntax tree. It inherits from the base parser for
|
|
14
20
|
# the parser gem, and overrides the parse* methods to parse with prism and
|
|
15
21
|
# then translate.
|
|
22
|
+
#
|
|
23
|
+
# Note that this version of the parser always parses using the latest
|
|
24
|
+
# version of Ruby syntax supported by Prism. If you want specific version
|
|
25
|
+
# support, use one of the version-specific subclasses, such as
|
|
26
|
+
# `Prism::Translation::Parser34`. If you want to parse using the same
|
|
27
|
+
# version of Ruby syntax as the currently running version of Ruby, use
|
|
28
|
+
# `Prism::Translation::ParserCurrent`.
|
|
16
29
|
class Parser < ::Parser::Base
|
|
17
30
|
Diagnostic = ::Parser::Diagnostic # :nodoc:
|
|
18
31
|
private_constant :Diagnostic
|
|
@@ -59,13 +72,19 @@ module Prism
|
|
|
59
72
|
# should be implemented as needed.
|
|
60
73
|
#
|
|
61
74
|
def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
|
|
75
|
+
if !builder.is_a?(Prism::Translation::Parser::Builder)
|
|
76
|
+
warn(<<~MSG, uplevel: 1, category: :deprecated)
|
|
77
|
+
[deprecation]: The builder passed to `Prism::Translation::Parser.new` is not a \
|
|
78
|
+
`Prism::Translation::Parser::Builder` subclass. This will raise in the next major version.
|
|
79
|
+
MSG
|
|
80
|
+
end
|
|
62
81
|
@parser = parser
|
|
63
82
|
|
|
64
83
|
super(builder)
|
|
65
84
|
end
|
|
66
85
|
|
|
67
86
|
def version # :nodoc:
|
|
68
|
-
|
|
87
|
+
35
|
|
69
88
|
end
|
|
70
89
|
|
|
71
90
|
# The default encoding for Ruby files is UTF-8.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# :markup: markdown
|
|
3
|
+
# typed: ignore
|
|
4
|
+
|
|
5
|
+
#
|
|
6
|
+
module Prism
|
|
7
|
+
module Translation
|
|
8
|
+
case RUBY_VERSION
|
|
9
|
+
when /^3\.3\./
|
|
10
|
+
ParserCurrent = Parser33
|
|
11
|
+
when /^3\.4\./
|
|
12
|
+
ParserCurrent = Parser34
|
|
13
|
+
when /^3\.5\./
|
|
14
|
+
ParserCurrent = Parser35
|
|
15
|
+
else
|
|
16
|
+
# Keep this in sync with released Ruby.
|
|
17
|
+
parser = Parser34
|
|
18
|
+
major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
|
|
19
|
+
warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
|
|
20
|
+
"but you are running #{major}.#{minor}."
|
|
21
|
+
ParserCurrent = parser
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|