prism 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +23 -1
  3. data/Makefile +2 -1
  4. data/README.md +1 -0
  5. data/config.yml +264 -37
  6. data/docs/parser_translation.md +8 -23
  7. data/docs/ripper_translation.md +1 -1
  8. data/ext/prism/api_node.c +2 -0
  9. data/ext/prism/extension.c +14 -1
  10. data/ext/prism/extension.h +1 -1
  11. data/include/prism/ast.h +275 -49
  12. data/include/prism/diagnostic.h +4 -0
  13. data/include/prism/options.h +43 -3
  14. data/include/prism/regexp.h +2 -2
  15. data/include/prism/util/pm_buffer.h +8 -0
  16. data/include/prism/util/pm_integer.h +4 -0
  17. data/include/prism/util/pm_list.h +6 -0
  18. data/include/prism/util/pm_string.h +12 -2
  19. data/include/prism/version.h +2 -2
  20. data/include/prism.h +39 -14
  21. data/lib/prism/compiler.rb +456 -151
  22. data/lib/prism/desugar_compiler.rb +1 -0
  23. data/lib/prism/dispatcher.rb +16 -0
  24. data/lib/prism/dot_visitor.rb +5 -1
  25. data/lib/prism/dsl.rb +3 -0
  26. data/lib/prism/ffi.rb +17 -7
  27. data/lib/prism/inspect_visitor.rb +3 -0
  28. data/lib/prism/lex_compat.rb +1 -0
  29. data/lib/prism/mutation_compiler.rb +3 -0
  30. data/lib/prism/node.rb +506 -335
  31. data/lib/prism/node_ext.rb +4 -1
  32. data/lib/prism/pack.rb +2 -0
  33. data/lib/prism/parse_result/comments.rb +1 -0
  34. data/lib/prism/parse_result/errors.rb +1 -0
  35. data/lib/prism/parse_result/newlines.rb +1 -0
  36. data/lib/prism/parse_result.rb +1 -0
  37. data/lib/prism/pattern.rb +1 -0
  38. data/lib/prism/polyfill/scan_byte.rb +14 -0
  39. data/lib/prism/polyfill/warn.rb +42 -0
  40. data/lib/prism/reflection.rb +3 -0
  41. data/lib/prism/relocation.rb +1 -0
  42. data/lib/prism/serialize.rb +24 -19
  43. data/lib/prism/string_query.rb +1 -0
  44. data/lib/prism/translation/parser/builder.rb +1 -0
  45. data/lib/prism/translation/parser/compiler.rb +47 -25
  46. data/lib/prism/translation/parser/lexer.rb +29 -21
  47. data/lib/prism/translation/parser.rb +13 -1
  48. data/lib/prism/translation/parser33.rb +1 -0
  49. data/lib/prism/translation/parser34.rb +1 -0
  50. data/lib/prism/translation/parser35.rb +1 -0
  51. data/lib/prism/translation/parser_current.rb +24 -0
  52. data/lib/prism/translation/ripper/sexp.rb +1 -0
  53. data/lib/prism/translation/ripper.rb +17 -1
  54. data/lib/prism/translation/ruby_parser.rb +286 -3
  55. data/lib/prism/translation.rb +2 -0
  56. data/lib/prism/visitor.rb +457 -152
  57. data/lib/prism.rb +2 -0
  58. data/prism.gemspec +5 -1
  59. data/rbi/prism/dsl.rbi +3 -3
  60. data/rbi/prism/node.rbi +21 -9
  61. data/sig/prism/dispatcher.rbs +3 -0
  62. data/sig/prism/dsl.rbs +3 -3
  63. data/sig/prism/node.rbs +444 -30
  64. data/sig/prism/node_ext.rbs +84 -17
  65. data/sig/prism/parse_result/comments.rbs +38 -0
  66. data/sig/prism/parse_result.rbs +4 -0
  67. data/sig/prism/reflection.rbs +1 -1
  68. data/src/diagnostic.c +7 -1
  69. data/src/node.c +2 -0
  70. data/src/options.c +2 -2
  71. data/src/prettyprint.c +2 -0
  72. data/src/prism.c +252 -130
  73. data/src/serialize.c +2 -0
  74. data/src/token_type.c +36 -34
  75. metadata +6 -2
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
4
+ #--
3
5
  # Here we are reopening the prism module to provide methods on nodes that aren't
4
6
  # templated and are meant as convenience methods.
7
+ #++
5
8
  module Prism
6
9
  class Node
7
10
  def deprecated(*replacements) # :nodoc:
@@ -9,7 +12,7 @@ module Prism
9
12
  location = location[0].label if location
10
13
  suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
11
14
 
12
- warn(<<~MSG, category: :deprecated)
15
+ warn(<<~MSG, uplevel: 1, category: :deprecated)
13
16
  [deprecation]: #{self.class}##{location} is deprecated and will be \
14
17
  removed in the next major version. Use #{suggest.join("/")} instead.
15
18
  #{(caller(1, 3) || []).join("\n")}
data/lib/prism/pack.rb CHANGED
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
  # typed: ignore
3
4
 
5
+ #
4
6
  module Prism
5
7
  # A parser for the pack template language.
6
8
  module Pack
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class ParseResult < Result
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "stringio"
4
5
 
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class ParseResult < Result
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # This represents a source of Ruby code that has been parsed. It is used in
data/lib/prism/pattern.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # A pattern is an object that wraps a Ruby pattern matching expression. The
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+
5
+ # Polyfill for StringScanner#scan_byte, which didn't exist until Ruby 3.4.
6
+ if !(StringScanner.instance_methods.include?(:scan_byte))
7
+ StringScanner.include(
8
+ Module.new {
9
+ def scan_byte # :nodoc:
10
+ get_byte&.b&.ord
11
+ end
12
+ }
13
+ )
14
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Polyfill for Kernel#warn with the category parameter. Not all Ruby engines
4
+ # have Method#parameters implemented, so we check the arity instead if
5
+ # necessary.
6
+ if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.parameters.none? { |_, name| name == :category } : (method.arity == -1)
7
+ Kernel.prepend(
8
+ Module.new {
9
+ def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
10
+ uplevel =
11
+ case uplevel
12
+ when nil
13
+ 1
14
+ when Integer
15
+ uplevel + 1
16
+ else
17
+ uplevel.to_int + 1
18
+ end
19
+
20
+ super(*msgs, uplevel: uplevel)
21
+ end
22
+ }
23
+ )
24
+
25
+ Object.prepend(
26
+ Module.new {
27
+ def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
28
+ uplevel =
29
+ case uplevel
30
+ when nil
31
+ 1
32
+ when Integer
33
+ uplevel + 1
34
+ else
35
+ uplevel.to_int + 1
36
+ end
37
+
38
+ super(*msgs, uplevel: uplevel)
39
+ end
40
+ }
41
+ )
42
+ end
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/reflection.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # Prism parses deterministically for the same input. This provides a nice
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/serialize.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  require "stringio"
@@ -18,7 +21,7 @@ module Prism
18
21
 
19
22
  # The minor version of prism that we are expecting to find in the serialized
20
23
  # strings.
21
- MINOR_VERSION = 4
24
+ MINOR_VERSION = 5
22
25
 
23
26
  # The patch version of prism that we are expecting to find in the serialized
24
27
  # strings.
@@ -435,6 +438,8 @@ module Prism
435
438
  :expect_for_delimiter,
436
439
  :expect_ident_req_parameter,
437
440
  :expect_in_delimiter,
441
+ :expect_lparen_after_not_lparen,
442
+ :expect_lparen_after_not_other,
438
443
  :expect_lparen_req_parameter,
439
444
  :expect_message,
440
445
  :expect_rbracket,
@@ -2219,8 +2224,21 @@ module Prism
2219
2224
  TOKEN_TYPES = [
2220
2225
  nil,
2221
2226
  :EOF,
2222
- :MISSING,
2223
- :NOT_PROVIDED,
2227
+ :BRACE_RIGHT,
2228
+ :COMMA,
2229
+ :EMBEXPR_END,
2230
+ :KEYWORD_DO,
2231
+ :KEYWORD_ELSE,
2232
+ :KEYWORD_ELSIF,
2233
+ :KEYWORD_END,
2234
+ :KEYWORD_ENSURE,
2235
+ :KEYWORD_IN,
2236
+ :KEYWORD_RESCUE,
2237
+ :KEYWORD_THEN,
2238
+ :KEYWORD_WHEN,
2239
+ :NEWLINE,
2240
+ :PARENTHESIS_RIGHT,
2241
+ :SEMICOLON,
2224
2242
  :AMPERSAND,
2225
2243
  :AMPERSAND_AMPERSAND,
2226
2244
  :AMPERSAND_AMPERSAND_EQUAL,
@@ -2232,7 +2250,6 @@ module Prism
2232
2250
  :BANG_EQUAL,
2233
2251
  :BANG_TILDE,
2234
2252
  :BRACE_LEFT,
2235
- :BRACE_RIGHT,
2236
2253
  :BRACKET_LEFT,
2237
2254
  :BRACKET_LEFT_ARRAY,
2238
2255
  :BRACKET_LEFT_RIGHT,
@@ -2244,7 +2261,6 @@ module Prism
2244
2261
  :CLASS_VARIABLE,
2245
2262
  :COLON,
2246
2263
  :COLON_COLON,
2247
- :COMMA,
2248
2264
  :COMMENT,
2249
2265
  :CONSTANT,
2250
2266
  :DOT,
@@ -2254,7 +2270,6 @@ module Prism
2254
2270
  :EMBDOC_END,
2255
2271
  :EMBDOC_LINE,
2256
2272
  :EMBEXPR_BEGIN,
2257
- :EMBEXPR_END,
2258
2273
  :EMBVAR,
2259
2274
  :EQUAL,
2260
2275
  :EQUAL_EQUAL,
@@ -2288,38 +2303,29 @@ module Prism
2288
2303
  :KEYWORD_CLASS,
2289
2304
  :KEYWORD_DEF,
2290
2305
  :KEYWORD_DEFINED,
2291
- :KEYWORD_DO,
2292
2306
  :KEYWORD_DO_LOOP,
2293
- :KEYWORD_ELSE,
2294
- :KEYWORD_ELSIF,
2295
- :KEYWORD_END,
2296
2307
  :KEYWORD_END_UPCASE,
2297
- :KEYWORD_ENSURE,
2298
2308
  :KEYWORD_FALSE,
2299
2309
  :KEYWORD_FOR,
2300
2310
  :KEYWORD_IF,
2301
2311
  :KEYWORD_IF_MODIFIER,
2302
- :KEYWORD_IN,
2303
2312
  :KEYWORD_MODULE,
2304
2313
  :KEYWORD_NEXT,
2305
2314
  :KEYWORD_NIL,
2306
2315
  :KEYWORD_NOT,
2307
2316
  :KEYWORD_OR,
2308
2317
  :KEYWORD_REDO,
2309
- :KEYWORD_RESCUE,
2310
2318
  :KEYWORD_RESCUE_MODIFIER,
2311
2319
  :KEYWORD_RETRY,
2312
2320
  :KEYWORD_RETURN,
2313
2321
  :KEYWORD_SELF,
2314
2322
  :KEYWORD_SUPER,
2315
- :KEYWORD_THEN,
2316
2323
  :KEYWORD_TRUE,
2317
2324
  :KEYWORD_UNDEF,
2318
2325
  :KEYWORD_UNLESS,
2319
2326
  :KEYWORD_UNLESS_MODIFIER,
2320
2327
  :KEYWORD_UNTIL,
2321
2328
  :KEYWORD_UNTIL_MODIFIER,
2322
- :KEYWORD_WHEN,
2323
2329
  :KEYWORD_WHILE,
2324
2330
  :KEYWORD_WHILE_MODIFIER,
2325
2331
  :KEYWORD_YIELD,
@@ -2338,11 +2344,9 @@ module Prism
2338
2344
  :MINUS,
2339
2345
  :MINUS_EQUAL,
2340
2346
  :MINUS_GREATER,
2341
- :NEWLINE,
2342
2347
  :NUMBERED_REFERENCE,
2343
2348
  :PARENTHESIS_LEFT,
2344
2349
  :PARENTHESIS_LEFT_PARENTHESES,
2345
- :PARENTHESIS_RIGHT,
2346
2350
  :PERCENT,
2347
2351
  :PERCENT_EQUAL,
2348
2352
  :PERCENT_LOWER_I,
@@ -2359,7 +2363,6 @@ module Prism
2359
2363
  :QUESTION_MARK,
2360
2364
  :REGEXP_BEGIN,
2361
2365
  :REGEXP_END,
2362
- :SEMICOLON,
2363
2366
  :SLASH,
2364
2367
  :SLASH_EQUAL,
2365
2368
  :STAR,
@@ -2382,7 +2385,9 @@ module Prism
2382
2385
  :USTAR_STAR,
2383
2386
  :WORDS_SEP,
2384
2387
  :__END__,
2385
- ]
2388
+ :MISSING,
2389
+ :NOT_PROVIDED,
2390
+ ].freeze
2386
2391
 
2387
2392
  private_constant :MAJOR_VERSION, :MINOR_VERSION, :PATCH_VERSION
2388
2393
  private_constant :ConstantPool, :FastStringIO, :Loader, :TOKEN_TYPES
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # Query methods that allow categorizing strings based on their context for
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -133,8 +134,8 @@ module Prism
133
134
  def visit_assoc_node(node)
134
135
  key = node.key
135
136
 
136
- if in_pattern
137
- if node.value.is_a?(ImplicitNode)
137
+ if node.value.is_a?(ImplicitNode)
138
+ if in_pattern
138
139
  if key.is_a?(SymbolNode)
139
140
  if key.opening.nil?
140
141
  builder.match_hash_var([key.unescaped, srange(key.location)])
@@ -144,23 +145,19 @@ module Prism
144
145
  else
145
146
  builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
146
147
  end
147
- elsif key.opening.nil?
148
- builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value))
149
148
  else
150
- builder.pair_quoted(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc), visit(node.value))
151
- end
152
- elsif node.value.is_a?(ImplicitNode)
153
- value = node.value.value
149
+ value = node.value.value
154
150
 
155
- implicit_value = if value.is_a?(CallNode)
156
- builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
157
- elsif value.is_a?(ConstantReadNode)
158
- builder.const([value.name, srange(key.value_loc)])
159
- else
160
- builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
161
- end
151
+ implicit_value = if value.is_a?(CallNode)
152
+ builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
153
+ elsif value.is_a?(ConstantReadNode)
154
+ builder.const([value.name, srange(key.value_loc)])
155
+ else
156
+ builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
157
+ end
162
158
 
163
- builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
159
+ builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
160
+ end
164
161
  elsif node.operator_loc
165
162
  builder.pair(visit(key), token(node.operator_loc), visit(node.value))
166
163
  elsif key.is_a?(SymbolNode) && key.opening_loc.nil?
@@ -696,13 +693,37 @@ module Prism
696
693
  # defined?(a)
697
694
  # ^^^^^^^^^^^
698
695
  def visit_defined_node(node)
699
- builder.keyword_cmd(
700
- :defined?,
701
- token(node.keyword_loc),
702
- token(node.lparen_loc),
703
- [visit(node.value)],
704
- token(node.rparen_loc)
705
- )
696
+ # Very weird circumstances here where something like:
697
+ #
698
+ # defined?
699
+ # (1)
700
+ #
701
+ # gets parsed in Ruby as having only the `1` expression but in parser
702
+ # it gets parsed as having a begin. In this case we need to synthesize
703
+ # that begin to match parser's behavior.
704
+ if node.lparen_loc && node.keyword_loc.join(node.lparen_loc).slice.include?("\n")
705
+ builder.keyword_cmd(
706
+ :defined?,
707
+ token(node.keyword_loc),
708
+ nil,
709
+ [
710
+ builder.begin(
711
+ token(node.lparen_loc),
712
+ visit(node.value),
713
+ token(node.rparen_loc)
714
+ )
715
+ ],
716
+ nil
717
+ )
718
+ else
719
+ builder.keyword_cmd(
720
+ :defined?,
721
+ token(node.keyword_loc),
722
+ token(node.lparen_loc),
723
+ [visit(node.value)],
724
+ token(node.rparen_loc)
725
+ )
726
+ end
706
727
  end
707
728
 
708
729
  # if foo then bar else baz end
@@ -1032,7 +1053,7 @@ module Prism
1032
1053
  builder.index_asgn(
1033
1054
  visit(node.receiver),
1034
1055
  token(node.opening_loc),
1035
- visit_all(node.arguments.arguments),
1056
+ visit_all(node.arguments&.arguments || []),
1036
1057
  token(node.closing_loc),
1037
1058
  )
1038
1059
  end
@@ -1461,7 +1482,8 @@ module Prism
1461
1482
  # foo => ^(bar)
1462
1483
  # ^^^^^^
1463
1484
  def visit_pinned_expression_node(node)
1464
- expression = builder.begin(token(node.lparen_loc), visit(node.expression), token(node.rparen_loc))
1485
+ parts = node.expression.accept(copy_compiler(in_pattern: false)) # Don't treat * and similar as match_rest
1486
+ expression = builder.begin(token(node.lparen_loc), parts, token(node.rparen_loc))
1465
1487
  builder.pin(token(node.operator_loc), expression)
1466
1488
  end
1467
1489
 
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "strscan"
4
5
  require_relative "../../polyfill/append_as_bytes"
6
+ require_relative "../../polyfill/scan_byte"
5
7
 
6
8
  module Prism
7
9
  module Translation
@@ -200,8 +202,8 @@ module Prism
200
202
  # The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
201
203
  # The following token types are listed as those classified as `tLPAREN`.
202
204
  LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
203
- :kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
204
- :tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
205
+ :kBREAK, :tCARET, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
206
+ :tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS, :tLCURLY
205
207
  ])
206
208
 
207
209
  # Types of tokens that are allowed to continue a method call with comments in-between.
@@ -275,20 +277,20 @@ module Prism
275
277
  when :tCOMMENT
276
278
  if token.type == :EMBDOC_BEGIN
277
279
 
278
- while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1)
280
+ while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
279
281
  value += next_token.value
280
282
  index += 1
281
283
  end
282
284
 
283
285
  value += next_token.value
284
- location = range(token.location.start_offset, lexed[index][0].location.end_offset)
286
+ location = range(token.location.start_offset, next_token.location.end_offset)
285
287
  index += 1
286
288
  else
287
289
  is_at_eol = value.chomp!.nil?
288
290
  location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
289
291
 
290
- prev_token = lexed[index - 2][0] if index - 2 >= 0
291
- next_token = lexed[index][0]
292
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
293
+ next_token, _ = lexed[index]
292
294
 
293
295
  is_inline_comment = prev_token&.location&.start_line == token.location.start_line
294
296
  if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
@@ -307,7 +309,7 @@ module Prism
307
309
  end
308
310
  end
309
311
  when :tNL
310
- next_token = next_token = lexed[index][0]
312
+ next_token, _ = lexed[index]
311
313
  # Newlines after comments are emitted out of order.
312
314
  if next_token&.type == :COMMENT
313
315
  comment_newline_location = location
@@ -344,8 +346,8 @@ module Prism
344
346
  location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
345
347
  value = nil
346
348
  when :tSTRING_BEG
347
- next_token = lexed[index][0]
348
- next_next_token = lexed[index + 1][0]
349
+ next_token, _ = lexed[index]
350
+ next_next_token, _ = lexed[index + 1]
349
351
  basic_quotes = value == '"' || value == "'"
350
352
 
351
353
  if basic_quotes && next_token&.type == :STRING_END
@@ -413,7 +415,8 @@ module Prism
413
415
  while token.type == :STRING_CONTENT
414
416
  current_length += token.value.bytesize
415
417
  # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
416
- is_first_token_on_line = lexed[index - 1] && token.location.start_line != lexed[index - 2][0].location&.start_line
418
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
419
+ is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
417
420
  # The parser gem only removes indentation when the heredoc is not nested
418
421
  not_nested = heredoc_stack.size == 1
419
422
  if is_percent_array
@@ -423,11 +426,16 @@ module Prism
423
426
  end
424
427
 
425
428
  current_string << unescape_string(value, quote_stack.last)
426
- if (backslash_count = token.value[/(\\{1,})\n/, 1]&.length).nil? || backslash_count.even? || !interpolation?(quote_stack.last)
429
+ relevant_backslash_count = if quote_stack.last.start_with?("%W", "%I")
430
+ 0 # the last backslash escapes the newline
431
+ else
432
+ token.value[/(\\{1,})\n/, 1]&.length || 0
433
+ end
434
+ if relevant_backslash_count.even? || !interpolation?(quote_stack.last)
427
435
  tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
428
436
  break
429
437
  end
430
- token = lexed[index][0]
438
+ token, _ = lexed[index]
431
439
  index += 1
432
440
  end
433
441
  else
@@ -482,7 +490,7 @@ module Prism
482
490
  end
483
491
 
484
492
  if percent_array?(quote_stack.pop)
485
- prev_token = lexed[index - 2][0] if index - 2 >= 0
493
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
486
494
  empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
487
495
  ends_with_whitespace = prev_token&.type == :WORDS_SEP
488
496
  # parser always emits a space token after content in a percent array, even if no actual whitespace is present.
@@ -491,7 +499,7 @@ module Prism
491
499
  end
492
500
  end
493
501
  when :tSYMBEG
494
- if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
502
+ if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
495
503
  next_location = token.location.join(next_token.location)
496
504
  type = :tSYMBOL
497
505
  value = next_token.value
@@ -506,13 +514,13 @@ module Prism
506
514
  type = :tIDENTIFIER
507
515
  end
508
516
  when :tXSTRING_BEG
509
- if (next_token = lexed[index][0]) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
517
+ if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
510
518
  # self.`()
511
519
  type = :tBACK_REF2
512
520
  end
513
521
  quote_stack.push(value)
514
522
  when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
515
- if (next_token = lexed[index][0]) && next_token.type == :WORDS_SEP
523
+ if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
516
524
  index += 1
517
525
  end
518
526
 
@@ -588,9 +596,9 @@ module Prism
588
596
  previous_line = -1
589
597
  result = Float::MAX
590
598
 
591
- while (lexed[next_token_index] && next_token = lexed[next_token_index][0])
599
+ while (next_token = lexed[next_token_index]&.first)
592
600
  next_token_index += 1
593
- next_next_token = lexed[next_token_index] && lexed[next_token_index][0]
601
+ next_next_token, _ = lexed[next_token_index]
594
602
  first_token_on_line = next_token.location.start_column == 0
595
603
 
596
604
  # String content inside nested heredocs and interpolation is ignored
@@ -761,12 +769,12 @@ module Prism
761
769
  elsif (value = scanner.scan(/M-\\?(?=[[:print:]])/))
762
770
  # \M-x where x is an ASCII printable character
763
771
  escape_read(result, scanner, control, true)
764
- elsif (byte = scanner.get_byte)
772
+ elsif (byte = scanner.scan_byte)
765
773
  # Something else after an escape.
766
- if control && byte == "?"
774
+ if control && byte == 0x3f # ASCII '?'
767
775
  result.append_as_bytes(escape_build(0x7f, false, meta))
768
776
  else
769
- result.append_as_bytes(escape_build(byte.ord, control, meta))
777
+ result.append_as_bytes(escape_build(byte, control, meta))
770
778
  end
771
779
  end
772
780
  end
@@ -1,9 +1,15 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  begin
5
+ required_version = ">= 3.3.7.2"
6
+ gem "parser", required_version
4
7
  require "parser"
5
8
  rescue LoadError
6
- warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.})
9
+ warn(<<~MSG)
10
+ Error: Unable to load parser #{required_version}. \
11
+ Add `gem "parser"` to your Gemfile or run `bundle update parser`.
12
+ MSG
7
13
  exit(1)
8
14
  end
9
15
 
@@ -59,6 +65,12 @@ module Prism
59
65
  # should be implemented as needed.
60
66
  #
61
67
  def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
68
+ if !builder.is_a?(Prism::Translation::Parser::Builder)
69
+ warn(<<~MSG, uplevel: 1, category: :deprecated)
70
+ [deprecation]: The builder passed to `Prism::Translation::Parser.new` is not a \
71
+ `Prism::Translation::Parser::Builder` subclass. This will raise in the next major version.
72
+ MSG
73
+ end
62
74
  @parser = parser
63
75
 
64
76
  super(builder)
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+ # :markup: markdown
3
+ # typed: ignore
4
+
5
+ #
6
+ module Prism
7
+ module Translation
8
+ case RUBY_VERSION
9
+ when /^3\.3\./
10
+ ParserCurrent = Parser33
11
+ when /^3\.4\./
12
+ ParserCurrent = Parser34
13
+ when /^3\.5\./
14
+ ParserCurrent = Parser35
15
+ else
16
+ # Keep this in sync with released Ruby.
17
+ parser = Parser34
18
+ major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
19
+ warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
20
+ "but you are running #{major}.#{minor}."
21
+ ParserCurrent = parser
22
+ end
23
+ end
24
+ end
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require_relative "../ripper"
4
5