prism 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +51 -1
  3. data/Makefile +4 -2
  4. data/README.md +2 -0
  5. data/config.yml +266 -38
  6. data/docs/design.md +2 -2
  7. data/docs/parser_translation.md +8 -23
  8. data/docs/releasing.md +5 -24
  9. data/docs/ripper_translation.md +1 -1
  10. data/ext/prism/api_node.c +2 -0
  11. data/ext/prism/extension.c +25 -3
  12. data/ext/prism/extension.h +1 -1
  13. data/include/prism/ast.h +306 -50
  14. data/include/prism/diagnostic.h +5 -0
  15. data/include/prism/options.h +43 -3
  16. data/include/prism/regexp.h +2 -2
  17. data/include/prism/util/pm_buffer.h +8 -0
  18. data/include/prism/util/pm_integer.h +4 -0
  19. data/include/prism/util/pm_list.h +6 -0
  20. data/include/prism/util/pm_string.h +12 -2
  21. data/include/prism/version.h +2 -2
  22. data/include/prism.h +39 -14
  23. data/lib/prism/compiler.rb +456 -151
  24. data/lib/prism/desugar_compiler.rb +1 -0
  25. data/lib/prism/dispatcher.rb +16 -0
  26. data/lib/prism/dot_visitor.rb +5 -1
  27. data/lib/prism/dsl.rb +3 -0
  28. data/lib/prism/ffi.rb +25 -9
  29. data/lib/prism/inspect_visitor.rb +3 -0
  30. data/lib/prism/lex_compat.rb +1 -0
  31. data/lib/prism/mutation_compiler.rb +3 -0
  32. data/lib/prism/node.rb +507 -336
  33. data/lib/prism/node_ext.rb +4 -1
  34. data/lib/prism/pack.rb +2 -0
  35. data/lib/prism/parse_result/comments.rb +1 -0
  36. data/lib/prism/parse_result/errors.rb +1 -0
  37. data/lib/prism/parse_result/newlines.rb +1 -0
  38. data/lib/prism/parse_result.rb +1 -0
  39. data/lib/prism/pattern.rb +1 -0
  40. data/lib/prism/polyfill/scan_byte.rb +14 -0
  41. data/lib/prism/polyfill/warn.rb +36 -0
  42. data/lib/prism/reflection.rb +3 -0
  43. data/lib/prism/relocation.rb +1 -0
  44. data/lib/prism/serialize.rb +25 -19
  45. data/lib/prism/string_query.rb +1 -0
  46. data/lib/prism/translation/parser/builder.rb +1 -0
  47. data/lib/prism/translation/parser/compiler.rb +47 -25
  48. data/lib/prism/translation/parser/lexer.rb +29 -21
  49. data/lib/prism/translation/parser.rb +21 -2
  50. data/lib/prism/translation/parser33.rb +1 -0
  51. data/lib/prism/translation/parser34.rb +1 -0
  52. data/lib/prism/translation/parser35.rb +1 -0
  53. data/lib/prism/translation/parser_current.rb +24 -0
  54. data/lib/prism/translation/ripper/sexp.rb +1 -0
  55. data/lib/prism/translation/ripper.rb +17 -1
  56. data/lib/prism/translation/ruby_parser.rb +287 -4
  57. data/lib/prism/translation.rb +2 -0
  58. data/lib/prism/visitor.rb +457 -152
  59. data/lib/prism.rb +23 -0
  60. data/prism.gemspec +5 -1
  61. data/rbi/prism/dsl.rbi +3 -3
  62. data/rbi/prism/node.rbi +21 -9
  63. data/sig/prism/dispatcher.rbs +3 -0
  64. data/sig/prism/dsl.rbs +3 -3
  65. data/sig/prism/node.rbs +444 -30
  66. data/sig/prism/node_ext.rbs +84 -17
  67. data/sig/prism/parse_result/comments.rbs +38 -0
  68. data/sig/prism/parse_result.rbs +4 -0
  69. data/sig/prism/reflection.rbs +1 -1
  70. data/sig/prism.rbs +4 -0
  71. data/src/diagnostic.c +9 -1
  72. data/src/node.c +2 -0
  73. data/src/options.c +2 -2
  74. data/src/prettyprint.c +2 -0
  75. data/src/prism.c +324 -147
  76. data/src/serialize.c +2 -0
  77. data/src/token_type.c +36 -34
  78. data/src/util/pm_string.c +6 -8
  79. metadata +7 -3
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
4
+ #--
3
5
  # Here we are reopening the prism module to provide methods on nodes that aren't
4
6
  # templated and are meant as convenience methods.
7
+ #++
5
8
  module Prism
6
9
  class Node
7
10
  def deprecated(*replacements) # :nodoc:
@@ -9,7 +12,7 @@ module Prism
9
12
  location = location[0].label if location
10
13
  suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
11
14
 
12
- warn(<<~MSG, category: :deprecated)
15
+ warn(<<~MSG, uplevel: 1, category: :deprecated)
13
16
  [deprecation]: #{self.class}##{location} is deprecated and will be \
14
17
  removed in the next major version. Use #{suggest.join("/")} instead.
15
18
  #{(caller(1, 3) || []).join("\n")}
data/lib/prism/pack.rb CHANGED
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
  # typed: ignore
3
4
 
5
+ #
4
6
  module Prism
5
7
  # A parser for the pack template language.
6
8
  module Pack
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class ParseResult < Result
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "stringio"
4
5
 
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class ParseResult < Result
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # This represents a source of Ruby code that has been parsed. It is used in
data/lib/prism/pattern.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # A pattern is an object that wraps a Ruby pattern matching expression. The
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+
5
+ # Polyfill for StringScanner#scan_byte, which didn't exist until Ruby 3.4.
6
+ if !(StringScanner.instance_methods.include?(:scan_byte))
7
+ StringScanner.include(
8
+ Module.new {
9
+ def scan_byte # :nodoc:
10
+ get_byte&.b&.ord
11
+ end
12
+ }
13
+ )
14
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Polyfill for Kernel#warn with the category parameter. Not all Ruby engines
4
+ # have Method#parameters implemented, so we check the arity instead if
5
+ # necessary.
6
+ if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.parameters.none? { |_, name| name == :category } : (method.arity == -1)
7
+ Kernel.prepend(
8
+ Module.new {
9
+ def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
10
+ case uplevel
11
+ when nil
12
+ super(*msgs)
13
+ when Integer
14
+ super(*msgs, uplevel: uplevel + 1)
15
+ else
16
+ super(*msgs, uplevel: uplevel.to_int + 1)
17
+ end
18
+ end
19
+ }
20
+ )
21
+
22
+ Object.prepend(
23
+ Module.new {
24
+ def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
25
+ case uplevel
26
+ when nil
27
+ super(*msgs)
28
+ when Integer
29
+ super(*msgs, uplevel: uplevel + 1)
30
+ else
31
+ super(*msgs, uplevel: uplevel.to_int + 1)
32
+ end
33
+ end
34
+ }
35
+ )
36
+ end
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/reflection.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # Prism parses deterministically for the same input. This provides a nice
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/serialize.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  require "stringio"
@@ -18,7 +21,7 @@ module Prism
18
21
 
19
22
  # The minor version of prism that we are expecting to find in the serialized
20
23
  # strings.
21
- MINOR_VERSION = 4
24
+ MINOR_VERSION = 6
22
25
 
23
26
  # The patch version of prism that we are expecting to find in the serialized
24
27
  # strings.
@@ -394,6 +397,7 @@ module Prism
394
397
  :conditional_while_predicate,
395
398
  :constant_path_colon_colon_constant,
396
399
  :def_endless,
400
+ :def_endless_parameters,
397
401
  :def_endless_setter,
398
402
  :def_name,
399
403
  :def_params_term,
@@ -435,6 +439,8 @@ module Prism
435
439
  :expect_for_delimiter,
436
440
  :expect_ident_req_parameter,
437
441
  :expect_in_delimiter,
442
+ :expect_lparen_after_not_lparen,
443
+ :expect_lparen_after_not_other,
438
444
  :expect_lparen_req_parameter,
439
445
  :expect_message,
440
446
  :expect_rbracket,
@@ -2219,8 +2225,21 @@ module Prism
2219
2225
  TOKEN_TYPES = [
2220
2226
  nil,
2221
2227
  :EOF,
2222
- :MISSING,
2223
- :NOT_PROVIDED,
2228
+ :BRACE_RIGHT,
2229
+ :COMMA,
2230
+ :EMBEXPR_END,
2231
+ :KEYWORD_DO,
2232
+ :KEYWORD_ELSE,
2233
+ :KEYWORD_ELSIF,
2234
+ :KEYWORD_END,
2235
+ :KEYWORD_ENSURE,
2236
+ :KEYWORD_IN,
2237
+ :KEYWORD_RESCUE,
2238
+ :KEYWORD_THEN,
2239
+ :KEYWORD_WHEN,
2240
+ :NEWLINE,
2241
+ :PARENTHESIS_RIGHT,
2242
+ :SEMICOLON,
2224
2243
  :AMPERSAND,
2225
2244
  :AMPERSAND_AMPERSAND,
2226
2245
  :AMPERSAND_AMPERSAND_EQUAL,
@@ -2232,7 +2251,6 @@ module Prism
2232
2251
  :BANG_EQUAL,
2233
2252
  :BANG_TILDE,
2234
2253
  :BRACE_LEFT,
2235
- :BRACE_RIGHT,
2236
2254
  :BRACKET_LEFT,
2237
2255
  :BRACKET_LEFT_ARRAY,
2238
2256
  :BRACKET_LEFT_RIGHT,
@@ -2244,7 +2262,6 @@ module Prism
2244
2262
  :CLASS_VARIABLE,
2245
2263
  :COLON,
2246
2264
  :COLON_COLON,
2247
- :COMMA,
2248
2265
  :COMMENT,
2249
2266
  :CONSTANT,
2250
2267
  :DOT,
@@ -2254,7 +2271,6 @@ module Prism
2254
2271
  :EMBDOC_END,
2255
2272
  :EMBDOC_LINE,
2256
2273
  :EMBEXPR_BEGIN,
2257
- :EMBEXPR_END,
2258
2274
  :EMBVAR,
2259
2275
  :EQUAL,
2260
2276
  :EQUAL_EQUAL,
@@ -2288,38 +2304,29 @@ module Prism
2288
2304
  :KEYWORD_CLASS,
2289
2305
  :KEYWORD_DEF,
2290
2306
  :KEYWORD_DEFINED,
2291
- :KEYWORD_DO,
2292
2307
  :KEYWORD_DO_LOOP,
2293
- :KEYWORD_ELSE,
2294
- :KEYWORD_ELSIF,
2295
- :KEYWORD_END,
2296
2308
  :KEYWORD_END_UPCASE,
2297
- :KEYWORD_ENSURE,
2298
2309
  :KEYWORD_FALSE,
2299
2310
  :KEYWORD_FOR,
2300
2311
  :KEYWORD_IF,
2301
2312
  :KEYWORD_IF_MODIFIER,
2302
- :KEYWORD_IN,
2303
2313
  :KEYWORD_MODULE,
2304
2314
  :KEYWORD_NEXT,
2305
2315
  :KEYWORD_NIL,
2306
2316
  :KEYWORD_NOT,
2307
2317
  :KEYWORD_OR,
2308
2318
  :KEYWORD_REDO,
2309
- :KEYWORD_RESCUE,
2310
2319
  :KEYWORD_RESCUE_MODIFIER,
2311
2320
  :KEYWORD_RETRY,
2312
2321
  :KEYWORD_RETURN,
2313
2322
  :KEYWORD_SELF,
2314
2323
  :KEYWORD_SUPER,
2315
- :KEYWORD_THEN,
2316
2324
  :KEYWORD_TRUE,
2317
2325
  :KEYWORD_UNDEF,
2318
2326
  :KEYWORD_UNLESS,
2319
2327
  :KEYWORD_UNLESS_MODIFIER,
2320
2328
  :KEYWORD_UNTIL,
2321
2329
  :KEYWORD_UNTIL_MODIFIER,
2322
- :KEYWORD_WHEN,
2323
2330
  :KEYWORD_WHILE,
2324
2331
  :KEYWORD_WHILE_MODIFIER,
2325
2332
  :KEYWORD_YIELD,
@@ -2338,11 +2345,9 @@ module Prism
2338
2345
  :MINUS,
2339
2346
  :MINUS_EQUAL,
2340
2347
  :MINUS_GREATER,
2341
- :NEWLINE,
2342
2348
  :NUMBERED_REFERENCE,
2343
2349
  :PARENTHESIS_LEFT,
2344
2350
  :PARENTHESIS_LEFT_PARENTHESES,
2345
- :PARENTHESIS_RIGHT,
2346
2351
  :PERCENT,
2347
2352
  :PERCENT_EQUAL,
2348
2353
  :PERCENT_LOWER_I,
@@ -2359,7 +2364,6 @@ module Prism
2359
2364
  :QUESTION_MARK,
2360
2365
  :REGEXP_BEGIN,
2361
2366
  :REGEXP_END,
2362
- :SEMICOLON,
2363
2367
  :SLASH,
2364
2368
  :SLASH_EQUAL,
2365
2369
  :STAR,
@@ -2382,7 +2386,9 @@ module Prism
2382
2386
  :USTAR_STAR,
2383
2387
  :WORDS_SEP,
2384
2388
  :__END__,
2385
- ]
2389
+ :MISSING,
2390
+ :NOT_PROVIDED,
2391
+ ].freeze
2386
2392
 
2387
2393
  private_constant :MAJOR_VERSION, :MINOR_VERSION, :PATCH_VERSION
2388
2394
  private_constant :ConstantPool, :FastStringIO, :Loader, :TOKEN_TYPES
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  # Query methods that allow categorizing strings based on their context for
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -133,8 +134,8 @@ module Prism
133
134
  def visit_assoc_node(node)
134
135
  key = node.key
135
136
 
136
- if in_pattern
137
- if node.value.is_a?(ImplicitNode)
137
+ if node.value.is_a?(ImplicitNode)
138
+ if in_pattern
138
139
  if key.is_a?(SymbolNode)
139
140
  if key.opening.nil?
140
141
  builder.match_hash_var([key.unescaped, srange(key.location)])
@@ -144,23 +145,19 @@ module Prism
144
145
  else
145
146
  builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
146
147
  end
147
- elsif key.opening.nil?
148
- builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value))
149
148
  else
150
- builder.pair_quoted(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc), visit(node.value))
151
- end
152
- elsif node.value.is_a?(ImplicitNode)
153
- value = node.value.value
149
+ value = node.value.value
154
150
 
155
- implicit_value = if value.is_a?(CallNode)
156
- builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
157
- elsif value.is_a?(ConstantReadNode)
158
- builder.const([value.name, srange(key.value_loc)])
159
- else
160
- builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
161
- end
151
+ implicit_value = if value.is_a?(CallNode)
152
+ builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
153
+ elsif value.is_a?(ConstantReadNode)
154
+ builder.const([value.name, srange(key.value_loc)])
155
+ else
156
+ builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
157
+ end
162
158
 
163
- builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
159
+ builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
160
+ end
164
161
  elsif node.operator_loc
165
162
  builder.pair(visit(key), token(node.operator_loc), visit(node.value))
166
163
  elsif key.is_a?(SymbolNode) && key.opening_loc.nil?
@@ -696,13 +693,37 @@ module Prism
696
693
  # defined?(a)
697
694
  # ^^^^^^^^^^^
698
695
  def visit_defined_node(node)
699
- builder.keyword_cmd(
700
- :defined?,
701
- token(node.keyword_loc),
702
- token(node.lparen_loc),
703
- [visit(node.value)],
704
- token(node.rparen_loc)
705
- )
696
+ # Very weird circumstances here where something like:
697
+ #
698
+ # defined?
699
+ # (1)
700
+ #
701
+ # gets parsed in Ruby as having only the `1` expression but in parser
702
+ # it gets parsed as having a begin. In this case we need to synthesize
703
+ # that begin to match parser's behavior.
704
+ if node.lparen_loc && node.keyword_loc.join(node.lparen_loc).slice.include?("\n")
705
+ builder.keyword_cmd(
706
+ :defined?,
707
+ token(node.keyword_loc),
708
+ nil,
709
+ [
710
+ builder.begin(
711
+ token(node.lparen_loc),
712
+ visit(node.value),
713
+ token(node.rparen_loc)
714
+ )
715
+ ],
716
+ nil
717
+ )
718
+ else
719
+ builder.keyword_cmd(
720
+ :defined?,
721
+ token(node.keyword_loc),
722
+ token(node.lparen_loc),
723
+ [visit(node.value)],
724
+ token(node.rparen_loc)
725
+ )
726
+ end
706
727
  end
707
728
 
708
729
  # if foo then bar else baz end
@@ -1032,7 +1053,7 @@ module Prism
1032
1053
  builder.index_asgn(
1033
1054
  visit(node.receiver),
1034
1055
  token(node.opening_loc),
1035
- visit_all(node.arguments.arguments),
1056
+ visit_all(node.arguments&.arguments || []),
1036
1057
  token(node.closing_loc),
1037
1058
  )
1038
1059
  end
@@ -1461,7 +1482,8 @@ module Prism
1461
1482
  # foo => ^(bar)
1462
1483
  # ^^^^^^
1463
1484
  def visit_pinned_expression_node(node)
1464
- expression = builder.begin(token(node.lparen_loc), visit(node.expression), token(node.rparen_loc))
1485
+ parts = node.expression.accept(copy_compiler(in_pattern: false)) # Don't treat * and similar as match_rest
1486
+ expression = builder.begin(token(node.lparen_loc), parts, token(node.rparen_loc))
1465
1487
  builder.pin(token(node.operator_loc), expression)
1466
1488
  end
1467
1489
 
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "strscan"
4
5
  require_relative "../../polyfill/append_as_bytes"
6
+ require_relative "../../polyfill/scan_byte"
5
7
 
6
8
  module Prism
7
9
  module Translation
@@ -200,8 +202,8 @@ module Prism
200
202
  # The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
201
203
  # The following token types are listed as those classified as `tLPAREN`.
202
204
  LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
203
- :kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
204
- :tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
205
+ :kBREAK, :tCARET, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
206
+ :tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS, :tLCURLY
205
207
  ])
206
208
 
207
209
  # Types of tokens that are allowed to continue a method call with comments in-between.
@@ -275,20 +277,20 @@ module Prism
275
277
  when :tCOMMENT
276
278
  if token.type == :EMBDOC_BEGIN
277
279
 
278
- while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1)
280
+ while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
279
281
  value += next_token.value
280
282
  index += 1
281
283
  end
282
284
 
283
285
  value += next_token.value
284
- location = range(token.location.start_offset, lexed[index][0].location.end_offset)
286
+ location = range(token.location.start_offset, next_token.location.end_offset)
285
287
  index += 1
286
288
  else
287
289
  is_at_eol = value.chomp!.nil?
288
290
  location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
289
291
 
290
- prev_token = lexed[index - 2][0] if index - 2 >= 0
291
- next_token = lexed[index][0]
292
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
293
+ next_token, _ = lexed[index]
292
294
 
293
295
  is_inline_comment = prev_token&.location&.start_line == token.location.start_line
294
296
  if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
@@ -307,7 +309,7 @@ module Prism
307
309
  end
308
310
  end
309
311
  when :tNL
310
- next_token = next_token = lexed[index][0]
312
+ next_token, _ = lexed[index]
311
313
  # Newlines after comments are emitted out of order.
312
314
  if next_token&.type == :COMMENT
313
315
  comment_newline_location = location
@@ -344,8 +346,8 @@ module Prism
344
346
  location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
345
347
  value = nil
346
348
  when :tSTRING_BEG
347
- next_token = lexed[index][0]
348
- next_next_token = lexed[index + 1][0]
349
+ next_token, _ = lexed[index]
350
+ next_next_token, _ = lexed[index + 1]
349
351
  basic_quotes = value == '"' || value == "'"
350
352
 
351
353
  if basic_quotes && next_token&.type == :STRING_END
@@ -413,7 +415,8 @@ module Prism
413
415
  while token.type == :STRING_CONTENT
414
416
  current_length += token.value.bytesize
415
417
  # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
416
- is_first_token_on_line = lexed[index - 1] && token.location.start_line != lexed[index - 2][0].location&.start_line
418
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
419
+ is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
417
420
  # The parser gem only removes indentation when the heredoc is not nested
418
421
  not_nested = heredoc_stack.size == 1
419
422
  if is_percent_array
@@ -423,11 +426,16 @@ module Prism
423
426
  end
424
427
 
425
428
  current_string << unescape_string(value, quote_stack.last)
426
- if (backslash_count = token.value[/(\\{1,})\n/, 1]&.length).nil? || backslash_count.even? || !interpolation?(quote_stack.last)
429
+ relevant_backslash_count = if quote_stack.last.start_with?("%W", "%I")
430
+ 0 # the last backslash escapes the newline
431
+ else
432
+ token.value[/(\\{1,})\n/, 1]&.length || 0
433
+ end
434
+ if relevant_backslash_count.even? || !interpolation?(quote_stack.last)
427
435
  tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
428
436
  break
429
437
  end
430
- token = lexed[index][0]
438
+ token, _ = lexed[index]
431
439
  index += 1
432
440
  end
433
441
  else
@@ -482,7 +490,7 @@ module Prism
482
490
  end
483
491
 
484
492
  if percent_array?(quote_stack.pop)
485
- prev_token = lexed[index - 2][0] if index - 2 >= 0
493
+ prev_token, _ = lexed[index - 2] if index - 2 >= 0
486
494
  empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
487
495
  ends_with_whitespace = prev_token&.type == :WORDS_SEP
488
496
  # parser always emits a space token after content in a percent array, even if no actual whitespace is present.
@@ -491,7 +499,7 @@ module Prism
491
499
  end
492
500
  end
493
501
  when :tSYMBEG
494
- if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
502
+ if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
495
503
  next_location = token.location.join(next_token.location)
496
504
  type = :tSYMBOL
497
505
  value = next_token.value
@@ -506,13 +514,13 @@ module Prism
506
514
  type = :tIDENTIFIER
507
515
  end
508
516
  when :tXSTRING_BEG
509
- if (next_token = lexed[index][0]) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
517
+ if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
510
518
  # self.`()
511
519
  type = :tBACK_REF2
512
520
  end
513
521
  quote_stack.push(value)
514
522
  when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
515
- if (next_token = lexed[index][0]) && next_token.type == :WORDS_SEP
523
+ if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
516
524
  index += 1
517
525
  end
518
526
 
@@ -588,9 +596,9 @@ module Prism
588
596
  previous_line = -1
589
597
  result = Float::MAX
590
598
 
591
- while (lexed[next_token_index] && next_token = lexed[next_token_index][0])
599
+ while (next_token = lexed[next_token_index]&.first)
592
600
  next_token_index += 1
593
- next_next_token = lexed[next_token_index] && lexed[next_token_index][0]
601
+ next_next_token, _ = lexed[next_token_index]
594
602
  first_token_on_line = next_token.location.start_column == 0
595
603
 
596
604
  # String content inside nested heredocs and interpolation is ignored
@@ -761,12 +769,12 @@ module Prism
761
769
  elsif (value = scanner.scan(/M-\\?(?=[[:print:]])/))
762
770
  # \M-x where x is an ASCII printable character
763
771
  escape_read(result, scanner, control, true)
764
- elsif (byte = scanner.get_byte)
772
+ elsif (byte = scanner.scan_byte)
765
773
  # Something else after an escape.
766
- if control && byte == "?"
774
+ if control && byte == 0x3f # ASCII '?'
767
775
  result.append_as_bytes(escape_build(0x7f, false, meta))
768
776
  else
769
- result.append_as_bytes(escape_build(byte.ord, control, meta))
777
+ result.append_as_bytes(escape_build(byte, control, meta))
770
778
  end
771
779
  end
772
780
  end
@@ -1,9 +1,15 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  begin
5
+ required_version = ">= 3.3.7.2"
6
+ gem "parser", required_version
4
7
  require "parser"
5
8
  rescue LoadError
6
- warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.})
9
+ warn(<<~MSG)
10
+ Error: Unable to load parser #{required_version}. \
11
+ Add `gem "parser"` to your Gemfile or run `bundle update parser`.
12
+ MSG
7
13
  exit(1)
8
14
  end
9
15
 
@@ -13,6 +19,13 @@ module Prism
13
19
  # whitequark/parser gem's syntax tree. It inherits from the base parser for
14
20
  # the parser gem, and overrides the parse* methods to parse with prism and
15
21
  # then translate.
22
+ #
23
+ # Note that this version of the parser always parses using the latest
24
+ # version of Ruby syntax supported by Prism. If you want specific version
25
+ # support, use one of the version-specific subclasses, such as
26
+ # `Prism::Translation::Parser34`. If you want to parse using the same
27
+ # version of Ruby syntax as the currently running version of Ruby, use
28
+ # `Prism::Translation::ParserCurrent`.
16
29
  class Parser < ::Parser::Base
17
30
  Diagnostic = ::Parser::Diagnostic # :nodoc:
18
31
  private_constant :Diagnostic
@@ -59,13 +72,19 @@ module Prism
59
72
  # should be implemented as needed.
60
73
  #
61
74
  def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
75
+ if !builder.is_a?(Prism::Translation::Parser::Builder)
76
+ warn(<<~MSG, uplevel: 1, category: :deprecated)
77
+ [deprecation]: The builder passed to `Prism::Translation::Parser.new` is not a \
78
+ `Prism::Translation::Parser::Builder` subclass. This will raise in the next major version.
79
+ MSG
80
+ end
62
81
  @parser = parser
63
82
 
64
83
  super(builder)
65
84
  end
66
85
 
67
86
  def version # :nodoc:
68
- 34
87
+ 35
69
88
  end
70
89
 
71
90
  # The default encoding for Ruby files is UTF-8.
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  module Translation
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+ # :markup: markdown
3
+ # typed: ignore
4
+
5
+ #
6
+ module Prism
7
+ module Translation
8
+ case RUBY_VERSION
9
+ when /^3\.3\./
10
+ ParserCurrent = Parser33
11
+ when /^3\.4\./
12
+ ParserCurrent = Parser34
13
+ when /^3\.5\./
14
+ ParserCurrent = Parser35
15
+ else
16
+ # Keep this in sync with released Ruby.
17
+ parser = Parser34
18
+ major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
19
+ warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
20
+ "but you are running #{major}.#{minor}."
21
+ ParserCurrent = parser
22
+ end
23
+ end
24
+ end