prism 0.24.0 → 0.29.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/BSDmakefile +58 -0
  3. data/CHANGELOG.md +132 -1
  4. data/Makefile +25 -18
  5. data/README.md +45 -6
  6. data/config.yml +828 -25
  7. data/docs/build_system.md +31 -0
  8. data/docs/configuration.md +4 -0
  9. data/docs/cruby_compilation.md +1 -1
  10. data/docs/parser_translation.md +14 -9
  11. data/docs/releasing.md +7 -9
  12. data/docs/ripper_translation.md +50 -0
  13. data/docs/ruby_api.md +1 -0
  14. data/docs/serialization.md +26 -5
  15. data/ext/prism/api_node.c +1037 -936
  16. data/ext/prism/api_pack.c +9 -0
  17. data/ext/prism/extconf.rb +62 -18
  18. data/ext/prism/extension.c +351 -71
  19. data/ext/prism/extension.h +5 -4
  20. data/include/prism/ast.h +539 -101
  21. data/include/prism/defines.h +106 -2
  22. data/include/prism/diagnostic.h +168 -74
  23. data/include/prism/encoding.h +22 -4
  24. data/include/prism/node.h +93 -0
  25. data/include/prism/options.h +84 -9
  26. data/include/prism/pack.h +11 -0
  27. data/include/prism/parser.h +213 -54
  28. data/include/prism/prettyprint.h +8 -0
  29. data/include/prism/static_literals.h +120 -0
  30. data/include/prism/util/pm_buffer.h +65 -2
  31. data/include/prism/util/pm_constant_pool.h +18 -1
  32. data/include/prism/util/pm_integer.h +119 -0
  33. data/include/prism/util/pm_list.h +1 -1
  34. data/include/prism/util/pm_newline_list.h +8 -0
  35. data/include/prism/util/pm_string.h +26 -2
  36. data/include/prism/version.h +2 -2
  37. data/include/prism.h +59 -1
  38. data/lib/prism/compiler.rb +8 -1
  39. data/lib/prism/debug.rb +46 -3
  40. data/lib/prism/desugar_compiler.rb +5 -3
  41. data/lib/prism/dispatcher.rb +29 -0
  42. data/lib/prism/dot_visitor.rb +141 -54
  43. data/lib/prism/dsl.rb +48 -36
  44. data/lib/prism/ffi.rb +82 -17
  45. data/lib/prism/inspect_visitor.rb +2156 -0
  46. data/lib/prism/lex_compat.rb +34 -15
  47. data/lib/prism/mutation_compiler.rb +13 -2
  48. data/lib/prism/node.rb +4453 -4459
  49. data/lib/prism/node_ext.rb +249 -30
  50. data/lib/prism/pack.rb +4 -0
  51. data/lib/prism/parse_result/comments.rb +35 -18
  52. data/lib/prism/parse_result/newlines.rb +2 -2
  53. data/lib/prism/parse_result.rb +218 -43
  54. data/lib/prism/pattern.rb +28 -10
  55. data/lib/prism/polyfill/byteindex.rb +13 -0
  56. data/lib/prism/polyfill/unpack1.rb +14 -0
  57. data/lib/prism/reflection.rb +411 -0
  58. data/lib/prism/serialize.rb +480 -112
  59. data/lib/prism/translation/parser/compiler.rb +376 -88
  60. data/lib/prism/translation/parser/lexer.rb +103 -22
  61. data/lib/prism/translation/parser/rubocop.rb +41 -13
  62. data/lib/prism/translation/parser.rb +123 -11
  63. data/lib/prism/translation/parser33.rb +1 -1
  64. data/lib/prism/translation/parser34.rb +1 -1
  65. data/lib/prism/translation/ripper/sexp.rb +125 -0
  66. data/lib/prism/translation/ripper/shim.rb +5 -0
  67. data/lib/prism/translation/ripper.rb +3216 -462
  68. data/lib/prism/translation/ruby_parser.rb +111 -56
  69. data/lib/prism/translation.rb +3 -1
  70. data/lib/prism/visitor.rb +10 -0
  71. data/lib/prism.rb +12 -20
  72. data/prism.gemspec +46 -14
  73. data/rbi/prism/compiler.rbi +12 -0
  74. data/rbi/prism/inspect_visitor.rbi +12 -0
  75. data/rbi/prism/node.rbi +8712 -0
  76. data/rbi/prism/node_ext.rbi +107 -0
  77. data/rbi/prism/parse_result.rbi +358 -0
  78. data/rbi/prism/reflection.rbi +58 -0
  79. data/rbi/prism/translation/parser.rbi +11 -0
  80. data/rbi/prism/translation/parser33.rbi +6 -0
  81. data/rbi/prism/translation/parser34.rbi +6 -0
  82. data/rbi/prism/translation/ripper.rbi +15 -0
  83. data/rbi/prism/visitor.rbi +470 -0
  84. data/rbi/prism.rbi +38 -7748
  85. data/sig/prism/compiler.rbs +9 -0
  86. data/sig/prism/dispatcher.rbs +16 -0
  87. data/sig/prism/dot_visitor.rbs +6 -0
  88. data/sig/prism/dsl.rbs +462 -0
  89. data/sig/prism/inspect_visitor.rbs +22 -0
  90. data/sig/prism/lex_compat.rbs +10 -0
  91. data/sig/prism/mutation_compiler.rbs +158 -0
  92. data/sig/prism/node.rbs +3558 -0
  93. data/sig/prism/node_ext.rbs +82 -0
  94. data/sig/prism/pack.rbs +43 -0
  95. data/sig/prism/parse_result.rbs +160 -0
  96. data/sig/prism/pattern.rbs +13 -0
  97. data/sig/prism/reflection.rbs +50 -0
  98. data/sig/prism/serialize.rbs +6 -0
  99. data/sig/prism/visitor.rbs +168 -0
  100. data/sig/prism.rbs +188 -4767
  101. data/src/diagnostic.c +636 -230
  102. data/src/encoding.c +211 -108
  103. data/src/node.c +7555 -451
  104. data/src/options.c +66 -31
  105. data/src/pack.c +33 -17
  106. data/src/prettyprint.c +1383 -1431
  107. data/src/prism.c +4734 -1310
  108. data/src/regexp.c +17 -2
  109. data/src/serialize.c +68 -46
  110. data/src/static_literals.c +638 -0
  111. data/src/token_type.c +10 -9
  112. data/src/util/pm_buffer.c +147 -20
  113. data/src/util/pm_char.c +4 -4
  114. data/src/util/pm_constant_pool.c +35 -11
  115. data/src/util/pm_integer.c +642 -0
  116. data/src/util/pm_list.c +1 -1
  117. data/src/util/pm_newline_list.c +14 -5
  118. data/src/util/pm_string.c +134 -5
  119. data/src/util/pm_string_list.c +2 -2
  120. metadata +41 -9
  121. data/docs/ripper.md +0 -36
  122. data/include/prism/util/pm_state_stack.h +0 -42
  123. data/lib/prism/node_inspector.rb +0 -68
  124. data/rbi/prism_static.rbi +0 -207
  125. data/sig/prism_static.rbs +0 -201
  126. data/src/util/pm_state_stack.c +0 -25
@@ -167,7 +167,7 @@ module Prism
167
167
  TILDE: :tTILDE,
168
168
  UAMPERSAND: :tAMPER,
169
169
  UCOLON_COLON: :tCOLON3,
170
- UDOT_DOT: :tDOT2,
170
+ UDOT_DOT: :tBDOT2,
171
171
  UDOT_DOT_DOT: :tBDOT3,
172
172
  UMINUS: :tUMINUS,
173
173
  UMINUS_NUM: :tUNARY_NUM,
@@ -177,12 +177,23 @@ module Prism
177
177
  WORDS_SEP: :tSPACE
178
178
  }
179
179
 
180
- private_constant :TYPES
180
+ # These constants represent flags in our lex state. We really, really
181
+ # don't want to be using them and we really, really don't want to be
182
+ # exposing them as part of our public API. Unfortunately, we don't have
183
+ # another way of matching the exact tokens that the parser gem expects
184
+ # without them. We should find another way to do this, but in the
185
+ # meantime we'll hide them from the documentation and mark them as
186
+ # private constants.
187
+ EXPR_BEG = 0x1 # :nodoc:
188
+ EXPR_LABEL = 0x400 # :nodoc:
189
+
190
+ private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL
181
191
 
182
192
  # The Parser::Source::Buffer that the tokens were lexed from.
183
193
  attr_reader :source_buffer
184
194
 
185
- # An array of prism tokens that we lexed.
195
+ # An array of tuples that contain prism tokens and their associated lex
196
+ # state when they were lexed.
186
197
  attr_reader :lexed
187
198
 
188
199
  # A hash that maps offsets in bytes to offsets in characters.
@@ -202,12 +213,16 @@ module Prism
202
213
  # Convert the prism tokens into the expected format for the parser gem.
203
214
  def to_a
204
215
  tokens = []
216
+
205
217
  index = 0
218
+ length = lexed.length
219
+
220
+ heredoc_identifier_stack = []
206
221
 
207
- while index < lexed.length
208
- token, = lexed[index]
222
+ while index < length
223
+ token, state = lexed[index]
209
224
  index += 1
210
- next if token.type == :IGNORED_NEWLINE || token.type == :EOF
225
+ next if %i[IGNORED_NEWLINE __END__ EOF].include?(token.type)
211
226
 
212
227
  type = TYPES.fetch(token.type)
213
228
  value = token.value
@@ -218,14 +233,18 @@ module Prism
218
233
  value.delete_prefix!("?")
219
234
  when :tCOMMENT
220
235
  if token.type == :EMBDOC_BEGIN
221
- until (next_token = lexed[index]) && next_token.type == :EMBDOC_END
236
+ start_index = index
237
+
238
+ while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1)
222
239
  value += next_token.value
223
240
  index += 1
224
241
  end
225
242
 
226
- value += next_token.value
227
- location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index].location.end_offset])
228
- index += 1
243
+ if start_index != index
244
+ value += next_token.value
245
+ location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index][0].location.end_offset])
246
+ index += 1
247
+ end
229
248
  else
230
249
  value.chomp!
231
250
  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - 1])
@@ -233,7 +252,7 @@ module Prism
233
252
  when :tNL
234
253
  value = nil
235
254
  when :tFLOAT
236
- value = Float(value)
255
+ value = parse_float(value)
237
256
  when :tIMAGINARY
238
257
  value = parse_complex(value)
239
258
  when :tINTEGER
@@ -242,13 +261,15 @@ module Prism
242
261
  location = Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset])
243
262
  end
244
263
 
245
- value = Integer(value)
264
+ value = parse_integer(value)
246
265
  when :tLABEL
247
266
  value.chomp!(":")
248
267
  when :tLABEL_END
249
268
  value.chomp!(":")
269
+ when :tLCURLY
270
+ type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
250
271
  when :tNTH_REF
251
- value = Integer(value.delete_prefix("$"))
272
+ value = parse_integer(value.delete_prefix("$"))
252
273
  when :tOP_ASGN
253
274
  value.chomp!("=")
254
275
  when :tRATIONAL
@@ -256,31 +277,69 @@ module Prism
256
277
  when :tSPACE
257
278
  value = nil
258
279
  when :tSTRING_BEG
259
- if ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_END
280
+ if token.type == :HEREDOC_START
281
+ heredoc_identifier_stack.push(value.match(/<<[-~]?["'`]?(?<heredoc_identifier>.*?)["'`]?\z/)[:heredoc_identifier])
282
+ end
283
+ if ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_END
260
284
  next_location = token.location.join(next_token.location)
261
285
  type = :tSTRING
262
286
  value = ""
263
287
  location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
264
288
  index += 1
265
- elsif ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_CONTENT && (next_next_token = lexed[index + 1]) && next_next_token.type == :STRING_END
289
+ elsif ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_CONTENT && next_token.value.lines.count <= 1 && (next_next_token = lexed[index + 1][0]) && next_next_token.type == :STRING_END
266
290
  next_location = token.location.join(next_next_token.location)
267
291
  type = :tSTRING
268
- value = next_token.value
292
+ value = next_token.value.gsub("\\\\", "\\")
269
293
  location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
270
294
  index += 2
271
295
  elsif value.start_with?("<<")
272
296
  quote = value[2] == "-" || value[2] == "~" ? value[3] : value[2]
273
- value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
297
+ if quote == "`"
298
+ type = :tXSTRING_BEG
299
+ value = "<<`"
300
+ else
301
+ value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
302
+ end
303
+ end
304
+ when :tSTRING_CONTENT
305
+ unless (lines = token.value.lines).one?
306
+ start_offset = offset_cache[token.location.start_offset]
307
+ lines.map do |line|
308
+ newline = line.end_with?("\r\n") ? "\r\n" : "\n"
309
+ chomped_line = line.chomp
310
+ if match = chomped_line.match(/(?<backslashes>\\+)\z/)
311
+ adjustment = match[:backslashes].size / 2
312
+ adjusted_line = chomped_line.delete_suffix("\\" * adjustment)
313
+ if match[:backslashes].size.odd?
314
+ adjusted_line.delete_suffix!("\\")
315
+ adjustment += 2
316
+ else
317
+ adjusted_line << newline
318
+ end
319
+ else
320
+ adjusted_line = line
321
+ adjustment = 0
322
+ end
323
+
324
+ end_offset = start_offset + adjusted_line.length + adjustment
325
+ tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
326
+ start_offset = end_offset
327
+ end
328
+ next
274
329
  end
275
330
  when :tSTRING_DVAR
276
331
  value = nil
277
332
  when :tSTRING_END
278
- if token.type == :REGEXP_END
333
+ if token.type == :HEREDOC_END && value.end_with?("\n")
334
+ newline_length = value.end_with?("\r\n") ? 2 : 1
335
+ value = heredoc_identifier_stack.pop
336
+ location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - newline_length])
337
+ elsif token.type == :REGEXP_END
279
338
  value = value[0]
280
339
  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
281
340
  end
282
341
  when :tSYMBEG
283
- if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
342
+ if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
284
343
  next_location = token.location.join(next_token.location)
285
344
  type = :tSYMBOL
286
345
  value = next_token.value
@@ -289,9 +348,13 @@ module Prism
289
348
  index += 1
290
349
  end
291
350
  when :tFID
292
- if tokens[-1][0] == :kDEF
351
+ if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
293
352
  type = :tIDENTIFIER
294
353
  end
354
+ when :tXSTRING_BEG
355
+ if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :STRING_END
356
+ type = :tBACK_REF2
357
+ end
295
358
  end
296
359
 
297
360
  tokens << [type, [value, location]]
@@ -306,6 +369,20 @@ module Prism
306
369
 
307
370
  private
308
371
 
372
+ # Parse an integer from the string representation.
373
+ def parse_integer(value)
374
+ Integer(value)
375
+ rescue ArgumentError
376
+ 0
377
+ end
378
+
379
+ # Parse a float from the string representation.
380
+ def parse_float(value)
381
+ Float(value)
382
+ rescue ArgumentError
383
+ 0.0
384
+ end
385
+
309
386
  # Parse a complex from the string representation.
310
387
  def parse_complex(value)
311
388
  value.chomp!("i")
@@ -313,10 +390,12 @@ module Prism
313
390
  if value.end_with?("r")
314
391
  Complex(0, parse_rational(value))
315
392
  elsif value.start_with?(/0[BbOoDdXx]/)
316
- Complex(0, Integer(value))
393
+ Complex(0, parse_integer(value))
317
394
  else
318
395
  Complex(0, value)
319
396
  end
397
+ rescue ArgumentError
398
+ 0i
320
399
  end
321
400
 
322
401
  # Parse a rational from the string representation.
@@ -324,10 +403,12 @@ module Prism
324
403
  value.chomp!("r")
325
404
 
326
405
  if value.start_with?(/0[BbOoDdXx]/)
327
- Rational(Integer(value))
406
+ Rational(parse_integer(value))
328
407
  else
329
408
  Rational(value)
330
409
  end
410
+ rescue ArgumentError
411
+ 0r
331
412
  end
332
413
  end
333
414
  end
@@ -1,10 +1,13 @@
1
1
  # frozen_string_literal: true
2
+ # typed: ignore
3
+
4
+ warn "WARN: Prism is directly supported since RuboCop 1.62. The `prism/translation/parser/rubocop` file is deprecated."
2
5
 
3
6
  require "parser"
4
7
  require "rubocop"
5
8
 
6
- require "prism"
7
- require "prism/translation/parser"
9
+ require_relative "../../prism"
10
+ require_relative "../parser"
8
11
 
9
12
  module Prism
10
13
  module Translation
@@ -20,17 +23,42 @@ module Prism
20
23
 
21
24
  # This module gets prepended into RuboCop::AST::ProcessedSource.
22
25
  module ProcessedSource
23
- # Redefine parser_class so that we can inject the prism parser into the
24
- # list of known parsers.
25
- def parser_class(ruby_version)
26
- if ruby_version == Prism::Translation::Parser::VERSION_3_3
27
- require "prism/translation/parser33"
28
- Prism::Translation::Parser33
29
- elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
30
- require "prism/translation/parser34"
31
- Prism::Translation::Parser34
32
- else
33
- super
26
+ # This condition is compatible with rubocop-ast versions up to 1.30.0.
27
+ if RuboCop::AST::ProcessedSource.instance_method(:parser_class).arity == 1
28
+ # Redefine parser_class so that we can inject the prism parser into the
29
+ # list of known parsers.
30
+ def parser_class(ruby_version)
31
+ if ruby_version == Prism::Translation::Parser::VERSION_3_3
32
+ warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
33
+ "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
34
+ require_relative "../parser33"
35
+ Prism::Translation::Parser33
36
+ elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
37
+ warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
38
+ "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
39
+ require_relative "../parser34"
40
+ Prism::Translation::Parser34
41
+ else
42
+ super
43
+ end
44
+ end
45
+ else
46
+ # Redefine parser_class so that we can inject the prism parser into the
47
+ # list of known parsers.
48
+ def parser_class(ruby_version, _parser_engine)
49
+ if ruby_version == Prism::Translation::Parser::VERSION_3_3
50
+ warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
51
+ "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
52
+ require_relative "../parser33"
53
+ Prism::Translation::Parser33
54
+ elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
55
+ warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
56
+ "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
57
+ require_relative "../parser34"
58
+ Prism::Translation::Parser34
59
+ else
60
+ super
61
+ end
34
62
  end
35
63
  end
36
64
  end
@@ -9,17 +9,20 @@ module Prism
9
9
  # the parser gem, and overrides the parse* methods to parse with prism and
10
10
  # then translate.
11
11
  class Parser < ::Parser::Base
12
+ Diagnostic = ::Parser::Diagnostic # :nodoc:
13
+ private_constant :Diagnostic
14
+
12
15
  # The parser gem has a list of diagnostics with a hard-coded set of error
13
16
  # messages. We create our own diagnostic class in order to set our own
14
17
  # error messages.
15
- class Diagnostic < ::Parser::Diagnostic
16
- # The message generated by prism.
18
+ class PrismDiagnostic < Diagnostic
19
+ # This is the cached message coming from prism.
17
20
  attr_reader :message
18
21
 
19
22
  # Initialize a new diagnostic with the given message and location.
20
- def initialize(message, location)
23
+ def initialize(message, level, reason, location)
21
24
  @message = message
22
- super(:error, :prism_error, {}, location, [])
25
+ super(level, reason, {}, location, [])
23
26
  end
24
27
  end
25
28
 
@@ -43,7 +46,7 @@ module Prism
43
46
  source = source_buffer.source
44
47
 
45
48
  offset_cache = build_offset_cache(source)
46
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
49
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
47
50
 
48
51
  build_ast(result.value, offset_cache)
49
52
  ensure
@@ -56,7 +59,7 @@ module Prism
56
59
  source = source_buffer.source
57
60
 
58
61
  offset_cache = build_offset_cache(source)
59
- result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
62
+ result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
60
63
 
61
64
  [
62
65
  build_ast(result.value, offset_cache),
@@ -75,7 +78,7 @@ module Prism
75
78
  offset_cache = build_offset_cache(source)
76
79
  result =
77
80
  begin
78
- unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
81
+ unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
79
82
  rescue ::Parser::SyntaxError
80
83
  raise if !recover
81
84
  end
@@ -106,14 +109,123 @@ module Prism
106
109
  true
107
110
  end
108
111
 
112
+ # This is a hook to allow consumers to disable some warnings if they don't
113
+ # want them to block creating the syntax tree.
114
+ def valid_warning?(warning)
115
+ true
116
+ end
117
+
118
+ # Build a diagnostic from the given prism parse error.
119
+ def error_diagnostic(error, offset_cache)
120
+ location = error.location
121
+ diagnostic_location = build_range(location, offset_cache)
122
+
123
+ case error.type
124
+ when :argument_block_multi
125
+ Diagnostic.new(:error, :block_and_blockarg, {}, diagnostic_location, [])
126
+ when :argument_formal_constant
127
+ Diagnostic.new(:error, :argument_const, {}, diagnostic_location, [])
128
+ when :argument_formal_class
129
+ Diagnostic.new(:error, :argument_cvar, {}, diagnostic_location, [])
130
+ when :argument_formal_global
131
+ Diagnostic.new(:error, :argument_gvar, {}, diagnostic_location, [])
132
+ when :argument_formal_ivar
133
+ Diagnostic.new(:error, :argument_ivar, {}, diagnostic_location, [])
134
+ when :argument_no_forwarding_amp
135
+ Diagnostic.new(:error, :no_anonymous_blockarg, {}, diagnostic_location, [])
136
+ when :argument_no_forwarding_star
137
+ Diagnostic.new(:error, :no_anonymous_restarg, {}, diagnostic_location, [])
138
+ when :argument_no_forwarding_star_star
139
+ Diagnostic.new(:error, :no_anonymous_kwrestarg, {}, diagnostic_location, [])
140
+ when :begin_lonely_else
141
+ location = location.copy(length: 4)
142
+ diagnostic_location = build_range(location, offset_cache)
143
+ Diagnostic.new(:error, :useless_else, {}, diagnostic_location, [])
144
+ when :class_name, :module_name
145
+ Diagnostic.new(:error, :module_name_const, {}, diagnostic_location, [])
146
+ when :class_in_method
147
+ Diagnostic.new(:error, :class_in_def, {}, diagnostic_location, [])
148
+ when :def_endless_setter
149
+ Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, [])
150
+ when :embdoc_term
151
+ Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, [])
152
+ when :incomplete_variable_class, :incomplete_variable_class_3_3
153
+ location = location.copy(length: location.length + 1)
154
+ diagnostic_location = build_range(location, offset_cache)
155
+
156
+ Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, [])
157
+ when :incomplete_variable_instance, :incomplete_variable_instance_3_3
158
+ location = location.copy(length: location.length + 1)
159
+ diagnostic_location = build_range(location, offset_cache)
160
+
161
+ Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, [])
162
+ when :invalid_variable_global, :invalid_variable_global_3_3
163
+ Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, [])
164
+ when :module_in_method
165
+ Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, [])
166
+ when :numbered_parameter_ordinary
167
+ Diagnostic.new(:error, :ordinary_param_defined, {}, diagnostic_location, [])
168
+ when :numbered_parameter_outer_scope
169
+ Diagnostic.new(:error, :numparam_used_in_outer_scope, {}, diagnostic_location, [])
170
+ when :parameter_circular
171
+ Diagnostic.new(:error, :circular_argument_reference, { var_name: location.slice }, diagnostic_location, [])
172
+ when :parameter_name_repeat
173
+ Diagnostic.new(:error, :duplicate_argument, {}, diagnostic_location, [])
174
+ when :parameter_numbered_reserved
175
+ Diagnostic.new(:error, :reserved_for_numparam, { name: location.slice }, diagnostic_location, [])
176
+ when :regexp_unknown_options
177
+ Diagnostic.new(:error, :regexp_options, { options: location.slice[1..] }, diagnostic_location, [])
178
+ when :singleton_for_literals
179
+ Diagnostic.new(:error, :singleton_literal, {}, diagnostic_location, [])
180
+ when :string_literal_eof
181
+ Diagnostic.new(:error, :string_eof, {}, diagnostic_location, [])
182
+ when :unexpected_token_ignore
183
+ Diagnostic.new(:error, :unexpected_token, { token: location.slice }, diagnostic_location, [])
184
+ when :write_target_in_method
185
+ Diagnostic.new(:error, :dynamic_const, {}, diagnostic_location, [])
186
+ else
187
+ PrismDiagnostic.new(error.message, :error, error.type, diagnostic_location)
188
+ end
189
+ end
190
+
191
+ # Build a diagnostic from the given prism parse warning.
192
+ def warning_diagnostic(warning, offset_cache)
193
+ diagnostic_location = build_range(warning.location, offset_cache)
194
+
195
+ case warning.type
196
+ when :ambiguous_first_argument_plus
197
+ Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "+" }, diagnostic_location, [])
198
+ when :ambiguous_first_argument_minus
199
+ Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "-" }, diagnostic_location, [])
200
+ when :ambiguous_prefix_ampersand
201
+ Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "&" }, diagnostic_location, [])
202
+ when :ambiguous_prefix_star
203
+ Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "*" }, diagnostic_location, [])
204
+ when :ambiguous_prefix_star_star
205
+ Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "**" }, diagnostic_location, [])
206
+ when :ambiguous_slash
207
+ Diagnostic.new(:warning, :ambiguous_regexp, {}, diagnostic_location, [])
208
+ when :dot_dot_dot_eol
209
+ Diagnostic.new(:warning, :triple_dot_at_eol, {}, diagnostic_location, [])
210
+ when :duplicated_hash_key
211
+ # skip, parser does this on its own
212
+ else
213
+ PrismDiagnostic.new(warning.message, :warning, warning.type, diagnostic_location)
214
+ end
215
+ end
216
+
109
217
  # If there was a error generated during the parse, then raise an
110
218
  # appropriate syntax error. Otherwise return the result.
111
219
  def unwrap(result, offset_cache)
112
220
  result.errors.each do |error|
113
221
  next unless valid_error?(error)
222
+ diagnostics.process(error_diagnostic(error, offset_cache))
223
+ end
114
224
 
115
- location = build_range(error.location, offset_cache)
116
- diagnostics.process(Diagnostic.new(error.message, location))
225
+ result.warnings.each do |warning|
226
+ next unless valid_warning?(warning)
227
+ diagnostic = warning_diagnostic(warning, offset_cache)
228
+ diagnostics.process(diagnostic) if diagnostic
117
229
  end
118
230
 
119
231
  result
@@ -156,7 +268,7 @@ module Prism
156
268
 
157
269
  # Build the parser gem tokens from the prism tokens.
158
270
  def build_tokens(tokens, offset_cache)
159
- Lexer.new(source_buffer, tokens.map(&:first), offset_cache).to_a
271
+ Lexer.new(source_buffer, tokens, offset_cache).to_a
160
272
  end
161
273
 
162
274
  # Build a range from a prism location.
@@ -172,7 +284,7 @@ module Prism
172
284
  def convert_for_prism(version)
173
285
  case version
174
286
  when 33
175
- "3.3.0"
287
+ "3.3.1"
176
288
  when 34
177
289
  "3.4.0"
178
290
  else
@@ -1,4 +1,4 @@
1
- require_relative "parser"
1
+ # frozen_string_literal: true
2
2
 
3
3
  module Prism
4
4
  module Translation
@@ -1,4 +1,4 @@
1
- require_relative "parser"
1
+ # frozen_string_literal: true
2
2
 
3
3
  module Prism
4
4
  module Translation
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ripper"
4
+
5
+ module Prism
6
+ module Translation
7
+ class Ripper
8
+ # This class mirrors the ::Ripper::SexpBuilder subclass of ::Ripper that
9
+ # returns the arrays of [type, *children].
10
+ class SexpBuilder < Ripper
11
+ # :stopdoc:
12
+
13
+ attr_reader :error
14
+
15
+ private
16
+
17
+ def dedent_element(e, width)
18
+ if (n = dedent_string(e[1], width)) > 0
19
+ e[2][1] += n
20
+ end
21
+ e
22
+ end
23
+
24
+ def on_heredoc_dedent(val, width)
25
+ sub = proc do |cont|
26
+ cont.map! do |e|
27
+ if Array === e
28
+ case e[0]
29
+ when :@tstring_content
30
+ e = dedent_element(e, width)
31
+ when /_add\z/
32
+ e[1] = sub[e[1]]
33
+ end
34
+ elsif String === e
35
+ dedent_string(e, width)
36
+ end
37
+ e
38
+ end
39
+ end
40
+ sub[val]
41
+ val
42
+ end
43
+
44
+ events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym}
45
+ (PARSER_EVENTS - events).each do |event|
46
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
47
+ def on_#{event}(*args)
48
+ args.unshift :#{event}
49
+ end
50
+ End
51
+ end
52
+
53
+ SCANNER_EVENTS.each do |event|
54
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
55
+ def on_#{event}(tok)
56
+ [:@#{event}, tok, [lineno(), column()]]
57
+ end
58
+ End
59
+ end
60
+
61
+ def on_error(mesg)
62
+ @error = mesg
63
+ end
64
+ remove_method :on_parse_error
65
+ alias on_parse_error on_error
66
+ alias compile_error on_error
67
+
68
+ # :startdoc:
69
+ end
70
+
71
+ # This class mirrors the ::Ripper::SexpBuilderPP subclass of ::Ripper that
72
+ # returns the same values as ::Ripper::SexpBuilder except with a couple of
73
+ # niceties that flatten linked lists into arrays.
74
+ class SexpBuilderPP < SexpBuilder
75
+ # :stopdoc:
76
+
77
+ private
78
+
79
+ def on_heredoc_dedent(val, width)
80
+ val.map! do |e|
81
+ next e if Symbol === e and /_content\z/ =~ e
82
+ if Array === e and e[0] == :@tstring_content
83
+ e = dedent_element(e, width)
84
+ elsif String === e
85
+ dedent_string(e, width)
86
+ end
87
+ e
88
+ end
89
+ val
90
+ end
91
+
92
+ def _dispatch_event_new
93
+ []
94
+ end
95
+
96
+ def _dispatch_event_push(list, item)
97
+ list.push item
98
+ list
99
+ end
100
+
101
+ def on_mlhs_paren(list)
102
+ [:mlhs, *list]
103
+ end
104
+
105
+ def on_mlhs_add_star(list, star)
106
+ list.push([:rest_param, star])
107
+ end
108
+
109
+ def on_mlhs_add_post(list, post)
110
+ list.concat(post)
111
+ end
112
+
113
+ PARSER_EVENT_TABLE.each do |event, arity|
114
+ if /_new\z/ =~ event and arity == 0
115
+ alias_method "on_#{event}", :_dispatch_event_new
116
+ elsif /_add\z/ =~ event
117
+ alias_method "on_#{event}", :_dispatch_event_push
118
+ end
119
+ end
120
+
121
+ # :startdoc:
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This writes the prism ripper translation into the Ripper constant so that
4
+ # users can transparently use Ripper without any changes.
5
+ Ripper = Prism::Translation::Ripper