prism 0.24.0 → 0.29.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/BSDmakefile +58 -0
- data/CHANGELOG.md +132 -1
- data/Makefile +25 -18
- data/README.md +45 -6
- data/config.yml +828 -25
- data/docs/build_system.md +31 -0
- data/docs/configuration.md +4 -0
- data/docs/cruby_compilation.md +1 -1
- data/docs/parser_translation.md +14 -9
- data/docs/releasing.md +7 -9
- data/docs/ripper_translation.md +50 -0
- data/docs/ruby_api.md +1 -0
- data/docs/serialization.md +26 -5
- data/ext/prism/api_node.c +1037 -936
- data/ext/prism/api_pack.c +9 -0
- data/ext/prism/extconf.rb +62 -18
- data/ext/prism/extension.c +351 -71
- data/ext/prism/extension.h +5 -4
- data/include/prism/ast.h +539 -101
- data/include/prism/defines.h +106 -2
- data/include/prism/diagnostic.h +168 -74
- data/include/prism/encoding.h +22 -4
- data/include/prism/node.h +93 -0
- data/include/prism/options.h +84 -9
- data/include/prism/pack.h +11 -0
- data/include/prism/parser.h +213 -54
- data/include/prism/prettyprint.h +8 -0
- data/include/prism/static_literals.h +120 -0
- data/include/prism/util/pm_buffer.h +65 -2
- data/include/prism/util/pm_constant_pool.h +18 -1
- data/include/prism/util/pm_integer.h +119 -0
- data/include/prism/util/pm_list.h +1 -1
- data/include/prism/util/pm_newline_list.h +8 -0
- data/include/prism/util/pm_string.h +26 -2
- data/include/prism/version.h +2 -2
- data/include/prism.h +59 -1
- data/lib/prism/compiler.rb +8 -1
- data/lib/prism/debug.rb +46 -3
- data/lib/prism/desugar_compiler.rb +5 -3
- data/lib/prism/dispatcher.rb +29 -0
- data/lib/prism/dot_visitor.rb +141 -54
- data/lib/prism/dsl.rb +48 -36
- data/lib/prism/ffi.rb +82 -17
- data/lib/prism/inspect_visitor.rb +2156 -0
- data/lib/prism/lex_compat.rb +34 -15
- data/lib/prism/mutation_compiler.rb +13 -2
- data/lib/prism/node.rb +4453 -4459
- data/lib/prism/node_ext.rb +249 -30
- data/lib/prism/pack.rb +4 -0
- data/lib/prism/parse_result/comments.rb +35 -18
- data/lib/prism/parse_result/newlines.rb +2 -2
- data/lib/prism/parse_result.rb +218 -43
- data/lib/prism/pattern.rb +28 -10
- data/lib/prism/polyfill/byteindex.rb +13 -0
- data/lib/prism/polyfill/unpack1.rb +14 -0
- data/lib/prism/reflection.rb +411 -0
- data/lib/prism/serialize.rb +480 -112
- data/lib/prism/translation/parser/compiler.rb +376 -88
- data/lib/prism/translation/parser/lexer.rb +103 -22
- data/lib/prism/translation/parser/rubocop.rb +41 -13
- data/lib/prism/translation/parser.rb +123 -11
- data/lib/prism/translation/parser33.rb +1 -1
- data/lib/prism/translation/parser34.rb +1 -1
- data/lib/prism/translation/ripper/sexp.rb +125 -0
- data/lib/prism/translation/ripper/shim.rb +5 -0
- data/lib/prism/translation/ripper.rb +3216 -462
- data/lib/prism/translation/ruby_parser.rb +111 -56
- data/lib/prism/translation.rb +3 -1
- data/lib/prism/visitor.rb +10 -0
- data/lib/prism.rb +12 -20
- data/prism.gemspec +46 -14
- data/rbi/prism/compiler.rbi +12 -0
- data/rbi/prism/inspect_visitor.rbi +12 -0
- data/rbi/prism/node.rbi +8712 -0
- data/rbi/prism/node_ext.rbi +107 -0
- data/rbi/prism/parse_result.rbi +358 -0
- data/rbi/prism/reflection.rbi +58 -0
- data/rbi/prism/translation/parser.rbi +11 -0
- data/rbi/prism/translation/parser33.rbi +6 -0
- data/rbi/prism/translation/parser34.rbi +6 -0
- data/rbi/prism/translation/ripper.rbi +15 -0
- data/rbi/prism/visitor.rbi +470 -0
- data/rbi/prism.rbi +38 -7748
- data/sig/prism/compiler.rbs +9 -0
- data/sig/prism/dispatcher.rbs +16 -0
- data/sig/prism/dot_visitor.rbs +6 -0
- data/sig/prism/dsl.rbs +462 -0
- data/sig/prism/inspect_visitor.rbs +22 -0
- data/sig/prism/lex_compat.rbs +10 -0
- data/sig/prism/mutation_compiler.rbs +158 -0
- data/sig/prism/node.rbs +3558 -0
- data/sig/prism/node_ext.rbs +82 -0
- data/sig/prism/pack.rbs +43 -0
- data/sig/prism/parse_result.rbs +160 -0
- data/sig/prism/pattern.rbs +13 -0
- data/sig/prism/reflection.rbs +50 -0
- data/sig/prism/serialize.rbs +6 -0
- data/sig/prism/visitor.rbs +168 -0
- data/sig/prism.rbs +188 -4767
- data/src/diagnostic.c +636 -230
- data/src/encoding.c +211 -108
- data/src/node.c +7555 -451
- data/src/options.c +66 -31
- data/src/pack.c +33 -17
- data/src/prettyprint.c +1383 -1431
- data/src/prism.c +4734 -1310
- data/src/regexp.c +17 -2
- data/src/serialize.c +68 -46
- data/src/static_literals.c +638 -0
- data/src/token_type.c +10 -9
- data/src/util/pm_buffer.c +147 -20
- data/src/util/pm_char.c +4 -4
- data/src/util/pm_constant_pool.c +35 -11
- data/src/util/pm_integer.c +642 -0
- data/src/util/pm_list.c +1 -1
- data/src/util/pm_newline_list.c +14 -5
- data/src/util/pm_string.c +134 -5
- data/src/util/pm_string_list.c +2 -2
- metadata +41 -9
- data/docs/ripper.md +0 -36
- data/include/prism/util/pm_state_stack.h +0 -42
- data/lib/prism/node_inspector.rb +0 -68
- data/rbi/prism_static.rbi +0 -207
- data/sig/prism_static.rbs +0 -201
- data/src/util/pm_state_stack.c +0 -25
@@ -167,7 +167,7 @@ module Prism
|
|
167
167
|
TILDE: :tTILDE,
|
168
168
|
UAMPERSAND: :tAMPER,
|
169
169
|
UCOLON_COLON: :tCOLON3,
|
170
|
-
UDOT_DOT: :
|
170
|
+
UDOT_DOT: :tBDOT2,
|
171
171
|
UDOT_DOT_DOT: :tBDOT3,
|
172
172
|
UMINUS: :tUMINUS,
|
173
173
|
UMINUS_NUM: :tUNARY_NUM,
|
@@ -177,12 +177,23 @@ module Prism
|
|
177
177
|
WORDS_SEP: :tSPACE
|
178
178
|
}
|
179
179
|
|
180
|
-
|
180
|
+
# These constants represent flags in our lex state. We really, really
|
181
|
+
# don't want to be using them and we really, really don't want to be
|
182
|
+
# exposing them as part of our public API. Unfortunately, we don't have
|
183
|
+
# another way of matching the exact tokens that the parser gem expects
|
184
|
+
# without them. We should find another way to do this, but in the
|
185
|
+
# meantime we'll hide them from the documentation and mark them as
|
186
|
+
# private constants.
|
187
|
+
EXPR_BEG = 0x1 # :nodoc:
|
188
|
+
EXPR_LABEL = 0x400 # :nodoc:
|
189
|
+
|
190
|
+
private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL
|
181
191
|
|
182
192
|
# The Parser::Source::Buffer that the tokens were lexed from.
|
183
193
|
attr_reader :source_buffer
|
184
194
|
|
185
|
-
# An array of prism tokens
|
195
|
+
# An array of tuples that contain prism tokens and their associated lex
|
196
|
+
# state when they were lexed.
|
186
197
|
attr_reader :lexed
|
187
198
|
|
188
199
|
# A hash that maps offsets in bytes to offsets in characters.
|
@@ -202,12 +213,16 @@ module Prism
|
|
202
213
|
# Convert the prism tokens into the expected format for the parser gem.
|
203
214
|
def to_a
|
204
215
|
tokens = []
|
216
|
+
|
205
217
|
index = 0
|
218
|
+
length = lexed.length
|
219
|
+
|
220
|
+
heredoc_identifier_stack = []
|
206
221
|
|
207
|
-
while index <
|
208
|
-
token, = lexed[index]
|
222
|
+
while index < length
|
223
|
+
token, state = lexed[index]
|
209
224
|
index += 1
|
210
|
-
next if
|
225
|
+
next if %i[IGNORED_NEWLINE __END__ EOF].include?(token.type)
|
211
226
|
|
212
227
|
type = TYPES.fetch(token.type)
|
213
228
|
value = token.value
|
@@ -218,14 +233,18 @@ module Prism
|
|
218
233
|
value.delete_prefix!("?")
|
219
234
|
when :tCOMMENT
|
220
235
|
if token.type == :EMBDOC_BEGIN
|
221
|
-
|
236
|
+
start_index = index
|
237
|
+
|
238
|
+
while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1)
|
222
239
|
value += next_token.value
|
223
240
|
index += 1
|
224
241
|
end
|
225
242
|
|
226
|
-
|
227
|
-
|
228
|
-
|
243
|
+
if start_index != index
|
244
|
+
value += next_token.value
|
245
|
+
location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index][0].location.end_offset])
|
246
|
+
index += 1
|
247
|
+
end
|
229
248
|
else
|
230
249
|
value.chomp!
|
231
250
|
location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - 1])
|
@@ -233,7 +252,7 @@ module Prism
|
|
233
252
|
when :tNL
|
234
253
|
value = nil
|
235
254
|
when :tFLOAT
|
236
|
-
value =
|
255
|
+
value = parse_float(value)
|
237
256
|
when :tIMAGINARY
|
238
257
|
value = parse_complex(value)
|
239
258
|
when :tINTEGER
|
@@ -242,13 +261,15 @@ module Prism
|
|
242
261
|
location = Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset])
|
243
262
|
end
|
244
263
|
|
245
|
-
value =
|
264
|
+
value = parse_integer(value)
|
246
265
|
when :tLABEL
|
247
266
|
value.chomp!(":")
|
248
267
|
when :tLABEL_END
|
249
268
|
value.chomp!(":")
|
269
|
+
when :tLCURLY
|
270
|
+
type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
|
250
271
|
when :tNTH_REF
|
251
|
-
value =
|
272
|
+
value = parse_integer(value.delete_prefix("$"))
|
252
273
|
when :tOP_ASGN
|
253
274
|
value.chomp!("=")
|
254
275
|
when :tRATIONAL
|
@@ -256,31 +277,69 @@ module Prism
|
|
256
277
|
when :tSPACE
|
257
278
|
value = nil
|
258
279
|
when :tSTRING_BEG
|
259
|
-
if
|
280
|
+
if token.type == :HEREDOC_START
|
281
|
+
heredoc_identifier_stack.push(value.match(/<<[-~]?["'`]?(?<heredoc_identifier>.*?)["'`]?\z/)[:heredoc_identifier])
|
282
|
+
end
|
283
|
+
if ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_END
|
260
284
|
next_location = token.location.join(next_token.location)
|
261
285
|
type = :tSTRING
|
262
286
|
value = ""
|
263
287
|
location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
|
264
288
|
index += 1
|
265
|
-
elsif ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_CONTENT && (next_next_token = lexed[index + 1]) && next_next_token.type == :STRING_END
|
289
|
+
elsif ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_CONTENT && next_token.value.lines.count <= 1 && (next_next_token = lexed[index + 1][0]) && next_next_token.type == :STRING_END
|
266
290
|
next_location = token.location.join(next_next_token.location)
|
267
291
|
type = :tSTRING
|
268
|
-
value = next_token.value
|
292
|
+
value = next_token.value.gsub("\\\\", "\\")
|
269
293
|
location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
|
270
294
|
index += 2
|
271
295
|
elsif value.start_with?("<<")
|
272
296
|
quote = value[2] == "-" || value[2] == "~" ? value[3] : value[2]
|
273
|
-
|
297
|
+
if quote == "`"
|
298
|
+
type = :tXSTRING_BEG
|
299
|
+
value = "<<`"
|
300
|
+
else
|
301
|
+
value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
|
302
|
+
end
|
303
|
+
end
|
304
|
+
when :tSTRING_CONTENT
|
305
|
+
unless (lines = token.value.lines).one?
|
306
|
+
start_offset = offset_cache[token.location.start_offset]
|
307
|
+
lines.map do |line|
|
308
|
+
newline = line.end_with?("\r\n") ? "\r\n" : "\n"
|
309
|
+
chomped_line = line.chomp
|
310
|
+
if match = chomped_line.match(/(?<backslashes>\\+)\z/)
|
311
|
+
adjustment = match[:backslashes].size / 2
|
312
|
+
adjusted_line = chomped_line.delete_suffix("\\" * adjustment)
|
313
|
+
if match[:backslashes].size.odd?
|
314
|
+
adjusted_line.delete_suffix!("\\")
|
315
|
+
adjustment += 2
|
316
|
+
else
|
317
|
+
adjusted_line << newline
|
318
|
+
end
|
319
|
+
else
|
320
|
+
adjusted_line = line
|
321
|
+
adjustment = 0
|
322
|
+
end
|
323
|
+
|
324
|
+
end_offset = start_offset + adjusted_line.length + adjustment
|
325
|
+
tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
|
326
|
+
start_offset = end_offset
|
327
|
+
end
|
328
|
+
next
|
274
329
|
end
|
275
330
|
when :tSTRING_DVAR
|
276
331
|
value = nil
|
277
332
|
when :tSTRING_END
|
278
|
-
if token.type == :
|
333
|
+
if token.type == :HEREDOC_END && value.end_with?("\n")
|
334
|
+
newline_length = value.end_with?("\r\n") ? 2 : 1
|
335
|
+
value = heredoc_identifier_stack.pop
|
336
|
+
location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - newline_length])
|
337
|
+
elsif token.type == :REGEXP_END
|
279
338
|
value = value[0]
|
280
339
|
location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
|
281
340
|
end
|
282
341
|
when :tSYMBEG
|
283
|
-
if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
|
342
|
+
if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
|
284
343
|
next_location = token.location.join(next_token.location)
|
285
344
|
type = :tSYMBOL
|
286
345
|
value = next_token.value
|
@@ -289,9 +348,13 @@ module Prism
|
|
289
348
|
index += 1
|
290
349
|
end
|
291
350
|
when :tFID
|
292
|
-
if tokens
|
351
|
+
if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
|
293
352
|
type = :tIDENTIFIER
|
294
353
|
end
|
354
|
+
when :tXSTRING_BEG
|
355
|
+
if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :STRING_END
|
356
|
+
type = :tBACK_REF2
|
357
|
+
end
|
295
358
|
end
|
296
359
|
|
297
360
|
tokens << [type, [value, location]]
|
@@ -306,6 +369,20 @@ module Prism
|
|
306
369
|
|
307
370
|
private
|
308
371
|
|
372
|
+
# Parse an integer from the string representation.
|
373
|
+
def parse_integer(value)
|
374
|
+
Integer(value)
|
375
|
+
rescue ArgumentError
|
376
|
+
0
|
377
|
+
end
|
378
|
+
|
379
|
+
# Parse a float from the string representation.
|
380
|
+
def parse_float(value)
|
381
|
+
Float(value)
|
382
|
+
rescue ArgumentError
|
383
|
+
0.0
|
384
|
+
end
|
385
|
+
|
309
386
|
# Parse a complex from the string representation.
|
310
387
|
def parse_complex(value)
|
311
388
|
value.chomp!("i")
|
@@ -313,10 +390,12 @@ module Prism
|
|
313
390
|
if value.end_with?("r")
|
314
391
|
Complex(0, parse_rational(value))
|
315
392
|
elsif value.start_with?(/0[BbOoDdXx]/)
|
316
|
-
Complex(0,
|
393
|
+
Complex(0, parse_integer(value))
|
317
394
|
else
|
318
395
|
Complex(0, value)
|
319
396
|
end
|
397
|
+
rescue ArgumentError
|
398
|
+
0i
|
320
399
|
end
|
321
400
|
|
322
401
|
# Parse a rational from the string representation.
|
@@ -324,10 +403,12 @@ module Prism
|
|
324
403
|
value.chomp!("r")
|
325
404
|
|
326
405
|
if value.start_with?(/0[BbOoDdXx]/)
|
327
|
-
Rational(
|
406
|
+
Rational(parse_integer(value))
|
328
407
|
else
|
329
408
|
Rational(value)
|
330
409
|
end
|
410
|
+
rescue ArgumentError
|
411
|
+
0r
|
331
412
|
end
|
332
413
|
end
|
333
414
|
end
|
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
# typed: ignore
|
3
|
+
|
4
|
+
warn "WARN: Prism is directly supported since RuboCop 1.62. The `prism/translation/parser/rubocop` file is deprecated."
|
2
5
|
|
3
6
|
require "parser"
|
4
7
|
require "rubocop"
|
5
8
|
|
6
|
-
|
7
|
-
|
9
|
+
require_relative "../../prism"
|
10
|
+
require_relative "../parser"
|
8
11
|
|
9
12
|
module Prism
|
10
13
|
module Translation
|
@@ -20,17 +23,42 @@ module Prism
|
|
20
23
|
|
21
24
|
# This module gets prepended into RuboCop::AST::ProcessedSource.
|
22
25
|
module ProcessedSource
|
23
|
-
#
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
Prism::Translation::
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
26
|
+
# This condition is compatible with rubocop-ast versions up to 1.30.0.
|
27
|
+
if RuboCop::AST::ProcessedSource.instance_method(:parser_class).arity == 1
|
28
|
+
# Redefine parser_class so that we can inject the prism parser into the
|
29
|
+
# list of known parsers.
|
30
|
+
def parser_class(ruby_version)
|
31
|
+
if ruby_version == Prism::Translation::Parser::VERSION_3_3
|
32
|
+
warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
|
33
|
+
"Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
|
34
|
+
require_relative "../parser33"
|
35
|
+
Prism::Translation::Parser33
|
36
|
+
elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
|
37
|
+
warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
|
38
|
+
"Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
|
39
|
+
require_relative "../parser34"
|
40
|
+
Prism::Translation::Parser34
|
41
|
+
else
|
42
|
+
super
|
43
|
+
end
|
44
|
+
end
|
45
|
+
else
|
46
|
+
# Redefine parser_class so that we can inject the prism parser into the
|
47
|
+
# list of known parsers.
|
48
|
+
def parser_class(ruby_version, _parser_engine)
|
49
|
+
if ruby_version == Prism::Translation::Parser::VERSION_3_3
|
50
|
+
warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
|
51
|
+
"Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
|
52
|
+
require_relative "../parser33"
|
53
|
+
Prism::Translation::Parser33
|
54
|
+
elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
|
55
|
+
warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
|
56
|
+
"Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
|
57
|
+
require_relative "../parser34"
|
58
|
+
Prism::Translation::Parser34
|
59
|
+
else
|
60
|
+
super
|
61
|
+
end
|
34
62
|
end
|
35
63
|
end
|
36
64
|
end
|
@@ -9,17 +9,20 @@ module Prism
|
|
9
9
|
# the parser gem, and overrides the parse* methods to parse with prism and
|
10
10
|
# then translate.
|
11
11
|
class Parser < ::Parser::Base
|
12
|
+
Diagnostic = ::Parser::Diagnostic # :nodoc:
|
13
|
+
private_constant :Diagnostic
|
14
|
+
|
12
15
|
# The parser gem has a list of diagnostics with a hard-coded set of error
|
13
16
|
# messages. We create our own diagnostic class in order to set our own
|
14
17
|
# error messages.
|
15
|
-
class
|
16
|
-
#
|
18
|
+
class PrismDiagnostic < Diagnostic
|
19
|
+
# This is the cached message coming from prism.
|
17
20
|
attr_reader :message
|
18
21
|
|
19
22
|
# Initialize a new diagnostic with the given message and location.
|
20
|
-
def initialize(message, location)
|
23
|
+
def initialize(message, level, reason, location)
|
21
24
|
@message = message
|
22
|
-
super(
|
25
|
+
super(level, reason, {}, location, [])
|
23
26
|
end
|
24
27
|
end
|
25
28
|
|
@@ -43,7 +46,7 @@ module Prism
|
|
43
46
|
source = source_buffer.source
|
44
47
|
|
45
48
|
offset_cache = build_offset_cache(source)
|
46
|
-
result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
|
49
|
+
result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
|
47
50
|
|
48
51
|
build_ast(result.value, offset_cache)
|
49
52
|
ensure
|
@@ -56,7 +59,7 @@ module Prism
|
|
56
59
|
source = source_buffer.source
|
57
60
|
|
58
61
|
offset_cache = build_offset_cache(source)
|
59
|
-
result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
|
62
|
+
result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
|
60
63
|
|
61
64
|
[
|
62
65
|
build_ast(result.value, offset_cache),
|
@@ -75,7 +78,7 @@ module Prism
|
|
75
78
|
offset_cache = build_offset_cache(source)
|
76
79
|
result =
|
77
80
|
begin
|
78
|
-
unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
|
81
|
+
unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
|
79
82
|
rescue ::Parser::SyntaxError
|
80
83
|
raise if !recover
|
81
84
|
end
|
@@ -106,14 +109,123 @@ module Prism
|
|
106
109
|
true
|
107
110
|
end
|
108
111
|
|
112
|
+
# This is a hook to allow consumers to disable some warnings if they don't
|
113
|
+
# want them to block creating the syntax tree.
|
114
|
+
def valid_warning?(warning)
|
115
|
+
true
|
116
|
+
end
|
117
|
+
|
118
|
+
# Build a diagnostic from the given prism parse error.
|
119
|
+
def error_diagnostic(error, offset_cache)
|
120
|
+
location = error.location
|
121
|
+
diagnostic_location = build_range(location, offset_cache)
|
122
|
+
|
123
|
+
case error.type
|
124
|
+
when :argument_block_multi
|
125
|
+
Diagnostic.new(:error, :block_and_blockarg, {}, diagnostic_location, [])
|
126
|
+
when :argument_formal_constant
|
127
|
+
Diagnostic.new(:error, :argument_const, {}, diagnostic_location, [])
|
128
|
+
when :argument_formal_class
|
129
|
+
Diagnostic.new(:error, :argument_cvar, {}, diagnostic_location, [])
|
130
|
+
when :argument_formal_global
|
131
|
+
Diagnostic.new(:error, :argument_gvar, {}, diagnostic_location, [])
|
132
|
+
when :argument_formal_ivar
|
133
|
+
Diagnostic.new(:error, :argument_ivar, {}, diagnostic_location, [])
|
134
|
+
when :argument_no_forwarding_amp
|
135
|
+
Diagnostic.new(:error, :no_anonymous_blockarg, {}, diagnostic_location, [])
|
136
|
+
when :argument_no_forwarding_star
|
137
|
+
Diagnostic.new(:error, :no_anonymous_restarg, {}, diagnostic_location, [])
|
138
|
+
when :argument_no_forwarding_star_star
|
139
|
+
Diagnostic.new(:error, :no_anonymous_kwrestarg, {}, diagnostic_location, [])
|
140
|
+
when :begin_lonely_else
|
141
|
+
location = location.copy(length: 4)
|
142
|
+
diagnostic_location = build_range(location, offset_cache)
|
143
|
+
Diagnostic.new(:error, :useless_else, {}, diagnostic_location, [])
|
144
|
+
when :class_name, :module_name
|
145
|
+
Diagnostic.new(:error, :module_name_const, {}, diagnostic_location, [])
|
146
|
+
when :class_in_method
|
147
|
+
Diagnostic.new(:error, :class_in_def, {}, diagnostic_location, [])
|
148
|
+
when :def_endless_setter
|
149
|
+
Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, [])
|
150
|
+
when :embdoc_term
|
151
|
+
Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, [])
|
152
|
+
when :incomplete_variable_class, :incomplete_variable_class_3_3
|
153
|
+
location = location.copy(length: location.length + 1)
|
154
|
+
diagnostic_location = build_range(location, offset_cache)
|
155
|
+
|
156
|
+
Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, [])
|
157
|
+
when :incomplete_variable_instance, :incomplete_variable_instance_3_3
|
158
|
+
location = location.copy(length: location.length + 1)
|
159
|
+
diagnostic_location = build_range(location, offset_cache)
|
160
|
+
|
161
|
+
Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, [])
|
162
|
+
when :invalid_variable_global, :invalid_variable_global_3_3
|
163
|
+
Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, [])
|
164
|
+
when :module_in_method
|
165
|
+
Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, [])
|
166
|
+
when :numbered_parameter_ordinary
|
167
|
+
Diagnostic.new(:error, :ordinary_param_defined, {}, diagnostic_location, [])
|
168
|
+
when :numbered_parameter_outer_scope
|
169
|
+
Diagnostic.new(:error, :numparam_used_in_outer_scope, {}, diagnostic_location, [])
|
170
|
+
when :parameter_circular
|
171
|
+
Diagnostic.new(:error, :circular_argument_reference, { var_name: location.slice }, diagnostic_location, [])
|
172
|
+
when :parameter_name_repeat
|
173
|
+
Diagnostic.new(:error, :duplicate_argument, {}, diagnostic_location, [])
|
174
|
+
when :parameter_numbered_reserved
|
175
|
+
Diagnostic.new(:error, :reserved_for_numparam, { name: location.slice }, diagnostic_location, [])
|
176
|
+
when :regexp_unknown_options
|
177
|
+
Diagnostic.new(:error, :regexp_options, { options: location.slice[1..] }, diagnostic_location, [])
|
178
|
+
when :singleton_for_literals
|
179
|
+
Diagnostic.new(:error, :singleton_literal, {}, diagnostic_location, [])
|
180
|
+
when :string_literal_eof
|
181
|
+
Diagnostic.new(:error, :string_eof, {}, diagnostic_location, [])
|
182
|
+
when :unexpected_token_ignore
|
183
|
+
Diagnostic.new(:error, :unexpected_token, { token: location.slice }, diagnostic_location, [])
|
184
|
+
when :write_target_in_method
|
185
|
+
Diagnostic.new(:error, :dynamic_const, {}, diagnostic_location, [])
|
186
|
+
else
|
187
|
+
PrismDiagnostic.new(error.message, :error, error.type, diagnostic_location)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
# Build a diagnostic from the given prism parse warning.
|
192
|
+
def warning_diagnostic(warning, offset_cache)
|
193
|
+
diagnostic_location = build_range(warning.location, offset_cache)
|
194
|
+
|
195
|
+
case warning.type
|
196
|
+
when :ambiguous_first_argument_plus
|
197
|
+
Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "+" }, diagnostic_location, [])
|
198
|
+
when :ambiguous_first_argument_minus
|
199
|
+
Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "-" }, diagnostic_location, [])
|
200
|
+
when :ambiguous_prefix_ampersand
|
201
|
+
Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "&" }, diagnostic_location, [])
|
202
|
+
when :ambiguous_prefix_star
|
203
|
+
Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "*" }, diagnostic_location, [])
|
204
|
+
when :ambiguous_prefix_star_star
|
205
|
+
Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "**" }, diagnostic_location, [])
|
206
|
+
when :ambiguous_slash
|
207
|
+
Diagnostic.new(:warning, :ambiguous_regexp, {}, diagnostic_location, [])
|
208
|
+
when :dot_dot_dot_eol
|
209
|
+
Diagnostic.new(:warning, :triple_dot_at_eol, {}, diagnostic_location, [])
|
210
|
+
when :duplicated_hash_key
|
211
|
+
# skip, parser does this on its own
|
212
|
+
else
|
213
|
+
PrismDiagnostic.new(warning.message, :warning, warning.type, diagnostic_location)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
109
217
|
# If there was a error generated during the parse, then raise an
|
110
218
|
# appropriate syntax error. Otherwise return the result.
|
111
219
|
def unwrap(result, offset_cache)
|
112
220
|
result.errors.each do |error|
|
113
221
|
next unless valid_error?(error)
|
222
|
+
diagnostics.process(error_diagnostic(error, offset_cache))
|
223
|
+
end
|
114
224
|
|
115
|
-
|
116
|
-
|
225
|
+
result.warnings.each do |warning|
|
226
|
+
next unless valid_warning?(warning)
|
227
|
+
diagnostic = warning_diagnostic(warning, offset_cache)
|
228
|
+
diagnostics.process(diagnostic) if diagnostic
|
117
229
|
end
|
118
230
|
|
119
231
|
result
|
@@ -156,7 +268,7 @@ module Prism
|
|
156
268
|
|
157
269
|
# Build the parser gem tokens from the prism tokens.
|
158
270
|
def build_tokens(tokens, offset_cache)
|
159
|
-
Lexer.new(source_buffer, tokens
|
271
|
+
Lexer.new(source_buffer, tokens, offset_cache).to_a
|
160
272
|
end
|
161
273
|
|
162
274
|
# Build a range from a prism location.
|
@@ -172,7 +284,7 @@ module Prism
|
|
172
284
|
def convert_for_prism(version)
|
173
285
|
case version
|
174
286
|
when 33
|
175
|
-
"3.3.
|
287
|
+
"3.3.1"
|
176
288
|
when 34
|
177
289
|
"3.4.0"
|
178
290
|
else
|
@@ -0,0 +1,125 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "../ripper"
|
4
|
+
|
5
|
+
module Prism
|
6
|
+
module Translation
|
7
|
+
class Ripper
|
8
|
+
# This class mirrors the ::Ripper::SexpBuilder subclass of ::Ripper that
|
9
|
+
# returns the arrays of [type, *children].
|
10
|
+
class SexpBuilder < Ripper
|
11
|
+
# :stopdoc:
|
12
|
+
|
13
|
+
attr_reader :error
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def dedent_element(e, width)
|
18
|
+
if (n = dedent_string(e[1], width)) > 0
|
19
|
+
e[2][1] += n
|
20
|
+
end
|
21
|
+
e
|
22
|
+
end
|
23
|
+
|
24
|
+
def on_heredoc_dedent(val, width)
|
25
|
+
sub = proc do |cont|
|
26
|
+
cont.map! do |e|
|
27
|
+
if Array === e
|
28
|
+
case e[0]
|
29
|
+
when :@tstring_content
|
30
|
+
e = dedent_element(e, width)
|
31
|
+
when /_add\z/
|
32
|
+
e[1] = sub[e[1]]
|
33
|
+
end
|
34
|
+
elsif String === e
|
35
|
+
dedent_string(e, width)
|
36
|
+
end
|
37
|
+
e
|
38
|
+
end
|
39
|
+
end
|
40
|
+
sub[val]
|
41
|
+
val
|
42
|
+
end
|
43
|
+
|
44
|
+
events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym}
|
45
|
+
(PARSER_EVENTS - events).each do |event|
|
46
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
47
|
+
def on_#{event}(*args)
|
48
|
+
args.unshift :#{event}
|
49
|
+
end
|
50
|
+
End
|
51
|
+
end
|
52
|
+
|
53
|
+
SCANNER_EVENTS.each do |event|
|
54
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
55
|
+
def on_#{event}(tok)
|
56
|
+
[:@#{event}, tok, [lineno(), column()]]
|
57
|
+
end
|
58
|
+
End
|
59
|
+
end
|
60
|
+
|
61
|
+
def on_error(mesg)
|
62
|
+
@error = mesg
|
63
|
+
end
|
64
|
+
remove_method :on_parse_error
|
65
|
+
alias on_parse_error on_error
|
66
|
+
alias compile_error on_error
|
67
|
+
|
68
|
+
# :startdoc:
|
69
|
+
end
|
70
|
+
|
71
|
+
# This class mirrors the ::Ripper::SexpBuilderPP subclass of ::Ripper that
|
72
|
+
# returns the same values as ::Ripper::SexpBuilder except with a couple of
|
73
|
+
# niceties that flatten linked lists into arrays.
|
74
|
+
class SexpBuilderPP < SexpBuilder
|
75
|
+
# :stopdoc:
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
def on_heredoc_dedent(val, width)
|
80
|
+
val.map! do |e|
|
81
|
+
next e if Symbol === e and /_content\z/ =~ e
|
82
|
+
if Array === e and e[0] == :@tstring_content
|
83
|
+
e = dedent_element(e, width)
|
84
|
+
elsif String === e
|
85
|
+
dedent_string(e, width)
|
86
|
+
end
|
87
|
+
e
|
88
|
+
end
|
89
|
+
val
|
90
|
+
end
|
91
|
+
|
92
|
+
def _dispatch_event_new
|
93
|
+
[]
|
94
|
+
end
|
95
|
+
|
96
|
+
def _dispatch_event_push(list, item)
|
97
|
+
list.push item
|
98
|
+
list
|
99
|
+
end
|
100
|
+
|
101
|
+
def on_mlhs_paren(list)
|
102
|
+
[:mlhs, *list]
|
103
|
+
end
|
104
|
+
|
105
|
+
def on_mlhs_add_star(list, star)
|
106
|
+
list.push([:rest_param, star])
|
107
|
+
end
|
108
|
+
|
109
|
+
def on_mlhs_add_post(list, post)
|
110
|
+
list.concat(post)
|
111
|
+
end
|
112
|
+
|
113
|
+
PARSER_EVENT_TABLE.each do |event, arity|
|
114
|
+
if /_new\z/ =~ event and arity == 0
|
115
|
+
alias_method "on_#{event}", :_dispatch_event_new
|
116
|
+
elsif /_add\z/ =~ event
|
117
|
+
alias_method "on_#{event}", :_dispatch_event_push
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# :startdoc:
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|