prism 0.24.0 → 0.29.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/BSDmakefile +58 -0
- data/CHANGELOG.md +132 -1
- data/Makefile +25 -18
- data/README.md +45 -6
- data/config.yml +828 -25
- data/docs/build_system.md +31 -0
- data/docs/configuration.md +4 -0
- data/docs/cruby_compilation.md +1 -1
- data/docs/parser_translation.md +14 -9
- data/docs/releasing.md +7 -9
- data/docs/ripper_translation.md +50 -0
- data/docs/ruby_api.md +1 -0
- data/docs/serialization.md +26 -5
- data/ext/prism/api_node.c +1037 -936
- data/ext/prism/api_pack.c +9 -0
- data/ext/prism/extconf.rb +62 -18
- data/ext/prism/extension.c +351 -71
- data/ext/prism/extension.h +5 -4
- data/include/prism/ast.h +539 -101
- data/include/prism/defines.h +106 -2
- data/include/prism/diagnostic.h +168 -74
- data/include/prism/encoding.h +22 -4
- data/include/prism/node.h +93 -0
- data/include/prism/options.h +84 -9
- data/include/prism/pack.h +11 -0
- data/include/prism/parser.h +213 -54
- data/include/prism/prettyprint.h +8 -0
- data/include/prism/static_literals.h +120 -0
- data/include/prism/util/pm_buffer.h +65 -2
- data/include/prism/util/pm_constant_pool.h +18 -1
- data/include/prism/util/pm_integer.h +119 -0
- data/include/prism/util/pm_list.h +1 -1
- data/include/prism/util/pm_newline_list.h +8 -0
- data/include/prism/util/pm_string.h +26 -2
- data/include/prism/version.h +2 -2
- data/include/prism.h +59 -1
- data/lib/prism/compiler.rb +8 -1
- data/lib/prism/debug.rb +46 -3
- data/lib/prism/desugar_compiler.rb +5 -3
- data/lib/prism/dispatcher.rb +29 -0
- data/lib/prism/dot_visitor.rb +141 -54
- data/lib/prism/dsl.rb +48 -36
- data/lib/prism/ffi.rb +82 -17
- data/lib/prism/inspect_visitor.rb +2156 -0
- data/lib/prism/lex_compat.rb +34 -15
- data/lib/prism/mutation_compiler.rb +13 -2
- data/lib/prism/node.rb +4453 -4459
- data/lib/prism/node_ext.rb +249 -30
- data/lib/prism/pack.rb +4 -0
- data/lib/prism/parse_result/comments.rb +35 -18
- data/lib/prism/parse_result/newlines.rb +2 -2
- data/lib/prism/parse_result.rb +218 -43
- data/lib/prism/pattern.rb +28 -10
- data/lib/prism/polyfill/byteindex.rb +13 -0
- data/lib/prism/polyfill/unpack1.rb +14 -0
- data/lib/prism/reflection.rb +411 -0
- data/lib/prism/serialize.rb +480 -112
- data/lib/prism/translation/parser/compiler.rb +376 -88
- data/lib/prism/translation/parser/lexer.rb +103 -22
- data/lib/prism/translation/parser/rubocop.rb +41 -13
- data/lib/prism/translation/parser.rb +123 -11
- data/lib/prism/translation/parser33.rb +1 -1
- data/lib/prism/translation/parser34.rb +1 -1
- data/lib/prism/translation/ripper/sexp.rb +125 -0
- data/lib/prism/translation/ripper/shim.rb +5 -0
- data/lib/prism/translation/ripper.rb +3216 -462
- data/lib/prism/translation/ruby_parser.rb +111 -56
- data/lib/prism/translation.rb +3 -1
- data/lib/prism/visitor.rb +10 -0
- data/lib/prism.rb +12 -20
- data/prism.gemspec +46 -14
- data/rbi/prism/compiler.rbi +12 -0
- data/rbi/prism/inspect_visitor.rbi +12 -0
- data/rbi/prism/node.rbi +8712 -0
- data/rbi/prism/node_ext.rbi +107 -0
- data/rbi/prism/parse_result.rbi +358 -0
- data/rbi/prism/reflection.rbi +58 -0
- data/rbi/prism/translation/parser.rbi +11 -0
- data/rbi/prism/translation/parser33.rbi +6 -0
- data/rbi/prism/translation/parser34.rbi +6 -0
- data/rbi/prism/translation/ripper.rbi +15 -0
- data/rbi/prism/visitor.rbi +470 -0
- data/rbi/prism.rbi +38 -7748
- data/sig/prism/compiler.rbs +9 -0
- data/sig/prism/dispatcher.rbs +16 -0
- data/sig/prism/dot_visitor.rbs +6 -0
- data/sig/prism/dsl.rbs +462 -0
- data/sig/prism/inspect_visitor.rbs +22 -0
- data/sig/prism/lex_compat.rbs +10 -0
- data/sig/prism/mutation_compiler.rbs +158 -0
- data/sig/prism/node.rbs +3558 -0
- data/sig/prism/node_ext.rbs +82 -0
- data/sig/prism/pack.rbs +43 -0
- data/sig/prism/parse_result.rbs +160 -0
- data/sig/prism/pattern.rbs +13 -0
- data/sig/prism/reflection.rbs +50 -0
- data/sig/prism/serialize.rbs +6 -0
- data/sig/prism/visitor.rbs +168 -0
- data/sig/prism.rbs +188 -4767
- data/src/diagnostic.c +636 -230
- data/src/encoding.c +211 -108
- data/src/node.c +7555 -451
- data/src/options.c +66 -31
- data/src/pack.c +33 -17
- data/src/prettyprint.c +1383 -1431
- data/src/prism.c +4734 -1310
- data/src/regexp.c +17 -2
- data/src/serialize.c +68 -46
- data/src/static_literals.c +638 -0
- data/src/token_type.c +10 -9
- data/src/util/pm_buffer.c +147 -20
- data/src/util/pm_char.c +4 -4
- data/src/util/pm_constant_pool.c +35 -11
- data/src/util/pm_integer.c +642 -0
- data/src/util/pm_list.c +1 -1
- data/src/util/pm_newline_list.c +14 -5
- data/src/util/pm_string.c +134 -5
- data/src/util/pm_string_list.c +2 -2
- metadata +41 -9
- data/docs/ripper.md +0 -36
- data/include/prism/util/pm_state_stack.h +0 -42
- data/lib/prism/node_inspector.rb +0 -68
- data/rbi/prism_static.rbi +0 -207
- data/sig/prism_static.rbs +0 -201
- data/src/util/pm_state_stack.c +0 -25
data/lib/prism/lex_compat.rb
CHANGED
@@ -10,6 +10,23 @@ module Prism
|
|
10
10
|
# generally lines up. However, there are a few cases that require special
|
11
11
|
# handling.
|
12
12
|
class LexCompat # :nodoc:
|
13
|
+
# A result class specialized for holding tokens produced by the lexer.
|
14
|
+
class Result < Prism::Result
|
15
|
+
# The list of tokens that were produced by the lexer.
|
16
|
+
attr_reader :value
|
17
|
+
|
18
|
+
# Create a new lex compat result object with the given values.
|
19
|
+
def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
|
20
|
+
@value = value
|
21
|
+
super(comments, magic_comments, data_loc, errors, warnings, source)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Implement the hash pattern matching interface for Result.
|
25
|
+
def deconstruct_keys(keys)
|
26
|
+
super.merge!(value: value)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
13
30
|
# This is a mapping of prism token types to Ripper token types. This is a
|
14
31
|
# many-to-one mapping because we split up our token types, whereas Ripper
|
15
32
|
# tends to group them.
|
@@ -185,6 +202,8 @@ module Prism
|
|
185
202
|
# However, we add a couple of convenience methods onto them to make them a
|
186
203
|
# little easier to work with. We delegate all other methods to the array.
|
187
204
|
class Token < SimpleDelegator
|
205
|
+
# @dynamic initialize, each, []
|
206
|
+
|
188
207
|
# The location of the token in the source.
|
189
208
|
def location
|
190
209
|
self[0]
|
@@ -241,10 +260,10 @@ module Prism
|
|
241
260
|
def ==(other) # :nodoc:
|
242
261
|
return false unless self[0...-1] == other[0...-1]
|
243
262
|
|
244
|
-
if self[
|
245
|
-
other[
|
263
|
+
if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
|
264
|
+
other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0
|
246
265
|
else
|
247
|
-
self[
|
266
|
+
self[3] == other[3]
|
248
267
|
end
|
249
268
|
end
|
250
269
|
end
|
@@ -308,7 +327,7 @@ module Prism
|
|
308
327
|
def to_a
|
309
328
|
embexpr_balance = 0
|
310
329
|
|
311
|
-
tokens.each_with_object([]) do |token, results|
|
330
|
+
tokens.each_with_object([]) do |token, results| #$ Array[Token]
|
312
331
|
case token.event
|
313
332
|
when :on_embexpr_beg
|
314
333
|
embexpr_balance += 1
|
@@ -409,7 +428,7 @@ module Prism
|
|
409
428
|
# If every line in the heredoc is blank, we still need to split up the
|
410
429
|
# string content token into multiple tokens.
|
411
430
|
if dedent.nil?
|
412
|
-
results = []
|
431
|
+
results = [] #: Array[Token]
|
413
432
|
embexpr_balance = 0
|
414
433
|
|
415
434
|
tokens.each do |token|
|
@@ -444,7 +463,7 @@ module Prism
|
|
444
463
|
# If the minimum common whitespace is 0, then we need to concatenate
|
445
464
|
# string nodes together that are immediately adjacent.
|
446
465
|
if dedent == 0
|
447
|
-
results = []
|
466
|
+
results = [] #: Array[Token]
|
448
467
|
embexpr_balance = 0
|
449
468
|
|
450
469
|
index = 0
|
@@ -477,7 +496,7 @@ module Prism
|
|
477
496
|
# insert on_ignored_sp tokens for the amount of dedent that we need to
|
478
497
|
# perform. We also need to remove the dedent from the beginning of
|
479
498
|
# each line of plain string content tokens.
|
480
|
-
results = []
|
499
|
+
results = [] #: Array[Token]
|
481
500
|
dedent_next = true
|
482
501
|
embexpr_balance = 0
|
483
502
|
|
@@ -526,7 +545,7 @@ module Prism
|
|
526
545
|
# dedent from the beginning of the line.
|
527
546
|
if (dedent > 0) && (dedent_next || index > 0)
|
528
547
|
deleting = 0
|
529
|
-
deleted_chars = []
|
548
|
+
deleted_chars = [] #: Array[String]
|
530
549
|
|
531
550
|
# Gather up all of the characters that we're going to
|
532
551
|
# delete, stopping when you hit a character that would put
|
@@ -603,15 +622,15 @@ module Prism
|
|
603
622
|
end
|
604
623
|
|
605
624
|
def result
|
606
|
-
tokens = []
|
625
|
+
tokens = [] #: Array[LexCompat::Token]
|
607
626
|
|
608
627
|
state = :default
|
609
|
-
heredoc_stack = [[]]
|
628
|
+
heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]]
|
610
629
|
|
611
630
|
result = Prism.lex(source, **options)
|
612
631
|
result_value = result.value
|
613
|
-
previous_state = nil
|
614
|
-
last_heredoc_end = nil
|
632
|
+
previous_state = nil #: Ripper::Lexer::State?
|
633
|
+
last_heredoc_end = nil #: Integer?
|
615
634
|
|
616
635
|
# In previous versions of Ruby, Ripper wouldn't flush the bom before the
|
617
636
|
# first token, so we had to have a hack in place to account for that. This
|
@@ -842,7 +861,7 @@ module Prism
|
|
842
861
|
# We sort by location to compare against Ripper's output
|
843
862
|
tokens.sort_by!(&:location)
|
844
863
|
|
845
|
-
|
864
|
+
Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source))
|
846
865
|
end
|
847
866
|
end
|
848
867
|
|
@@ -858,8 +877,8 @@ module Prism
|
|
858
877
|
end
|
859
878
|
|
860
879
|
def result
|
861
|
-
previous = []
|
862
|
-
results = []
|
880
|
+
previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
|
881
|
+
results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
|
863
882
|
|
864
883
|
lex(source).each do |token|
|
865
884
|
case token[1]
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
=begin
|
3
4
|
This file is generated by the templates/template.rb script and should not be
|
4
5
|
modified manually. See templates/lib/prism/mutation_compiler.rb.erb
|
@@ -192,7 +193,7 @@ module Prism
|
|
192
193
|
|
193
194
|
# Copy a ConstantPathNode node
|
194
195
|
def visit_constant_path_node(node)
|
195
|
-
node.copy(parent: visit(node.parent)
|
196
|
+
node.copy(parent: visit(node.parent))
|
196
197
|
end
|
197
198
|
|
198
199
|
# Copy a ConstantPathOperatorWriteNode node
|
@@ -207,7 +208,7 @@ module Prism
|
|
207
208
|
|
208
209
|
# Copy a ConstantPathTargetNode node
|
209
210
|
def visit_constant_path_target_node(node)
|
210
|
-
node.copy(parent: visit(node.parent)
|
211
|
+
node.copy(parent: visit(node.parent))
|
211
212
|
end
|
212
213
|
|
213
214
|
# Copy a ConstantPathWriteNode node
|
@@ -445,6 +446,11 @@ module Prism
|
|
445
446
|
node.copy(parts: visit_all(node.parts))
|
446
447
|
end
|
447
448
|
|
449
|
+
# Copy a ItParametersNode node
|
450
|
+
def visit_it_parameters_node(node)
|
451
|
+
node.copy
|
452
|
+
end
|
453
|
+
|
448
454
|
# Copy a KeywordHashNode node
|
449
455
|
def visit_keyword_hash_node(node)
|
450
456
|
node.copy(elements: visit_all(node.elements))
|
@@ -665,6 +671,11 @@ module Prism
|
|
665
671
|
node.copy
|
666
672
|
end
|
667
673
|
|
674
|
+
# Copy a ShareableConstantNode node
|
675
|
+
def visit_shareable_constant_node(node)
|
676
|
+
node.copy(write: visit(node.write))
|
677
|
+
end
|
678
|
+
|
668
679
|
# Copy a SingletonClassNode node
|
669
680
|
def visit_singleton_class_node(node)
|
670
681
|
node.copy(expression: visit(node.expression), body: visit(node.body))
|