prism 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -1
- data/README.md +4 -1
- data/config.yml +96 -35
- data/docs/fuzzing.md +5 -10
- data/docs/prism.png +0 -0
- data/docs/serialization.md +10 -0
- data/ext/prism/api_node.c +239 -86
- data/ext/prism/extension.c +35 -48
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +170 -118
- data/include/prism/diagnostic.h +1 -0
- data/include/prism/node.h +8 -0
- data/include/prism/parser.h +26 -0
- data/include/prism/util/pm_buffer.h +3 -0
- data/include/prism/util/pm_constant_pool.h +21 -2
- data/include/prism/util/pm_string.h +2 -1
- data/include/prism/version.h +2 -2
- data/include/prism.h +1 -2
- data/lib/prism/compiler.rb +150 -141
- data/lib/prism/debug.rb +30 -26
- data/lib/prism/dispatcher.rb +42 -0
- data/lib/prism/dsl.rb +23 -8
- data/lib/prism/ffi.rb +4 -4
- data/lib/prism/lex_compat.rb +42 -8
- data/lib/prism/mutation_compiler.rb +18 -3
- data/lib/prism/node.rb +2061 -191
- data/lib/prism/node_ext.rb +44 -0
- data/lib/prism/parse_result.rb +32 -5
- data/lib/prism/pattern.rb +1 -1
- data/lib/prism/serialize.rb +95 -87
- data/lib/prism/visitor.rb +9 -0
- data/prism.gemspec +2 -3
- data/src/diagnostic.c +2 -1
- data/src/node.c +99 -32
- data/src/prettyprint.c +137 -80
- data/src/prism.c +1960 -843
- data/src/serialize.c +140 -79
- data/src/util/pm_buffer.c +9 -7
- data/src/util/pm_constant_pool.c +25 -11
- metadata +3 -4
- data/include/prism/unescape.h +0 -48
- data/src/unescape.c +0 -637
data/lib/prism/debug.rb
CHANGED
@@ -45,29 +45,16 @@ module Prism
|
|
45
45
|
# For the given source, compiles with CRuby and returns a list of all of the
|
46
46
|
# sets of local variables that were encountered.
|
47
47
|
def self.cruby_locals(source)
|
48
|
-
verbose = $VERBOSE
|
49
|
-
$VERBOSE = nil
|
48
|
+
verbose, $VERBOSE = $VERBOSE, nil
|
50
49
|
|
51
50
|
begin
|
52
51
|
locals = []
|
53
52
|
stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
|
54
53
|
|
55
54
|
while (iseq = stack.pop)
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
# CRuby will push on a special local variable when there are keyword
|
60
|
-
# arguments. We get rid of that here.
|
61
|
-
names = names.grep_v(Integer)
|
62
|
-
|
63
|
-
# For some reason, CRuby occasionally pushes this special local
|
64
|
-
# variable when there are splat arguments. We get rid of that here.
|
65
|
-
names = names.grep_v(:"#arg_rest")
|
66
|
-
|
67
|
-
# Now push them onto the list of locals.
|
68
|
-
locals << names
|
69
|
-
end
|
70
|
-
|
55
|
+
# For some reason, CRuby occasionally pushes this special local
|
56
|
+
# variable when there are splat arguments. We get rid of that here.
|
57
|
+
locals << (iseq.local_table - [:"#arg_rest"])
|
71
58
|
iseq.each_child { |child| stack << child }
|
72
59
|
end
|
73
60
|
|
@@ -77,6 +64,8 @@ module Prism
|
|
77
64
|
end
|
78
65
|
end
|
79
66
|
|
67
|
+
AnonymousLocal = Object.new
|
68
|
+
|
80
69
|
# For the given source, parses with prism and returns a list of all of the
|
81
70
|
# sets of local variables that were encountered.
|
82
71
|
def self.prism_locals(source)
|
@@ -97,20 +86,27 @@ module Prism
|
|
97
86
|
# order here so that we can compare properly.
|
98
87
|
if params
|
99
88
|
sorted = [
|
100
|
-
*params.requireds.
|
89
|
+
*params.requireds.map do |required|
|
90
|
+
if required.is_a?(RequiredParameterNode)
|
91
|
+
required.name
|
92
|
+
else
|
93
|
+
AnonymousLocal
|
94
|
+
end
|
95
|
+
end,
|
101
96
|
*params.optionals.map(&:name),
|
102
97
|
*((params.rest.name || :*) if params.rest && params.rest.operator != ","),
|
103
|
-
*params.posts.
|
98
|
+
*params.posts.map do |post|
|
99
|
+
if post.is_a?(RequiredParameterNode)
|
100
|
+
post.name
|
101
|
+
else
|
102
|
+
AnonymousLocal
|
103
|
+
end
|
104
|
+
end,
|
104
105
|
*params.keywords.reject(&:value).map(&:name),
|
105
106
|
*params.keywords.select(&:value).map(&:name)
|
106
107
|
]
|
107
108
|
|
108
|
-
|
109
|
-
# onto the local list. We don't do that yet, so we need to add them
|
110
|
-
# in here.
|
111
|
-
if params.keyword_rest.is_a?(ForwardingParameterNode)
|
112
|
-
sorted.push(:*, :&, :"...")
|
113
|
-
end
|
109
|
+
sorted << AnonymousLocal if params.keywords.any?
|
114
110
|
|
115
111
|
# Recurse down the parameter tree to find any destructured
|
116
112
|
# parameters and add them after the other parameters.
|
@@ -129,11 +125,19 @@ module Prism
|
|
129
125
|
names = sorted.concat(names - sorted)
|
130
126
|
end
|
131
127
|
|
128
|
+
names.map!.with_index do |name, index|
|
129
|
+
if name == AnonymousLocal
|
130
|
+
names.length - index + 1
|
131
|
+
else
|
132
|
+
name
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
132
136
|
locals << names
|
133
137
|
when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
|
134
138
|
locals << node.locals
|
135
139
|
when ForNode
|
136
|
-
locals << []
|
140
|
+
locals << [2]
|
137
141
|
when PostExecutionNode
|
138
142
|
locals.push([], [])
|
139
143
|
when InterpolatedRegularExpressionNode
|
data/lib/prism/dispatcher.rb
CHANGED
@@ -608,6 +608,30 @@ module Prism
|
|
608
608
|
listeners[:on_in_node_leave]&.each { |listener| listener.on_in_node_leave(node) }
|
609
609
|
end
|
610
610
|
|
611
|
+
# Dispatch enter and leave events for IndexAndWriteNode nodes and continue
|
612
|
+
# walking the tree.
|
613
|
+
def visit_index_and_write_node(node)
|
614
|
+
listeners[:on_index_and_write_node_enter]&.each { |listener| listener.on_index_and_write_node_enter(node) }
|
615
|
+
super
|
616
|
+
listeners[:on_index_and_write_node_leave]&.each { |listener| listener.on_index_and_write_node_leave(node) }
|
617
|
+
end
|
618
|
+
|
619
|
+
# Dispatch enter and leave events for IndexOperatorWriteNode nodes and continue
|
620
|
+
# walking the tree.
|
621
|
+
def visit_index_operator_write_node(node)
|
622
|
+
listeners[:on_index_operator_write_node_enter]&.each { |listener| listener.on_index_operator_write_node_enter(node) }
|
623
|
+
super
|
624
|
+
listeners[:on_index_operator_write_node_leave]&.each { |listener| listener.on_index_operator_write_node_leave(node) }
|
625
|
+
end
|
626
|
+
|
627
|
+
# Dispatch enter and leave events for IndexOrWriteNode nodes and continue
|
628
|
+
# walking the tree.
|
629
|
+
def visit_index_or_write_node(node)
|
630
|
+
listeners[:on_index_or_write_node_enter]&.each { |listener| listener.on_index_or_write_node_enter(node) }
|
631
|
+
super
|
632
|
+
listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
|
633
|
+
end
|
634
|
+
|
611
635
|
# Dispatch enter and leave events for InstanceVariableAndWriteNode nodes and continue
|
612
636
|
# walking the tree.
|
613
637
|
def visit_instance_variable_and_write_node(node)
|
@@ -1607,6 +1631,24 @@ module Prism
|
|
1607
1631
|
listeners[:on_in_node_leave]&.each { |listener| listener.on_in_node_leave(node) }
|
1608
1632
|
end
|
1609
1633
|
|
1634
|
+
# Dispatch enter and leave events for IndexAndWriteNode nodes.
|
1635
|
+
def visit_index_and_write_node(node)
|
1636
|
+
listeners[:on_index_and_write_node_enter]&.each { |listener| listener.on_index_and_write_node_enter(node) }
|
1637
|
+
listeners[:on_index_and_write_node_leave]&.each { |listener| listener.on_index_and_write_node_leave(node) }
|
1638
|
+
end
|
1639
|
+
|
1640
|
+
# Dispatch enter and leave events for IndexOperatorWriteNode nodes.
|
1641
|
+
def visit_index_operator_write_node(node)
|
1642
|
+
listeners[:on_index_operator_write_node_enter]&.each { |listener| listener.on_index_operator_write_node_enter(node) }
|
1643
|
+
listeners[:on_index_operator_write_node_leave]&.each { |listener| listener.on_index_operator_write_node_leave(node) }
|
1644
|
+
end
|
1645
|
+
|
1646
|
+
# Dispatch enter and leave events for IndexOrWriteNode nodes.
|
1647
|
+
def visit_index_or_write_node(node)
|
1648
|
+
listeners[:on_index_or_write_node_enter]&.each { |listener| listener.on_index_or_write_node_enter(node) }
|
1649
|
+
listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
|
1650
|
+
end
|
1651
|
+
|
1610
1652
|
# Dispatch enter and leave events for InstanceVariableAndWriteNode nodes.
|
1611
1653
|
def visit_instance_variable_and_write_node(node)
|
1612
1654
|
listeners[:on_instance_variable_and_write_node_enter]&.each { |listener| listener.on_instance_variable_and_write_node_enter(node) }
|
data/lib/prism/dsl.rb
CHANGED
@@ -88,8 +88,8 @@ module Prism
|
|
88
88
|
end
|
89
89
|
|
90
90
|
# Create a new BackReferenceReadNode node
|
91
|
-
def BackReferenceReadNode(location = Location())
|
92
|
-
BackReferenceReadNode.new(location)
|
91
|
+
def BackReferenceReadNode(name, location = Location())
|
92
|
+
BackReferenceReadNode.new(name, location)
|
93
93
|
end
|
94
94
|
|
95
95
|
# Create a new BeginNode node
|
@@ -128,8 +128,8 @@ module Prism
|
|
128
128
|
end
|
129
129
|
|
130
130
|
# Create a new CallAndWriteNode node
|
131
|
-
def CallAndWriteNode(receiver, call_operator_loc, message_loc,
|
132
|
-
CallAndWriteNode.new(receiver, call_operator_loc, message_loc,
|
131
|
+
def CallAndWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
|
132
|
+
CallAndWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
|
133
133
|
end
|
134
134
|
|
135
135
|
# Create a new CallNode node
|
@@ -138,13 +138,13 @@ module Prism
|
|
138
138
|
end
|
139
139
|
|
140
140
|
# Create a new CallOperatorWriteNode node
|
141
|
-
def CallOperatorWriteNode(receiver, call_operator_loc, message_loc,
|
142
|
-
CallOperatorWriteNode.new(receiver, call_operator_loc, message_loc,
|
141
|
+
def CallOperatorWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location = Location())
|
142
|
+
CallOperatorWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location)
|
143
143
|
end
|
144
144
|
|
145
145
|
# Create a new CallOrWriteNode node
|
146
|
-
def CallOrWriteNode(receiver, call_operator_loc, message_loc,
|
147
|
-
CallOrWriteNode.new(receiver, call_operator_loc, message_loc,
|
146
|
+
def CallOrWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
|
147
|
+
CallOrWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
|
148
148
|
end
|
149
149
|
|
150
150
|
# Create a new CapturePatternNode node
|
@@ -382,6 +382,21 @@ module Prism
|
|
382
382
|
InNode.new(pattern, statements, in_loc, then_loc, location)
|
383
383
|
end
|
384
384
|
|
385
|
+
# Create a new IndexAndWriteNode node
|
386
|
+
def IndexAndWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
|
387
|
+
IndexAndWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
|
388
|
+
end
|
389
|
+
|
390
|
+
# Create a new IndexOperatorWriteNode node
|
391
|
+
def IndexOperatorWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location = Location())
|
392
|
+
IndexOperatorWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location)
|
393
|
+
end
|
394
|
+
|
395
|
+
# Create a new IndexOrWriteNode node
|
396
|
+
def IndexOrWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
|
397
|
+
IndexOrWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
|
398
|
+
end
|
399
|
+
|
385
400
|
# Create a new InstanceVariableAndWriteNode node
|
386
401
|
def InstanceVariableAndWriteNode(name, name_loc, operator_loc, value, location = Location())
|
387
402
|
InstanceVariableAndWriteNode.new(name, name_loc, operator_loc, value, location)
|
data/lib/prism/ffi.rb
CHANGED
@@ -25,8 +25,8 @@ module Prism
|
|
25
25
|
# void -> :void
|
26
26
|
#
|
27
27
|
def self.resolve_type(type)
|
28
|
-
type = type.strip
|
29
|
-
type.end_with?("*") ? :pointer : type.to_sym
|
28
|
+
type = type.strip
|
29
|
+
type.end_with?("*") ? :pointer : type.delete_prefix("const ").to_sym
|
30
30
|
end
|
31
31
|
|
32
32
|
# Read through the given header file and find the declaration of each of the
|
@@ -234,11 +234,11 @@ module Prism
|
|
234
234
|
loader = Serialize::Loader.new(source, buffer.read)
|
235
235
|
|
236
236
|
tokens = loader.load_tokens
|
237
|
-
node, comments, errors, warnings = loader.load_nodes
|
237
|
+
node, comments, magic_comments, errors, warnings = loader.load_nodes
|
238
238
|
|
239
239
|
tokens.each { |token,| token.value.force_encoding(loader.encoding) }
|
240
240
|
|
241
|
-
ParseResult.new([node, tokens], comments, errors, warnings, source)
|
241
|
+
ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
|
242
242
|
end
|
243
243
|
end
|
244
244
|
|
data/lib/prism/lex_compat.rb
CHANGED
@@ -357,6 +357,7 @@ module Prism
|
|
357
357
|
@dedent_next = true
|
358
358
|
@dedent = nil
|
359
359
|
@embexpr_balance = 0
|
360
|
+
@ended_on_newline = false
|
360
361
|
end
|
361
362
|
|
362
363
|
# As tokens are coming in, we track the minimum amount of common leading
|
@@ -366,14 +367,14 @@ module Prism
|
|
366
367
|
case token.event
|
367
368
|
when :on_embexpr_beg, :on_heredoc_beg
|
368
369
|
@embexpr_balance += 1
|
370
|
+
@dedent = 0 if @dedent_next && @ended_on_newline
|
369
371
|
when :on_embexpr_end, :on_heredoc_end
|
370
372
|
@embexpr_balance -= 1
|
371
373
|
when :on_tstring_content
|
372
374
|
if embexpr_balance == 0
|
373
|
-
token.value
|
374
|
-
next if line.strip.empty? && line.end_with?("\n")
|
375
|
-
next if !(dedent_next || index > 0)
|
375
|
+
line = token.value
|
376
376
|
|
377
|
+
if dedent_next && !(line.strip.empty? && line.end_with?("\n"))
|
377
378
|
leading = line[/\A(\s*)\n?/, 1]
|
378
379
|
next_dedent = 0
|
379
380
|
|
@@ -386,11 +387,16 @@ module Prism
|
|
386
387
|
end
|
387
388
|
|
388
389
|
@dedent = [dedent, next_dedent].compact.min
|
390
|
+
@dedent_next = true
|
391
|
+
@ended_on_newline = line.end_with?("\n")
|
392
|
+
tokens << token
|
393
|
+
return
|
389
394
|
end
|
390
395
|
end
|
391
396
|
end
|
392
397
|
|
393
398
|
@dedent_next = token.event == :on_tstring_content && embexpr_balance == 0
|
399
|
+
@ended_on_newline = false
|
394
400
|
tokens << token
|
395
401
|
end
|
396
402
|
|
@@ -430,6 +436,38 @@ module Prism
|
|
430
436
|
return results
|
431
437
|
end
|
432
438
|
|
439
|
+
# If the minimum common whitespace is 0, then we need to concatenate
|
440
|
+
# string nodes together that are immediately adjacent.
|
441
|
+
if dedent == 0
|
442
|
+
results = []
|
443
|
+
embexpr_balance = 0
|
444
|
+
|
445
|
+
index = 0
|
446
|
+
max_index = tokens.length
|
447
|
+
|
448
|
+
while index < max_index
|
449
|
+
token = tokens[index]
|
450
|
+
results << token
|
451
|
+
index += 1
|
452
|
+
|
453
|
+
case token.event
|
454
|
+
when :on_embexpr_beg, :on_heredoc_beg
|
455
|
+
embexpr_balance += 1
|
456
|
+
when :on_embexpr_end, :on_heredoc_end
|
457
|
+
embexpr_balance -= 1
|
458
|
+
when :on_tstring_content
|
459
|
+
if embexpr_balance == 0
|
460
|
+
while index < max_index && tokens[index].event == :on_tstring_content
|
461
|
+
token.value << tokens[index].value
|
462
|
+
index += 1
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
return results
|
469
|
+
end
|
470
|
+
|
433
471
|
# Otherwise, we're going to run through each token in the list and
|
434
472
|
# insert on_ignored_sp tokens for the amount of dedent that we need to
|
435
473
|
# perform. We also need to remove the dedent from the beginning of
|
@@ -787,11 +825,7 @@ module Prism
|
|
787
825
|
# We sort by location to compare against Ripper's output
|
788
826
|
tokens.sort_by!(&:location)
|
789
827
|
|
790
|
-
|
791
|
-
raise StandardError, "Lost tokens when performing lex_compat"
|
792
|
-
end
|
793
|
-
|
794
|
-
ParseResult.new(tokens, result.comments, result.errors, result.warnings, [])
|
828
|
+
ParseResult.new(tokens, result.comments, result.magic_comments, result.errors, result.warnings, [])
|
795
829
|
end
|
796
830
|
end
|
797
831
|
|
@@ -97,7 +97,7 @@ module Prism
|
|
97
97
|
|
98
98
|
# Copy a CallAndWriteNode node
|
99
99
|
def visit_call_and_write_node(node)
|
100
|
-
node.copy(receiver: visit(node.receiver),
|
100
|
+
node.copy(receiver: visit(node.receiver), value: visit(node.value))
|
101
101
|
end
|
102
102
|
|
103
103
|
# Copy a CallNode node
|
@@ -107,12 +107,12 @@ module Prism
|
|
107
107
|
|
108
108
|
# Copy a CallOperatorWriteNode node
|
109
109
|
def visit_call_operator_write_node(node)
|
110
|
-
node.copy(receiver: visit(node.receiver),
|
110
|
+
node.copy(receiver: visit(node.receiver), value: visit(node.value))
|
111
111
|
end
|
112
112
|
|
113
113
|
# Copy a CallOrWriteNode node
|
114
114
|
def visit_call_or_write_node(node)
|
115
|
-
node.copy(receiver: visit(node.receiver),
|
115
|
+
node.copy(receiver: visit(node.receiver), value: visit(node.value))
|
116
116
|
end
|
117
117
|
|
118
118
|
# Copy a CapturePatternNode node
|
@@ -350,6 +350,21 @@ module Prism
|
|
350
350
|
node.copy(pattern: visit(node.pattern), statements: visit(node.statements))
|
351
351
|
end
|
352
352
|
|
353
|
+
# Copy a IndexAndWriteNode node
|
354
|
+
def visit_index_and_write_node(node)
|
355
|
+
node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
|
356
|
+
end
|
357
|
+
|
358
|
+
# Copy a IndexOperatorWriteNode node
|
359
|
+
def visit_index_operator_write_node(node)
|
360
|
+
node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
|
361
|
+
end
|
362
|
+
|
363
|
+
# Copy a IndexOrWriteNode node
|
364
|
+
def visit_index_or_write_node(node)
|
365
|
+
node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
|
366
|
+
end
|
367
|
+
|
353
368
|
# Copy a InstanceVariableAndWriteNode node
|
354
369
|
def visit_instance_variable_and_write_node(node)
|
355
370
|
node.copy(value: visit(node.value))
|