prism 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -1
- data/Makefile +6 -0
- data/README.md +1 -1
- data/config.yml +50 -35
- data/docs/fuzzing.md +1 -1
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +802 -770
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +464 -162
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3173 -763
- data/include/prism/defines.h +32 -9
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +118 -28
- data/include/prism/node.h +38 -13
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -200
- data/include/prism/prettyprint.h +12 -1
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +94 -16
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +126 -32
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +70 -27
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +27 -2
- data/include/prism.h +224 -31
- data/lib/prism/compiler.rb +6 -3
- data/lib/prism/debug.rb +23 -7
- data/lib/prism/dispatcher.rb +33 -18
- data/lib/prism/dsl.rb +10 -5
- data/lib/prism/ffi.rb +132 -80
- data/lib/prism/lex_compat.rb +25 -15
- data/lib/prism/mutation_compiler.rb +10 -5
- data/lib/prism/node.rb +370 -135
- data/lib/prism/node_ext.rb +1 -1
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +150 -30
- data/lib/prism/pattern.rb +11 -0
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +86 -54
- data/lib/prism/visitor.rb +10 -3
- data/lib/prism.rb +20 -2
- data/prism.gemspec +4 -2
- data/rbi/prism.rbi +104 -60
- data/rbi/prism_static.rbi +16 -2
- data/sig/prism.rbs +72 -43
- data/sig/prism_static.rbs +14 -1
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +53 -8
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +334 -321
- data/src/options.c +170 -0
- data/src/prettyprint.c +74 -47
- data/src/prism.c +1642 -856
- data/src/regexp.c +151 -95
- data/src/serialize.c +44 -20
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +45 -15
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +51 -21
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +20 -12
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +5 -3
- data/docs/prism.png +0 -0
data/lib/prism/debug.rb
CHANGED
@@ -4,7 +4,9 @@ module Prism
|
|
4
4
|
# This module is used for testing and debugging and is not meant to be used by
|
5
5
|
# consumers of this library.
|
6
6
|
module Debug
|
7
|
-
|
7
|
+
# A wrapper around a RubyVM::InstructionSequence that provides a more
|
8
|
+
# convenient interface for accessing parts of the iseq.
|
9
|
+
class ISeq # :nodoc:
|
8
10
|
attr_reader :parts
|
9
11
|
|
10
12
|
def initialize(parts)
|
@@ -42,6 +44,11 @@ module Prism
|
|
42
44
|
end
|
43
45
|
end
|
44
46
|
|
47
|
+
private_constant :ISeq
|
48
|
+
|
49
|
+
# :call-seq:
|
50
|
+
# Debug::cruby_locals(source) -> Array
|
51
|
+
#
|
45
52
|
# For the given source, compiles with CRuby and returns a list of all of the
|
46
53
|
# sets of local variables that were encountered.
|
47
54
|
def self.cruby_locals(source)
|
@@ -76,8 +83,16 @@ module Prism
|
|
76
83
|
end
|
77
84
|
end
|
78
85
|
|
86
|
+
# Used to hold the place of a local that will be in the local table but
|
87
|
+
# cannot be accessed directly from the source code. For example, the
|
88
|
+
# iteration variable in a for loop or the positional parameter on a method
|
89
|
+
# definition that is destructured.
|
79
90
|
AnonymousLocal = Object.new
|
91
|
+
private_constant :AnonymousLocal
|
80
92
|
|
93
|
+
# :call-seq:
|
94
|
+
# Debug::prism_locals(source) -> Array
|
95
|
+
#
|
81
96
|
# For the given source, parses with prism and returns a list of all of the
|
82
97
|
# sets of local variables that were encountered.
|
83
98
|
def self.prism_locals(source)
|
@@ -114,8 +129,8 @@ module Prism
|
|
114
129
|
AnonymousLocal
|
115
130
|
end
|
116
131
|
end,
|
117
|
-
*params.keywords.
|
118
|
-
*params.keywords.
|
132
|
+
*params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
|
133
|
+
*params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
|
119
134
|
]
|
120
135
|
|
121
136
|
sorted << AnonymousLocal if params.keywords.any?
|
@@ -164,12 +179,13 @@ module Prism
|
|
164
179
|
locals
|
165
180
|
end
|
166
181
|
|
182
|
+
# :call-seq:
|
183
|
+
# Debug::newlines(source) -> Array
|
184
|
+
#
|
185
|
+
# For the given source string, return the byte offsets of every newline in
|
186
|
+
# the source.
|
167
187
|
def self.newlines(source)
|
168
188
|
Prism.parse(source).source.offsets
|
169
189
|
end
|
170
|
-
|
171
|
-
def self.parse_serialize_file(filepath)
|
172
|
-
parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
|
173
|
-
end
|
174
190
|
end
|
175
191
|
end
|
data/lib/prism/dispatcher.rb
CHANGED
@@ -41,23 +41,24 @@ module Prism
|
|
41
41
|
# attr_reader listeners: Hash[Symbol, Array[Listener]]
|
42
42
|
attr_reader :listeners
|
43
43
|
|
44
|
+
# Initialize a new dispatcher.
|
44
45
|
def initialize
|
45
46
|
@listeners = {}
|
46
47
|
end
|
47
48
|
|
48
|
-
# Register a listener for one or more events
|
49
|
+
# Register a listener for one or more events.
|
49
50
|
#
|
50
51
|
# def register: (Listener, *Symbol) -> void
|
51
52
|
def register(listener, *events)
|
52
53
|
events.each { |event| (listeners[event] ||= []) << listener }
|
53
54
|
end
|
54
55
|
|
55
|
-
# Walks `root` dispatching events to all registered listeners
|
56
|
+
# Walks `root` dispatching events to all registered listeners.
|
56
57
|
#
|
57
58
|
# def dispatch: (Node) -> void
|
58
59
|
alias dispatch visit
|
59
60
|
|
60
|
-
# Dispatches a single event for `node` to all registered listeners
|
61
|
+
# Dispatches a single event for `node` to all registered listeners.
|
61
62
|
#
|
62
63
|
# def dispatch_once: (Node) -> void
|
63
64
|
def dispatch_once(node)
|
@@ -736,14 +737,6 @@ module Prism
|
|
736
737
|
listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
|
737
738
|
end
|
738
739
|
|
739
|
-
# Dispatch enter and leave events for KeywordParameterNode nodes and continue
|
740
|
-
# walking the tree.
|
741
|
-
def visit_keyword_parameter_node(node)
|
742
|
-
listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
|
743
|
-
super
|
744
|
-
listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
|
745
|
-
end
|
746
|
-
|
747
740
|
# Dispatch enter and leave events for KeywordRestParameterNode nodes and continue
|
748
741
|
# walking the tree.
|
749
742
|
def visit_keyword_rest_parameter_node(node)
|
@@ -904,6 +897,14 @@ module Prism
|
|
904
897
|
listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
|
905
898
|
end
|
906
899
|
|
900
|
+
# Dispatch enter and leave events for OptionalKeywordParameterNode nodes and continue
|
901
|
+
# walking the tree.
|
902
|
+
def visit_optional_keyword_parameter_node(node)
|
903
|
+
listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
|
904
|
+
super
|
905
|
+
listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
|
906
|
+
end
|
907
|
+
|
907
908
|
# Dispatch enter and leave events for OptionalParameterNode nodes and continue
|
908
909
|
# walking the tree.
|
909
910
|
def visit_optional_parameter_node(node)
|
@@ -1008,6 +1009,14 @@ module Prism
|
|
1008
1009
|
listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
|
1009
1010
|
end
|
1010
1011
|
|
1012
|
+
# Dispatch enter and leave events for RequiredKeywordParameterNode nodes and continue
|
1013
|
+
# walking the tree.
|
1014
|
+
def visit_required_keyword_parameter_node(node)
|
1015
|
+
listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
|
1016
|
+
super
|
1017
|
+
listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
|
1018
|
+
end
|
1019
|
+
|
1011
1020
|
# Dispatch enter and leave events for RequiredParameterNode nodes and continue
|
1012
1021
|
# walking the tree.
|
1013
1022
|
def visit_required_parameter_node(node)
|
@@ -1208,7 +1217,7 @@ module Prism
|
|
1208
1217
|
listeners[:on_yield_node_leave]&.each { |listener| listener.on_yield_node_leave(node) }
|
1209
1218
|
end
|
1210
1219
|
|
1211
|
-
class DispatchOnce < Visitor
|
1220
|
+
class DispatchOnce < Visitor # :nodoc:
|
1212
1221
|
attr_reader :listeners
|
1213
1222
|
|
1214
1223
|
def initialize(listeners)
|
@@ -1719,12 +1728,6 @@ module Prism
|
|
1719
1728
|
listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
|
1720
1729
|
end
|
1721
1730
|
|
1722
|
-
# Dispatch enter and leave events for KeywordParameterNode nodes.
|
1723
|
-
def visit_keyword_parameter_node(node)
|
1724
|
-
listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
|
1725
|
-
listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
|
1726
|
-
end
|
1727
|
-
|
1728
1731
|
# Dispatch enter and leave events for KeywordRestParameterNode nodes.
|
1729
1732
|
def visit_keyword_rest_parameter_node(node)
|
1730
1733
|
listeners[:on_keyword_rest_parameter_node_enter]&.each { |listener| listener.on_keyword_rest_parameter_node_enter(node) }
|
@@ -1845,6 +1848,12 @@ module Prism
|
|
1845
1848
|
listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
|
1846
1849
|
end
|
1847
1850
|
|
1851
|
+
# Dispatch enter and leave events for OptionalKeywordParameterNode nodes.
|
1852
|
+
def visit_optional_keyword_parameter_node(node)
|
1853
|
+
listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
|
1854
|
+
listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
|
1855
|
+
end
|
1856
|
+
|
1848
1857
|
# Dispatch enter and leave events for OptionalParameterNode nodes.
|
1849
1858
|
def visit_optional_parameter_node(node)
|
1850
1859
|
listeners[:on_optional_parameter_node_enter]&.each { |listener| listener.on_optional_parameter_node_enter(node) }
|
@@ -1923,6 +1932,12 @@ module Prism
|
|
1923
1932
|
listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
|
1924
1933
|
end
|
1925
1934
|
|
1935
|
+
# Dispatch enter and leave events for RequiredKeywordParameterNode nodes.
|
1936
|
+
def visit_required_keyword_parameter_node(node)
|
1937
|
+
listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
|
1938
|
+
listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
|
1939
|
+
end
|
1940
|
+
|
1926
1941
|
# Dispatch enter and leave events for RequiredParameterNode nodes.
|
1927
1942
|
def visit_required_parameter_node(node)
|
1928
1943
|
listeners[:on_required_parameter_node_enter]&.each { |listener| listener.on_required_parameter_node_enter(node) }
|
data/lib/prism/dsl.rb
CHANGED
@@ -462,11 +462,6 @@ module Prism
|
|
462
462
|
KeywordHashNode.new(elements, location)
|
463
463
|
end
|
464
464
|
|
465
|
-
# Create a new KeywordParameterNode node
|
466
|
-
def KeywordParameterNode(name, name_loc, value, location = Location())
|
467
|
-
KeywordParameterNode.new(name, name_loc, value, location)
|
468
|
-
end
|
469
|
-
|
470
465
|
# Create a new KeywordRestParameterNode node
|
471
466
|
def KeywordRestParameterNode(name, name_loc, operator_loc, location = Location())
|
472
467
|
KeywordRestParameterNode.new(name, name_loc, operator_loc, location)
|
@@ -567,6 +562,11 @@ module Prism
|
|
567
562
|
NumberedReferenceReadNode.new(number, location)
|
568
563
|
end
|
569
564
|
|
565
|
+
# Create a new OptionalKeywordParameterNode node
|
566
|
+
def OptionalKeywordParameterNode(name, name_loc, value, location = Location())
|
567
|
+
OptionalKeywordParameterNode.new(name, name_loc, value, location)
|
568
|
+
end
|
569
|
+
|
570
570
|
# Create a new OptionalParameterNode node
|
571
571
|
def OptionalParameterNode(name, name_loc, operator_loc, value, location = Location())
|
572
572
|
OptionalParameterNode.new(name, name_loc, operator_loc, value, location)
|
@@ -632,6 +632,11 @@ module Prism
|
|
632
632
|
RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
|
633
633
|
end
|
634
634
|
|
635
|
+
# Create a new RequiredKeywordParameterNode node
|
636
|
+
def RequiredKeywordParameterNode(name, name_loc, location = Location())
|
637
|
+
RequiredKeywordParameterNode.new(name, name_loc, location)
|
638
|
+
end
|
639
|
+
|
635
640
|
# Create a new RequiredParameterNode node
|
636
641
|
def RequiredParameterNode(name, location = Location())
|
637
642
|
RequiredParameterNode.new(name, location)
|
data/lib/prism/ffi.rb
CHANGED
@@ -9,7 +9,7 @@ require "ffi"
|
|
9
9
|
module Prism
|
10
10
|
BACKEND = :FFI
|
11
11
|
|
12
|
-
module LibRubyParser
|
12
|
+
module LibRubyParser # :nodoc:
|
13
13
|
extend FFI::Library
|
14
14
|
|
15
15
|
# Define the library that we will be pulling functions from. Note that this
|
@@ -69,10 +69,10 @@ module Prism
|
|
69
69
|
load_exported_functions_from(
|
70
70
|
"prism.h",
|
71
71
|
"pm_version",
|
72
|
-
"
|
73
|
-
"
|
74
|
-
"
|
75
|
-
"
|
72
|
+
"pm_serialize_parse",
|
73
|
+
"pm_serialize_parse_comments",
|
74
|
+
"pm_serialize_lex",
|
75
|
+
"pm_serialize_parse_lex"
|
76
76
|
)
|
77
77
|
|
78
78
|
load_exported_functions_from(
|
@@ -95,7 +95,7 @@ module Prism
|
|
95
95
|
|
96
96
|
# This object represents a pm_buffer_t. We only use it as an opaque pointer,
|
97
97
|
# so it doesn't need to know the fields of pm_buffer_t.
|
98
|
-
class PrismBuffer
|
98
|
+
class PrismBuffer # :nodoc:
|
99
99
|
SIZEOF = LibRubyParser.pm_buffer_sizeof
|
100
100
|
|
101
101
|
attr_reader :pointer
|
@@ -133,7 +133,7 @@ module Prism
|
|
133
133
|
|
134
134
|
# This object represents a pm_string_t. We only use it as an opaque pointer,
|
135
135
|
# so it doesn't have to be an FFI::Struct.
|
136
|
-
class PrismString
|
136
|
+
class PrismString # :nodoc:
|
137
137
|
SIZEOF = LibRubyParser.pm_string_sizeof
|
138
138
|
|
139
139
|
attr_reader :pointer
|
@@ -167,14 +167,6 @@ module Prism
|
|
167
167
|
end
|
168
168
|
end
|
169
169
|
end
|
170
|
-
|
171
|
-
def self.dump_internal(source, source_size, filepath)
|
172
|
-
PrismBuffer.with do |buffer|
|
173
|
-
metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
|
174
|
-
pm_parse_serialize(source, source_size, buffer.pointer, metadata)
|
175
|
-
buffer.read
|
176
|
-
end
|
177
|
-
end
|
178
170
|
end
|
179
171
|
|
180
172
|
# Mark the LibRubyParser module as private as it should only be called through
|
@@ -184,93 +176,153 @@ module Prism
|
|
184
176
|
# The version constant is set by reading the result of calling pm_version.
|
185
177
|
VERSION = LibRubyParser.pm_version.read_string
|
186
178
|
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
179
|
+
class << self
|
180
|
+
# Mirror the Prism.dump API by using the serialization API.
|
181
|
+
def dump(code, **options)
|
182
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
183
|
+
LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
|
184
|
+
buffer.read
|
185
|
+
end
|
186
|
+
end
|
191
187
|
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
188
|
+
# Mirror the Prism.dump_file API by using the serialization API.
|
189
|
+
def dump_file(filepath, **options)
|
190
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
191
|
+
dump(string.read, **options, filepath: filepath)
|
192
|
+
end
|
196
193
|
end
|
197
|
-
end
|
198
194
|
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
195
|
+
# Mirror the Prism.lex API by using the serialization API.
|
196
|
+
def lex(code, **options)
|
197
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
198
|
+
LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
|
199
|
+
Serialize.load_tokens(Source.new(code), buffer.read)
|
200
|
+
end
|
204
201
|
end
|
205
|
-
end
|
206
202
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
203
|
+
# Mirror the Prism.lex_file API by using the serialization API.
|
204
|
+
def lex_file(filepath, **options)
|
205
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
206
|
+
lex(string.read, **options, filepath: filepath)
|
207
|
+
end
|
211
208
|
end
|
212
|
-
end
|
213
209
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
210
|
+
# Mirror the Prism.parse API by using the serialization API.
|
211
|
+
def parse(code, **options)
|
212
|
+
Prism.load(code, dump(code, **options))
|
213
|
+
end
|
218
214
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
215
|
+
# Mirror the Prism.parse_file API by using the serialization API. This uses
|
216
|
+
# native strings instead of Ruby strings because it allows us to use mmap when
|
217
|
+
# it is available.
|
218
|
+
def parse_file(filepath, **options)
|
219
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
220
|
+
parse(string.read, **options, filepath: filepath)
|
221
|
+
end
|
225
222
|
end
|
226
|
-
end
|
227
223
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata)
|
224
|
+
# Mirror the Prism.parse_comments API by using the serialization API.
|
225
|
+
def parse_comments(code, **options)
|
226
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
227
|
+
LibRubyParser.pm_serialize_parse_comments(buffer.pointer, code, code.bytesize, dump_options(options))
|
233
228
|
|
234
|
-
|
235
|
-
|
229
|
+
source = Source.new(code)
|
230
|
+
loader = Serialize::Loader.new(source, buffer.read)
|
236
231
|
|
237
|
-
|
238
|
-
|
239
|
-
|
232
|
+
loader.load_header
|
233
|
+
loader.load_force_encoding
|
234
|
+
loader.load_start_line
|
235
|
+
loader.load_comments
|
236
|
+
end
|
240
237
|
end
|
241
|
-
end
|
242
238
|
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
239
|
+
# Mirror the Prism.parse_file_comments API by using the serialization
|
240
|
+
# API. This uses native strings instead of Ruby strings because it allows us
|
241
|
+
# to use mmap when it is available.
|
242
|
+
def parse_file_comments(filepath, **options)
|
243
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
244
|
+
parse_comments(string.read, **options, filepath: filepath)
|
245
|
+
end
|
249
246
|
end
|
250
|
-
end
|
251
247
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
|
248
|
+
# Mirror the Prism.parse_lex API by using the serialization API.
|
249
|
+
def parse_lex(code, **options)
|
250
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
251
|
+
LibRubyParser.pm_serialize_parse_lex(buffer.pointer, code, code.bytesize, dump_options(options))
|
257
252
|
|
258
|
-
|
259
|
-
|
253
|
+
source = Source.new(code)
|
254
|
+
loader = Serialize::Loader.new(source, buffer.read)
|
260
255
|
|
261
|
-
|
262
|
-
|
256
|
+
tokens = loader.load_tokens
|
257
|
+
node, comments, magic_comments, errors, warnings = loader.load_nodes
|
258
|
+
tokens.each { |token,| token.value.force_encoding(loader.encoding) }
|
263
259
|
|
264
|
-
|
260
|
+
ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
|
261
|
+
end
|
262
|
+
end
|
265
263
|
|
266
|
-
|
264
|
+
# Mirror the Prism.parse_lex_file API by using the serialization API.
|
265
|
+
def parse_lex_file(filepath, **options)
|
266
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
267
|
+
parse_lex(string.read, **options, filepath: filepath)
|
268
|
+
end
|
267
269
|
end
|
268
|
-
end
|
269
270
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
271
|
+
private
|
272
|
+
|
273
|
+
# Convert the given options into a serialized options string.
|
274
|
+
def dump_options(options)
|
275
|
+
template = +""
|
276
|
+
values = []
|
277
|
+
|
278
|
+
template << "L"
|
279
|
+
if (filepath = options[:filepath])
|
280
|
+
values.push(filepath.bytesize, filepath.b)
|
281
|
+
template << "A*"
|
282
|
+
else
|
283
|
+
values << 0
|
284
|
+
end
|
285
|
+
|
286
|
+
template << "L"
|
287
|
+
values << options.fetch(:line, 1)
|
288
|
+
|
289
|
+
template << "L"
|
290
|
+
if (encoding = options[:encoding])
|
291
|
+
name = encoding.name
|
292
|
+
values.push(name.bytesize, name.b)
|
293
|
+
template << "A*"
|
294
|
+
else
|
295
|
+
values << 0
|
296
|
+
end
|
297
|
+
|
298
|
+
template << "C"
|
299
|
+
values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
|
300
|
+
|
301
|
+
template << "C"
|
302
|
+
values << (options[:verbose] ? 0 : 1)
|
303
|
+
|
304
|
+
template << "L"
|
305
|
+
if (scopes = options[:scopes])
|
306
|
+
values << scopes.length
|
307
|
+
|
308
|
+
scopes.each do |scope|
|
309
|
+
template << "L"
|
310
|
+
values << scope.length
|
311
|
+
|
312
|
+
scope.each do |local|
|
313
|
+
name = local.name
|
314
|
+
template << "L"
|
315
|
+
values << name.bytesize
|
316
|
+
|
317
|
+
template << "A*"
|
318
|
+
values << name.b
|
319
|
+
end
|
320
|
+
end
|
321
|
+
else
|
322
|
+
values << 0
|
323
|
+
end
|
324
|
+
|
325
|
+
values.pack(template)
|
274
326
|
end
|
275
327
|
end
|
276
328
|
end
|
data/lib/prism/lex_compat.rb
CHANGED
@@ -8,7 +8,7 @@ module Prism
|
|
8
8
|
# of cases, this is a one-to-one mapping of the token type. Everything else
|
9
9
|
# generally lines up. However, there are a few cases that require special
|
10
10
|
# handling.
|
11
|
-
class LexCompat
|
11
|
+
class LexCompat # :nodoc:
|
12
12
|
# This is a mapping of prism token types to Ripper token types. This is a
|
13
13
|
# many-to-one mapping because we split up our token types, whereas Ripper
|
14
14
|
# tends to group them.
|
@@ -184,18 +184,22 @@ module Prism
|
|
184
184
|
# However, we add a couple of convenience methods onto them to make them a
|
185
185
|
# little easier to work with. We delegate all other methods to the array.
|
186
186
|
class Token < SimpleDelegator
|
187
|
+
# The location of the token in the source.
|
187
188
|
def location
|
188
189
|
self[0]
|
189
190
|
end
|
190
191
|
|
192
|
+
# The type of the token.
|
191
193
|
def event
|
192
194
|
self[1]
|
193
195
|
end
|
194
196
|
|
197
|
+
# The slice of the source that this token represents.
|
195
198
|
def value
|
196
199
|
self[2]
|
197
200
|
end
|
198
201
|
|
202
|
+
# The state of the lexer when this token was produced.
|
199
203
|
def state
|
200
204
|
self[3]
|
201
205
|
end
|
@@ -204,7 +208,7 @@ module Prism
|
|
204
208
|
# Ripper doesn't include the rest of the token in the event, so we need to
|
205
209
|
# trim it down to just the content on the first line when comparing.
|
206
210
|
class EndContentToken < Token
|
207
|
-
def ==(other)
|
211
|
+
def ==(other) # :nodoc:
|
208
212
|
[self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other
|
209
213
|
end
|
210
214
|
end
|
@@ -212,7 +216,7 @@ module Prism
|
|
212
216
|
# Tokens where state should be ignored
|
213
217
|
# used for :on_comment, :on_heredoc_end, :on_embexpr_end
|
214
218
|
class IgnoreStateToken < Token
|
215
|
-
def ==(other)
|
219
|
+
def ==(other) # :nodoc:
|
216
220
|
self[0...-1] == other[0...-1]
|
217
221
|
end
|
218
222
|
end
|
@@ -222,7 +226,7 @@ module Prism
|
|
222
226
|
# through named captures in regular expressions). In that case we don't
|
223
227
|
# compare the state.
|
224
228
|
class IdentToken < Token
|
225
|
-
def ==(other)
|
229
|
+
def ==(other) # :nodoc:
|
226
230
|
(self[0...-1] == other[0...-1]) && (
|
227
231
|
(other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) ||
|
228
232
|
(other[3] & Ripper::EXPR_ARG_ANY != 0)
|
@@ -233,7 +237,7 @@ module Prism
|
|
233
237
|
# Ignored newlines can occasionally have a LABEL state attached to them, so
|
234
238
|
# we compare the state differently here.
|
235
239
|
class IgnoredNewlineToken < Token
|
236
|
-
def ==(other)
|
240
|
+
def ==(other) # :nodoc:
|
237
241
|
return false unless self[0...-1] == other[0...-1]
|
238
242
|
|
239
243
|
if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
|
@@ -253,7 +257,7 @@ module Prism
|
|
253
257
|
# more accurately, so we need to allow comparing against both END and
|
254
258
|
# END|LABEL.
|
255
259
|
class ParamToken < Token
|
256
|
-
def ==(other)
|
260
|
+
def ==(other) # :nodoc:
|
257
261
|
(self[0...-1] == other[0...-1]) && (
|
258
262
|
(other[3] == Ripper::EXPR_END) ||
|
259
263
|
(other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
|
@@ -264,12 +268,12 @@ module Prism
|
|
264
268
|
# A heredoc in this case is a list of tokens that belong to the body of the
|
265
269
|
# heredoc that should be appended onto the list of tokens when the heredoc
|
266
270
|
# closes.
|
267
|
-
module Heredoc
|
271
|
+
module Heredoc # :nodoc:
|
268
272
|
# Heredocs that are no dash or tilde heredocs are just a list of tokens.
|
269
273
|
# We need to keep them around so that we can insert them in the correct
|
270
274
|
# order back into the token stream and set the state of the last token to
|
271
275
|
# the state that the heredoc was opened in.
|
272
|
-
class PlainHeredoc
|
276
|
+
class PlainHeredoc # :nodoc:
|
273
277
|
attr_reader :tokens
|
274
278
|
|
275
279
|
def initialize
|
@@ -288,7 +292,7 @@ module Prism
|
|
288
292
|
# Dash heredocs are a little more complicated. They are a list of tokens
|
289
293
|
# that need to be split on "\\\n" to mimic Ripper's behavior. We also need
|
290
294
|
# to keep track of the state that the heredoc was opened in.
|
291
|
-
class DashHeredoc
|
295
|
+
class DashHeredoc # :nodoc:
|
292
296
|
attr_reader :split, :tokens
|
293
297
|
|
294
298
|
def initialize(split)
|
@@ -347,7 +351,7 @@ module Prism
|
|
347
351
|
# insert them into the stream in the correct order. As such, we can do
|
348
352
|
# some extra manipulation on the tokens to make them match Ripper's
|
349
353
|
# output by mirroring the dedent logic that Ripper uses.
|
350
|
-
class DedentingHeredoc
|
354
|
+
class DedentingHeredoc # :nodoc:
|
351
355
|
TAB_WIDTH = 8
|
352
356
|
|
353
357
|
attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance
|
@@ -588,11 +592,13 @@ module Prism
|
|
588
592
|
end
|
589
593
|
end
|
590
594
|
|
591
|
-
|
595
|
+
private_constant :Heredoc
|
592
596
|
|
593
|
-
|
597
|
+
attr_reader :source, :options
|
598
|
+
|
599
|
+
def initialize(source, **options)
|
594
600
|
@source = source
|
595
|
-
@
|
601
|
+
@options = options
|
596
602
|
end
|
597
603
|
|
598
604
|
def result
|
@@ -601,7 +607,7 @@ module Prism
|
|
601
607
|
state = :default
|
602
608
|
heredoc_stack = [[]]
|
603
609
|
|
604
|
-
result = Prism.lex(source,
|
610
|
+
result = Prism.lex(source, **options)
|
605
611
|
result_value = result.value
|
606
612
|
previous_state = nil
|
607
613
|
|
@@ -829,9 +835,11 @@ module Prism
|
|
829
835
|
end
|
830
836
|
end
|
831
837
|
|
838
|
+
private_constant :LexCompat
|
839
|
+
|
832
840
|
# This is a class that wraps the Ripper lexer to produce almost exactly the
|
833
841
|
# same tokens.
|
834
|
-
class LexRipper
|
842
|
+
class LexRipper # :nodoc:
|
835
843
|
attr_reader :source
|
836
844
|
|
837
845
|
def initialize(source)
|
@@ -869,4 +877,6 @@ module Prism
|
|
869
877
|
results
|
870
878
|
end
|
871
879
|
end
|
880
|
+
|
881
|
+
private_constant :LexRipper
|
872
882
|
end
|
@@ -430,11 +430,6 @@ module Prism
|
|
430
430
|
node.copy(elements: visit_all(node.elements))
|
431
431
|
end
|
432
432
|
|
433
|
-
# Copy a KeywordParameterNode node
|
434
|
-
def visit_keyword_parameter_node(node)
|
435
|
-
node.copy(value: visit(node.value))
|
436
|
-
end
|
437
|
-
|
438
433
|
# Copy a KeywordRestParameterNode node
|
439
434
|
def visit_keyword_rest_parameter_node(node)
|
440
435
|
node.copy
|
@@ -535,6 +530,11 @@ module Prism
|
|
535
530
|
node.copy
|
536
531
|
end
|
537
532
|
|
533
|
+
# Copy a OptionalKeywordParameterNode node
|
534
|
+
def visit_optional_keyword_parameter_node(node)
|
535
|
+
node.copy(value: visit(node.value))
|
536
|
+
end
|
537
|
+
|
538
538
|
# Copy a OptionalParameterNode node
|
539
539
|
def visit_optional_parameter_node(node)
|
540
540
|
node.copy(value: visit(node.value))
|
@@ -600,6 +600,11 @@ module Prism
|
|
600
600
|
node.copy
|
601
601
|
end
|
602
602
|
|
603
|
+
# Copy a RequiredKeywordParameterNode node
|
604
|
+
def visit_required_keyword_parameter_node(node)
|
605
|
+
node.copy
|
606
|
+
end
|
607
|
+
|
603
608
|
# Copy a RequiredParameterNode node
|
604
609
|
def visit_required_parameter_node(node)
|
605
610
|
node.copy
|