prism 0.16.0 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -1
- data/Makefile +6 -0
- data/README.md +1 -1
- data/config.yml +50 -35
- data/docs/fuzzing.md +1 -1
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +802 -770
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +464 -162
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3173 -763
- data/include/prism/defines.h +32 -9
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +118 -28
- data/include/prism/node.h +38 -13
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -200
- data/include/prism/prettyprint.h +12 -1
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +94 -16
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +126 -32
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +70 -27
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +27 -2
- data/include/prism.h +224 -31
- data/lib/prism/compiler.rb +6 -3
- data/lib/prism/debug.rb +23 -7
- data/lib/prism/dispatcher.rb +33 -18
- data/lib/prism/dsl.rb +10 -5
- data/lib/prism/ffi.rb +132 -80
- data/lib/prism/lex_compat.rb +25 -15
- data/lib/prism/mutation_compiler.rb +10 -5
- data/lib/prism/node.rb +370 -135
- data/lib/prism/node_ext.rb +1 -1
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +150 -30
- data/lib/prism/pattern.rb +11 -0
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +86 -54
- data/lib/prism/visitor.rb +10 -3
- data/lib/prism.rb +20 -2
- data/prism.gemspec +4 -2
- data/rbi/prism.rbi +104 -60
- data/rbi/prism_static.rbi +16 -2
- data/sig/prism.rbs +72 -43
- data/sig/prism_static.rbs +14 -1
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +53 -8
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +334 -321
- data/src/options.c +170 -0
- data/src/prettyprint.c +74 -47
- data/src/prism.c +1642 -856
- data/src/regexp.c +151 -95
- data/src/serialize.c +44 -20
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +45 -15
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +51 -21
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +20 -12
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +5 -3
- data/docs/prism.png +0 -0
data/lib/prism/debug.rb
CHANGED
@@ -4,7 +4,9 @@ module Prism
|
|
4
4
|
# This module is used for testing and debugging and is not meant to be used by
|
5
5
|
# consumers of this library.
|
6
6
|
module Debug
|
7
|
-
|
7
|
+
# A wrapper around a RubyVM::InstructionSequence that provides a more
|
8
|
+
# convenient interface for accessing parts of the iseq.
|
9
|
+
class ISeq # :nodoc:
|
8
10
|
attr_reader :parts
|
9
11
|
|
10
12
|
def initialize(parts)
|
@@ -42,6 +44,11 @@ module Prism
|
|
42
44
|
end
|
43
45
|
end
|
44
46
|
|
47
|
+
private_constant :ISeq
|
48
|
+
|
49
|
+
# :call-seq:
|
50
|
+
# Debug::cruby_locals(source) -> Array
|
51
|
+
#
|
45
52
|
# For the given source, compiles with CRuby and returns a list of all of the
|
46
53
|
# sets of local variables that were encountered.
|
47
54
|
def self.cruby_locals(source)
|
@@ -76,8 +83,16 @@ module Prism
|
|
76
83
|
end
|
77
84
|
end
|
78
85
|
|
86
|
+
# Used to hold the place of a local that will be in the local table but
|
87
|
+
# cannot be accessed directly from the source code. For example, the
|
88
|
+
# iteration variable in a for loop or the positional parameter on a method
|
89
|
+
# definition that is destructured.
|
79
90
|
AnonymousLocal = Object.new
|
91
|
+
private_constant :AnonymousLocal
|
80
92
|
|
93
|
+
# :call-seq:
|
94
|
+
# Debug::prism_locals(source) -> Array
|
95
|
+
#
|
81
96
|
# For the given source, parses with prism and returns a list of all of the
|
82
97
|
# sets of local variables that were encountered.
|
83
98
|
def self.prism_locals(source)
|
@@ -114,8 +129,8 @@ module Prism
|
|
114
129
|
AnonymousLocal
|
115
130
|
end
|
116
131
|
end,
|
117
|
-
*params.keywords.
|
118
|
-
*params.keywords.
|
132
|
+
*params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
|
133
|
+
*params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
|
119
134
|
]
|
120
135
|
|
121
136
|
sorted << AnonymousLocal if params.keywords.any?
|
@@ -164,12 +179,13 @@ module Prism
|
|
164
179
|
locals
|
165
180
|
end
|
166
181
|
|
182
|
+
# :call-seq:
|
183
|
+
# Debug::newlines(source) -> Array
|
184
|
+
#
|
185
|
+
# For the given source string, return the byte offsets of every newline in
|
186
|
+
# the source.
|
167
187
|
def self.newlines(source)
|
168
188
|
Prism.parse(source).source.offsets
|
169
189
|
end
|
170
|
-
|
171
|
-
def self.parse_serialize_file(filepath)
|
172
|
-
parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
|
173
|
-
end
|
174
190
|
end
|
175
191
|
end
|
data/lib/prism/dispatcher.rb
CHANGED
@@ -41,23 +41,24 @@ module Prism
|
|
41
41
|
# attr_reader listeners: Hash[Symbol, Array[Listener]]
|
42
42
|
attr_reader :listeners
|
43
43
|
|
44
|
+
# Initialize a new dispatcher.
|
44
45
|
def initialize
|
45
46
|
@listeners = {}
|
46
47
|
end
|
47
48
|
|
48
|
-
# Register a listener for one or more events
|
49
|
+
# Register a listener for one or more events.
|
49
50
|
#
|
50
51
|
# def register: (Listener, *Symbol) -> void
|
51
52
|
def register(listener, *events)
|
52
53
|
events.each { |event| (listeners[event] ||= []) << listener }
|
53
54
|
end
|
54
55
|
|
55
|
-
# Walks `root` dispatching events to all registered listeners
|
56
|
+
# Walks `root` dispatching events to all registered listeners.
|
56
57
|
#
|
57
58
|
# def dispatch: (Node) -> void
|
58
59
|
alias dispatch visit
|
59
60
|
|
60
|
-
# Dispatches a single event for `node` to all registered listeners
|
61
|
+
# Dispatches a single event for `node` to all registered listeners.
|
61
62
|
#
|
62
63
|
# def dispatch_once: (Node) -> void
|
63
64
|
def dispatch_once(node)
|
@@ -736,14 +737,6 @@ module Prism
|
|
736
737
|
listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
|
737
738
|
end
|
738
739
|
|
739
|
-
# Dispatch enter and leave events for KeywordParameterNode nodes and continue
|
740
|
-
# walking the tree.
|
741
|
-
def visit_keyword_parameter_node(node)
|
742
|
-
listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
|
743
|
-
super
|
744
|
-
listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
|
745
|
-
end
|
746
|
-
|
747
740
|
# Dispatch enter and leave events for KeywordRestParameterNode nodes and continue
|
748
741
|
# walking the tree.
|
749
742
|
def visit_keyword_rest_parameter_node(node)
|
@@ -904,6 +897,14 @@ module Prism
|
|
904
897
|
listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
|
905
898
|
end
|
906
899
|
|
900
|
+
# Dispatch enter and leave events for OptionalKeywordParameterNode nodes and continue
|
901
|
+
# walking the tree.
|
902
|
+
def visit_optional_keyword_parameter_node(node)
|
903
|
+
listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
|
904
|
+
super
|
905
|
+
listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
|
906
|
+
end
|
907
|
+
|
907
908
|
# Dispatch enter and leave events for OptionalParameterNode nodes and continue
|
908
909
|
# walking the tree.
|
909
910
|
def visit_optional_parameter_node(node)
|
@@ -1008,6 +1009,14 @@ module Prism
|
|
1008
1009
|
listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
|
1009
1010
|
end
|
1010
1011
|
|
1012
|
+
# Dispatch enter and leave events for RequiredKeywordParameterNode nodes and continue
|
1013
|
+
# walking the tree.
|
1014
|
+
def visit_required_keyword_parameter_node(node)
|
1015
|
+
listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
|
1016
|
+
super
|
1017
|
+
listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
|
1018
|
+
end
|
1019
|
+
|
1011
1020
|
# Dispatch enter and leave events for RequiredParameterNode nodes and continue
|
1012
1021
|
# walking the tree.
|
1013
1022
|
def visit_required_parameter_node(node)
|
@@ -1208,7 +1217,7 @@ module Prism
|
|
1208
1217
|
listeners[:on_yield_node_leave]&.each { |listener| listener.on_yield_node_leave(node) }
|
1209
1218
|
end
|
1210
1219
|
|
1211
|
-
class DispatchOnce < Visitor
|
1220
|
+
class DispatchOnce < Visitor # :nodoc:
|
1212
1221
|
attr_reader :listeners
|
1213
1222
|
|
1214
1223
|
def initialize(listeners)
|
@@ -1719,12 +1728,6 @@ module Prism
|
|
1719
1728
|
listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
|
1720
1729
|
end
|
1721
1730
|
|
1722
|
-
# Dispatch enter and leave events for KeywordParameterNode nodes.
|
1723
|
-
def visit_keyword_parameter_node(node)
|
1724
|
-
listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
|
1725
|
-
listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
|
1726
|
-
end
|
1727
|
-
|
1728
1731
|
# Dispatch enter and leave events for KeywordRestParameterNode nodes.
|
1729
1732
|
def visit_keyword_rest_parameter_node(node)
|
1730
1733
|
listeners[:on_keyword_rest_parameter_node_enter]&.each { |listener| listener.on_keyword_rest_parameter_node_enter(node) }
|
@@ -1845,6 +1848,12 @@ module Prism
|
|
1845
1848
|
listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
|
1846
1849
|
end
|
1847
1850
|
|
1851
|
+
# Dispatch enter and leave events for OptionalKeywordParameterNode nodes.
|
1852
|
+
def visit_optional_keyword_parameter_node(node)
|
1853
|
+
listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
|
1854
|
+
listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
|
1855
|
+
end
|
1856
|
+
|
1848
1857
|
# Dispatch enter and leave events for OptionalParameterNode nodes.
|
1849
1858
|
def visit_optional_parameter_node(node)
|
1850
1859
|
listeners[:on_optional_parameter_node_enter]&.each { |listener| listener.on_optional_parameter_node_enter(node) }
|
@@ -1923,6 +1932,12 @@ module Prism
|
|
1923
1932
|
listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
|
1924
1933
|
end
|
1925
1934
|
|
1935
|
+
# Dispatch enter and leave events for RequiredKeywordParameterNode nodes.
|
1936
|
+
def visit_required_keyword_parameter_node(node)
|
1937
|
+
listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
|
1938
|
+
listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
|
1939
|
+
end
|
1940
|
+
|
1926
1941
|
# Dispatch enter and leave events for RequiredParameterNode nodes.
|
1927
1942
|
def visit_required_parameter_node(node)
|
1928
1943
|
listeners[:on_required_parameter_node_enter]&.each { |listener| listener.on_required_parameter_node_enter(node) }
|
data/lib/prism/dsl.rb
CHANGED
@@ -462,11 +462,6 @@ module Prism
|
|
462
462
|
KeywordHashNode.new(elements, location)
|
463
463
|
end
|
464
464
|
|
465
|
-
# Create a new KeywordParameterNode node
|
466
|
-
def KeywordParameterNode(name, name_loc, value, location = Location())
|
467
|
-
KeywordParameterNode.new(name, name_loc, value, location)
|
468
|
-
end
|
469
|
-
|
470
465
|
# Create a new KeywordRestParameterNode node
|
471
466
|
def KeywordRestParameterNode(name, name_loc, operator_loc, location = Location())
|
472
467
|
KeywordRestParameterNode.new(name, name_loc, operator_loc, location)
|
@@ -567,6 +562,11 @@ module Prism
|
|
567
562
|
NumberedReferenceReadNode.new(number, location)
|
568
563
|
end
|
569
564
|
|
565
|
+
# Create a new OptionalKeywordParameterNode node
|
566
|
+
def OptionalKeywordParameterNode(name, name_loc, value, location = Location())
|
567
|
+
OptionalKeywordParameterNode.new(name, name_loc, value, location)
|
568
|
+
end
|
569
|
+
|
570
570
|
# Create a new OptionalParameterNode node
|
571
571
|
def OptionalParameterNode(name, name_loc, operator_loc, value, location = Location())
|
572
572
|
OptionalParameterNode.new(name, name_loc, operator_loc, value, location)
|
@@ -632,6 +632,11 @@ module Prism
|
|
632
632
|
RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
|
633
633
|
end
|
634
634
|
|
635
|
+
# Create a new RequiredKeywordParameterNode node
|
636
|
+
def RequiredKeywordParameterNode(name, name_loc, location = Location())
|
637
|
+
RequiredKeywordParameterNode.new(name, name_loc, location)
|
638
|
+
end
|
639
|
+
|
635
640
|
# Create a new RequiredParameterNode node
|
636
641
|
def RequiredParameterNode(name, location = Location())
|
637
642
|
RequiredParameterNode.new(name, location)
|
data/lib/prism/ffi.rb
CHANGED
@@ -9,7 +9,7 @@ require "ffi"
|
|
9
9
|
module Prism
|
10
10
|
BACKEND = :FFI
|
11
11
|
|
12
|
-
module LibRubyParser
|
12
|
+
module LibRubyParser # :nodoc:
|
13
13
|
extend FFI::Library
|
14
14
|
|
15
15
|
# Define the library that we will be pulling functions from. Note that this
|
@@ -69,10 +69,10 @@ module Prism
|
|
69
69
|
load_exported_functions_from(
|
70
70
|
"prism.h",
|
71
71
|
"pm_version",
|
72
|
-
"
|
73
|
-
"
|
74
|
-
"
|
75
|
-
"
|
72
|
+
"pm_serialize_parse",
|
73
|
+
"pm_serialize_parse_comments",
|
74
|
+
"pm_serialize_lex",
|
75
|
+
"pm_serialize_parse_lex"
|
76
76
|
)
|
77
77
|
|
78
78
|
load_exported_functions_from(
|
@@ -95,7 +95,7 @@ module Prism
|
|
95
95
|
|
96
96
|
# This object represents a pm_buffer_t. We only use it as an opaque pointer,
|
97
97
|
# so it doesn't need to know the fields of pm_buffer_t.
|
98
|
-
class PrismBuffer
|
98
|
+
class PrismBuffer # :nodoc:
|
99
99
|
SIZEOF = LibRubyParser.pm_buffer_sizeof
|
100
100
|
|
101
101
|
attr_reader :pointer
|
@@ -133,7 +133,7 @@ module Prism
|
|
133
133
|
|
134
134
|
# This object represents a pm_string_t. We only use it as an opaque pointer,
|
135
135
|
# so it doesn't have to be an FFI::Struct.
|
136
|
-
class PrismString
|
136
|
+
class PrismString # :nodoc:
|
137
137
|
SIZEOF = LibRubyParser.pm_string_sizeof
|
138
138
|
|
139
139
|
attr_reader :pointer
|
@@ -167,14 +167,6 @@ module Prism
|
|
167
167
|
end
|
168
168
|
end
|
169
169
|
end
|
170
|
-
|
171
|
-
def self.dump_internal(source, source_size, filepath)
|
172
|
-
PrismBuffer.with do |buffer|
|
173
|
-
metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
|
174
|
-
pm_parse_serialize(source, source_size, buffer.pointer, metadata)
|
175
|
-
buffer.read
|
176
|
-
end
|
177
|
-
end
|
178
170
|
end
|
179
171
|
|
180
172
|
# Mark the LibRubyParser module as private as it should only be called through
|
@@ -184,93 +176,153 @@ module Prism
|
|
184
176
|
# The version constant is set by reading the result of calling pm_version.
|
185
177
|
VERSION = LibRubyParser.pm_version.read_string
|
186
178
|
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
179
|
+
class << self
|
180
|
+
# Mirror the Prism.dump API by using the serialization API.
|
181
|
+
def dump(code, **options)
|
182
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
183
|
+
LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
|
184
|
+
buffer.read
|
185
|
+
end
|
186
|
+
end
|
191
187
|
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
188
|
+
# Mirror the Prism.dump_file API by using the serialization API.
|
189
|
+
def dump_file(filepath, **options)
|
190
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
191
|
+
dump(string.read, **options, filepath: filepath)
|
192
|
+
end
|
196
193
|
end
|
197
|
-
end
|
198
194
|
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
195
|
+
# Mirror the Prism.lex API by using the serialization API.
|
196
|
+
def lex(code, **options)
|
197
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
198
|
+
LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
|
199
|
+
Serialize.load_tokens(Source.new(code), buffer.read)
|
200
|
+
end
|
204
201
|
end
|
205
|
-
end
|
206
202
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
203
|
+
# Mirror the Prism.lex_file API by using the serialization API.
|
204
|
+
def lex_file(filepath, **options)
|
205
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
206
|
+
lex(string.read, **options, filepath: filepath)
|
207
|
+
end
|
211
208
|
end
|
212
|
-
end
|
213
209
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
210
|
+
# Mirror the Prism.parse API by using the serialization API.
|
211
|
+
def parse(code, **options)
|
212
|
+
Prism.load(code, dump(code, **options))
|
213
|
+
end
|
218
214
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
215
|
+
# Mirror the Prism.parse_file API by using the serialization API. This uses
|
216
|
+
# native strings instead of Ruby strings because it allows us to use mmap when
|
217
|
+
# it is available.
|
218
|
+
def parse_file(filepath, **options)
|
219
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
220
|
+
parse(string.read, **options, filepath: filepath)
|
221
|
+
end
|
225
222
|
end
|
226
|
-
end
|
227
223
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata)
|
224
|
+
# Mirror the Prism.parse_comments API by using the serialization API.
|
225
|
+
def parse_comments(code, **options)
|
226
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
227
|
+
LibRubyParser.pm_serialize_parse_comments(buffer.pointer, code, code.bytesize, dump_options(options))
|
233
228
|
|
234
|
-
|
235
|
-
|
229
|
+
source = Source.new(code)
|
230
|
+
loader = Serialize::Loader.new(source, buffer.read)
|
236
231
|
|
237
|
-
|
238
|
-
|
239
|
-
|
232
|
+
loader.load_header
|
233
|
+
loader.load_force_encoding
|
234
|
+
loader.load_start_line
|
235
|
+
loader.load_comments
|
236
|
+
end
|
240
237
|
end
|
241
|
-
end
|
242
238
|
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
239
|
+
# Mirror the Prism.parse_file_comments API by using the serialization
|
240
|
+
# API. This uses native strings instead of Ruby strings because it allows us
|
241
|
+
# to use mmap when it is available.
|
242
|
+
def parse_file_comments(filepath, **options)
|
243
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
244
|
+
parse_comments(string.read, **options, filepath: filepath)
|
245
|
+
end
|
249
246
|
end
|
250
|
-
end
|
251
247
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
|
248
|
+
# Mirror the Prism.parse_lex API by using the serialization API.
|
249
|
+
def parse_lex(code, **options)
|
250
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
251
|
+
LibRubyParser.pm_serialize_parse_lex(buffer.pointer, code, code.bytesize, dump_options(options))
|
257
252
|
|
258
|
-
|
259
|
-
|
253
|
+
source = Source.new(code)
|
254
|
+
loader = Serialize::Loader.new(source, buffer.read)
|
260
255
|
|
261
|
-
|
262
|
-
|
256
|
+
tokens = loader.load_tokens
|
257
|
+
node, comments, magic_comments, errors, warnings = loader.load_nodes
|
258
|
+
tokens.each { |token,| token.value.force_encoding(loader.encoding) }
|
263
259
|
|
264
|
-
|
260
|
+
ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
|
261
|
+
end
|
262
|
+
end
|
265
263
|
|
266
|
-
|
264
|
+
# Mirror the Prism.parse_lex_file API by using the serialization API.
|
265
|
+
def parse_lex_file(filepath, **options)
|
266
|
+
LibRubyParser::PrismString.with(filepath) do |string|
|
267
|
+
parse_lex(string.read, **options, filepath: filepath)
|
268
|
+
end
|
267
269
|
end
|
268
|
-
end
|
269
270
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
271
|
+
private
|
272
|
+
|
273
|
+
# Convert the given options into a serialized options string.
|
274
|
+
def dump_options(options)
|
275
|
+
template = +""
|
276
|
+
values = []
|
277
|
+
|
278
|
+
template << "L"
|
279
|
+
if (filepath = options[:filepath])
|
280
|
+
values.push(filepath.bytesize, filepath.b)
|
281
|
+
template << "A*"
|
282
|
+
else
|
283
|
+
values << 0
|
284
|
+
end
|
285
|
+
|
286
|
+
template << "L"
|
287
|
+
values << options.fetch(:line, 1)
|
288
|
+
|
289
|
+
template << "L"
|
290
|
+
if (encoding = options[:encoding])
|
291
|
+
name = encoding.name
|
292
|
+
values.push(name.bytesize, name.b)
|
293
|
+
template << "A*"
|
294
|
+
else
|
295
|
+
values << 0
|
296
|
+
end
|
297
|
+
|
298
|
+
template << "C"
|
299
|
+
values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
|
300
|
+
|
301
|
+
template << "C"
|
302
|
+
values << (options[:verbose] ? 0 : 1)
|
303
|
+
|
304
|
+
template << "L"
|
305
|
+
if (scopes = options[:scopes])
|
306
|
+
values << scopes.length
|
307
|
+
|
308
|
+
scopes.each do |scope|
|
309
|
+
template << "L"
|
310
|
+
values << scope.length
|
311
|
+
|
312
|
+
scope.each do |local|
|
313
|
+
name = local.name
|
314
|
+
template << "L"
|
315
|
+
values << name.bytesize
|
316
|
+
|
317
|
+
template << "A*"
|
318
|
+
values << name.b
|
319
|
+
end
|
320
|
+
end
|
321
|
+
else
|
322
|
+
values << 0
|
323
|
+
end
|
324
|
+
|
325
|
+
values.pack(template)
|
274
326
|
end
|
275
327
|
end
|
276
328
|
end
|
data/lib/prism/lex_compat.rb
CHANGED
@@ -8,7 +8,7 @@ module Prism
|
|
8
8
|
# of cases, this is a one-to-one mapping of the token type. Everything else
|
9
9
|
# generally lines up. However, there are a few cases that require special
|
10
10
|
# handling.
|
11
|
-
class LexCompat
|
11
|
+
class LexCompat # :nodoc:
|
12
12
|
# This is a mapping of prism token types to Ripper token types. This is a
|
13
13
|
# many-to-one mapping because we split up our token types, whereas Ripper
|
14
14
|
# tends to group them.
|
@@ -184,18 +184,22 @@ module Prism
|
|
184
184
|
# However, we add a couple of convenience methods onto them to make them a
|
185
185
|
# little easier to work with. We delegate all other methods to the array.
|
186
186
|
class Token < SimpleDelegator
|
187
|
+
# The location of the token in the source.
|
187
188
|
def location
|
188
189
|
self[0]
|
189
190
|
end
|
190
191
|
|
192
|
+
# The type of the token.
|
191
193
|
def event
|
192
194
|
self[1]
|
193
195
|
end
|
194
196
|
|
197
|
+
# The slice of the source that this token represents.
|
195
198
|
def value
|
196
199
|
self[2]
|
197
200
|
end
|
198
201
|
|
202
|
+
# The state of the lexer when this token was produced.
|
199
203
|
def state
|
200
204
|
self[3]
|
201
205
|
end
|
@@ -204,7 +208,7 @@ module Prism
|
|
204
208
|
# Ripper doesn't include the rest of the token in the event, so we need to
|
205
209
|
# trim it down to just the content on the first line when comparing.
|
206
210
|
class EndContentToken < Token
|
207
|
-
def ==(other)
|
211
|
+
def ==(other) # :nodoc:
|
208
212
|
[self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other
|
209
213
|
end
|
210
214
|
end
|
@@ -212,7 +216,7 @@ module Prism
|
|
212
216
|
# Tokens where state should be ignored
|
213
217
|
# used for :on_comment, :on_heredoc_end, :on_embexpr_end
|
214
218
|
class IgnoreStateToken < Token
|
215
|
-
def ==(other)
|
219
|
+
def ==(other) # :nodoc:
|
216
220
|
self[0...-1] == other[0...-1]
|
217
221
|
end
|
218
222
|
end
|
@@ -222,7 +226,7 @@ module Prism
|
|
222
226
|
# through named captures in regular expressions). In that case we don't
|
223
227
|
# compare the state.
|
224
228
|
class IdentToken < Token
|
225
|
-
def ==(other)
|
229
|
+
def ==(other) # :nodoc:
|
226
230
|
(self[0...-1] == other[0...-1]) && (
|
227
231
|
(other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) ||
|
228
232
|
(other[3] & Ripper::EXPR_ARG_ANY != 0)
|
@@ -233,7 +237,7 @@ module Prism
|
|
233
237
|
# Ignored newlines can occasionally have a LABEL state attached to them, so
|
234
238
|
# we compare the state differently here.
|
235
239
|
class IgnoredNewlineToken < Token
|
236
|
-
def ==(other)
|
240
|
+
def ==(other) # :nodoc:
|
237
241
|
return false unless self[0...-1] == other[0...-1]
|
238
242
|
|
239
243
|
if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
|
@@ -253,7 +257,7 @@ module Prism
|
|
253
257
|
# more accurately, so we need to allow comparing against both END and
|
254
258
|
# END|LABEL.
|
255
259
|
class ParamToken < Token
|
256
|
-
def ==(other)
|
260
|
+
def ==(other) # :nodoc:
|
257
261
|
(self[0...-1] == other[0...-1]) && (
|
258
262
|
(other[3] == Ripper::EXPR_END) ||
|
259
263
|
(other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
|
@@ -264,12 +268,12 @@ module Prism
|
|
264
268
|
# A heredoc in this case is a list of tokens that belong to the body of the
|
265
269
|
# heredoc that should be appended onto the list of tokens when the heredoc
|
266
270
|
# closes.
|
267
|
-
module Heredoc
|
271
|
+
module Heredoc # :nodoc:
|
268
272
|
# Heredocs that are no dash or tilde heredocs are just a list of tokens.
|
269
273
|
# We need to keep them around so that we can insert them in the correct
|
270
274
|
# order back into the token stream and set the state of the last token to
|
271
275
|
# the state that the heredoc was opened in.
|
272
|
-
class PlainHeredoc
|
276
|
+
class PlainHeredoc # :nodoc:
|
273
277
|
attr_reader :tokens
|
274
278
|
|
275
279
|
def initialize
|
@@ -288,7 +292,7 @@ module Prism
|
|
288
292
|
# Dash heredocs are a little more complicated. They are a list of tokens
|
289
293
|
# that need to be split on "\\\n" to mimic Ripper's behavior. We also need
|
290
294
|
# to keep track of the state that the heredoc was opened in.
|
291
|
-
class DashHeredoc
|
295
|
+
class DashHeredoc # :nodoc:
|
292
296
|
attr_reader :split, :tokens
|
293
297
|
|
294
298
|
def initialize(split)
|
@@ -347,7 +351,7 @@ module Prism
|
|
347
351
|
# insert them into the stream in the correct order. As such, we can do
|
348
352
|
# some extra manipulation on the tokens to make them match Ripper's
|
349
353
|
# output by mirroring the dedent logic that Ripper uses.
|
350
|
-
class DedentingHeredoc
|
354
|
+
class DedentingHeredoc # :nodoc:
|
351
355
|
TAB_WIDTH = 8
|
352
356
|
|
353
357
|
attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance
|
@@ -588,11 +592,13 @@ module Prism
|
|
588
592
|
end
|
589
593
|
end
|
590
594
|
|
591
|
-
|
595
|
+
private_constant :Heredoc
|
592
596
|
|
593
|
-
|
597
|
+
attr_reader :source, :options
|
598
|
+
|
599
|
+
def initialize(source, **options)
|
594
600
|
@source = source
|
595
|
-
@
|
601
|
+
@options = options
|
596
602
|
end
|
597
603
|
|
598
604
|
def result
|
@@ -601,7 +607,7 @@ module Prism
|
|
601
607
|
state = :default
|
602
608
|
heredoc_stack = [[]]
|
603
609
|
|
604
|
-
result = Prism.lex(source,
|
610
|
+
result = Prism.lex(source, **options)
|
605
611
|
result_value = result.value
|
606
612
|
previous_state = nil
|
607
613
|
|
@@ -829,9 +835,11 @@ module Prism
|
|
829
835
|
end
|
830
836
|
end
|
831
837
|
|
838
|
+
private_constant :LexCompat
|
839
|
+
|
832
840
|
# This is a class that wraps the Ripper lexer to produce almost exactly the
|
833
841
|
# same tokens.
|
834
|
-
class LexRipper
|
842
|
+
class LexRipper # :nodoc:
|
835
843
|
attr_reader :source
|
836
844
|
|
837
845
|
def initialize(source)
|
@@ -869,4 +877,6 @@ module Prism
|
|
869
877
|
results
|
870
878
|
end
|
871
879
|
end
|
880
|
+
|
881
|
+
private_constant :LexRipper
|
872
882
|
end
|
@@ -430,11 +430,6 @@ module Prism
|
|
430
430
|
node.copy(elements: visit_all(node.elements))
|
431
431
|
end
|
432
432
|
|
433
|
-
# Copy a KeywordParameterNode node
|
434
|
-
def visit_keyword_parameter_node(node)
|
435
|
-
node.copy(value: visit(node.value))
|
436
|
-
end
|
437
|
-
|
438
433
|
# Copy a KeywordRestParameterNode node
|
439
434
|
def visit_keyword_rest_parameter_node(node)
|
440
435
|
node.copy
|
@@ -535,6 +530,11 @@ module Prism
|
|
535
530
|
node.copy
|
536
531
|
end
|
537
532
|
|
533
|
+
# Copy a OptionalKeywordParameterNode node
|
534
|
+
def visit_optional_keyword_parameter_node(node)
|
535
|
+
node.copy(value: visit(node.value))
|
536
|
+
end
|
537
|
+
|
538
538
|
# Copy a OptionalParameterNode node
|
539
539
|
def visit_optional_parameter_node(node)
|
540
540
|
node.copy(value: visit(node.value))
|
@@ -600,6 +600,11 @@ module Prism
|
|
600
600
|
node.copy
|
601
601
|
end
|
602
602
|
|
603
|
+
# Copy a RequiredKeywordParameterNode node
|
604
|
+
def visit_required_keyword_parameter_node(node)
|
605
|
+
node.copy
|
606
|
+
end
|
607
|
+
|
603
608
|
# Copy a RequiredParameterNode node
|
604
609
|
def visit_required_parameter_node(node)
|
605
610
|
node.copy
|