prism 0.19.0 → 0.24.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +102 -1
- data/Makefile +5 -0
- data/README.md +9 -6
- data/config.yml +236 -38
- data/docs/build_system.md +19 -2
- data/docs/cruby_compilation.md +27 -0
- data/docs/parser_translation.md +34 -0
- data/docs/parsing_rules.md +19 -0
- data/docs/releasing.md +84 -16
- data/docs/ruby_api.md +1 -1
- data/docs/ruby_parser_translation.md +19 -0
- data/docs/serialization.md +19 -5
- data/ext/prism/api_node.c +1989 -1525
- data/ext/prism/extension.c +130 -30
- data/ext/prism/extension.h +2 -2
- data/include/prism/ast.h +1700 -505
- data/include/prism/defines.h +8 -0
- data/include/prism/diagnostic.h +49 -7
- data/include/prism/encoding.h +17 -0
- data/include/prism/options.h +40 -14
- data/include/prism/parser.h +34 -18
- data/include/prism/util/pm_buffer.h +9 -0
- data/include/prism/util/pm_constant_pool.h +18 -0
- data/include/prism/util/pm_newline_list.h +4 -14
- data/include/prism/util/pm_strpbrk.h +4 -1
- data/include/prism/version.h +2 -2
- data/include/prism.h +19 -2
- data/lib/prism/debug.rb +11 -5
- data/lib/prism/desugar_compiler.rb +225 -80
- data/lib/prism/dot_visitor.rb +36 -14
- data/lib/prism/dsl.rb +302 -299
- data/lib/prism/ffi.rb +107 -76
- data/lib/prism/lex_compat.rb +17 -1
- data/lib/prism/node.rb +4580 -2607
- data/lib/prism/node_ext.rb +27 -4
- data/lib/prism/parse_result.rb +75 -29
- data/lib/prism/serialize.rb +633 -305
- data/lib/prism/translation/parser/compiler.rb +1838 -0
- data/lib/prism/translation/parser/lexer.rb +335 -0
- data/lib/prism/translation/parser/rubocop.rb +45 -0
- data/lib/prism/translation/parser.rb +190 -0
- data/lib/prism/translation/parser33.rb +12 -0
- data/lib/prism/translation/parser34.rb +12 -0
- data/lib/prism/translation/ripper.rb +696 -0
- data/lib/prism/translation/ruby_parser.rb +1521 -0
- data/lib/prism/translation.rb +11 -0
- data/lib/prism.rb +1 -1
- data/prism.gemspec +18 -7
- data/rbi/prism.rbi +150 -88
- data/rbi/prism_static.rbi +15 -3
- data/sig/prism.rbs +996 -961
- data/sig/prism_static.rbs +123 -46
- data/src/diagnostic.c +264 -219
- data/src/encoding.c +21 -26
- data/src/node.c +2 -6
- data/src/options.c +29 -5
- data/src/prettyprint.c +176 -44
- data/src/prism.c +1499 -564
- data/src/serialize.c +35 -21
- data/src/token_type.c +353 -4
- data/src/util/pm_buffer.c +11 -0
- data/src/util/pm_constant_pool.c +37 -11
- data/src/util/pm_newline_list.c +6 -15
- data/src/util/pm_string.c +0 -7
- data/src/util/pm_strpbrk.c +122 -14
- metadata +16 -5
- data/docs/building.md +0 -29
- data/lib/prism/ripper_compat.rb +0 -207
data/lib/prism/ffi.rb
CHANGED
@@ -119,15 +119,12 @@ module Prism
|
|
119
119
|
|
120
120
|
# Initialize a new buffer and yield it to the block. The buffer will be
|
121
121
|
# automatically freed when the block returns.
|
122
|
-
def self.with
|
123
|
-
|
124
|
-
|
125
|
-
begin
|
122
|
+
def self.with
|
123
|
+
FFI::MemoryPointer.new(SIZEOF) do |pointer|
|
126
124
|
raise unless LibRubyParser.pm_buffer_init(pointer)
|
127
|
-
yield new(pointer)
|
125
|
+
return yield new(pointer)
|
128
126
|
ensure
|
129
127
|
LibRubyParser.pm_buffer_free(pointer)
|
130
|
-
pointer.free
|
131
128
|
end
|
132
129
|
end
|
133
130
|
end
|
@@ -137,34 +134,47 @@ module Prism
|
|
137
134
|
class PrismString # :nodoc:
|
138
135
|
SIZEOF = LibRubyParser.pm_string_sizeof
|
139
136
|
|
140
|
-
attr_reader :pointer
|
137
|
+
attr_reader :pointer, :length
|
141
138
|
|
142
|
-
def initialize(pointer)
|
139
|
+
def initialize(pointer, length, from_string)
|
143
140
|
@pointer = pointer
|
144
|
-
|
145
|
-
|
146
|
-
def source
|
147
|
-
LibRubyParser.pm_string_source(pointer)
|
148
|
-
end
|
149
|
-
|
150
|
-
def length
|
151
|
-
LibRubyParser.pm_string_length(pointer)
|
141
|
+
@length = length
|
142
|
+
@from_string = from_string
|
152
143
|
end
|
153
144
|
|
154
145
|
def read
|
155
|
-
|
146
|
+
raise "should use the original String instead" if @from_string
|
147
|
+
@pointer.read_string(@length)
|
156
148
|
end
|
157
149
|
|
158
150
|
# Yields a pm_string_t pointer to the given block.
|
159
|
-
def self.
|
160
|
-
|
151
|
+
def self.with_string(string)
|
152
|
+
raise TypeError unless string.is_a?(String)
|
153
|
+
|
154
|
+
length = string.bytesize
|
155
|
+
# + 1 to never get an address of 0, which pm_parser_init() asserts
|
156
|
+
FFI::MemoryPointer.new(:char, length + 1, false) do |pointer|
|
157
|
+
pointer.write_string(string)
|
158
|
+
# since we have the extra byte we might as well \0-terminate
|
159
|
+
pointer.put_char(length, 0)
|
160
|
+
return yield new(pointer, length, true)
|
161
|
+
end
|
162
|
+
end
|
161
163
|
|
162
|
-
|
163
|
-
|
164
|
-
|
164
|
+
# Yields a pm_string_t pointer to the given block.
|
165
|
+
def self.with_file(filepath)
|
166
|
+
raise TypeError unless filepath.is_a?(String)
|
167
|
+
|
168
|
+
FFI::MemoryPointer.new(SIZEOF) do |pm_string|
|
169
|
+
if LibRubyParser.pm_string_mapped_init(pm_string, filepath)
|
170
|
+
pointer = LibRubyParser.pm_string_source(pm_string)
|
171
|
+
length = LibRubyParser.pm_string_length(pm_string)
|
172
|
+
return yield new(pointer, length, false)
|
173
|
+
else
|
174
|
+
raise SystemCallError.new(filepath, FFI.errno)
|
175
|
+
end
|
165
176
|
ensure
|
166
|
-
LibRubyParser.pm_string_free(
|
167
|
-
pointer.free
|
177
|
+
LibRubyParser.pm_string_free(pm_string)
|
168
178
|
end
|
169
179
|
end
|
170
180
|
end
|
@@ -180,52 +190,100 @@ module Prism
|
|
180
190
|
class << self
|
181
191
|
# Mirror the Prism.dump API by using the serialization API.
|
182
192
|
def dump(code, **options)
|
183
|
-
LibRubyParser::
|
184
|
-
LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
|
185
|
-
buffer.read
|
186
|
-
end
|
193
|
+
LibRubyParser::PrismString.with_string(code) { |string| dump_common(string, options) }
|
187
194
|
end
|
188
195
|
|
189
196
|
# Mirror the Prism.dump_file API by using the serialization API.
|
190
197
|
def dump_file(filepath, **options)
|
191
|
-
|
192
|
-
|
193
|
-
end
|
198
|
+
options[:filepath] = filepath
|
199
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| dump_common(string, options) }
|
194
200
|
end
|
195
201
|
|
196
202
|
# Mirror the Prism.lex API by using the serialization API.
|
197
203
|
def lex(code, **options)
|
198
|
-
LibRubyParser::
|
199
|
-
LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
|
200
|
-
Serialize.load_tokens(Source.new(code), buffer.read)
|
201
|
-
end
|
204
|
+
LibRubyParser::PrismString.with_string(code) { |string| lex_common(string, code, options) }
|
202
205
|
end
|
203
206
|
|
204
207
|
# Mirror the Prism.lex_file API by using the serialization API.
|
205
208
|
def lex_file(filepath, **options)
|
206
|
-
|
207
|
-
|
208
|
-
end
|
209
|
+
options[:filepath] = filepath
|
210
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| lex_common(string, string.read, options) }
|
209
211
|
end
|
210
212
|
|
211
213
|
# Mirror the Prism.parse API by using the serialization API.
|
212
214
|
def parse(code, **options)
|
213
|
-
|
215
|
+
LibRubyParser::PrismString.with_string(code) { |string| parse_common(string, code, options) }
|
214
216
|
end
|
215
217
|
|
216
218
|
# Mirror the Prism.parse_file API by using the serialization API. This uses
|
217
219
|
# native strings instead of Ruby strings because it allows us to use mmap when
|
218
220
|
# it is available.
|
219
221
|
def parse_file(filepath, **options)
|
220
|
-
|
221
|
-
|
222
|
-
end
|
222
|
+
options[:filepath] = filepath
|
223
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_common(string, string.read, options) }
|
223
224
|
end
|
224
225
|
|
225
226
|
# Mirror the Prism.parse_comments API by using the serialization API.
|
226
227
|
def parse_comments(code, **options)
|
228
|
+
LibRubyParser::PrismString.with_string(code) { |string| parse_comments_common(string, code, options) }
|
229
|
+
end
|
230
|
+
|
231
|
+
# Mirror the Prism.parse_file_comments API by using the serialization
|
232
|
+
# API. This uses native strings instead of Ruby strings because it allows us
|
233
|
+
# to use mmap when it is available.
|
234
|
+
def parse_file_comments(filepath, **options)
|
235
|
+
options[:filepath] = filepath
|
236
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_comments_common(string, string.read, options) }
|
237
|
+
end
|
238
|
+
|
239
|
+
# Mirror the Prism.parse_lex API by using the serialization API.
|
240
|
+
def parse_lex(code, **options)
|
241
|
+
LibRubyParser::PrismString.with_string(code) { |string| parse_lex_common(string, code, options) }
|
242
|
+
end
|
243
|
+
|
244
|
+
# Mirror the Prism.parse_lex_file API by using the serialization API.
|
245
|
+
def parse_lex_file(filepath, **options)
|
246
|
+
options[:filepath] = filepath
|
247
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_lex_common(string, string.read, options) }
|
248
|
+
end
|
249
|
+
|
250
|
+
# Mirror the Prism.parse_success? API by using the serialization API.
|
251
|
+
def parse_success?(code, **options)
|
252
|
+
LibRubyParser::PrismString.with_string(code) { |string| parse_file_success_common(string, options) }
|
253
|
+
end
|
254
|
+
|
255
|
+
# Mirror the Prism.parse_file_success? API by using the serialization API.
|
256
|
+
def parse_file_success?(filepath, **options)
|
257
|
+
options[:filepath] = filepath
|
258
|
+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_file_success_common(string, options) }
|
259
|
+
end
|
260
|
+
|
261
|
+
private
|
262
|
+
|
263
|
+
def dump_common(string, options) # :nodoc:
|
227
264
|
LibRubyParser::PrismBuffer.with do |buffer|
|
228
|
-
LibRubyParser.
|
265
|
+
LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
|
266
|
+
buffer.read
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
def lex_common(string, code, options) # :nodoc:
|
271
|
+
serialized = LibRubyParser::PrismBuffer.with do |buffer|
|
272
|
+
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
|
273
|
+
buffer.read
|
274
|
+
end
|
275
|
+
|
276
|
+
Serialize.load_tokens(Source.new(code), serialized)
|
277
|
+
end
|
278
|
+
|
279
|
+
def parse_common(string, code, options) # :nodoc:
|
280
|
+
serialized = dump_common(string, options)
|
281
|
+
Prism.load(code, serialized)
|
282
|
+
end
|
283
|
+
|
284
|
+
def parse_comments_common(string, code, options) # :nodoc:
|
285
|
+
LibRubyParser::PrismBuffer.with do |buffer|
|
286
|
+
LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
|
229
287
|
|
230
288
|
source = Source.new(code)
|
231
289
|
loader = Serialize::Loader.new(source, buffer.read)
|
@@ -237,19 +295,9 @@ module Prism
|
|
237
295
|
end
|
238
296
|
end
|
239
297
|
|
240
|
-
|
241
|
-
# API. This uses native strings instead of Ruby strings because it allows us
|
242
|
-
# to use mmap when it is available.
|
243
|
-
def parse_file_comments(filepath, **options)
|
244
|
-
LibRubyParser::PrismString.with(filepath) do |string|
|
245
|
-
parse_comments(string.read, **options, filepath: filepath)
|
246
|
-
end
|
247
|
-
end
|
248
|
-
|
249
|
-
# Mirror the Prism.parse_lex API by using the serialization API.
|
250
|
-
def parse_lex(code, **options)
|
298
|
+
def parse_lex_common(string, code, options) # :nodoc:
|
251
299
|
LibRubyParser::PrismBuffer.with do |buffer|
|
252
|
-
LibRubyParser.pm_serialize_parse_lex(buffer.pointer,
|
300
|
+
LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
|
253
301
|
|
254
302
|
source = Source.new(code)
|
255
303
|
loader = Serialize::Loader.new(source, buffer.read)
|
@@ -262,27 +310,10 @@ module Prism
|
|
262
310
|
end
|
263
311
|
end
|
264
312
|
|
265
|
-
|
266
|
-
|
267
|
-
LibRubyParser::PrismString.with(filepath) do |string|
|
268
|
-
parse_lex(string.read, **options, filepath: filepath)
|
269
|
-
end
|
270
|
-
end
|
271
|
-
|
272
|
-
# Mirror the Prism.parse_success? API by using the serialization API.
|
273
|
-
def parse_success?(code, **options)
|
274
|
-
LibRubyParser.pm_parse_success_p(code, code.bytesize, dump_options(options))
|
275
|
-
end
|
276
|
-
|
277
|
-
# Mirror the Prism.parse_file_success? API by using the serialization API.
|
278
|
-
def parse_file_success?(filepath, **options)
|
279
|
-
LibRubyParser::PrismString.with(filepath) do |string|
|
280
|
-
parse_success?(string.read, **options, filepath: filepath)
|
281
|
-
end
|
313
|
+
def parse_file_success_common(string, options) # :nodoc:
|
314
|
+
LibRubyParser.pm_parse_success_p(string.pointer, string.length, dump_options(options))
|
282
315
|
end
|
283
316
|
|
284
|
-
private
|
285
|
-
|
286
317
|
# Convert the given options into a serialized options string.
|
287
318
|
def dump_options(options)
|
288
319
|
template = +""
|
@@ -296,7 +327,7 @@ module Prism
|
|
296
327
|
values << 0
|
297
328
|
end
|
298
329
|
|
299
|
-
template << "
|
330
|
+
template << "l"
|
300
331
|
values << options.fetch(:line, 1)
|
301
332
|
|
302
333
|
template << "L"
|
@@ -312,7 +343,7 @@ module Prism
|
|
312
343
|
values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
|
313
344
|
|
314
345
|
template << "C"
|
315
|
-
values <<
|
346
|
+
values << { nil => 0, "3.3.0" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version])
|
316
347
|
|
317
348
|
template << "L"
|
318
349
|
if (scopes = options[:scopes])
|
data/lib/prism/lex_compat.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "delegate"
|
4
|
+
require "ripper"
|
4
5
|
|
5
6
|
module Prism
|
6
7
|
# This class is responsible for lexing the source using prism and then
|
@@ -860,7 +861,7 @@ module Prism
|
|
860
861
|
previous = []
|
861
862
|
results = []
|
862
863
|
|
863
|
-
|
864
|
+
lex(source).each do |token|
|
864
865
|
case token[1]
|
865
866
|
when :on_sp
|
866
867
|
# skip
|
@@ -886,6 +887,21 @@ module Prism
|
|
886
887
|
|
887
888
|
results
|
888
889
|
end
|
890
|
+
|
891
|
+
private
|
892
|
+
|
893
|
+
if Ripper.method(:lex).parameters.assoc(:keyrest)
|
894
|
+
def lex(source)
|
895
|
+
Ripper.lex(source, raise_errors: true)
|
896
|
+
end
|
897
|
+
else
|
898
|
+
def lex(source)
|
899
|
+
ripper = Ripper::Lexer.new(source)
|
900
|
+
ripper.lex.tap do |result|
|
901
|
+
raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
|
902
|
+
end
|
903
|
+
end
|
904
|
+
end
|
889
905
|
end
|
890
906
|
|
891
907
|
private_constant :LexRipper
|