prism 0.29.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +77 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/README.md +4 -0
  5. data/config.yml +498 -145
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/parsing_rules.md +4 -1
  8. data/docs/ripper_translation.md +22 -0
  9. data/docs/serialization.md +3 -0
  10. data/ext/prism/api_node.c +2858 -2082
  11. data/ext/prism/extconf.rb +1 -1
  12. data/ext/prism/extension.c +203 -421
  13. data/ext/prism/extension.h +2 -2
  14. data/include/prism/ast.h +1732 -453
  15. data/include/prism/defines.h +36 -0
  16. data/include/prism/diagnostic.h +23 -6
  17. data/include/prism/node.h +0 -21
  18. data/include/prism/options.h +94 -3
  19. data/include/prism/parser.h +57 -28
  20. data/include/prism/regexp.h +18 -8
  21. data/include/prism/static_literals.h +3 -2
  22. data/include/prism/util/pm_char.h +1 -2
  23. data/include/prism/util/pm_constant_pool.h +0 -8
  24. data/include/prism/util/pm_integer.h +22 -15
  25. data/include/prism/util/pm_newline_list.h +11 -0
  26. data/include/prism/util/pm_string.h +28 -12
  27. data/include/prism/version.h +3 -3
  28. data/include/prism.h +0 -11
  29. data/lib/prism/compiler.rb +3 -0
  30. data/lib/prism/desugar_compiler.rb +111 -74
  31. data/lib/prism/dispatcher.rb +16 -1
  32. data/lib/prism/dot_visitor.rb +45 -34
  33. data/lib/prism/dsl.rb +660 -468
  34. data/lib/prism/ffi.rb +64 -6
  35. data/lib/prism/inspect_visitor.rb +294 -64
  36. data/lib/prism/lex_compat.rb +1 -1
  37. data/lib/prism/mutation_compiler.rb +11 -6
  38. data/lib/prism/node.rb +2469 -4973
  39. data/lib/prism/node_ext.rb +91 -14
  40. data/lib/prism/parse_result/comments.rb +0 -7
  41. data/lib/prism/parse_result/errors.rb +65 -0
  42. data/lib/prism/parse_result/newlines.rb +101 -11
  43. data/lib/prism/parse_result.rb +43 -3
  44. data/lib/prism/reflection.rb +10 -8
  45. data/lib/prism/serialize.rb +484 -609
  46. data/lib/prism/translation/parser/compiler.rb +152 -132
  47. data/lib/prism/translation/parser/lexer.rb +26 -4
  48. data/lib/prism/translation/parser.rb +9 -4
  49. data/lib/prism/translation/ripper.rb +22 -20
  50. data/lib/prism/translation/ruby_parser.rb +73 -13
  51. data/lib/prism/visitor.rb +3 -0
  52. data/lib/prism.rb +0 -4
  53. data/prism.gemspec +3 -5
  54. data/rbi/prism/dsl.rbi +521 -0
  55. data/rbi/prism/node.rbi +744 -4837
  56. data/rbi/prism/visitor.rbi +3 -0
  57. data/rbi/prism.rbi +36 -30
  58. data/sig/prism/dsl.rbs +190 -303
  59. data/sig/prism/mutation_compiler.rbs +1 -0
  60. data/sig/prism/node.rbs +759 -628
  61. data/sig/prism/parse_result.rbs +2 -0
  62. data/sig/prism/visitor.rbs +1 -0
  63. data/sig/prism.rbs +103 -64
  64. data/src/diagnostic.c +62 -28
  65. data/src/node.c +499 -1754
  66. data/src/options.c +76 -27
  67. data/src/prettyprint.c +156 -112
  68. data/src/prism.c +2773 -2081
  69. data/src/regexp.c +202 -69
  70. data/src/serialize.c +170 -50
  71. data/src/static_literals.c +63 -84
  72. data/src/token_type.c +4 -4
  73. data/src/util/pm_constant_pool.c +0 -8
  74. data/src/util/pm_integer.c +53 -25
  75. data/src/util/pm_newline_list.c +29 -0
  76. data/src/util/pm_string.c +130 -80
  77. data/src/util/pm_strpbrk.c +32 -6
  78. metadata +4 -6
  79. data/include/prism/util/pm_string_list.h +0 -44
  80. data/lib/prism/debug.rb +0 -249
  81. data/lib/prism/translation/parser/rubocop.rb +0 -73
  82. data/src/util/pm_string_list.c +0 -28
data/lib/prism/ffi.rb CHANGED
@@ -72,6 +72,7 @@ module Prism
72
72
  end
73
73
 
74
74
  callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
75
+ enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
75
76
 
76
77
  load_exported_functions_from(
77
78
  "prism.h",
@@ -176,13 +177,26 @@ module Prism
176
177
  def self.with_file(filepath)
177
178
  raise TypeError unless filepath.is_a?(String)
178
179
 
180
+ # On Windows and Mac, it's expected that filepaths will be encoded in
181
+ # UTF-8. If they are not, we need to convert them to UTF-8 before
182
+ # passing them into pm_string_mapped_init.
183
+ if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
184
+ (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
185
+ filepath = filepath.encode(Encoding::UTF_8)
186
+ end
187
+
179
188
  FFI::MemoryPointer.new(SIZEOF) do |pm_string|
180
- if LibRubyParser.pm_string_mapped_init(pm_string, filepath)
189
+ case (result = LibRubyParser.pm_string_mapped_init(pm_string, filepath))
190
+ when :PM_STRING_INIT_SUCCESS
181
191
  pointer = LibRubyParser.pm_string_source(pm_string)
182
192
  length = LibRubyParser.pm_string_length(pm_string)
183
193
  return yield new(pointer, length, false)
184
- else
194
+ when :PM_STRING_INIT_ERROR_GENERIC
185
195
  raise SystemCallError.new(filepath, FFI.errno)
196
+ when :PM_STRING_INIT_ERROR_DIRECTORY
197
+ raise Errno::EISDIR.new(filepath)
198
+ else
199
+ raise "Unknown error initializing pm_string_t: #{result.inspect}"
186
200
  end
187
201
  ensure
188
202
  LibRubyParser.pm_string_free(pm_string)
@@ -200,8 +214,8 @@ module Prism
200
214
 
201
215
  class << self
202
216
  # Mirror the Prism.dump API by using the serialization API.
203
- def dump(code, **options)
204
- LibRubyParser::PrismString.with_string(code) { |string| dump_common(string, options) }
217
+ def dump(source, **options)
218
+ LibRubyParser::PrismString.with_string(source) { |string| dump_common(string, options) }
205
219
  end
206
220
 
207
221
  # Mirror the Prism.dump_file API by using the serialization API.
@@ -302,6 +316,27 @@ module Prism
302
316
  !parse_file_success?(filepath, **options)
303
317
  end
304
318
 
319
+ # Mirror the Prism.profile API by using the serialization API.
320
+ def profile(source, **options)
321
+ LibRubyParser::PrismString.with_string(source) do |string|
322
+ LibRubyParser::PrismBuffer.with do |buffer|
323
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
324
+ nil
325
+ end
326
+ end
327
+ end
328
+
329
+ # Mirror the Prism.profile_file API by using the serialization API.
330
+ def profile_file(filepath, **options)
331
+ LibRubyParser::PrismString.with_file(filepath) do |string|
332
+ LibRubyParser::PrismBuffer.with do |buffer|
333
+ options[:filepath] = filepath
334
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
335
+ nil
336
+ end
337
+ end
338
+ end
339
+
305
340
  private
306
341
 
307
342
  def dump_common(string, options) # :nodoc:
@@ -376,6 +411,20 @@ module Prism
376
411
  end
377
412
  end
378
413
 
414
+ # Return the value that should be dumped for the version option.
415
+ def dump_options_version(version)
416
+ case version
417
+ when nil, "latest"
418
+ 0
419
+ when /\A3\.3(\.\d+)?\z/
420
+ 1
421
+ when /\A3\.4(\.\d+)?\z/
422
+ 0
423
+ else
424
+ raise ArgumentError, "invalid version: #{version}"
425
+ end
426
+ end
427
+
379
428
  # Convert the given options into a serialized options string.
380
429
  def dump_options(options)
381
430
  template = +""
@@ -394,7 +443,7 @@ module Prism
394
443
 
395
444
  template << "L"
396
445
  if (encoding = options[:encoding])
397
- name = encoding.name
446
+ name = encoding.is_a?(Encoding) ? encoding.name : encoding
398
447
  values.push(name.bytesize, name.b)
399
448
  template << "A*"
400
449
  else
@@ -408,7 +457,16 @@ module Prism
408
457
  values << dump_options_command_line(options)
409
458
 
410
459
  template << "C"
411
- values << { nil => 0, "3.3.0" => 1, "3.3.1" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version])
460
+ values << dump_options_version(options[:version])
461
+
462
+ template << "C"
463
+ values << (options[:encoding] == false ? 1 : 0)
464
+
465
+ template << "C"
466
+ values << (options.fetch(:main_script, false) ? 1 : 0)
467
+
468
+ template << "C"
469
+ values << (options.fetch(:partial_script, false) ? 1 : 0)
412
470
 
413
471
  template << "L"
414
472
  if (scopes = options[:scopes])