prism 1.4.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +73 -1
  3. data/Makefile +7 -5
  4. data/README.md +3 -1
  5. data/config.yml +294 -41
  6. data/docs/build_system.md +2 -2
  7. data/docs/cruby_compilation.md +1 -1
  8. data/docs/design.md +2 -2
  9. data/docs/parser_translation.md +8 -23
  10. data/docs/releasing.md +6 -25
  11. data/docs/ripper_translation.md +1 -1
  12. data/ext/prism/api_node.c +9 -3
  13. data/ext/prism/extconf.rb +1 -1
  14. data/ext/prism/extension.c +24 -3
  15. data/ext/prism/extension.h +1 -1
  16. data/include/prism/ast.h +360 -70
  17. data/include/prism/diagnostic.h +7 -0
  18. data/include/prism/options.h +49 -3
  19. data/include/prism/parser.h +3 -0
  20. data/include/prism/regexp.h +2 -2
  21. data/include/prism/util/pm_buffer.h +8 -0
  22. data/include/prism/util/pm_integer.h +4 -0
  23. data/include/prism/util/pm_list.h +6 -0
  24. data/include/prism/util/pm_string.h +12 -2
  25. data/include/prism/version.h +2 -2
  26. data/include/prism.h +40 -15
  27. data/lib/prism/compiler.rb +456 -151
  28. data/lib/prism/desugar_compiler.rb +1 -0
  29. data/lib/prism/dispatcher.rb +16 -0
  30. data/lib/prism/dot_visitor.rb +10 -1
  31. data/lib/prism/dsl.rb +5 -2
  32. data/lib/prism/ffi.rb +28 -10
  33. data/lib/prism/inspect_visitor.rb +4 -0
  34. data/lib/prism/lex_compat.rb +1 -0
  35. data/lib/prism/mutation_compiler.rb +3 -0
  36. data/lib/prism/node.rb +559 -349
  37. data/lib/prism/node_ext.rb +4 -1
  38. data/lib/prism/pack.rb +2 -0
  39. data/lib/prism/parse_result/comments.rb +1 -0
  40. data/lib/prism/parse_result/errors.rb +1 -0
  41. data/lib/prism/parse_result/newlines.rb +1 -0
  42. data/lib/prism/parse_result.rb +3 -15
  43. data/lib/prism/pattern.rb +1 -0
  44. data/lib/prism/polyfill/scan_byte.rb +14 -0
  45. data/lib/prism/polyfill/warn.rb +36 -0
  46. data/lib/prism/reflection.rb +4 -1
  47. data/lib/prism/relocation.rb +1 -0
  48. data/lib/prism/serialize.rb +30 -22
  49. data/lib/prism/string_query.rb +1 -0
  50. data/lib/prism/translation/parser/builder.rb +1 -0
  51. data/lib/prism/translation/parser/compiler.rb +63 -41
  52. data/lib/prism/translation/parser/lexer.rb +29 -21
  53. data/lib/prism/translation/parser.rb +25 -4
  54. data/lib/prism/translation/parser33.rb +1 -0
  55. data/lib/prism/translation/parser34.rb +1 -0
  56. data/lib/prism/translation/parser35.rb +2 -6
  57. data/lib/prism/translation/parser40.rb +13 -0
  58. data/lib/prism/translation/parser41.rb +13 -0
  59. data/lib/prism/translation/parser_current.rb +26 -0
  60. data/lib/prism/translation/ripper/sexp.rb +1 -0
  61. data/lib/prism/translation/ripper.rb +19 -3
  62. data/lib/prism/translation/ruby_parser.rb +340 -22
  63. data/lib/prism/translation.rb +4 -0
  64. data/lib/prism/visitor.rb +457 -152
  65. data/lib/prism.rb +22 -0
  66. data/prism.gemspec +9 -1
  67. data/rbi/prism/dsl.rbi +6 -6
  68. data/rbi/prism/node.rbi +42 -17
  69. data/rbi/prism/translation/parser35.rbi +0 -2
  70. data/rbi/prism/translation/parser40.rbi +6 -0
  71. data/rbi/prism/translation/parser41.rbi +6 -0
  72. data/sig/prism/dispatcher.rbs +3 -0
  73. data/sig/prism/dsl.rbs +5 -5
  74. data/sig/prism/node.rbs +462 -38
  75. data/sig/prism/node_ext.rbs +84 -17
  76. data/sig/prism/parse_result/comments.rbs +38 -0
  77. data/sig/prism/parse_result.rbs +4 -0
  78. data/sig/prism/reflection.rbs +1 -1
  79. data/sig/prism.rbs +4 -0
  80. data/src/diagnostic.c +13 -1
  81. data/src/encoding.c +172 -67
  82. data/src/node.c +11 -0
  83. data/src/options.c +17 -7
  84. data/src/prettyprint.c +18 -0
  85. data/src/prism.c +1495 -2021
  86. data/src/serialize.c +9 -1
  87. data/src/token_type.c +38 -36
  88. data/src/util/pm_constant_pool.c +1 -1
  89. data/src/util/pm_string.c +6 -8
  90. metadata +11 -3
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class DesugarAndWriteNode # :nodoc:
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dispatcher.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -52,6 +55,19 @@ module Prism
52
55
  #
53
56
  # def register: (Listener, *Symbol) -> void
54
57
  def register(listener, *events)
58
+ register_events(listener, events)
59
+ end
60
+
61
+ # Register all public methods of a listener that match the pattern
62
+ # `on_<node_name>_(enter|leave)`.
63
+ #
64
+ # def register_public_methods: (Listener) -> void
65
+ def register_public_methods(listener)
66
+ register_events(listener, listener.public_methods(false).grep(/\Aon_.+_(?:enter|leave)\z/))
67
+ end
68
+
69
+ # Register a listener for the given events.
70
+ private def register_events(listener, events)
55
71
  events.each { |event| (listeners[event] ||= []) << listener }
56
72
  end
57
73
 
@@ -1,12 +1,16 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dot_visitor.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
- require "cgi"
12
+ require "cgi/escape"
13
+ require "cgi/util" unless defined?(CGI::EscapeExt)
10
14
 
11
15
  module Prism
12
16
  # This visitor provides the ability to call Node#to_dot, which converts a
@@ -720,6 +724,11 @@ module Prism
720
724
  table.field("closing_loc", location_inspect(closing_loc))
721
725
  end
722
726
 
727
+ # equal_loc
728
+ unless (equal_loc = node.equal_loc).nil?
729
+ table.field("equal_loc", location_inspect(equal_loc))
730
+ end
731
+
723
732
  # block
724
733
  unless (block = node.block).nil?
725
734
  table.field("block", port: true)
data/lib/prism/dsl.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dsl.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -164,8 +167,8 @@ module Prism
164
167
  end
165
168
 
166
169
  # Create a new CallNode node.
167
- def call_node(source: default_source, node_id: 0, location: default_location, flags: 0, receiver: nil, call_operator_loc: nil, name: :"", message_loc: nil, opening_loc: nil, arguments: nil, closing_loc: nil, block: nil)
168
- CallNode.new(source, node_id, location, flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, block)
170
+ def call_node(source: default_source, node_id: 0, location: default_location, flags: 0, receiver: nil, call_operator_loc: nil, name: :"", message_loc: nil, opening_loc: nil, arguments: nil, closing_loc: nil, equal_loc: nil, block: nil)
171
+ CallNode.new(source, node_id, location, flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, equal_loc, block)
169
172
  end
170
173
 
171
174
  # Create a new CallOperatorWriteNode node.
data/lib/prism/ffi.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
  # typed: ignore
3
4
 
4
5
  # This file is responsible for mirroring the API provided by the C extension by
@@ -7,6 +8,10 @@
7
8
  require "rbconfig"
8
9
  require "ffi"
9
10
 
11
+ # We want to eagerly load this file if there are Ractors so that it does not get
12
+ # autoloaded from within a non-main Ractor.
13
+ require "prism/serialize" if defined?(Ractor)
14
+
10
15
  module Prism
11
16
  module LibRubyParser # :nodoc:
12
17
  extend FFI::Library
@@ -81,6 +86,7 @@ module Prism
81
86
  end
82
87
 
83
88
  callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
89
+ callback :pm_parse_stream_feof_t, [:pointer], :int
84
90
  enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
85
91
  enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
86
92
 
@@ -96,7 +102,7 @@ module Prism
96
102
  "pm_string_query_local",
97
103
  "pm_string_query_constant",
98
104
  "pm_string_query_method_name",
99
- [:pm_parse_stream_fgets_t]
105
+ [:pm_parse_stream_fgets_t, :pm_parse_stream_feof_t]
100
106
  )
101
107
 
102
108
  load_exported_functions_from(
@@ -159,6 +165,9 @@ module Prism
159
165
  class PrismString # :nodoc:
160
166
  SIZEOF = LibRubyParser.pm_string_sizeof
161
167
 
168
+ PLATFORM_EXPECTS_UTF8 =
169
+ RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i)
170
+
162
171
  attr_reader :pointer, :length
163
172
 
164
173
  def initialize(pointer, length, from_string)
@@ -193,8 +202,7 @@ module Prism
193
202
  # On Windows and Mac, it's expected that filepaths will be encoded in
194
203
  # UTF-8. If they are not, we need to convert them to UTF-8 before
195
204
  # passing them into pm_string_mapped_init.
196
- if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
197
- (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
205
+ if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
198
206
  filepath = filepath.encode(Encoding::UTF_8)
199
207
  end
200
208
 
@@ -223,7 +231,7 @@ module Prism
223
231
  private_constant :LibRubyParser
224
232
 
225
233
  # The version constant is set by reading the result of calling pm_version.
226
- VERSION = LibRubyParser.pm_version.read_string
234
+ VERSION = LibRubyParser.pm_version.read_string.freeze
227
235
 
228
236
  class << self
229
237
  # Mirror the Prism.dump API by using the serialization API.
@@ -274,12 +282,14 @@ module Prism
274
282
  end
275
283
  }
276
284
 
285
+ eof_callback = -> (_) { stream.eof? }
286
+
277
287
  # In the pm_serialize_parse_stream function it accepts a pointer to the
278
288
  # IO object as a void* and then passes it through to the callback as the
279
289
  # third argument, but it never touches it itself. As such, since we have
280
290
  # access to the IO object already through the closure of the lambda, we
281
291
  # can pass a null pointer here and not worry.
282
- LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
292
+ LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, eof_callback, dump_options(options))
283
293
  Prism.load(source, buffer.read, options.fetch(:freeze, false))
284
294
  end
285
295
  end
@@ -413,17 +423,25 @@ module Prism
413
423
 
414
424
  # Return the value that should be dumped for the version option.
415
425
  def dump_options_version(version)
416
- case version
426
+ current = version == "current"
427
+
428
+ case current ? RUBY_VERSION : version
417
429
  when nil, "latest"
418
- 0
430
+ 0 # Handled in pm_parser_init
419
431
  when /\A3\.3(\.\d+)?\z/
420
432
  1
421
433
  when /\A3\.4(\.\d+)?\z/
422
434
  2
423
- when /\A3\.5(\.\d+)?\z/
424
- 0
435
+ when /\A3\.5(\.\d+)?\z/, /\A4\.0(\.\d+)?\z/
436
+ 3
437
+ when /\A4\.1(\.\d+)?\z/
438
+ 4
425
439
  else
426
- raise ArgumentError, "invalid version: #{version}"
440
+ if current
441
+ raise CurrentVersionError, RUBY_VERSION
442
+ else
443
+ raise ArgumentError, "invalid version: #{version}"
444
+ end
427
445
  end
428
446
  end
429
447
 
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/inspect_visitor.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -399,6 +402,7 @@ module Prism
399
402
  commands << [arguments, "#{indent}│ "]
400
403
  end
401
404
  commands << ["├── closing_loc: #{inspect_location(node.closing_loc)}\n", indent]
405
+ commands << ["├── equal_loc: #{inspect_location(node.equal_loc)}\n", indent]
402
406
  if (block = node.block).nil?
403
407
  commands << ["└── block: ∅\n", indent]
404
408
  else
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "delegate"
4
5
  require "ripper"
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/mutation_compiler.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism