prism 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +23 -1
  3. data/Makefile +2 -1
  4. data/README.md +1 -0
  5. data/config.yml +264 -37
  6. data/docs/parser_translation.md +8 -23
  7. data/docs/ripper_translation.md +1 -1
  8. data/ext/prism/api_node.c +2 -0
  9. data/ext/prism/extension.c +14 -1
  10. data/ext/prism/extension.h +1 -1
  11. data/include/prism/ast.h +275 -49
  12. data/include/prism/diagnostic.h +4 -0
  13. data/include/prism/options.h +43 -3
  14. data/include/prism/regexp.h +2 -2
  15. data/include/prism/util/pm_buffer.h +8 -0
  16. data/include/prism/util/pm_integer.h +4 -0
  17. data/include/prism/util/pm_list.h +6 -0
  18. data/include/prism/util/pm_string.h +12 -2
  19. data/include/prism/version.h +2 -2
  20. data/include/prism.h +39 -14
  21. data/lib/prism/compiler.rb +456 -151
  22. data/lib/prism/desugar_compiler.rb +1 -0
  23. data/lib/prism/dispatcher.rb +16 -0
  24. data/lib/prism/dot_visitor.rb +5 -1
  25. data/lib/prism/dsl.rb +3 -0
  26. data/lib/prism/ffi.rb +17 -7
  27. data/lib/prism/inspect_visitor.rb +3 -0
  28. data/lib/prism/lex_compat.rb +1 -0
  29. data/lib/prism/mutation_compiler.rb +3 -0
  30. data/lib/prism/node.rb +506 -335
  31. data/lib/prism/node_ext.rb +4 -1
  32. data/lib/prism/pack.rb +2 -0
  33. data/lib/prism/parse_result/comments.rb +1 -0
  34. data/lib/prism/parse_result/errors.rb +1 -0
  35. data/lib/prism/parse_result/newlines.rb +1 -0
  36. data/lib/prism/parse_result.rb +1 -0
  37. data/lib/prism/pattern.rb +1 -0
  38. data/lib/prism/polyfill/scan_byte.rb +14 -0
  39. data/lib/prism/polyfill/warn.rb +42 -0
  40. data/lib/prism/reflection.rb +3 -0
  41. data/lib/prism/relocation.rb +1 -0
  42. data/lib/prism/serialize.rb +24 -19
  43. data/lib/prism/string_query.rb +1 -0
  44. data/lib/prism/translation/parser/builder.rb +1 -0
  45. data/lib/prism/translation/parser/compiler.rb +47 -25
  46. data/lib/prism/translation/parser/lexer.rb +29 -21
  47. data/lib/prism/translation/parser.rb +13 -1
  48. data/lib/prism/translation/parser33.rb +1 -0
  49. data/lib/prism/translation/parser34.rb +1 -0
  50. data/lib/prism/translation/parser35.rb +1 -0
  51. data/lib/prism/translation/parser_current.rb +24 -0
  52. data/lib/prism/translation/ripper/sexp.rb +1 -0
  53. data/lib/prism/translation/ripper.rb +17 -1
  54. data/lib/prism/translation/ruby_parser.rb +286 -3
  55. data/lib/prism/translation.rb +2 -0
  56. data/lib/prism/visitor.rb +457 -152
  57. data/lib/prism.rb +2 -0
  58. data/prism.gemspec +5 -1
  59. data/rbi/prism/dsl.rbi +3 -3
  60. data/rbi/prism/node.rbi +21 -9
  61. data/sig/prism/dispatcher.rbs +3 -0
  62. data/sig/prism/dsl.rbs +3 -3
  63. data/sig/prism/node.rbs +444 -30
  64. data/sig/prism/node_ext.rbs +84 -17
  65. data/sig/prism/parse_result/comments.rbs +38 -0
  66. data/sig/prism/parse_result.rbs +4 -0
  67. data/sig/prism/reflection.rbs +1 -1
  68. data/src/diagnostic.c +7 -1
  69. data/src/node.c +2 -0
  70. data/src/options.c +2 -2
  71. data/src/prettyprint.c +2 -0
  72. data/src/prism.c +252 -130
  73. data/src/serialize.c +2 -0
  74. data/src/token_type.c +36 -34
  75. metadata +6 -2
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  module Prism
4
5
  class DesugarAndWriteNode # :nodoc:
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dispatcher.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -52,6 +55,19 @@ module Prism
52
55
  #
53
56
  # def register: (Listener, *Symbol) -> void
54
57
  def register(listener, *events)
58
+ register_events(listener, events)
59
+ end
60
+
61
+ # Register all public methods of a listener that match the pattern
62
+ # `on_<node_name>_(enter|leave)`.
63
+ #
64
+ # def register_public_methods: (Listener) -> void
65
+ def register_public_methods(listener)
66
+ register_events(listener, listener.public_methods(false).grep(/\Aon_.+_(?:enter|leave)\z/))
67
+ end
68
+
69
+ # Register a listener for the given events.
70
+ private def register_events(listener, events)
55
71
  events.each { |event| (listeners[event] ||= []) << listener }
56
72
  end
57
73
 
@@ -1,12 +1,16 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dot_visitor.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
- require "cgi"
12
+ require "cgi/escape"
13
+ require "cgi/util" unless defined?(CGI::EscapeExt)
10
14
 
11
15
  module Prism
12
16
  # This visitor provides the ability to call Node#to_dot, which converts a
data/lib/prism/dsl.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/dsl.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
data/lib/prism/ffi.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
  # typed: ignore
3
4
 
4
5
  # This file is responsible for mirroring the API provided by the C extension by
@@ -7,6 +8,10 @@
7
8
  require "rbconfig"
8
9
  require "ffi"
9
10
 
11
+ # We want to eagerly load this file if there are Ractors so that it does not get
12
+ # autoloaded from within a non-main Ractor.
13
+ require "prism/serialize" if defined?(Ractor)
14
+
10
15
  module Prism
11
16
  module LibRubyParser # :nodoc:
12
17
  extend FFI::Library
@@ -81,6 +86,7 @@ module Prism
81
86
  end
82
87
 
83
88
  callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
89
+ callback :pm_parse_stream_feof_t, [:pointer], :int
84
90
  enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
85
91
  enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
86
92
 
@@ -96,7 +102,7 @@ module Prism
96
102
  "pm_string_query_local",
97
103
  "pm_string_query_constant",
98
104
  "pm_string_query_method_name",
99
- [:pm_parse_stream_fgets_t]
105
+ [:pm_parse_stream_fgets_t, :pm_parse_stream_feof_t]
100
106
  )
101
107
 
102
108
  load_exported_functions_from(
@@ -159,6 +165,9 @@ module Prism
159
165
  class PrismString # :nodoc:
160
166
  SIZEOF = LibRubyParser.pm_string_sizeof
161
167
 
168
+ PLATFORM_EXPECTS_UTF8 =
169
+ RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i)
170
+
162
171
  attr_reader :pointer, :length
163
172
 
164
173
  def initialize(pointer, length, from_string)
@@ -193,8 +202,7 @@ module Prism
193
202
  # On Windows and Mac, it's expected that filepaths will be encoded in
194
203
  # UTF-8. If they are not, we need to convert them to UTF-8 before
195
204
  # passing them into pm_string_mapped_init.
196
- if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
197
- (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
205
+ if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
198
206
  filepath = filepath.encode(Encoding::UTF_8)
199
207
  end
200
208
 
@@ -223,7 +231,7 @@ module Prism
223
231
  private_constant :LibRubyParser
224
232
 
225
233
  # The version constant is set by reading the result of calling pm_version.
226
- VERSION = LibRubyParser.pm_version.read_string
234
+ VERSION = LibRubyParser.pm_version.read_string.freeze
227
235
 
228
236
  class << self
229
237
  # Mirror the Prism.dump API by using the serialization API.
@@ -274,12 +282,14 @@ module Prism
274
282
  end
275
283
  }
276
284
 
285
+ eof_callback = -> (_) { stream.eof? }
286
+
277
287
  # In the pm_serialize_parse_stream function it accepts a pointer to the
278
288
  # IO object as a void* and then passes it through to the callback as the
279
289
  # third argument, but it never touches it itself. As such, since we have
280
290
  # access to the IO object already through the closure of the lambda, we
281
291
  # can pass a null pointer here and not worry.
282
- LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
292
+ LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, eof_callback, dump_options(options))
283
293
  Prism.load(source, buffer.read, options.fetch(:freeze, false))
284
294
  end
285
295
  end
@@ -415,13 +425,13 @@ module Prism
415
425
  def dump_options_version(version)
416
426
  case version
417
427
  when nil, "latest"
418
- 0
428
+ 0 # Handled in pm_parser_init
419
429
  when /\A3\.3(\.\d+)?\z/
420
430
  1
421
431
  when /\A3\.4(\.\d+)?\z/
422
432
  2
423
433
  when /\A3\.5(\.\d+)?\z/
424
- 0
434
+ 3
425
435
  else
426
436
  raise ArgumentError, "invalid version: #{version}"
427
437
  end
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/inspect_visitor.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  require "delegate"
4
5
  require "ripper"
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # :markup: markdown
2
3
 
3
4
  =begin
5
+ --
4
6
  This file is generated by the templates/template.rb script and should not be
5
7
  modified manually. See templates/lib/prism/mutation_compiler.rb.erb
6
8
  if you are looking to modify the template
9
+ ++
7
10
  =end
8
11
 
9
12
  module Prism