prism 1.3.0 → 1.5.0

data/lib/prism/desugar_compiler.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   class DesugarAndWriteNode # :nodoc:

data/lib/prism/dispatcher.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dispatcher.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -52,6 +55,19 @@ module Prism
     #
     # def register: (Listener, *Symbol) -> void
     def register(listener, *events)
+      register_events(listener, events)
+    end
+
+    # Register all public methods of a listener that match the pattern
+    # `on_<node_name>_(enter|leave)`.
+    #
+    # def register_public_methods: (Listener) -> void
+    def register_public_methods(listener)
+      register_events(listener, listener.public_methods(false).grep(/\Aon_.+_(?:enter|leave)\z/))
+    end
+
+    # Register a listener for the given events.
+    private def register_events(listener, events)
       events.each { |event| (listeners[event] ||= []) << listener }
     end
 

data/lib/prism/dot_visitor.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dot_visitor.rb.erb
 if you are looking to modify the template
+++
 =end
 
-require "cgi"
+require "cgi/escape"
+require "cgi/util" unless defined?(CGI::EscapeExt)
 
 module Prism
   # This visitor provides the ability to call Node#to_dot, which converts a
@@ -3608,6 +3612,9 @@ module Prism
       table = Table.new("ParenthesesNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parentheses_node_flags_inspect(node))
+
       # body
       unless (body = node.body).nil?
         table.field("body", port: true)
@@ -3954,6 +3961,11 @@ module Prism
         digraph.edge("#{id}:reference -> #{node_id(reference)};")
       end
 
+      # then_keyword_loc
+      unless (then_keyword_loc = node.then_keyword_loc).nil?
+        table.field("then_keyword_loc", location_inspect(then_keyword_loc))
+      end
+
       # statements
       unless (statements = node.statements).nil?
         table.field("statements", port: true)
@@ -4682,6 +4694,14 @@ module Prism
       flags.join(", ")
     end
 
+    # Inspect a node that has parentheses_node_flags flags to display the flags as a
+    # comma-separated list.
+    def parentheses_node_flags_inspect(node)
+      flags = [] #: Array[String]
+      flags << "multiple_statements" if node.multiple_statements?
+      flags.join(", ")
+    end
+
     # Inspect a node that has range_flags flags to display the flags as a
     # comma-separated list.
     def range_flags_inspect(node)

data/lib/prism/dsl.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dsl.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -714,8 +717,8 @@ module Prism
     end
 
     # Create a new RescueNode node.
-    def rescue_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, exceptions: [], operator_loc: nil, reference: nil, statements: nil, subsequent: nil)
-      RescueNode.new(source, node_id, location, flags, keyword_loc, exceptions, operator_loc, reference, statements, subsequent)
+    def rescue_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, exceptions: [], operator_loc: nil, reference: nil, then_keyword_loc: nil, statements: nil, subsequent: nil)
+      RescueNode.new(source, node_id, location, flags, keyword_loc, exceptions, operator_loc, reference, then_keyword_loc, statements, subsequent)
     end
 
     # Create a new RestParameterNode node.
@@ -912,6 +915,14 @@ module Prism
       end
     end
 
+    # Retrieve the value of one of the ParenthesesNodeFlags flags.
+    def parentheses_node_flag(name)
+      case name
+      when :multiple_statements then ParenthesesNodeFlags::MULTIPLE_STATEMENTS
+      else Kernel.raise ArgumentError, "invalid ParenthesesNodeFlags flag: #{name.inspect}"
+      end
+    end
+
     # Retrieve the value of one of the RangeFlags flags.
     def range_flag(name)
       case name

data/lib/prism/ffi.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 # typed: ignore
 
 # This file is responsible for mirroring the API provided by the C extension by
@@ -7,6 +8,10 @@
 require "rbconfig"
 require "ffi"
 
+# We want to eagerly load this file if there are Ractors so that it does not get
+# autoloaded from within a non-main Ractor.
+require "prism/serialize" if defined?(Ractor)
+
 module Prism
   module LibRubyParser # :nodoc:
     extend FFI::Library
@@ -15,7 +20,8 @@ module Prism
     # must align with the build shared library from make/rake.
     libprism_in_build = File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
     libprism_in_libdir = "#{RbConfig::CONFIG["libdir"]}/prism/libprism.#{RbConfig::CONFIG["SOEXT"]}"
-    if File.exist? libprism_in_build
+
+    if File.exist?(libprism_in_build)
       INCLUDE_DIR = File.expand_path("../../include", __dir__)
       ffi_lib libprism_in_build
     else
@@ -80,6 +86,7 @@ module Prism
     end
 
     callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
+    callback :pm_parse_stream_feof_t, [:pointer], :int
     enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
     enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
 
@@ -95,7 +102,7 @@ module Prism
       "pm_string_query_local",
       "pm_string_query_constant",
       "pm_string_query_method_name",
-      [:pm_parse_stream_fgets_t]
+      [:pm_parse_stream_fgets_t, :pm_parse_stream_feof_t]
     )
 
     load_exported_functions_from(
@@ -158,6 +165,9 @@ module Prism
     class PrismString # :nodoc:
       SIZEOF = LibRubyParser.pm_string_sizeof
 
+      PLATFORM_EXPECTS_UTF8 =
+        RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i)
+
       attr_reader :pointer, :length
 
       def initialize(pointer, length, from_string)
@@ -192,8 +202,7 @@ module Prism
         # On Windows and Mac, it's expected that filepaths will be encoded in
         # UTF-8. If they are not, we need to convert them to UTF-8 before
         # passing them into pm_string_mapped_init.
-        if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
-           (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
+        if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
           filepath = filepath.encode(Encoding::UTF_8)
         end
 
@@ -222,7 +231,7 @@ module Prism
   private_constant :LibRubyParser
 
   # The version constant is set by reading the result of calling pm_version.
-  VERSION = LibRubyParser.pm_version.read_string
+  VERSION = LibRubyParser.pm_version.read_string.freeze
 
   class << self
     # Mirror the Prism.dump API by using the serialization API.
@@ -273,13 +282,15 @@ module Prism
           end
         }
 
+        eof_callback = -> (_) { stream.eof?  }
+
         # In the pm_serialize_parse_stream function it accepts a pointer to the
         # IO object as a void* and then passes it through to the callback as the
         # third argument, but it never touches it itself. As such, since we have
         # access to the IO object already through the closure of the lambda, we
         # can pass a null pointer here and not worry.
-        LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
-        Prism.load(source, buffer.read)
+        LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, eof_callback, dump_options(options))
+        Prism.load(source, buffer.read, options.fetch(:freeze, false))
       end
     end
 
@@ -354,50 +365,37 @@ module Prism
     def dump_common(string, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+
+        dumped = buffer.read
+        dumped.freeze if options.fetch(:freeze, false)
+
+        dumped
       end
     end
 
     def lex_common(string, code, options) # :nodoc:
-      serialized = LibRubyParser::PrismBuffer.with do |buffer|
+      LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+        Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
       end
-
-      Serialize.load_tokens(Source.for(code), serialized)
     end
 
     def parse_common(string, code, options) # :nodoc:
       serialized = dump_common(string, options)
-      Prism.load(code, serialized)
+      Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
     end
 
     def parse_comments_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        loader.load_header
-        loader.load_encoding
-        loader.load_start_line
-        loader.load_comments
+        Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
     def parse_lex_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        tokens = loader.load_tokens
-        node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
-        tokens.each { |token,| token.value.force_encoding(loader.encoding) }
-
-        ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
+        Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
@@ -427,11 +425,13 @@ module Prism
     def dump_options_version(version)
       case version
       when nil, "latest"
-        0
+        0 # Handled in pm_parser_init
       when /\A3\.3(\.\d+)?\z/
         1
       when /\A3\.4(\.\d+)?\z/
-        0
+        2
+      when /\A3\.5(\.\d+)?\z/
+        3
       else
         raise ArgumentError, "invalid version: #{version}"
       end
@@ -480,15 +480,43 @@ module Prism
       template << "C"
       values << (options.fetch(:partial_script, false) ? 1 : 0)
 
+      template << "C"
+      values << (options.fetch(:freeze, false) ? 1 : 0)
+
       template << "L"
       if (scopes = options[:scopes])
         values << scopes.length
 
         scopes.each do |scope|
+          locals = nil
+          forwarding = 0
+
+          case scope
+          when Array
+            locals = scope
+          when Scope
+            locals = scope.locals
+
+            scope.forwarding.each do |forward|
+              case forward
+              when :*     then forwarding |= 0x1
+              when :**    then forwarding |= 0x2
+              when :&     then forwarding |= 0x4
+              when :"..." then forwarding |= 0x8
+              else raise ArgumentError, "invalid forwarding value: #{forward}"
+              end
+            end
+          else
+            raise TypeError, "wrong argument type #{scope.class.inspect} (expected Array or Prism::Scope)"
+          end
+
           template << "L"
-          values << scope.length
+          values << locals.length
+
+          template << "C"
+          values << forwarding
 
-          scope.each do |local|
+          locals.each do |local|
             name = local.name
             template << "L"
             values << name.bytesize

data/lib/prism/inspect_visitor.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/inspect_visitor.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -1879,7 +1882,7 @@ module Prism
     # Inspect a ParenthesesNode node.
     def visit_parentheses_node(node)
       commands << [inspect_node("ParenthesesNode", node), indent]
-      flags = [("newline" if node.newline?), ("static_literal" if node.static_literal?), ].compact
+      flags = [("newline" if node.newline?), ("static_literal" if node.static_literal?), ("multiple_statements" if node.multiple_statements?)].compact
       commands << ["├── flags: #{flags.empty? ? "∅" : flags.join(", ")}\n", indent]
       if (body = node.body).nil?
         commands << ["├── body: ∅\n", indent]
@@ -2053,6 +2056,7 @@ module Prism
         commands << ["├── reference:\n", indent]
         commands << [reference, "#{indent}│   "]
       end
+      commands << ["├── then_keyword_loc: #{inspect_location(node.then_keyword_loc)}\n", indent]
       if (statements = node.statements).nil?
         commands << ["├── statements: ∅\n", indent]
       else

data/lib/prism/lex_compat.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 require "delegate"
 require "ripper"

data/lib/prism/mutation_compiler.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/mutation_compiler.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism