prism 1.4.0 → 1.8.0

data/lib/prism/desugar_compiler.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 module Prism
   class DesugarAndWriteNode # :nodoc:

data/lib/prism/dispatcher.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dispatcher.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -52,6 +55,19 @@ module Prism
     #
     # def register: (Listener, *Symbol) -> void
     def register(listener, *events)
+      register_events(listener, events)
+    end
+
+    # Register all public methods of a listener that match the pattern
+    # `on_<node_name>_(enter|leave)`.
+    #
+    # def register_public_methods: (Listener) -> void
+    def register_public_methods(listener)
+      register_events(listener, listener.public_methods(false).grep(/\Aon_.+_(?:enter|leave)\z/))
+    end
+
+    # Register a listener for the given events.
+    private def register_events(listener, events)
       events.each { |event| (listeners[event] ||= []) << listener }
     end
 

data/lib/prism/dot_visitor.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dot_visitor.rb.erb
 if you are looking to modify the template
+++
 =end
 
-require "cgi"
+require "cgi/escape"
+require "cgi/util" unless defined?(CGI::EscapeExt)
 
 module Prism
   # This visitor provides the ability to call Node#to_dot, which converts a
@@ -720,6 +724,11 @@ module Prism
         table.field("closing_loc", location_inspect(closing_loc))
       end
 
+      # equal_loc
+      unless (equal_loc = node.equal_loc).nil?
+        table.field("equal_loc", location_inspect(equal_loc))
+      end
+
       # block
       unless (block = node.block).nil?
         table.field("block", port: true)

data/lib/prism/dsl.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dsl.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -164,8 +167,8 @@ module Prism
     end
 
     # Create a new CallNode node.
-    def call_node(source: default_source, node_id: 0, location: default_location, flags: 0, receiver: nil, call_operator_loc: nil, name: :"", message_loc: nil, opening_loc: nil, arguments: nil, closing_loc: nil, block: nil)
-      CallNode.new(source, node_id, location, flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, block)
+    def call_node(source: default_source, node_id: 0, location: default_location, flags: 0, receiver: nil, call_operator_loc: nil, name: :"", message_loc: nil, opening_loc: nil, arguments: nil, closing_loc: nil, equal_loc: nil, block: nil)
+      CallNode.new(source, node_id, location, flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, equal_loc, block)
     end
 
     # Create a new CallOperatorWriteNode node.

data/lib/prism/ffi.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# :markup: markdown
 # typed: ignore
 
 # This file is responsible for mirroring the API provided by the C extension by
@@ -7,6 +8,10 @@
 require "rbconfig"
 require "ffi"
 
+# We want to eagerly load this file if there are Ractors so that it does not get
+# autoloaded from within a non-main Ractor.
+require "prism/serialize" if defined?(Ractor)
+
 module Prism
   module LibRubyParser # :nodoc:
     extend FFI::Library
@@ -81,6 +86,7 @@ module Prism
     end
 
     callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
+    callback :pm_parse_stream_feof_t, [:pointer], :int
     enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
     enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
 
@@ -96,7 +102,7 @@ module Prism
       "pm_string_query_local",
       "pm_string_query_constant",
       "pm_string_query_method_name",
-      [:pm_parse_stream_fgets_t]
+      [:pm_parse_stream_fgets_t, :pm_parse_stream_feof_t]
     )
 
     load_exported_functions_from(
@@ -159,6 +165,9 @@ module Prism
     class PrismString # :nodoc:
       SIZEOF = LibRubyParser.pm_string_sizeof
 
+      PLATFORM_EXPECTS_UTF8 =
+        RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i)
+
       attr_reader :pointer, :length
 
       def initialize(pointer, length, from_string)
@@ -193,8 +202,7 @@ module Prism
         # On Windows and Mac, it's expected that filepaths will be encoded in
         # UTF-8. If they are not, we need to convert them to UTF-8 before
         # passing them into pm_string_mapped_init.
-        if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
-           (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
+        if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
           filepath = filepath.encode(Encoding::UTF_8)
         end
 
@@ -223,7 +231,7 @@ module Prism
   private_constant :LibRubyParser
 
   # The version constant is set by reading the result of calling pm_version.
-  VERSION = LibRubyParser.pm_version.read_string
+  VERSION = LibRubyParser.pm_version.read_string.freeze
 
   class << self
     # Mirror the Prism.dump API by using the serialization API.
@@ -274,12 +282,14 @@ module Prism
           end
         }
 
+        eof_callback = -> (_) { stream.eof?  }
+
         # In the pm_serialize_parse_stream function it accepts a pointer to the
         # IO object as a void* and then passes it through to the callback as the
         # third argument, but it never touches it itself. As such, since we have
         # access to the IO object already through the closure of the lambda, we
         # can pass a null pointer here and not worry.
-        LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
+        LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, eof_callback, dump_options(options))
         Prism.load(source, buffer.read, options.fetch(:freeze, false))
       end
     end
@@ -413,17 +423,25 @@ module Prism
 
     # Return the value that should be dumped for the version option.
     def dump_options_version(version)
-      case version
+      current = version == "current"
+
+      case current ? RUBY_VERSION : version
       when nil, "latest"
-        0
+        0 # Handled in pm_parser_init
       when /\A3\.3(\.\d+)?\z/
         1
       when /\A3\.4(\.\d+)?\z/
         2
-      when /\A3\.5(\.\d+)?\z/
-        0
+      when /\A3\.5(\.\d+)?\z/, /\A4\.0(\.\d+)?\z/
+        3
+      when /\A4\.1(\.\d+)?\z/
+        4
       else
-        raise ArgumentError, "invalid version: #{version}"
+        if current
+          raise CurrentVersionError, RUBY_VERSION
+        else
+          raise ArgumentError, "invalid version: #{version}"
+        end
       end
     end
 

data/lib/prism/inspect_visitor.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/inspect_visitor.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism
@@ -399,6 +402,7 @@ module Prism
         commands << [arguments, "#{indent}│   "]
       end
       commands << ["├── closing_loc: #{inspect_location(node.closing_loc)}\n", indent]
+      commands << ["├── equal_loc: #{inspect_location(node.equal_loc)}\n", indent]
       if (block = node.block).nil?
         commands << ["└── block: ∅\n", indent]
       else

data/lib/prism/lex_compat.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 require "delegate"
-require "ripper"
 
 module Prism
   # This class is responsible for lexing the source using prism and then
@@ -248,8 +248,8 @@ module Prism
     class IdentToken < Token
       def ==(other) # :nodoc:
         (self[0...-1] == other[0...-1]) && (
-          (other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) ||
-          (other[3] & Ripper::EXPR_ARG_ANY != 0)
+          (other[3] == Translation::Ripper::EXPR_LABEL | Translation::Ripper::EXPR_END) ||
+          (other[3] & (Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_CMDARG) != 0)
         )
       end
     end
@@ -260,8 +260,8 @@ module Prism
       def ==(other) # :nodoc:
         return false unless self[0...-1] == other[0...-1]
 
-        if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
-          other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0
+        if self[3] == Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED
+          other[3] & Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED != 0
         else
           self[3] == other[3]
         end
@@ -279,8 +279,8 @@ module Prism
     class ParamToken < Token
       def ==(other) # :nodoc:
         (self[0...-1] == other[0...-1]) && (
-          (other[3] == Ripper::EXPR_END) ||
-          (other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
+          (other[3] == Translation::Ripper::EXPR_END) ||
+          (other[3] == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL)
         )
       end
     end
@@ -614,6 +614,11 @@ module Prism
 
     private_constant :Heredoc
 
+    # In previous versions of Ruby, Ripper wouldn't flush the bom before the
+    # first token, so we had to have a hack in place to account for that.
+    BOM_FLUSHED = RUBY_VERSION >= "3.3.0"
+    private_constant :BOM_FLUSHED
+
     attr_reader :source, :options
 
     def initialize(source, **options)
@@ -629,13 +634,9 @@ module Prism
 
       result = Prism.lex(source, **options)
       result_value = result.value
-      previous_state = nil #: Ripper::Lexer::State?
+      previous_state = nil #: State?
       last_heredoc_end = nil #: Integer?
 
-      # In previous versions of Ruby, Ripper wouldn't flush the bom before the
-      # first token, so we had to have a hack in place to account for that. This
-      # checks for that behavior.
-      bom_flushed = Ripper.lex("\xEF\xBB\xBF# test")[0][0][1] == 0
       bom = source.byteslice(0..2) == "\xEF\xBB\xBF"
 
       result_value.each_with_index do |(token, lex_state), index|
@@ -650,7 +651,7 @@ module Prism
         if bom && lineno == 1
           column -= 3
 
-          if index == 0 && column == 0 && !bom_flushed
+          if index == 0 && column == 0 && !BOM_FLUSHED
             flushed =
               case token.type
               when :BACK_REFERENCE, :INSTANCE_VARIABLE, :CLASS_VARIABLE,
@@ -674,7 +675,7 @@ module Prism
 
         event = RIPPER.fetch(token.type)
         value = token.value
-        lex_state = Ripper::Lexer::State.new(lex_state)
+        lex_state = Translation::Ripper::Lexer::State.new(lex_state)
 
         token =
           case event
@@ -688,7 +689,7 @@ module Prism
             last_heredoc_end = token.location.end_offset
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
           when :on_ident
-            if lex_state == Ripper::EXPR_END
+            if lex_state == Translation::Ripper::EXPR_END
               # If we have an identifier that follows a method name like:
               #
               #     def foo bar
@@ -698,7 +699,7 @@ module Prism
               # yet. We do this more accurately, so we need to allow comparing
               # against both END and END|LABEL.
               ParamToken.new([[lineno, column], event, value, lex_state])
-            elsif lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL
+            elsif lex_state == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL
               # In the event that we're comparing identifiers, we're going to
               # allow a little divergence. Ripper doesn't account for local
               # variables introduced through named captures in regexes, and we
@@ -738,7 +739,7 @@ module Prism
                   counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
                 end
 
-                Ripper::Lexer::State.new(result_value[current_index][1])
+                Translation::Ripper::Lexer::State.new(result_value[current_index][1])
               else
                 previous_state
               end
@@ -866,62 +867,4 @@ module Prism
   end
 
   private_constant :LexCompat
-
-  # This is a class that wraps the Ripper lexer to produce almost exactly the
-  # same tokens.
-  class LexRipper # :nodoc:
-    attr_reader :source
-
-    def initialize(source)
-      @source = source
-    end
-
-    def result
-      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
-      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
-
-      lex(source).each do |token|
-        case token[1]
-        when :on_sp
-          # skip
-        when :on_tstring_content
-          if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
-        when :on_words_sep
-          if previous[1] == :on_words_sep
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
-        else
-          results << token
-          previous = token
-        end
-      end
-
-      results
-    end
-
-    private
-
-    if Ripper.method(:lex).parameters.assoc(:keyrest)
-      def lex(source)
-        Ripper.lex(source, raise_errors: true)
-      end
-    else
-      def lex(source)
-        ripper = Ripper::Lexer.new(source)
-        ripper.lex.tap do |result|
-          raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
-        end
-      end
-    end
-  end
-
-  private_constant :LexRipper
 end

data/lib/prism/lex_ripper.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+require "ripper"
+
+module Prism
+  # This is a class that wraps the Ripper lexer to produce almost exactly the
+  # same tokens.
+  class LexRipper # :nodoc:
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+    end
+
+    def result
+      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
+      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
+
+      lex(source).each do |token|
+        case token[1]
+        when :on_sp
+          # skip
+        when :on_tstring_content
+          if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
+            previous[2] << token[2]
+          else
+            results << token
+            previous = token
+          end
+        when :on_words_sep
+          if previous[1] == :on_words_sep
+            previous[2] << token[2]
+          else
+            results << token
+            previous = token
+          end
+        else
+          results << token
+          previous = token
+        end
+      end
+
+      results
+    end
+
+    private
+
+    if Ripper.method(:lex).parameters.assoc(:keyrest)
+      def lex(source)
+        Ripper.lex(source, raise_errors: true)
+      end
+    else
+      def lex(source)
+        ripper = Ripper::Lexer.new(source)
+        ripper.lex.tap do |result|
+          raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any?
+        end
+      end
+    end
+  end
+
+  private_constant :LexRipper
+end

data/lib/prism/mutation_compiler.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
+# :markup: markdown
 
 =begin
+--
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/mutation_compiler.rb.erb
 if you are looking to modify the template
+++
 =end
 
 module Prism