prism 0.24.0 → 0.25.0

data/lib/prism/dot_visitor.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dot_visitor.rb.erb
@@ -24,7 +25,7 @@ module Prism
         if port
           "<tr><td align=\"left\" colspan=\"2\" port=\"#{name}\">#{name}</td></tr>"
         else
-          "<tr><td align=\"left\">#{name}</td><td>#{CGI.escapeHTML(value)}</td></tr>"
+          "<tr><td align=\"left\">#{name}</td><td>#{CGI.escapeHTML(value || raise)}</td></tr>"
         end
       end
     end
@@ -1155,9 +1156,7 @@ module Prism
       digraph.edge("#{id}:value -> #{node_id(node.value)};")
 
       # operator_loc
-      unless (operator_loc = node.operator_loc).nil?
-        table.field("operator_loc", location_inspect(operator_loc))
-      end
+      table.field("operator_loc", location_inspect(node.operator_loc))
 
       digraph.nodes << <<~DOT
         #{id} [
@@ -1773,6 +1772,9 @@ module Prism
       table = Table.new("FloatNode")
       id = node_id(node)
 
+      # value
+      table.field("value", node.value.inspect)
+
       digraph.nodes << <<~DOT
         #{id} [
           label=<#{table.to_dot.gsub(/\n/, "\n  ")}>
@@ -2580,6 +2582,9 @@ module Prism
       # flags
       table.field("flags", integer_base_flags_inspect(node))
 
+      # value
+      table.field("value", node.value.inspect)
+
       digraph.nodes << <<~DOT
         #{id} [
           label=<#{table.to_dot.gsub(/\n/, "\n  ")}>
@@ -2666,6 +2671,9 @@ module Prism
       table = Table.new("InterpolatedStringNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", interpolated_string_node_flags_inspect(node))
+
       # opening_loc
       unless (opening_loc = node.opening_loc).nil?
         table.field("opening_loc", location_inspect(opening_loc))
@@ -2768,6 +2776,20 @@ module Prism
       super
     end
 
+    # Visit a ItParametersNode node.
+    def visit_it_parameters_node(node)
+      table = Table.new("ItParametersNode")
+      id = node_id(node)
+
+      digraph.nodes << <<~DOT
+        #{id} [
+          label=<#{table.to_dot.gsub(/\n/, "\n  ")}>
+        ];
+      DOT
+
+      super
+    end
+
     # Visit a KeywordHashNode node.
     def visit_keyword_hash_node(node)
       table = Table.new("KeywordHashNode")
@@ -4009,6 +4031,27 @@ module Prism
       super
     end
 
+    # Visit a ShareableConstantNode node.
+    def visit_shareable_constant_node(node)
+      table = Table.new("ShareableConstantNode")
+      id = node_id(node)
+
+      # flags
+      table.field("flags", shareable_constant_node_flags_inspect(node))
+
+      # write
+      table.field("write", port: true)
+      digraph.edge("#{id}:write -> #{node_id(node.write)};")
+
+      digraph.nodes << <<~DOT
+        #{id} [
+          label=<#{table.to_dot.gsub(/\n/, "\n  ")}>
+        ];
+      DOT
+
+      super
+    end
+
     # Visit a SingletonClassNode node.
     def visit_singleton_class_node(node)
       table = Table.new("SingletonClassNode")
@@ -4064,6 +4107,9 @@ module Prism
       table = Table.new("SourceFileNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", string_flags_inspect(node))
+
       # filepath
       table.field("filepath", node.filepath.inspect)
 
@@ -4390,6 +4436,11 @@ module Prism
         table.field("conditions", "[]")
       end
 
+      # then_keyword_loc
+      unless (then_keyword_loc = node.then_keyword_loc).nil?
+        table.field("then_keyword_loc", location_inspect(then_keyword_loc))
+      end
+
       # statements
       unless (statements = node.statements).nil?
         table.field("statements", port: true)
@@ -4517,7 +4568,7 @@ module Prism
     # Inspect a node that has arguments_node_flags flags to display the flags as a
     # comma-separated list.
     def arguments_node_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "contains_keyword_splat" if node.contains_keyword_splat?
       flags.join(", ")
     end
@@ -4525,7 +4576,7 @@ module Prism
     # Inspect a node that has array_node_flags flags to display the flags as a
     # comma-separated list.
     def array_node_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "contains_splat" if node.contains_splat?
       flags.join(", ")
     end
@@ -4533,7 +4584,7 @@ module Prism
     # Inspect a node that has call_node_flags flags to display the flags as a
     # comma-separated list.
     def call_node_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "safe_navigation" if node.safe_navigation?
       flags << "variable_call" if node.variable_call?
       flags << "attribute_write" if node.attribute_write?
@@ -4544,7 +4595,7 @@ module Prism
     # Inspect a node that has encoding_flags flags to display the flags as a
     # comma-separated list.
     def encoding_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "forced_utf8_encoding" if node.forced_utf8_encoding?
       flags << "forced_binary_encoding" if node.forced_binary_encoding?
       flags.join(", ")
@@ -4553,7 +4604,7 @@ module Prism
     # Inspect a node that has integer_base_flags flags to display the flags as a
     # comma-separated list.
     def integer_base_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "binary" if node.binary?
       flags << "decimal" if node.decimal?
       flags << "octal" if node.octal?
@@ -4561,10 +4612,19 @@ module Prism
       flags.join(", ")
     end
 
+    # Inspect a node that has interpolated_string_node_flags flags to display the flags as a
+    # comma-separated list.
+    def interpolated_string_node_flags_inspect(node)
+      flags = [] #: Array[String]
+      flags << "frozen" if node.frozen?
+      flags << "mutable" if node.mutable?
+      flags.join(", ")
+    end
+
     # Inspect a node that has keyword_hash_node_flags flags to display the flags as a
     # comma-separated list.
     def keyword_hash_node_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "symbol_keys" if node.symbol_keys?
       flags.join(", ")
     end
@@ -4572,7 +4632,7 @@ module Prism
     # Inspect a node that has loop_flags flags to display the flags as a
     # comma-separated list.
     def loop_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "begin_modifier" if node.begin_modifier?
       flags.join(", ")
     end
@@ -4580,7 +4640,7 @@ module Prism
     # Inspect a node that has parameter_flags flags to display the flags as a
     # comma-separated list.
     def parameter_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "repeated_parameter" if node.repeated_parameter?
       flags.join(", ")
     end
@@ -4588,7 +4648,7 @@ module Prism
     # Inspect a node that has range_flags flags to display the flags as a
     # comma-separated list.
     def range_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "exclude_end" if node.exclude_end?
       flags.join(", ")
     end
@@ -4596,7 +4656,7 @@ module Prism
     # Inspect a node that has regular_expression_flags flags to display the flags as a
     # comma-separated list.
     def regular_expression_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "ignore_case" if node.ignore_case?
       flags << "extended" if node.extended?
       flags << "multi_line" if node.multi_line?
@@ -4611,20 +4671,31 @@ module Prism
       flags.join(", ")
     end
 
+    # Inspect a node that has shareable_constant_node_flags flags to display the flags as a
+    # comma-separated list.
+    def shareable_constant_node_flags_inspect(node)
+      flags = [] #: Array[String]
+      flags << "literal" if node.literal?
+      flags << "experimental_everything" if node.experimental_everything?
+      flags << "experimental_copy" if node.experimental_copy?
+      flags.join(", ")
+    end
+
     # Inspect a node that has string_flags flags to display the flags as a
     # comma-separated list.
     def string_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "forced_utf8_encoding" if node.forced_utf8_encoding?
       flags << "forced_binary_encoding" if node.forced_binary_encoding?
       flags << "frozen" if node.frozen?
+      flags << "mutable" if node.mutable?
       flags.join(", ")
     end
 
     # Inspect a node that has symbol_flags flags to display the flags as a
     # comma-separated list.
     def symbol_flags_inspect(node)
-      flags = []
+      flags = [] #: Array[String]
       flags << "forced_utf8_encoding" if node.forced_utf8_encoding?
       flags << "forced_binary_encoding" if node.forced_binary_encoding?
       flags << "forced_us_ascii_encoding" if node.forced_us_ascii_encoding?

data/lib/prism/dsl.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dsl.rb.erb
@@ -15,6 +16,7 @@ module Prism
   #       [
   #         Prism::IntegerNode.new(
   #           Prism::IntegerBaseFlags::DECIMAL,
+  #           1,
   #           Prism::Location.new(source, 1, 1),
   #           source
   #         )
@@ -29,7 +31,7 @@ module Prism
   #     source = Prism::Source.new("[1]")
   #
   #     ArrayNode(
-  #       IntegerNode(Prism::IntegerBaseFlags::DECIMAL, Location(source, 1, 1)), source),
+  #       IntegerNode(Prism::IntegerBaseFlags::DECIMAL, 1, Location(source, 1, 1)), source),
   #       Location(source, 0, 1),
   #       Location(source, 2, 1),
   #       source
@@ -42,7 +44,7 @@ module Prism
 
     # Create a new Location object
     def Location(source = nil, start_offset = 0, length = 0)
-      Location.new(source, start_offset, length)
+      Location.new(source, start_offset, length) # steep:ignore
     end
 
     # Create a new AliasGlobalVariableNode node
@@ -311,8 +313,8 @@ module Prism
     end
 
     # Create a new FloatNode node
-    def FloatNode(source = nil, location = Location())
-      FloatNode.new(source, location)
+    def FloatNode(value, source = nil, location = Location())
+      FloatNode.new(source, value, location)
     end
 
     # Create a new ForNode node
@@ -451,8 +453,8 @@ module Prism
     end
 
     # Create a new IntegerNode node
-    def IntegerNode(flags, source = nil, location = Location())
-      IntegerNode.new(source, flags, location)
+    def IntegerNode(flags, value, source = nil, location = Location())
+      IntegerNode.new(source, flags, value, location)
     end
 
     # Create a new InterpolatedMatchLastLineNode node
@@ -466,8 +468,8 @@ module Prism
     end
 
     # Create a new InterpolatedStringNode node
-    def InterpolatedStringNode(opening_loc, parts, closing_loc, source = nil, location = Location())
-      InterpolatedStringNode.new(source, opening_loc, parts, closing_loc, location)
+    def InterpolatedStringNode(flags, opening_loc, parts, closing_loc, source = nil, location = Location())
+      InterpolatedStringNode.new(source, flags, opening_loc, parts, closing_loc, location)
     end
 
     # Create a new InterpolatedSymbolNode node
@@ -480,6 +482,11 @@ module Prism
       InterpolatedXStringNode.new(source, opening_loc, parts, closing_loc, location)
     end
 
+    # Create a new ItParametersNode node
+    def ItParametersNode(source = nil, location = Location())
+      ItParametersNode.new(source, location)
+    end
+
     # Create a new KeywordHashNode node
     def KeywordHashNode(flags, elements, source = nil, location = Location())
       KeywordHashNode.new(source, flags, elements, location)
@@ -700,6 +707,11 @@ module Prism
       SelfNode.new(source, location)
     end
 
+    # Create a new ShareableConstantNode node
+    def ShareableConstantNode(flags, write, source = nil, location = Location())
+      ShareableConstantNode.new(source, flags, write, location)
+    end
+
     # Create a new SingletonClassNode node
     def SingletonClassNode(locals, class_keyword_loc, operator_loc, expression, body, end_keyword_loc, source = nil, location = Location())
       SingletonClassNode.new(source, locals, class_keyword_loc, operator_loc, expression, body, end_keyword_loc, location)
@@ -711,8 +723,8 @@ module Prism
     end
 
     # Create a new SourceFileNode node
-    def SourceFileNode(filepath, source = nil, location = Location())
-      SourceFileNode.new(source, filepath, location)
+    def SourceFileNode(flags, filepath, source = nil, location = Location())
+      SourceFileNode.new(source, flags, filepath, location)
     end
 
     # Create a new SourceLineNode node
@@ -766,8 +778,8 @@ module Prism
     end
 
     # Create a new WhenNode node
-    def WhenNode(keyword_loc, conditions, statements, source = nil, location = Location())
-      WhenNode.new(source, keyword_loc, conditions, statements, location)
+    def WhenNode(keyword_loc, conditions, then_keyword_loc, statements, source = nil, location = Location())
+      WhenNode.new(source, keyword_loc, conditions, then_keyword_loc, statements, location)
     end
 
     # Create a new WhileNode node

data/lib/prism/ffi.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# typed: ignore
 
 # This file is responsible for mirroring the API provided by the C extension by
 # using FFI to call into the shared library.
@@ -7,8 +8,6 @@ require "rbconfig"
 require "ffi"
 
 module Prism
-  BACKEND = :FFI
-
   module LibRubyParser # :nodoc:
     extend FFI::Library
 
@@ -24,15 +23,21 @@ module Prism
     #     size_t       -> :size_t
     #     void         -> :void
     #
-    def self.resolve_type(type)
+    def self.resolve_type(type, callbacks)
       type = type.strip
-      type.end_with?("*") ? :pointer : type.delete_prefix("const ").to_sym
+
+      if !type.end_with?("*")
+        type.delete_prefix("const ").to_sym
+      else
+        type = type.delete_suffix("*").rstrip
+        callbacks.include?(type.to_sym) ? type.to_sym : :pointer
+      end
     end
 
     # Read through the given header file and find the declaration of each of the
     # given functions. For each one, define a function with the same name and
     # signature as the C function.
-    def self.load_exported_functions_from(header, *functions)
+    def self.load_exported_functions_from(header, *functions, callbacks)
       File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line|
         # We only want to attempt to load exported functions.
         next unless line.start_with?("PRISM_EXPORTED_FUNCTION ")
@@ -56,24 +61,28 @@ module Prism
 
         # Resolve the type of the argument by dropping the name of the argument
         # first if it is present.
-        arg_types.map! { |type| resolve_type(type.sub(/\w+$/, "")) }
+        arg_types.map! { |type| resolve_type(type.sub(/\w+$/, ""), callbacks) }
 
         # Attach the function using the FFI library.
-        attach_function name, arg_types, resolve_type(return_type)
+        attach_function name, arg_types, resolve_type(return_type, [])
       end
 
       # If we didn't find all of the functions, raise an error.
       raise "Could not find functions #{functions.inspect}" unless functions.empty?
     end
 
+    callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
+
     load_exported_functions_from(
       "prism.h",
       "pm_version",
       "pm_serialize_parse",
+      "pm_serialize_parse_stream",
       "pm_serialize_parse_comments",
       "pm_serialize_lex",
       "pm_serialize_parse_lex",
-      "pm_parse_success_p"
+      "pm_parse_success_p",
+      [:pm_parse_stream_fgets_t]
     )
 
     load_exported_functions_from(
@@ -82,7 +91,8 @@ module Prism
       "pm_buffer_init",
       "pm_buffer_value",
       "pm_buffer_length",
-      "pm_buffer_free"
+      "pm_buffer_free",
+      []
     )
 
     load_exported_functions_from(
@@ -91,7 +101,8 @@ module Prism
       "pm_string_free",
       "pm_string_source",
       "pm_string_length",
-      "pm_string_sizeof"
+      "pm_string_sizeof",
+      []
     )
 
     # This object represents a pm_buffer_t. We only use it as an opaque pointer,
@@ -216,13 +227,36 @@ module Prism
     end
 
     # Mirror the Prism.parse_file API by using the serialization API. This uses
-    # native strings instead of Ruby strings because it allows us to use mmap when
-    # it is available.
+    # native strings instead of Ruby strings because it allows us to use mmap
+    # when it is available.
     def parse_file(filepath, **options)
       options[:filepath] = filepath
       LibRubyParser::PrismString.with_file(filepath) { |string| parse_common(string, string.read, options) }
     end
 
+    # Mirror the Prism.parse_stream API by using the serialization API.
+    def parse_stream(stream, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        source = +""
+        callback = -> (string, size, _) {
+          raise "Expected size to be >= 0, got: #{size}" if size <= 0
+
+          if !(line = stream.gets(size - 1)).nil?
+            source << line
+            string.write_string("#{line}\x00", line.bytesize + 1)
+          end
+        }
+
+        # In the pm_serialize_parse_stream function it accepts a pointer to the
+        # IO object as a void* and then passes it through to the callback as the
+        # third argument, but it never touches it itself. As such, since we have
+        # access to the IO object already through the closure of the lambda, we
+        # can pass a null pointer here and not worry.
+        LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
+        Prism.load(source, buffer.read)
+      end
+    end
+
     # Mirror the Prism.parse_comments API by using the serialization API.
     def parse_comments(code, **options)
       LibRubyParser::PrismString.with_string(code) { |string| parse_comments_common(string, code, options) }
@@ -314,6 +348,24 @@ module Prism
       LibRubyParser.pm_parse_success_p(string.pointer, string.length, dump_options(options))
     end
 
+    # Return the value that should be dumped for the command_line option.
+    def dump_options_command_line(options)
+      command_line = options.fetch(:command_line, "")
+      raise ArgumentError, "command_line must be a string" unless command_line.is_a?(String)
+
+      command_line.each_char.inject(0) do |value, char|
+        case char
+        when "a" then value | 0b000001
+        when "e" then value | 0b000010
+        when "l" then value | 0b000100
+        when "n" then value | 0b001000
+        when "p" then value | 0b010000
+        when "x" then value | 0b100000
+        else raise ArgumentError, "invalid command_line option: #{char}"
+        end
+      end
+    end
+
     # Convert the given options into a serialized options string.
     def dump_options(options)
       template = +""
@@ -342,6 +394,9 @@ module Prism
       template << "C"
       values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
 
+      template << "C"
+      values << dump_options_command_line(options)
+
       template << "C"
       values << { nil => 0, "3.3.0" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version])
 

data/lib/prism/lex_compat.rb

@@ -185,6 +185,8 @@ module Prism
     # However, we add a couple of convenience methods onto them to make them a
     # little easier to work with. We delegate all other methods to the array.
     class Token < SimpleDelegator
+      # @dynamic initialize, each, []
+
       # The location of the token in the source.
       def location
         self[0]
@@ -241,10 +243,10 @@ module Prism
       def ==(other) # :nodoc:
         return false unless self[0...-1] == other[0...-1]
 
-        if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
-          other[4] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED > 0
+        if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
+          other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0
         else
-          self[4] == other[4]
+          self[3] == other[3]
         end
       end
     end
@@ -308,7 +310,7 @@ module Prism
         def to_a
           embexpr_balance = 0
 
-          tokens.each_with_object([]) do |token, results|
+          tokens.each_with_object([]) do |token, results| #$ Array[Token]
             case token.event
             when :on_embexpr_beg
               embexpr_balance += 1
@@ -409,7 +411,7 @@ module Prism
           # If every line in the heredoc is blank, we still need to split up the
           # string content token into multiple tokens.
           if dedent.nil?
-            results = []
+            results = [] #: Array[Token]
             embexpr_balance = 0
 
             tokens.each do |token|
@@ -444,7 +446,7 @@ module Prism
           # If the minimum common whitespace is 0, then we need to concatenate
           # string nodes together that are immediately adjacent.
           if dedent == 0
-            results = []
+            results = [] #: Array[Token]
             embexpr_balance = 0
 
             index = 0
@@ -477,7 +479,7 @@ module Prism
           # insert on_ignored_sp tokens for the amount of dedent that we need to
           # perform. We also need to remove the dedent from the beginning of
           # each line of plain string content tokens.
-          results = []
+          results = [] #: Array[Token]
           dedent_next = true
           embexpr_balance = 0
 
@@ -526,7 +528,7 @@ module Prism
                   # dedent from the beginning of the line.
                   if (dedent > 0) && (dedent_next || index > 0)
                     deleting = 0
-                    deleted_chars = []
+                    deleted_chars = [] #: Array[String]
 
                     # Gather up all of the characters that we're going to
                     # delete, stopping when you hit a character that would put
@@ -603,15 +605,15 @@ module Prism
     end
 
     def result
-      tokens = []
+      tokens = [] #: Array[LexCompat::Token]
 
       state = :default
-      heredoc_stack = [[]]
+      heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]]
 
       result = Prism.lex(source, **options)
       result_value = result.value
-      previous_state = nil
-      last_heredoc_end = nil
+      previous_state = nil #: Ripper::Lexer::State?
+      last_heredoc_end = nil #: Integer?
 
       # In previous versions of Ruby, Ripper wouldn't flush the bom before the
       # first token, so we had to have a hack in place to account for that. This
@@ -842,7 +844,7 @@ module Prism
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, [])
+      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source))
     end
   end
 
@@ -858,8 +860,8 @@ module Prism
     end
 
     def result
-      previous = []
-      results = []
+      previous = [] #: [[Integer, Integer], Symbol, String, untyped] | []
+      results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]]
 
       lex(source).each do |token|
         case token[1]

data/lib/prism/mutation_compiler.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/mutation_compiler.rb.erb
@@ -445,6 +446,11 @@ module Prism
       node.copy(parts: visit_all(node.parts))
     end
 
+    # Copy a ItParametersNode node
+    def visit_it_parameters_node(node)
+      node.copy
+    end
+
     # Copy a KeywordHashNode node
     def visit_keyword_hash_node(node)
       node.copy(elements: visit_all(node.elements))
@@ -665,6 +671,11 @@ module Prism
       node.copy
     end
 
+    # Copy a ShareableConstantNode node
+    def visit_shareable_constant_node(node)
+      node.copy(write: visit(node.write))
+    end
+
     # Copy a SingletonClassNode node
     def visit_singleton_class_node(node)
       node.copy(expression: visit(node.expression), body: visit(node.body))