prism 0.15.1 → 0.17.0

data/lib/prism/compiler.rb

@@ -291,9 +291,6 @@ module Prism
     # Compile a KeywordHashNode node
     alias visit_keyword_hash_node visit_child_nodes
 
-    # Compile a KeywordParameterNode node
-    alias visit_keyword_parameter_node visit_child_nodes
-
     # Compile a KeywordRestParameterNode node
     alias visit_keyword_rest_parameter_node visit_child_nodes
 
@@ -354,6 +351,9 @@ module Prism
     # Compile a NumberedReferenceReadNode node
     alias visit_numbered_reference_read_node visit_child_nodes
 
+    # Compile a OptionalKeywordParameterNode node
+    alias visit_optional_keyword_parameter_node visit_child_nodes
+
     # Compile a OptionalParameterNode node
     alias visit_optional_parameter_node visit_child_nodes
 
@@ -393,8 +393,8 @@ module Prism
     # Compile a RegularExpressionNode node
     alias visit_regular_expression_node visit_child_nodes
 
-    # Compile a RequiredDestructuredParameterNode node
-    alias visit_required_destructured_parameter_node visit_child_nodes
+    # Compile a RequiredKeywordParameterNode node
+    alias visit_required_keyword_parameter_node visit_child_nodes
 
     # Compile a RequiredParameterNode node
     alias visit_required_parameter_node visit_child_nodes

data/lib/prism/debug.rb

@@ -4,7 +4,9 @@ module Prism
   # This module is used for testing and debugging and is not meant to be used by
   # consumers of this library.
   module Debug
-    class ISeq
+    # A wrapper around a RubyVM::InstructionSequence that provides a more
+    # convenient interface for accessing parts of the iseq.
+    class ISeq # :nodoc:
       attr_reader :parts
 
       def initialize(parts)
@@ -42,6 +44,11 @@ module Prism
       end
     end
 
+    private_constant :ISeq
+
+    # :call-seq:
+    #   Debug::cruby_locals(source) -> Array
+    #
     # For the given source, compiles with CRuby and returns a list of all of the
     # sets of local variables that were encountered.
     def self.cruby_locals(source)
@@ -52,9 +59,21 @@ module Prism
         stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
 
         while (iseq = stack.pop)
-          # For some reason, CRuby occasionally pushes this special local
-          # variable when there are splat arguments. We get rid of that here.
-          locals << (iseq.local_table - [:"#arg_rest"])
+          names = [*iseq.local_table]
+          names.map!.with_index do |name, index|
+            # When an anonymous local variable is present in the iseq's local
+            # table, it is represented as the stack offset from the top.
+            # However, when these are dumped to binary and read back in, they
+            # are replaced with the symbol :#arg_rest. To consistently handle
+            # this, we replace them here with their index.
+            if name == :"#arg_rest"
+              names.length - index + 1
+            else
+              name
+            end
+          end
+
+          locals << names
           iseq.each_child { |child| stack << child }
         end
 
@@ -64,8 +83,16 @@ module Prism
       end
     end
 
+    # Used to hold the place of a local that will be in the local table but
+    # cannot be accessed directly from the source code. For example, the
+    # iteration variable in a for loop or the positional parameter on a method
+    # definition that is destructured.
     AnonymousLocal = Object.new
+    private_constant :AnonymousLocal
 
+    # :call-seq:
+    #   Debug::prism_locals(source) -> Array
+    #
     # For the given source, parses with prism and returns a list of all of the
     # sets of local variables that were encountered.
     def self.prism_locals(source)
@@ -102,19 +129,21 @@ module Prism
                   AnonymousLocal
                 end
               end,
-              *params.keywords.reject(&:value).map(&:name),
-              *params.keywords.select(&:value).map(&:name)
+              *params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
+              *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
             ]
 
             sorted << AnonymousLocal if params.keywords.any?
 
             # Recurse down the parameter tree to find any destructured
             # parameters and add them after the other parameters.
-            param_stack = params.requireds.concat(params.posts).grep(RequiredDestructuredParameterNode).reverse
+            param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
             while (param = param_stack.pop)
               case param
-              when RequiredDestructuredParameterNode
-                param_stack.concat(param.parameters.reverse)
+              when MultiTargetNode
+                param_stack.concat(param.rights.reverse)
+                param_stack << param.rest
+                param_stack.concat(param.lefts.reverse)
               when RequiredParameterNode
                 sorted << param.name
               when SplatNode
@@ -150,12 +179,13 @@ module Prism
       locals
     end
 
+    # :call-seq:
+    #   Debug::newlines(source) -> Array
+    #
+    # For the given source string, return the byte offsets of every newline in
+    # the source.
     def self.newlines(source)
       Prism.parse(source).source.offsets
     end
-
-    def self.parse_serialize_file(filepath)
-      parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
-    end
   end
 end

data/lib/prism/desugar_compiler.rb

@@ -161,7 +161,7 @@ module Prism
           nil,
           node.operator_loc.copy(length: node.operator_loc.length - 1),
           nil,
-          ArgumentsNode.new([node.value], node.value.location),
+          ArgumentsNode.new([node.value], 0, node.value.location),
           nil,
           nil,
           0,

data/lib/prism/dispatcher.rb

@@ -41,23 +41,24 @@ module Prism
     # attr_reader listeners: Hash[Symbol, Array[Listener]]
     attr_reader :listeners
 
+    # Initialize a new dispatcher.
     def initialize
       @listeners = {}
     end
 
-    # Register a listener for one or more events
+    # Register a listener for one or more events.
     #
     # def register: (Listener, *Symbol) -> void
     def register(listener, *events)
       events.each { |event| (listeners[event] ||= []) << listener }
     end
 
-    # Walks `root` dispatching events to all registered listeners
+    # Walks `root` dispatching events to all registered listeners.
     #
     # def dispatch: (Node) -> void
     alias dispatch visit
 
-    # Dispatches a single event for `node` to all registered listeners
+    # Dispatches a single event for `node` to all registered listeners.
     #
     # def dispatch_once: (Node) -> void
     def dispatch_once(node)
@@ -736,14 +737,6 @@ module Prism
       listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
     end
 
-    # Dispatch enter and leave events for KeywordParameterNode nodes and continue
-    # walking the tree.
-    def visit_keyword_parameter_node(node)
-      listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
-      super
-      listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
-    end
-
     # Dispatch enter and leave events for KeywordRestParameterNode nodes and continue
     # walking the tree.
     def visit_keyword_rest_parameter_node(node)
@@ -904,6 +897,14 @@ module Prism
       listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
     end
 
+    # Dispatch enter and leave events for OptionalKeywordParameterNode nodes and continue
+    # walking the tree.
+    def visit_optional_keyword_parameter_node(node)
+      listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
+      super
+      listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
+    end
+
     # Dispatch enter and leave events for OptionalParameterNode nodes and continue
     # walking the tree.
     def visit_optional_parameter_node(node)
@@ -1008,12 +1009,12 @@ module Prism
       listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
     end
 
-    # Dispatch enter and leave events for RequiredDestructuredParameterNode nodes and continue
+    # Dispatch enter and leave events for RequiredKeywordParameterNode nodes and continue
     # walking the tree.
-    def visit_required_destructured_parameter_node(node)
-      listeners[:on_required_destructured_parameter_node_enter]&.each { |listener| listener.on_required_destructured_parameter_node_enter(node) }
+    def visit_required_keyword_parameter_node(node)
+      listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
       super
-      listeners[:on_required_destructured_parameter_node_leave]&.each { |listener| listener.on_required_destructured_parameter_node_leave(node) }
+      listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
     end
 
     # Dispatch enter and leave events for RequiredParameterNode nodes and continue
@@ -1216,7 +1217,7 @@ module Prism
       listeners[:on_yield_node_leave]&.each { |listener| listener.on_yield_node_leave(node) }
     end
 
-    class DispatchOnce < Visitor
+    class DispatchOnce < Visitor # :nodoc:
       attr_reader :listeners
 
       def initialize(listeners)
@@ -1727,12 +1728,6 @@ module Prism
         listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
       end
 
-      # Dispatch enter and leave events for KeywordParameterNode nodes.
-      def visit_keyword_parameter_node(node)
-        listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
-        listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
-      end
-
       # Dispatch enter and leave events for KeywordRestParameterNode nodes.
       def visit_keyword_rest_parameter_node(node)
         listeners[:on_keyword_rest_parameter_node_enter]&.each { |listener| listener.on_keyword_rest_parameter_node_enter(node) }
@@ -1853,6 +1848,12 @@ module Prism
         listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
       end
 
+      # Dispatch enter and leave events for OptionalKeywordParameterNode nodes.
+      def visit_optional_keyword_parameter_node(node)
+        listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
+        listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
+      end
+
       # Dispatch enter and leave events for OptionalParameterNode nodes.
       def visit_optional_parameter_node(node)
         listeners[:on_optional_parameter_node_enter]&.each { |listener| listener.on_optional_parameter_node_enter(node) }
@@ -1931,10 +1932,10 @@ module Prism
         listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
       end
 
-      # Dispatch enter and leave events for RequiredDestructuredParameterNode nodes.
-      def visit_required_destructured_parameter_node(node)
-        listeners[:on_required_destructured_parameter_node_enter]&.each { |listener| listener.on_required_destructured_parameter_node_enter(node) }
-        listeners[:on_required_destructured_parameter_node_leave]&.each { |listener| listener.on_required_destructured_parameter_node_leave(node) }
+      # Dispatch enter and leave events for RequiredKeywordParameterNode nodes.
+      def visit_required_keyword_parameter_node(node)
+        listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
+        listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
       end
 
       # Dispatch enter and leave events for RequiredParameterNode nodes.

data/lib/prism/dsl.rb

@@ -63,8 +63,8 @@ module Prism
     end
 
     # Create a new ArgumentsNode node
-    def ArgumentsNode(arguments, location = Location())
-      ArgumentsNode.new(arguments, location)
+    def ArgumentsNode(arguments, flags, location = Location())
+      ArgumentsNode.new(arguments, flags, location)
     end
 
     # Create a new ArrayNode node
@@ -358,8 +358,8 @@ module Prism
     end
 
     # Create a new HashPatternNode node
-    def HashPatternNode(constant, assocs, kwrest, opening_loc, closing_loc, location = Location())
-      HashPatternNode.new(constant, assocs, kwrest, opening_loc, closing_loc, location)
+    def HashPatternNode(constant, elements, rest, opening_loc, closing_loc, location = Location())
+      HashPatternNode.new(constant, elements, rest, opening_loc, closing_loc, location)
     end
 
     # Create a new IfNode node
@@ -462,11 +462,6 @@ module Prism
       KeywordHashNode.new(elements, location)
     end
 
-    # Create a new KeywordParameterNode node
-    def KeywordParameterNode(name, name_loc, value, location = Location())
-      KeywordParameterNode.new(name, name_loc, value, location)
-    end
-
     # Create a new KeywordRestParameterNode node
     def KeywordRestParameterNode(name, name_loc, operator_loc, location = Location())
       KeywordRestParameterNode.new(name, name_loc, operator_loc, location)
@@ -538,13 +533,13 @@ module Prism
     end
 
     # Create a new MultiTargetNode node
-    def MultiTargetNode(targets, lparen_loc, rparen_loc, location = Location())
-      MultiTargetNode.new(targets, lparen_loc, rparen_loc, location)
+    def MultiTargetNode(lefts, rest, rights, lparen_loc, rparen_loc, location = Location())
+      MultiTargetNode.new(lefts, rest, rights, lparen_loc, rparen_loc, location)
     end
 
     # Create a new MultiWriteNode node
-    def MultiWriteNode(targets, lparen_loc, rparen_loc, operator_loc, value, location = Location())
-      MultiWriteNode.new(targets, lparen_loc, rparen_loc, operator_loc, value, location)
+    def MultiWriteNode(lefts, rest, rights, lparen_loc, rparen_loc, operator_loc, value, location = Location())
+      MultiWriteNode.new(lefts, rest, rights, lparen_loc, rparen_loc, operator_loc, value, location)
     end
 
     # Create a new NextNode node
@@ -567,6 +562,11 @@ module Prism
       NumberedReferenceReadNode.new(number, location)
     end
 
+    # Create a new OptionalKeywordParameterNode node
+    def OptionalKeywordParameterNode(name, name_loc, value, location = Location())
+      OptionalKeywordParameterNode.new(name, name_loc, value, location)
+    end
+
     # Create a new OptionalParameterNode node
     def OptionalParameterNode(name, name_loc, operator_loc, value, location = Location())
       OptionalParameterNode.new(name, name_loc, operator_loc, value, location)
@@ -632,9 +632,9 @@ module Prism
       RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
     end
 
-    # Create a new RequiredDestructuredParameterNode node
-    def RequiredDestructuredParameterNode(parameters, opening_loc, closing_loc, location = Location())
-      RequiredDestructuredParameterNode.new(parameters, opening_loc, closing_loc, location)
+    # Create a new RequiredKeywordParameterNode node
+    def RequiredKeywordParameterNode(name, name_loc, location = Location())
+      RequiredKeywordParameterNode.new(name, name_loc, location)
     end
 
     # Create a new RequiredParameterNode node

data/lib/prism/ffi.rb

@@ -9,7 +9,7 @@ require "ffi"
 module Prism
   BACKEND = :FFI
 
-  module LibRubyParser
+  module LibRubyParser # :nodoc:
     extend FFI::Library
 
     # Define the library that we will be pulling functions from. Note that this
@@ -69,9 +69,10 @@ module Prism
     load_exported_functions_from(
       "prism.h",
       "pm_version",
-      "pm_parse_serialize",
-      "pm_lex_serialize",
-      "pm_parse_lex_serialize"
+      "pm_serialize_parse",
+      "pm_serialize_parse_comments",
+      "pm_serialize_lex",
+      "pm_serialize_parse_lex"
     )
 
     load_exported_functions_from(
@@ -94,7 +95,7 @@ module Prism
 
     # This object represents a pm_buffer_t. We only use it as an opaque pointer,
     # so it doesn't need to know the fields of pm_buffer_t.
-    class PrismBuffer
+    class PrismBuffer # :nodoc:
       SIZEOF = LibRubyParser.pm_buffer_sizeof
 
       attr_reader :pointer
@@ -132,7 +133,7 @@ module Prism
 
     # This object represents a pm_string_t. We only use it as an opaque pointer,
     # so it doesn't have to be an FFI::Struct.
-    class PrismString
+    class PrismString # :nodoc:
       SIZEOF = LibRubyParser.pm_string_sizeof
 
       attr_reader :pointer
@@ -166,14 +167,6 @@ module Prism
         end
       end
     end
-
-    def self.dump_internal(source, source_size, filepath)
-      PrismBuffer.with do |buffer|
-        metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-        pm_parse_serialize(source, source_size, buffer.pointer, metadata)
-        buffer.read
-      end
-    end
   end
 
   # Mark the LibRubyParser module as private as it should only be called through
@@ -183,69 +176,153 @@ module Prism
   # The version constant is set by reading the result of calling pm_version.
   VERSION = LibRubyParser.pm_version.read_string
 
-  # Mirror the Prism.dump API by using the serialization API.
-  def self.dump(code, filepath = nil)
-    LibRubyParser.dump_internal(code, code.bytesize, filepath)
-  end
+  class << self
+    # Mirror the Prism.dump API by using the serialization API.
+    def dump(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
+        buffer.read
+      end
+    end
 
-  # Mirror the Prism.dump_file API by using the serialization API.
-  def self.dump_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      LibRubyParser.dump_internal(string.source, string.length, filepath)
+    # Mirror the Prism.dump_file API by using the serialization API.
+    def dump_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        dump(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.lex API by using the serialization API.
-  def self.lex(code, filepath = nil)
-    LibRubyParser::PrismBuffer.with do |buffer|
-      LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
-      Serialize.load_tokens(Source.new(code), buffer.read)
+    # Mirror the Prism.lex API by using the serialization API.
+    def lex(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
+        Serialize.load_tokens(Source.new(code), buffer.read)
+      end
     end
-  end
 
-  # Mirror the Prism.lex_file API by using the serialization API.
-  def self.lex_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      lex(string.read, filepath)
+    # Mirror the Prism.lex_file API by using the serialization API.
+    def lex_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        lex(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.parse API by using the serialization API.
-  def self.parse(code, filepath = nil)
-    Prism.load(code, dump(code, filepath))
-  end
+    # Mirror the Prism.parse API by using the serialization API.
+    def parse(code, **options)
+      Prism.load(code, dump(code, **options))
+    end
 
-  # Mirror the Prism.parse_file API by using the serialization API. This uses
-  # native strings instead of Ruby strings because it allows us to use mmap when
-  # it is available.
-  def self.parse_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      parse(string.read, filepath)
+    # Mirror the Prism.parse_file API by using the serialization API. This uses
+    # native strings instead of Ruby strings because it allows us to use mmap when
+    # it is available.
+    def parse_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.parse_lex API by using the serialization API.
-  def self.parse_lex(code, filepath = nil)
-    LibRubyParser::PrismBuffer.with do |buffer|
-      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
+    # Mirror the Prism.parse_comments API by using the serialization API.
+    def parse_comments(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_serialize_parse_comments(buffer.pointer, code, code.bytesize, dump_options(options))
 
-      source = Source.new(code)
-      loader = Serialize::Loader.new(source, buffer.read)
+        source = Source.new(code)
+        loader = Serialize::Loader.new(source, buffer.read)
 
-      tokens = loader.load_tokens
-      node, comments, magic_comments, errors, warnings = loader.load_nodes
+        loader.load_header
+        loader.load_force_encoding
+        loader.load_start_line
+        loader.load_comments
+      end
+    end
+
+    # Mirror the Prism.parse_file_comments API by using the serialization
+    # API. This uses native strings instead of Ruby strings because it allows us
+    # to use mmap when it is available.
+    def parse_file_comments(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse_comments(string.read, **options, filepath: filepath)
+      end
+    end
 
-      tokens.each { |token,| token.value.force_encoding(loader.encoding) }
+    # Mirror the Prism.parse_lex API by using the serialization API.
+    def parse_lex(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_serialize_parse_lex(buffer.pointer, code, code.bytesize, dump_options(options))
 
-      ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
+        source = Source.new(code)
+        loader = Serialize::Loader.new(source, buffer.read)
+
+        tokens = loader.load_tokens
+        node, comments, magic_comments, errors, warnings = loader.load_nodes
+        tokens.each { |token,| token.value.force_encoding(loader.encoding) }
+
+        ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
+      end
     end
-  end
 
-  # Mirror the Prism.parse_lex_file API by using the serialization API.
-  def self.parse_lex_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      parse_lex(string.read, filepath)
+    # Mirror the Prism.parse_lex_file API by using the serialization API.
+    def parse_lex_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse_lex(string.read, **options, filepath: filepath)
+      end
+    end
+
+    private
+
+    # Convert the given options into a serialized options string.
+    def dump_options(options)
+      template = +""
+      values = []
+
+      template << "L"
+      if (filepath = options[:filepath])
+        values.push(filepath.bytesize, filepath.b)
+        template << "A*"
+      else
+        values << 0
+      end
+
+      template << "L"
+      values << options.fetch(:line, 1)
+
+      template << "L"
+      if (encoding = options[:encoding])
+        name = encoding.name
+        values.push(name.bytesize, name.b)
+        template << "A*"
+      else
+        values << 0
+      end
+
+      template << "C"
+      values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
+
+      template << "C"
+      values << (options[:verbose] ? 0 : 1)
+
+      template << "L"
+      if (scopes = options[:scopes])
+        values << scopes.length
+
+        scopes.each do |scope|
+          template << "L"
+          values << scope.length
+
+          scope.each do |local|
+            name = local.name
+            template << "L"
+            values << name.bytesize
+
+            template << "A*"
+            values << name.b
+          end
+        end
+      else
+        values << 0
+      end
+
+      values.pack(template)
     end
   end
 end