prism 0.13.0 → 0.15.0

data/lib/prism/debug.rb

@@ -45,29 +45,16 @@ module Prism
     # For the given source, compiles with CRuby and returns a list of all of the
     # sets of local variables that were encountered.
     def self.cruby_locals(source)
-      verbose = $VERBOSE
-      $VERBOSE = nil
+      verbose, $VERBOSE = $VERBOSE, nil
 
       begin
         locals = []
         stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
 
         while (iseq = stack.pop)
-          if iseq.type != :once
-            names = iseq.local_table
-
-            # CRuby will push on a special local variable when there are keyword
-            # arguments. We get rid of that here.
-            names = names.grep_v(Integer)
-
-            # For some reason, CRuby occasionally pushes this special local
-            # variable when there are splat arguments. We get rid of that here.
-            names = names.grep_v(:"#arg_rest")
-
-            # Now push them onto the list of locals.
-            locals << names
-          end
-
+          # For some reason, CRuby occasionally pushes this special local
+          # variable when there are splat arguments. We get rid of that here.
+          locals << (iseq.local_table - [:"#arg_rest"])
           iseq.each_child { |child| stack << child }
         end
 
@@ -77,6 +64,8 @@ module Prism
       end
     end
 
+    AnonymousLocal = Object.new
+
     # For the given source, parses with prism and returns a list of all of the
     # sets of local variables that were encountered.
     def self.prism_locals(source)
@@ -97,20 +86,27 @@ module Prism
           # order here so that we can compare properly.
           if params
             sorted = [
-              *params.requireds.grep(RequiredParameterNode).map(&:name),
+              *params.requireds.map do |required|
+                if required.is_a?(RequiredParameterNode)
+                  required.name
+                else
+                  AnonymousLocal
+                end
+              end,
               *params.optionals.map(&:name),
               *((params.rest.name || :*) if params.rest && params.rest.operator != ","),
-              *params.posts.grep(RequiredParameterNode).map(&:name),
+              *params.posts.map do |post|
+                if post.is_a?(RequiredParameterNode)
+                  post.name
+                else
+                  AnonymousLocal
+                end
+              end,
               *params.keywords.reject(&:value).map(&:name),
               *params.keywords.select(&:value).map(&:name)
             ]
 
-            # TODO: When we get a ... parameter, we should be pushing * and &
-            # onto the local list. We don't do that yet, so we need to add them
-            # in here.
-            if params.keyword_rest.is_a?(ForwardingParameterNode)
-              sorted.push(:*, :&, :"...")
-            end
+            sorted << AnonymousLocal if params.keywords.any?
 
             # Recurse down the parameter tree to find any destructured
             # parameters and add them after the other parameters.
@@ -129,11 +125,19 @@ module Prism
             names = sorted.concat(names - sorted)
           end
 
+          names.map!.with_index do |name, index|
+            if name == AnonymousLocal
+              names.length - index + 1
+            else
+              name
+            end
+          end
+
           locals << names
         when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
           locals << node.locals
         when ForNode
-          locals << []
+          locals << [2]
         when PostExecutionNode
           locals.push([], [])
         when InterpolatedRegularExpressionNode

data/lib/prism/dispatcher.rb

@@ -608,6 +608,30 @@ module Prism
       listeners[:on_in_node_leave]&.each { |listener| listener.on_in_node_leave(node) }
     end
 
+    # Dispatch enter and leave events for IndexAndWriteNode nodes and continue
+    # walking the tree.
+    def visit_index_and_write_node(node)
+      listeners[:on_index_and_write_node_enter]&.each { |listener| listener.on_index_and_write_node_enter(node) }
+      super
+      listeners[:on_index_and_write_node_leave]&.each { |listener| listener.on_index_and_write_node_leave(node) }
+    end
+
+    # Dispatch enter and leave events for IndexOperatorWriteNode nodes and continue
+    # walking the tree.
+    def visit_index_operator_write_node(node)
+      listeners[:on_index_operator_write_node_enter]&.each { |listener| listener.on_index_operator_write_node_enter(node) }
+      super
+      listeners[:on_index_operator_write_node_leave]&.each { |listener| listener.on_index_operator_write_node_leave(node) }
+    end
+
+    # Dispatch enter and leave events for IndexOrWriteNode nodes and continue
+    # walking the tree.
+    def visit_index_or_write_node(node)
+      listeners[:on_index_or_write_node_enter]&.each { |listener| listener.on_index_or_write_node_enter(node) }
+      super
+      listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
+    end
+
     # Dispatch enter and leave events for InstanceVariableAndWriteNode nodes and continue
     # walking the tree.
     def visit_instance_variable_and_write_node(node)
@@ -1607,6 +1631,24 @@ module Prism
         listeners[:on_in_node_leave]&.each { |listener| listener.on_in_node_leave(node) }
       end
 
+      # Dispatch enter and leave events for IndexAndWriteNode nodes.
+      def visit_index_and_write_node(node)
+        listeners[:on_index_and_write_node_enter]&.each { |listener| listener.on_index_and_write_node_enter(node) }
+        listeners[:on_index_and_write_node_leave]&.each { |listener| listener.on_index_and_write_node_leave(node) }
+      end
+
+      # Dispatch enter and leave events for IndexOperatorWriteNode nodes.
+      def visit_index_operator_write_node(node)
+        listeners[:on_index_operator_write_node_enter]&.each { |listener| listener.on_index_operator_write_node_enter(node) }
+        listeners[:on_index_operator_write_node_leave]&.each { |listener| listener.on_index_operator_write_node_leave(node) }
+      end
+
+      # Dispatch enter and leave events for IndexOrWriteNode nodes.
+      def visit_index_or_write_node(node)
+        listeners[:on_index_or_write_node_enter]&.each { |listener| listener.on_index_or_write_node_enter(node) }
+        listeners[:on_index_or_write_node_leave]&.each { |listener| listener.on_index_or_write_node_leave(node) }
+      end
+
       # Dispatch enter and leave events for InstanceVariableAndWriteNode nodes.
       def visit_instance_variable_and_write_node(node)
         listeners[:on_instance_variable_and_write_node_enter]&.each { |listener| listener.on_instance_variable_and_write_node_enter(node) }

data/lib/prism/dsl.rb

@@ -88,8 +88,8 @@ module Prism
     end
 
     # Create a new BackReferenceReadNode node
-    def BackReferenceReadNode(location = Location())
-      BackReferenceReadNode.new(location)
+    def BackReferenceReadNode(name, location = Location())
+      BackReferenceReadNode.new(name, location)
     end
 
     # Create a new BeginNode node
@@ -128,8 +128,8 @@ module Prism
     end
 
     # Create a new CallAndWriteNode node
-    def CallAndWriteNode(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator_loc, value, location = Location())
-      CallAndWriteNode.new(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator_loc, value, location)
+    def CallAndWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
+      CallAndWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
     end
 
     # Create a new CallNode node
@@ -138,13 +138,13 @@ module Prism
     end
 
     # Create a new CallOperatorWriteNode node
-    def CallOperatorWriteNode(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator, operator_loc, value, location = Location())
-      CallOperatorWriteNode.new(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator, operator_loc, value, location)
+    def CallOperatorWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location = Location())
+      CallOperatorWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location)
     end
 
     # Create a new CallOrWriteNode node
-    def CallOrWriteNode(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator_loc, value, location = Location())
-      CallOrWriteNode.new(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, flags, read_name, write_name, operator_loc, value, location)
+    def CallOrWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
+      CallOrWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
     end
 
     # Create a new CapturePatternNode node
@@ -382,6 +382,21 @@ module Prism
       InNode.new(pattern, statements, in_loc, then_loc, location)
     end
 
+    # Create a new IndexAndWriteNode node
+    def IndexAndWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
+      IndexAndWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
+    end
+
+    # Create a new IndexOperatorWriteNode node
+    def IndexOperatorWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location = Location())
+      IndexOperatorWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location)
+    end
+
+    # Create a new IndexOrWriteNode node
+    def IndexOrWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
+      IndexOrWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
+    end
+
     # Create a new InstanceVariableAndWriteNode node
     def InstanceVariableAndWriteNode(name, name_loc, operator_loc, value, location = Location())
       InstanceVariableAndWriteNode.new(name, name_loc, operator_loc, value, location)

data/lib/prism/ffi.rb

@@ -25,8 +25,8 @@ module Prism
     #     void         -> :void
     #
     def self.resolve_type(type)
-      type = type.strip.delete_prefix("const ")
-      type.end_with?("*") ? :pointer : type.to_sym
+      type = type.strip
+      type.end_with?("*") ? :pointer : type.delete_prefix("const ").to_sym
     end
 
     # Read through the given header file and find the declaration of each of the
@@ -234,11 +234,11 @@ module Prism
       loader = Serialize::Loader.new(source, buffer.read)
 
       tokens = loader.load_tokens
-      node, comments, errors, warnings = loader.load_nodes
+      node, comments, magic_comments, errors, warnings = loader.load_nodes
 
       tokens.each { |token,| token.value.force_encoding(loader.encoding) }
 
-      ParseResult.new([node, tokens], comments, errors, warnings, source)
+      ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
     end
   end
 

data/lib/prism/lex_compat.rb

@@ -357,6 +357,7 @@ module Prism
           @dedent_next = true
           @dedent = nil
           @embexpr_balance = 0
+          @ended_on_newline = false
         end
 
         # As tokens are coming in, we track the minimum amount of common leading
@@ -366,14 +367,14 @@ module Prism
           case token.event
           when :on_embexpr_beg, :on_heredoc_beg
             @embexpr_balance += 1
+            @dedent = 0 if @dedent_next && @ended_on_newline
           when :on_embexpr_end, :on_heredoc_end
             @embexpr_balance -= 1
           when :on_tstring_content
             if embexpr_balance == 0
-              token.value.split(/(?<=\n)/).each_with_index do |line, index|
-                next if line.strip.empty? && line.end_with?("\n")
-                next if !(dedent_next || index > 0)
+              line = token.value
 
+              if dedent_next && !(line.strip.empty? && line.end_with?("\n"))
                 leading = line[/\A(\s*)\n?/, 1]
                 next_dedent = 0
 
@@ -386,11 +387,16 @@ module Prism
                 end
 
                 @dedent = [dedent, next_dedent].compact.min
+                @dedent_next = true
+                @ended_on_newline = line.end_with?("\n")
+                tokens << token
+                return
               end
             end
           end
 
           @dedent_next = token.event == :on_tstring_content && embexpr_balance == 0
+          @ended_on_newline = false
           tokens << token
         end
 
@@ -430,6 +436,38 @@ module Prism
             return results
           end
 
+          # If the minimum common whitespace is 0, then we need to concatenate
+          # string nodes together that are immediately adjacent.
+          if dedent == 0
+            results = []
+            embexpr_balance = 0
+
+            index = 0
+            max_index = tokens.length
+
+            while index < max_index
+              token = tokens[index]
+              results << token
+              index += 1
+
+              case token.event
+              when :on_embexpr_beg, :on_heredoc_beg
+                embexpr_balance += 1
+              when :on_embexpr_end, :on_heredoc_end
+                embexpr_balance -= 1
+              when :on_tstring_content
+                if embexpr_balance == 0
+                  while index < max_index && tokens[index].event == :on_tstring_content
+                    token.value << tokens[index].value
+                    index += 1
+                  end
+                end
+              end
+            end
+
+            return results
+          end
+
           # Otherwise, we're going to run through each token in the list and
           # insert on_ignored_sp tokens for the amount of dedent that we need to
           # perform. We also need to remove the dedent from the beginning of
@@ -787,11 +825,7 @@ module Prism
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      if result_value.size - 1 > tokens.size
-        raise StandardError, "Lost tokens when performing lex_compat"
-      end
-
-      ParseResult.new(tokens, result.comments, result.errors, result.warnings, [])
+      ParseResult.new(tokens, result.comments, result.magic_comments, result.errors, result.warnings, [])
     end
   end
 

data/lib/prism/mutation_compiler.rb

@@ -97,7 +97,7 @@ module Prism
 
     # Copy a CallAndWriteNode node
     def visit_call_and_write_node(node)
-      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), value: visit(node.value))
+      node.copy(receiver: visit(node.receiver), value: visit(node.value))
     end
 
     # Copy a CallNode node
@@ -107,12 +107,12 @@ module Prism
 
     # Copy a CallOperatorWriteNode node
     def visit_call_operator_write_node(node)
-      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), value: visit(node.value))
+      node.copy(receiver: visit(node.receiver), value: visit(node.value))
     end
 
     # Copy a CallOrWriteNode node
     def visit_call_or_write_node(node)
-      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), value: visit(node.value))
+      node.copy(receiver: visit(node.receiver), value: visit(node.value))
     end
 
     # Copy a CapturePatternNode node
@@ -350,6 +350,21 @@ module Prism
       node.copy(pattern: visit(node.pattern), statements: visit(node.statements))
     end
 
+    # Copy a IndexAndWriteNode node
+    def visit_index_and_write_node(node)
+      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
+    end
+
+    # Copy a IndexOperatorWriteNode node
+    def visit_index_operator_write_node(node)
+      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
+    end
+
+    # Copy a IndexOrWriteNode node
+    def visit_index_or_write_node(node)
+      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
+    end
+
     # Copy a InstanceVariableAndWriteNode node
     def visit_instance_variable_and_write_node(node)
       node.copy(value: visit(node.value))