syntax_tree 0.1.0 → 1.2.0

data/lib/syntax_tree.rb

@@ -7,7 +7,7 @@ require "stringio"
 
 require_relative "syntax_tree/version"
 
-# If PrettyPrint::Assign isn't defined, then we haven't gotten the updated
+# If PrettyPrint::Align isn't defined, then we haven't gotten the updated
 # version of prettyprint. In that case we'll define our own. This is going to
 # overwrite a bunch of methods, so silencing them as well.
 unless PrettyPrint.const_defined?(:Align)
@@ -26,12 +26,14 @@ class SyntaxTree < Ripper
   # every character in the string is 1 byte in length, so we can just return the
   # start of the line + the index.
   class SingleByteString
+    attr_reader :start
+
     def initialize(start)
       @start = start
     end
 
     def [](byteindex)
-      @start + byteindex
+      start + byteindex
     end
   end
 
@@ -40,7 +42,10 @@ class SyntaxTree < Ripper
   # an array of indices, such that array[byteindex] will be equal to the index
   # of the character within the string.
   class MultiByteString
+    attr_reader :start, :indices
+
     def initialize(start, line)
+      @start = start
       @indices = []
 
       line.each_char.with_index(start) do |char, index|
@@ -48,8 +53,11 @@ class SyntaxTree < Ripper
       end
     end
 
+    # Technically it's possible for the column index to be a negative value if
+    # there's a BOM at the beginning of the file, which is the reason we need to
+    # compare it to 0 here.
     def [](byteindex)
-      @indices[byteindex]
+      indices[byteindex < 0 ? 0 : byteindex]
     end
   end
 
@@ -66,8 +74,7 @@ class SyntaxTree < Ripper
 
     def ==(other)
       other.is_a?(Location) && start_line == other.start_line &&
-        start_char == other.start_char &&
-        end_line == other.end_line &&
+        start_char == other.start_char && end_line == other.end_line &&
         end_char == other.end_char
     end
 
@@ -75,7 +82,7 @@ class SyntaxTree < Ripper
       Location.new(
         start_line: start_line,
         start_char: start_char,
-        end_line: other.end_line,
+        end_line: [end_line, other.end_line].max,
         end_char: other.end_char
       )
     end
@@ -113,16 +120,21 @@ class SyntaxTree < Ripper
   # A slightly enhanced PP that knows how to format recursively including
   # comments.
   class Formatter < PP
-    attr_reader :stack, :quote
+    COMMENT_PRIORITY = 1
+    HEREDOC_PRIORITY = 2
+
+    attr_reader :source, :stack, :quote
 
-    def initialize(*)
-      super
+    def initialize(source, ...)
+      super(...)
+
+      @source = source
       @stack = []
       @quote = "\""
     end
 
-    def format(node)
-      stack << node
+    def format(node, stackable: true)
+      stack << node if stackable
       doc = nil
 
       # If there are comments, then we're going to format them around the node
@@ -136,11 +148,17 @@ class SyntaxTree < Ripper
           breakable(force: true)
         end
 
-        doc = node.format(self)
+        # If the node has a stree-ignore comment right before it, then we're
+        # going to just print out the node as it was seen in the source.
+        if leading.last&.ignore?
+          doc = text(source[node.location.start_char...node.location.end_char])
+        else
+          doc = node.format(self)
+        end
 
         # Print all comments that were found after the node.
         trailing.each do |comment|
-          line_suffix do
+          line_suffix(priority: COMMENT_PRIORITY) do
             text(" ")
             comment.format(self)
             break_parent
@@ -150,7 +168,7 @@ class SyntaxTree < Ripper
         doc = node.format(self)
       end
 
-      stack.pop
+      stack.pop if stackable
       doc
     end
 
@@ -173,6 +191,10 @@ class SyntaxTree < Ripper
   # [Array[ String ]] the list of lines in the source
   attr_reader :lines
 
+  # [Array[ SingleByteString | MultiByteString ]] the list of objects that
+  # represent the start of each line in character offsets
+  attr_reader :line_counts
+
   # [Array[ untyped ]] a running list of tokens that have been found in the
   # source. This list changes a lot as certain nodes will "consume" these tokens
   # to determine their bounds.
@@ -248,6 +270,12 @@ class SyntaxTree < Ripper
 
       last_index += line.size
     end
+
+    # Make sure line counts is filled out with the first and last line at
+    # minimum so that it has something to compare against if the parser is in a
+    # lineno=2 state for an empty file.
+    @line_counts << SingleByteString.new(0) if @line_counts.empty?
+    @line_counts << SingleByteString.new(last_index)
   end
 
   def self.parse(source)
@@ -259,13 +287,26 @@ class SyntaxTree < Ripper
   def self.format(source)
     output = []
 
-    formatter = Formatter.new(output)
+    formatter = Formatter.new(source, output)
     parse(source).format(formatter)
 
     formatter.flush
     output.join
   end
 
+  # Returns the source from the given filepath taking into account any potential
+  # magic encoding comments.
+  def self.read(filepath)
+    encoding =
+      File.open(filepath, "r") do |file|
+        header = file.readline
+        header += file.readline if header.start_with?("#!")
+        Ripper.new(header).tap(&:parse).encoding
+      end
+
+    File.read(filepath, encoding: encoding)
+  end
+
   private
 
   # ----------------------------------------------------------------------------
@@ -280,7 +321,7 @@ class SyntaxTree < Ripper
   # this line, then we add the number of columns into this line that we've gone
   # through.
   def char_pos
-    @line_counts[lineno - 1][column]
+    line_counts[lineno - 1][column]
   end
 
   # As we build up a list of tokens, we'll periodically need to go backwards and
@@ -294,7 +335,7 @@ class SyntaxTree < Ripper
   # (which would happen to be the innermost keyword). Then the outer one would
   # only be able to grab the first one. In this way all of the tokens act as
   # their own stack.
-  def find_token(type, value = :any, consume: true)
+  def find_token(type, value = :any, consume: true, location: nil)
     index =
       tokens.rindex do |token|
         token.is_a?(type) && (value == :any || (token.value == value))
@@ -307,8 +348,16 @@ class SyntaxTree < Ripper
       # could also be caused by accidentally attempting to consume a token twice
       # by two different parser event handlers.
       unless index
-        message = "Cannot find expected #{value == :any ? type : value}"
-        raise ParseError.new(message, lineno, column)
+        token = value == :any ? type.name.split("::", 2).last : value
+        message = "Cannot find expected #{token}"
+
+        if location
+          lineno = location.start_line
+          column = location.start_char - line_counts[lineno - 1].start
+          raise ParseError.new(message, lineno, column)
+        else
+          raise ParseError.new(message, lineno, column)
+        end
       end
 
       tokens.delete_at(index)
@@ -476,7 +525,7 @@ class SyntaxTree < Ripper
         q.text(value)
       else
         q.text(q.quote)
-        q.text(value[1])
+        q.text(value[1] == "\"" ? "\\\"" : value[1])
         q.text(q.quote)
       end
     end
@@ -628,7 +677,9 @@ class SyntaxTree < Ripper
     def format(q)
       q.text("__END__")
       q.breakable(force: true)
-      q.text(value)
+
+      separator = -> { q.breakable(indent: false, force: true) }
+      q.seplist(value.split(/\r?\n/, -1), separator) { |line| q.text(line) }
     end
 
     def pretty_print(q)
@@ -654,7 +705,7 @@ class SyntaxTree < Ripper
   def on___end__(value)
     @__end__ =
       EndContent.new(
-        value: lines[lineno..-1].join("\n"),
+        value: source[(char_pos + value.length)..-1],
         location: Location.token(line: lineno, char: char_pos, size: value.size)
       )
   end
@@ -725,11 +776,11 @@ class SyntaxTree < Ripper
 
       q.group do
         q.text(keyword)
-        q.format(left_argument)
+        q.format(left_argument, stackable: false)
         q.group do
           q.nest(keyword.length) do
             q.breakable(force: left_argument.comments.any?)
-            q.format(AliasArgumentFormatter.new(right))
+            q.format(AliasArgumentFormatter.new(right), stackable: false)
           end
         end
       end
@@ -1122,7 +1173,7 @@ class SyntaxTree < Ripper
   #     method(&expression)
   #
   class ArgBlock
-    # [untyped] the expression being turned into a block
+    # [nil | untyped] the expression being turned into a block
     attr_reader :value
 
     # [Location] the location of this node
@@ -1143,15 +1194,17 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.text("&")
-      q.format(value)
+      q.format(value) if value
     end
 
     def pretty_print(q)
       q.group(2, "(", ")") do
         q.text("arg_block")
 
-        q.breakable
-        q.pp(value)
+        if value
+          q.breakable
+          q.pp(value)
+        end
 
         q.pp(Comment::List.new(comments))
       end
@@ -1170,17 +1223,34 @@ class SyntaxTree < Ripper
   #     (false | untyped) block
   #   ) -> Args
   def on_args_add_block(arguments, block)
-    return arguments unless block
+    operator = find_token(Op, "&", consume: false)
 
-    arg_block =
-      ArgBlock.new(
-        value: block,
-        location: find_token(Op, "&").location.to(block.location)
-      )
+    # If we can't find the & operator, then there's no block to add to the list,
+    # so we're just going to return the arguments as-is.
+    return arguments unless operator
+
+    # Now we know we have an & operator, so we're going to delete it from the
+    # list of tokens to make sure it doesn't get confused with anything else.
+    tokens.delete(operator)
+
+    # Construct the location that represents the block argument.
+    location = operator.location
+    location = operator.location.to(block.location) if block
+
+    # If there are any arguments and the operator we found from the list is not
+    # after them, then we're going to return the arguments as-is because we're
+    # looking at an & that occurs before the arguments are done.
+    if arguments.parts.any? && location.start_char < arguments.location.end_char
+      return arguments
+    end
+
+    # Otherwise, we're looking at an actual block argument (with or without a
+    # block, which could be missing because it could be a bare & since 3.1.0).
+    arg_block = ArgBlock.new(value: block, location: location)
 
     Args.new(
       parts: arguments.parts << arg_block,
-      location: arguments.location.to(arg_block.location)
+      location: arguments.location.to(location)
     )
   end
 
@@ -1349,7 +1419,11 @@ class SyntaxTree < Ripper
           q.indent do
             q.breakable("")
             q.seplist(contents.parts, -> { q.breakable }) do |part|
-              q.format(part.parts.first)
+              if part.is_a?(StringLiteral)
+                q.format(part.parts.first)
+              else
+                q.text(part.value[1..-1])
+              end
             end
           end
           q.breakable("")
@@ -1373,10 +1447,39 @@ class SyntaxTree < Ripper
               q.format(part.value)
             end
           end
+          q.breakable("")
+        end
+      end
+    end
+
+    class VarRefsFormatter
+      # [Args] the contents of the array
+      attr_reader :contents
+
+      def initialize(contents)
+        @contents = contents
+      end
+
+      def format(q)
+        q.group(0, "[", "]") do
+          q.indent do
+            q.breakable("")
+
+            separator = -> do
+              q.text(",")
+              q.fill_breakable
+            end
+
+            q.seplist(contents.parts, separator) { |part| q.format(part) }
+          end
+          q.breakable("")
         end
       end
     end
 
+    # [LBracket] the bracket that opens this array
+    attr_reader :lbracket
+
     # [nil | Args] the contents of the array
     attr_reader :contents
 
@@ -1386,22 +1489,18 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(contents:, location:, comments: [])
+    def initialize(lbracket:, contents:, location:, comments: [])
+      @lbracket = lbracket
       @contents = contents
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [contents]
+      [lbracket, contents]
     end
 
     def format(q)
-      unless contents
-        q.text("[]")
-        return
-      end
-
       if qwords?
         QWordsFormatter.new(contents).format(q)
         return
@@ -1412,12 +1511,23 @@ class SyntaxTree < Ripper
         return
       end
 
-      q.group(0, "[", "]") do
-        q.indent do
-          q.breakable("")
-          q.format(contents)
+      if var_refs?(q)
+        VarRefsFormatter.new(contents).format(q)
+        return
+      end
+
+      q.group do
+        q.format(lbracket)
+
+        if contents
+          q.indent do
+            q.breakable("")
+            q.format(contents)
+          end
         end
+
         q.breakable("")
+        q.text("]")
       end
     end
 
@@ -1441,21 +1551,40 @@ class SyntaxTree < Ripper
     private
 
     def qwords?
-      contents && contents.comments.empty? && contents.parts.length > 1 &&
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.length > 1 &&
         contents.parts.all? do |part|
-          part.is_a?(StringLiteral) && part.comments.empty? &&
-            part.parts.length == 1 &&
-            part.parts.first.is_a?(TStringContent) &&
-            !part.parts.first.value.match?(/[\s\\\]]/)
+          case part
+          when StringLiteral
+            part.comments.empty? && part.parts.length == 1 &&
+              part.parts.first.is_a?(TStringContent) &&
+              !part.parts.first.value.match?(/[\s\[\]\\]/)
+          when CHAR
+            !part.value.match?(/[\[\]\\]/)
+          else
+            false
+          end
         end
     end
 
     def qsymbols?
-      contents && contents.comments.empty? && contents.parts.length > 1 &&
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.length > 1 &&
         contents.parts.all? do |part|
           part.is_a?(SymbolLiteral) && part.comments.empty?
         end
     end
+
+    def var_refs?(q)
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.all? do |part|
+          part.is_a?(VarRef) && part.comments.empty?
+        end &&
+        (
+          contents.parts.sum { |part| part.value.value.length + 2 } >
+            q.maxwidth * 2
+        )
+    end
   end
 
   # :call-seq:
@@ -1467,13 +1596,16 @@ class SyntaxTree < Ripper
       rbracket = find_token(RBracket)
 
       ArrayLiteral.new(
+        lbracket: lbracket,
         contents: contents,
         location: lbracket.location.to(rbracket.location)
       )
     else
-      tstring_end = find_token(TStringEnd)
+      tstring_end =
+        find_token(TStringEnd, location: contents.beginning.location)
 
       contents.class.new(
+        beginning: contents.beginning,
         elements: contents.elements,
         location: contents.location.to(tstring_end.location)
       )
@@ -1554,7 +1686,7 @@ class SyntaxTree < Ripper
     end
 
     def child_nodes
-      [constant, *required, rest, *posts]
+      [constant, *requireds, rest, *posts]
     end
 
     def format(q)
@@ -1563,20 +1695,23 @@ class SyntaxTree < Ripper
       parts += posts
 
       if constant
-        q.format(constant)
-        q.text("[")
-        q.seplist(parts) { |part| q.format(part) }
-        q.text("]")
+        q.group do
+          q.format(constant)
+          q.text("[")
+          q.seplist(parts) { |part| q.format(part) }
+          q.text("]")
+        end
+
         return
       end
 
       parent = q.parent
-      if parts.length == 1 || PATTERNS.any? { |pattern| parent.is_a?(pattern) }
+      if parts.length == 1 || PATTERNS.include?(parent.class)
         q.text("[")
         q.seplist(parts) { |part| q.format(part) }
         q.text("]")
       else
-        q.seplist(parts) { |part| q.format(part) }
+        q.group { q.seplist(parts) { |part| q.format(part) } }
       end
     end
 
@@ -1642,6 +1777,23 @@ class SyntaxTree < Ripper
     )
   end
 
+  # Determins if the following value should be indented or not.
+  module AssignFormatting
+    def self.skip_indent?(value)
+      (value.is_a?(Call) && skip_indent?(value.receiver)) ||
+        [
+          ArrayLiteral,
+          HashLiteral,
+          Heredoc,
+          Lambda,
+          QSymbols,
+          QWords,
+          Symbols,
+          Words
+        ].include?(value.class)
+    end
+  end
+
   # Assign represents assigning something to a variable or constant. Generally,
   # the left side of the assignment is going to be any node that ends with the
   # name "Field".
@@ -1717,10 +1869,8 @@ class SyntaxTree < Ripper
     private
 
     def skip_indent?
-      target.is_a?(ARefField) || value.is_a?(ArrayLiteral) ||
-        value.is_a?(HashLiteral) ||
-        value.is_a?(Heredoc) ||
-        value.is_a?(Lambda)
+      target.comments.empty? &&
+        (target.is_a?(ARefField) || AssignFormatting.skip_indent?(value))
     end
   end
 
@@ -1768,15 +1918,11 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      contents = -> do
-        q.parent.format_key(q, key)
-        q.indent do
-          q.breakable
-          q.format(value)
-        end
+      if value&.is_a?(HashLiteral)
+        format_contents(q)
+      else
+        q.group { format_contents(q) }
       end
-
-      value.is_a?(HashLiteral) ? contents.call : q.group(&contents)
     end
 
     def pretty_print(q)
@@ -1786,8 +1932,10 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(key)
 
-        q.breakable
-        q.pp(value)
+        if value
+          q.breakable
+          q.pp(value)
+        end
 
         q.pp(Comment::List.new(comments))
       end
@@ -1802,12 +1950,32 @@ class SyntaxTree < Ripper
         cmts: comments
       }.to_json(*opts)
     end
+
+    private
+
+    def format_contents(q)
+      q.parent.format_key(q, key)
+      return unless value
+
+      if key.comments.empty? && AssignFormatting.skip_indent?(value)
+        q.text(" ")
+        q.format(value)
+      else
+        q.indent do
+          q.breakable
+          q.format(value)
+        end
+      end
+    end
   end
 
   # :call-seq:
   #   on_assoc_new: (untyped key, untyped value) -> Assoc
   def on_assoc_new(key, value)
-    Assoc.new(key: key, value: value, location: key.location.to(value.location))
+    location = key.location
+    location = location.to(value.location) if value
+
+    Assoc.new(key: key, value: value, location: location)
   end
 
   # AssocSplat represents double-splatting a value into a hash (either a hash
@@ -1990,25 +2158,8 @@ class SyntaxTree < Ripper
   # This module is responsible for formatting the assocs contained within a
   # hash or bare hash. It first determines if every key in the hash can use
   # labels. If it can, it uses labels. Otherwise it uses hash rockets.
-  module HashFormatter
-    class Base
-      # [HashLiteral | BareAssocHash] the source of the assocs
-      attr_reader :container
-
-      def initialize(container)
-        @container = container
-      end
-
-      def comments
-        container.comments
-      end
-
-      def format(q)
-        q.seplist(container.assocs) { |assoc| q.format(assoc) }
-      end
-    end
-
-    class Labels < Base
+  module HashKeyFormatter
+    class Labels
       def format_key(q, key)
         case key
         when Label
@@ -2023,7 +2174,7 @@ class SyntaxTree < Ripper
       end
     end
 
-    class Rockets < Base
+    class Rockets
       def format_key(q, key)
         case key
         when Label
@@ -2063,7 +2214,7 @@ class SyntaxTree < Ripper
           end
         end
 
-      (labels ? Labels : Rockets).new(container)
+      (labels ? Labels : Rockets).new
     end
   end
 
@@ -2094,7 +2245,11 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.format(HashFormatter.for(self))
+      q.seplist(assocs) { |assoc| q.format(assoc) }
+    end
+
+    def format_key(q, key)
+      (@key_formatter ||= HashKeyFormatter.for(self)).format_key(q, key)
     end
 
     def pretty_print(q)
@@ -2188,24 +2343,95 @@ class SyntaxTree < Ripper
     end
   end
 
+  # PinnedBegin represents a pinning a nested statement within pattern matching.
+  #
+  #     case value
+  #     in ^(statement)
+  #     end
+  #
+  class PinnedBegin
+    # [untyped] the expression being pinned
+    attr_reader :statement
+
+    # [Location] the location of this node
+    attr_reader :location
+
+    # [Array[ Comment | EmbDoc ]] the comments attached to this node
+    attr_reader :comments
+
+    def initialize(statement:, location:, comments: [])
+      @statement = statement
+      @location = location
+      @comments = comments
+    end
+
+    def child_nodes
+      [statement]
+    end
+
+    def format(q)
+      q.group do
+        q.text("^(")
+        q.nest(1) do
+          q.indent do
+            q.breakable("")
+            q.format(statement)
+          end
+          q.breakable("")
+          q.text(")")
+        end
+      end
+    end
+
+    def pretty_print(q)
+      q.group(2, "(", ")") do
+        q.text("pinned_begin")
+
+        q.breakable
+        q.pp(statement)
+
+        q.pp(Comment::List.new(comments))
+      end
+    end
+
+    def to_json(*opts)
+      {
+        type: :pinned_begin,
+        stmt: statement,
+        loc: location,
+        cmts: comments
+      }.to_json(*opts)
+    end
+  end
+
   # :call-seq:
-  #   on_begin: (BodyStmt bodystmt) -> Begin
+  #   on_begin: (untyped bodystmt) -> Begin | PinnedBegin
   def on_begin(bodystmt)
-    keyword = find_token(Kw, "begin")
-    end_char =
-      if bodystmt.rescue_clause || bodystmt.ensure_clause ||
-           bodystmt.else_clause
-        bodystmt.location.end_char
-      else
-        find_token(Kw, "end").location.end_char
-      end
+    pin = find_token(Op, "^", consume: false)
 
-    bodystmt.bind(keyword.location.end_char, end_char)
+    if pin && pin.location.start_char < bodystmt.location.start_char
+      tokens.delete(pin)
+      find_token(LParen)
 
-    Begin.new(
-      bodystmt: bodystmt,
-      location: keyword.location.to(bodystmt.location)
-    )
+      rparen = find_token(RParen)
+      location = pin.location.to(rparen.location)
+
+      PinnedBegin.new(statement: bodystmt, location: location)
+    else
+      keyword = find_token(Kw, "begin")
+      end_char =
+        if bodystmt.rescue_clause || bodystmt.ensure_clause ||
+            bodystmt.else_clause
+          bodystmt.location.end_char
+        else
+          find_token(Kw, "end").location.end_char
+        end
+
+      bodystmt.bind(keyword.location.end_char, end_char)
+      location = keyword.location.to(bodystmt.location)
+
+      Begin.new(bodystmt: bodystmt, location: location)
+    end
   end
 
   # Binary represents any expression that involves two sub-expressions with an
@@ -2252,11 +2478,14 @@ class SyntaxTree < Ripper
       q.group do
         q.group { q.format(left) }
         q.text(" ") unless power
-        q.text(operator)
 
-        q.indent do
-          q.breakable(power ? "" : " ")
-          q.format(right)
+        q.group do
+          q.text(operator)
+
+          q.indent do
+            q.breakable(power ? "" : " ")
+            q.format(right)
+          end
         end
       end
     end
@@ -2293,12 +2522,29 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_binary: (untyped left, (Op | Symbol) operator, untyped right) -> Binary
   def on_binary(left, operator, right)
-    # On most Ruby implementations, operator is a Symbol that represents that
-    # operation being performed. For instance in the example `1 < 2`, the
-    # `operator` object would be `:<`. However, on JRuby, it's an `@op` node,
-    # so here we're going to explicitly convert it into the same normalized
-    # form.
-    operator = tokens.delete(operator).value unless operator.is_a?(Symbol)
+    if operator.is_a?(Symbol)
+      # Here, we're going to search backward for the token that's between the
+      # two operands that matches the operator so we can delete it from the
+      # list.
+      index =
+        tokens.rindex do |token|
+          location = token.location
+
+          token.is_a?(Op) &&
+            token.value == operator.to_s &&
+            location.start_char > left.location.end_char &&
+            location.end_char < right.location.start_char
+        end
+
+      tokens.delete_at(index) if index
+    else
+      # On most Ruby implementations, operator is a Symbol that represents that
+      # operation being performed. For instance in the example `1 < 2`, the
+      # `operator` object would be `:<`. However, on JRuby, it's an `@op` node,
+      # so here we're going to explicitly convert it into the same normalized
+      # form.
+      operator = tokens.delete(operator).value
+    end
 
     Binary.new(
       left: left,
@@ -2448,7 +2694,7 @@ class SyntaxTree < Ripper
   #     def method(&block); end
   #
   class BlockArg
-    # [Ident] the name of the block argument
+    # [nil | Ident] the name of the block argument
     attr_reader :name
 
     # [Location] the location of this node
@@ -2469,15 +2715,17 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.text("&")
-      q.format(name)
+      q.format(name) if name
     end
 
     def pretty_print(q)
       q.group(2, "(", ")") do
         q.text("blockarg")
 
-        q.breakable
-        q.pp(name)
+        if name
+          q.breakable
+          q.pp(name)
+        end
 
         q.pp(Comment::List.new(comments))
       end
@@ -2495,7 +2743,10 @@ class SyntaxTree < Ripper
   def on_blockarg(name)
     operator = find_token(Op, "&")
 
-    BlockArg.new(name: name, location: operator.location.to(name.location))
+    location = operator.location
+    location = location.to(name.location) if name
+
+    BlockArg.new(name: name, location: location)
   end
 
   # bodystmt can't actually determine its bounds appropriately because it
@@ -2685,59 +2936,137 @@ class SyntaxTree < Ripper
     # [LBrace | Keyword] the node that opens the block
     attr_reader :block_open
 
+    # [String] the string that closes the block
+    attr_reader :block_close
+
     # [BodyStmt | Statements] the statements inside the block
     attr_reader :statements
 
-    def initialize(node, block_open, statements)
+    def initialize(node, block_open, block_close, statements)
       @node = node
       @block_open = block_open
+      @block_close = block_close
       @statements = statements
     end
 
     def format(q)
-      q.group do
-        q.text(" ")
+      # If this is nested anywhere inside of a Command or CommandCall node, then
+      # we can't change which operators we're using for the bounds of the block.
+      break_opening, break_closing, flat_opening, flat_closing =
+        if unchangeable_bounds?(q)
+          [block_open.value, block_close, block_open.value, block_close]
+        elsif forced_do_end_bounds?(q)
+          %w[do end do end]
+        elsif forced_brace_bounds?(q)
+          %w[{ } { }]
+        else
+          %w[do end { }]
+        end
 
-        q.if_break do
-          q.format(BlockOpenFormatter.new("do", block_open))
+      # If the receiver of this block a Command or CommandCall node, then there
+      # are no parentheses around the arguments to that command, so we need to
+      # break the block.
+      receiver = q.parent.call
+      if receiver.is_a?(Command) || receiver.is_a?(CommandCall)
+        q.break_parent
+        format_break(q, break_opening, break_closing)
+        return
+      end
 
-          if node.block_var
-            q.text(" ")
-            q.format(node.block_var)
-          end
+      q.group do
+        q.if_break { format_break(q, break_opening, break_closing) }.if_flat do
+          format_flat(q, flat_opening, flat_closing)
+        end
+      end
+    end
 
-          unless statements.empty?
-            q.indent do
-              q.breakable
-              q.format(statements)
-            end
-          end
+    private
 
-          q.breakable
-          q.text("end")
-        end.if_flat do
-          q.format(BlockOpenFormatter.new("{", block_open))
+    # If this is nested anywhere inside certain nodes, then we can't change
+    # which operators/keywords we're using for the bounds of the block.
+    def unchangeable_bounds?(q)
+      q.parents.any? do |parent|
+        # If we hit a statements, then we're safe to use whatever since we
+        # know for certain we're going to get split over multiple lines
+        # anyway.
+        break false if parent.is_a?(Statements)
 
-          if node.block_var
-            q.breakable
-            q.format(node.block_var)
-            q.breakable
-          end
+        [Command, CommandCall].include?(parent.class)
+      end
+    end
 
-          unless statements.empty?
-            q.breakable unless node.block_var
-            q.format(statements)
-            q.breakable
-          end
+    # If we're a sibling of a control-flow keyword, then we're going to have to
+    # use the do..end bounds.
+    def forced_do_end_bounds?(q)
+      [Break, Next, Return, Super].include?(q.parent.call.class)
+    end
 
-          q.text("}")
-        end
+    # If we're the predicate of a loop or conditional, then we're going to have
+    # to go with the {..} bounds.
+    def forced_brace_bounds?(q)
+      parents = q.parents.to_a
+      parents.each_with_index.any? do |parent, index|
+        # If we hit certain breakpoints then we know we're safe.
+        break false if [Paren, Statements].include?(parent.class)
+
+        [
+          If,
+          IfMod,
+          IfOp,
+          Unless,
+          UnlessMod,
+          While,
+          WhileMod,
+          Until,
+          UntilMod
+        ].include?(parent.class) && parent.predicate == parents[index - 1]
       end
     end
-  end
 
-  # BraceBlock represents passing a block to a method call using the { }
-  # operators.
+    def format_break(q, opening, closing)
+      q.text(" ")
+      q.format(BlockOpenFormatter.new(opening, block_open), stackable: false)
+
+      if node.block_var
+        q.text(" ")
+        q.format(node.block_var)
+      end
+
+      unless statements.empty?
+        q.indent do
+          q.breakable
+          q.format(statements)
+        end
+      end
+
+      q.breakable
+      q.text(closing)
+    end
+
+    def format_flat(q, opening, closing)
+      q.text(" ")
+      q.format(BlockOpenFormatter.new(opening, block_open), stackable: false)
+
+      if node.block_var
+        q.breakable
+        q.format(node.block_var)
+        q.breakable
+      end
+
+      if statements.empty?
+        q.text(" ") if opening == "do"
+      else
+        q.breakable unless node.block_var
+        q.format(statements)
+        q.breakable
+      end
+
+      q.text(closing)
+    end
+  end
+
+  # BraceBlock represents passing a block to a method call using the { }
+  # operators.
   #
   #     method { |variable| variable + 1 }
   #
@@ -2770,7 +3099,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      BlockFormatter.new(self, lbrace, statements).format(q)
+      BlockFormatter.new(self, lbrace, "}", statements).format(q)
     end
 
     def pretty_print(q)
@@ -2851,15 +3180,35 @@ class SyntaxTree < Ripper
         q.text(keyword)
 
         if arguments.parts.any?
-          if arguments.parts.length == 1 && arguments.parts.first.is_a?(Paren)
-            q.format(arguments)
+          if arguments.parts.length == 1
+            part = arguments.parts.first
+
+            if part.is_a?(Paren)
+              q.format(arguments)
+            elsif part.is_a?(ArrayLiteral)
+              q.text(" ")
+              q.format(arguments)
+            else
+              format_arguments(q, "(", ")")
+            end
           else
-            q.text(" ")
-            q.nest(keyword.length + 1) { q.format(arguments) }
+            format_arguments(q, " [", "]")
           end
         end
       end
     end
+
+    private
+
+    def format_arguments(q, opening, closing)
+      q.if_break { q.text(opening) }
+      q.indent do
+        q.breakable(" ")
+        q.format(node.arguments)
+      end
+      q.breakable("")
+      q.if_break { q.text(closing) }
+    end
   end
 
   # Break represents using the +break+ keyword.
@@ -2946,9 +3295,7 @@ class SyntaxTree < Ripper
     end
   end
 
-  # Call represents a method call. This node doesn't contain the arguments being
-  # passed (if arguments are passed, this node will get nested under a
-  # MethodAddArg node).
+  # Call represents a method call.
   #
   #     receiver.message
   #
@@ -2962,32 +3309,64 @@ class SyntaxTree < Ripper
     # [:call | Backtick | Const | Ident | Op] the message being sent
     attr_reader :message
 
+    # [nil | ArgParen | Args] the arguments to the method call
+    attr_reader :arguments
+
     # [Location] the location of this node
     attr_reader :location
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(receiver:, operator:, message:, location:, comments: [])
+    def initialize(
+      receiver:,
+      operator:,
+      message:,
+      arguments:,
+      location:,
+      comments: []
+    )
       @receiver = receiver
       @operator = operator
       @message = message
+      @arguments = arguments
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [receiver, (operator if operator != :"::"), (message if message != :call)]
+      [
+        receiver,
+        (operator if operator != :"::"),
+        (message if message != :call),
+        arguments
+      ]
     end
 
     def format(q)
+      call_operator = CallOperatorFormatter.new(operator)
+
       q.group do
         q.format(receiver)
+
+        # If there are trailing comments on the call operator, then we need to
+        # use the trailing form as opposed to the leading form.
+        q.format(call_operator) if call_operator.comments.any?
+
         q.group do
           q.indent do
-            q.format(CallOperatorFormatter.new(operator))
+            if receiver.comments.any? || call_operator.comments.any?
+              q.breakable(force: true)
+            end
+
+            if call_operator.comments.empty?
+              q.format(call_operator, stackable: false)
+            end
+
             q.format(message) if message != :call
           end
+
+          q.format(arguments) if arguments
         end
       end
     end
@@ -3005,6 +3384,11 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(message)
 
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
+
         q.pp(Comment::List.new(comments))
       end
     end
@@ -3015,6 +3399,7 @@ class SyntaxTree < Ripper
         receiver: receiver,
         op: operator,
         message: message,
+        args: arguments,
         loc: location,
         cmts: comments
       }.to_json(*opts)
@@ -3035,13 +3420,8 @@ class SyntaxTree < Ripper
       receiver: receiver,
       operator: operator,
       message: message,
-      location:
-        Location.new(
-          start_line: receiver.location.start_line,
-          start_char: receiver.location.start_char,
-          end_line: [ending.location.end_line, receiver.location.end_line].max,
-          end_char: ending.location.end_char
-        )
+      arguments: nil,
+      location: receiver.location.to(ending.location)
     )
   end
 
@@ -3057,6 +3437,9 @@ class SyntaxTree < Ripper
   #     end
   #
   class Case
+    # [Kw] the keyword that opens this expression
+    attr_reader :keyword
+
     # [nil | untyped] optional value being switched on
     attr_reader :value
 
@@ -3069,7 +3452,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(value:, consequent:, location:, comments: [])
+    def initialize(keyword:, value:, consequent:, location:, comments: [])
+      @keyword = keyword
       @value = value
       @consequent = consequent
       @location = location
@@ -3077,11 +3461,13 @@ class SyntaxTree < Ripper
     end
 
     def child_nodes
-      [value, consequent]
+      [keyword, value, consequent]
     end
 
     def format(q)
-      q.group(0, "case", "end") do
+      q.group do
+        q.format(keyword)
+
         if value
           q.text(" ")
           q.format(value)
@@ -3090,6 +3476,8 @@ class SyntaxTree < Ripper
         q.breakable(force: true)
         q.format(consequent)
         q.breakable(force: true)
+
+        q.text("end")
       end
     end
 
@@ -3097,6 +3485,9 @@ class SyntaxTree < Ripper
       q.group(2, "(", ")") do
         q.text("case")
 
+        q.breakable
+        q.pp(keyword)
+
         if value
           q.breakable
           q.pp(value)
@@ -3204,6 +3595,7 @@ class SyntaxTree < Ripper
       tokens.delete(keyword)
 
       Case.new(
+        keyword: keyword,
         value: value,
         consequent: consequent,
         location: keyword.location.to(consequent.location)
@@ -3484,7 +3876,7 @@ class SyntaxTree < Ripper
     # [Const | Ident | Op] the message being send
     attr_reader :message
 
-    # [Args] the arguments going along with the message
+    # [nil | Args] the arguments going along with the message
     attr_reader :arguments
 
     # [Location] the location of this node
@@ -3515,16 +3907,16 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.group do
-        doc = q.format(receiver)
-        q.format(CallOperatorFormatter.new(operator))
-        q.format(message)
-        q.text(" ")
+        doc =
+          q.nest(0) do
+            q.format(receiver)
+            q.format(CallOperatorFormatter.new(operator), stackable: false)
+            q.format(message)
+          end
 
-        width = doc_width(doc)
-        if width > (q.maxwidth / 2) || width < 2
-          q.indent { q.format(arguments) }
-        else
-          q.nest(width) { q.format(arguments) }
+        if arguments
+          q.text(" ")
+          q.nest(argument_alignment(q, doc)) { q.format(arguments) }
         end
       end
     end
@@ -3542,8 +3934,10 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(message)
 
-        q.breakable
-        q.pp(arguments)
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
 
         q.pp(Comment::List.new(comments))
       end
@@ -3577,16 +3971,38 @@ class SyntaxTree < Ripper
         when PrettyPrint::Text
           width += doc.width
         when PrettyPrint::Indent, PrettyPrint::Align, PrettyPrint::Group
-          queue += doc.contents.reverse
+          queue = doc.contents + queue
         when PrettyPrint::IfBreak
-          queue += doc.flat_contents.reverse
+          queue = doc.break_contents + queue
         when PrettyPrint::Breakable
-          width = doc.force? ? 0 : width + doc.width
+          width = 0
         end
       end
 
       width
     end
+
+    def argument_alignment(q, doc)
+      # Very special handling case for rspec matchers. In general with rspec
+      # matchers you expect to see something like:
+      #
+      #     expect(foo).to receive(:bar).with(
+      #       'one',
+      #       'two',
+      #       'three',
+      #       'four',
+      #       'five'
+      #     )
+      #
+      # In this case the arguments are aligned to the left side as opposed to
+      # being aligned with the `receive` call.
+      if %w[to not_to to_not].include?(message.value)
+        0
+      else
+        width = doc_width(doc) + 1
+        width > (q.maxwidth / 2) ? 0 : width
+      end
+    end
   end
 
   # :call-seq:
@@ -3594,7 +4010,7 @@ class SyntaxTree < Ripper
   #     untyped receiver,
   #     (:"::" | Op | Period) operator,
   #     (Const | Ident | Op) message,
-  #     Args arguments
+  #     (nil | Args) arguments
   #   ) -> CommandCall
   def on_command_call(receiver, operator, message, arguments)
     ending = arguments || message
@@ -3665,10 +4081,18 @@ class SyntaxTree < Ripper
       @trailing
     end
 
+    def ignore?
+      value[1..-1].strip == "stree-ignore"
+    end
+
     def comments
       []
     end
 
+    def child_nodes
+      []
+    end
+
     def format(q)
       q.text(value)
     end
@@ -4062,14 +4486,7 @@ class SyntaxTree < Ripper
         q.group do
           q.text("def ")
           q.format(name)
-
-          if params.is_a?(Params) && !params.empty?
-            q.text("(")
-            q.format(params)
-            q.text(")")
-          else
-            q.format(params)
-          end
+          q.format(params) if !params.is_a?(Params) || !params.empty?
         end
 
         unless bodystmt.empty?
@@ -4118,10 +4535,16 @@ class SyntaxTree < Ripper
   #     def method = result
   #
   class DefEndless
+    # [untyped] the target where the method is being defined
+    attr_reader :target
+
+    # [Op | Period] the operator being used to declare the method
+    attr_reader :operator
+
     # [Backtick | Const | Ident | Kw | Op] the name of the method
     attr_reader :name
 
-    # [nil | Paren] the parameter declaration for the method
+    # [nil | Params | Paren] the parameter declaration for the method
     attr_reader :paren
 
     # [untyped] the expression to be executed by the method
@@ -4133,7 +4556,17 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(name:, paren:, statement:, location:, comments: [])
+    def initialize(
+      target:,
+      operator:,
+      name:,
+      paren:,
+      statement:,
+      location:,
+      comments: []
+    )
+      @target = target
+      @operator = operator
       @name = name
       @paren = paren
       @statement = statement
@@ -4142,14 +4575,26 @@ class SyntaxTree < Ripper
     end
 
     def child_nodes
-      [name, paren, statement]
+      [target, operator, name, paren, statement]
     end
 
     def format(q)
       q.group do
         q.text("def ")
+
+        if target
+          q.format(target)
+          q.format(CallOperatorFormatter.new(operator), stackable: false)
+        end
+
         q.format(name)
-        q.format(paren) if paren && !paren.contents.empty?
+
+        if paren
+          params = paren
+          params = params.contents if params.is_a?(Paren)
+          q.format(paren) unless params.empty?
+        end
+
         q.text(" =")
         q.group do
           q.indent do
@@ -4164,6 +4609,14 @@ class SyntaxTree < Ripper
       q.group(2, "(", ")") do
         q.text("def_endless")
 
+        if target
+          q.breakable
+          q.pp(target)
+
+          q.breakable
+          q.pp(operator)
+        end
+
         q.breakable
         q.pp(name)
 
@@ -4206,19 +4659,6 @@ class SyntaxTree < Ripper
     # and normal method definitions.
     beginning = find_token(Kw, "def")
 
-    # If we don't have a bodystmt node, then we have a single-line method
-    unless bodystmt.is_a?(BodyStmt)
-      node =
-        DefEndless.new(
-          name: name,
-          paren: params,
-          statement: bodystmt,
-          location: beginning.location.to(bodystmt.location)
-        )
-
-      return node
-    end
-
     # If there aren't any params then we need to correct the params node
     # location information
     if params.is_a?(Params) && params.empty?
@@ -4234,18 +4674,35 @@ class SyntaxTree < Ripper
       params = Params.new(location: location)
     end
 
-    ending = find_token(Kw, "end")
-    bodystmt.bind(
-      find_next_statement_start(params.location.end_char),
-      ending.location.start_char
-    )
+    ending = find_token(Kw, "end", consume: false)
 
-    Def.new(
-      name: name,
-      params: params,
-      bodystmt: bodystmt,
-      location: beginning.location.to(ending.location)
-    )
+    if ending
+      tokens.delete(ending)
+      bodystmt.bind(
+        find_next_statement_start(params.location.end_char),
+        ending.location.start_char
+      )
+
+      Def.new(
+        name: name,
+        params: params,
+        bodystmt: bodystmt,
+        location: beginning.location.to(ending.location)
+      )
+    else
+      # In Ruby >= 3.1.0, this is a BodyStmt that wraps a single statement in
+      # the statements list. Before, it was just the individual statement.
+      statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt
+
+      DefEndless.new(
+        target: nil,
+        operator: nil,
+        name: name,
+        paren: params,
+        statement: statement,
+        location: beginning.location.to(bodystmt.location)
+      )
+    end
   end
 
   # Defined represents the use of the +defined?+ operator. It can be used with
@@ -4369,16 +4826,9 @@ class SyntaxTree < Ripper
         q.group do
           q.text("def ")
           q.format(target)
-          q.format(CallOperatorFormatter.new(operator))
+          q.format(CallOperatorFormatter.new(operator), stackable: false)
           q.format(name)
-
-          if params.is_a?(Params) && !params.empty?
-            q.text("(")
-            q.format(params)
-            q.text(")")
-          else
-            q.format(params)
-          end
+          q.format(params) if !params.is_a?(Params) || !params.empty?
         end
 
         unless bodystmt.empty?
@@ -4460,21 +4910,37 @@ class SyntaxTree < Ripper
     end
 
     beginning = find_token(Kw, "def")
-    ending = find_token(Kw, "end")
+    ending = find_token(Kw, "end", consume: false)
 
-    bodystmt.bind(
-      find_next_statement_start(params.location.end_char),
-      ending.location.start_char
-    )
+    if ending
+      tokens.delete(ending)
+      bodystmt.bind(
+        find_next_statement_start(params.location.end_char),
+        ending.location.start_char
+      )
 
-    Defs.new(
-      target: target,
-      operator: operator,
-      name: name,
-      params: params,
-      bodystmt: bodystmt,
-      location: beginning.location.to(ending.location)
-    )
+      Defs.new(
+        target: target,
+        operator: operator,
+        name: name,
+        params: params,
+        bodystmt: bodystmt,
+        location: beginning.location.to(ending.location)
+      )
+    else
+      # In Ruby >= 3.1.0, this is a BodyStmt that wraps a single statement in
+      # the statements list. Before, it was just the individual statement.
+      statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt
+
+      DefEndless.new(
+        target: target,
+        operator: operator,
+        name: name,
+        paren: params,
+        statement: statement,
+        location: beginning.location.to(bodystmt.location)
+      )
+    end
   end
 
   # DoBlock represents passing a block to a method call using the +do+ and +end+
@@ -4512,7 +4978,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      BlockFormatter.new(self, keyword, bodystmt).format(q)
+      BlockFormatter.new(self, keyword, "end", bodystmt).format(q)
     end
 
     def pretty_print(q)
@@ -4576,8 +5042,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      parent = q.parent
-      space = parent.is_a?(If) || parent.is_a?(Unless)
+      space = [If, IfMod, Unless, UnlessMod].include?(q.parent.class)
 
       left = node.left
       right = node.right
@@ -4890,9 +5355,9 @@ class SyntaxTree < Ripper
         matching = Quotes.matching(quote[2])
         pattern = /[\n#{Regexp.escape(matching)}'"]/
 
-        if parts.any? do |part|
+        if parts.any? { |part|
              part.is_a?(TStringContent) && part.value.match?(pattern)
-           end
+           }
           [quote, matching]
         elsif Quotes.locked?(self)
           ["#{":" unless hash_key}'", "'"]
@@ -4917,7 +5382,7 @@ class SyntaxTree < Ripper
     if find_token(SymBeg, consume: false)
       # A normal dynamic symbol
       symbeg = find_token(SymBeg)
-      tstring_end = find_token(TStringEnd)
+      tstring_end = find_token(TStringEnd, location: symbeg.location)
 
       DynaSymbol.new(
         quote: symbeg.value,
@@ -5009,6 +5474,7 @@ class SyntaxTree < Ripper
 
     node = tokens[index]
     ending = node.value == "end" ? tokens.delete_at(index) : node
+    # ending = node
 
     statements.bind(beginning.location.end_char, ending.location.start_char)
 
@@ -5155,6 +5621,10 @@ class SyntaxTree < Ripper
       false
     end
 
+    def ignore?
+      false
+    end
+
     def comments
       []
     end
@@ -5480,24 +5950,29 @@ class SyntaxTree < Ripper
     # [Const | Ident] the name of the method
     attr_reader :value
 
+    # [nil | ArgParen | Args] the arguments to the method call
+    attr_reader :arguments
+
     # [Location] the location of this node
     attr_reader :location
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(value:, location:, comments: [])
+    def initialize(value:, arguments:, location:, comments: [])
       @value = value
+      @arguments = arguments
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [value]
+      [value, arguments]
     end
 
     def format(q)
       q.format(value)
+      q.format(arguments)
     end
 
     def pretty_print(q)
@@ -5507,21 +5982,30 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(value)
 
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
+
         q.pp(Comment::List.new(comments))
       end
     end
 
     def to_json(*opts)
-      { type: :fcall, value: value, loc: location, cmts: comments }.to_json(
-        *opts
-      )
+      {
+        type: :fcall,
+        value: value,
+        args: arguments,
+        loc: location,
+        cmts: comments
+      }.to_json(*opts)
     end
   end
 
   # :call-seq:
   #   on_fcall: ((Const | Ident) value) -> FCall
   def on_fcall(value)
-    FCall.new(value: value, location: value.location)
+    FCall.new(value: value, arguments: nil, location: value.location)
   end
 
   # Field is always the child of an assignment. It represents assigning to a
@@ -5560,7 +6044,7 @@ class SyntaxTree < Ripper
     def format(q)
       q.group do
         q.format(parent)
-        q.format(CallOperatorFormatter.new(operator))
+        q.format(CallOperatorFormatter.new(operator), stackable: false)
         q.format(name)
       end
     end
@@ -5864,6 +6348,7 @@ class SyntaxTree < Ripper
   #   ) -> For
   def on_for(index, collection, statements)
     beginning = find_token(Kw, "for")
+    in_keyword = find_token(Kw, "in")
     ending = find_token(Kw, "end")
 
     # Consume the do keyword if it exists so that it doesn't get confused for
@@ -5879,6 +6364,11 @@ class SyntaxTree < Ripper
       ending.location.start_char
     )
 
+    if index.is_a?(MLHS)
+      comma_range = index.location.end_char...in_keyword.location.start_char
+      index.comma = true if source[comma_range].strip.start_with?(",")
+    end
+
     For.new(
       index: index,
       collection: collection,
@@ -5947,6 +6437,9 @@ class SyntaxTree < Ripper
   #     { key => value }
   #
   class HashLiteral
+    # [LBrace] the left brace that opens this hash
+    attr_reader :lbrace
+
     # [Array[ AssocNew | AssocSplat ]] the optional contents of the hash
     attr_reader :assocs
 
@@ -5956,28 +6449,27 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(assocs:, location:, comments: [])
+    def initialize(lbrace:, assocs:, location:, comments: [])
+      @lbrace = lbrace
       @assocs = assocs
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      assocs
+      [lbrace] + assocs
     end
 
     def format(q)
-      contents = -> do
-        q.text("{")
-        q.indent do
-          q.breakable
-          q.format(HashFormatter.for(self))
-        end
-        q.breakable
-        q.text("}")
+      if q.parent.is_a?(Assoc)
+        format_contents(q)
+      else
+        q.group { format_contents(q) }
       end
+    end
 
-      q.parent.is_a?(Assoc) ? contents.call : q.group(&contents)
+    def format_key(q, key)
+      (@key_formatter ||= HashKeyFormatter.for(self)).format_key(q, key)
     end
 
     def pretty_print(q)
@@ -5998,6 +6490,24 @@ class SyntaxTree < Ripper
         *opts
       )
     end
+
+    private
+
+    def format_contents(q)
+      q.format(lbrace)
+
+      if assocs.empty?
+        q.breakable("")
+      else
+        q.indent do
+          q.breakable
+          q.seplist(assocs) { |assoc| q.format(assoc) }
+        end
+        q.breakable
+      end
+
+      q.text("}")
+    end
   end
 
   # :call-seq:
@@ -6007,6 +6517,7 @@ class SyntaxTree < Ripper
     rbrace = find_token(RBrace)
 
     HashLiteral.new(
+      lbrace: lbrace,
       assocs: assocs || [],
       location: lbrace.location.to(rbrace.location)
     )
@@ -6059,7 +6570,7 @@ class SyntaxTree < Ripper
       q.group do
         q.format(beginning)
 
-        q.line_suffix do
+        q.line_suffix(priority: Formatter::HEREDOC_PRIORITY) do
           q.group do
             breakable.call
 
@@ -6282,7 +6793,9 @@ class SyntaxTree < Ripper
     def format(q)
       parts = keywords.map { |(key, value)| KeywordFormatter.new(key, value) }
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
-      contents = -> { q.seplist(parts) { |part| q.format(part) } }
+      contents = -> do
+        q.seplist(parts) { |part| q.format(part, stackable: false) }
+      end
 
       if constant
         q.format(constant)
@@ -6293,7 +6806,7 @@ class SyntaxTree < Ripper
       end
 
       parent = q.parent
-      if PATTERNS.any? { |pattern| parent.is_a?(pattern) }
+      if PATTERNS.include?(parent.class)
         q.text("{ ")
         contents.call
         q.text(" }")
@@ -6428,6 +6941,23 @@ class SyntaxTree < Ripper
     )
   end
 
+  # If the predicate of a conditional or loop contains an assignment (in which
+  # case we can't know for certain that that assignment doesn't impact the
+  # statements inside the conditional) then we can't use the modifier form
+  # and we must use the block form.
+  module ContainsAssignment
+    def self.call(parent)
+      queue = [parent]
+
+      while node = queue.shift
+        return true if [Assign, MAssign, OpAssign].include?(node.class)
+        queue += node.child_nodes
+      end
+
+      false
+    end
+  end
+
   # Formats an If or Unless node.
   class ConditionalFormatter
     # [String] the keyword associated with this conditional
@@ -6442,38 +6972,50 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      statements = node.statements
-      break_format = ->(force:) do
-        q.text("#{keyword} ")
-        q.nest(keyword.length + 1) { q.format(node.predicate) }
+      # If the predicate of the conditional contains an assignment (in which
+      # case we can't know for certain that that assignment doesn't impact the
+      # statements inside the conditional) then we can't use the modifier form
+      # and we must use the block form.
+      if ContainsAssignment.call(node.predicate)
+        format_break(q, force: true)
+        return
+      end
 
-        unless statements.empty?
-          q.indent do
-            q.breakable(force: force)
-            q.format(statements)
+      if node.consequent || node.statements.empty?
+        q.group { format_break(q, force: true) }
+      else
+        q.group do
+          q.if_break { format_break(q, force: false) }.if_flat do
+            Parentheses.flat(q) do
+              q.format(node.statements)
+              q.text(" #{keyword} ")
+              q.format(node.predicate)
+            end
           end
         end
+      end
+    end
+
+    private
+
+    def format_break(q, force:)
+      q.text("#{keyword} ")
+      q.nest(keyword.length + 1) { q.format(node.predicate) }
 
-        if node.consequent
+      unless node.statements.empty?
+        q.indent do
           q.breakable(force: force)
-          q.format(node.consequent)
+          q.format(node.statements)
         end
+      end
 
+      if node.consequent
         q.breakable(force: force)
-        q.text("end")
+        q.format(node.consequent)
       end
 
-      if node.consequent || statements.empty?
-        q.group { break_format.call(force: true) }
-      else
-        q.group do
-          q.if_break { break_format.call(force: false) }.if_flat do
-            q.format(node.statements)
-            q.text(" #{keyword} ")
-            q.format(node.predicate)
-          end
-        end
-      end
+      q.breakable(force: force)
+      q.text("end")
     end
   end
 
@@ -6604,38 +7146,19 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group do
-        q.if_break do
-          q.text("if ")
-          q.nest("if ".length) { q.format(predicate) }
-
-          q.indent do
-            q.breakable
-            q.format(truthy)
-          end
-
-          q.breakable
-          q.text("else")
-
-          q.indent do
-            q.breakable
-            q.format(falsy)
-          end
-
-          q.breakable
-          q.text("end")
-        end.if_flat do
-          q.format(predicate)
-          q.text(" ?")
-
-          q.breakable
-          q.format(truthy)
-          q.text(" :")
+      force_flat = [
+        Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp,
+        Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super,
+        Undef, Unless, UnlessMod, UntilMod, VarAlias, VoidStmt, WhileMod, Yield,
+        Yield0, ZSuper
+      ]
 
-          q.breakable
-          q.format(falsy)
-        end
+      if force_flat.include?(truthy.class) || force_flat.include?(falsy.class)
+        q.group { format_flat(q) }
+        return
       end
+
+      q.group { q.if_break { format_break(q) }.if_flat { format_flat(q) } }
     end
 
     def pretty_print(q)
@@ -6665,6 +7188,43 @@ class SyntaxTree < Ripper
         cmts: comments
       }.to_json(*opts)
     end
+
+    private
+
+    def format_break(q)
+      Parentheses.break(q) do
+        q.text("if ")
+        q.nest("if ".length) { q.format(predicate) }
+
+        q.indent do
+          q.breakable
+          q.format(truthy)
+        end
+
+        q.breakable
+        q.text("else")
+
+        q.indent do
+          q.breakable
+          q.format(falsy)
+        end
+
+        q.breakable
+        q.text("end")
+      end
+    end
+
+    def format_flat(q)
+      q.format(predicate)
+      q.text(" ?")
+
+      q.breakable
+      q.format(truthy)
+      q.text(" :")
+
+      q.breakable
+      q.format(falsy)
+    end
   end
 
   # :call-seq:
@@ -6692,21 +7252,31 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group do
-        q.if_break do
-          q.text("#{keyword} ")
-          q.nest(keyword.length + 1) { q.format(node.predicate) }
-          q.indent do
-            q.breakable
-            q.format(node.statement)
-          end
-          q.breakable
-          q.text("end")
-        end.if_flat do
-          q.format(node.statement)
-          q.text(" #{keyword} ")
-          q.format(node.predicate)
-        end
+      if ContainsAssignment.call(node.statement)
+        q.group { format_flat(q) }
+      else
+        q.group { q.if_break { format_break(q) }.if_flat { format_flat(q) } }
+      end
+    end
+
+    private
+
+    def format_break(q)
+      q.text("#{keyword} ")
+      q.nest(keyword.length + 1) { q.format(node.predicate) }
+      q.indent do
+        q.breakable
+        q.format(node.statement)
+      end
+      q.breakable
+      q.text("end")
+    end
+
+    def format_flat(q)
+      Parentheses.flat(q) do
+        q.format(node.statement)
+        q.text(" #{keyword} ")
+        q.format(node.predicate)
       end
     end
   end
@@ -6944,8 +7514,14 @@ class SyntaxTree < Ripper
     beginning = find_token(Kw, "in")
     ending = consequent || find_token(Kw, "end")
 
+    statements_start = pattern
+    if token = find_token(Kw, "then", consume: false)
+      tokens.delete(token)
+      statements_start = token
+    end
+
     statements.bind(
-      find_next_statement_start(pattern.location.end_char),
+      find_next_statement_start(statements_start.location.end_char),
       ending.location.start_char
     )
 
@@ -6982,7 +7558,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      if !value.start_with?("0") && value.length >= 5 && !value.include?("_")
+      if !value.start_with?(/\+?0/) && value.length >= 5 && !value.include?("_")
         # If it's a plain integer and it doesn't have any underscores separating
         # the values, then we're going to insert them every 3 characters
         # starting from the right.
@@ -7330,9 +7906,11 @@ class SyntaxTree < Ripper
         if params.is_a?(Paren)
           q.format(params) unless params.contents.empty?
         elsif !params.empty?
-          q.text("(")
-          q.format(params)
-          q.text(")")
+          q.group do
+            q.text("(")
+            q.format(params)
+            q.text(")")
+          end
         end
 
         q.text(" ")
@@ -7391,8 +7969,11 @@ class SyntaxTree < Ripper
   def on_lambda(params, statements)
     beginning = find_token(TLambda)
 
-    if token = find_token(TLamBeg, consume: false)
-      opening = tokens.delete(token)
+    if tokens.any? { |token|
+         token.is_a?(TLamBeg) &&
+           token.location.start_char > beginning.location.start_char
+       }
+      opening = find_token(TLamBeg)
       closing = find_token(RBrace)
     else
       opening = find_token(Kw, "do")
@@ -7472,9 +8053,38 @@ class SyntaxTree < Ripper
     # [Location] the location of this node
     attr_reader :location
 
-    def initialize(value:, location:)
+    # [Array[ Comment | EmbDoc ]] the comments attached to this node
+    attr_reader :comments
+
+    def initialize(value:, location:, comments: [])
       @value = value
       @location = location
+      @comments = comments
+    end
+
+    def child_nodes
+      []
+    end
+
+    def format(q)
+      q.text(value)
+    end
+
+    def pretty_print(q)
+      q.group(2, "(", ")") do
+        q.text("lbracket")
+
+        q.breakable
+        q.pp(value)
+
+        q.pp(Comment::List.new(comments))
+      end
+    end
+
+    def to_json(*opts)
+      { type: :lbracket, value: value, loc: location, cmts: comments }.to_json(
+        *opts
+      )
     end
   end
 
@@ -7565,104 +8175,12 @@ class SyntaxTree < Ripper
   #
   #     first, = value
   #
-  class MAssign
-    # [MLHS | MLHSParen] the target of the multiple assignment
-    attr_reader :target
-
-    # [untyped] the value being assigned
-    attr_reader :value
-
-    # [Location] the location of this node
-    attr_reader :location
-
-    # [Array[ Comment | EmbDoc ]] the comments attached to this node
-    attr_reader :comments
-
-    def initialize(target:, value:, location:, comments: [])
-      @target = target
-      @value = value
-      @location = location
-      @comments = comments
-    end
-
-    def child_nodes
-      [target, value]
-    end
-
-    def format(q)
-      q.group do
-        q.group do
-          q.format(target)
-          q.text(",") if target.is_a?(MLHS) && target.comma
-        end
-
-        q.text(" =")
-        q.indent do
-          q.breakable
-          q.format(value)
-        end
-      end
-    end
-
-    def pretty_print(q)
-      q.group(2, "(", ")") do
-        q.text("massign")
-
-        q.breakable
-        q.pp(target)
-
-        q.breakable
-        q.pp(value)
-
-        q.pp(Comment::List.new(comments))
-      end
-    end
-
-    def to_json(*opts)
-      {
-        type: :massign,
-        target: target,
-        value: value,
-        loc: location,
-        cmts: comments
-      }.to_json(*opts)
-    end
-  end
-
-  # :call-seq:
-  #   on_massign: ((MLHS | MLHSParen) target, untyped value) -> MAssign
-  def on_massign(target, value)
-    comma_range = target.location.end_char...value.location.start_char
-    target.comma = true if source[comma_range].strip.start_with?(",")
-
-    MAssign.new(
-      target: target,
-      value: value,
-      location: target.location.to(value.location)
-    )
-  end
-
-  # MethodAddArg represents a method call with arguments and parentheses.
-  #
-  #     method(argument)
-  #
-  # MethodAddArg can also represent with a method on an object, as in:
-  #
-  #     object.method(argument)
-  #
-  # Finally, MethodAddArg can represent calling a method with no receiver that
-  # ends in a ?. In this case, the parser knows it's a method call and not a
-  # local variable, so it uses a MethodAddArg node as opposed to a VCall node,
-  # as in:
-  #
-  #     method?
-  #
-  class MethodAddArg
-    # [Call | FCall] the method call
-    attr_reader :call
+  class MAssign
+    # [MLHS | MLHSParen] the target of the multiple assignment
+    attr_reader :target
 
-    # [ArgParen | Args] the arguments to the method call
-    attr_reader :arguments
+    # [untyped] the value being assigned
+    attr_reader :value
 
     # [Location] the location of this node
     attr_reader :location
@@ -7670,32 +8188,37 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(call:, arguments:, location:, comments: [])
-      @call = call
-      @arguments = arguments
+    def initialize(target:, value:, location:, comments: [])
+      @target = target
+      @value = value
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [call, arguments]
+      [target, value]
     end
 
     def format(q)
-      q.format(call)
-      q.text(" ") if !arguments.is_a?(ArgParen) && arguments.parts.any?
-      q.format(arguments)
+      q.group do
+        q.group { q.format(target) }
+        q.text(" =")
+        q.indent do
+          q.breakable
+          q.format(value)
+        end
+      end
     end
 
     def pretty_print(q)
       q.group(2, "(", ")") do
-        q.text("method_add_arg")
+        q.text("massign")
 
         q.breakable
-        q.pp(call)
+        q.pp(target)
 
         q.breakable
-        q.pp(arguments)
+        q.pp(value)
 
         q.pp(Comment::List.new(comments))
       end
@@ -7703,25 +8226,48 @@ class SyntaxTree < Ripper
 
     def to_json(*opts)
       {
-        type: :method_add_arg,
-        call: call,
-        args: arguments,
+        type: :massign,
+        target: target,
+        value: value,
         loc: location,
         cmts: comments
       }.to_json(*opts)
     end
   end
 
+  # :call-seq:
+  #   on_massign: ((MLHS | MLHSParen) target, untyped value) -> MAssign
+  def on_massign(target, value)
+    comma_range = target.location.end_char...value.location.start_char
+    target.comma = true if source[comma_range].strip.start_with?(",")
+
+    MAssign.new(
+      target: target,
+      value: value,
+      location: target.location.to(value.location)
+    )
+  end
+
   # :call-seq:
   #   on_method_add_arg: (
   #     (Call | FCall) call,
   #     (ArgParen | Args) arguments
-  #   ) -> MethodAddArg
+  #   ) -> Call | FCall
   def on_method_add_arg(call, arguments)
     location = call.location
-    location = location.to(arguments.location) unless arguments.is_a?(Args)
+    location = location.to(arguments.location) if arguments.is_a?(ArgParen)
 
-    MethodAddArg.new(call: call, arguments: arguments, location: location)
+    if call.is_a?(FCall)
+      FCall.new(value: call.value, arguments: arguments, location: location)
+    else
+      Call.new(
+        receiver: call.receiver,
+        operator: call.operator,
+        message: call.message,
+        arguments: arguments,
+        location: location
+      )
+    end
   end
 
   # MethodAddBlock represents a method call with a block argument.
@@ -7729,7 +8275,7 @@ class SyntaxTree < Ripper
   #     method {}
   #
   class MethodAddBlock
-    # [Call | Command | CommandCall | FCall | MethodAddArg] the method call
+    # [Call | Command | CommandCall | FCall] the method call
     attr_reader :call
 
     # [BraceBlock | DoBlock] the block being sent with the method call
@@ -7784,7 +8330,7 @@ class SyntaxTree < Ripper
 
   # :call-seq:
   #   on_method_add_block: (
-  #     (Call | Command | CommandCall | FCall | MethodAddArg) call,
+  #     (Call | Command | CommandCall | FCall) call,
   #     (BraceBlock | DoBlock) block
   #   ) -> MethodAddBlock
   def on_method_add_block(call, block)
@@ -7809,7 +8355,6 @@ class SyntaxTree < Ripper
     # list, which impacts destructuring. It's an attr_accessor so that while
     # the syntax tree is being built it can be set by its parent node
     attr_accessor :comma
-    alias comma? comma
 
     # [Location] the location of this node
     attr_reader :location
@@ -7830,6 +8375,7 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.seplist(parts) { |part| q.format(part) }
+      q.text(",") if comma
     end
 
     def pretty_print(q)
@@ -7932,7 +8478,6 @@ class SyntaxTree < Ripper
           q.indent do
             q.breakable("")
             q.format(contents)
-            q.text(",") if contents.is_a?(MLHS) && contents.comma?
           end
 
           q.breakable("")
@@ -8400,6 +8945,61 @@ class SyntaxTree < Ripper
     )
   end
 
+  # If you have a modifier statement (for instance a modifier if statement or a
+  # modifier while loop) there are times when you need to wrap the entire
+  # statement in parentheses. This occurs when you have something like:
+  #
+  #     foo[:foo] =
+  #       if bar?
+  #         baz
+  #       end
+  #
+  # Normally we would shorten this to an inline version, which would result in:
+  #
+  #     foo[:foo] = baz if bar?
+  #
+  # but this actually has different semantic meaning. The first example will
+  # result in a nil being inserted into the hash for the :foo key, whereas the
+  # second example will result in an empty hash because the if statement applies
+  # to the entire assignment.
+  #
+  # We can fix this in a couple of ways. We can use the then keyword, as in:
+  #
+  #     foo[:foo] = if bar? then baz end
+  #
+  # But this isn't used very often. We can also just leave it as is with the
+  # multi-line version, but for a short predicate and short value it looks
+  # verbose. The last option and the one used here is to add parentheses on
+  # both sides of the expression, as in:
+  #
+  #     foo[:foo] = (baz if bar?)
+  #
+  # This approach maintains the nice conciseness of the inline version, while
+  # keeping the correct semantic meaning.
+  module Parentheses
+    NODES = [Args, Assign, Assoc, Binary, Call, Defined, MAssign, OpAssign]
+
+    def self.flat(q)
+      return yield unless NODES.include?(q.parent.class)
+
+      q.text("(")
+      yield
+      q.text(")")
+    end
+
+    def self.break(q)
+      return yield unless NODES.include?(q.parent.class)
+
+      q.text("(")
+      q.indent do
+        q.breakable("")
+        yield
+      end
+      q.breakable("")
+      q.text(")")
+    end
+  end
+
   # def on_operator_ambiguous(value)
   #   value
   # end
@@ -8459,7 +9059,7 @@ class SyntaxTree < Ripper
     end
 
     class KeywordRestFormatter
-      # [:nil | KwRestParam] the value of the parameter
+      # [:nil | ArgsForward | KwRestParam] the value of the parameter
       attr_reader :value
 
       def initialize(value)
@@ -8538,9 +9138,7 @@ class SyntaxTree < Ripper
     # it's missing.
     def empty?
       requireds.empty? && optionals.empty? && !rest && posts.empty? &&
-        keywords.empty? &&
-        !keyword_rest &&
-        !block
+        keywords.empty? && !keyword_rest && !block
     end
 
     def child_nodes
@@ -8571,10 +9169,22 @@ class SyntaxTree < Ripper
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
       parts << block if block
 
-      q.nest(0) do
+      contents = -> do
         q.seplist(parts) { |part| q.format(part) }
         q.format(rest) if rest && rest.is_a?(ExcessedComma)
       end
+
+      if [Def, Defs, DefEndless].include?(q.parent.class)
+        q.group(0, "(", ")") do
+          q.indent do
+            q.breakable("")
+            contents.call
+          end
+          q.breakable("")
+        end
+      else
+        q.nest(0, &contents)
+      end
     end
 
     def pretty_print(q)
@@ -8664,8 +9274,8 @@ class SyntaxTree < Ripper
   #     (nil | ArgsForward | ExcessedComma | RestParam) rest,
   #     (nil | Array[Ident]) posts,
   #     (nil | Array[[Ident, nil | untyped]]) keywords,
-  #     (nil | :nil | KwRestParam) keyword_rest,
-  #     (nil | BlockArg) block
+  #     (nil | :nil | ArgsForward | KwRestParam) keyword_rest,
+  #     (nil | :& | BlockArg) block
   #   ) -> Params
   def on_params(
     requireds,
@@ -8676,16 +9286,15 @@ class SyntaxTree < Ripper
     keyword_rest,
     block
   )
-    parts =
-      [
-        *requireds,
-        *optionals&.flatten(1),
-        rest,
-        *posts,
-        *keywords&.flat_map { |(key, value)| [key, value || nil] },
-        (keyword_rest if keyword_rest != :nil),
-        block
-      ].compact
+    parts = [
+      *requireds,
+      *optionals&.flatten(1),
+      rest,
+      *posts,
+      *keywords&.flat_map { |(key, value)| [key, value || nil] },
+      (keyword_rest if keyword_rest != :nil),
+      (block if block != :&)
+    ].compact
 
     location =
       if parts.any?
@@ -8701,7 +9310,7 @@ class SyntaxTree < Ripper
       posts: posts || [],
       keywords: keywords || [],
       keyword_rest: keyword_rest,
-      block: block,
+      block: (block if block != :&),
       location: location
     )
   end
@@ -8716,7 +9325,7 @@ class SyntaxTree < Ripper
     # [LParen] the left parenthesis that opened this statement
     attr_reader :lparen
 
-    # [untyped] the expression inside the parentheses
+    # [nil | untyped] the expression inside the parentheses
     attr_reader :contents
 
     # [Location] the location of this node
@@ -8740,7 +9349,7 @@ class SyntaxTree < Ripper
       q.group do
         q.format(lparen)
 
-        if !contents.is_a?(Params) || !contents.empty?
+        if contents && (!contents.is_a?(Params) || !contents.empty?)
           q.indent do
             q.breakable("")
             q.format(contents)
@@ -8805,7 +9414,7 @@ class SyntaxTree < Ripper
 
     Paren.new(
       lparen: lparen,
-      contents: contents,
+      contents: contents || nil,
       location: lparen.location.to(rparen.location)
     )
   end
@@ -8896,7 +9505,11 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.format(statements)
-      q.breakable(force: true)
+
+      # We're going to put a newline on the end so that it always has one unless
+      # it ends with the special __END__ syntax. In that case we want to
+      # replicate the text exactly so we will just let it be.
+      q.breakable(force: true) unless statements.body.last.is_a?(EndContent)
     end
 
     def pretty_print(q)
@@ -9035,6 +9648,9 @@ class SyntaxTree < Ripper
   #     %i[one two three]
   #
   class QSymbols
+    # [QSymbolsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ TStringContent ]] the elements of the array
     attr_reader :elements
 
@@ -9044,7 +9660,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -9055,7 +9672,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%i[", "]") do
+      opening, closing = "%i[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -9091,6 +9715,7 @@ class SyntaxTree < Ripper
   #   on_qsymbols_add: (QSymbols qsymbols, TStringContent element) -> QSymbols
   def on_qsymbols_add(qsymbols, element)
     QSymbols.new(
+      beginning: qsymbols.beginning,
       elements: qsymbols.elements << element,
       location: qsymbols.location.to(element.location)
     )
@@ -9132,9 +9757,13 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_qsymbols_new: () -> QSymbols
   def on_qsymbols_new
-    qsymbols_beg = find_token(QSymbolsBeg)
+    beginning = find_token(QSymbolsBeg)
 
-    QSymbols.new(elements: [], location: qsymbols_beg.location)
+    QSymbols.new(
+      beginning: beginning,
+      elements: [],
+      location: beginning.location
+    )
   end
 
   # QWords represents a string literal array without interpolation.
@@ -9142,6 +9771,9 @@ class SyntaxTree < Ripper
   #     %w[one two three]
   #
   class QWords
+    # [QWordsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ TStringContent ]] the elements of the array
     attr_reader :elements
 
@@ -9151,7 +9783,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -9162,7 +9795,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%w[", "]") do
+      opening, closing = "%w[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -9195,6 +9835,7 @@ class SyntaxTree < Ripper
   #   on_qwords_add: (QWords qwords, TStringContent element) -> QWords
   def on_qwords_add(qwords, element)
     QWords.new(
+      beginning: qwords.beginning,
       elements: qwords.elements << element,
       location: qwords.location.to(element.location)
     )
@@ -9236,9 +9877,9 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_qwords_new: () -> QWords
   def on_qwords_new
-    qwords_beg = find_token(QWordsBeg)
+    beginning = find_token(QWordsBeg)
 
-    QWords.new(elements: [], location: qwords_beg.location)
+    QWords.new(beginning: beginning, elements: [], location: beginning.location)
   end
 
   # RationalLiteral represents the use of a rational number literal.
@@ -9550,11 +10191,32 @@ class SyntaxTree < Ripper
           q.format_each(parts)
           q.text(ending)
         end
+      elsif braces
+        q.group do
+          q.text("%r{")
+
+          if beginning == "/"
+            # If we're changing from a forward slash to a %r{, then we can
+            # replace any escaped forward slashes with regular forward slashes.
+            parts.each do |part|
+              if part.is_a?(TStringContent)
+                q.text(part.value.gsub("\\/", "/"))
+              else
+                q.format(part)
+              end
+            end
+          else
+            q.format_each(parts)
+          end
+
+          q.text("}")
+          q.text(ending[1..-1])
+        end
       else
         q.group do
-          q.text(braces ? "%r{" : "/")
+          q.text("/")
           q.format_each(parts)
-          q.text(braces ? "}" : "/")
+          q.text("/")
           q.text(ending[1..-1])
         end
       end
@@ -10285,21 +10947,6 @@ class SyntaxTree < Ripper
   #   value
   # end
 
-  # stmts_add is a parser event that represents a single statement inside a
-  # list of statements within any lexical block. It accepts as arguments the
-  # parent stmts node as well as an stmt which can be any expression in
-  # Ruby.
-  def on_stmts_add(statements, statement)
-    location =
-      if statements.body.empty?
-        statement.location
-      else
-        statements.location.to(statement.location)
-      end
-
-    Statements.new(self, body: statements.body << statement, location: location)
-  end
-
   # Everything that has a block of code inside of it has a list of statements.
   # Normally we would just track those as a node that has an array body, but we
   # have some special handling in order to handle empty statement lists. They
@@ -10379,12 +11026,22 @@ class SyntaxTree < Ripper
       # the only value is a comment. In that case a lot of nodes like
       # brace_block will attempt to format as a single line, but since that
       # wouldn't work with a comment, we intentionally break the parent group.
-      if body.length == 2 && body.first.is_a?(VoidStmt)
-        q.format(body.last)
-        q.break_parent
-        return
+      if body.length == 2
+        void_stmt, comment = body
+
+        if void_stmt.is_a?(VoidStmt) && comment.is_a?(Comment)
+          q.format(comment)
+          q.break_parent
+          return
+        end
       end
 
+      access_controls =
+        Hash.new do |hash, node|
+          hash[node] = node.is_a?(VCall) &&
+            %w[private protected public].include?(node.value.value)
+        end
+
       body.each_with_index do |statement, index|
         next if statement.is_a?(VoidStmt)
 
@@ -10394,7 +11051,7 @@ class SyntaxTree < Ripper
           q.breakable(force: true)
           q.breakable(force: true)
           q.format(statement)
-        elsif statement.is_a?(AccessCtrl) || body[index - 1].is_a?(AccessCtrl)
+        elsif access_controls[statement] || access_controls[body[index - 1]]
           q.breakable(force: true)
           q.breakable(force: true)
           q.format(statement)
@@ -10446,7 +11103,7 @@ class SyntaxTree < Ripper
         location = comment.location
 
         if !comment.inline? && (start_char <= location.start_char) &&
-             (end_char >= location.end_char)
+             (end_char >= location.end_char) && !comment.ignore?
           parser_comments.delete_at(comment_index)
 
           while (node = body[body_index]) &&
@@ -10465,6 +11122,21 @@ class SyntaxTree < Ripper
     end
   end
 
+  # stmts_add is a parser event that represents a single statement inside a
+  # list of statements within any lexical block. It accepts as arguments the
+  # parent stmts node as well as an stmt which can be any expression in
+  # Ruby.
+  def on_stmts_add(statements, statement)
+    location =
+      if statements.body.empty?
+        statement.location
+      else
+        statements.location.to(statement.location)
+      end
+
+    Statements.new(self, body: statements.body << statement, location: location)
+  end
+
   # :call-seq:
   #   on_stmts_new: () -> Statements
   def on_stmts_new
@@ -10736,10 +11408,18 @@ class SyntaxTree < Ripper
       embexpr_end.location.start_char
     )
 
-    StringEmbExpr.new(
-      statements: statements,
-      location: embexpr_beg.location.to(embexpr_end.location)
-    )
+    location =
+      Location.new(
+        start_line: embexpr_beg.location.start_line,
+        start_char: embexpr_beg.location.start_char,
+        end_line: [
+          embexpr_end.location.end_line,
+          statements.location.end_line
+        ].max,
+        end_char: embexpr_end.location.end_char
+      )
+
+    StringEmbExpr.new(statements: statements, location: location)
   end
 
   # StringLiteral represents a string literal.
@@ -10839,12 +11519,23 @@ class SyntaxTree < Ripper
       )
     else
       tstring_beg = find_token(TStringBeg)
-      tstring_end = find_token(TStringEnd)
+      tstring_end = find_token(TStringEnd, location: tstring_beg.location)
+
+      location =
+        Location.new(
+          start_line: tstring_beg.location.start_line,
+          start_char: tstring_beg.location.start_char,
+          end_line: [
+            tstring_end.location.end_line,
+            string.location.end_line
+          ].max,
+          end_char: tstring_end.location.end_char
+        )
 
       StringLiteral.new(
         parts: string.parts,
         quote: tstring_beg.value,
-        location: tstring_beg.location.to(tstring_end.location)
+        location: location
       )
     end
   end
@@ -11066,6 +11757,9 @@ class SyntaxTree < Ripper
   #     %I[one two three]
   #
   class Symbols
+    # [SymbolsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ Word ]] the words in the symbol array literal
     attr_reader :elements
 
@@ -11075,7 +11769,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -11086,7 +11781,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%I[", "]") do
+      opening, closing = "%I[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -11122,6 +11824,7 @@ class SyntaxTree < Ripper
   #   on_symbols_add: (Symbols symbols, Word word) -> Symbols
   def on_symbols_add(symbols, word)
     Symbols.new(
+      beginning: symbols.beginning,
       elements: symbols.elements << word,
       location: symbols.location.to(word.location)
     )
@@ -11164,9 +11867,13 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_symbols_new: () -> Symbols
   def on_symbols_new
-    symbols_beg = find_token(SymbolsBeg)
+    beginning = find_token(SymbolsBeg)
 
-    Symbols.new(elements: [], location: symbols_beg.location)
+    Symbols.new(
+      beginning: beginning,
+      elements: [],
+      location: beginning.location
+    )
   end
 
   # TLambda represents the beginning of a lambda literal.
@@ -11258,6 +11965,11 @@ class SyntaxTree < Ripper
       [constant]
     end
 
+    def format(q)
+      q.text("::")
+      q.format(constant)
+    end
+
     def pretty_print(q)
       q.group(2, "(", ")") do
         q.text("top_const_field")
@@ -11412,6 +12124,10 @@ class SyntaxTree < Ripper
       @comments = comments
     end
 
+    def match?(pattern)
+      value.match?(pattern)
+    end
+
     def child_nodes
       []
     end
@@ -11914,23 +12630,35 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
+      if ContainsAssignment.call(node.predicate)
+        format_break(q)
+        q.break_parent
+        return
+      end
+
       q.group do
-        q.if_break do
-          q.text("#{keyword} ")
-          q.nest(keyword.length + 1) { q.format(node.predicate) }
-          q.indent do
-            q.breakable("")
+        q.if_break { format_break(q) }.if_flat do
+          Parentheses.flat(q) do
             q.format(statements)
+            q.text(" #{keyword} ")
+            q.format(node.predicate)
           end
-          q.breakable("")
-          q.text("end")
-        end.if_flat do
-          q.format(statements)
-          q.text(" #{keyword} ")
-          q.format(node.predicate)
         end
       end
     end
+
+    private
+
+    def format_break(q)
+      q.text("#{keyword} ")
+      q.nest(keyword.length + 1) { q.format(node.predicate) }
+      q.indent do
+        q.breakable("")
+        q.format(statements)
+      end
+      q.breakable("")
+      q.text("end")
+    end
   end
 
   # Until represents an +until+ loop.
@@ -12055,7 +12783,27 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      LoopFormatter.new("until", self, statement).format(q)
+      # If we're in the modifier form and we're modifying a `begin`, then this
+      # is a special case where we need to explicitly use the modifier form
+      # because otherwise the semantic meaning changes. This looks like:
+      #
+      #     begin
+      #       foo
+      #     end until bar
+      #
+      # Also, if the statement of the modifier includes an assignment, then we
+      # can't know for certain that it won't impact the predicate, so we need to
+      # force it to stay as it is. This looks like:
+      #
+      #     foo = bar until foo
+      #
+      if statement.is_a?(Begin) || ContainsAssignment.call(statement)
+        q.format(statement)
+        q.text(" until ")
+        q.format(predicate)
+      else
+        LoopFormatter.new("until", self, statement).format(q)
+      end
     end
 
     def pretty_print(q)
@@ -12285,19 +13033,17 @@ class SyntaxTree < Ripper
     end
   end
 
-  # :call-seq:
-  #   on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef
-  def on_var_ref(value)
-    VarRef.new(value: value, location: value.location)
-  end
-
-  # AccessCtrl represents a call to a method visibility control, i.e., +public+,
-  # +protected+, or +private+.
+  # PinnedVarRef represents a pinned variable reference within a pattern
+  # matching pattern.
   #
-  #     private
+  #     case value
+  #     in ^variable
+  #     end
   #
-  class AccessCtrl
-    # [Ident] the value of this expression
+  # This can be a plain local variable like the example above. It can also be a
+  # a class variable, a global variable, or an instance variable.
+  class PinnedVarRef
+    # [VarRef] the value of this node
     attr_reader :value
 
     # [Location] the location of this node
@@ -12317,12 +13063,15 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.format(value)
+      q.group do
+        q.text("^")
+        q.format(value)
+      end
     end
 
     def pretty_print(q)
       q.group(2, "(", ")") do
-        q.text("access_ctrl")
+        q.text("pinned_var_ref")
 
         q.breakable
         q.pp(value)
@@ -12332,12 +13081,21 @@ class SyntaxTree < Ripper
     end
 
     def to_json(*opts)
-      {
-        type: :access_ctrl,
-        value: value,
-        loc: location,
-        cmts: comments
-      }.to_json(*opts)
+      { type: :pinned_var_ref, value: value, loc: location, cmts: comments }
+        .to_json(*opts)
+    end
+  end
+
+  # :call-seq:
+  #   on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef
+  def on_var_ref(value)
+    pin = find_token(Op, "^", consume: false)
+
+    if pin && pin.location.start_char == value.location.start_char - 1
+      tokens.delete(pin)
+      PinnedVarRef.new(value: value, location: pin.location.to(value.location))
+    else
+      VarRef.new(value: value, location: value.location)
     end
   end
 
@@ -12389,19 +13147,9 @@ class SyntaxTree < Ripper
   end
 
   # :call-seq:
-  #   on_vcall: (Ident ident) -> AccessCtrl | VCall
+  #   on_vcall: (Ident ident) -> VCall
   def on_vcall(ident)
-    @controls ||= %w[private protected public].freeze
-
-    if @controls.include?(ident.value) && ident.value == lines[lineno - 1].strip
-      # Access controls like private, protected, and public are reported as
-      # vcall nodes since they're technically method calls. We want to be able
-      # add new lines around them as necessary, so here we're going to
-      # explicitly track those as a different node type.
-      AccessCtrl.new(value: ident, location: ident.location)
-    else
-      VCall.new(value: ident, location: ident.location)
-    end
+    VCall.new(value: ident, location: ident.location)
   end
 
   # VoidStmt represents an empty lexical block of code.
@@ -12420,6 +13168,10 @@ class SyntaxTree < Ripper
       @comments = comments
     end
 
+    def child_nodes
+      []
+    end
+
     def format(q)
     end
 
@@ -12494,6 +13246,14 @@ class SyntaxTree < Ripper
               separator = -> { q.group { q.comma_breakable } }
               q.seplist(arguments.parts, separator) { |part| q.format(part) }
             end
+
+            # Very special case here. If you're inside of a when clause and the
+            # last argument to the predicate is and endless range, then you are
+            # forced to use the "then" keyword to make it parse properly.
+            last = arguments.parts.last
+            if (last.is_a?(Dot2) || last.is_a?(Dot3)) && !last.right
+              q.text(" then")
+            end
           end
         end
 
@@ -12552,7 +13312,16 @@ class SyntaxTree < Ripper
     beginning = find_token(Kw, "when")
     ending = consequent || find_token(Kw, "end")
 
-    statements.bind(arguments.location.end_char, ending.location.start_char)
+    statements_start = arguments
+    if token = find_token(Kw, "then", consume: false)
+      tokens.delete(token)
+      statements_start = token
+    end
+
+    statements.bind(
+      find_next_statement_start(statements_start.location.end_char),
+      ending.location.start_char
+    )
 
     When.new(
       arguments: arguments,
@@ -12684,7 +13453,27 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      LoopFormatter.new("while", self, statement).format(q)
+      # If we're in the modifier form and we're modifying a `begin`, then this
+      # is a special case where we need to explicitly use the modifier form
+      # because otherwise the semantic meaning changes. This looks like:
+      #
+      #     begin
+      #       foo
+      #     end while bar
+      #
+      # Also, if the statement of the modifier includes an assignment, then we
+      # can't know for certain that it won't impact the predicate, so we need to
+      # force it to stay as it is. This looks like:
+      #
+      #     foo = bar while foo
+      #
+      if statement.is_a?(Begin) || ContainsAssignment.call(statement)
+        q.format(statement)
+        q.text(" while ")
+        q.format(predicate)
+      else
+        LoopFormatter.new("while", self, statement).format(q)
+      end
     end
 
     def pretty_print(q)
@@ -12748,6 +13537,10 @@ class SyntaxTree < Ripper
       @comments = comments
     end
 
+    def match?(pattern)
+      parts.any? { |part| part.is_a?(TStringContent) && part.match?(pattern) }
+    end
+
     def child_nodes
       parts
     end
@@ -12797,6 +13590,9 @@ class SyntaxTree < Ripper
   #     %W[one two three]
   #
   class Words
+    # [WordsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ Word ]] the elements of this array
     attr_reader :elements
 
@@ -12806,7 +13602,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -12817,7 +13614,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%W[", "]") do
+      opening, closing = "%W[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -12850,6 +13654,7 @@ class SyntaxTree < Ripper
   #   on_words_add: (Words words, Word word) -> Words
   def on_words_add(words, word)
     Words.new(
+      beginning: words.beginning,
       elements: words.elements << word,
       location: words.location.to(word.location)
     )
@@ -12892,9 +13697,9 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_words_new: () -> Words
   def on_words_new
-    words_beg = find_token(WordsBeg)
+    beginning = find_token(WordsBeg)
 
-    Words.new(elements: [], location: words_beg.location)
+    Words.new(beginning: beginning, elements: [], location: beginning.location)
   end
 
   # def on_words_sep(value)
@@ -13011,7 +13816,7 @@ class SyntaxTree < Ripper
         location: heredoc.location
       )
     else
-      ending = find_token(TStringEnd)
+      ending = find_token(TStringEnd, location: xstring.location)
 
       XStringLiteral.new(
         parts: xstring.parts,
@@ -13047,8 +13852,18 @@ class SyntaxTree < Ripper
     def format(q)
       q.group do
         q.text("yield")
-        q.text(" ") if arguments.is_a?(Args)
-        q.format(arguments)
+
+        if arguments.is_a?(Paren)
+          q.format(arguments)
+        else
+          q.if_break { q.text("(") }.if_flat { q.text(" ") }
+          q.indent do
+            q.breakable("")
+            q.format(arguments)
+          end
+          q.breakable("")
+          q.if_break { q.text(")") }
+        end
       end
     end