syntax_tree 0.1.0 → 1.0.0

data/lib/syntax_tree.rb

@@ -7,7 +7,7 @@ require "stringio"
 
 require_relative "syntax_tree/version"
 
-# If PrettyPrint::Assign isn't defined, then we haven't gotten the updated
+# If PrettyPrint::Align isn't defined, then we haven't gotten the updated
 # version of prettyprint. In that case we'll define our own. This is going to
 # overwrite a bunch of methods, so silencing them as well.
 unless PrettyPrint.const_defined?(:Align)
@@ -26,12 +26,14 @@ class SyntaxTree < Ripper
   # every character in the string is 1 byte in length, so we can just return the
   # start of the line + the index.
   class SingleByteString
+    attr_reader :start
+
     def initialize(start)
       @start = start
     end
 
     def [](byteindex)
-      @start + byteindex
+      start + byteindex
     end
   end
 
@@ -40,7 +42,10 @@ class SyntaxTree < Ripper
   # an array of indices, such that array[byteindex] will be equal to the index
   # of the character within the string.
   class MultiByteString
+    attr_reader :start, :indices
+
     def initialize(start, line)
+      @start = start
       @indices = []
 
       line.each_char.with_index(start) do |char, index|
@@ -48,8 +53,11 @@ class SyntaxTree < Ripper
       end
     end
 
+    # Technically it's possible for the column index to be a negative value if
+    # there's a BOM at the beginning of the file, which is the reason we need to
+    # compare it to 0 here.
     def [](byteindex)
-      @indices[byteindex]
+      indices[byteindex < 0 ? 0 : byteindex]
     end
   end
 
@@ -66,8 +74,7 @@ class SyntaxTree < Ripper
 
     def ==(other)
       other.is_a?(Location) && start_line == other.start_line &&
-        start_char == other.start_char &&
-        end_line == other.end_line &&
+        start_char == other.start_char && end_line == other.end_line &&
         end_char == other.end_char
     end
 
@@ -75,7 +82,7 @@ class SyntaxTree < Ripper
       Location.new(
         start_line: start_line,
         start_char: start_char,
-        end_line: other.end_line,
+        end_line: [end_line, other.end_line].max,
         end_char: other.end_char
       )
     end
@@ -113,10 +120,15 @@ class SyntaxTree < Ripper
   # A slightly enhanced PP that knows how to format recursively including
   # comments.
   class Formatter < PP
-    attr_reader :stack, :quote
+    COMMENT_PRIORITY = 1
+    HEREDOC_PRIORITY = 2
+
+    attr_reader :source, :stack, :quote
 
-    def initialize(*)
-      super
+    def initialize(source, ...)
+      super(...)
+
+      @source = source
       @stack = []
       @quote = "\""
     end
@@ -136,11 +148,17 @@ class SyntaxTree < Ripper
           breakable(force: true)
         end
 
-        doc = node.format(self)
+        # If the node has a stree-ignore comment right before it, then we're
+        # going to just print out the node as it was seen in the source.
+        if leading.last&.ignore?
+          doc = text(source[node.location.start_char...node.location.end_char])
+        else
+          doc = node.format(self)
+        end
 
         # Print all comments that were found after the node.
         trailing.each do |comment|
-          line_suffix do
+          line_suffix(priority: COMMENT_PRIORITY) do
             text(" ")
             comment.format(self)
             break_parent
@@ -173,6 +191,10 @@ class SyntaxTree < Ripper
   # [Array[ String ]] the list of lines in the source
   attr_reader :lines
 
+  # [Array[ SingleByteString | MultiByteString ]] the list of objects that
+  # represent the start of each line in character offsets
+  attr_reader :line_counts
+
   # [Array[ untyped ]] a running list of tokens that have been found in the
   # source. This list changes a lot as certain nodes will "consume" these tokens
   # to determine their bounds.
@@ -248,6 +270,12 @@ class SyntaxTree < Ripper
 
       last_index += line.size
     end
+
+    # Make sure line counts is filled out with the first and last line at
+    # minimum so that it has something to compare against if the parser is in a
+    # lineno=2 state for an empty file.
+    @line_counts << SingleByteString.new(0) if @line_counts.empty?
+    @line_counts << SingleByteString.new(last_index)
   end
 
   def self.parse(source)
@@ -259,7 +287,7 @@ class SyntaxTree < Ripper
   def self.format(source)
     output = []
 
-    formatter = Formatter.new(output)
+    formatter = Formatter.new(source, output)
     parse(source).format(formatter)
 
     formatter.flush
@@ -280,7 +308,7 @@ class SyntaxTree < Ripper
   # this line, then we add the number of columns into this line that we've gone
   # through.
   def char_pos
-    @line_counts[lineno - 1][column]
+    line_counts[lineno - 1][column]
   end
 
   # As we build up a list of tokens, we'll periodically need to go backwards and
@@ -294,7 +322,7 @@ class SyntaxTree < Ripper
   # (which would happen to be the innermost keyword). Then the outer one would
   # only be able to grab the first one. In this way all of the tokens act as
   # their own stack.
-  def find_token(type, value = :any, consume: true)
+  def find_token(type, value = :any, consume: true, location: nil)
     index =
       tokens.rindex do |token|
         token.is_a?(type) && (value == :any || (token.value == value))
@@ -307,8 +335,16 @@ class SyntaxTree < Ripper
       # could also be caused by accidentally attempting to consume a token twice
       # by two different parser event handlers.
       unless index
-        message = "Cannot find expected #{value == :any ? type : value}"
-        raise ParseError.new(message, lineno, column)
+        token = value == :any ? type.name.split("::", 2).last : value
+        message = "Cannot find expected #{token}"
+
+        if location
+          lineno = location.start_line
+          column = location.start_char - line_counts[lineno - 1].start
+          raise ParseError.new(message, lineno, column)
+        else
+          raise ParseError.new(message, lineno, column)
+        end
       end
 
       tokens.delete_at(index)
@@ -476,7 +512,7 @@ class SyntaxTree < Ripper
         q.text(value)
       else
         q.text(q.quote)
-        q.text(value[1])
+        q.text(value[1] == "\"" ? "\\\"" : value[1])
         q.text(q.quote)
       end
     end
@@ -628,7 +664,9 @@ class SyntaxTree < Ripper
     def format(q)
       q.text("__END__")
       q.breakable(force: true)
-      q.text(value)
+
+      separator = -> { q.breakable(indent: false, force: true) }
+      q.seplist(value.split(/\r?\n/, -1), separator) { |line| q.text(line) }
     end
 
     def pretty_print(q)
@@ -654,7 +692,7 @@ class SyntaxTree < Ripper
   def on___end__(value)
     @__end__ =
       EndContent.new(
-        value: lines[lineno..-1].join("\n"),
+        value: source[(char_pos + value.length)..-1],
         location: Location.token(line: lineno, char: char_pos, size: value.size)
       )
   end
@@ -1349,7 +1387,11 @@ class SyntaxTree < Ripper
           q.indent do
             q.breakable("")
             q.seplist(contents.parts, -> { q.breakable }) do |part|
-              q.format(part.parts.first)
+              if part.is_a?(StringLiteral)
+                q.format(part.parts.first)
+              else
+                q.text(part.value[1..-1])
+              end
             end
           end
           q.breakable("")
@@ -1373,10 +1415,39 @@ class SyntaxTree < Ripper
               q.format(part.value)
             end
           end
+          q.breakable("")
+        end
+      end
+    end
+
+    class VarRefsFormatter
+      # [Args] the contents of the array
+      attr_reader :contents
+
+      def initialize(contents)
+        @contents = contents
+      end
+
+      def format(q)
+        q.group(0, "[", "]") do
+          q.indent do
+            q.breakable("")
+
+            separator = -> do
+              q.text(",")
+              q.fill_breakable
+            end
+
+            q.seplist(contents.parts, separator) { |part| q.format(part) }
+          end
+          q.breakable("")
         end
       end
     end
 
+    # [LBracket] the bracket that opens this array
+    attr_reader :lbracket
+
     # [nil | Args] the contents of the array
     attr_reader :contents
 
@@ -1386,22 +1457,18 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(contents:, location:, comments: [])
+    def initialize(lbracket:, contents:, location:, comments: [])
+      @lbracket = lbracket
       @contents = contents
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [contents]
+      [lbracket, contents]
     end
 
     def format(q)
-      unless contents
-        q.text("[]")
-        return
-      end
-
       if qwords?
         QWordsFormatter.new(contents).format(q)
         return
@@ -1412,12 +1479,23 @@ class SyntaxTree < Ripper
         return
       end
 
-      q.group(0, "[", "]") do
-        q.indent do
-          q.breakable("")
-          q.format(contents)
+      if var_refs?(q)
+        VarRefsFormatter.new(contents).format(q)
+        return
+      end
+
+      q.group do
+        q.format(lbracket)
+
+        if contents
+          q.indent do
+            q.breakable("")
+            q.format(contents)
+          end
         end
+
         q.breakable("")
+        q.text("]")
       end
     end
 
@@ -1441,21 +1519,40 @@ class SyntaxTree < Ripper
     private
 
     def qwords?
-      contents && contents.comments.empty? && contents.parts.length > 1 &&
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.length > 1 &&
         contents.parts.all? do |part|
-          part.is_a?(StringLiteral) && part.comments.empty? &&
-            part.parts.length == 1 &&
-            part.parts.first.is_a?(TStringContent) &&
-            !part.parts.first.value.match?(/[\s\\\]]/)
+          case part
+          when StringLiteral
+            part.comments.empty? && part.parts.length == 1 &&
+              part.parts.first.is_a?(TStringContent) &&
+              !part.parts.first.value.match?(/[\s\[\]\\]/)
+          when CHAR
+            !part.value.match?(/[\[\]\\]/)
+          else
+            false
+          end
         end
     end
 
     def qsymbols?
-      contents && contents.comments.empty? && contents.parts.length > 1 &&
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.length > 1 &&
         contents.parts.all? do |part|
           part.is_a?(SymbolLiteral) && part.comments.empty?
         end
     end
+
+    def var_refs?(q)
+      lbracket.comments.empty? && contents && contents.comments.empty? &&
+        contents.parts.all? do |part|
+          part.is_a?(VarRef) && part.comments.empty?
+        end &&
+        (
+          contents.parts.sum { |part| part.value.value.length + 2 } >
+            q.maxwidth * 2
+        )
+    end
   end
 
   # :call-seq:
@@ -1467,13 +1564,16 @@ class SyntaxTree < Ripper
       rbracket = find_token(RBracket)
 
       ArrayLiteral.new(
+        lbracket: lbracket,
         contents: contents,
         location: lbracket.location.to(rbracket.location)
       )
     else
-      tstring_end = find_token(TStringEnd)
+      tstring_end =
+        find_token(TStringEnd, location: contents.beginning.location)
 
       contents.class.new(
+        beginning: contents.beginning,
         elements: contents.elements,
         location: contents.location.to(tstring_end.location)
       )
@@ -1571,7 +1671,7 @@ class SyntaxTree < Ripper
       end
 
       parent = q.parent
-      if parts.length == 1 || PATTERNS.any? { |pattern| parent.is_a?(pattern) }
+      if parts.length == 1 || PATTERNS.include?(parent.class)
         q.text("[")
         q.seplist(parts) { |part| q.format(part) }
         q.text("]")
@@ -1678,7 +1778,7 @@ class SyntaxTree < Ripper
         q.format(target)
         q.text(" =")
 
-        if skip_indent?
+        if target.comments.empty? && (skip_indent_target? || skip_indent_value?)
           q.text(" ")
           q.format(value)
         else
@@ -1716,11 +1816,21 @@ class SyntaxTree < Ripper
 
     private
 
-    def skip_indent?
-      target.is_a?(ARefField) || value.is_a?(ArrayLiteral) ||
-        value.is_a?(HashLiteral) ||
-        value.is_a?(Heredoc) ||
-        value.is_a?(Lambda)
+    def skip_indent_target?
+      target.is_a?(ARefField)
+    end
+
+    def skip_indent_value?
+      [
+        ArrayLiteral,
+        HashLiteral,
+        Heredoc,
+        Lambda,
+        QSymbols,
+        QWords,
+        Symbols,
+        Words
+      ].any? { |type| value.is_a?(type) }
     end
   end
 
@@ -2252,11 +2362,14 @@ class SyntaxTree < Ripper
       q.group do
         q.group { q.format(left) }
         q.text(" ") unless power
-        q.text(operator)
 
-        q.indent do
-          q.breakable(power ? "" : " ")
-          q.format(right)
+        q.group do
+          q.text(operator)
+
+          q.indent do
+            q.breakable(power ? "" : " ")
+            q.format(right)
+          end
         end
       end
     end
@@ -2685,54 +2798,132 @@ class SyntaxTree < Ripper
     # [LBrace | Keyword] the node that opens the block
     attr_reader :block_open
 
+    # [String] the string that closes the block
+    attr_reader :block_close
+
     # [BodyStmt | Statements] the statements inside the block
     attr_reader :statements
 
-    def initialize(node, block_open, statements)
+    def initialize(node, block_open, block_close, statements)
       @node = node
       @block_open = block_open
+      @block_close = block_close
       @statements = statements
     end
 
     def format(q)
+      # If this is nested anywhere inside of a Command or CommandCall node, then
+      # we can't change which operators we're using for the bounds of the block.
+      break_opening, break_closing, flat_opening, flat_closing =
+        if unchangeable_bounds?(q)
+          [block_open.value, block_close, block_open.value, block_close]
+        elsif forced_do_end_bounds?(q)
+          %w[do end do end]
+        elsif forced_brace_bounds?(q)
+          %w[{ } { }]
+        else
+          %w[do end { }]
+        end
+
+      # If the receiver of this block a Command or CommandCall node, then there
+      # are no parentheses around the arguments to that command, so we need to
+      # break the block.
+      receiver = q.parent.call
+      if receiver.is_a?(Command) || receiver.is_a?(CommandCall)
+        q.break_parent
+        format_break(q, break_opening, break_closing)
+        return
+      end
+
       q.group do
-        q.text(" ")
+        q.if_break { format_break(q, break_opening, break_closing) }.if_flat do
+          format_flat(q, flat_opening, flat_closing)
+        end
+      end
+    end
 
-        q.if_break do
-          q.format(BlockOpenFormatter.new("do", block_open))
+    private
 
-          if node.block_var
-            q.text(" ")
-            q.format(node.block_var)
-          end
+    # If this is nested anywhere inside certain nodes, then we can't change
+    # which operators/keywords we're using for the bounds of the block.
+    def unchangeable_bounds?(q)
+      q.parents.any? do |parent|
+        # If we hit a statements, then we're safe to use whatever since we
+        # know for certain we're going to get split over multiple lines
+        # anyway.
+        break false if parent.is_a?(Statements)
 
-          unless statements.empty?
-            q.indent do
-              q.breakable
-              q.format(statements)
-            end
-          end
+        [Command, CommandCall].include?(parent.class)
+      end
+    end
 
-          q.breakable
-          q.text("end")
-        end.if_flat do
-          q.format(BlockOpenFormatter.new("{", block_open))
+    # If we're a sibling of a control-flow keyword, then we're going to have to
+    # use the do..end bounds.
+    def forced_do_end_bounds?(q)
+      [Break, Next, Return, Super].include?(q.parent.call.class)
+    end
 
-          if node.block_var
-            q.breakable
-            q.format(node.block_var)
-            q.breakable
-          end
+    # If we're the predicate of a loop or conditional, then we're going to have
+    # to go with the {..} bounds.
+    def forced_brace_bounds?(q)
+      parents = q.parents.to_a
+      parents.each_with_index.any? do |parent, index|
+        # If we hit certain breakpoints then we know we're safe.
+        break false if [Paren, Statements].include?(parent.class)
 
-          unless statements.empty?
-            q.breakable unless node.block_var
-            q.format(statements)
-            q.breakable
-          end
+        [
+          If,
+          IfMod,
+          IfOp,
+          Unless,
+          UnlessMod,
+          While,
+          WhileMod,
+          Until,
+          UntilMod
+        ].include?(parent.class) && parent.predicate == parents[index - 1]
+      end
+    end
 
-          q.text("}")
+    def format_break(q, opening, closing)
+      q.text(" ")
+      q.format(BlockOpenFormatter.new(opening, block_open))
+
+      if node.block_var
+        q.text(" ")
+        q.format(node.block_var)
+      end
+
+      unless statements.empty?
+        q.indent do
+          q.breakable
+          q.format(statements)
         end
       end
+
+      q.breakable
+      q.text(closing)
+    end
+
+    def format_flat(q, opening, closing)
+      q.text(" ")
+      q.format(BlockOpenFormatter.new(opening, block_open))
+
+      if node.block_var
+        q.breakable
+        q.format(node.block_var)
+        q.breakable
+      end
+
+      if statements.empty?
+        q.text(" ") if opening == "do"
+      else
+        q.breakable unless node.block_var
+        q.format(statements)
+        q.breakable
+      end
+
+      q.text(closing)
     end
   end
 
@@ -2770,7 +2961,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      BlockFormatter.new(self, lbrace, statements).format(q)
+      BlockFormatter.new(self, lbrace, "}", statements).format(q)
     end
 
     def pretty_print(q)
@@ -2851,15 +3042,35 @@ class SyntaxTree < Ripper
         q.text(keyword)
 
         if arguments.parts.any?
-          if arguments.parts.length == 1 && arguments.parts.first.is_a?(Paren)
-            q.format(arguments)
+          if arguments.parts.length == 1
+            part = arguments.parts.first
+
+            if part.is_a?(Paren)
+              q.format(arguments)
+            elsif part.is_a?(ArrayLiteral)
+              q.text(" ")
+              q.format(arguments)
+            else
+              format_arguments(q, "(", ")")
+            end
           else
-            q.text(" ")
-            q.nest(keyword.length + 1) { q.format(arguments) }
+            format_arguments(q, " [", "]")
           end
         end
       end
     end
+
+    private
+
+    def format_arguments(q, opening, closing)
+      q.if_break { q.text(opening) }
+      q.indent do
+        q.breakable(" ")
+        q.format(node.arguments)
+      end
+      q.breakable("")
+      q.if_break { q.text(closing) }
+    end
   end
 
   # Break represents using the +break+ keyword.
@@ -2946,9 +3157,7 @@ class SyntaxTree < Ripper
     end
   end
 
-  # Call represents a method call. This node doesn't contain the arguments being
-  # passed (if arguments are passed, this node will get nested under a
-  # MethodAddArg node).
+  # Call represents a method call.
   #
   #     receiver.message
   #
@@ -2962,32 +3171,60 @@ class SyntaxTree < Ripper
     # [:call | Backtick | Const | Ident | Op] the message being sent
     attr_reader :message
 
+    # [nil | ArgParen | Args] the arguments to the method call
+    attr_reader :arguments
+
     # [Location] the location of this node
     attr_reader :location
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(receiver:, operator:, message:, location:, comments: [])
+    def initialize(
+      receiver:,
+      operator:,
+      message:,
+      arguments:,
+      location:,
+      comments: []
+    )
       @receiver = receiver
       @operator = operator
       @message = message
+      @arguments = arguments
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [receiver, (operator if operator != :"::"), (message if message != :call)]
+      [
+        receiver,
+        (operator if operator != :"::"),
+        (message if message != :call),
+        arguments
+      ]
     end
 
     def format(q)
+      call_operator = CallOperatorFormatter.new(operator)
+
       q.group do
         q.format(receiver)
+
+        # If there are trailing comments on the call operator, then we need to
+        # use the trailing form as opposed to the leading form.
+        q.format(call_operator) if call_operator.comments.any?
+
         q.group do
           q.indent do
-            q.format(CallOperatorFormatter.new(operator))
+            if receiver.comments.any? || call_operator.comments.any?
+              q.breakable(force: true)
+            end
+            q.format(call_operator) if call_operator.comments.empty?
             q.format(message) if message != :call
           end
+
+          q.format(arguments) if arguments
         end
       end
     end
@@ -3005,6 +3242,11 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(message)
 
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
+
         q.pp(Comment::List.new(comments))
       end
     end
@@ -3015,6 +3257,7 @@ class SyntaxTree < Ripper
         receiver: receiver,
         op: operator,
         message: message,
+        args: arguments,
         loc: location,
         cmts: comments
       }.to_json(*opts)
@@ -3035,13 +3278,8 @@ class SyntaxTree < Ripper
       receiver: receiver,
       operator: operator,
       message: message,
-      location:
-        Location.new(
-          start_line: receiver.location.start_line,
-          start_char: receiver.location.start_char,
-          end_line: [ending.location.end_line, receiver.location.end_line].max,
-          end_char: ending.location.end_char
-        )
+      arguments: nil,
+      location: receiver.location.to(ending.location)
     )
   end
 
@@ -3057,6 +3295,9 @@ class SyntaxTree < Ripper
   #     end
   #
   class Case
+    # [Kw] the keyword that opens this expression
+    attr_reader :keyword
+
     # [nil | untyped] optional value being switched on
     attr_reader :value
 
@@ -3069,7 +3310,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(value:, consequent:, location:, comments: [])
+    def initialize(keyword:, value:, consequent:, location:, comments: [])
+      @keyword = keyword
       @value = value
       @consequent = consequent
       @location = location
@@ -3077,11 +3319,13 @@ class SyntaxTree < Ripper
     end
 
     def child_nodes
-      [value, consequent]
+      [keyword, value, consequent]
     end
 
     def format(q)
-      q.group(0, "case", "end") do
+      q.group do
+        q.format(keyword)
+
         if value
           q.text(" ")
           q.format(value)
@@ -3090,6 +3334,8 @@ class SyntaxTree < Ripper
         q.breakable(force: true)
         q.format(consequent)
         q.breakable(force: true)
+
+        q.text("end")
       end
     end
 
@@ -3097,6 +3343,9 @@ class SyntaxTree < Ripper
       q.group(2, "(", ")") do
         q.text("case")
 
+        q.breakable
+        q.pp(keyword)
+
         if value
           q.breakable
           q.pp(value)
@@ -3204,6 +3453,7 @@ class SyntaxTree < Ripper
       tokens.delete(keyword)
 
       Case.new(
+        keyword: keyword,
         value: value,
         consequent: consequent,
         location: keyword.location.to(consequent.location)
@@ -3484,7 +3734,7 @@ class SyntaxTree < Ripper
     # [Const | Ident | Op] the message being send
     attr_reader :message
 
-    # [Args] the arguments going along with the message
+    # [nil | Args] the arguments going along with the message
     attr_reader :arguments
 
     # [Location] the location of this node
@@ -3515,16 +3765,22 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.group do
-        doc = q.format(receiver)
-        q.format(CallOperatorFormatter.new(operator))
-        q.format(message)
-        q.text(" ")
+        doc =
+          q.nest(0) do
+            q.format(receiver)
+            q.format(CallOperatorFormatter.new(operator))
+            q.format(message)
+          end
 
-        width = doc_width(doc)
-        if width > (q.maxwidth / 2) || width < 2
-          q.indent { q.format(arguments) }
-        else
-          q.nest(width) { q.format(arguments) }
+        if arguments
+          width = doc_width(doc) + 1
+          q.text(" ")
+
+          if width > (q.maxwidth / 2)
+            q.format(arguments)
+          else
+            q.nest(width) { q.format(arguments) }
+          end
         end
       end
     end
@@ -3542,8 +3798,10 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(message)
 
-        q.breakable
-        q.pp(arguments)
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
 
         q.pp(Comment::List.new(comments))
       end
@@ -3577,11 +3835,11 @@ class SyntaxTree < Ripper
         when PrettyPrint::Text
           width += doc.width
         when PrettyPrint::Indent, PrettyPrint::Align, PrettyPrint::Group
-          queue += doc.contents.reverse
+          queue = doc.contents + queue
         when PrettyPrint::IfBreak
-          queue += doc.flat_contents.reverse
+          queue = doc.break_contents + queue
         when PrettyPrint::Breakable
-          width = doc.force? ? 0 : width + doc.width
+          width = 0
         end
       end
 
@@ -3594,7 +3852,7 @@ class SyntaxTree < Ripper
   #     untyped receiver,
   #     (:"::" | Op | Period) operator,
   #     (Const | Ident | Op) message,
-  #     Args arguments
+  #     (nil | Args) arguments
   #   ) -> CommandCall
   def on_command_call(receiver, operator, message, arguments)
     ending = arguments || message
@@ -3665,6 +3923,10 @@ class SyntaxTree < Ripper
       @trailing
     end
 
+    def ignore?
+      value[1..-1].strip == "stree-ignore"
+    end
+
     def comments
       []
     end
@@ -4062,14 +4324,7 @@ class SyntaxTree < Ripper
         q.group do
           q.text("def ")
           q.format(name)
-
-          if params.is_a?(Params) && !params.empty?
-            q.text("(")
-            q.format(params)
-            q.text(")")
-          else
-            q.format(params)
-          end
+          q.format(params) if !params.is_a?(Params) || !params.empty?
         end
 
         unless bodystmt.empty?
@@ -4118,6 +4373,12 @@ class SyntaxTree < Ripper
   #     def method = result
   #
   class DefEndless
+    # [untyped] the target where the method is being defined
+    attr_reader :target
+
+    # [Op | Period] the operator being used to declare the method
+    attr_reader :operator
+
     # [Backtick | Const | Ident | Kw | Op] the name of the method
     attr_reader :name
 
@@ -4133,7 +4394,17 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(name:, paren:, statement:, location:, comments: [])
+    def initialize(
+      target:,
+      operator:,
+      name:,
+      paren:,
+      statement:,
+      location:,
+      comments: []
+    )
+      @target = target
+      @operator = operator
       @name = name
       @paren = paren
       @statement = statement
@@ -4142,14 +4413,21 @@ class SyntaxTree < Ripper
     end
 
     def child_nodes
-      [name, paren, statement]
+      [target, operator, name, paren, statement]
     end
 
     def format(q)
       q.group do
         q.text("def ")
+
+        if target
+          q.format(target)
+          q.format(CallOperatorFormatter.new(operator))
+        end
+
         q.format(name)
         q.format(paren) if paren && !paren.contents.empty?
+
         q.text(" =")
         q.group do
           q.indent do
@@ -4164,6 +4442,14 @@ class SyntaxTree < Ripper
       q.group(2, "(", ")") do
         q.text("def_endless")
 
+        if target
+          q.breakable
+          q.pp(target)
+
+          q.breakable
+          q.pp(operator)
+        end
+
         q.breakable
         q.pp(name)
 
@@ -4210,6 +4496,8 @@ class SyntaxTree < Ripper
     unless bodystmt.is_a?(BodyStmt)
       node =
         DefEndless.new(
+          target: nil,
+          operator: nil,
           name: name,
           paren: params,
           statement: bodystmt,
@@ -4371,14 +4659,7 @@ class SyntaxTree < Ripper
           q.format(target)
           q.format(CallOperatorFormatter.new(operator))
           q.format(name)
-
-          if params.is_a?(Params) && !params.empty?
-            q.text("(")
-            q.format(params)
-            q.text(")")
-          else
-            q.format(params)
-          end
+          q.format(params) if !params.is_a?(Params) || !params.empty?
         end
 
         unless bodystmt.empty?
@@ -4460,6 +4741,22 @@ class SyntaxTree < Ripper
     end
 
     beginning = find_token(Kw, "def")
+
+    # If we don't have a bodystmt node, then we have a single-line method
+    unless bodystmt.is_a?(BodyStmt)
+      node =
+        DefEndless.new(
+          target: target,
+          operator: operator,
+          name: name,
+          paren: params,
+          statement: bodystmt,
+          location: beginning.location.to(bodystmt.location)
+        )
+
+      return node
+    end
+
     ending = find_token(Kw, "end")
 
     bodystmt.bind(
@@ -4512,7 +4809,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      BlockFormatter.new(self, keyword, bodystmt).format(q)
+      BlockFormatter.new(self, keyword, "end", bodystmt).format(q)
     end
 
     def pretty_print(q)
@@ -4576,8 +4873,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      parent = q.parent
-      space = parent.is_a?(If) || parent.is_a?(Unless)
+      space = [If, IfMod, Unless, UnlessMod].include?(q.parent.class)
 
       left = node.left
       right = node.right
@@ -4890,9 +5186,9 @@ class SyntaxTree < Ripper
         matching = Quotes.matching(quote[2])
         pattern = /[\n#{Regexp.escape(matching)}'"]/
 
-        if parts.any? do |part|
+        if parts.any? { |part|
              part.is_a?(TStringContent) && part.value.match?(pattern)
-           end
+           }
           [quote, matching]
         elsif Quotes.locked?(self)
           ["#{":" unless hash_key}'", "'"]
@@ -4917,7 +5213,7 @@ class SyntaxTree < Ripper
     if find_token(SymBeg, consume: false)
       # A normal dynamic symbol
       symbeg = find_token(SymBeg)
-      tstring_end = find_token(TStringEnd)
+      tstring_end = find_token(TStringEnd, location: symbeg.location)
 
       DynaSymbol.new(
         quote: symbeg.value,
@@ -5009,6 +5305,7 @@ class SyntaxTree < Ripper
 
     node = tokens[index]
     ending = node.value == "end" ? tokens.delete_at(index) : node
+    # ending = node
 
     statements.bind(beginning.location.end_char, ending.location.start_char)
 
@@ -5155,6 +5452,10 @@ class SyntaxTree < Ripper
       false
     end
 
+    def ignore?
+      false
+    end
+
     def comments
       []
     end
@@ -5480,24 +5781,29 @@ class SyntaxTree < Ripper
     # [Const | Ident] the name of the method
     attr_reader :value
 
+    # [nil | ArgParen | Args] the arguments to the method call
+    attr_reader :arguments
+
     # [Location] the location of this node
     attr_reader :location
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(value:, location:, comments: [])
+    def initialize(value:, arguments:, location:, comments: [])
       @value = value
+      @arguments = arguments
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      [value]
+      [value, arguments]
     end
 
     def format(q)
       q.format(value)
+      q.format(arguments)
     end
 
     def pretty_print(q)
@@ -5507,21 +5813,30 @@ class SyntaxTree < Ripper
         q.breakable
         q.pp(value)
 
+        if arguments
+          q.breakable
+          q.pp(arguments)
+        end
+
         q.pp(Comment::List.new(comments))
       end
     end
 
     def to_json(*opts)
-      { type: :fcall, value: value, loc: location, cmts: comments }.to_json(
-        *opts
-      )
+      {
+        type: :fcall,
+        value: value,
+        args: arguments,
+        loc: location,
+        cmts: comments
+      }.to_json(*opts)
     end
   end
 
   # :call-seq:
   #   on_fcall: ((Const | Ident) value) -> FCall
   def on_fcall(value)
-    FCall.new(value: value, location: value.location)
+    FCall.new(value: value, arguments: nil, location: value.location)
   end
 
   # Field is always the child of an assignment. It represents assigning to a
@@ -5864,6 +6179,7 @@ class SyntaxTree < Ripper
   #   ) -> For
   def on_for(index, collection, statements)
     beginning = find_token(Kw, "for")
+    in_keyword = find_token(Kw, "in")
     ending = find_token(Kw, "end")
 
     # Consume the do keyword if it exists so that it doesn't get confused for
@@ -5879,6 +6195,11 @@ class SyntaxTree < Ripper
       ending.location.start_char
     )
 
+    if index.is_a?(MLHS)
+      comma_range = index.location.end_char...in_keyword.location.start_char
+      index.comma = true if source[comma_range].strip.start_with?(",")
+    end
+
     For.new(
       index: index,
       collection: collection,
@@ -5947,6 +6268,9 @@ class SyntaxTree < Ripper
   #     { key => value }
   #
   class HashLiteral
+    # [LBrace] the left brace that opens this hash
+    attr_reader :lbrace
+
     # [Array[ AssocNew | AssocSplat ]] the optional contents of the hash
     attr_reader :assocs
 
@@ -5956,24 +6280,31 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(assocs:, location:, comments: [])
+    def initialize(lbrace:, assocs:, location:, comments: [])
+      @lbrace = lbrace
       @assocs = assocs
       @location = location
       @comments = comments
     end
 
     def child_nodes
-      assocs
+      [lbrace] + assocs
     end
 
     def format(q)
       contents = -> do
-        q.text("{")
-        q.indent do
+        q.format(lbrace)
+
+        if assocs.empty?
+          q.breakable("")
+        else
+          q.indent do
+            q.breakable
+            q.format(HashFormatter.for(self))
+          end
           q.breakable
-          q.format(HashFormatter.for(self))
         end
-        q.breakable
+
         q.text("}")
       end
 
@@ -6007,6 +6338,7 @@ class SyntaxTree < Ripper
     rbrace = find_token(RBrace)
 
     HashLiteral.new(
+      lbrace: lbrace,
       assocs: assocs || [],
       location: lbrace.location.to(rbrace.location)
     )
@@ -6059,7 +6391,7 @@ class SyntaxTree < Ripper
       q.group do
         q.format(beginning)
 
-        q.line_suffix do
+        q.line_suffix(priority: Formatter::HEREDOC_PRIORITY) do
           q.group do
             breakable.call
 
@@ -6219,6 +6551,12 @@ class SyntaxTree < Ripper
         @value = value
       end
 
+      # This is here so that when checking if its contained within a parent
+      # pattern that it will return true.
+      def class
+        HshPtn
+      end
+
       def comments
         []
       end
@@ -6293,7 +6631,7 @@ class SyntaxTree < Ripper
       end
 
       parent = q.parent
-      if PATTERNS.any? { |pattern| parent.is_a?(pattern) }
+      if PATTERNS.include?(parent.class)
         q.text("{ ")
         contents.call
         q.text(" }")
@@ -6442,38 +6780,50 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      statements = node.statements
-      break_format = ->(force:) do
-        q.text("#{keyword} ")
-        q.nest(keyword.length + 1) { q.format(node.predicate) }
+      # If the predicate of the conditional contains an assignment (in which
+      # case we can't know for certain that that assignment doesn't impact the
+      # statements inside the conditional) then we can't use the modifier form
+      # and we must use the block form.
+      if [Assign, MAssign, OpAssign].include?(node.predicate.class)
+        format_break(q, force: true)
+        return
+      end
 
-        unless statements.empty?
-          q.indent do
-            q.breakable(force: force)
-            q.format(statements)
+      if node.consequent || node.statements.empty?
+        q.group { format_break(q, force: true) }
+      else
+        q.group do
+          q.if_break { format_break(q, force: false) }.if_flat do
+            Parentheses.flat(q) do
+              q.format(node.statements)
+              q.text(" #{keyword} ")
+              q.format(node.predicate)
+            end
           end
         end
+      end
+    end
+
+    private
 
-        if node.consequent
+    def format_break(q, force:)
+      q.text("#{keyword} ")
+      q.nest(keyword.length + 1) { q.format(node.predicate) }
+
+      unless node.statements.empty?
+        q.indent do
           q.breakable(force: force)
-          q.format(node.consequent)
+          q.format(node.statements)
         end
+      end
 
+      if node.consequent
         q.breakable(force: force)
-        q.text("end")
+        q.format(node.consequent)
       end
 
-      if node.consequent || statements.empty?
-        q.group { break_format.call(force: true) }
-      else
-        q.group do
-          q.if_break { break_format.call(force: false) }.if_flat do
-            q.format(node.statements)
-            q.text(" #{keyword} ")
-            q.format(node.predicate)
-          end
-        end
-      end
+      q.breakable(force: force)
+      q.text("end")
     end
   end
 
@@ -6604,38 +6954,19 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group do
-        q.if_break do
-          q.text("if ")
-          q.nest("if ".length) { q.format(predicate) }
-
-          q.indent do
-            q.breakable
-            q.format(truthy)
-          end
-
-          q.breakable
-          q.text("else")
-
-          q.indent do
-            q.breakable
-            q.format(falsy)
-          end
-
-          q.breakable
-          q.text("end")
-        end.if_flat do
-          q.format(predicate)
-          q.text(" ?")
-
-          q.breakable
-          q.format(truthy)
-          q.text(" :")
+      force_flat = [
+        Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp,
+        Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super,
+        Undef, Unless, UnlessMod, UntilMod, VarAlias, VoidStmt, WhileMod, Yield,
+        Yield0, ZSuper
+      ]
 
-          q.breakable
-          q.format(falsy)
-        end
+      if force_flat.include?(truthy.class) || force_flat.include?(falsy.class)
+        q.group { format_flat(q) }
+        return
       end
+
+      q.group { q.if_break { format_break(q) }.if_flat { format_flat(q) } }
     end
 
     def pretty_print(q)
@@ -6665,6 +6996,43 @@ class SyntaxTree < Ripper
         cmts: comments
       }.to_json(*opts)
     end
+
+    private
+
+    def format_break(q)
+      Parentheses.break(q) do
+        q.text("if ")
+        q.nest("if ".length) { q.format(predicate) }
+
+        q.indent do
+          q.breakable
+          q.format(truthy)
+        end
+
+        q.breakable
+        q.text("else")
+
+        q.indent do
+          q.breakable
+          q.format(falsy)
+        end
+
+        q.breakable
+        q.text("end")
+      end
+    end
+
+    def format_flat(q)
+      q.format(predicate)
+      q.text(" ?")
+
+      q.breakable
+      q.format(truthy)
+      q.text(" :")
+
+      q.breakable
+      q.format(falsy)
+    end
   end
 
   # :call-seq:
@@ -6703,9 +7071,11 @@ class SyntaxTree < Ripper
           q.breakable
           q.text("end")
         end.if_flat do
-          q.format(node.statement)
-          q.text(" #{keyword} ")
-          q.format(node.predicate)
+          Parentheses.flat(q) do
+            q.format(node.statement)
+            q.text(" #{keyword} ")
+            q.format(node.predicate)
+          end
         end
       end
     end
@@ -6944,8 +7314,9 @@ class SyntaxTree < Ripper
     beginning = find_token(Kw, "in")
     ending = consequent || find_token(Kw, "end")
 
+    statements_start = find_token(Kw, "then", consume: false) || pattern
     statements.bind(
-      find_next_statement_start(pattern.location.end_char),
+      find_next_statement_start(statements_start.location.end_char),
       ending.location.start_char
     )
 
@@ -6982,7 +7353,7 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      if !value.start_with?("0") && value.length >= 5 && !value.include?("_")
+      if !value.start_with?(/\+?0/) && value.length >= 5 && !value.include?("_")
         # If it's a plain integer and it doesn't have any underscores separating
         # the values, then we're going to insert them every 3 characters
         # starting from the right.
@@ -7330,9 +7701,11 @@ class SyntaxTree < Ripper
         if params.is_a?(Paren)
           q.format(params) unless params.contents.empty?
         elsif !params.empty?
-          q.text("(")
-          q.format(params)
-          q.text(")")
+          q.group do
+            q.text("(")
+            q.format(params)
+            q.text(")")
+          end
         end
 
         q.text(" ")
@@ -7391,8 +7764,11 @@ class SyntaxTree < Ripper
   def on_lambda(params, statements)
     beginning = find_token(TLambda)
 
-    if token = find_token(TLamBeg, consume: false)
-      opening = tokens.delete(token)
+    if tokens.any? { |token|
+         token.is_a?(TLamBeg) &&
+           token.location.start_char > beginning.location.start_char
+       }
+      opening = find_token(TLamBeg)
       closing = find_token(RBrace)
     else
       opening = find_token(Kw, "do")
@@ -7472,9 +7848,34 @@ class SyntaxTree < Ripper
     # [Location] the location of this node
     attr_reader :location
 
-    def initialize(value:, location:)
+    # [Array[ Comment | EmbDoc ]] the comments attached to this node
+    attr_reader :comments
+
+    def initialize(value:, location:, comments: [])
       @value = value
       @location = location
+      @comments = comments
+    end
+
+    def format(q)
+      q.text(value)
+    end
+
+    def pretty_print(q)
+      q.group(2, "(", ")") do
+        q.text("lbracket")
+
+        q.breakable
+        q.pp(value)
+
+        q.pp(Comment::List.new(comments))
+      end
+    end
+
+    def to_json(*opts)
+      { type: :lbracket, value: value, loc: location, cmts: comments }.to_json(
+        *opts
+      )
     end
   end
 
@@ -7591,11 +7992,7 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.group do
-        q.group do
-          q.format(target)
-          q.text(",") if target.is_a?(MLHS) && target.comma
-        end
-
+        q.group { q.format(target) }
         q.text(" =")
         q.indent do
           q.breakable
@@ -7642,86 +8039,26 @@ class SyntaxTree < Ripper
     )
   end
 
-  # MethodAddArg represents a method call with arguments and parentheses.
-  #
-  #     method(argument)
-  #
-  # MethodAddArg can also represent with a method on an object, as in:
-  #
-  #     object.method(argument)
-  #
-  # Finally, MethodAddArg can represent calling a method with no receiver that
-  # ends in a ?. In this case, the parser knows it's a method call and not a
-  # local variable, so it uses a MethodAddArg node as opposed to a VCall node,
-  # as in:
-  #
-  #     method?
-  #
-  class MethodAddArg
-    # [Call | FCall] the method call
-    attr_reader :call
-
-    # [ArgParen | Args] the arguments to the method call
-    attr_reader :arguments
-
-    # [Location] the location of this node
-    attr_reader :location
-
-    # [Array[ Comment | EmbDoc ]] the comments attached to this node
-    attr_reader :comments
-
-    def initialize(call:, arguments:, location:, comments: [])
-      @call = call
-      @arguments = arguments
-      @location = location
-      @comments = comments
-    end
-
-    def child_nodes
-      [call, arguments]
-    end
-
-    def format(q)
-      q.format(call)
-      q.text(" ") if !arguments.is_a?(ArgParen) && arguments.parts.any?
-      q.format(arguments)
-    end
-
-    def pretty_print(q)
-      q.group(2, "(", ")") do
-        q.text("method_add_arg")
-
-        q.breakable
-        q.pp(call)
-
-        q.breakable
-        q.pp(arguments)
-
-        q.pp(Comment::List.new(comments))
-      end
-    end
-
-    def to_json(*opts)
-      {
-        type: :method_add_arg,
-        call: call,
-        args: arguments,
-        loc: location,
-        cmts: comments
-      }.to_json(*opts)
-    end
-  end
-
   # :call-seq:
   #   on_method_add_arg: (
   #     (Call | FCall) call,
   #     (ArgParen | Args) arguments
-  #   ) -> MethodAddArg
+  #   ) -> Call | FCall
   def on_method_add_arg(call, arguments)
     location = call.location
-    location = location.to(arguments.location) unless arguments.is_a?(Args)
+    location = location.to(arguments.location) if arguments.is_a?(ArgParen)
 
-    MethodAddArg.new(call: call, arguments: arguments, location: location)
+    if call.is_a?(FCall)
+      FCall.new(value: call.value, arguments: arguments, location: location)
+    else
+      Call.new(
+        receiver: call.receiver,
+        operator: call.operator,
+        message: call.message,
+        arguments: arguments,
+        location: location
+      )
+    end
   end
 
   # MethodAddBlock represents a method call with a block argument.
@@ -7729,7 +8066,7 @@ class SyntaxTree < Ripper
   #     method {}
   #
   class MethodAddBlock
-    # [Call | Command | CommandCall | FCall | MethodAddArg] the method call
+    # [Call | Command | CommandCall | FCall] the method call
     attr_reader :call
 
     # [BraceBlock | DoBlock] the block being sent with the method call
@@ -7784,7 +8121,7 @@ class SyntaxTree < Ripper
 
   # :call-seq:
   #   on_method_add_block: (
-  #     (Call | Command | CommandCall | FCall | MethodAddArg) call,
+  #     (Call | Command | CommandCall | FCall) call,
   #     (BraceBlock | DoBlock) block
   #   ) -> MethodAddBlock
   def on_method_add_block(call, block)
@@ -7809,7 +8146,6 @@ class SyntaxTree < Ripper
     # list, which impacts destructuring. It's an attr_accessor so that while
     # the syntax tree is being built it can be set by its parent node
     attr_accessor :comma
-    alias comma? comma
 
     # [Location] the location of this node
     attr_reader :location
@@ -7830,6 +8166,7 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.seplist(parts) { |part| q.format(part) }
+      q.text(",") if comma
     end
 
     def pretty_print(q)
@@ -7932,7 +8269,6 @@ class SyntaxTree < Ripper
           q.indent do
             q.breakable("")
             q.format(contents)
-            q.text(",") if contents.is_a?(MLHS) && contents.comma?
           end
 
           q.breakable("")
@@ -8400,6 +8736,61 @@ class SyntaxTree < Ripper
     )
   end
 
+  # If you have a modifier statement (for instance a modifier if statement or a
+  # modifier while loop) there are times when you need to wrap the entire
+  # statement in parentheses. This occurs when you have something like:
+  #
+  #     foo[:foo] =
+  #       if bar?
+  #         baz
+  #       end
+  #
+  # Normally we would shorten this to an inline version, which would result in:
+  #
+  #     foo[:foo] = baz if bar?
+  #
+  # but this actually has different semantic meaning. The first example will
+  # result in a nil being inserted into the hash for the :foo key, whereas the
+  # second example will result in an empty hash because the if statement applies
+  # to the entire assignment.
+  #
+  # We can fix this in a couple of ways. We can use the then keyword, as in:
+  #
+  #     foo[:foo] = if bar? then baz end
+  #
+  # But this isn't used very often. We can also just leave it as is with the
+  # multi-line version, but for a short predicate and short value it looks
+  # verbose. The last option and the one used here is to add parentheses on
+  # both sides of the expression, as in:
+  #
+  #     foo[:foo] = (baz if bar?)
+  #
+  # This approach maintains the nice conciseness of the inline version, while
+  # keeping the correct semantic meaning.
+  module Parentheses
+    NODES = [Args, Assign, Assoc, Binary, Call, Defined, MAssign, OpAssign]
+
+    def self.flat(q)
+      return yield unless NODES.include?(q.parent.class)
+
+      q.text("(")
+      yield
+      q.text(")")
+    end
+
+    def self.break(q)
+      return yield unless NODES.include?(q.parent.class)
+
+      q.text("(")
+      q.indent do
+        q.breakable("")
+        yield
+      end
+      q.breakable("")
+      q.text(")")
+    end
+  end
+
   # def on_operator_ambiguous(value)
   #   value
   # end
@@ -8538,9 +8929,7 @@ class SyntaxTree < Ripper
     # it's missing.
     def empty?
       requireds.empty? && optionals.empty? && !rest && posts.empty? &&
-        keywords.empty? &&
-        !keyword_rest &&
-        !block
+        keywords.empty? && !keyword_rest && !block
     end
 
     def child_nodes
@@ -8571,10 +8960,22 @@ class SyntaxTree < Ripper
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
       parts << block if block
 
-      q.nest(0) do
+      contents = -> do
         q.seplist(parts) { |part| q.format(part) }
         q.format(rest) if rest && rest.is_a?(ExcessedComma)
       end
+
+      if [Def, Defs].include?(q.parent.class)
+        q.group(0, "(", ")") do
+          q.indent do
+            q.breakable("")
+            contents.call
+          end
+          q.breakable("")
+        end
+      else
+        q.nest(0, &contents)
+      end
     end
 
     def pretty_print(q)
@@ -8716,7 +9117,7 @@ class SyntaxTree < Ripper
     # [LParen] the left parenthesis that opened this statement
     attr_reader :lparen
 
-    # [untyped] the expression inside the parentheses
+    # [nil | untyped] the expression inside the parentheses
     attr_reader :contents
 
     # [Location] the location of this node
@@ -8740,7 +9141,7 @@ class SyntaxTree < Ripper
       q.group do
         q.format(lparen)
 
-        if !contents.is_a?(Params) || !contents.empty?
+        if contents && (!contents.is_a?(Params) || !contents.empty?)
           q.indent do
             q.breakable("")
             q.format(contents)
@@ -8805,7 +9206,7 @@ class SyntaxTree < Ripper
 
     Paren.new(
       lparen: lparen,
-      contents: contents,
+      contents: contents || nil,
       location: lparen.location.to(rparen.location)
     )
   end
@@ -8896,7 +9297,11 @@ class SyntaxTree < Ripper
 
     def format(q)
       q.format(statements)
-      q.breakable(force: true)
+
+      # We're going to put a newline on the end so that it always has one unless
+      # it ends with the special __END__ syntax. In that case we want to
+      # replicate the text exactly so we will just let it be.
+      q.breakable(force: true) unless statements.body.last.is_a?(EndContent)
     end
 
     def pretty_print(q)
@@ -9035,6 +9440,9 @@ class SyntaxTree < Ripper
   #     %i[one two three]
   #
   class QSymbols
+    # [QSymbolsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ TStringContent ]] the elements of the array
     attr_reader :elements
 
@@ -9044,7 +9452,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -9055,7 +9464,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%i[", "]") do
+      opening, closing = "%i[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -9091,6 +9507,7 @@ class SyntaxTree < Ripper
   #   on_qsymbols_add: (QSymbols qsymbols, TStringContent element) -> QSymbols
   def on_qsymbols_add(qsymbols, element)
     QSymbols.new(
+      beginning: qsymbols.beginning,
       elements: qsymbols.elements << element,
       location: qsymbols.location.to(element.location)
     )
@@ -9132,9 +9549,13 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_qsymbols_new: () -> QSymbols
   def on_qsymbols_new
-    qsymbols_beg = find_token(QSymbolsBeg)
+    beginning = find_token(QSymbolsBeg)
 
-    QSymbols.new(elements: [], location: qsymbols_beg.location)
+    QSymbols.new(
+      beginning: beginning,
+      elements: [],
+      location: beginning.location
+    )
   end
 
   # QWords represents a string literal array without interpolation.
@@ -9142,6 +9563,9 @@ class SyntaxTree < Ripper
   #     %w[one two three]
   #
   class QWords
+    # [QWordsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ TStringContent ]] the elements of the array
     attr_reader :elements
 
@@ -9151,7 +9575,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -9162,7 +9587,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%w[", "]") do
+      opening, closing = "%w[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -9195,6 +9627,7 @@ class SyntaxTree < Ripper
   #   on_qwords_add: (QWords qwords, TStringContent element) -> QWords
   def on_qwords_add(qwords, element)
     QWords.new(
+      beginning: qwords.beginning,
       elements: qwords.elements << element,
       location: qwords.location.to(element.location)
     )
@@ -9236,9 +9669,9 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_qwords_new: () -> QWords
   def on_qwords_new
-    qwords_beg = find_token(QWordsBeg)
+    beginning = find_token(QWordsBeg)
 
-    QWords.new(elements: [], location: qwords_beg.location)
+    QWords.new(beginning: beginning, elements: [], location: beginning.location)
   end
 
   # RationalLiteral represents the use of a rational number literal.
@@ -9550,11 +9983,32 @@ class SyntaxTree < Ripper
           q.format_each(parts)
           q.text(ending)
         end
+      elsif braces
+        q.group do
+          q.text("%r{")
+
+          if beginning == "/"
+            # If we're changing from a forward slash to a %r{, then we can
+            # replace any escaped forward slashes with regular forward slashes.
+            parts.each do |part|
+              if part.is_a?(TStringContent)
+                q.text(part.value.gsub("\\/", "/"))
+              else
+                q.format(part)
+              end
+            end
+          else
+            q.format_each(parts)
+          end
+
+          q.text("}")
+          q.text(ending[1..-1])
+        end
       else
         q.group do
-          q.text(braces ? "%r{" : "/")
+          q.text("/")
           q.format_each(parts)
-          q.text(braces ? "}" : "/")
+          q.text("/")
           q.text(ending[1..-1])
         end
       end
@@ -10285,21 +10739,6 @@ class SyntaxTree < Ripper
   #   value
   # end
 
-  # stmts_add is a parser event that represents a single statement inside a
-  # list of statements within any lexical block. It accepts as arguments the
-  # parent stmts node as well as an stmt which can be any expression in
-  # Ruby.
-  def on_stmts_add(statements, statement)
-    location =
-      if statements.body.empty?
-        statement.location
-      else
-        statements.location.to(statement.location)
-      end
-
-    Statements.new(self, body: statements.body << statement, location: location)
-  end
-
   # Everything that has a block of code inside of it has a list of statements.
   # Normally we would just track those as a node that has an array body, but we
   # have some special handling in order to handle empty statement lists. They
@@ -10379,12 +10818,22 @@ class SyntaxTree < Ripper
       # the only value is a comment. In that case a lot of nodes like
       # brace_block will attempt to format as a single line, but since that
       # wouldn't work with a comment, we intentionally break the parent group.
-      if body.length == 2 && body.first.is_a?(VoidStmt)
-        q.format(body.last)
-        q.break_parent
-        return
+      if body.length == 2
+        void_stmt, comment = body
+
+        if void_stmt.is_a?(VoidStmt) && comment.is_a?(Comment)
+          q.format(comment)
+          q.break_parent
+          return
+        end
       end
 
+      access_controls =
+        Hash.new do |hash, node|
+          hash[node] = node.is_a?(VCall) &&
+            %w[private protected public].include?(node.value.value)
+        end
+
       body.each_with_index do |statement, index|
         next if statement.is_a?(VoidStmt)
 
@@ -10394,7 +10843,7 @@ class SyntaxTree < Ripper
           q.breakable(force: true)
           q.breakable(force: true)
           q.format(statement)
-        elsif statement.is_a?(AccessCtrl) || body[index - 1].is_a?(AccessCtrl)
+        elsif access_controls[statement] || access_controls[body[index - 1]]
           q.breakable(force: true)
           q.breakable(force: true)
           q.format(statement)
@@ -10446,7 +10895,7 @@ class SyntaxTree < Ripper
         location = comment.location
 
         if !comment.inline? && (start_char <= location.start_char) &&
-             (end_char >= location.end_char)
+             (end_char >= location.end_char) && !comment.ignore?
           parser_comments.delete_at(comment_index)
 
           while (node = body[body_index]) &&
@@ -10465,6 +10914,21 @@ class SyntaxTree < Ripper
     end
   end
 
+  # stmts_add is a parser event that represents a single statement inside a
+  # list of statements within any lexical block. It accepts as arguments the
+  # parent stmts node as well as an stmt which can be any expression in
+  # Ruby.
+  def on_stmts_add(statements, statement)
+    location =
+      if statements.body.empty?
+        statement.location
+      else
+        statements.location.to(statement.location)
+      end
+
+    Statements.new(self, body: statements.body << statement, location: location)
+  end
+
   # :call-seq:
   #   on_stmts_new: () -> Statements
   def on_stmts_new
@@ -10736,10 +11200,16 @@ class SyntaxTree < Ripper
       embexpr_end.location.start_char
     )
 
-    StringEmbExpr.new(
-      statements: statements,
-      location: embexpr_beg.location.to(embexpr_end.location)
-    )
+    location =
+      Location.new(
+        start_line: embexpr_beg.location.start_line,
+        start_char: embexpr_beg.location.start_char,
+        end_line:
+          [embexpr_end.location.end_line, statements.location.end_line].max,
+        end_char: embexpr_end.location.end_char
+      )
+
+    StringEmbExpr.new(statements: statements, location: location)
   end
 
   # StringLiteral represents a string literal.
@@ -10839,12 +11309,21 @@ class SyntaxTree < Ripper
       )
     else
       tstring_beg = find_token(TStringBeg)
-      tstring_end = find_token(TStringEnd)
+      tstring_end = find_token(TStringEnd, location: tstring_beg.location)
+
+      location =
+        Location.new(
+          start_line: tstring_beg.location.start_line,
+          start_char: tstring_beg.location.start_char,
+          end_line:
+            [tstring_end.location.end_line, string.location.end_line].max,
+          end_char: tstring_end.location.end_char
+        )
 
       StringLiteral.new(
         parts: string.parts,
         quote: tstring_beg.value,
-        location: tstring_beg.location.to(tstring_end.location)
+        location: location
       )
     end
   end
@@ -11066,6 +11545,9 @@ class SyntaxTree < Ripper
   #     %I[one two three]
   #
   class Symbols
+    # [SymbolsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ Word ]] the words in the symbol array literal
     attr_reader :elements
 
@@ -11075,7 +11557,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -11086,7 +11569,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%I[", "]") do
+      opening, closing = "%I[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -11122,6 +11612,7 @@ class SyntaxTree < Ripper
   #   on_symbols_add: (Symbols symbols, Word word) -> Symbols
   def on_symbols_add(symbols, word)
     Symbols.new(
+      beginning: symbols.beginning,
       elements: symbols.elements << word,
       location: symbols.location.to(word.location)
     )
@@ -11164,9 +11655,13 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_symbols_new: () -> Symbols
   def on_symbols_new
-    symbols_beg = find_token(SymbolsBeg)
+    beginning = find_token(SymbolsBeg)
 
-    Symbols.new(elements: [], location: symbols_beg.location)
+    Symbols.new(
+      beginning: beginning,
+      elements: [],
+      location: beginning.location
+    )
   end
 
   # TLambda represents the beginning of a lambda literal.
@@ -11258,6 +11753,11 @@ class SyntaxTree < Ripper
       [constant]
     end
 
+    def format(q)
+      q.text("::")
+      q.format(constant)
+    end
+
     def pretty_print(q)
       q.group(2, "(", ")") do
         q.text("top_const_field")
@@ -11412,6 +11912,10 @@ class SyntaxTree < Ripper
       @comments = comments
     end
 
+    def match?(pattern)
+      value.match?(pattern)
+    end
+
     def child_nodes
       []
     end
@@ -11914,23 +12418,39 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
+      # If the predicate of the loop contains an assignment (in which case we
+      # can't know for certain that that assignment doesn't impact the
+      # statements inside the loop) then we can't use the modifier form and we
+      # must use the block form.
+      if [Assign, MAssign, OpAssign].include?(node.predicate.class)
+        format_break(q)
+        q.break_parent
+        return
+      end
+
       q.group do
-        q.if_break do
-          q.text("#{keyword} ")
-          q.nest(keyword.length + 1) { q.format(node.predicate) }
-          q.indent do
-            q.breakable("")
+        q.if_break { format_break(q) }.if_flat do
+          Parentheses.flat(q) do
             q.format(statements)
+            q.text(" #{keyword} ")
+            q.format(node.predicate)
           end
-          q.breakable("")
-          q.text("end")
-        end.if_flat do
-          q.format(statements)
-          q.text(" #{keyword} ")
-          q.format(node.predicate)
         end
       end
     end
+
+    private
+
+    def format_break(q)
+      q.text("#{keyword} ")
+      q.nest(keyword.length + 1) { q.format(node.predicate) }
+      q.indent do
+        q.breakable("")
+        q.format(statements)
+      end
+      q.breakable("")
+      q.text("end")
+    end
   end
 
   # Until represents an +until+ loop.
@@ -12055,7 +12575,22 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      LoopFormatter.new("until", self, statement).format(q)
+      # If we're in the modifier form and we're modifying a `begin`, then this
+      # is a special case where we need to explicitly use the modifier form
+      # because otherwise the semantic meaning changes. This looks like:
+      #
+      #     begin
+      #       foo
+      #     end until bar
+      #
+      # The above is effectively a `do...until` loop.
+      if statement.is_a?(Begin)
+        q.format(statement)
+        q.text(" until ")
+        q.format(predicate)
+      else
+        LoopFormatter.new("until", self, statement).format(q)
+      end
     end
 
     def pretty_print(q)
@@ -12291,56 +12826,6 @@ class SyntaxTree < Ripper
     VarRef.new(value: value, location: value.location)
   end
 
-  # AccessCtrl represents a call to a method visibility control, i.e., +public+,
-  # +protected+, or +private+.
-  #
-  #     private
-  #
-  class AccessCtrl
-    # [Ident] the value of this expression
-    attr_reader :value
-
-    # [Location] the location of this node
-    attr_reader :location
-
-    # [Array[ Comment | EmbDoc ]] the comments attached to this node
-    attr_reader :comments
-
-    def initialize(value:, location:, comments: [])
-      @value = value
-      @location = location
-      @comments = comments
-    end
-
-    def child_nodes
-      [value]
-    end
-
-    def format(q)
-      q.format(value)
-    end
-
-    def pretty_print(q)
-      q.group(2, "(", ")") do
-        q.text("access_ctrl")
-
-        q.breakable
-        q.pp(value)
-
-        q.pp(Comment::List.new(comments))
-      end
-    end
-
-    def to_json(*opts)
-      {
-        type: :access_ctrl,
-        value: value,
-        loc: location,
-        cmts: comments
-      }.to_json(*opts)
-    end
-  end
-
   # VCall represent any plain named object with Ruby that could be either a
   # local variable or a method call.
   #
@@ -12389,19 +12874,9 @@ class SyntaxTree < Ripper
   end
 
   # :call-seq:
-  #   on_vcall: (Ident ident) -> AccessCtrl | VCall
+  #   on_vcall: (Ident ident) -> VCall
   def on_vcall(ident)
-    @controls ||= %w[private protected public].freeze
-
-    if @controls.include?(ident.value) && ident.value == lines[lineno - 1].strip
-      # Access controls like private, protected, and public are reported as
-      # vcall nodes since they're technically method calls. We want to be able
-      # add new lines around them as necessary, so here we're going to
-      # explicitly track those as a different node type.
-      AccessCtrl.new(value: ident, location: ident.location)
-    else
-      VCall.new(value: ident, location: ident.location)
-    end
+    VCall.new(value: ident, location: ident.location)
   end
 
   # VoidStmt represents an empty lexical block of code.
@@ -12552,7 +13027,11 @@ class SyntaxTree < Ripper
     beginning = find_token(Kw, "when")
     ending = consequent || find_token(Kw, "end")
 
-    statements.bind(arguments.location.end_char, ending.location.start_char)
+    statements_start = find_token(Kw, "then", consume: false) || arguments
+    statements.bind(
+      find_next_statement_start(statements_start.location.end_char),
+      ending.location.start_char
+    )
 
     When.new(
       arguments: arguments,
@@ -12684,7 +13163,22 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      LoopFormatter.new("while", self, statement).format(q)
+      # If we're in the modifier form and we're modifying a `begin`, then this
+      # is a special case where we need to explicitly use the modifier form
+      # because otherwise the semantic meaning changes. This looks like:
+      #
+      #     begin
+      #       foo
+      #     end while bar
+      #
+      # The above is effectively a `do...while` loop.
+      if statement.is_a?(Begin)
+        q.format(statement)
+        q.text(" while ")
+        q.format(predicate)
+      else
+        LoopFormatter.new("while", self, statement).format(q)
+      end
     end
 
     def pretty_print(q)
@@ -12748,6 +13242,10 @@ class SyntaxTree < Ripper
       @comments = comments
     end
 
+    def match?(pattern)
+      parts.any? { |part| part.is_a?(TStringContent) && part.match?(pattern) }
+    end
+
     def child_nodes
       parts
     end
@@ -12797,6 +13295,9 @@ class SyntaxTree < Ripper
   #     %W[one two three]
   #
   class Words
+    # [WordsBeg] the token that opens this array literal
+    attr_reader :beginning
+
     # [Array[ Word ]] the elements of this array
     attr_reader :elements
 
@@ -12806,7 +13307,8 @@ class SyntaxTree < Ripper
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(elements:, location:, comments: [])
+    def initialize(beginning:, elements:, location:, comments: [])
+      @beginning = beginning
       @elements = elements
       @location = location
       @comments = comments
@@ -12817,7 +13319,14 @@ class SyntaxTree < Ripper
     end
 
     def format(q)
-      q.group(0, "%W[", "]") do
+      opening, closing = "%W[", "]"
+
+      if elements.any? { |element| element.match?(/[\[\]]/) }
+        opening = beginning.value
+        closing = Quotes.matching(opening[2])
+      end
+
+      q.group(0, opening, closing) do
         q.indent do
           q.breakable("")
           q.seplist(elements, -> { q.breakable }) do |element|
@@ -12850,6 +13359,7 @@ class SyntaxTree < Ripper
   #   on_words_add: (Words words, Word word) -> Words
   def on_words_add(words, word)
     Words.new(
+      beginning: words.beginning,
       elements: words.elements << word,
       location: words.location.to(word.location)
     )
@@ -12892,9 +13402,9 @@ class SyntaxTree < Ripper
   # :call-seq:
   #   on_words_new: () -> Words
   def on_words_new
-    words_beg = find_token(WordsBeg)
+    beginning = find_token(WordsBeg)
 
-    Words.new(elements: [], location: words_beg.location)
+    Words.new(beginning: beginning, elements: [], location: beginning.location)
   end
 
   # def on_words_sep(value)
@@ -13011,7 +13521,7 @@ class SyntaxTree < Ripper
         location: heredoc.location
       )
     else
-      ending = find_token(TStringEnd)
+      ending = find_token(TStringEnd, location: xstring.location)
 
       XStringLiteral.new(
         parts: xstring.parts,
@@ -13047,8 +13557,18 @@ class SyntaxTree < Ripper
     def format(q)
       q.group do
         q.text("yield")
-        q.text(" ") if arguments.is_a?(Args)
-        q.format(arguments)
+
+        if arguments.is_a?(Paren)
+          q.format(arguments)
+        else
+          q.if_break { q.text("(") }.if_flat { q.text(" ") }
+          q.indent do
+            q.breakable("")
+            q.format(arguments)
+          end
+          q.breakable("")
+          q.if_break { q.text(")") }
+        end
       end
     end