syntax_tree 2.0.1 → 2.2.0

data/lib/syntax_tree/parser.rb

@@ -163,6 +163,13 @@ module SyntaxTree
       line_counts[lineno - 1][column]
     end
 
+    # This represents the current column we're in relative to the beginning of
+    # the current line.
+    def current_column
+      line = line_counts[lineno - 1]
+      line[column].to_i - line.start
+    end
+
     # As we build up a list of tokens, we'll periodically need to go backwards
     # and find the ones that we've already hit in order to determine the
     # location information for nodes that use them. For example, if you have a
@@ -251,10 +258,13 @@ module SyntaxTree
     def on_BEGIN(statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      start_char = find_next_statement_start(lbrace.location.end_char)
 
       statements.bind(
-        find_next_statement_start(lbrace.location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[lbrace.location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column,
       )
 
       keyword = find_token(Kw, "BEGIN")
@@ -271,7 +281,7 @@ module SyntaxTree
     def on_CHAR(value)
       CHAR.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -280,10 +290,13 @@ module SyntaxTree
     def on_END(statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      start_char = find_next_statement_start(lbrace.location.end_char)
 
       statements.bind(
-        find_next_statement_start(lbrace.location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[lbrace.location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column
       )
 
       keyword = find_token(Kw, "END")
@@ -301,7 +314,7 @@ module SyntaxTree
       @__end__ =
         EndContent.new(
           value: source[(char_pos + value.length)..-1],
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
     end
 
@@ -401,12 +414,19 @@ module SyntaxTree
     #     (false | untyped) block
     #   ) -> Args
     def on_args_add_block(arguments, block)
+      # First, see if there is an & operator that could potentially be
+      # associated with the block part of this args_add_block. If there is not,
+      # then just return the arguments.
       operator = find_token(Op, "&", consume: false)
-
-      # If we can't find the & operator, then there's no block to add to the
-      # list, so we're just going to return the arguments as-is.
       return arguments unless operator
 
+      # If there are any arguments and the operator we found from the list is
+      # not after them, then we're going to return the arguments as-is because
+      # we're looking at an & that occurs before the arguments are done.
+      if arguments.parts.any? && operator.location.start_char < arguments.location.end_char
+        return arguments
+      end
+
       # Now we know we have an & operator, so we're going to delete it from the
       # list of tokens to make sure it doesn't get confused with anything else.
       tokens.delete(operator)
@@ -415,13 +435,6 @@ module SyntaxTree
       location = operator.location
       location = operator.location.to(block.location) if block
 
-      # If there are any arguments and the operator we found from the list is
-      # not after them, then we're going to return the arguments as-is because
-      # we're looking at an & that occurs before the arguments are done.
-      if arguments.parts.any? && location.start_char < arguments.location.end_char
-        return arguments
-      end
-
       # Otherwise, we're looking at an actual block argument (with or without a
       # block, which could be missing because it could be a bare & since 3.1.0).
       arg_block = ArgBlock.new(value: block, location: location)
@@ -465,7 +478,7 @@ module SyntaxTree
     # :call-seq:
     #   on_args_new: () -> Args
     def on_args_new
-      Args.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      Args.new(parts: [], location: Location.fixed(line: lineno, column: current_column, char: char_pos))
     end
 
     # :call-seq:
@@ -503,12 +516,46 @@ module SyntaxTree
     def on_aryptn(constant, requireds, rest, posts)
       parts = [constant, *requireds, rest, *posts].compact
 
+      # If there aren't any parts (no constant, no positional arguments), then
+      # we're matching an empty array. In this case, we're going to look for the
+      # left and right brackets explicitly. Otherwise, we'll just use the bounds
+      # of the various parts.
+      location =
+        if parts.empty?
+          find_token(LBracket).location.to(find_token(RBracket).location)
+        else
+          parts[0].location.to(parts[-1].location)
+        end
+
+      # If there's the optional then keyword, then we'll delete that and use it
+      # as the end bounds of the location.
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+        location = location.to(token.location)
+      end
+
+      # If there is a plain *, then we're going to fix up the location of it
+      # here because it currently doesn't have anything to use for its precise
+      # location. If we hit a comma, then we've gone too far.
+      if rest.is_a?(VarField) && rest.value.nil?
+        tokens.rindex do |token|
+          case token
+          in Op[value: "*"]
+            rest = VarField.new(value: nil, location: token.location)
+            break
+          in Comma
+            break
+          else
+          end
+        end
+      end
+
       AryPtn.new(
         constant: constant,
         requireds: requireds || [],
         rest: rest,
         posts: posts || [],
-        location: parts[0].location.to(parts[-1].location)
+        location: location
       )
     end
 
@@ -551,7 +598,7 @@ module SyntaxTree
     def on_backref(value)
       Backref.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -561,7 +608,7 @@ module SyntaxTree
       node =
         Backtick.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -592,15 +639,20 @@ module SyntaxTree
         PinnedBegin.new(statement: bodystmt, location: location)
       else
         keyword = find_token(Kw, "begin")
-        end_char =
+        end_location =
           if bodystmt.rescue_clause || bodystmt.ensure_clause ||
               bodystmt.else_clause
-            bodystmt.location.end_char
+            bodystmt.location
           else
-            find_token(Kw, "end").location.end_char
+            find_token(Kw, "end").location
           end
 
-        bodystmt.bind(keyword.location.end_char, end_char)
+        bodystmt.bind(
+          keyword.location.end_char,
+          keyword.location.end_column,
+          end_location.end_char,
+          end_location.end_column
+        )
         location = keyword.location.to(bodystmt.location)
 
         Begin.new(bodystmt: bodystmt, location: location)
@@ -682,9 +734,10 @@ module SyntaxTree
       BodyStmt.new(
         statements: statements,
         rescue_clause: rescue_clause,
+        else_keyword: else_clause && find_token(Kw, "else"),
         else_clause: else_clause,
         ensure_clause: ensure_clause,
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -696,18 +749,24 @@ module SyntaxTree
     def on_brace_block(block_var, statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      location = (block_var || lbrace).location
+      start_char = find_next_statement_start(location.end_char)
 
       statements.bind(
-        find_next_statement_start((block_var || lbrace).location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column
       )
 
       location =
         Location.new(
           start_line: lbrace.location.start_line,
           start_char: lbrace.location.start_char,
+          start_column: lbrace.location.start_column,
           end_line: [rbrace.location.end_line, statements.location.end_line].max,
-          end_char: rbrace.location.end_char
+          end_char: rbrace.location.end_char,
+          end_column: rbrace.location.end_column
         )
 
       BraceBlock.new(
@@ -736,8 +795,14 @@ module SyntaxTree
     #     (:call | Backtick | Const | Ident | Op) message
     #   ) -> Call
     def on_call(receiver, operator, message)
-      ending = message
-      ending = operator if message == :call
+      ending =
+        if message != :call
+          message
+        elsif operator != :"::"
+          operator
+        else
+          receiver
+        end
 
       Call.new(
         receiver: receiver,
@@ -781,10 +846,14 @@ module SyntaxTree
     def on_class(constant, superclass, bodystmt)
       beginning = find_token(Kw, "class")
       ending = find_token(Kw, "end")
+      location = (superclass || constant).location
+      start_char = find_next_statement_start(location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start((superclass || constant).location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       ClassDeclaration.new(
@@ -801,7 +870,7 @@ module SyntaxTree
       node =
         Comma.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -846,7 +915,7 @@ module SyntaxTree
           value: value.chomp,
           inline: value.strip != lines[line - 1].strip,
           location:
-            Location.token(line: line, char: char_pos, size: value.size - 1)
+            Location.token(line: line, char: char_pos, column: current_column, size: value.size - 1)
         )
 
       @comments << comment
@@ -858,7 +927,7 @@ module SyntaxTree
     def on_const(value)
       Const.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -893,7 +962,7 @@ module SyntaxTree
     def on_cvar(value)
       CVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -917,12 +986,15 @@ module SyntaxTree
       # location information
       if params.is_a?(Params) && params.empty?
         end_char = name.location.end_char
+        end_column = name.location.end_column
         location =
           Location.new(
             start_line: params.location.start_line,
             start_char: end_char,
+            start_column: end_column,
             end_line: params.location.end_line,
-            end_char: end_char
+            end_char: end_char,
+            end_column: end_column
           )
 
         params = Params.new(location: location)
@@ -932,9 +1004,13 @@ module SyntaxTree
 
       if ending
         tokens.delete(ending)
+        start_char = find_next_statement_start(params.location.end_char)
+
         bodystmt.bind(
-          find_next_statement_start(params.location.end_char),
-          ending.location.start_char
+          start_char,
+          start_char - line_counts[params.location.start_line - 1].start,
+          ending.location.start_char,
+          ending.location.start_column
         )
 
         Def.new(
@@ -992,12 +1068,15 @@ module SyntaxTree
       # location information
       if params.is_a?(Params) && params.empty?
         end_char = name.location.end_char
+        end_column = name.location.end_column
         location =
           Location.new(
             start_line: params.location.start_line,
             start_char: end_char,
+            start_column: end_column,
             end_line: params.location.end_line,
-            end_char: end_char
+            end_char: end_char,
+            end_column: end_column
           )
 
         params = Params.new(location: location)
@@ -1008,9 +1087,13 @@ module SyntaxTree
 
       if ending
         tokens.delete(ending)
+        start_char = find_next_statement_start(params.location.end_char)
+
         bodystmt.bind(
-          find_next_statement_start(params.location.end_char),
-          ending.location.start_char
+          start_char,
+          start_char - line_counts[params.location.start_line - 1].start,
+          ending.location.start_char,
+          ending.location.start_column
         )
 
         Defs.new(
@@ -1042,10 +1125,14 @@ module SyntaxTree
     def on_do_block(block_var, bodystmt)
       beginning = find_token(Kw, "do")
       ending = find_token(Kw, "end")
+      location = (block_var || beginning).location
+      start_char = find_next_statement_start(location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start((block_var || beginning).location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       DoBlock.new(
@@ -1115,7 +1202,7 @@ module SyntaxTree
     # :call-seq:
     #   on_else: (Statements statements) -> Else
     def on_else(statements)
-      beginning = find_token(Kw, "else")
+      keyword = find_token(Kw, "else")
 
       # else can either end with an end keyword (in which case we'll want to
       # consume that event) or it can end with an ensure keyword (in which case
@@ -1127,13 +1214,19 @@ module SyntaxTree
 
       node = tokens[index]
       ending = node.value == "end" ? tokens.delete_at(index) : node
-      # ending = node
+      start_char = find_next_statement_start(keyword.location.end_char)
 
-      statements.bind(beginning.location.end_char, ending.location.start_char)
+      statements.bind(
+        start_char,
+        start_char - line_counts[keyword.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Else.new(
+        keyword: keyword,
         statements: statements,
-        location: beginning.location.to(ending.location)
+        location: keyword.location.to(ending.location)
       )
     end
 
@@ -1147,7 +1240,12 @@ module SyntaxTree
       beginning = find_token(Kw, "elsif")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Elsif.new(
         predicate: predicate,
@@ -1170,7 +1268,7 @@ module SyntaxTree
       @embdoc =
         EmbDoc.new(
           value: value,
-          location: Location.fixed(line: lineno, char: char_pos)
+          location: Location.fixed(line: lineno, column: current_column, char: char_pos)
         )
     end
 
@@ -1185,8 +1283,10 @@ module SyntaxTree
             Location.new(
               start_line: location.start_line,
               start_char: location.start_char,
+              start_column: location.start_column,
               end_line: lineno,
-              end_char: char_pos + value.length - 1
+              end_char: char_pos + value.length - 1,
+              end_column: current_column + value.length - 1
             )
         )
 
@@ -1202,7 +1302,7 @@ module SyntaxTree
       node =
         EmbExprBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1215,7 +1315,7 @@ module SyntaxTree
       node =
         EmbExprEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1228,7 +1328,7 @@ module SyntaxTree
       node =
         EmbVar.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1243,9 +1343,12 @@ module SyntaxTree
       # We don't want to consume the :@kw event, because that would break
       # def..ensure..end chains.
       ending = find_token(Kw, "end", consume: false)
+      start_char = find_next_statement_start(keyword.location.end_char)
       statements.bind(
-        find_next_statement_start(keyword.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[keyword.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       Ensure.new(
@@ -1292,7 +1395,7 @@ module SyntaxTree
     def on_float(value)
       FloatLiteral.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1304,15 +1407,35 @@ module SyntaxTree
     #     VarField right
     #   ) -> FndPtn
     def on_fndptn(constant, left, values, right)
-      beginning = constant || find_token(LBracket)
-      ending = find_token(RBracket)
+      # The opening of this find pattern is either going to be a left bracket, a
+      # right left parenthesis, or the left splat. We're going to use this to
+      # determine how to find the closing of the pattern, as well as determining
+      # the location of the node.
+      opening =
+        find_token(LBracket, consume: false) ||
+        find_token(LParen, consume: false) ||
+        left
+
+      # The closing is based on the opening, which is either the matched
+      # punctuation or the right splat.
+      closing =
+        case opening
+        in LBracket
+          tokens.delete(opening)
+          find_token(RBracket)
+        in LParen
+          tokens.delete(opening)
+          find_token(RParen)
+        else
+          right
+        end
 
       FndPtn.new(
         constant: constant,
         left: left,
         values: values,
         right: right,
-        location: beginning.location.to(ending.location)
+        location: (constant || opening).location.to(closing.location)
       )
     end
 
@@ -1337,7 +1460,9 @@ module SyntaxTree
 
       statements.bind(
         (keyword || collection).location.end_char,
-        ending.location.start_char
+        (keyword || collection).location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       if index.is_a?(MLHS)
@@ -1358,7 +1483,7 @@ module SyntaxTree
     def on_gvar(value)
       GVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1379,7 +1504,7 @@ module SyntaxTree
     #   on_heredoc_beg: (String value) -> HeredocBeg
     def on_heredoc_beg(value)
       location =
-        Location.token(line: lineno, char: char_pos, size: value.size + 1)
+        Location.token(line: lineno, char: char_pos, column: current_column, size: value.size + 1)
 
       # Here we're going to artificially create an extra node type so that if
       # there are comments after the declaration of a heredoc, they get printed.
@@ -1397,6 +1522,7 @@ module SyntaxTree
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: heredoc.ending,
+        dedent: width,
         parts: string.parts,
         location: heredoc.location
       )
@@ -1410,13 +1536,16 @@ module SyntaxTree
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: value.chomp,
+        dedent: heredoc.dedent,
         parts: heredoc.parts,
         location:
           Location.new(
             start_line: heredoc.location.start_line,
             start_char: heredoc.location.start_char,
+            start_column: heredoc.location.start_column,
             end_line: lineno,
-            end_char: char_pos
+            end_char: char_pos,
+            end_column: current_column,
           )
       )
     end
@@ -1428,13 +1557,30 @@ module SyntaxTree
     #     (nil | VarField) keyword_rest
     #   ) -> HshPtn
     def on_hshptn(constant, keywords, keyword_rest)
-      parts = [constant, keywords, keyword_rest].flatten(2).compact
+      # Create an artificial VarField if we find an extra ** on the end
+      if !keyword_rest && (token = find_token(Op, "**", consume: false))
+        tokens.delete(token)
+        keyword_rest = VarField.new(value: nil, location: token.location)
+      end
+
+      # Delete the optional then keyword
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+      end
+
+      parts = [constant, *keywords&.flatten(1), keyword_rest].compact
+      location =
+        if parts.any?
+          parts[0].location.to(parts[-1].location)
+        else
+          find_token(LBrace).location.to(find_token(RBrace).location)
+        end
 
       HshPtn.new(
         constant: constant,
-        keywords: keywords,
+        keywords: keywords || [],
         keyword_rest: keyword_rest,
-        location: parts[0].location.to(parts[-1].location)
+        location: location
       )
     end
 
@@ -1443,7 +1589,7 @@ module SyntaxTree
     def on_ident(value)
       Ident.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1457,7 +1603,12 @@ module SyntaxTree
       beginning = find_token(Kw, "if")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       If.new(
         predicate: predicate,
@@ -1503,7 +1654,7 @@ module SyntaxTree
     def on_imaginary(value)
       Imaginary.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1527,9 +1678,12 @@ module SyntaxTree
         statements_start = token
       end
 
+      start_char = find_next_statement_start(statements_start.location.end_char)
       statements.bind(
-        find_next_statement_start(statements_start.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[statements_start.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       In.new(
@@ -1545,7 +1699,7 @@ module SyntaxTree
     def on_int(value)
       Int.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1554,7 +1708,7 @@ module SyntaxTree
     def on_ivar(value)
       IVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1564,7 +1718,7 @@ module SyntaxTree
       node =
         Kw.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1585,7 +1739,7 @@ module SyntaxTree
     def on_label(value)
       Label.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1595,7 +1749,7 @@ module SyntaxTree
       node =
         LabelEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1621,7 +1775,12 @@ module SyntaxTree
         closing = find_token(Kw, "end")
       end
 
-      statements.bind(opening.location.end_char, closing.location.start_char)
+      statements.bind(
+        opening.location.end_char,
+        opening.location.end_column,
+        closing.location.start_char,
+        closing.location.start_column
+      )
 
       Lambda.new(
         params: params,
@@ -1636,7 +1795,7 @@ module SyntaxTree
       node =
         LBrace.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1649,7 +1808,7 @@ module SyntaxTree
       node =
         LBracket.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1662,7 +1821,7 @@ module SyntaxTree
       node =
         LParen.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1761,7 +1920,7 @@ module SyntaxTree
     # :call-seq:
     #   on_mlhs_new: () -> MLHS
     def on_mlhs_new
-      MLHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      MLHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -1787,10 +1946,13 @@ module SyntaxTree
     def on_module(constant, bodystmt)
       beginning = find_token(Kw, "module")
       ending = find_token(Kw, "end")
+      start_char = find_next_statement_start(constant.location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start(constant.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[constant.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       ModuleDeclaration.new(
@@ -1803,7 +1965,7 @@ module SyntaxTree
     # :call-seq:
     #   on_mrhs_new: () -> MRHS
     def on_mrhs_new
-      MRHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      MRHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -1872,7 +2034,7 @@ module SyntaxTree
       node =
         Op.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1931,7 +2093,7 @@ module SyntaxTree
         if parts.any?
           parts[0].location.to(parts[-1].location)
         else
-          Location.fixed(line: lineno, char: char_pos)
+          Location.fixed(line: lineno, char: char_pos, column: current_column)
         end
 
       Params.new(
@@ -1954,12 +2116,15 @@ module SyntaxTree
 
       if contents && contents.is_a?(Params)
         location = contents.location
+        start_char = find_next_statement_start(lparen.location.end_char)
         location =
           Location.new(
             start_line: location.start_line,
-            start_char: find_next_statement_start(lparen.location.end_char),
+            start_char: start_char,
+            start_column: start_char - line_counts[lparen.location.start_line - 1].start,
             end_line: location.end_line,
-            end_char: rparen.location.start_char
+            end_char: rparen.location.start_char,
+            end_column: rparen.location.start_column
           )
 
         contents =
@@ -1997,23 +2162,26 @@ module SyntaxTree
     def on_period(value)
       Period.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
     # :call-seq:
     #   on_program: (Statements statements) -> Program
     def on_program(statements)
+      last_column = source.length - line_counts[lines.length - 1].start
       location =
         Location.new(
           start_line: 1,
           start_char: 0,
+          start_column: 0,
           end_line: lines.length,
-          end_char: source.length
+          end_char: source.length,
+          end_column: last_column
         )
 
       statements.body << @__end__ if @__end__
-      statements.bind(0, source.length)
+      statements.bind(0, 0, source.length, last_column)
 
       program = Program.new(statements: statements, location: location)
       attach_comments(program, @comments)
@@ -2126,7 +2294,7 @@ module SyntaxTree
       node =
         QSymbolsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2161,7 +2329,7 @@ module SyntaxTree
       node =
         QWordsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2181,7 +2349,7 @@ module SyntaxTree
     def on_rational(value)
       RationalLiteral.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2191,7 +2359,7 @@ module SyntaxTree
       node =
         RBrace.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2204,7 +2372,7 @@ module SyntaxTree
       node =
         RBracket.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2238,7 +2406,7 @@ module SyntaxTree
       node =
         RegexpBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2250,7 +2418,7 @@ module SyntaxTree
     def on_regexp_end(value)
       RegexpEnd.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2292,9 +2460,12 @@ module SyntaxTree
       exceptions = exceptions[0] if exceptions.is_a?(Array)
 
       last_node = variable || exceptions || keyword
+      start_char = find_next_statement_start(last_node.location.end_char)
       statements.bind(
-        find_next_statement_start(last_node.location.end_char),
-        char_pos
+        start_char,
+        start_char - line_counts[last_node.location.start_line - 1].start,
+        char_pos,
+        current_column
       )
 
       # We add an additional inner node here that ripper doesn't provide so that
@@ -2309,13 +2480,16 @@ module SyntaxTree
               Location.new(
                 start_line: keyword.location.start_line,
                 start_char: keyword.location.end_char + 1,
+                start_column: keyword.location.end_column + 1,
                 end_line: last_node.location.end_line,
-                end_char: last_node.location.end_char
+                end_char: last_node.location.end_char,
+                end_column: last_node.location.end_column
               )
           )
         end
 
       Rescue.new(
+        keyword: keyword,
         exception: rescue_ex,
         statements: statements,
         consequent: consequent,
@@ -2323,8 +2497,10 @@ module SyntaxTree
           Location.new(
             start_line: keyword.location.start_line,
             start_char: keyword.location.start_char,
+            start_column: keyword.location.start_column,
             end_line: lineno,
-            end_char: char_pos
+            end_char: char_pos,
+            end_column: current_column
           )
       )
     end
@@ -2383,7 +2559,7 @@ module SyntaxTree
       node =
         RParen.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2395,10 +2571,13 @@ module SyntaxTree
     def on_sclass(target, bodystmt)
       beginning = find_token(Kw, "class")
       ending = find_token(Kw, "end")
+      start_char = find_next_statement_start(target.location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start(target.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[target.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       SClass.new(
@@ -2437,7 +2616,7 @@ module SyntaxTree
       Statements.new(
         self,
         body: [],
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -2471,7 +2650,7 @@ module SyntaxTree
     def on_string_content
       StringContent.new(
         parts: [],
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -2494,18 +2673,22 @@ module SyntaxTree
 
       statements.bind(
         embexpr_beg.location.end_char,
-        embexpr_end.location.start_char
+        embexpr_beg.location.end_column,
+        embexpr_end.location.start_char,
+        embexpr_end.location.start_column
       )
 
       location =
         Location.new(
           start_line: embexpr_beg.location.start_line,
           start_char: embexpr_beg.location.start_char,
+          start_column: embexpr_beg.location.start_column,
           end_line: [
             embexpr_end.location.end_line,
             statements.location.end_line
           ].max,
-          end_char: embexpr_end.location.end_char
+          end_char: embexpr_end.location.end_char,
+          end_column: embexpr_end.location.end_column
         )
 
       StringEmbExpr.new(statements: statements, location: location)
@@ -2522,6 +2705,7 @@ module SyntaxTree
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: string.parts,
           location: heredoc.location
         )
@@ -2533,11 +2717,13 @@ module SyntaxTree
           Location.new(
             start_line: tstring_beg.location.start_line,
             start_char: tstring_beg.location.start_char,
+            start_column: tstring_beg.location.start_column,
             end_line: [
               tstring_end.location.end_line,
               string.location.end_line
             ].max,
-            end_char: tstring_end.location.end_char
+            end_char: tstring_end.location.end_char,
+            end_column: tstring_end.location.end_column
           )
 
         StringLiteral.new(
@@ -2566,7 +2752,7 @@ module SyntaxTree
       node =
         SymBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2620,7 +2806,7 @@ module SyntaxTree
       node =
         SymbolsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2645,7 +2831,7 @@ module SyntaxTree
       node =
         TLambda.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2658,7 +2844,7 @@ module SyntaxTree
       node =
         TLamBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2693,7 +2879,7 @@ module SyntaxTree
       node =
         TStringBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2705,7 +2891,7 @@ module SyntaxTree
     def on_tstring_content(value)
       TStringContent.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2715,7 +2901,7 @@ module SyntaxTree
       node =
         TStringEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2731,19 +2917,17 @@ module SyntaxTree
         # parentheses they don't get reported as a paren node for some reason.
 
         beginning = find_token(Kw, "not")
-        ending = statement
-
-        range = beginning.location.end_char...statement.location.start_char
-        paren = source[range].include?("(")
+        ending = statement || beginning
+        parentheses = source[beginning.location.end_char] == "("
 
-        if paren
+        if parentheses
           find_token(LParen)
           ending = find_token(RParen)
         end
 
         Not.new(
           statement: statement,
-          parentheses: paren,
+          parentheses: parentheses,
           location: beginning.location.to(ending.location)
         )
       else
@@ -2789,7 +2973,12 @@ module SyntaxTree
       beginning = find_token(Kw, "unless")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Unless.new(
         predicate: predicate,
@@ -2826,7 +3015,12 @@ module SyntaxTree
       end
 
       # Update the Statements location information
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Until.new(
         predicate: predicate,
@@ -2865,12 +3059,12 @@ module SyntaxTree
     #   ) -> VarField
     def on_var_field(value)
       location =
-        if value
+        if value && value != :nil
           value.location
         else
           # You can hit this pattern if you're assigning to a splat using
           # pattern matching syntax in Ruby 2.7+
-          Location.fixed(line: lineno, char: char_pos)
+          Location.fixed(line: lineno, char: char_pos, column: current_column)
         end
 
       VarField.new(value: value, location: location)
@@ -2898,7 +3092,7 @@ module SyntaxTree
     # :call-seq:
     #   on_void_stmt: () -> VoidStmt
     def on_void_stmt
-      VoidStmt.new(location: Location.fixed(line: lineno, char: char_pos))
+      VoidStmt.new(location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -2917,9 +3111,13 @@ module SyntaxTree
         statements_start = token
       end
 
+      start_char = find_next_statement_start(statements_start.location.end_char)
+
       statements.bind(
-        find_next_statement_start(statements_start.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[statements_start.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       When.new(
@@ -2945,7 +3143,12 @@ module SyntaxTree
       end
 
       # Update the Statements location information
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       While.new(
         predicate: predicate,
@@ -2981,7 +3184,7 @@ module SyntaxTree
     # :call-seq:
     #   on_word_new: () -> Word
     def on_word_new
-      Word.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      Word.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -3000,7 +3203,7 @@ module SyntaxTree
       node =
         WordsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -3055,6 +3258,7 @@ module SyntaxTree
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: xstring.parts,
           location: heredoc.location
         )