syntax_tree 2.0.0 → 2.1.1

data/lib/syntax_tree/parser.rb

@@ -163,6 +163,13 @@ module SyntaxTree
       line_counts[lineno - 1][column]
     end
 
+    # This represents the current column we're in relative to the beginning of
+    # the current line.
+    def current_column
+      line = line_counts[lineno - 1]
+      line[column].to_i - line.start
+    end
+
     # As we build up a list of tokens, we'll periodically need to go backwards
     # and find the ones that we've already hit in order to determine the
     # location information for nodes that use them. For example, if you have a
@@ -251,10 +258,13 @@ module SyntaxTree
     def on_BEGIN(statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      start_char = find_next_statement_start(lbrace.location.end_char)
 
       statements.bind(
-        find_next_statement_start(lbrace.location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[lbrace.location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column,
       )
 
       keyword = find_token(Kw, "BEGIN")
@@ -271,7 +281,7 @@ module SyntaxTree
     def on_CHAR(value)
       CHAR.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -280,10 +290,13 @@ module SyntaxTree
     def on_END(statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      start_char = find_next_statement_start(lbrace.location.end_char)
 
       statements.bind(
-        find_next_statement_start(lbrace.location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[lbrace.location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column
       )
 
       keyword = find_token(Kw, "END")
@@ -301,7 +314,7 @@ module SyntaxTree
       @__end__ =
         EndContent.new(
           value: source[(char_pos + value.length)..-1],
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
     end
 
@@ -401,12 +414,19 @@ module SyntaxTree
     #     (false | untyped) block
     #   ) -> Args
     def on_args_add_block(arguments, block)
+      # First, see if there is an & operator that could potentially be
+      # associated with the block part of this args_add_block. If there is not,
+      # then just return the arguments.
       operator = find_token(Op, "&", consume: false)
-
-      # If we can't find the & operator, then there's no block to add to the
-      # list, so we're just going to return the arguments as-is.
       return arguments unless operator
 
+      # If there are any arguments and the operator we found from the list is
+      # not after them, then we're going to return the arguments as-is because
+      # we're looking at an & that occurs before the arguments are done.
+      if arguments.parts.any? && operator.location.start_char < arguments.location.end_char
+        return arguments
+      end
+
       # Now we know we have an & operator, so we're going to delete it from the
       # list of tokens to make sure it doesn't get confused with anything else.
       tokens.delete(operator)
@@ -415,13 +435,6 @@ module SyntaxTree
       location = operator.location
       location = operator.location.to(block.location) if block
 
-      # If there are any arguments and the operator we found from the list is
-      # not after them, then we're going to return the arguments as-is because
-      # we're looking at an & that occurs before the arguments are done.
-      if arguments.parts.any? && location.start_char < arguments.location.end_char
-        return arguments
-      end
-
       # Otherwise, we're looking at an actual block argument (with or without a
       # block, which could be missing because it could be a bare & since 3.1.0).
       arg_block = ArgBlock.new(value: block, location: location)
@@ -465,7 +478,7 @@ module SyntaxTree
     # :call-seq:
     #   on_args_new: () -> Args
     def on_args_new
-      Args.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      Args.new(parts: [], location: Location.fixed(line: lineno, column: current_column, char: char_pos))
     end
 
     # :call-seq:
@@ -551,7 +564,7 @@ module SyntaxTree
     def on_backref(value)
       Backref.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -561,7 +574,7 @@ module SyntaxTree
       node =
         Backtick.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -592,15 +605,20 @@ module SyntaxTree
         PinnedBegin.new(statement: bodystmt, location: location)
       else
         keyword = find_token(Kw, "begin")
-        end_char =
+        end_location =
           if bodystmt.rescue_clause || bodystmt.ensure_clause ||
               bodystmt.else_clause
-            bodystmt.location.end_char
+            bodystmt.location
           else
-            find_token(Kw, "end").location.end_char
+            find_token(Kw, "end").location
           end
 
-        bodystmt.bind(keyword.location.end_char, end_char)
+        bodystmt.bind(
+          keyword.location.end_char,
+          keyword.location.end_column,
+          end_location.end_char,
+          end_location.end_column
+        )
         location = keyword.location.to(bodystmt.location)
 
         Begin.new(bodystmt: bodystmt, location: location)
@@ -682,9 +700,10 @@ module SyntaxTree
       BodyStmt.new(
         statements: statements,
         rescue_clause: rescue_clause,
+        else_keyword: else_clause && find_token(Kw, "else"),
         else_clause: else_clause,
         ensure_clause: ensure_clause,
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -696,18 +715,24 @@ module SyntaxTree
     def on_brace_block(block_var, statements)
       lbrace = find_token(LBrace)
       rbrace = find_token(RBrace)
+      location = (block_var || lbrace).location
+      start_char = find_next_statement_start(location.end_char)
 
       statements.bind(
-        find_next_statement_start((block_var || lbrace).location.end_char),
-        rbrace.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        rbrace.location.start_char,
+        rbrace.location.start_column
       )
 
       location =
         Location.new(
           start_line: lbrace.location.start_line,
           start_char: lbrace.location.start_char,
+          start_column: lbrace.location.start_column,
           end_line: [rbrace.location.end_line, statements.location.end_line].max,
-          end_char: rbrace.location.end_char
+          end_char: rbrace.location.end_char,
+          end_column: rbrace.location.end_column
         )
 
       BraceBlock.new(
@@ -736,8 +761,14 @@ module SyntaxTree
     #     (:call | Backtick | Const | Ident | Op) message
     #   ) -> Call
     def on_call(receiver, operator, message)
-      ending = message
-      ending = operator if message == :call
+      ending =
+        if message != :call
+          message
+        elsif operator != :"::"
+          operator
+        else
+          receiver
+        end
 
       Call.new(
         receiver: receiver,
@@ -781,10 +812,14 @@ module SyntaxTree
     def on_class(constant, superclass, bodystmt)
       beginning = find_token(Kw, "class")
       ending = find_token(Kw, "end")
+      location = (superclass || constant).location
+      start_char = find_next_statement_start(location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start((superclass || constant).location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       ClassDeclaration.new(
@@ -801,7 +836,7 @@ module SyntaxTree
       node =
         Comma.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -846,7 +881,7 @@ module SyntaxTree
           value: value.chomp,
           inline: value.strip != lines[line - 1].strip,
           location:
-            Location.token(line: line, char: char_pos, size: value.size - 1)
+            Location.token(line: line, char: char_pos, column: current_column, size: value.size - 1)
         )
 
       @comments << comment
@@ -858,7 +893,7 @@ module SyntaxTree
     def on_const(value)
       Const.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -893,7 +928,7 @@ module SyntaxTree
     def on_cvar(value)
       CVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -917,12 +952,15 @@ module SyntaxTree
       # location information
       if params.is_a?(Params) && params.empty?
         end_char = name.location.end_char
+        end_column = name.location.end_column
         location =
           Location.new(
             start_line: params.location.start_line,
             start_char: end_char,
+            start_column: end_column,
             end_line: params.location.end_line,
-            end_char: end_char
+            end_char: end_char,
+            end_column: end_column
           )
 
         params = Params.new(location: location)
@@ -932,9 +970,13 @@ module SyntaxTree
 
       if ending
         tokens.delete(ending)
+        start_char = find_next_statement_start(params.location.end_char)
+
         bodystmt.bind(
-          find_next_statement_start(params.location.end_char),
-          ending.location.start_char
+          start_char,
+          start_char - line_counts[params.location.start_line - 1].start,
+          ending.location.start_char,
+          ending.location.start_column
         )
 
         Def.new(
@@ -992,12 +1034,15 @@ module SyntaxTree
       # location information
       if params.is_a?(Params) && params.empty?
         end_char = name.location.end_char
+        end_column = name.location.end_column
         location =
           Location.new(
             start_line: params.location.start_line,
             start_char: end_char,
+            start_column: end_column,
             end_line: params.location.end_line,
-            end_char: end_char
+            end_char: end_char,
+            end_column: end_column
           )
 
         params = Params.new(location: location)
@@ -1008,9 +1053,13 @@ module SyntaxTree
 
       if ending
         tokens.delete(ending)
+        start_char = find_next_statement_start(params.location.end_char)
+
         bodystmt.bind(
-          find_next_statement_start(params.location.end_char),
-          ending.location.start_char
+          start_char,
+          start_char - line_counts[params.location.start_line - 1].start,
+          ending.location.start_char,
+          ending.location.start_column
         )
 
         Defs.new(
@@ -1042,10 +1091,14 @@ module SyntaxTree
     def on_do_block(block_var, bodystmt)
       beginning = find_token(Kw, "do")
       ending = find_token(Kw, "end")
+      location = (block_var || beginning).location
+      start_char = find_next_statement_start(location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start((block_var || beginning).location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       DoBlock.new(
@@ -1115,7 +1168,7 @@ module SyntaxTree
     # :call-seq:
     #   on_else: (Statements statements) -> Else
     def on_else(statements)
-      beginning = find_token(Kw, "else")
+      keyword = find_token(Kw, "else")
 
       # else can either end with an end keyword (in which case we'll want to
       # consume that event) or it can end with an ensure keyword (in which case
@@ -1127,13 +1180,19 @@ module SyntaxTree
 
       node = tokens[index]
       ending = node.value == "end" ? tokens.delete_at(index) : node
-      # ending = node
+      start_char = find_next_statement_start(keyword.location.end_char)
 
-      statements.bind(beginning.location.end_char, ending.location.start_char)
+      statements.bind(
+        start_char,
+        start_char - line_counts[keyword.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Else.new(
+        keyword: keyword,
         statements: statements,
-        location: beginning.location.to(ending.location)
+        location: keyword.location.to(ending.location)
       )
     end
 
@@ -1147,7 +1206,12 @@ module SyntaxTree
       beginning = find_token(Kw, "elsif")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Elsif.new(
         predicate: predicate,
@@ -1170,7 +1234,7 @@ module SyntaxTree
       @embdoc =
         EmbDoc.new(
           value: value,
-          location: Location.fixed(line: lineno, char: char_pos)
+          location: Location.fixed(line: lineno, column: current_column, char: char_pos)
         )
     end
 
@@ -1185,8 +1249,10 @@ module SyntaxTree
             Location.new(
               start_line: location.start_line,
               start_char: location.start_char,
+              start_column: location.start_column,
               end_line: lineno,
-              end_char: char_pos + value.length - 1
+              end_char: char_pos + value.length - 1,
+              end_column: current_column + value.length - 1
             )
         )
 
@@ -1202,7 +1268,7 @@ module SyntaxTree
       node =
         EmbExprBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1215,7 +1281,7 @@ module SyntaxTree
       node =
         EmbExprEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1228,7 +1294,7 @@ module SyntaxTree
       node =
         EmbVar.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1243,9 +1309,12 @@ module SyntaxTree
       # We don't want to consume the :@kw event, because that would break
       # def..ensure..end chains.
       ending = find_token(Kw, "end", consume: false)
+      start_char = find_next_statement_start(keyword.location.end_char)
       statements.bind(
-        find_next_statement_start(keyword.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[keyword.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       Ensure.new(
@@ -1292,7 +1361,7 @@ module SyntaxTree
     def on_float(value)
       FloatLiteral.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1337,7 +1406,9 @@ module SyntaxTree
 
       statements.bind(
         (keyword || collection).location.end_char,
-        ending.location.start_char
+        (keyword || collection).location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       if index.is_a?(MLHS)
@@ -1358,7 +1429,7 @@ module SyntaxTree
     def on_gvar(value)
       GVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1379,7 +1450,7 @@ module SyntaxTree
     #   on_heredoc_beg: (String value) -> HeredocBeg
     def on_heredoc_beg(value)
       location =
-        Location.token(line: lineno, char: char_pos, size: value.size + 1)
+        Location.token(line: lineno, char: char_pos, column: current_column, size: value.size + 1)
 
       # Here we're going to artificially create an extra node type so that if
       # there are comments after the declaration of a heredoc, they get printed.
@@ -1415,8 +1486,10 @@ module SyntaxTree
           Location.new(
             start_line: heredoc.location.start_line,
             start_char: heredoc.location.start_char,
+            start_column: heredoc.location.start_column,
             end_line: lineno,
-            end_char: char_pos
+            end_char: char_pos,
+            end_column: current_column,
           )
       )
     end
@@ -1428,13 +1501,19 @@ module SyntaxTree
     #     (nil | VarField) keyword_rest
     #   ) -> HshPtn
     def on_hshptn(constant, keywords, keyword_rest)
-      parts = [constant, keywords, keyword_rest].flatten(2).compact
+      parts = [constant, *keywords&.flatten(1), keyword_rest].compact
+      location =
+        if parts.empty?
+          find_token(LBrace).location.to(find_token(RBrace).location)
+        else
+          parts[0].location.to(parts[-1].location)
+        end
 
       HshPtn.new(
         constant: constant,
-        keywords: keywords,
+        keywords: keywords || [],
         keyword_rest: keyword_rest,
-        location: parts[0].location.to(parts[-1].location)
+        location: location
       )
     end
 
@@ -1443,7 +1522,7 @@ module SyntaxTree
     def on_ident(value)
       Ident.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1457,7 +1536,12 @@ module SyntaxTree
       beginning = find_token(Kw, "if")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       If.new(
         predicate: predicate,
@@ -1503,7 +1587,7 @@ module SyntaxTree
     def on_imaginary(value)
       Imaginary.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1527,9 +1611,12 @@ module SyntaxTree
         statements_start = token
       end
 
+      start_char = find_next_statement_start(statements_start.location.end_char)
       statements.bind(
-        find_next_statement_start(statements_start.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[statements_start.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       In.new(
@@ -1545,7 +1632,7 @@ module SyntaxTree
     def on_int(value)
       Int.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1554,7 +1641,7 @@ module SyntaxTree
     def on_ivar(value)
       IVar.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1564,7 +1651,7 @@ module SyntaxTree
       node =
         Kw.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1585,7 +1672,7 @@ module SyntaxTree
     def on_label(value)
       Label.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -1595,7 +1682,7 @@ module SyntaxTree
       node =
         LabelEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1621,7 +1708,12 @@ module SyntaxTree
         closing = find_token(Kw, "end")
       end
 
-      statements.bind(opening.location.end_char, closing.location.start_char)
+      statements.bind(
+        opening.location.end_char,
+        opening.location.end_column,
+        closing.location.start_char,
+        closing.location.start_column
+      )
 
       Lambda.new(
         params: params,
@@ -1636,7 +1728,7 @@ module SyntaxTree
       node =
         LBrace.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1649,7 +1741,7 @@ module SyntaxTree
       node =
         LBracket.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1662,7 +1754,7 @@ module SyntaxTree
       node =
         LParen.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1761,7 +1853,7 @@ module SyntaxTree
     # :call-seq:
     #   on_mlhs_new: () -> MLHS
     def on_mlhs_new
-      MLHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      MLHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -1787,10 +1879,13 @@ module SyntaxTree
     def on_module(constant, bodystmt)
       beginning = find_token(Kw, "module")
       ending = find_token(Kw, "end")
+      start_char = find_next_statement_start(constant.location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start(constant.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[constant.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       ModuleDeclaration.new(
@@ -1803,7 +1898,7 @@ module SyntaxTree
     # :call-seq:
     #   on_mrhs_new: () -> MRHS
     def on_mrhs_new
-      MRHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      MRHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -1872,7 +1967,7 @@ module SyntaxTree
       node =
         Op.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -1931,7 +2026,7 @@ module SyntaxTree
         if parts.any?
           parts[0].location.to(parts[-1].location)
         else
-          Location.fixed(line: lineno, char: char_pos)
+          Location.fixed(line: lineno, char: char_pos, column: current_column)
         end
 
       Params.new(
@@ -1954,12 +2049,15 @@ module SyntaxTree
 
       if contents && contents.is_a?(Params)
         location = contents.location
+        start_char = find_next_statement_start(lparen.location.end_char)
         location =
           Location.new(
             start_line: location.start_line,
-            start_char: find_next_statement_start(lparen.location.end_char),
+            start_char: start_char,
+            start_column: start_char - line_counts[lparen.location.start_line - 1].start,
             end_line: location.end_line,
-            end_char: rparen.location.start_char
+            end_char: rparen.location.start_char,
+            end_column: rparen.location.start_column
           )
 
         contents =
@@ -1997,23 +2095,26 @@ module SyntaxTree
     def on_period(value)
       Period.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
     # :call-seq:
     #   on_program: (Statements statements) -> Program
     def on_program(statements)
+      last_column = source.length - line_counts[lines.length - 1].start
       location =
         Location.new(
           start_line: 1,
           start_char: 0,
+          start_column: 0,
           end_line: lines.length,
-          end_char: source.length
+          end_char: source.length,
+          end_column: last_column
         )
 
       statements.body << @__end__ if @__end__
-      statements.bind(0, source.length)
+      statements.bind(0, 0, source.length, last_column)
 
       program = Program.new(statements: statements, location: location)
       attach_comments(program, @comments)
@@ -2126,7 +2227,7 @@ module SyntaxTree
       node =
         QSymbolsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2161,7 +2262,7 @@ module SyntaxTree
       node =
         QWordsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2181,7 +2282,7 @@ module SyntaxTree
     def on_rational(value)
       RationalLiteral.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2191,7 +2292,7 @@ module SyntaxTree
       node =
         RBrace.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2204,7 +2305,7 @@ module SyntaxTree
       node =
         RBracket.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2238,7 +2339,7 @@ module SyntaxTree
       node =
         RegexpBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2250,7 +2351,7 @@ module SyntaxTree
     def on_regexp_end(value)
       RegexpEnd.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2292,9 +2393,12 @@ module SyntaxTree
       exceptions = exceptions[0] if exceptions.is_a?(Array)
 
       last_node = variable || exceptions || keyword
+      start_char = find_next_statement_start(last_node.location.end_char)
       statements.bind(
-        find_next_statement_start(last_node.location.end_char),
-        char_pos
+        start_char,
+        start_char - line_counts[last_node.location.start_line - 1].start,
+        char_pos,
+        current_column
       )
 
       # We add an additional inner node here that ripper doesn't provide so that
@@ -2309,13 +2413,16 @@ module SyntaxTree
               Location.new(
                 start_line: keyword.location.start_line,
                 start_char: keyword.location.end_char + 1,
+                start_column: keyword.location.end_column + 1,
                 end_line: last_node.location.end_line,
-                end_char: last_node.location.end_char
+                end_char: last_node.location.end_char,
+                end_column: last_node.location.end_column
               )
           )
         end
 
       Rescue.new(
+        keyword: keyword,
         exception: rescue_ex,
         statements: statements,
         consequent: consequent,
@@ -2323,8 +2430,10 @@ module SyntaxTree
           Location.new(
             start_line: keyword.location.start_line,
             start_char: keyword.location.start_char,
+            start_column: keyword.location.start_column,
             end_line: lineno,
-            end_char: char_pos
+            end_char: char_pos,
+            end_column: current_column
           )
       )
     end
@@ -2383,7 +2492,7 @@ module SyntaxTree
       node =
         RParen.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2395,10 +2504,13 @@ module SyntaxTree
     def on_sclass(target, bodystmt)
       beginning = find_token(Kw, "class")
       ending = find_token(Kw, "end")
+      start_char = find_next_statement_start(target.location.end_char)
 
       bodystmt.bind(
-        find_next_statement_start(target.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[target.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       SClass.new(
@@ -2437,7 +2549,7 @@ module SyntaxTree
       Statements.new(
         self,
         body: [],
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -2471,7 +2583,7 @@ module SyntaxTree
     def on_string_content
       StringContent.new(
         parts: [],
-        location: Location.fixed(line: lineno, char: char_pos)
+        location: Location.fixed(line: lineno, char: char_pos, column: current_column)
       )
     end
 
@@ -2494,18 +2606,22 @@ module SyntaxTree
 
       statements.bind(
         embexpr_beg.location.end_char,
-        embexpr_end.location.start_char
+        embexpr_beg.location.end_column,
+        embexpr_end.location.start_char,
+        embexpr_end.location.start_column
       )
 
       location =
         Location.new(
           start_line: embexpr_beg.location.start_line,
           start_char: embexpr_beg.location.start_char,
+          start_column: embexpr_beg.location.start_column,
           end_line: [
             embexpr_end.location.end_line,
             statements.location.end_line
           ].max,
-          end_char: embexpr_end.location.end_char
+          end_char: embexpr_end.location.end_char,
+          end_column: embexpr_end.location.end_column
         )
 
       StringEmbExpr.new(statements: statements, location: location)
@@ -2533,11 +2649,13 @@ module SyntaxTree
           Location.new(
             start_line: tstring_beg.location.start_line,
             start_char: tstring_beg.location.start_char,
+            start_column: tstring_beg.location.start_column,
             end_line: [
               tstring_end.location.end_line,
               string.location.end_line
             ].max,
-            end_char: tstring_end.location.end_char
+            end_char: tstring_end.location.end_char,
+            end_column: tstring_end.location.end_column
           )
 
         StringLiteral.new(
@@ -2566,7 +2684,7 @@ module SyntaxTree
       node =
         SymBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2620,7 +2738,7 @@ module SyntaxTree
       node =
         SymbolsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2645,7 +2763,7 @@ module SyntaxTree
       node =
         TLambda.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2658,7 +2776,7 @@ module SyntaxTree
       node =
         TLamBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2693,7 +2811,7 @@ module SyntaxTree
       node =
         TStringBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2705,7 +2823,7 @@ module SyntaxTree
     def on_tstring_content(value)
       TStringContent.new(
         value: value,
-        location: Location.token(line: lineno, char: char_pos, size: value.size)
+        location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
       )
     end
 
@@ -2715,7 +2833,7 @@ module SyntaxTree
       node =
         TStringEnd.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node
@@ -2731,19 +2849,17 @@ module SyntaxTree
         # parentheses they don't get reported as a paren node for some reason.
 
         beginning = find_token(Kw, "not")
-        ending = statement
-
-        range = beginning.location.end_char...statement.location.start_char
-        paren = source[range].include?("(")
+        ending = statement || beginning
+        parentheses = source[beginning.location.end_char] == "("
 
-        if paren
+        if parentheses
           find_token(LParen)
           ending = find_token(RParen)
         end
 
         Not.new(
           statement: statement,
-          parentheses: paren,
+          parentheses: parentheses,
           location: beginning.location.to(ending.location)
         )
       else
@@ -2789,7 +2905,12 @@ module SyntaxTree
       beginning = find_token(Kw, "unless")
       ending = consequent || find_token(Kw, "end")
 
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Unless.new(
         predicate: predicate,
@@ -2826,7 +2947,12 @@ module SyntaxTree
       end
 
       # Update the Statements location information
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       Until.new(
         predicate: predicate,
@@ -2865,12 +2991,12 @@ module SyntaxTree
     #   ) -> VarField
     def on_var_field(value)
       location =
-        if value
+        if value && value != :nil
           value.location
         else
           # You can hit this pattern if you're assigning to a splat using
           # pattern matching syntax in Ruby 2.7+
-          Location.fixed(line: lineno, char: char_pos)
+          Location.fixed(line: lineno, char: char_pos, column: current_column)
         end
 
       VarField.new(value: value, location: location)
@@ -2898,7 +3024,7 @@ module SyntaxTree
     # :call-seq:
     #   on_void_stmt: () -> VoidStmt
     def on_void_stmt
-      VoidStmt.new(location: Location.fixed(line: lineno, char: char_pos))
+      VoidStmt.new(location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -2917,9 +3043,13 @@ module SyntaxTree
         statements_start = token
       end
 
+      start_char = find_next_statement_start(statements_start.location.end_char)
+
       statements.bind(
-        find_next_statement_start(statements_start.location.end_char),
-        ending.location.start_char
+        start_char,
+        start_char - line_counts[statements_start.location.start_line - 1].start,
+        ending.location.start_char,
+        ending.location.start_column
       )
 
       When.new(
@@ -2945,7 +3075,12 @@ module SyntaxTree
       end
 
       # Update the Statements location information
-      statements.bind(predicate.location.end_char, ending.location.start_char)
+      statements.bind(
+        predicate.location.end_char,
+        predicate.location.end_column,
+        ending.location.start_char,
+        ending.location.start_column
+      )
 
       While.new(
         predicate: predicate,
@@ -2981,7 +3116,7 @@ module SyntaxTree
     # :call-seq:
     #   on_word_new: () -> Word
     def on_word_new
-      Word.new(parts: [], location: Location.fixed(line: lineno, char: char_pos))
+      Word.new(parts: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column))
     end
 
     # :call-seq:
@@ -3000,7 +3135,7 @@ module SyntaxTree
       node =
         WordsBeg.new(
           value: value,
-          location: Location.token(line: lineno, char: char_pos, size: value.size)
+          location: Location.token(line: lineno, char: char_pos, column: current_column, size: value.size)
         )
 
       tokens << node