syntax_tree 3.6.3 → 4.0.0

data/lib/syntax_tree/parser.rb

@@ -60,29 +60,46 @@ module SyntaxTree
     # This represents all of the tokens coming back from the lexer. It is
     # replacing a simple array because it keeps track of the last deleted token
     # from the list for better error messages.
-    class TokenList < SimpleDelegator
-      attr_reader :last_deleted
+    class TokenList
+      attr_reader :tokens, :last_deleted
 
-      def initialize(object)
-        super
+      def initialize
+        @tokens = []
         @last_deleted = nil
       end
 
+      def <<(token)
+        tokens << token
+      end
+
+      def [](index)
+        tokens[index]
+      end
+
+      def any?(&block)
+        tokens.any?(&block)
+      end
+
+      def reverse_each(&block)
+        tokens.reverse_each(&block)
+      end
+
+      def rindex(&block)
+        tokens.rindex(&block)
+      end
+
       def delete(value)
-        @last_deleted = super || @last_deleted
+        @last_deleted = tokens.delete(value) || @last_deleted
       end
 
       def delete_at(index)
-        @last_deleted = super
+        @last_deleted = tokens.delete_at(index)
       end
     end
 
     # [String] the source being parsed
     attr_reader :source
 
-    # [Array[ String ]] the list of lines in the source
-    attr_reader :lines
-
     # [Array[ SingleByteString | MultiByteString ]] the list of objects that
     # represent the start of each line in character offsets
     attr_reader :line_counts
@@ -105,12 +122,6 @@ module SyntaxTree
       # example.
       @source = source
 
-      # Similarly, we keep the lines of the source string around to be able to
-      # check if certain lines contain certain characters. For example, we'll
-      # use this to generate the content that goes after the __END__ keyword.
-      # Or we'll use this to check if a comment has other content on its line.
-      @lines = source.split(/\r?\n/)
-
       # This is the full set of comments that have been found by the parser.
       # It's a running list. At the end of every block of statements, they will
       # go in and attempt to grab any comments that are on their own line and
@@ -144,7 +155,7 @@ module SyntaxTree
       # Most of the time, when a parser event consumes one of these events, it
       # will be deleted from the list. So ideally, this list stays pretty short
       # over the course of parsing a source string.
-      @tokens = TokenList.new([])
+      @tokens = TokenList.new
 
       # Here we're going to build up a list of SingleByteString or
       # MultiByteString objects. They're each going to represent a string in the
@@ -153,7 +164,7 @@ module SyntaxTree
       @line_counts = []
       last_index = 0
 
-      @source.lines.each do |line|
+      @source.each_line do |line|
         @line_counts << if line.size == line.bytesize
           SingleByteString.new(last_index)
         else
@@ -233,28 +244,55 @@ module SyntaxTree
     # "module" (which would happen to be the innermost keyword). Then the outer
     # one would only be able to grab the first one. In this way all of the
     # tokens act as their own stack.
-    def find_token(type, value = :any, consume: true, location: nil)
-      index =
-        tokens.rindex do |token|
-          token.is_a?(type) && (value == :any || (token.value == value))
-        end
+    #
+    # If we're expecting to be able to find a token and consume it, but can't
+    # actually find it, then we need to raise an error. This is _usually_ caused
+    # by a syntax error in the source that we're printing. It could also be
+    # caused by accidentally attempting to consume a token twice by two
+    # different parser event handlers.
 
-      if consume
-        # If we're expecting to be able to find a token and consume it, but
-        # can't actually find it, then we need to raise an error. This is
-        # _usually_ caused by a syntax error in the source that we're printing.
-        # It could also be caused by accidentally attempting to consume a token
-        # twice by two different parser event handlers.
-        unless index
-          token = value == :any ? type.name.split("::", 2).last : value
-          message = "Cannot find expected #{token}"
-          raise ParseError.new(message, *find_token_error(location))
-        end
+    def find_token(type)
+      index = tokens.rindex { |token| token.is_a?(type) }
+      tokens[index] if index
+    end
 
-        tokens.delete_at(index)
-      elsif index
-        tokens[index]
-      end
+    def find_keyword(name)
+      index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) }
+      tokens[index] if index
+    end
+
+    def find_operator(name)
+      index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) }
+      tokens[index] if index
+    end
+
+    def consume_error(name, location)
+      message = "Cannot find expected #{name}"
+      raise ParseError.new(message, *find_token_error(location))
+    end
+
+    def consume_token(type)
+      index = tokens.rindex { |token| token.is_a?(type) }
+      consume_error(type.name.split("::", 2).last, nil) unless index
+      tokens.delete_at(index)
+    end
+
+    def consume_tstring_end(location)
+      index = tokens.rindex { |token| token.is_a?(TStringEnd) }
+      consume_error("string ending", location) unless index
+      tokens.delete_at(index)
+    end
+
+    def consume_keyword(name)
+      index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) }
+      consume_error(name, nil) unless index
+      tokens.delete_at(index)
+    end
+
+    def consume_operator(name)
+      index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) }
+      consume_error(name, nil) unless index
+      tokens.delete_at(index)
     end
 
     # A helper function to find a :: operator. We do special handling instead of
@@ -283,13 +321,18 @@ module SyntaxTree
     # By finding the next non-space character, we can make sure that the bounds
     # of the statement list are correct.
     def find_next_statement_start(position)
-      remaining = source[position..]
-
-      if remaining.sub(/\A +/, "")[0] == "#"
-        return position + remaining.index("\n")
+      maximum = source.length
+
+      position.upto(maximum) do |pound_index|
+        case source[pound_index]
+        when "#"
+          return source.index("\n", pound_index + 1) || maximum
+        when " "
+          # continue
+        else
+          return position
+        end
       end
-
-      position
     end
 
     # --------------------------------------------------------------------------
@@ -300,8 +343,8 @@ module SyntaxTree
     # :call-seq:
     #   on_BEGIN: (Statements statements) -> BEGINBlock
     def on_BEGIN(statements)
-      lbrace = find_token(LBrace)
-      rbrace = find_token(RBrace)
+      lbrace = consume_token(LBrace)
+      rbrace = consume_token(RBrace)
 
       start_char = find_next_statement_start(lbrace.location.end_char)
       statements.bind(
@@ -311,7 +354,7 @@ module SyntaxTree
         rbrace.location.start_column
       )
 
-      keyword = find_token(Kw, "BEGIN")
+      keyword = consume_keyword(:BEGIN)
 
       BEGINBlock.new(
         lbrace: lbrace,
@@ -338,8 +381,8 @@ module SyntaxTree
     # :call-seq:
     #   on_END: (Statements statements) -> ENDBlock
     def on_END(statements)
-      lbrace = find_token(LBrace)
-      rbrace = find_token(RBrace)
+      lbrace = consume_token(LBrace)
+      rbrace = consume_token(RBrace)
 
       start_char = find_next_statement_start(lbrace.location.end_char)
       statements.bind(
@@ -349,7 +392,7 @@ module SyntaxTree
         rbrace.location.start_column
       )
 
-      keyword = find_token(Kw, "END")
+      keyword = consume_keyword(:END)
 
       ENDBlock.new(
         lbrace: lbrace,
@@ -380,7 +423,7 @@ module SyntaxTree
     #     (DynaSymbol | SymbolLiteral) right
     #   ) -> Alias
     def on_alias(left, right)
-      keyword = find_token(Kw, "alias")
+      keyword = consume_keyword(:alias)
 
       Alias.new(
         left: left,
@@ -392,8 +435,8 @@ module SyntaxTree
     # :call-seq:
     #   on_aref: (untyped collection, (nil | Args) index) -> ARef
     def on_aref(collection, index)
-      find_token(LBracket)
-      rbracket = find_token(RBracket)
+      consume_token(LBracket)
+      rbracket = consume_token(RBracket)
 
       ARef.new(
         collection: collection,
@@ -408,8 +451,8 @@ module SyntaxTree
     #     (nil | Args) index
     #   ) -> ARefField
     def on_aref_field(collection, index)
-      find_token(LBracket)
-      rbracket = find_token(RBracket)
+      consume_token(LBracket)
+      rbracket = consume_token(RBracket)
 
       ARefField.new(
         collection: collection,
@@ -427,8 +470,8 @@ module SyntaxTree
     #     (nil | Args | ArgsForward) arguments
     #   ) -> ArgParen
     def on_arg_paren(arguments)
-      lparen = find_token(LParen)
-      rparen = find_token(RParen)
+      lparen = consume_token(LParen)
+      rparen = consume_token(RParen)
 
       # If the arguments exceed the ending of the parentheses, then we know we
       # have a heredoc in the arguments, and we need to use the bounds of the
@@ -470,23 +513,26 @@ module SyntaxTree
     #     (false | untyped) block
     #   ) -> Args
     def on_args_add_block(arguments, block)
+      end_char = arguments.parts.any? && arguments.location.end_char
+
       # First, see if there is an & operator that could potentially be
       # associated with the block part of this args_add_block. If there is not,
       # then just return the arguments.
-      operator = find_token(Op, "&", consume: false)
-      return arguments unless operator
-
-      # If there are any arguments and the operator we found from the list is
-      # not after them, then we're going to return the arguments as-is because
-      # we're looking at an & that occurs before the arguments are done.
-      if arguments.parts.any? &&
-           operator.location.start_char < arguments.location.end_char
-        return arguments
-      end
+      index =
+        tokens.rindex do |token|
+          # If there are any arguments and the operator we found from the list
+          # is not after them, then we're going to return the arguments as-is
+          # because we're looking at an & that occurs before the arguments are
+          # done.
+          return arguments if end_char && token.location.start_char < end_char
+          token.is_a?(Op) && (token.name == :&)
+        end
+
+      return arguments unless index
 
       # Now we know we have an & operator, so we're going to delete it from the
       # list of tokens to make sure it doesn't get confused with anything else.
-      tokens.delete(operator)
+      operator = tokens.delete_at(index)
 
       # Construct the location that represents the block argument.
       location = operator.location
@@ -505,7 +551,7 @@ module SyntaxTree
     # :call-seq:
     #   on_args_add_star: (Args arguments, untyped star) -> Args
     def on_args_add_star(arguments, argument)
-      beginning = find_token(Op, "*")
+      beginning = consume_operator(:*)
       ending = argument || beginning
 
       location =
@@ -527,7 +573,7 @@ module SyntaxTree
     # :call-seq:
     #   on_args_forward: () -> ArgsForward
     def on_args_forward
-      op = find_token(Op, "...")
+      op = consume_operator(:"...")
 
       ArgsForward.new(value: op.value, location: op.location)
     end
@@ -547,8 +593,8 @@ module SyntaxTree
     #     ArrayLiteral | QSymbols | QWords | Symbols | Words
     def on_array(contents)
       if !contents || contents.is_a?(Args)
-        lbracket = find_token(LBracket)
-        rbracket = find_token(RBracket)
+        lbracket = consume_token(LBracket)
+        rbracket = consume_token(RBracket)
 
         ArrayLiteral.new(
           lbracket: lbracket,
@@ -556,8 +602,7 @@ module SyntaxTree
           location: lbracket.location.to(rbracket.location)
         )
       else
-        tstring_end =
-          find_token(TStringEnd, location: contents.beginning.location)
+        tstring_end = consume_tstring_end(contents.beginning.location)
 
         contents.class.new(
           beginning: contents.beginning,
@@ -567,6 +612,56 @@ module SyntaxTree
       end
     end
 
+    # Ugh... I really do not like this class. Basically, ripper doesn't provide
+    # enough information about where pins are located in the tree. It only gives
+    # events for ^ ops and var_ref nodes. You have to piece it together
+    # yourself.
+    #
+    # Note that there are edge cases here that we straight up do not address,
+    # because I honestly think it's going to be faster to write a new parser
+    # than to address them. For example, this will not work properly:
+    #
+    #     foo in ^((bar = 0; bar; baz))
+    #
+    # If someone actually does something like that, we'll have to find another
+    # way to make this work.
+    class PinVisitor < Visitor
+      attr_reader :pins, :stack
+
+      def initialize(pins)
+        @pins = pins
+        @stack = []
+      end
+
+      def visit(node)
+        return if pins.empty?
+        stack << node
+        super
+        stack.pop
+      end
+
+      def visit_var_ref(node)
+        pins.shift
+        node.pin(stack[-2])
+      end
+
+      def self.visit(node, tokens)
+        start_char = node.location.start_char
+        allocated = []
+
+        tokens.reverse_each do |token|
+          char = token.location.start_char
+          break if char <= start_char
+
+          if token.is_a?(Op) && token.value == "^"
+            allocated.unshift(tokens.delete(token))
+          end
+        end
+
+        new(allocated).visit(node) if allocated.any?
+      end
+    end
+
     # :call-seq:
     #   on_aryptn: (
     #     (nil | VarRef) constant,
@@ -583,7 +678,7 @@ module SyntaxTree
       # of the various parts.
       location =
         if parts.empty?
-          find_token(LBracket).location.to(find_token(RBracket).location)
+          consume_token(LBracket).location.to(consume_token(RBracket).location)
         else
           parts[0].location.to(parts[-1].location)
         end
@@ -594,12 +689,13 @@ module SyntaxTree
       if rest.is_a?(VarField) && rest.value.nil?
         tokens.rindex do |rtoken|
           case rtoken
-          in Op[value: "*"]
-            rest = VarField.new(value: nil, location: rtoken.location)
+          when Comma
             break
-          in Comma
-            break
-          else
+          when Op
+            if rtoken.value == "*"
+              rest = VarField.new(value: nil, location: rtoken.location)
+              break
+            end
           end
         end
       end
@@ -644,7 +740,7 @@ module SyntaxTree
     # :call-seq:
     #   on_assoc_splat: (untyped value) -> AssocSplat
     def on_assoc_splat(value)
-      operator = find_token(Op, "**")
+      operator = consume_operator(:**)
 
       AssocSplat.new(
         value: value,
@@ -704,23 +800,23 @@ module SyntaxTree
     # :call-seq:
     #   on_begin: (untyped bodystmt) -> Begin | PinnedBegin
     def on_begin(bodystmt)
-      pin = find_token(Op, "^", consume: false)
+      pin = find_operator(:^)
 
       if pin && pin.location.start_char < bodystmt.location.start_char
         tokens.delete(pin)
-        find_token(LParen)
+        consume_token(LParen)
 
-        rparen = find_token(RParen)
+        rparen = consume_token(RParen)
         location = pin.location.to(rparen.location)
 
         PinnedBegin.new(statement: bodystmt, location: location)
       else
-        keyword = find_token(Kw, "begin")
+        keyword = consume_keyword(:begin)
         end_location =
           if bodystmt.else_clause
             bodystmt.location
           else
-            find_token(Kw, "end").location
+            consume_keyword(:end).location
           end
 
         bodystmt.bind(
@@ -746,13 +842,11 @@ module SyntaxTree
         # Here, we're going to search backward for the token that's between the
         # two operands that matches the operator so we can delete it from the
         # list.
+        range = (left.location.end_char + 1)...right.location.start_char
         index =
           tokens.rindex do |token|
-            location = token.location
-
-            token.is_a?(Op) && token.value == operator.to_s &&
-              location.start_char > left.location.end_char &&
-              location.end_char < right.location.start_char
+            token.is_a?(Op) && token.name == operator &&
+              range.cover?(token.location.start_char)
           end
 
         tokens.delete_at(index) if index
@@ -795,7 +889,7 @@ module SyntaxTree
     # :call-seq:
     #   on_blockarg: (Ident name) -> BlockArg
     def on_blockarg(name)
-      operator = find_token(Op, "&")
+      operator = consume_operator(:&)
 
       location = operator.location
       location = location.to(name.location) if name
@@ -814,7 +908,7 @@ module SyntaxTree
       BodyStmt.new(
         statements: statements,
         rescue_clause: rescue_clause,
-        else_keyword: else_clause && find_token(Kw, "else"),
+        else_keyword: else_clause && consume_keyword(:else),
         else_clause: else_clause,
         ensure_clause: ensure_clause,
         location:
@@ -828,8 +922,8 @@ module SyntaxTree
     #     Statements statements
     #   ) -> BraceBlock
     def on_brace_block(block_var, statements)
-      lbrace = find_token(LBrace)
-      rbrace = find_token(RBrace)
+      lbrace = consume_token(LBrace)
+      rbrace = consume_token(RBrace)
       location = (block_var || lbrace).location
 
       start_char = find_next_statement_start(location.end_char)
@@ -864,7 +958,7 @@ module SyntaxTree
     # :call-seq:
     #   on_break: (Args arguments) -> Break
     def on_break(arguments)
-      keyword = find_token(Kw, "break")
+      keyword = consume_keyword(:break)
 
       location = keyword.location
       location = location.to(arguments.location) if arguments.parts.any?
@@ -900,7 +994,7 @@ module SyntaxTree
     # :call-seq:
     #   on_case: (untyped value, untyped consequent) -> Case | RAssign
     def on_case(value, consequent)
-      if (keyword = find_token(Kw, "case", consume: false))
+      if (keyword = find_keyword(:case))
         tokens.delete(keyword)
 
         Case.new(
@@ -911,18 +1005,22 @@ module SyntaxTree
         )
       else
         operator =
-          if (keyword = find_token(Kw, "in", consume: false))
+          if (keyword = find_keyword(:in))
             tokens.delete(keyword)
           else
-            find_token(Op, "=>")
+            consume_operator(:"=>")
           end
 
-        RAssign.new(
-          value: value,
-          operator: operator,
-          pattern: consequent,
-          location: value.location.to(consequent.location)
-        )
+        node =
+          RAssign.new(
+            value: value,
+            operator: operator,
+            pattern: consequent,
+            location: value.location.to(consequent.location)
+          )
+
+        PinVisitor.visit(node, tokens)
+        node
       end
     end
 
@@ -933,8 +1031,8 @@ module SyntaxTree
     #     BodyStmt bodystmt
     #   ) -> ClassDeclaration
     def on_class(constant, superclass, bodystmt)
-      beginning = find_token(Kw, "class")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:class)
+      ending = consume_keyword(:end)
       location = (superclass || constant).location
       start_char = find_next_statement_start(location.end_char)
 
@@ -1004,20 +1102,20 @@ module SyntaxTree
     # :call-seq:
     #   on_comment: (String value) -> Comment
     def on_comment(value)
-      line = lineno
-      comment =
-        Comment.new(
-          value: value.chomp,
-          inline: value.strip != lines[line - 1].strip,
-          location:
-            Location.token(
-              line: line,
-              char: char_pos,
-              column: current_column,
-              size: value.size - 1
-            )
+      char = char_pos
+      location =
+        Location.token(
+          line: lineno,
+          char: char,
+          column: current_column,
+          size: value.size - 1
         )
 
+      index = source.rindex(/[^\t ]/, char - 1) if char != 0
+      inline = index && (source[index] != "\n")
+      comment =
+        Comment.new(value: value.chomp, inline: inline, location: location)
+
       @comments << comment
       comment
     end
@@ -1092,7 +1190,7 @@ module SyntaxTree
 
       # Find the beginning of the method definition, which works for single-line
       # and normal method definitions.
-      beginning = find_token(Kw, "def")
+      beginning = consume_keyword(:def)
 
       # If there aren't any params then we need to correct the params node
       # location information
@@ -1112,7 +1210,7 @@ module SyntaxTree
         params = Params.new(location: location)
       end
 
-      ending = find_token(Kw, "end", consume: false)
+      ending = find_keyword(:end)
 
       if ending
         tokens.delete(ending)
@@ -1150,13 +1248,13 @@ module SyntaxTree
     # :call-seq:
     #   on_defined: (untyped value) -> Defined
     def on_defined(value)
-      beginning = find_token(Kw, "defined?")
+      beginning = consume_keyword(:defined?)
       ending = value
 
       range = beginning.location.end_char...value.location.start_char
       if source[range].include?("(")
-        find_token(LParen)
-        ending = find_token(RParen)
+        consume_token(LParen)
+        ending = consume_token(RParen)
       end
 
       Defined.new(
@@ -1197,8 +1295,8 @@ module SyntaxTree
         params = Params.new(location: location)
       end
 
-      beginning = find_token(Kw, "def")
-      ending = find_token(Kw, "end", consume: false)
+      beginning = consume_keyword(:def)
+      ending = find_keyword(:end)
 
       if ending
         tokens.delete(ending)
@@ -1238,8 +1336,8 @@ module SyntaxTree
     # :call-seq:
     #   on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock
     def on_do_block(block_var, bodystmt)
-      beginning = find_token(Kw, "do")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:do)
+      ending = consume_keyword(:end)
       location = (block_var || beginning).location
       start_char = find_next_statement_start(location.end_char)
 
@@ -1261,7 +1359,7 @@ module SyntaxTree
     # :call-seq:
     #   on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2
     def on_dot2(left, right)
-      operator = find_token(Op, "..")
+      operator = consume_operator(:"..")
 
       beginning = left || operator
       ending = right || operator
@@ -1276,7 +1374,7 @@ module SyntaxTree
     # :call-seq:
     #   on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3
     def on_dot3(left, right)
-      operator = find_token(Op, "...")
+      operator = consume_operator(:"...")
 
       beginning = left || operator
       ending = right || operator
@@ -1291,10 +1389,10 @@ module SyntaxTree
     # :call-seq:
     #   on_dyna_symbol: (StringContent string_content) -> DynaSymbol
     def on_dyna_symbol(string_content)
-      if find_token(SymBeg, consume: false)
+      if (symbeg = find_token(SymBeg))
         # A normal dynamic symbol
-        symbeg = find_token(SymBeg)
-        tstring_end = find_token(TStringEnd, location: symbeg.location)
+        tokens.delete(symbeg)
+        tstring_end = consume_tstring_end(symbeg.location)
 
         DynaSymbol.new(
           quote: symbeg.value,
@@ -1303,8 +1401,8 @@ module SyntaxTree
         )
       else
         # A dynamic symbol as a hash key
-        tstring_beg = find_token(TStringBeg)
-        label_end = find_token(LabelEnd)
+        tstring_beg = consume_token(TStringBeg)
+        label_end = consume_token(LabelEnd)
 
         DynaSymbol.new(
           parts: string_content.parts,
@@ -1317,7 +1415,7 @@ module SyntaxTree
     # :call-seq:
     #   on_else: (Statements statements) -> Else
     def on_else(statements)
-      keyword = find_token(Kw, "else")
+      keyword = consume_keyword(:else)
 
       # else can either end with an end keyword (in which case we'll want to
       # consume that event) or it can end with an ensure keyword (in which case
@@ -1357,8 +1455,8 @@ module SyntaxTree
     #     (nil | Elsif | Else) consequent
     #   ) -> Elsif
     def on_elsif(predicate, statements, consequent)
-      beginning = find_token(Kw, "elsif")
-      ending = consequent || find_token(Kw, "end")
+      beginning = consume_keyword(:elsif)
+      ending = consequent || consume_keyword(:end)
 
       start_char = find_next_statement_start(predicate.location.end_char)
       statements.bind(
@@ -1478,11 +1576,11 @@ module SyntaxTree
     # :call-seq:
     #   on_ensure: (Statements statements) -> Ensure
     def on_ensure(statements)
-      keyword = find_token(Kw, "ensure")
+      keyword = consume_keyword(:ensure)
 
       # We don't want to consume the :@kw event, because that would break
       # def..ensure..end chains.
-      ending = find_token(Kw, "end", consume: false)
+      ending = find_keyword(:end)
       start_char = find_next_statement_start(keyword.location.end_char)
       statements.bind(
         start_char,
@@ -1504,7 +1602,7 @@ module SyntaxTree
     # :call-seq:
     #   on_excessed_comma: () -> ExcessedComma
     def on_excessed_comma(*)
-      comma = find_token(Comma)
+      comma = consume_token(Comma)
 
       ExcessedComma.new(value: comma.value, location: comma.location)
     end
@@ -1557,20 +1655,18 @@ module SyntaxTree
       # right left parenthesis, or the left splat. We're going to use this to
       # determine how to find the closing of the pattern, as well as determining
       # the location of the node.
-      opening =
-        find_token(LBracket, consume: false) ||
-          find_token(LParen, consume: false) || left
+      opening = find_token(LBracket) || find_token(LParen) || left
 
       # The closing is based on the opening, which is either the matched
       # punctuation or the right splat.
       closing =
         case opening
-        in LBracket
+        when LBracket
           tokens.delete(opening)
-          find_token(RBracket)
-        in LParen
+          consume_token(RBracket)
+        when LParen
           tokens.delete(opening)
-          find_token(RParen)
+          consume_token(RParen)
         else
           right
         end
@@ -1591,13 +1687,13 @@ module SyntaxTree
     #     Statements statements
     #   ) -> For
     def on_for(index, collection, statements)
-      beginning = find_token(Kw, "for")
-      in_keyword = find_token(Kw, "in")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:for)
+      in_keyword = consume_keyword(:in)
+      ending = consume_keyword(:end)
 
       # Consume the do keyword if it exists so that it doesn't get confused for
       # some other block
-      keyword = find_token(Kw, "do", consume: false)
+      keyword = find_keyword(:do)
       if keyword &&
            keyword.location.start_char > collection.location.end_char &&
            keyword.location.end_char < ending.location.start_char
@@ -1645,8 +1741,8 @@ module SyntaxTree
     # :call-seq:
     #   on_hash: ((nil | Array[AssocNew | AssocSplat]) assocs) -> HashLiteral
     def on_hash(assocs)
-      lbrace = find_token(LBrace)
-      rbrace = find_token(RBrace)
+      lbrace = consume_token(LBrace)
+      rbrace = consume_token(RBrace)
 
       HashLiteral.new(
         lbrace: lbrace,
@@ -1730,8 +1826,8 @@ module SyntaxTree
       if keyword_rest
         # We're doing this to delete the token from the list so that it doesn't
         # confuse future patterns by thinking they have an extra ** on the end.
-        find_token(Op, "**")
-      elsif (token = find_token(Op, "**", consume: false))
+        consume_operator(:**)
+      elsif (token = find_operator(:**))
         tokens.delete(token)
 
         # Create an artificial VarField if we find an extra ** on the end. This
@@ -1744,8 +1840,8 @@ module SyntaxTree
       # If there's no constant, there may be braces, so we're going to look for
       # those to get our bounds.
       unless constant
-        lbrace = find_token(LBrace, consume: false)
-        rbrace = find_token(RBrace, consume: false)
+        lbrace = find_token(LBrace)
+        rbrace = find_token(RBrace)
 
         if lbrace && rbrace
           parts = [lbrace, *parts, rbrace]
@@ -1784,8 +1880,8 @@ module SyntaxTree
     #     (nil | Elsif | Else) consequent
     #   ) -> If
     def on_if(predicate, statements, consequent)
-      beginning = find_token(Kw, "if")
-      ending = consequent || find_token(Kw, "end")
+      beginning = consume_keyword(:if)
+      ending = consequent || consume_keyword(:end)
 
       start_char = find_next_statement_start(predicate.location.end_char)
       statements.bind(
@@ -1817,7 +1913,7 @@ module SyntaxTree
     # :call-seq:
     #   on_if_mod: (untyped predicate, untyped statement) -> IfMod
     def on_if_mod(predicate, statement)
-      find_token(Kw, "if")
+      consume_keyword(:if)
 
       IfMod.new(
         statement: statement,
@@ -1860,11 +1956,11 @@ module SyntaxTree
       # Here we have a rightward assignment
       return pattern unless statements
 
-      beginning = find_token(Kw, "in")
-      ending = consequent || find_token(Kw, "end")
+      beginning = consume_keyword(:in)
+      ending = consequent || consume_keyword(:end)
 
       statements_start = pattern
-      if (token = find_token(Kw, "then", consume: false))
+      if (token = find_keyword(:then))
         tokens.delete(token)
         statements_start = token
       end
@@ -1878,12 +1974,16 @@ module SyntaxTree
         ending.location.start_column
       )
 
-      In.new(
-        pattern: pattern,
-        statements: statements,
-        consequent: consequent,
-        location: beginning.location.to(ending.location)
-      )
+      node =
+        In.new(
+          pattern: pattern,
+          statements: statements,
+          consequent: consequent,
+          location: beginning.location.to(ending.location)
+        )
+
+      PinVisitor.visit(node, tokens)
+      node
     end
 
     # :call-seq:
@@ -1938,7 +2038,7 @@ module SyntaxTree
     # :call-seq:
     #   on_kwrest_param: ((nil | Ident) name) -> KwRestParam
     def on_kwrest_param(name)
-      location = find_token(Op, "**").location
+      location = consume_operator(:**).location
       location = location.to(name.location) if name
 
       KwRestParam.new(name: name, location: location)
@@ -1984,7 +2084,7 @@ module SyntaxTree
     #     (BodyStmt | Statements) statements
     #   ) -> Lambda
     def on_lambda(params, statements)
-      beginning = find_token(TLambda)
+      beginning = consume_token(TLambda)
       braces =
         tokens.any? do |token|
           token.is_a?(TLamBeg) &&
@@ -1995,7 +2095,7 @@ module SyntaxTree
       # capturing lambda var until 3.2, we need to normalize all of that here.
       params =
         case params
-        in Paren[contents: Params]
+        when Paren
           # In this case we've gotten to the <3.2 parentheses wrapping a set of
           # parameters case. Here we need to manually scan for lambda locals.
           range = (params.location.start_char + 1)...params.location.end_char
@@ -2015,23 +2115,23 @@ module SyntaxTree
             location: params.location,
             comments: params.comments
           )
-        in Params
+        when Params
           # In this case we've gotten to the <3.2 plain set of parameters. In
           # this case there cannot be lambda locals, so we will wrap the
           # parameters into a lambda var that has no locals.
           LambdaVar.new(params: params, locals: [], location: params.location)
-        in LambdaVar
+        when LambdaVar
           # In this case we've gotten to 3.2+ lambda var. In this case we don't
           # need to do anything and can just the value as given.
           params
         end
 
       if braces
-        opening = find_token(TLamBeg)
-        closing = find_token(RBrace)
+        opening = consume_token(TLamBeg)
+        closing = consume_token(RBrace)
       else
-        opening = find_token(Kw, "do")
-        closing = find_token(Kw, "end")
+        opening = consume_keyword(:do)
+        closing = consume_keyword(:end)
       end
 
       start_char = find_next_statement_start(opening.location.end_char)
@@ -2262,7 +2362,7 @@ module SyntaxTree
     #     (nil | ARefField | Field | Ident | VarField) part
     #   ) -> MLHS
     def on_mlhs_add_star(mlhs, part)
-      beginning = find_token(Op, "*")
+      beginning = consume_operator(:*)
       ending = part || beginning
 
       location = beginning.location.to(ending.location)
@@ -2285,8 +2385,8 @@ module SyntaxTree
     # :call-seq:
     #   on_mlhs_paren: ((MLHS | MLHSParen) contents) -> MLHSParen
     def on_mlhs_paren(contents)
-      lparen = find_token(LParen)
-      rparen = find_token(RParen)
+      lparen = consume_token(LParen)
+      rparen = consume_token(RParen)
 
       comma_range = lparen.location.end_char...rparen.location.start_char
       contents.comma = true if source[comma_range].strip.end_with?(",")
@@ -2303,8 +2403,8 @@ module SyntaxTree
     #     BodyStmt bodystmt
     #   ) -> ModuleDeclaration
     def on_module(constant, bodystmt)
-      beginning = find_token(Kw, "module")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:module)
+      ending = consume_keyword(:end)
       start_char = find_next_statement_start(constant.location.end_char)
 
       bodystmt.bind(
@@ -2343,7 +2443,7 @@ module SyntaxTree
     # :call-seq:
     #   on_mrhs_add_star: (MRHS mrhs, untyped value) -> MRHS
     def on_mrhs_add_star(mrhs, value)
-      beginning = find_token(Op, "*")
+      beginning = consume_operator(:*)
       ending = value || beginning
 
       arg_star =
@@ -2371,7 +2471,7 @@ module SyntaxTree
     # :call-seq:
     #   on_next: (Args arguments) -> Next
     def on_next(arguments)
-      keyword = find_token(Kw, "next")
+      keyword = consume_keyword(:next)
 
       location = keyword.location
       location = location.to(arguments.location) if arguments.parts.any?
@@ -2486,8 +2586,8 @@ module SyntaxTree
     # :call-seq:
     #   on_paren: (untyped contents) -> Paren
     def on_paren(contents)
-      lparen = find_token(LParen)
-      rparen = find_token(RParen)
+      lparen = consume_token(LParen)
+      rparen = consume_token(RParen)
 
       if contents.is_a?(Params)
         location = contents.location
@@ -2551,13 +2651,13 @@ module SyntaxTree
     # :call-seq:
     #   on_program: (Statements statements) -> Program
     def on_program(statements)
-      last_column = source.length - line_counts[lines.length - 1].start
+      last_column = source.length - line_counts.last.start
       location =
         Location.new(
           start_line: 1,
           start_char: 0,
           start_column: 0,
-          end_line: lines.length,
+          end_line: line_counts.length - 1,
           end_char: source.length,
           end_column: last_column
         )
@@ -2692,7 +2792,7 @@ module SyntaxTree
     # :call-seq:
     #   on_qsymbols_new: () -> QSymbols
     def on_qsymbols_new
-      beginning = find_token(QSymbolsBeg)
+      beginning = consume_token(QSymbolsBeg)
 
       QSymbols.new(
         beginning: beginning,
@@ -2733,7 +2833,7 @@ module SyntaxTree
     # :call-seq:
     #   on_qwords_new: () -> QWords
     def on_qwords_new
-      beginning = find_token(QWordsBeg)
+      beginning = consume_token(QWordsBeg)
 
       QWords.new(
         beginning: beginning,
@@ -2798,7 +2898,7 @@ module SyntaxTree
     # :call-seq:
     #   on_redo: () -> Redo
     def on_redo
-      keyword = find_token(Kw, "redo")
+      keyword = consume_keyword(:redo)
 
       Redo.new(value: keyword.value, location: keyword.location)
     end
@@ -2874,7 +2974,7 @@ module SyntaxTree
     # :call-seq:
     #   on_regexp_new: () -> RegexpContent
     def on_regexp_new
-      regexp_beg = find_token(RegexpBeg)
+      regexp_beg = consume_token(RegexpBeg)
 
       RegexpContent.new(
         beginning: regexp_beg.value,
@@ -2891,7 +2991,7 @@ module SyntaxTree
     #     (nil | Rescue) consequent
     #   ) -> Rescue
     def on_rescue(exceptions, variable, statements, consequent)
-      keyword = find_token(Kw, "rescue")
+      keyword = consume_keyword(:rescue)
       exceptions = exceptions[0] if exceptions.is_a?(Array)
 
       last_node = variable || exceptions || keyword
@@ -2943,7 +3043,7 @@ module SyntaxTree
     # :call-seq:
     #   on_rescue_mod: (untyped statement, untyped value) -> RescueMod
     def on_rescue_mod(statement, value)
-      find_token(Kw, "rescue")
+      consume_keyword(:rescue)
 
       RescueMod.new(
         statement: statement,
@@ -2955,7 +3055,7 @@ module SyntaxTree
     # :call-seq:
     #   on_rest_param: ((nil | Ident) name) -> RestParam
     def on_rest_param(name)
-      location = find_token(Op, "*").location
+      location = consume_operator(:*).location
       location = location.to(name.location) if name
 
       RestParam.new(name: name, location: location)
@@ -2964,7 +3064,7 @@ module SyntaxTree
     # :call-seq:
     #   on_retry: () -> Retry
     def on_retry
-      keyword = find_token(Kw, "retry")
+      keyword = consume_keyword(:retry)
 
       Retry.new(value: keyword.value, location: keyword.location)
     end
@@ -2972,7 +3072,7 @@ module SyntaxTree
     # :call-seq:
     #   on_return: (Args arguments) -> Return
     def on_return(arguments)
-      keyword = find_token(Kw, "return")
+      keyword = consume_keyword(:return)
 
       Return.new(
         arguments: arguments,
@@ -2983,7 +3083,7 @@ module SyntaxTree
     # :call-seq:
     #   on_return0: () -> Return0
     def on_return0
-      keyword = find_token(Kw, "return")
+      keyword = consume_keyword(:return)
 
       Return0.new(value: keyword.value, location: keyword.location)
     end
@@ -3010,8 +3110,8 @@ module SyntaxTree
     # :call-seq:
     #   on_sclass: (untyped target, BodyStmt bodystmt) -> SClass
     def on_sclass(target, bodystmt)
-      beginning = find_token(Kw, "class")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:class)
+      ending = consume_keyword(:end)
       start_char = find_next_statement_start(target.location.end_char)
 
       bodystmt.bind(
@@ -3109,7 +3209,7 @@ module SyntaxTree
     # :call-seq:
     #   on_string_dvar: ((Backref | VarRef) variable) -> StringDVar
     def on_string_dvar(variable)
-      embvar = find_token(EmbVar)
+      embvar = consume_token(EmbVar)
 
       StringDVar.new(
         variable: variable,
@@ -3120,8 +3220,8 @@ module SyntaxTree
     # :call-seq:
     #   on_string_embexpr: (Statements statements) -> StringEmbExpr
     def on_string_embexpr(statements)
-      embexpr_beg = find_token(EmbExprBeg)
-      embexpr_end = find_token(EmbExprEnd)
+      embexpr_beg = consume_token(EmbExprBeg)
+      embexpr_end = consume_token(EmbExprEnd)
 
       statements.bind(
         embexpr_beg.location.end_char,
@@ -3162,8 +3262,8 @@ module SyntaxTree
           location: heredoc.location
         )
       else
-        tstring_beg = find_token(TStringBeg)
-        tstring_end = find_token(TStringEnd, location: tstring_beg.location)
+        tstring_beg = consume_token(TStringBeg)
+        tstring_end = consume_tstring_end(tstring_beg.location)
 
         location =
           Location.new(
@@ -3189,7 +3289,7 @@ module SyntaxTree
     # :call-seq:
     #   on_super: ((ArgParen | Args) arguments) -> Super
     def on_super(arguments)
-      keyword = find_token(Kw, "super")
+      keyword = consume_keyword(:super)
 
       Super.new(
         arguments: arguments,
@@ -3236,7 +3336,7 @@ module SyntaxTree
     #   ) -> SymbolLiteral
     def on_symbol_literal(value)
       if value.is_a?(SymbolContent)
-        symbeg = find_token(SymBeg)
+        symbeg = consume_token(SymBeg)
 
         SymbolLiteral.new(
           value: value.value,
@@ -3280,7 +3380,7 @@ module SyntaxTree
     # :call-seq:
     #   on_symbols_new: () -> Symbols
     def on_symbols_new
-      beginning = find_token(SymbolsBeg)
+      beginning = consume_token(SymbolsBeg)
 
       Symbols.new(
         beginning: beginning,
@@ -3410,13 +3510,13 @@ module SyntaxTree
         # We have somewhat special handling of the not operator since if it has
         # parentheses they don't get reported as a paren node for some reason.
 
-        beginning = find_token(Kw, "not")
+        beginning = consume_keyword(:not)
         ending = statement || beginning
         parentheses = source[beginning.location.end_char] == "("
 
         if parentheses
-          find_token(LParen)
-          ending = find_token(RParen)
+          consume_token(LParen)
+          ending = consume_token(RParen)
         end
 
         Not.new(
@@ -3449,7 +3549,7 @@ module SyntaxTree
     # :call-seq:
     #   on_undef: (Array[DynaSymbol | SymbolLiteral] symbols) -> Undef
     def on_undef(symbols)
-      keyword = find_token(Kw, "undef")
+      keyword = consume_keyword(:undef)
 
       Undef.new(
         symbols: symbols,
@@ -3464,8 +3564,8 @@ module SyntaxTree
     #     ((nil | Elsif | Else) consequent)
     #   ) -> Unless
     def on_unless(predicate, statements, consequent)
-      beginning = find_token(Kw, "unless")
-      ending = consequent || find_token(Kw, "end")
+      beginning = consume_keyword(:unless)
+      ending = consequent || consume_keyword(:end)
 
       start_char = find_next_statement_start(predicate.location.end_char)
       statements.bind(
@@ -3486,7 +3586,7 @@ module SyntaxTree
     # :call-seq:
     #   on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod
     def on_unless_mod(predicate, statement)
-      find_token(Kw, "unless")
+      consume_keyword(:unless)
 
       UnlessMod.new(
         statement: statement,
@@ -3498,12 +3598,12 @@ module SyntaxTree
     # :call-seq:
     #   on_until: (untyped predicate, Statements statements) -> Until
     def on_until(predicate, statements)
-      beginning = find_token(Kw, "until")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:until)
+      ending = consume_keyword(:end)
 
       # Consume the do keyword if it exists so that it doesn't get confused for
       # some other block
-      keyword = find_token(Kw, "do", consume: false)
+      keyword = find_keyword(:do)
       if keyword && keyword.location.start_char > predicate.location.end_char &&
            keyword.location.end_char < ending.location.start_char
         tokens.delete(keyword)
@@ -3528,7 +3628,7 @@ module SyntaxTree
     # :call-seq:
     #   on_until_mod: (untyped predicate, untyped statement) -> UntilMod
     def on_until_mod(predicate, statement)
-      find_token(Kw, "until")
+      consume_keyword(:until)
 
       UntilMod.new(
         statement: statement,
@@ -3540,7 +3640,7 @@ module SyntaxTree
     # :call-seq:
     #   on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias
     def on_var_alias(left, right)
-      keyword = find_token(Kw, "alias")
+      keyword = consume_keyword(:alias)
 
       VarAlias.new(
         left: left,
@@ -3569,17 +3669,7 @@ module SyntaxTree
     # :call-seq:
     #   on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef
     def on_var_ref(value)
-      pin = find_token(Op, "^", consume: false)
-
-      if pin && pin.location.start_char == value.location.start_char - 1
-        tokens.delete(pin)
-        PinnedVarRef.new(
-          value: value,
-          location: pin.location.to(value.location)
-        )
-      else
-        VarRef.new(value: value, location: value.location)
-      end
+      VarRef.new(value: value, location: value.location)
     end
 
     # :call-seq:
@@ -3604,11 +3694,11 @@ module SyntaxTree
     #     (nil | Else | When) consequent
     #   ) -> When
     def on_when(arguments, statements, consequent)
-      beginning = find_token(Kw, "when")
-      ending = consequent || find_token(Kw, "end")
+      beginning = consume_keyword(:when)
+      ending = consequent || consume_keyword(:end)
 
       statements_start = arguments
-      if (token = find_token(Kw, "then", consume: false))
+      if (token = find_keyword(:then))
         tokens.delete(token)
         statements_start = token
       end
@@ -3634,12 +3724,12 @@ module SyntaxTree
     # :call-seq:
     #   on_while: (untyped predicate, Statements statements) -> While
     def on_while(predicate, statements)
-      beginning = find_token(Kw, "while")
-      ending = find_token(Kw, "end")
+      beginning = consume_keyword(:while)
+      ending = consume_keyword(:end)
 
       # Consume the do keyword if it exists so that it doesn't get confused for
       # some other block
-      keyword = find_token(Kw, "do", consume: false)
+      keyword = find_keyword(:do)
       if keyword && keyword.location.start_char > predicate.location.end_char &&
            keyword.location.end_char < ending.location.start_char
         tokens.delete(keyword)
@@ -3664,7 +3754,7 @@ module SyntaxTree
     # :call-seq:
     #   on_while_mod: (untyped predicate, untyped statement) -> WhileMod
     def on_while_mod(predicate, statement)
-      find_token(Kw, "while")
+      consume_keyword(:while)
 
       WhileMod.new(
         statement: statement,
@@ -3727,7 +3817,7 @@ module SyntaxTree
     # :call-seq:
     #   on_words_new: () -> Words
     def on_words_new
-      beginning = find_token(WordsBeg)
+      beginning = consume_token(WordsBeg)
 
       Words.new(
         beginning: beginning,
@@ -3761,7 +3851,7 @@ module SyntaxTree
         if heredoc && heredoc.beginning.value.include?("`")
           heredoc.location
         else
-          find_token(Backtick).location
+          consume_token(Backtick).location
         end
 
       XString.new(parts: [], location: location)
@@ -3781,7 +3871,7 @@ module SyntaxTree
           location: heredoc.location
         )
       else
-        ending = find_token(TStringEnd, location: xstring.location)
+        ending = consume_tstring_end(xstring.location)
 
         XStringLiteral.new(
           parts: xstring.parts,
@@ -3793,7 +3883,7 @@ module SyntaxTree
     # :call-seq:
     #   on_yield: ((Args | Paren) arguments) -> Yield
     def on_yield(arguments)
-      keyword = find_token(Kw, "yield")
+      keyword = consume_keyword(:yield)
 
       Yield.new(
         arguments: arguments,
@@ -3804,7 +3894,7 @@ module SyntaxTree
     # :call-seq:
     #   on_yield0: () -> Yield0
     def on_yield0
-      keyword = find_token(Kw, "yield")
+      keyword = consume_keyword(:yield)
 
       Yield0.new(value: keyword.value, location: keyword.location)
     end
@@ -3812,7 +3902,7 @@ module SyntaxTree
     # :call-seq:
     #   on_zsuper: () -> ZSuper
     def on_zsuper
-      keyword = find_token(Kw, "super")
+      keyword = consume_keyword(:super)
 
       ZSuper.new(value: keyword.value, location: keyword.location)
     end