prism 1.8.1 → 2.0.0.rc1

data/lib/prism/lex_compat.rb

@@ -1,28 +1,64 @@
 # frozen_string_literal: true
 # :markup: markdown
-
-require "delegate"
+#--
+# rbs_inline: enabled
 
 module Prism
+  # @rbs!
+  #    module Translation
+  #      class Ripper
+  #        EXPR_NONE: Integer
+  #        EXPR_BEG: Integer
+  #        EXPR_MID: Integer
+  #        EXPR_END: Integer
+  #        EXPR_CLASS: Integer
+  #        EXPR_VALUE: Integer
+  #        EXPR_ARG: Integer
+  #        EXPR_CMDARG: Integer
+  #        EXPR_ENDARG: Integer
+  #        EXPR_ENDFN: Integer
+  #
+  #        class Lexer < Ripper
+  #          class State
+  #            def self.[]: (Integer value) -> State
+  #          end
+  #        end
+  #
+  #        class LineAndColumnCache
+  #          def initialize: (Source source) -> void
+  #
+  #          def line_and_column: (Integer byte_offset) -> [Integer, Integer]
+  #        end
+  #      end
+  #    end
+
   # This class is responsible for lexing the source using prism and then
   # converting those tokens to be compatible with Ripper. In the vast majority
   # of cases, this is a one-to-one mapping of the token type. Everything else
   # generally lines up. However, there are a few cases that require special
   # handling.
   class LexCompat # :nodoc:
+    # @rbs!
+    #    # A token produced by the Ripper lexer that Prism is replicating.
+    #    type lex_compat_token = [[Integer, Integer], Symbol, String, untyped]
+
     # A result class specialized for holding tokens produced by the lexer.
     class Result < Prism::Result
       # The list of tokens that were produced by the lexer.
-      attr_reader :value
+      attr_reader :value #: Array[lex_compat_token]
 
       # Create a new lex compat result object with the given values.
-      def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      #
+      #: (Array[lex_compat_token] value, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+      def initialize(value, magic_comments, data_loc, errors, warnings, continuable, source)
         @value = value
-        super(comments, magic_comments, data_loc, errors, warnings, source)
+        super(magic_comments, data_loc, errors, warnings, continuable, source)
       end
 
       # Implement the hash pattern matching interface for Result.
-      def deconstruct_keys(keys)
+      #
+      #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+      def deconstruct_keys(keys) # :nodoc:
         super.merge!(value: value)
       end
     end
@@ -42,6 +78,8 @@ module Prism
       BANG_EQUAL: :on_op,
       BANG_TILDE: :on_op,
       BRACE_LEFT: :on_lbrace,
+      BRACE_LEFT_ARGUMENT: :on_lbrace,
+      BRACE_LEFT_HASH: :on_lbrace,
       BRACE_RIGHT: :on_rbrace,
       BRACKET_LEFT: :on_lbracket,
       BRACKET_LEFT_ARRAY: :on_lbracket,
@@ -104,6 +142,8 @@ module Prism
       KEYWORD_DEF: :on_kw,
       KEYWORD_DEFINED: :on_kw,
       KEYWORD_DO: :on_kw,
+      KEYWORD_DO_BLOCK: :on_kw,
+      KEYWORD_DO_LAMBDA: :on_kw,
       KEYWORD_DO_LOOP: :on_kw,
       KEYWORD_ELSE: :on_kw,
       KEYWORD_ELSIF: :on_kw,
@@ -151,8 +191,10 @@ module Prism
       MINUS_EQUAL: :on_op,
       MINUS_GREATER: :on_tlambda,
       NEWLINE: :on_nl,
+      NEWLINE_TERMINATOR: :on_ignored_nl,
       NUMBERED_REFERENCE: :on_backref,
       PARENTHESIS_LEFT: :on_lparen,
+      PARENTHESIS_LEFT_GROUPING: :on_lparen,
       PARENTHESIS_LEFT_PARENTHESES: :on_lparen,
       PARENTHESIS_RIGHT: :on_rparen,
       PERCENT: :on_op,
@@ -195,96 +237,10 @@ module Prism
       USTAR: :on_op,
       USTAR_STAR: :on_op,
       WORDS_SEP: :on_words_sep,
+      XSTRING_BEGIN: :on_backtick,
       "__END__": :on___end__
     }.freeze
 
-    # When we produce tokens, we produce the same arrays that Ripper does.
-    # However, we add a couple of convenience methods onto them to make them a
-    # little easier to work with. We delegate all other methods to the array.
-    class Token < SimpleDelegator
-      # @dynamic initialize, each, []
-
-      # The location of the token in the source.
-      def location
-        self[0]
-      end
-
-      # The type of the token.
-      def event
-        self[1]
-      end
-
-      # The slice of the source that this token represents.
-      def value
-        self[2]
-      end
-
-      # The state of the lexer when this token was produced.
-      def state
-        self[3]
-      end
-    end
-
-    # Ripper doesn't include the rest of the token in the event, so we need to
-    # trim it down to just the content on the first line when comparing.
-    class EndContentToken < Token
-      def ==(other) # :nodoc:
-        [self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other
-      end
-    end
-
-    # Tokens where state should be ignored
-    # used for :on_comment, :on_heredoc_end, :on_embexpr_end
-    class IgnoreStateToken < Token
-      def ==(other) # :nodoc:
-        self[0...-1] == other[0...-1]
-      end
-    end
-
-    # Ident tokens for the most part are exactly the same, except sometimes we
-    # know an ident is a local when ripper doesn't (when they are introduced
-    # through named captures in regular expressions). In that case we don't
-    # compare the state.
-    class IdentToken < Token
-      def ==(other) # :nodoc:
-        (self[0...-1] == other[0...-1]) && (
-          (other[3] == Translation::Ripper::EXPR_LABEL | Translation::Ripper::EXPR_END) ||
-          (other[3] & (Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_CMDARG) != 0)
-        )
-      end
-    end
-
-    # Ignored newlines can occasionally have a LABEL state attached to them, so
-    # we compare the state differently here.
-    class IgnoredNewlineToken < Token
-      def ==(other) # :nodoc:
-        return false unless self[0...-1] == other[0...-1]
-
-        if self[3] == Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED
-          other[3] & Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED != 0
-        else
-          self[3] == other[3]
-        end
-      end
-    end
-
-    # If we have an identifier that follows a method name like:
-    #
-    #     def foo bar
-    #
-    # then Ripper will mark bar as END|LABEL if there is a local in a parent
-    # scope named bar because it hasn't pushed the local table yet. We do this
-    # more accurately, so we need to allow comparing against both END and
-    # END|LABEL.
-    class ParamToken < Token
-      def ==(other) # :nodoc:
-        (self[0...-1] == other[0...-1]) && (
-          (other[3] == Translation::Ripper::EXPR_END) ||
-          (other[3] == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL)
-        )
-      end
-    end
-
     # A heredoc in this case is a list of tokens that belong to the body of the
     # heredoc that should be appended onto the list of tokens when the heredoc
     # closes.
@@ -294,16 +250,19 @@ module Prism
       # order back into the token stream and set the state of the last token to
       # the state that the heredoc was opened in.
       class PlainHeredoc # :nodoc:
-        attr_reader :tokens
+        attr_reader :tokens #: Array[lex_compat_token]
 
+        #: () -> void
         def initialize
           @tokens = []
         end
 
+        #: (lex_compat_token token) -> void
         def <<(token)
           tokens << token
         end
 
+        #: () -> Array[lex_compat_token]
         def to_a
           tokens
         end
@@ -313,22 +272,26 @@ module Prism
       # that need to be split on "\\\n" to mimic Ripper's behavior. We also need
       # to keep track of the state that the heredoc was opened in.
       class DashHeredoc # :nodoc:
-        attr_reader :split, :tokens
+        attr_reader :split #: bool
+        attr_reader :tokens #: Array[lex_compat_token]
 
+        #: (bool split) -> void
         def initialize(split)
           @split = split
           @tokens = []
         end
 
+        #: (lex_compat_token token) -> void
         def <<(token)
           tokens << token
         end
 
+        #: () -> Array[lex_compat_token]
         def to_a
           embexpr_balance = 0
 
-          tokens.each_with_object([]) do |token, results| #$ Array[Token]
-            case token.event
+          tokens.each_with_object([]) do |token, results| #$ Array[lex_compat_token]
+            case token[1]
             when :on_embexpr_beg
               embexpr_balance += 1
               results << token
@@ -343,9 +306,9 @@ module Prism
                 if split
                   # Split on "\\\n" to mimic Ripper's behavior. Use a lookbehind
                   # to keep the delimiter in the result.
-                  token.value.split(/(?<=[^\\]\\\n)|(?<=[^\\]\\\r\n)/).each_with_index do |value, index|
+                  token[2].split(/(?<=[^\\]\\\n)|(?<=[^\\]\\\r\n)/).each_with_index do |value, index|
                     column = 0 if index > 0
-                    results << Token.new([[lineno, column], :on_tstring_content, value, token.state])
+                    results << [[lineno, column], :on_tstring_content, value, token[3]]
                     lineno += value.count("\n")
                   end
                 else
@@ -374,8 +337,13 @@ module Prism
       class DedentingHeredoc # :nodoc:
         TAB_WIDTH = 8
 
-        attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance
+        attr_reader :tokens #: Array[lex_compat_token]
+        attr_reader :dedent_next #: bool
+        attr_reader :dedent #: Integer?
+        attr_reader :embexpr_balance #: Integer
+        # @rbs @ended_on_newline: bool
 
+        #: () -> void
         def initialize
           @tokens = []
           @dedent_next = true
@@ -387,8 +355,10 @@ module Prism
         # As tokens are coming in, we track the minimum amount of common leading
         # whitespace on plain string content tokens. This allows us to later
         # remove that amount of whitespace from the beginning of each line.
+        #
+        #: (lex_compat_token token) -> void
         def <<(token)
-          case token.event
+          case token[1]
           when :on_embexpr_beg, :on_heredoc_beg
             @embexpr_balance += 1
             @dedent = 0 if @dedent_next && @ended_on_newline
@@ -396,10 +366,10 @@ module Prism
             @embexpr_balance -= 1
           when :on_tstring_content
             if embexpr_balance == 0
-              line = token.value
+              line = token[2]
 
               if dedent_next && !(line.strip.empty? && line.end_with?("\n"))
-                leading = line[/\A(\s*)\n?/, 1]
+                leading = line[/\A(\s*)\n?/, 1] #: String
                 next_dedent = 0
 
                 leading.each_char do |char|
@@ -419,20 +389,21 @@ module Prism
             end
           end
 
-          @dedent_next = token.event == :on_tstring_content && embexpr_balance == 0
+          @dedent_next = token[1] == :on_tstring_content && embexpr_balance == 0
           @ended_on_newline = false
           tokens << token
         end
 
+        #: () -> Array[lex_compat_token]
         def to_a
           # If every line in the heredoc is blank, we still need to split up the
           # string content token into multiple tokens.
           if dedent.nil?
-            results = [] #: Array[Token]
+            results = [] #: Array[lex_compat_token]
             embexpr_balance = 0
 
             tokens.each do |token|
-              case token.event
+              case token[1]
               when :on_embexpr_beg, :on_heredoc_beg
                 embexpr_balance += 1
                 results << token
@@ -444,9 +415,9 @@ module Prism
                   lineno = token[0][0]
                   column = token[0][1]
 
-                  token.value.split(/(?<=\n)/).each_with_index do |value, index|
+                  token[2].split(/(?<=\n)/).each_with_index do |value, index|
                     column = 0 if index > 0
-                    results << Token.new([[lineno, column], :on_tstring_content, value, token.state])
+                    results << [[lineno, column], :on_tstring_content, value, token[3]]
                     lineno += 1
                   end
                 else
@@ -463,7 +434,7 @@ module Prism
           # If the minimum common whitespace is 0, then we need to concatenate
           # string nodes together that are immediately adjacent.
           if dedent == 0
-            results = [] #: Array[Token]
+            results = [] #: Array[lex_compat_token]
             embexpr_balance = 0
 
             index = 0
@@ -474,15 +445,15 @@ module Prism
               results << token
               index += 1
 
-              case token.event
+              case token[1]
               when :on_embexpr_beg, :on_heredoc_beg
                 embexpr_balance += 1
               when :on_embexpr_end, :on_heredoc_end
                 embexpr_balance -= 1
               when :on_tstring_content
                 if embexpr_balance == 0
-                  while index < max_index && tokens[index].event == :on_tstring_content && !token.value.match?(/\\\r?\n\z/)
-                    token.value << tokens[index].value
+                  while index < max_index && tokens[index][1] == :on_tstring_content && !token[2].match?(/\\\r?\n\z/)
+                    token[2] << tokens[index][2]
                     index += 1
                   end
                 end
@@ -496,7 +467,7 @@ module Prism
           # insert on_ignored_sp tokens for the amount of dedent that we need to
           # perform. We also need to remove the dedent from the beginning of
           # each line of plain string content tokens.
-          results = [] #: Array[Token]
+          results = [] #: Array[lex_compat_token]
           dedent_next = true
           embexpr_balance = 0
 
@@ -505,7 +476,7 @@ module Prism
             # whitespace calculation we performed above. This is because
             # checking if the subsequent token needs to be dedented is common to
             # both the dedent calculation and the ignored_sp insertion.
-            case token.event
+            case token[1]
             when :on_embexpr_beg
               embexpr_balance += 1
               results << token
@@ -517,7 +488,7 @@ module Prism
                 # Here we're going to split the string on newlines, but maintain
                 # the newlines in the resulting array. We'll do that with a look
                 # behind assertion.
-                splits = token.value.split(/(?<=\n)/)
+                splits = token[2].split(/(?<=\n)/)
                 index = 0
 
                 while index < splits.length
@@ -535,7 +506,8 @@ module Prism
                   # line or this line doesn't start with whitespace, then we
                   # should concatenate the rest of the string to match ripper.
                   if dedent == 0 && (!dedent_next || !line.start_with?(/\s/))
-                    line = splits[index..].join
+                    unjoined = splits[index..] #: Array[String]
+                    line = unjoined.join
                     index = splits.length
                   end
 
@@ -574,12 +546,12 @@ module Prism
                       ignored = deleted_chars.join
                       line.delete_prefix!(ignored)
 
-                      results << Token.new([[lineno, 0], :on_ignored_sp, ignored, token[3]])
+                      results << [[lineno, 0], :on_ignored_sp, ignored, token[3]]
                       column = ignored.length
                     end
                   end
 
-                  results << Token.new([[lineno, column], token[1], line, token[3]]) unless line.empty?
+                  results << [[lineno, column], token[1], line, token[3]] unless line.empty?
                   index += 1
                 end
               else
@@ -590,7 +562,7 @@ module Prism
             end
 
             dedent_next =
-              ((token.event == :on_tstring_content) || (token.event == :on_heredoc_end)) &&
+              ((token[1] == :on_tstring_content) || (token[1] == :on_heredoc_end)) &&
               embexpr_balance == 0
           end
 
@@ -600,12 +572,14 @@ module Prism
 
       # Here we will split between the two types of heredocs and return the
       # object that will store their tokens.
+      #
+      #: (lex_compat_token opening) -> (PlainHeredoc | DashHeredoc | DedentingHeredoc)
       def self.build(opening)
-        case opening.value[2]
+        case opening[2][2]
         when "~"
           DedentingHeredoc.new
         when "-"
-          DashHeredoc.new(opening.value[3] != "'")
+          DashHeredoc.new(opening[2][3] != "'")
         else
           PlainHeredoc.new
         end
@@ -619,29 +593,74 @@ module Prism
     BOM_FLUSHED = RUBY_VERSION >= "3.3.0"
     private_constant :BOM_FLUSHED
 
-    attr_reader :source, :options
+    attr_reader :options #: Hash[Symbol, untyped]
+    # @rbs @source: String
 
+    #: (String source, **untyped options) -> void
     def initialize(source, **options)
       @source = source
       @options = options
     end
 
+    #: () -> Result
     def result
-      tokens = [] #: Array[LexCompat::Token]
+      tokens = [] #: Array[lex_compat_token]
 
       state = :default
       heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]]
 
-      result = Prism.lex(source, **options)
+      result = Prism.lex(@source, **options)
+      source = result.source
       result_value = result.value
-      previous_state = nil #: State?
+      previous_state = nil #: Translation::Ripper::Lexer::State?
       last_heredoc_end = nil #: Integer?
-
-      bom = source.byteslice(0..2) == "\xEF\xBB\xBF"
-
-      result_value.each_with_index do |(token, lex_state), index|
-        lineno = token.location.start_line
-        column = token.location.start_column
+      eof_token = nil #: Token?
+
+      # The two tokens Ripper scanned most recently, oldest last. Tokens that
+      # this loop does not emit are not part of Ripper's view, so they never
+      # become either one.
+      previous_token = nil #: Token?
+      previous_previous_token = nil #: Token?
+
+      # The lex state in effect at each embedded expression that is still open,
+      # innermost last, and the state at the opener of the one that closed most
+      # recently. Ending an embedded expression restores the state its opener
+      # had, which on_regexp_end below has to reproduce.
+      embexpr_states = [] #: Array[Integer]
+      last_embexpr_state = nil #: Integer?
+
+      bom = source.slice(0, 3) == "\xEF\xBB\xBF"
+
+      last_comment_token = nil #: lex_compat_token?
+      last_comment_end = nil #: Integer?
+
+      result_value.each_with_index do |prism_token, index|
+        lineno = prism_token.location.start_line
+        column = prism_token.location.start_column
+
+        # Ripper is a scanner, so every event it emits corresponds to source
+        # characters. Implicit word separators have none.
+        next if prism_token.type == :WORDS_SEP_IMPLICIT
+
+        event = RIPPER.fetch(prism_token.type)
+        value = prism_token.value
+        lex_state = Translation::Ripper::Lexer::State[prism_token._ripper_state]
+
+        # A comment token does not include its terminating newline, but
+        # ripper's comment value does, so the newline token that directly
+        # follows a comment is folded back into it.
+        if last_comment_token && last_comment_end == prism_token.location.start_offset && (event == :on_nl || event == :on_ignored_nl)
+          last_comment_token[2] += value
+          last_comment_token = nil
+          last_comment_end = nil
+
+          # The newline's characters are part of the comment Ripper emits, so
+          # it is still the token Ripper scanned most recently, and the trailing
+          # whitespace check at on_eof measures from its end.
+          previous_previous_token = previous_token
+          previous_token = prism_token
+          next
+        end
 
         # If there's a UTF-8 byte-order mark as the start of the file, then for
         # certain tokens ripper sets the first token back by 3 bytes. It also
@@ -653,68 +672,55 @@ module Prism
 
           if index == 0 && column == 0 && !BOM_FLUSHED
             flushed =
-              case token.type
+              case prism_token.type
               when :BACK_REFERENCE, :INSTANCE_VARIABLE, :CLASS_VARIABLE,
                   :GLOBAL_VARIABLE, :NUMBERED_REFERENCE, :PERCENT_LOWER_I,
                   :PERCENT_LOWER_X, :PERCENT_LOWER_W, :PERCENT_UPPER_I,
                   :PERCENT_UPPER_W, :STRING_BEGIN
                 true
               when :REGEXP_BEGIN, :SYMBOL_BEGIN
-                token.value.start_with?("%")
+                value.start_with?("%")
               else
                 false
               end
 
             unless flushed
               column -= 3
-              value = token.value
               value.prepend(String.new("\xEF\xBB\xBF", encoding: value.encoding))
             end
           end
         end
 
-        event = RIPPER.fetch(token.type)
-        value = token.value
-        lex_state = Translation::Ripper::Lexer::State.new(lex_state)
-
-        token =
+        lex_compat_token =
           case event
           when :on___end__
-            EndContentToken.new([[lineno, column], event, value, lex_state])
+            # Ripper doesn't include the rest of the token in the event, so we need to
+            # trim it down to just the content on the first line.
+            value = value[0..value.index("\n")] #: String
+            [[lineno, column], event, value, lex_state]
           when :on_comment
-            IgnoreStateToken.new([[lineno, column], event, value, lex_state])
+            [[lineno, column], event, value, lex_state]
           when :on_heredoc_end
             # Heredoc end tokens can be emitted in an odd order, so we don't
             # want to bother comparing the state on them.
-            last_heredoc_end = token.location.end_offset
-            IgnoreStateToken.new([[lineno, column], event, value, lex_state])
-          when :on_ident
-            if lex_state == Translation::Ripper::EXPR_END
-              # If we have an identifier that follows a method name like:
-              #
-              #     def foo bar
-              #
-              # then Ripper will mark bar as END|LABEL if there is a local in a
-              # parent scope named bar because it hasn't pushed the local table
-              # yet. We do this more accurately, so we need to allow comparing
-              # against both END and END|LABEL.
-              ParamToken.new([[lineno, column], event, value, lex_state])
-            elsif lex_state == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL
-              # In the event that we're comparing identifiers, we're going to
-              # allow a little divergence. Ripper doesn't account for local
-              # variables introduced through named captures in regexes, and we
-              # do, which accounts for this difference.
-              IdentToken.new([[lineno, column], event, value, lex_state])
-            else
-              Token.new([[lineno, column], event, value, lex_state])
-            end
+            last_heredoc_end = prism_token.location.end_offset
+            [[lineno, column], event, value, lex_state]
+          when :on_embexpr_beg
+            embexpr_states.push(prism_token._ripper_state)
+            [[lineno, column], event, value, lex_state]
           when :on_embexpr_end
-            IgnoreStateToken.new([[lineno, column], event, value, lex_state])
-          when :on_ignored_nl
-            # Ignored newlines can occasionally have a LABEL state attached to
-            # them which doesn't actually impact anything. We don't mirror that
-            # state so we ignored it.
-            IgnoredNewlineToken.new([[lineno, column], event, value, lex_state])
+            last_embexpr_state = embexpr_states.pop
+            [[lineno, column], event, value, lex_state]
+          when :on_words_sep
+            # Ripper emits one token each per line.
+            value.each_line.with_index do |line, index|
+              if index > 0
+                lineno += 1
+                column = 0
+              end
+              tokens << [[lineno, column], event, line, lex_state]
+            end
+            tokens.pop #: lex_compat_token
           when :on_regexp_end
             # On regex end, Ripper scans and then sets end state, so the ripper
             # lexed output is begin, when it should be end. prism sets lex state
@@ -722,62 +728,64 @@ module Prism
             # Ripper's lexed state. So here, if it's a regexp end token, we
             # output the state as the previous state, solely for the sake of
             # comparison.
-            previous_token = result_value[index - 1][0]
             lex_state =
-              if RIPPER.fetch(previous_token.type) == :on_embexpr_end
-                # If the previous token is embexpr_end, then we have to do even
-                # more processing. The end of an embedded expression sets the
-                # state to the state that it had at the beginning of the
-                # embedded expression. So we have to go and find that state and
-                # set it here.
-                counter = 1
-                current_index = index - 1
-
-                until counter == 0
-                  current_index -= 1
-                  current_event = RIPPER.fetch(result_value[current_index][0].type)
-                  counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
-                end
-
-                Translation::Ripper::Lexer::State.new(result_value[current_index][1])
+              if previous_token&.type == :EMBEXPR_END && last_embexpr_state
+                # An embedded expression that just closed restored the state its
+                # opener had, so that is the state Ripper reports here.
+                Translation::Ripper::Lexer::State[last_embexpr_state]
               else
                 previous_state
               end
 
-            Token.new([[lineno, column], event, value, lex_state])
+            [[lineno, column], event, value, lex_state]
           when :on_eof
-            previous_token = result_value[index - 1][0]
-
-            # If we're at the end of the file and the previous token was a
-            # comment and there is still whitespace after the comment, then
-            # Ripper will append a on_nl token (even though there isn't
-            # necessarily a newline). We mirror that here.
-            if previous_token.type == :COMMENT
-              # If the comment is at the start of a heredoc: <<HEREDOC # comment
-              # then the comment's end_offset is up near the heredoc_beg.
-              # This is not the correct offset to use for figuring out if
-              # there is trailing whitespace after the last token.
-              # Use the greater offset of the two to determine the start of
-              # the trailing whitespace.
-              start_offset = [previous_token.location.end_offset, last_heredoc_end].compact.max
-              end_offset = token.location.start_offset
-
-              if start_offset < end_offset
-                if bom
-                  start_offset += 3
-                  end_offset += 3
-                end
+            eof_token = prism_token
+
+            if (previous = previous_token)
+              # A newline that was folded back into a comment still marks the
+              # comment boundary for the check below.
+              before = previous_previous_token
+              comment_boundary = previous.type == :COMMENT ||
+                (before && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous.type) && before.type == :COMMENT && before.location.end_offset == previous.location.start_offset)
+
+              # If we're at the end of the file and the previous token was a
+              # comment and there is still whitespace after the comment, then
+              # Ripper will append a on_nl token (even though there isn't
+              # necessarily a newline). We mirror that here.
+              if comment_boundary
+                # If the comment is at the start of a heredoc: <<HEREDOC # comment
+                # then the comment's end_offset is up near the heredoc_beg.
+                # This is not the correct offset to use for figuring out if
+                # there is trailing whitespace after the last token.
+                # Use the greater offset of the two to determine the start of
+                # the trailing whitespace.
+                start_offset = [previous.location.end_offset, last_heredoc_end].compact.max
+                end_offset = prism_token.location.start_offset
+
+                if start_offset < end_offset
+                  if bom
+                    start_offset += 3
+                    end_offset += 3
+                  end
 
-                tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state])
+                  tokens << [[lineno, 0], :on_nl, source.slice(start_offset, end_offset - start_offset), lex_state]
+                end
               end
             end
 
-            Token.new([[lineno, column], event, value, lex_state])
+            [[lineno, column], event, value, lex_state]
           else
-            Token.new([[lineno, column], event, value, lex_state])
-          end
+            [[lineno, column], event, value, lex_state]
+          end #: lex_compat_token
 
         previous_state = lex_state
+        previous_previous_token = previous_token
+        previous_token = prism_token
+
+        if event == :on_comment
+          last_comment_token = lex_compat_token
+          last_comment_end = prism_token.location.end_offset
+        end
 
         # The order in which tokens appear in our lexer is different from the
         # order that they appear in Ripper. When we hit the declaration of a
@@ -792,19 +800,19 @@ module Prism
         when :default
           # The default state is when there are no heredocs at all. In this
           # state we can append the token to the list of tokens and move on.
-          tokens << token
+          tokens << lex_compat_token
 
           # If we get the declaration of a heredoc, then we open a new heredoc
           # and move into the heredoc_opened state.
           if event == :on_heredoc_beg
             state = :heredoc_opened
-            heredoc_stack.last << Heredoc.build(token)
+            heredoc_stack.last << Heredoc.build(lex_compat_token)
           end
         when :heredoc_opened
           # The heredoc_opened state is when we've seen the declaration of a
           # heredoc and are now lexing the body of the heredoc. In this state we
           # push tokens onto the most recently created heredoc.
-          heredoc_stack.last.last << token
+          heredoc_stack.last.last << lex_compat_token
 
           case event
           when :on_heredoc_beg
@@ -812,7 +820,7 @@ module Prism
             # heredoc, this means we have nested heredocs. In this case we'll
             # push a new heredoc onto the stack and stay in the heredoc_opened
             # state since we're now lexing the body of the new heredoc.
-            heredoc_stack << [Heredoc.build(token)]
+            heredoc_stack << [Heredoc.build(lex_compat_token)]
           when :on_heredoc_end
             # If we receive the end of a heredoc, then we're done lexing the
             # body of the heredoc. In this case we now have a completed heredoc
@@ -821,10 +829,10 @@ module Prism
             state = :heredoc_closed
           end
         when :heredoc_closed
-          if %i[on_nl on_ignored_nl on_comment].include?(event) || (event == :on_tstring_content && value.end_with?("\n"))
+          if %i[on_nl on_ignored_nl on_comment].include?(event) || ((event == :on_tstring_content) && value.end_with?("\n"))
             if heredoc_stack.size > 1
-              flushing = heredoc_stack.pop
-              heredoc_stack.last.last << token
+              flushing = heredoc_stack.pop #: Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]
+              heredoc_stack.last.last << lex_compat_token
 
               flushing.each do |heredoc|
                 heredoc.to_a.each do |flushed_token|
@@ -836,12 +844,12 @@ module Prism
               next
             end
           elsif event == :on_heredoc_beg
-            tokens << token
+            tokens << lex_compat_token
             state = :heredoc_opened
-            heredoc_stack.last << Heredoc.build(token)
+            heredoc_stack.last << Heredoc.build(lex_compat_token)
             next
           elsif heredoc_stack.size > 1
-            heredoc_stack[-2].last << token
+            heredoc_stack[-2].last << lex_compat_token
             next
           end
 
@@ -852,17 +860,92 @@ module Prism
           heredoc_stack.last.clear
           state = :default
 
-          tokens << token
+          tokens << lex_compat_token
         end
       end
 
-      # Drop the EOF token from the list
-      tokens = tokens[0...-1]
+      # Drop the EOF token from the list. The EOF token may not be
+      # present if the source was syntax invalid
+      if tokens.dig(-1, 1) == :on_eof
+        tokens = tokens[0...-1] #: Array[lex_compat_token]
+      end
+
+      # We sort by location because Ripper.lex sorts.
+      tokens.sort_by! do |token|
+        line, column = token[0]
+        source.byte_offset(line, column)
+      end
+
+      tokens = post_process_tokens(tokens, source, result.data_loc, bom, eof_token)
 
-      # We sort by location to compare against Ripper's output
-      tokens.sort_by!(&:location)
+      Result.new(tokens, result.magic_comments, result.data_loc, result.errors, result.warnings, result.continuable?, source)
+    end
+
+    private
+
+    #: (Array[lex_compat_token] tokens, Source source, Location? data_loc, bool bom, Token? eof_token) -> Array[lex_compat_token]
+    def post_process_tokens(tokens, source, data_loc, bom, eof_token)
+      new_tokens = [] #: Array[lex_compat_token]
+
+      prev_token_state = Translation::Ripper::Lexer::State[Translation::Ripper::EXPR_BEG]
+      prev_token_end = bom ? 3 : 0
+
+      cache = Translation::Ripper::LineAndColumnCache.new(source)
+
+      tokens.each do |token|
+        # Skip missing heredoc ends.
+        next if token[1] == :on_heredoc_end && token[2] == ""
+
+        # Add :on_sp tokens.
+        line, column = token[0]
+        start_offset = source.byte_offset(line, column)
+
+        # Ripper reports columns on line 1 without counting the BOM, so we
+        # adjust to get the real offset
+        start_offset += 3 if line == 1 && bom
+
+        if start_offset > prev_token_end
+          sp_value = source.slice(prev_token_end, start_offset - prev_token_end)
+          sp_line, sp_column = cache.line_and_column(prev_token_end)
+          # Ripper reports columns on line 1 without counting the BOM
+          sp_column -= 3 if sp_line == 1 && bom
+          continuation_index = sp_value.byteindex("\\")
+
+          # ripper emits up to three :on_sp tokens when line continuations are used
+          if continuation_index
+            next_whitespace_index = continuation_index + 1
+            next_whitespace_index += 1 if sp_value.byteslice(next_whitespace_index) == "\r"
+            next_whitespace_index += 1
+            first_whitespace = sp_value[0...continuation_index] #: String
+            continuation = sp_value[continuation_index...next_whitespace_index] #: String
+            second_whitespace = sp_value[next_whitespace_index..] || ""
+
+            new_tokens << [[sp_line, sp_column], :on_sp, first_whitespace, prev_token_state] unless first_whitespace.empty?
+            new_tokens << [[sp_line, sp_column + continuation_index], :on_sp, continuation, prev_token_state]
+            new_tokens << [[sp_line + 1, 0], :on_sp, second_whitespace, prev_token_state] unless second_whitespace.empty?
+          else
+            new_tokens << [[sp_line, sp_column], :on_sp, sp_value, prev_token_state]
+          end
+        end
+
+        new_tokens << token
+        prev_token_state = token[3]
+        prev_token_end = start_offset + token[2].bytesize
+      end
+
+      if !data_loc && eof_token # no trailing :on_sp with __END__ as it is always preceded by :on_nl
+        end_offset = eof_token.location.end_offset
+        if prev_token_end < end_offset
+          new_tokens << [
+            [source.line(prev_token_end), source.column(prev_token_end)],
+            :on_sp,
+            source.slice(prev_token_end, end_offset - prev_token_end),
+            prev_token_state
+          ]
+        end
+      end
 
-      Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source))
+      new_tokens
     end
   end