prism 1.8.1 → 2.0.0.rc1

data/lib/prism/parse_result.rb

@@ -1,7 +1,16 @@
 # frozen_string_literal: true
 # :markup: markdown
+#--
+# rbs_inline: enabled
 
 module Prism
+  # @rbs!
+  #    # An internal interface for a cache that can be used to compute code
+  #    # units from byte offsets.
+  #    interface _CodeUnitsCache
+  #      def []: (Integer byte_offset) -> Integer
+  #    end
+
   # This represents a source of Ruby code that has been parsed. It is used in
   # conjunction with locations to allow them to resolve line numbers and source
   # ranges.
@@ -10,14 +19,30 @@ module Prism
     # be used instead of `new` and it will return either a `Source` or a
     # specialized and more performant `ASCIISource` if no multibyte characters
     # are present in the source code.
-    def self.for(source, start_line = 1, offsets = [])
+    #
+    # Note that if you are calling this method manually, you will need to supply
+    # the start_line and offsets parameters. start_line is the line number that
+    # the source starts on, which is typically 1 but can be different if this
+    # source is a subset of a larger source or if this is an eval. offsets is an
+    # array of byte offsets for the start of each line in the source code, which
+    # can be calculated by iterating through the source code and recording the
+    # byte offset whenever a newline character is encountered.  The first
+    # element is always 0 to mark the first line.
+    #
+    # The comments, comment_attachments, and comment_ranges parameters are the
+    # lists described by the attributes of the same names. They are passed in
+    # empty when the source is created by the parser and filled in afterward,
+    # because the comments themselves refer to the source.
+    #
+    #: (String source, Integer start_line, Array[Integer] offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges) -> Source
+    def self.for(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [])
       if source.ascii_only?
-        ASCIISource.new(source, start_line, offsets)
+        ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
       elsif source.encoding == Encoding::BINARY
         source.force_encoding(Encoding::UTF_8)
 
         if source.valid_encoding?
-          new(source, start_line, offsets)
+          new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
         else
           # This is an extremely niche use case where the file is marked as
           # binary, contains multi-byte characters, and those characters are not
@@ -26,85 +51,162 @@ module Prism
           # problems when asking for code units, but it appears to be the
           # cleanest solution at the moment.
           source.force_encoding(Encoding::BINARY)
-          ASCIISource.new(source, start_line, offsets)
+          ASCIISource.new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
         end
       else
-        new(source, start_line, offsets)
+        new(source, start_line, offsets, comments, comment_attachments, comment_ranges)
       end
     end
 
     # The source code that this source object represents.
-    attr_reader :source
+    attr_reader :source #: String
 
     # The line number where this source starts.
-    attr_reader :start_line
-
-    # The list of newline byte offsets in the source code.
-    attr_reader :offsets
+    attr_reader :start_line #: Integer
+
+    # The list of comments that were encountered during parsing. When the source
+    # is parsed with the attach_comments option, these are sorted by start
+    # offset.
+    attr_reader :comments #: Array[Comment]
+
+    # The comments attached to nodes, grouped by node in order of node id. This
+    # is empty unless the source was parsed with the attach_comments option.
+    attr_reader :comment_attachments #: Array[Comment]
+
+    # Indices into comment_attachments, indexed by node id. For each node that
+    # owns comments, the index at which its comments begin is stored at its node
+    # id and the index at which they end is stored at the following node id. The
+    # other entries are nil. This is empty unless the source was parsed with the
+    # attach_comments option.
+    attr_reader :comment_ranges #: Array[Integer?]
+
+    # The list of newline byte offsets in the source code. When initialized from
+    # the C extension, this may be a packed binary string of uint32_t values
+    # that is lazily unpacked on first access.
+    #
+    #: () -> Array[Integer]
+    def offsets
+      offsets = @offsets
+      return offsets if offsets.is_a?(Array)
+      @offsets = offsets.unpack("L*")
+    end
 
-    # Create a new source object with the given source code.
-    def initialize(source, start_line = 1, offsets = [])
+    # Create a new source object with the given source code. The offsets
+    # parameter can be either an Array of Integer byte offsets or a packed
+    # binary string of uint32_t values (from the C extension).
+    #
+    #: (String source, Integer start_line, Array[Integer] | String offsets, ?Array[Comment] comments, ?Array[Comment] comment_attachments, ?Array[Integer?] comment_ranges) -> void
+    def initialize(source, start_line, offsets, comments = [], comment_attachments = [], comment_ranges = [])
       @source = source
-      @start_line = start_line # set after parsing is done
-      @offsets = offsets # set after parsing is done
+      @start_line = start_line
+      @offsets = offsets
+      @comments = comments
+      @comment_attachments = comment_attachments
+      @comment_ranges = comment_ranges
     end
 
     # Replace the value of start_line with the given value.
+    #
+    #: (Integer start_line) -> void
     def replace_start_line(start_line)
       @start_line = start_line
     end
 
     # Replace the value of offsets with the given value.
+    #
+    #: (Array[Integer] offsets) -> void
     def replace_offsets(offsets)
-      @offsets.replace(offsets)
+      @offsets = offsets
+    end
+
+    # The comments attached to the node with the given id, ordered by start
+    # offset. Comments are only attached when the source is parsed with the
+    # attach_comments option.
+    #
+    #: (Integer node_id) -> Array[Comment]
+    def comments_for(node_id)
+      start = comment_ranges[node_id]
+      finish = comment_ranges[node_id + 1]
+      return [] if start.nil? || finish.nil?
+
+      comment_attachments[start...finish] || []
     end
 
     # Returns the encoding of the source code, which is set by parameters to the
     # parser or by the encoding magic comment.
+    #
+    #: () -> Encoding
     def encoding
       source.encoding
     end
 
     # Returns the lines of the source code as an array of strings.
+    #
+    #: () -> Array[String]
     def lines
       source.lines
     end
 
     # Perform a byteslice on the source code using the given byte offset and
     # byte length.
+    #
+    #: (Integer byte_offset, Integer length) -> String
     def slice(byte_offset, length)
       source.byteslice(byte_offset, length) or raise
     end
 
+    # Converts the line number and column in bytes to a byte offset.
+    #
+    #: (Integer line, Integer column) -> Integer
+    def byte_offset(line, column)
+      normal = line - @start_line
+      raise IndexError if normal < 0
+      offsets.fetch(normal) + column
+    rescue IndexError
+      raise ArgumentError, "line #{line} is out of range"
+    end
+
     # Binary search through the offsets to find the line number for the given
     # byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def line(byte_offset)
       start_line + find_line(byte_offset)
     end
 
     # Return the byte offset of the start of the line corresponding to the given
     # byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def line_start(byte_offset)
       offsets[find_line(byte_offset)]
     end
 
     # Returns the byte offset of the end of the line corresponding to the given
     # byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def line_end(byte_offset)
       offsets[find_line(byte_offset) + 1] || source.bytesize
     end
 
-    # Return the column number for the given byte offset.
+    # Return the column in bytes for the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def column(byte_offset)
       byte_offset - line_start(byte_offset)
     end
 
     # Return the character offset for the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def character_offset(byte_offset)
       (source.byteslice(0, byte_offset) or raise).length
     end
 
-    # Return the column number in characters for the given byte offset.
+    # Return the column in characters for the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def character_column(byte_offset)
       character_offset(byte_offset) - character_offset(line_start(byte_offset))
     end
@@ -121,7 +223,11 @@ module Prism
     # possible that the given byte offset will not occur on a character
     # boundary. Second, it's possible that the source code will contain a
     # character that has no equivalent in the given encoding.
+    #
+    #: (Integer byte_offset, Encoding encoding) -> Integer
     def code_units_offset(byte_offset, encoding)
+      return byte_offset if encoding == Encoding::UTF_8
+
       byteslice = (source.byteslice(0, byte_offset) or raise).encode(encoding, invalid: :replace, undef: :replace)
 
       if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
@@ -133,28 +239,37 @@ module Prism
 
     # Generate a cache that targets a specific encoding for calculating code
     # unit offsets.
+    #
+    #: (Encoding encoding) -> CodeUnitsCache
     def code_units_cache(encoding)
       CodeUnitsCache.new(source, encoding)
     end
 
-    # Returns the column number in code units for the given encoding for the
+    # Returns the column in code units for the given encoding for the
     # given byte offset.
+    #
+    #: (Integer byte_offset, Encoding encoding) -> Integer
     def code_units_column(byte_offset, encoding)
       code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
     end
 
     # Freeze this object and the objects it contains.
+    #
+    #: () -> void
     def deep_freeze
       source.freeze
       offsets.freeze
+      comments.freeze
+      comment_attachments.freeze
+      comment_ranges.freeze
       freeze
     end
 
-    private
-
-    # Binary search through the offsets to find the line number for the given
+    # Binary search through the offsets to find the index for the given
     # byte offset.
-    def find_line(byte_offset)
+    #
+    #: (Integer byte_offset) -> Integer
+    def find_line(byte_offset) # :nodoc:
       index = offsets.bsearch_index { |offset| offset > byte_offset } || offsets.length
       index - 1
     end
@@ -175,38 +290,69 @@ module Prism
   #   has not yet been implemented.
   #
   class CodeUnitsCache
+    # Counter used for UTF-8, where one code unit equals one byte.
+    class UTF8Counter # :nodoc:
+      #: (Integer byte_offset, Integer byte_length) -> Integer
+      def count(byte_offset, byte_length)
+        byte_length
+      end
+    end
+
     class UTF16Counter # :nodoc:
+      # @rbs @source: String
+      # @rbs @encoding: Encoding
+
+      #: (String source, Encoding encoding) -> void
       def initialize(source, encoding)
         @source = source
         @encoding = encoding
       end
 
+      #: (Integer byte_offset, Integer byte_length) -> Integer
       def count(byte_offset, byte_length)
-        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).bytesize / 2
+        (@source.byteslice(byte_offset, byte_length) or raise).encode(@encoding, invalid: :replace, undef: :replace).bytesize / 2
       end
     end
 
-    class LengthCounter # :nodoc:
+    # Counter used for UTF-32, where one code unit equals one code point and
+    # matches String#length. Also used as a best-effort fallback for any other
+    # encoding that does not have a dedicated counter.
+    class UTF32Counter # :nodoc:
+      # @rbs @source: String
+      # @rbs @encoding: Encoding
+
+      #: (String source, Encoding encoding) -> void
       def initialize(source, encoding)
         @source = source
         @encoding = encoding
       end
 
+      #: (Integer byte_offset, Integer byte_length) -> Integer
       def count(byte_offset, byte_length)
-        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).length
+        (@source.byteslice(byte_offset, byte_length) or raise).encode(@encoding, invalid: :replace, undef: :replace).length
       end
     end
 
-    private_constant :UTF16Counter, :LengthCounter
+    private_constant :UTF8Counter, :UTF16Counter, :UTF32Counter
+
+    # @rbs @source: String
+    # @rbs @counter: UTF8Counter | UTF16Counter | UTF32Counter
+    # @rbs @cache: Hash[Integer, Integer]
+    # @rbs @offsets: Array[Integer]
 
     # Initialize a new cache with the given source and encoding.
+    #
+    #: (String source, Encoding encoding) -> void
     def initialize(source, encoding)
       @source = source
       @counter =
-        if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
+        case encoding
+        when Encoding::UTF_8
+          UTF8Counter.new
+        when Encoding::UTF_16LE, Encoding::UTF_16BE
           UTF16Counter.new(source, encoding)
         else
-          LengthCounter.new(source, encoding)
+          UTF32Counter.new(source, encoding)
         end
 
       @cache = {} #: Hash[Integer, Integer]
@@ -214,6 +360,8 @@ module Prism
     end
 
     # Retrieve the code units offset from the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def [](byte_offset)
       @cache[byte_offset] ||=
         if (index = @offsets.bsearch_index { |offset| offset > byte_offset }).nil?
@@ -240,11 +388,15 @@ module Prism
   # at that point we will treat everything as single-byte characters.
   class ASCIISource < Source
     # Return the character offset for the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def character_offset(byte_offset)
       byte_offset
     end
 
-    # Return the column number in characters for the given byte offset.
+    # Return the column in characters for the given byte offset.
+    #
+    #: (Integer byte_offset) -> Integer
     def character_column(byte_offset)
       byte_offset - line_start(byte_offset)
     end
@@ -255,6 +407,8 @@ module Prism
     # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the
     # concept of code units that differs from the number of characters in other
     # encodings, it is not captured here.
+    #
+    #: (Integer byte_offset, Encoding encoding) -> Integer
     def code_units_offset(byte_offset, encoding)
       byte_offset
     end
@@ -262,6 +416,8 @@ module Prism
     # Returns a cache that is the identity function in order to maintain the
     # same interface. We can do this because code units are always equivalent to
     # byte offsets for ASCII-only sources.
+    #
+    #: (Encoding encoding) -> _CodeUnitsCache
     def code_units_cache(encoding)
       ->(byte_offset) { byte_offset }
     end
@@ -269,6 +425,8 @@ module Prism
     # Specialized version of `code_units_column` that does not depend on
     # `code_units_offset`, which is a more expensive operation. This is
     # essentially the same as `Prism::Source#column`.
+    #
+    #: (Integer byte_offset, Encoding encoding) -> Integer
     def code_units_column(byte_offset, encoding)
       byte_offset - line_start(byte_offset)
     end
@@ -278,79 +436,57 @@ module Prism
   class Location
     # A Source object that is used to determine more information from the given
     # offset and length.
-    attr_reader :source
+    attr_reader :source #: Source
     protected :source
 
     # The byte offset from the beginning of the source where this location
     # starts.
-    attr_reader :start_offset
+    attr_reader :start_offset #: Integer
 
     # The length of this location in bytes.
-    attr_reader :length
+    attr_reader :length #: Integer
 
     # Create a new location object with the given source, start byte offset, and
     # byte length.
+    #
+    #: (Source source, Integer start_offset, Integer length) -> void
     def initialize(source, start_offset, length)
       @source = source
       @start_offset = start_offset
       @length = length
-
-      # These are used to store comments that are associated with this location.
-      # They are initialized to `nil` to save on memory when there are no
-      # comments to be attached and/or the comment-related APIs are not used.
-      @leading_comments = nil
-      @trailing_comments = nil
-    end
-
-    # These are the comments that are associated with this location that exist
-    # before the start of this location.
-    def leading_comments
-      @leading_comments ||= []
-    end
-
-    # Attach a comment to the leading comments of this location.
-    def leading_comment(comment)
-      leading_comments << comment
-    end
-
-    # These are the comments that are associated with this location that exist
-    # after the end of this location.
-    def trailing_comments
-      @trailing_comments ||= []
-    end
-
-    # Attach a comment to the trailing comments of this location.
-    def trailing_comment(comment)
-      trailing_comments << comment
-    end
-
-    # Returns all comments that are associated with this location (both leading
-    # and trailing comments).
-    def comments
-      [*@leading_comments, *@trailing_comments]
     end
 
     # Create a new location object with the given options.
+    #
+    #: (?source: Source, ?start_offset: Integer, ?length: Integer) -> Location
     def copy(source: self.source, start_offset: self.start_offset, length: self.length)
       Location.new(source, start_offset, length)
     end
 
     # Returns a new location that is the result of chopping off the last byte.
+    #
+    #: () -> Location
     def chop
       copy(length: length == 0 ? length : length - 1)
     end
 
     # Returns a string representation of this location.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::Location @start_offset=#{@start_offset} @length=#{@length} start_line=#{start_line}>"
     end
 
     # Returns all of the lines of the source code associated with this location.
+    #
+    #: () -> Array[String]
     def source_lines
       source.lines
     end
 
     # The source code that this location represents.
+    #
+    #: () -> String
     def slice
       source.slice(start_offset, length)
     end
@@ -358,6 +494,8 @@ module Prism
     # The source code that this location represents starting from the beginning
     # of the line that this location starts on to the end of the line that this
     # location ends on.
+    #
+    #: () -> String
     def slice_lines
       line_start = source.line_start(start_offset)
       line_end = source.line_end(end_offset)
@@ -366,118 +504,160 @@ module Prism
 
     # The character offset from the beginning of the source where this location
     # starts.
+    #
+    #: () -> Integer
     def start_character_offset
       source.character_offset(start_offset)
     end
 
     # The offset from the start of the file in code units of the given encoding.
+    #
+    #: (Encoding encoding) -> Integer
     def start_code_units_offset(encoding = Encoding::UTF_16LE)
       source.code_units_offset(start_offset, encoding)
     end
 
     # The start offset from the start of the file in code units using the given
     # cache to fetch or calculate the value.
+    #
+    #: (_CodeUnitsCache cache) -> Integer
     def cached_start_code_units_offset(cache)
       cache[start_offset]
     end
 
     # The byte offset from the beginning of the source where this location ends.
+    #
+    #: () -> Integer
     def end_offset
       start_offset + length
     end
 
     # The character offset from the beginning of the source where this location
     # ends.
+    #
+    #: () -> Integer
     def end_character_offset
       source.character_offset(end_offset)
     end
 
     # The offset from the start of the file in code units of the given encoding.
+    #
+    #: (Encoding encoding) -> Integer
     def end_code_units_offset(encoding = Encoding::UTF_16LE)
       source.code_units_offset(end_offset, encoding)
     end
 
     # The end offset from the start of the file in code units using the given
     # cache to fetch or calculate the value.
+    #
+    #: (_CodeUnitsCache cache) -> Integer
     def cached_end_code_units_offset(cache)
       cache[end_offset]
     end
 
     # The line number where this location starts.
+    #
+    #: () -> Integer
     def start_line
       source.line(start_offset)
     end
 
     # The content of the line where this location starts before this location.
+    #
+    #: () -> String
     def start_line_slice
       offset = source.line_start(start_offset)
       source.slice(offset, start_offset - offset)
     end
 
     # The line number where this location ends.
+    #
+    #: () -> Integer
     def end_line
       source.line(end_offset)
     end
 
-    # The column number in bytes where this location starts from the start of
+    # The column in bytes where this location starts from the start of
     # the line.
+    #
+    #: () -> Integer
     def start_column
       source.column(start_offset)
     end
 
-    # The column number in characters where this location ends from the start of
+    # The column in characters where this location ends from the start of
     # the line.
+    #
+    #: () -> Integer
     def start_character_column
       source.character_column(start_offset)
     end
 
-    # The column number in code units of the given encoding where this location
+    # The column in code units of the given encoding where this location
     # starts from the start of the line.
+    #
+    #: (?Encoding encoding) -> Integer
     def start_code_units_column(encoding = Encoding::UTF_16LE)
       source.code_units_column(start_offset, encoding)
     end
 
     # The start column in code units using the given cache to fetch or calculate
     # the value.
+    #
+    #: (_CodeUnitsCache cache) -> Integer
     def cached_start_code_units_column(cache)
       cache[start_offset] - cache[source.line_start(start_offset)]
     end
 
-    # The column number in bytes where this location ends from the start of the
+    # The column in bytes where this location ends from the start of the
     # line.
+    #
+    #: () -> Integer
     def end_column
       source.column(end_offset)
     end
 
-    # The column number in characters where this location ends from the start of
+    # The column in characters where this location ends from the start of
     # the line.
+    #
+    #: () -> Integer
     def end_character_column
       source.character_column(end_offset)
     end
 
-    # The column number in code units of the given encoding where this location
+    # The column in code units of the given encoding where this location
     # ends from the start of the line.
+    #
+    #: (?Encoding encoding) -> Integer
     def end_code_units_column(encoding = Encoding::UTF_16LE)
       source.code_units_column(end_offset, encoding)
     end
 
     # The end column in code units using the given cache to fetch or calculate
     # the value.
+    #
+    #: (_CodeUnitsCache cache) -> Integer
     def cached_end_code_units_column(cache)
       cache[end_offset] - cache[source.line_start(end_offset)]
     end
 
     # Implement the hash pattern matching interface for Location.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { start_offset: start_offset, end_offset: end_offset }
     end
 
     # Implement the pretty print interface for Location.
-    def pretty_print(q)
+    #
+    #: (PP q) -> void
+    def pretty_print(q) # :nodoc:
       q.text("(#{start_line},#{start_column})-(#{end_line},#{end_column})")
     end
 
     # Returns true if the given other location is equal to this location.
+    #
+    #: (untyped other) -> bool
     def ==(other)
       Location === other &&
         other.start_offset == start_offset &&
@@ -487,6 +667,8 @@ module Prism
     # Returns a new location that stretches from this location to the given
     # other location. Raises an error if this location is not before the other
     # location or if they don't share the same source.
+    #
+    #: (Location other) -> Location
     def join(other)
       raise "Incompatible sources" if source != other.source
       raise "Incompatible locations" if start_offset > other.start_offset
@@ -497,6 +679,8 @@ module Prism
     # Join this location with the first occurrence of the string in the source
     # that occurs after this location on the same line, and return the new
     # location. This will raise an error if the string does not exist.
+    #
+    #: (String string) -> Location
     def adjoin(string)
       line_suffix = source.slice(end_offset, source.line_end(end_offset) - end_offset)
 
@@ -510,23 +694,38 @@ module Prism
   # This represents a comment that was encountered during parsing. It is the
   # base class for all comment types.
   class Comment
-    # The location of this comment in the source.
-    attr_reader :location
+    # The Location of this comment in the source.
+    attr_reader :location #: Location
 
     # Create a new comment object with the given location.
+    #
+    #: (Location location) -> void
     def initialize(location)
       @location = location
     end
 
     # Implement the hash pattern matching interface for Comment.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { location: location }
     end
 
     # Returns the content of the comment by slicing it from the source code.
+    #
+    #: () -> String
     def slice
       location.slice
     end
+
+    # Returns true if this comment happens on the same line as other code and
+    # false if the comment is by itself. This can only be true for inline
+    # comments and should be false for block comments.
+    #
+    #: () -> bool
+    def trailing?
+      raise NotImplementedError, "trailing? is not implemented for #{self.class}"
+    end
   end
 
   # InlineComment objects are the most common. They correspond to comments in
@@ -534,12 +733,18 @@ module Prism
   class InlineComment < Comment
     # Returns true if this comment happens on the same line as other code and
     # false if the comment is by itself.
+    #
+    #: () -> bool
     def trailing?
       !location.start_line_slice.strip.empty?
+    rescue ArgumentError # invalid bytes, never whitespace only
+      false
     end
 
     # Returns a string representation of this comment.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::InlineComment @location=#{location.inspect}>"
     end
   end
@@ -547,13 +752,17 @@ module Prism
   # EmbDocComment objects correspond to comments that are surrounded by =begin
   # and =end.
   class EmbDocComment < Comment
-    # This can only be true for inline comments.
+    # Returns false. This can only be true for inline comments.
+    #
+    #: () -> bool
     def trailing?
       false
     end
 
     # Returns a string representation of this comment.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::EmbDocComment @location=#{location.inspect}>"
     end
   end
@@ -561,34 +770,44 @@ module Prism
   # This represents a magic comment that was encountered during parsing.
   class MagicComment
     # A Location object representing the location of the key in the source.
-    attr_reader :key_loc
+    attr_reader :key_loc #: Location
 
     # A Location object representing the location of the value in the source.
-    attr_reader :value_loc
+    attr_reader :value_loc #: Location
 
     # Create a new magic comment object with the given key and value locations.
+    #
+    #: (Location key_loc, Location value_loc) -> void
     def initialize(key_loc, value_loc)
       @key_loc = key_loc
       @value_loc = value_loc
     end
 
     # Returns the key of the magic comment by slicing it from the source code.
+    #
+    #: () -> String
     def key
       key_loc.slice
     end
 
     # Returns the value of the magic comment by slicing it from the source code.
+    #
+    #: () -> String
     def value
       value_loc.slice
     end
 
     # Implement the hash pattern matching interface for MagicComment.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { key_loc: key_loc, value_loc: value_loc }
     end
 
     # Returns a string representation of this magic comment.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::MagicComment @key=#{key.inspect} @value=#{value.inspect}>"
     end
   end
@@ -597,18 +816,20 @@ module Prism
   class ParseError
     # The type of error. This is an _internal_ symbol that is used for
     # communicating with translation layers. It is not meant to be public API.
-    attr_reader :type
+    attr_reader :type #: Symbol
 
     # The message associated with this error.
-    attr_reader :message
+    attr_reader :message #: String
 
     # A Location object representing the location of this error in the source.
-    attr_reader :location
+    attr_reader :location #: Location
 
     # The level of this error.
-    attr_reader :level
+    attr_reader :level #: Symbol
 
     # Create a new error object with the given message and location.
+    #
+    #: (Symbol type, String message, Location location, Symbol level) -> void
     def initialize(type, message, location, level)
       @type = type
       @message = message
@@ -617,12 +838,16 @@ module Prism
     end
 
     # Implement the hash pattern matching interface for ParseError.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { type: type, message: message, location: location, level: level }
     end
 
     # Returns a string representation of this error.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::ParseError @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
     end
   end
@@ -631,18 +856,20 @@ module Prism
   class ParseWarning
     # The type of warning. This is an _internal_ symbol that is used for
     # communicating with translation layers. It is not meant to be public API.
-    attr_reader :type
+    attr_reader :type #: Symbol
 
     # The message associated with this warning.
-    attr_reader :message
+    attr_reader :message #: String
 
     # A Location object representing the location of this warning in the source.
-    attr_reader :location
+    attr_reader :location #: Location
 
     # The level of this warning.
-    attr_reader :level
+    attr_reader :level #: Symbol
 
     # Create a new warning object with the given message and location.
+    #
+    #: (Symbol type, String message, Location location, Symbol level) -> void
     def initialize(type, message, location, level)
       @type = type
       @message = message
@@ -651,73 +878,118 @@ module Prism
     end
 
     # Implement the hash pattern matching interface for ParseWarning.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { type: type, message: message, location: location, level: level }
     end
 
     # Returns a string representation of this warning.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       "#<Prism::ParseWarning @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
     end
   end
 
-  # This represents the result of a call to ::parse or ::parse_file. It contains
-  # the requested structure, any comments that were encounters, and any errors
-  # that were encountered.
+  # This represents the result of a call to Prism.parse or Prism.parse_file.
+  # It contains the requested structure, any comments that were encounters,
+  # and any errors that were encountered.
   class Result
-    # The list of comments that were encountered during parsing.
-    attr_reader :comments
-
     # The list of magic comments that were encountered during parsing.
-    attr_reader :magic_comments
+    attr_reader :magic_comments #: Array[MagicComment]
 
     # An optional location that represents the location of the __END__ marker
     # and the rest of the content of the file. This content is loaded into the
     # DATA constant when the file being parsed is the main file being executed.
-    attr_reader :data_loc
+    attr_reader :data_loc #: Location?
 
     # The list of errors that were generated during parsing.
-    attr_reader :errors
+    attr_reader :errors #: Array[ParseError]
 
     # The list of warnings that were generated during parsing.
-    attr_reader :warnings
+    attr_reader :warnings #: Array[ParseWarning]
 
     # A Source instance that represents the source code that was parsed.
-    attr_reader :source
+    attr_reader :source #: Source
 
     # Create a new result object with the given values.
-    def initialize(comments, magic_comments, data_loc, errors, warnings, source)
-      @comments = comments
+    #
+    #: (Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(magic_comments, data_loc, errors, warnings, continuable, source)
       @magic_comments = magic_comments
       @data_loc = data_loc
       @errors = errors
       @warnings = warnings
+      @continuable = continuable
       @source = source
     end
 
+    # The list of comments that were encountered during parsing.
+    #
+    #: () -> Array[Comment]
+    def comments
+      source.comments
+    end
+
     # Implement the hash pattern matching interface for Result.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
     end
 
     # Returns the encoding of the source code that was parsed.
+    #
+    #: () -> Encoding
     def encoding
       source.encoding
     end
 
     # Returns true if there were no errors during parsing and false if there
     # were.
+    #
+    #: () -> bool
     def success?
       errors.empty?
     end
 
     # Returns true if there were errors during parsing and false if there were
     # not.
+    #
+    #: () -> bool
     def failure?
       !success?
     end
 
+    # Returns true if the parsed source is an incomplete expression that could
+    # become valid with additional input. This is useful for REPL contexts (such
+    # as IRB) where the user may be entering a multi-line expression one line at
+    # a time and the implementation needs to determine whether to wait for more
+    # input or to evaluate what has been entered so far.
+    #
+    # Concretely, this returns true when every error present is caused by the
+    # parser reaching the end of the input before a construct was closed (e.g.
+    # an unclosed string, array, block, or keyword), and returns false when any
+    # error is caused by a token that makes the input structurally invalid
+    # regardless of what might follow (e.g. a stray `end`, `]`, or `)` with no
+    # matching opener).
+    #
+    # Examples:
+    #
+    #     Prism.parse("1 + [").continuable?      #=> true  (unclosed array)
+    #     Prism.parse("1 + ]").continuable?      #=> false (stray ])
+    #     Prism.parse("tap do").continuable?     #=> true  (unclosed block)
+    #     Prism.parse("end.tap do").continuable? #=> false (stray end)
+    #
+    #: () -> bool
+    def continuable?
+      @continuable
+    end
+
     # Create a code units cache for the given encoding.
+    #
+    #: (Encoding encoding) -> _CodeUnitsCache
     def code_units_cache(encoding)
       source.code_units_cache(encoding)
     end
@@ -725,41 +997,42 @@ module Prism
 
   # This is a result specific to the `parse` and `parse_file` methods.
   class ParseResult < Result
-    autoload :Comments, "prism/parse_result/comments"
     autoload :Errors, "prism/parse_result/errors"
     autoload :Newlines, "prism/parse_result/newlines"
 
-    private_constant :Comments
     private_constant :Errors
     private_constant :Newlines
 
     # The syntax tree that was parsed from the source code.
-    attr_reader :value
+    attr_reader :value #: ProgramNode
 
     # Create a new parse result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #
+    #: (ProgramNode value, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for ParseResult.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       super.merge!(value: value)
     end
 
-    # Attach the list of comments to their respective locations in the tree.
-    def attach_comments!
-      Comments.new(self).attach! # steep:ignore
-    end
-
     # Walk the tree and mark nodes that are on a new line, loosely emulating
     # the behavior of CRuby's `:line` tracepoint event.
+    #
+    #: () -> void
     def mark_newlines!
       value.accept(Newlines.new(source.offsets.size)) # steep:ignore
     end
 
     # Returns a string representation of the syntax tree with the errors
     # displayed inline.
+    #
+    #: () -> String
     def errors_format
       Errors.new(self).format
     end
@@ -768,16 +1041,20 @@ module Prism
   # This is a result specific to the `lex` and `lex_file` methods.
   class LexResult < Result
     # The list of tokens that were parsed from the source code.
-    attr_reader :value
+    attr_reader :value #: Array[Token]
 
     # Create a new lex result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #
+    #: (Array[Token] value, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for LexResult.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       super.merge!(value: value)
     end
   end
@@ -786,16 +1063,20 @@ module Prism
   class ParseLexResult < Result
     # A tuple of the syntax tree and the list of tokens that were parsed from
     # the source code.
-    attr_reader :value
+    attr_reader :value #: [ProgramNode, Array[Token]]
 
     # Create a new parse lex result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+    #
+    #: ([ProgramNode, Array[Token]] value, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    def initialize(value, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
-      super(comments, magic_comments, data_loc, errors, warnings, source)
+      super(magic_comments, data_loc, errors, warnings, continuable, source)
     end
 
     # Implement the hash pattern matching interface for ParseLexResult.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       super.merge!(value: value)
     end
   end
@@ -803,29 +1084,38 @@ module Prism
   # This represents a token from the Ruby source.
   class Token
     # The Source object that represents the source this token came from.
-    attr_reader :source
+    attr_reader :source #: Source
     private :source
 
     # The type of token that this token is.
-    attr_reader :type
+    attr_reader :type #: Symbol
 
     # A byteslice of the source that this token represents.
-    attr_reader :value
+    attr_reader :value #: String
+
+    # @rbs @location: Location | Integer
 
     # Create a new token object with the given type, value, and location.
-    def initialize(source, type, value, location)
+    #
+    #: (Source source, Symbol type, String value, Location | Integer location, Integer state) -> void
+    def initialize(source, type, value, location, state)
       @source = source
       @type = type
       @value = value
       @location = location
+      @state = state
     end
 
     # Implement the hash pattern matching interface for Token.
-    def deconstruct_keys(keys)
+    #
+    #: (Array[Symbol]? keys) -> Hash[Symbol, untyped]
+    def deconstruct_keys(keys) # :nodoc:
       { type: type, value: value, location: location }
     end
 
     # A Location object representing the location of this token in the source.
+    #
+    #: () -> Location
     def location
       location = @location
       return location if location.is_a?(Location)
@@ -833,7 +1123,9 @@ module Prism
     end
 
     # Implement the pretty print interface for Token.
-    def pretty_print(q)
+    #
+    #: (PP q) -> void
+    def pretty_print(q) # :nodoc:
       q.group do
         q.text(type.to_s)
         self.location.pretty_print(q)
@@ -848,6 +1140,8 @@ module Prism
     end
 
     # Returns true if the given other token is equal to this token.
+    #
+    #: (untyped other) -> bool
     def ==(other)
       Token === other &&
         other.type == type &&
@@ -855,17 +1149,42 @@ module Prism
     end
 
     # Returns a string representation of this token.
-    def inspect
+    #
+    #: () -> String
+    def inspect # :nodoc:
       location
       super
     end
 
     # Freeze this object and the objects it contains.
+    #
+    #: () -> void
     def deep_freeze
       value.freeze
       location.freeze
       freeze
     end
+
+    # For internal use only.
+    #: () -> Integer
+    def _ripper_state # :nodoc:
+      @state
+    end
+
+    # Backwards compatibility for Prism.lex/Prism.lex_file/Prism.parse_lex
+    # when they returned a 2-element array.
+
+    #: (Integer index) -> Token | Integer
+    def [](index) # :nodoc:
+      return self if index == 0
+      return @state if index == 1
+      raise ArgumentError, "Invalid index #{index}"
+    end
+
+    #: () -> Token
+    def first # :nodoc:
+      self
+    end
   end
 
   # This object is passed to the various Prism.* methods that accept the
@@ -875,14 +1194,16 @@ module Prism
   class Scope
     # The list of local variables that are defined in this scope. This should be
     # defined as an array of symbols.
-    attr_reader :locals
+    attr_reader :locals #: Array[Symbol]
 
     # The list of local variables that are forwarded to the next scope. This
     # should by defined as an array of symbols containing the specific values of
     # :*, :**, :&, or :"...".
-    attr_reader :forwarding
+    attr_reader :forwarding #: Array[Symbol]
 
     # Create a new scope object with the given locals and forwarding.
+    #
+    #: (Array[Symbol] locals, Array[Symbol] forwarding) -> void
     def initialize(locals, forwarding)
       @locals = locals
       @forwarding = forwarding
@@ -892,6 +1213,8 @@ module Prism
   # Create a new scope with the given locals and forwarding options that is
   # suitable for passing into one of the Prism.* methods that accepts the
   # `scopes` option.
+  #
+  #: (?locals: Array[Symbol], ?forwarding: Array[Symbol]) -> Scope
   def self.scope(locals: [], forwarding: [])
     Scope.new(locals, forwarding)
   end