jruby-prism-parser 0.24.0-java → 1.4.0-java

data/lib/prism/parse_result.rb

@@ -5,6 +5,33 @@ module Prism
   # conjunction with locations to allow them to resolve line numbers and source
   # ranges.
   class Source
+    # Create a new source object with the given source code. This method should
+    # be used instead of `new` and it will return either a `Source` or a
+    # specialized and more performant `ASCIISource` if no multibyte characters
+    # are present in the source code.
+    def self.for(source, start_line = 1, offsets = [])
+      if source.ascii_only?
+        ASCIISource.new(source, start_line, offsets)
+      elsif source.encoding == Encoding::BINARY
+        source.force_encoding(Encoding::UTF_8)
+
+        if source.valid_encoding?
+          new(source, start_line, offsets)
+        else
+          # This is an extremely niche use case where the file is marked as
+          # binary, contains multi-byte characters, and those characters are not
+          # valid UTF-8. In this case we'll mark it as binary and fall back to
+          # treating everything as a single-byte character. This _may_ cause
+          # problems when asking for code units, but it appears to be the
+          # cleanest solution at the moment.
+          source.force_encoding(Encoding::BINARY)
+          ASCIISource.new(source, start_line, offsets)
+        end
+      else
+        new(source, start_line, offsets)
+      end
+    end
+
     # The source code that this source object represents.
     attr_reader :source
 
@@ -21,10 +48,31 @@ module Prism
       @offsets = offsets # set after parsing is done
     end
 
+    # Replace the value of start_line with the given value.
+    def replace_start_line(start_line)
+      @start_line = start_line
+    end
+
+    # Replace the value of offsets with the given value.
+    def replace_offsets(offsets)
+      @offsets.replace(offsets)
+    end
+
+    # Returns the encoding of the source code, which is set by parameters to the
+    # parser or by the encoding magic comment.
+    def encoding
+      source.encoding
+    end
+
+    # Returns the lines of the source code as an array of strings.
+    def lines
+      source.lines
+    end
+
     # Perform a byteslice on the source code using the given byte offset and
     # byte length.
     def slice(byte_offset, length)
-      source.byteslice(byte_offset, length)
+      source.byteslice(byte_offset, length) or raise
     end
 
     # Binary search through the offsets to find the line number for the given
@@ -39,6 +87,12 @@ module Prism
       offsets[find_line(byte_offset)]
     end
 
+    # Returns the byte offset of the end of the line corresponding to the given
+    # byte offset.
+    def line_end(byte_offset)
+      offsets[find_line(byte_offset) + 1] || source.bytesize
+    end
+
     # Return the column number for the given byte offset.
     def column(byte_offset)
       byte_offset - line_start(byte_offset)
@@ -46,7 +100,7 @@ module Prism
 
     # Return the character offset for the given byte offset.
     def character_offset(byte_offset)
-      source.byteslice(0, byte_offset).length
+      (source.byteslice(0, byte_offset) or raise).length
     end
 
     # Return the column number in characters for the given byte offset.
@@ -60,9 +114,26 @@ module Prism
     # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the
     # concept of code units that differs from the number of characters in other
     # encodings, it is not captured here.
+    #
+    # We purposefully replace invalid and undefined characters with replacement
+    # characters in this conversion. This happens for two reasons. First, it's
+    # possible that the given byte offset will not occur on a character
+    # boundary. Second, it's possible that the source code will contain a
+    # character that has no equivalent in the given encoding.
     def code_units_offset(byte_offset, encoding)
-      byteslice = source.byteslice(0, byte_offset).encode(encoding)
-      (encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE) ? (byteslice.bytesize / 2) : byteslice.length
+      byteslice = (source.byteslice(0, byte_offset) or raise).encode(encoding, invalid: :replace, undef: :replace)
+
+      if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
+        byteslice.bytesize / 2
+      else
+        byteslice.length
+      end
+    end
+
+    # Generate a cache that targets a specific encoding for calculating code
+    # unit offsets.
+    def code_units_cache(encoding)
+      CodeUnitsCache.new(source, encoding)
     end
 
     # Returns the column number in code units for the given encoding for the
@@ -71,6 +142,13 @@ module Prism
       code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
     end
 
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      source.freeze
+      offsets.freeze
+      freeze
+    end
+
     private
 
     # Binary search through the offsets to find the line number for the given
@@ -81,9 +159,9 @@ module Prism
 
       while left <= right
         mid = left + (right - left) / 2
-        return mid if offsets[mid] == byte_offset
+        return mid if (offset = offsets[mid]) == byte_offset
 
-        if offsets[mid] < byte_offset
+        if offset < byte_offset
           left = mid + 1
         else
           right = mid - 1
@@ -94,6 +172,120 @@ module Prism
     end
   end
 
+  # A cache that can be used to quickly compute code unit offsets from byte
+  # offsets. It purposefully provides only a single #[] method to access the
+  # cache in order to minimize surface area.
+  #
+  # Note that there are some known issues here that may or may not be addressed
+  # in the future:
+  #
+  # * The first is that there are issues when the cache computes values that are
+  #   not on character boundaries. This can result in subsequent computations
+  #   being off by one or more code units.
+  # * The second is that this cache is currently unbounded. In theory we could
+  #   introduce some kind of LRU cache to limit the number of entries, but this
+  #   has not yet been implemented.
+  #
+  class CodeUnitsCache
+    class UTF16Counter # :nodoc:
+      def initialize(source, encoding)
+        @source = source
+        @encoding = encoding
+      end
+
+      def count(byte_offset, byte_length)
+        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).bytesize / 2
+      end
+    end
+
+    class LengthCounter # :nodoc:
+      def initialize(source, encoding)
+        @source = source
+        @encoding = encoding
+      end
+
+      def count(byte_offset, byte_length)
+        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).length
+      end
+    end
+
+    private_constant :UTF16Counter, :LengthCounter
+
+    # Initialize a new cache with the given source and encoding.
+    def initialize(source, encoding)
+      @source = source
+      @counter =
+        if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
+          UTF16Counter.new(source, encoding)
+        else
+          LengthCounter.new(source, encoding)
+        end
+
+      @cache = {} #: Hash[Integer, Integer]
+      @offsets = [] #: Array[Integer]
+    end
+
+    # Retrieve the code units offset from the given byte offset.
+    def [](byte_offset)
+      @cache[byte_offset] ||=
+        if (index = @offsets.bsearch_index { |offset| offset > byte_offset }).nil?
+          @offsets << byte_offset
+          @counter.count(0, byte_offset)
+        elsif index == 0
+          @offsets.unshift(byte_offset)
+          @counter.count(0, byte_offset)
+        else
+          @offsets.insert(index, byte_offset)
+          offset = @offsets[index - 1]
+          @cache[offset] + @counter.count(offset, byte_offset - offset)
+        end
+    end
+  end
+
+  # Specialized version of Prism::Source for source code that includes ASCII
+  # characters only. This class is used to apply performance optimizations that
+  # cannot be applied to sources that include multibyte characters.
+  #
+  # In the extremely rare case that a source includes multi-byte characters but
+  # is marked as binary because of a magic encoding comment and it cannot be
+  # eagerly converted to UTF-8, this class will be used as well. This is because
+  # at that point we will treat everything as single-byte characters.
+  class ASCIISource < Source
+    # Return the character offset for the given byte offset.
+    def character_offset(byte_offset)
+      byte_offset
+    end
+
+    # Return the column number in characters for the given byte offset.
+    def character_column(byte_offset)
+      byte_offset - line_start(byte_offset)
+    end
+
+    # Returns the offset from the start of the file for the given byte offset
+    # counting in code units for the given encoding.
+    #
+    # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the
+    # concept of code units that differs from the number of characters in other
+    # encodings, it is not captured here.
+    def code_units_offset(byte_offset, encoding)
+      byte_offset
+    end
+
+    # Returns a cache that is the identity function in order to maintain the
+    # same interface. We can do this because code units are always equivalent to
+    # byte offsets for ASCII-only sources.
+    def code_units_cache(encoding)
+      ->(byte_offset) { byte_offset }
+    end
+
+    # Specialized version of `code_units_column` that does not depend on
+    # `code_units_offset`, which is a more expensive operation. This is
+    # essentially the same as `Prism::Source#column`.
+    def code_units_column(byte_offset, encoding)
+      byte_offset - line_start(byte_offset)
+    end
+  end
+
   # This represents a location in the source.
   class Location
     # A Source object that is used to determine more information from the given
@@ -108,25 +300,56 @@ module Prism
     # The length of this location in bytes.
     attr_reader :length
 
-    # The list of comments attached to this location
-    attr_reader :comments
-
     # Create a new location object with the given source, start byte offset, and
     # byte length.
     def initialize(source, start_offset, length)
       @source = source
       @start_offset = start_offset
       @length = length
-      @comments = []
+
+      # These are used to store comments that are associated with this location.
+      # They are initialized to `nil` to save on memory when there are no
+      # comments to be attached and/or the comment-related APIs are not used.
+      @leading_comments = nil
+      @trailing_comments = nil
+    end
+
+    # These are the comments that are associated with this location that exist
+    # before the start of this location.
+    def leading_comments
+      @leading_comments ||= []
+    end
+
+    # Attach a comment to the leading comments of this location.
+    def leading_comment(comment)
+      leading_comments << comment
+    end
+
+    # These are the comments that are associated with this location that exist
+    # after the end of this location.
+    def trailing_comments
+      @trailing_comments ||= []
+    end
+
+    # Attach a comment to the trailing comments of this location.
+    def trailing_comment(comment)
+      trailing_comments << comment
+    end
+
+    # Returns all comments that are associated with this location (both leading
+    # and trailing comments).
+    def comments
+      [*@leading_comments, *@trailing_comments]
     end
 
     # Create a new location object with the given options.
-    def copy(**options)
-      Location.new(
-        options.fetch(:source) { source },
-        options.fetch(:start_offset) { start_offset },
-        options.fetch(:length) { length }
-      )
+    def copy(source: self.source, start_offset: self.start_offset, length: self.length)
+      Location.new(source, start_offset, length)
+    end
+
+    # Returns a new location that is the result of chopping off the last byte.
+    def chop
+      copy(length: length == 0 ? length : length - 1)
     end
 
     # Returns a string representation of this location.
@@ -134,11 +357,25 @@ module Prism
       "#<Prism::Location @start_offset=#{@start_offset} @length=#{@length} start_line=#{start_line}>"
     end
 
+    # Returns all of the lines of the source code associated with this location.
+    def source_lines
+      source.lines
+    end
+
     # The source code that this location represents.
     def slice
       source.slice(start_offset, length)
     end
 
+    # The source code that this location represents starting from the beginning
+    # of the line that this location starts on to the end of the line that this
+    # location ends on.
+    def slice_lines
+      line_start = source.line_start(start_offset)
+      line_end = source.line_end(end_offset)
+      source.slice(line_start, line_end - line_start)
+    end
+
     # The character offset from the beginning of the source where this location
     # starts.
     def start_character_offset
@@ -150,6 +387,12 @@ module Prism
       source.code_units_offset(start_offset, encoding)
     end
 
+    # The start offset from the start of the file in code units using the given
+    # cache to fetch or calculate the value.
+    def cached_start_code_units_offset(cache)
+      cache[start_offset]
+    end
+
     # The byte offset from the beginning of the source where this location ends.
     def end_offset
       start_offset + length
@@ -166,6 +409,12 @@ module Prism
       source.code_units_offset(end_offset, encoding)
     end
 
+    # The end offset from the start of the file in code units using the given
+    # cache to fetch or calculate the value.
+    def cached_end_code_units_offset(cache)
+      cache[end_offset]
+    end
+
     # The line number where this location starts.
     def start_line
       source.line(start_offset)
@@ -200,6 +449,12 @@ module Prism
       source.code_units_column(start_offset, encoding)
     end
 
+    # The start column in code units using the given cache to fetch or calculate
+    # the value.
+    def cached_start_code_units_column(cache)
+      cache[start_offset] - cache[source.line_start(start_offset)]
+    end
+
     # The column number in bytes where this location ends from the start of the
     # line.
     def end_column
@@ -218,6 +473,12 @@ module Prism
       source.code_units_column(end_offset, encoding)
     end
 
+    # The end column in code units using the given cache to fetch or calculate
+    # the value.
+    def cached_end_code_units_column(cache)
+      cache[end_offset] - cache[source.line_start(end_offset)]
+    end
+
     # Implement the hash pattern matching interface for Location.
     def deconstruct_keys(keys)
       { start_offset: start_offset, end_offset: end_offset }
@@ -230,7 +491,7 @@ module Prism
 
     # Returns true if the given other location is equal to this location.
     def ==(other)
-      other.is_a?(Location) &&
+      Location === other &&
         other.start_offset == start_offset &&
         other.end_offset == end_offset
     end
@@ -245,11 +506,16 @@ module Prism
       Location.new(source, start_offset, other.end_offset - start_offset)
     end
 
-    # Returns a null location that does not correspond to a source and points to
-    # the beginning of the file. Useful for when you want a location object but
-    # do not care where it points.
-    def self.null
-      new(nil, 0, 0)
+    # Join this location with the first occurrence of the string in the source
+    # that occurs after this location on the same line, and return the new
+    # location. This will raise an error if the string does not exist.
+    def adjoin(string)
+      line_suffix = source.slice(end_offset, source.line_end(end_offset) - end_offset)
+
+      line_suffix_index = line_suffix.byteindex(string)
+      raise "Could not find #{string}" if line_suffix_index.nil?
+
+      Location.new(source, start_offset, length + line_suffix_index + string.bytesize)
     end
   end
 
@@ -268,6 +534,11 @@ module Prism
     def deconstruct_keys(keys)
       { location: location }
     end
+
+    # Returns the content of the comment by slicing it from the source code.
+    def slice
+      location.slice
+    end
   end
 
   # InlineComment objects are the most common. They correspond to comments in
@@ -336,6 +607,10 @@ module Prism
 
   # This represents an error that was encountered during parsing.
   class ParseError
+    # The type of error. This is an _internal_ symbol that is used for
+    # communicating with translation layers. It is not meant to be public API.
+    attr_reader :type
+
     # The message associated with this error.
     attr_reader :message
 
@@ -346,7 +621,8 @@ module Prism
     attr_reader :level
 
     # Create a new error object with the given message and location.
-    def initialize(message, location, level)
+    def initialize(type, message, location, level)
+      @type = type
       @message = message
       @location = location
       @level = level
@@ -354,17 +630,21 @@ module Prism
 
     # Implement the hash pattern matching interface for ParseError.
     def deconstruct_keys(keys)
-      { message: message, location: location, level: level }
+      { type: type, message: message, location: location, level: level }
     end
 
     # Returns a string representation of this error.
     def inspect
-      "#<Prism::ParseError @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
+      "#<Prism::ParseError @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
     end
   end
 
   # This represents a warning that was encountered during parsing.
   class ParseWarning
+    # The type of warning. This is an _internal_ symbol that is used for
+    # communicating with translation layers. It is not meant to be public API.
+    attr_reader :type
+
     # The message associated with this warning.
     attr_reader :message
 
@@ -375,7 +655,8 @@ module Prism
     attr_reader :level
 
     # Create a new warning object with the given message and location.
-    def initialize(message, location, level)
+    def initialize(type, message, location, level)
+      @type = type
       @message = message
       @location = location
       @level = level
@@ -383,24 +664,19 @@ module Prism
 
     # Implement the hash pattern matching interface for ParseWarning.
     def deconstruct_keys(keys)
-      { message: message, location: location, level: level }
+      { type: type, message: message, location: location, level: level }
     end
 
     # Returns a string representation of this warning.
     def inspect
-      "#<Prism::ParseWarning @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
+      "#<Prism::ParseWarning @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>"
     end
   end
 
   # This represents the result of a call to ::parse or ::parse_file. It contains
-  # the AST, any comments that were encounters, and any errors that were
-  # encountered.
-  class ParseResult
-    # The value that was generated by parsing. Normally this holds the AST, but
-    # it can sometimes how a list of tokens or other results passed back from
-    # the parser.
-    attr_reader :value
-
+  # the requested structure, any comments that were encounters, and any errors
+  # that were encountered.
+  class Result
     # The list of comments that were encountered during parsing.
     attr_reader :comments
 
@@ -421,9 +697,8 @@ module Prism
     # A Source instance that represents the source code that was parsed.
     attr_reader :source
 
-    # Create a new parse result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
-      @value = value
+    # Create a new result object with the given values.
+    def initialize(comments, magic_comments, data_loc, errors, warnings, source)
       @comments = comments
       @magic_comments = magic_comments
       @data_loc = data_loc
@@ -432,9 +707,14 @@ module Prism
       @source = source
     end
 
-    # Implement the hash pattern matching interface for ParseResult.
+    # Implement the hash pattern matching interface for Result.
     def deconstruct_keys(keys)
-      { value: value, comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
+      { comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
+    end
+
+    # Returns the encoding of the source code that was parsed.
+    def encoding
+      source.encoding
     end
 
     # Returns true if there were no errors during parsing and false if there
@@ -448,6 +728,88 @@ module Prism
     def failure?
       !success?
     end
+
+    # Create a code units cache for the given encoding.
+    def code_units_cache(encoding)
+      source.code_units_cache(encoding)
+    end
+  end
+
+  # This is a result specific to the `parse` and `parse_file` methods.
+  class ParseResult < Result
+    autoload :Comments, "prism/parse_result/comments"
+    autoload :Errors, "prism/parse_result/errors"
+    autoload :Newlines, "prism/parse_result/newlines"
+
+    private_constant :Comments
+    private_constant :Errors
+    private_constant :Newlines
+
+    # The syntax tree that was parsed from the source code.
+    attr_reader :value
+
+    # Create a new parse result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+
+    # Attach the list of comments to their respective locations in the tree.
+    def attach_comments!
+      Comments.new(self).attach! # steep:ignore
+    end
+
+    # Walk the tree and mark nodes that are on a new line, loosely emulating
+    # the behavior of CRuby's `:line` tracepoint event.
+    def mark_newlines!
+      value.accept(Newlines.new(source.offsets.size)) # steep:ignore
+    end
+
+    # Returns a string representation of the syntax tree with the errors
+    # displayed inline.
+    def errors_format
+      Errors.new(self).format
+    end
+  end
+
+  # This is a result specific to the `lex` and `lex_file` methods.
+  class LexResult < Result
+    # The list of tokens that were parsed from the source code.
+    attr_reader :value
+
+    # Create a new lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for LexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
+  # This is a result specific to the `parse_lex` and `parse_lex_file` methods.
+  class ParseLexResult < Result
+    # A tuple of the syntax tree and the list of tokens that were parsed from
+    # the source code.
+    attr_reader :value
+
+    # Create a new parse lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseLexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
   end
 
   # This represents a token from the Ruby source.
@@ -477,8 +839,9 @@ module Prism
 
     # A Location object representing the location of this token in the source.
     def location
-      return @location if @location.is_a?(Location)
-      @location = Location.new(source, @location >> 32, @location & 0xFFFFFFFF)
+      location = @location
+      return location if location.is_a?(Location)
+      @location = Location.new(source, location >> 32, location & 0xFFFFFFFF)
     end
 
     # Implement the pretty print interface for Token.
@@ -498,9 +861,50 @@ module Prism
 
     # Returns true if the given other token is equal to this token.
     def ==(other)
-      other.is_a?(Token) &&
+      Token === other &&
         other.type == type &&
         other.value == value
     end
+
+    # Returns a string representation of this token.
+    def inspect
+      location
+      super
+    end
+
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      value.freeze
+      location.freeze
+      freeze
+    end
+  end
+
+  # This object is passed to the various Prism.* methods that accept the
+  # `scopes` option as an element of the list. It defines both the local
+  # variables visible at that scope as well as the forwarding parameters
+  # available at that scope.
+  class Scope
+    # The list of local variables that are defined in this scope. This should be
+    # defined as an array of symbols.
+    attr_reader :locals
+
+    # The list of local variables that are forwarded to the next scope. This
+    # should by defined as an array of symbols containing the specific values of
+    # :*, :**, :&, or :"...".
+    attr_reader :forwarding
+
+    # Create a new scope object with the given locals and forwarding.
+    def initialize(locals, forwarding)
+      @locals = locals
+      @forwarding = forwarding
+    end
+  end
+
+  # Create a new scope with the given locals and forwarding options that is
+  # suitable for passing into one of the Prism.* methods that accepts the
+  # `scopes` option.
+  def self.scope(locals: [], forwarding: [])
+    Scope.new(locals, forwarding)
   end
 end