prism 0.29.0 → 1.3.0

data/lib/prism/node_ext.rb

@@ -5,10 +5,13 @@
 module Prism
   class Node
     def deprecated(*replacements) # :nodoc:
+      location = caller_locations(1, 1)
+      location = location[0].label if location
       suggest = replacements.map { |replacement| "#{self.class}##{replacement}" }
+
       warn(<<~MSG, category: :deprecated)
-        [deprecation]: #{self.class}##{caller_locations(1, 1)[0].label} is deprecated \
-        and will be removed in the next major version. Use #{suggest.join("/")} instead.
+        [deprecation]: #{self.class}##{location} is deprecated and will be \
+        removed in the next major version. Use #{suggest.join("/")} instead.
         #{(caller(1, 3) || []).join("\n")}
       MSG
     end
@@ -18,7 +21,10 @@ module Prism
     # Returns a numeric value that represents the flags that were used to create
     # the regular expression.
     def options
-      o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
+      o = 0
+      o |= Regexp::IGNORECASE if flags.anybits?(RegularExpressionFlags::IGNORE_CASE)
+      o |= Regexp::EXTENDED if flags.anybits?(RegularExpressionFlags::EXTENDED)
+      o |= Regexp::MULTILINE if flags.anybits?(RegularExpressionFlags::MULTI_LINE)
       o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
       o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
       o
@@ -66,11 +72,12 @@ module Prism
     def to_interpolated
       InterpolatedStringNode.new(
         source,
+        -1,
+        location,
         frozen? ? InterpolatedStringNodeFlags::FROZEN : 0,
         opening_loc,
-        [copy(opening_loc: nil, closing_loc: nil, location: content_loc)],
-        closing_loc,
-        location
+        [copy(location: content_loc, opening_loc: nil, closing_loc: nil)],
+        closing_loc
       )
     end
   end
@@ -83,10 +90,12 @@ module Prism
     def to_interpolated
       InterpolatedXStringNode.new(
         source,
+        -1,
+        location,
+        flags,
         opening_loc,
-        [StringNode.new(source, 0, nil, content_loc, nil, unescaped, content_loc)],
-        closing_loc,
-        location
+        [StringNode.new(source, node_id, content_loc, 0, nil, content_loc, nil, unescaped)],
+        closing_loc
       )
     end
   end
@@ -103,7 +112,19 @@ module Prism
   class RationalNode < Node
     # Returns the value of the node as a Ruby Rational.
     def value
-      Rational(numeric.is_a?(IntegerNode) ? numeric.value : slice.chomp("r"))
+      Rational(numerator, denominator)
+    end
+
+    # Returns the value of the node as an IntegerNode or a FloatNode. This
+    # method is deprecated in favor of #value or #numerator/#denominator.
+    def numeric
+      deprecated("value", "numerator", "denominator")
+
+      if denominator == 1
+        IntegerNode.new(source, -1, location.chop, flags, numerator)
+      else
+        FloatNode.new(source, -1, location.chop, 0, numerator.to_f / denominator)
+      end
     end
   end
 
@@ -180,7 +201,12 @@ module Prism
     # continue to supply that API.
     def child
       deprecated("name", "name_loc")
-      name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location)
+
+      if name
+        ConstantReadNode.new(source, -1, name_loc, 0, name)
+      else
+        MissingNode.new(source, -1, location, 0)
+      end
     end
   end
 
@@ -216,7 +242,12 @@ module Prism
     # continue to supply that API.
     def child
       deprecated("name", "name_loc")
-      name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location)
+
+      if name
+        ConstantReadNode.new(source, -1, name_loc, 0, name)
+      else
+        MissingNode.new(source, -1, location, 0)
+      end
     end
   end
 
@@ -249,9 +280,10 @@ module Prism
       end
 
       posts.each do |param|
-        if param.is_a?(MultiTargetNode)
+        case param
+        when MultiTargetNode
           names << [:req]
-        elsif param.is_a?(NoKeywordsParameterNode)
+        when NoKeywordsParameterNode, KeywordRestParameterNode, ForwardingParameterNode
           # Invalid syntax, e.g. "def f(**nil, ...)" moves the NoKeywordsParameterNode to posts
           raise "Invalid syntax"
         else
@@ -428,4 +460,49 @@ module Prism
       binary_operator_loc
     end
   end
+
+  class CaseMatchNode < Node
+    # Returns the else clause of the case match node. This method is deprecated
+    # in favor of #else_clause.
+    def consequent
+      deprecated("else_clause")
+      else_clause
+    end
+  end
+
+  class CaseNode < Node
+    # Returns the else clause of the case node. This method is deprecated in
+    # favor of #else_clause.
+    def consequent
+      deprecated("else_clause")
+      else_clause
+    end
+  end
+
+  class IfNode < Node
+    # Returns the subsequent if/elsif/else clause of the if node. This method is
+    # deprecated in favor of #subsequent.
+    def consequent
+      deprecated("subsequent")
+      subsequent
+    end
+  end
+
+  class RescueNode < Node
+    # Returns the subsequent rescue clause of the rescue node. This method is
+    # deprecated in favor of #subsequent.
+    def consequent
+      deprecated("subsequent")
+      subsequent
+    end
+  end
+
+  class UnlessNode < Node
+    # Returns the else clause of the unless node. This method is deprecated in
+    # favor of #else_clause.
+    def consequent
+      deprecated("else_clause")
+      else_clause
+    end
+  end
 end

data/lib/prism/parse_result/comments.rb

@@ -183,12 +183,5 @@ module Prism
         [preceding, NodeTarget.new(node), following]
       end
     end
-
-    private_constant :Comments
-
-    # Attach the list of comments to their respective locations in the tree.
-    def attach_comments!
-      Comments.new(self).attach! # steep:ignore
-    end
   end
 end

data/lib/prism/parse_result/errors.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Prism
+  class ParseResult < Result
+    # An object to represent the set of errors on a parse result. This object
+    # can be used to format the errors in a human-readable way.
+    class Errors
+      # The parse result that contains the errors.
+      attr_reader :parse_result
+
+      # Initialize a new set of errors from the given parse result.
+      def initialize(parse_result)
+        @parse_result = parse_result
+      end
+
+      # Formats the errors in a human-readable way and return them as a string.
+      def format
+        error_lines = {} #: Hash[Integer, Array[ParseError]]
+        parse_result.errors.each do |error|
+          location = error.location
+          (location.start_line..location.end_line).each do |line|
+            error_lines[line] ||= []
+            error_lines[line] << error
+          end
+        end
+
+        source_lines = parse_result.source.source.lines
+        source_lines << "" if error_lines.key?(source_lines.size + 1)
+
+        io = StringIO.new
+        source_lines.each.with_index(1) do |line, line_number|
+          io.puts(line)
+
+          (error_lines.delete(line_number) || []).each do |error|
+            location = error.location
+
+            case line_number
+            when location.start_line
+              io.print(" " * location.start_column + "^")
+
+              if location.start_line == location.end_line
+                if location.start_column != location.end_column
+                  io.print("~" * (location.end_column - location.start_column - 1))
+                end
+
+                io.puts(" " + error.message)
+              else
+                io.puts("~" * (line.bytesize - location.start_column))
+              end
+            when location.end_line
+              io.puts("~" * location.end_column + " " + error.message)
+            else
+              io.puts("~" * line.bytesize)
+            end
+          end
+        end
+
+        io.puts
+        io.string
+      end
+    end
+  end
+end

data/lib/prism/parse_result/newlines.rb

@@ -17,21 +17,27 @@ module Prism
     # Note that the logic in this file should be kept in sync with the Java
     # MarkNewlinesVisitor, since that visitor is responsible for marking the
     # newlines for JRuby/TruffleRuby.
+    #
+    # This file is autoloaded only when `mark_newlines!` is called, so the
+    # re-opening of the various nodes in this file will only be performed in
+    # that case. We do that to avoid storing the extra `@newline` instance
+    # variable on every node if we don't need it.
     class Newlines < Visitor
       # Create a new Newlines visitor with the given newline offsets.
-      def initialize(newline_marked)
-        @newline_marked = newline_marked
+      def initialize(lines)
+        # @type var lines: Integer
+        @lines = Array.new(1 + lines, false)
       end
 
       # Permit block/lambda nodes to mark newlines within themselves.
       def visit_block_node(node)
-        old_newline_marked = @newline_marked
-        @newline_marked = Array.new(old_newline_marked.size, false)
+        old_lines = @lines
+        @lines = Array.new(old_lines.size, false)
 
         begin
           super(node)
         ensure
-          @newline_marked = old_newline_marked
+          @lines = old_lines
         end
       end
 
@@ -39,7 +45,7 @@ module Prism
 
       # Mark if/unless nodes as newlines.
       def visit_if_node(node)
-        node.set_newline_flag(@newline_marked)
+        node.newline_flag!(@lines)
         super(node)
       end
 
@@ -48,17 +54,101 @@ module Prism
       # Permit statements lists to mark newlines within themselves.
       def visit_statements_node(node)
         node.body.each do |child|
-          child.set_newline_flag(@newline_marked)
+          child.newline_flag!(@lines)
         end
         super(node)
       end
     end
+  end
+
+  class Node
+    def newline_flag? # :nodoc:
+      @newline_flag ? true : false
+    end
+
+    def newline_flag!(lines) # :nodoc:
+      line = location.start_line
+      unless lines[line]
+        lines[line] = true
+        @newline_flag = true
+      end
+    end
+  end
+
+  class BeginNode < Node
+    def newline_flag!(lines) # :nodoc:
+      # Never mark BeginNode with a newline flag, mark children instead.
+    end
+  end
+
+  class ParenthesesNode < Node
+    def newline_flag!(lines) # :nodoc:
+      # Never mark ParenthesesNode with a newline flag, mark children instead.
+    end
+  end
+
+  class IfNode < Node
+    def newline_flag!(lines) # :nodoc:
+      predicate.newline_flag!(lines)
+    end
+  end
+
+  class UnlessNode < Node
+    def newline_flag!(lines) # :nodoc:
+      predicate.newline_flag!(lines)
+    end
+  end
+
+  class UntilNode < Node
+    def newline_flag!(lines) # :nodoc:
+      predicate.newline_flag!(lines)
+    end
+  end
+
+  class WhileNode < Node
+    def newline_flag!(lines) # :nodoc:
+      predicate.newline_flag!(lines)
+    end
+  end
+
+  class RescueModifierNode < Node
+    def newline_flag!(lines) # :nodoc:
+      expression.newline_flag!(lines)
+    end
+  end
+
+  class InterpolatedMatchLastLineNode < Node
+    def newline_flag!(lines) # :nodoc:
+      first = parts.first
+      first.newline_flag!(lines) if first
+    end
+  end
+
+  class InterpolatedRegularExpressionNode < Node
+    def newline_flag!(lines) # :nodoc:
+      first = parts.first
+      first.newline_flag!(lines) if first
+    end
+  end
+
+  class InterpolatedStringNode < Node
+    def newline_flag!(lines) # :nodoc:
+      first = parts.first
+      first.newline_flag!(lines) if first
+    end
+  end
 
-    private_constant :Newlines
+  class InterpolatedSymbolNode < Node
+    def newline_flag!(lines) # :nodoc:
+      first = parts.first
+      first.newline_flag!(lines) if first
+    end
+  end
 
-    # Walk the tree and mark nodes that are on a new line.
-    def mark_newlines!
-      value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore
+  class InterpolatedXStringNode < Node
+    def newline_flag!(lines) # :nodoc:
+      first = parts.first
+      first.newline_flag!(lines) if first
     end
   end
 end

data/lib/prism/parse_result.rb

@@ -10,7 +10,26 @@ module Prism
     # specialized and more performant `ASCIISource` if no multibyte characters
     # are present in the source code.
     def self.for(source, start_line = 1, offsets = [])
-      source.ascii_only? ? ASCIISource.new(source, start_line, offsets): new(source, start_line, offsets)
+      if source.ascii_only?
+        ASCIISource.new(source, start_line, offsets)
+      elsif source.encoding == Encoding::BINARY
+        source.force_encoding(Encoding::UTF_8)
+
+        if source.valid_encoding?
+          new(source, start_line, offsets)
+        else
+          # This is an extremely niche use case where the file is marked as
+          # binary, contains multi-byte characters, and those characters are not
+          # valid UTF-8. In this case we'll mark it as binary and fall back to
+          # treating everything as a single-byte character. This _may_ cause
+          # problems when asking for code units, but it appears to be the
+          # cleanest solution at the moment.
+          source.force_encoding(Encoding::BINARY)
+          ASCIISource.new(source, start_line, offsets)
+        end
+      else
+        new(source, start_line, offsets)
+      end
     end
 
     # The source code that this source object represents.
@@ -85,9 +104,26 @@ module Prism
     # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the
     # concept of code units that differs from the number of characters in other
     # encodings, it is not captured here.
+    #
+    # We purposefully replace invalid and undefined characters with replacement
+    # characters in this conversion. This happens for two reasons. First, it's
+    # possible that the given byte offset will not occur on a character
+    # boundary. Second, it's possible that the source code will contain a
+    # character that has no equivalent in the given encoding.
     def code_units_offset(byte_offset, encoding)
-      byteslice = (source.byteslice(0, byte_offset) or raise).encode(encoding)
-      (encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE) ? (byteslice.bytesize / 2) : byteslice.length
+      byteslice = (source.byteslice(0, byte_offset) or raise).encode(encoding, invalid: :replace, undef: :replace)
+
+      if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
+        byteslice.bytesize / 2
+      else
+        byteslice.length
+      end
+    end
+
+    # Generate a cache that targets a specific encoding for calculating code
+    # unit offsets.
+    def code_units_cache(encoding)
+      CodeUnitsCache.new(source, encoding)
     end
 
     # Returns the column number in code units for the given encoding for the
@@ -119,10 +155,84 @@ module Prism
     end
   end
 
+  # A cache that can be used to quickly compute code unit offsets from byte
+  # offsets. It purposefully provides only a single #[] method to access the
+  # cache in order to minimize surface area.
+  #
+  # Note that there are some known issues here that may or may not be addressed
+  # in the future:
+  #
+  # * The first is that there are issues when the cache computes values that are
+  #   not on character boundaries. This can result in subsequent computations
+  #   being off by one or more code units.
+  # * The second is that this cache is currently unbounded. In theory we could
+  #   introduce some kind of LRU cache to limit the number of entries, but this
+  #   has not yet been implemented.
+  #
+  class CodeUnitsCache
+    class UTF16Counter # :nodoc:
+      def initialize(source, encoding)
+        @source = source
+        @encoding = encoding
+      end
+
+      def count(byte_offset, byte_length)
+        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).bytesize / 2
+      end
+    end
+
+    class LengthCounter # :nodoc:
+      def initialize(source, encoding)
+        @source = source
+        @encoding = encoding
+      end
+
+      def count(byte_offset, byte_length)
+        @source.byteslice(byte_offset, byte_length).encode(@encoding, invalid: :replace, undef: :replace).length
+      end
+    end
+
+    private_constant :UTF16Counter, :LengthCounter
+
+    # Initialize a new cache with the given source and encoding.
+    def initialize(source, encoding)
+      @source = source
+      @counter =
+        if encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE
+          UTF16Counter.new(source, encoding)
+        else
+          LengthCounter.new(source, encoding)
+        end
+
+      @cache = {} #: Hash[Integer, Integer]
+      @offsets = [] #: Array[Integer]
+    end
+
+    # Retrieve the code units offset from the given byte offset.
+    def [](byte_offset)
+      @cache[byte_offset] ||=
+        if (index = @offsets.bsearch_index { |offset| offset > byte_offset }).nil?
+          @offsets << byte_offset
+          @counter.count(0, byte_offset)
+        elsif index == 0
+          @offsets.unshift(byte_offset)
+          @counter.count(0, byte_offset)
+        else
+          @offsets.insert(index, byte_offset)
+          offset = @offsets[index - 1]
+          @cache[offset] + @counter.count(offset, byte_offset - offset)
+        end
+    end
+  end
+
   # Specialized version of Prism::Source for source code that includes ASCII
   # characters only. This class is used to apply performance optimizations that
-  # cannot be applied to sources that include multibyte characters. Sources that
-  # include multibyte characters are represented by the Prism::Source class.
+  # cannot be applied to sources that include multibyte characters.
+  #
+  # In the extremely rare case that a source includes multi-byte characters but
+  # is marked as binary because of a magic encoding comment and it cannot be
+  # eagerly converted to UTF-8, this class will be used as well. This is because
+  # at that point we will treat everything as single-byte characters.
   class ASCIISource < Source
     # Return the character offset for the given byte offset.
     def character_offset(byte_offset)
@@ -144,9 +254,16 @@ module Prism
       byte_offset
     end
 
+    # Returns a cache that is the identity function in order to maintain the
+    # same interface. We can do this because code units are always equivalent to
+    # byte offsets for ASCII-only sources.
+    def code_units_cache(encoding)
+      ->(byte_offset) { byte_offset }
+    end
+
     # Specialized version of `code_units_column` that does not depend on
     # `code_units_offset`, which is a more expensive operation. This is
-    # essentialy the same as `Prism::Source#column`.
+    # essentially the same as `Prism::Source#column`.
     def code_units_column(byte_offset, encoding)
       byte_offset - line_start(byte_offset)
     end
@@ -253,6 +370,12 @@ module Prism
       source.code_units_offset(start_offset, encoding)
     end
 
+    # The start offset from the start of the file in code units using the given
+    # cache to fetch or calculate the value.
+    def cached_start_code_units_offset(cache)
+      cache[start_offset]
+    end
+
     # The byte offset from the beginning of the source where this location ends.
     def end_offset
       start_offset + length
@@ -269,6 +392,12 @@ module Prism
       source.code_units_offset(end_offset, encoding)
     end
 
+    # The end offset from the start of the file in code units using the given
+    # cache to fetch or calculate the value.
+    def cached_end_code_units_offset(cache)
+      cache[end_offset]
+    end
+
     # The line number where this location starts.
     def start_line
       source.line(start_offset)
@@ -303,6 +432,12 @@ module Prism
       source.code_units_column(start_offset, encoding)
     end
 
+    # The start column in code units using the given cache to fetch or calculate
+    # the value.
+    def cached_start_code_units_column(cache)
+      cache[start_offset] - cache[source.line_start(start_offset)]
+    end
+
     # The column number in bytes where this location ends from the start of the
     # line.
     def end_column
@@ -321,6 +456,12 @@ module Prism
       source.code_units_column(end_offset, encoding)
     end
 
+    # The end column in code units using the given cache to fetch or calculate
+    # the value.
+    def cached_end_code_units_column(cache)
+      cache[end_offset] - cache[source.line_start(end_offset)]
+    end
+
     # Implement the hash pattern matching interface for Location.
     def deconstruct_keys(keys)
       { start_offset: start_offset, end_offset: end_offset }
@@ -570,10 +711,23 @@ module Prism
     def failure?
       !success?
     end
+
+    # Create a code units cache for the given encoding.
+    def code_units_cache(encoding)
+      source.code_units_cache(encoding)
+    end
   end
 
   # This is a result specific to the `parse` and `parse_file` methods.
   class ParseResult < Result
+    autoload :Comments, "prism/parse_result/comments"
+    autoload :Errors, "prism/parse_result/errors"
+    autoload :Newlines, "prism/parse_result/newlines"
+
+    private_constant :Comments
+    private_constant :Errors
+    private_constant :Newlines
+
     # The syntax tree that was parsed from the source code.
     attr_reader :value
 
@@ -587,6 +741,23 @@ module Prism
     def deconstruct_keys(keys)
       super.merge!(value: value)
     end
+
+    # Attach the list of comments to their respective locations in the tree.
+    def attach_comments!
+      Comments.new(self).attach! # steep:ignore
+    end
+
+    # Walk the tree and mark nodes that are on a new line, loosely emulating
+    # the behavior of CRuby's `:line` tracepoint event.
+    def mark_newlines!
+      value.accept(Newlines.new(source.offsets.size)) # steep:ignore
+    end
+
+    # Returns a string representation of the syntax tree with the errors
+    # displayed inline.
+    def errors_format
+      Errors.new(self).format
+    end
   end
 
   # This is a result specific to the `lex` and `lex_file` methods.
@@ -677,5 +848,11 @@ module Prism
         other.type == type &&
         other.value == value
     end
+
+    # Returns a string representation of this token.
+    def inspect
+      location
+      super
+    end
   end
 end