prism 0.21.0 → 0.23.0

data/lib/prism/node_ext.rb

@@ -94,7 +94,7 @@ module Prism
 
     # Returns the full name of this constant. For example: "Foo"
     def full_name
-      name.name
+      name.to_s
     end
   end
 
@@ -135,7 +135,17 @@ module Prism
     # Returns the list of parts for the full name of this constant path.
     # For example: [:Foo, :Bar]
     def full_name_parts
-      (parent&.full_name_parts || [:""]).push(child.name)
+      parts = case parent
+      when ConstantPathNode, ConstantReadNode
+        parent.full_name_parts
+      when nil
+        [:""]
+      else
+        raise ConstantPathNode::DynamicPartsInConstantPathError,
+          "Constant path target contains dynamic parts. Cannot compute full name"
+      end
+
+      parts.push(child.name)
     end
 
     # Returns the full name of this constant path. For example: "Foo::Bar"
@@ -144,6 +154,19 @@ module Prism
     end
   end
 
+  class ConstantTargetNode < Node
+    # Returns the list of parts for the full name of this constant.
+    # For example: [:Foo]
+    def full_name_parts
+      [name]
+    end
+
+    # Returns the full name of this constant. For example: "Foo"
+    def full_name
+      name.to_s
+    end
+  end
+
   class ParametersNode < Node
     # Mirrors the Method#parameters method.
     def signature

data/lib/prism/parse_result.rb

@@ -9,18 +9,16 @@ module Prism
     attr_reader :source
 
     # The line number where this source starts.
-    attr_accessor :start_line
+    attr_reader :start_line
 
     # The list of newline byte offsets in the source code.
     attr_reader :offsets
 
-    # Create a new source object with the given source code and newline byte
-    # offsets. If no newline byte offsets are given, they will be computed from
-    # the source code.
-    def initialize(source, start_line = 1, offsets = compute_offsets(source))
+    # Create a new source object with the given source code.
+    def initialize(source, start_line = 1, offsets = [])
       @source = source
-      @start_line = start_line
-      @offsets = offsets
+      @start_line = start_line # set after parsing is done
+      @offsets = offsets # set after parsing is done
     end
 
     # Perform a byteslice on the source code using the given byte offset and
@@ -56,6 +54,23 @@ module Prism
       character_offset(byte_offset) - character_offset(line_start(byte_offset))
     end
 
+    # Returns the offset from the start of the file for the given byte offset
+    # counting in code units for the given encoding.
+    #
+    # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the
+    # concept of code units that differs from the number of characters in other
+    # encodings, it is not captured here.
+    def code_units_offset(byte_offset, encoding)
+      byteslice = source.byteslice(0, byte_offset).encode(encoding)
+      (encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE) ? (byteslice.bytesize / 2) : byteslice.length
+    end
+
+    # Returns the column number in code units for the given encoding for the
+    # given byte offset.
+    def code_units_column(byte_offset, encoding)
+      code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
+    end
+
     private
 
     # Binary search through the offsets to find the line number for the given
@@ -77,21 +92,14 @@ module Prism
 
       left - 1
     end
-
-    # Find all of the newlines in the source code and return their byte offsets
-    # from the start of the string an array.
-    def compute_offsets(code)
-      offsets = [0]
-      code.b.scan("\n") { offsets << $~.end(0) }
-      offsets
-    end
   end
 
   # This represents a location in the source.
   class Location
     # A Source object that is used to determine more information from the given
     # offset and length.
-    protected attr_reader :source
+    attr_reader :source
+    protected :source
 
     # The byte offset from the beginning of the source where this location
     # starts.
@@ -137,6 +145,11 @@ module Prism
       source.character_offset(start_offset)
     end
 
+    # The offset from the start of the file in code units of the given encoding.
+    def start_code_units_offset(encoding = Encoding::UTF_16LE)
+      source.code_units_offset(start_offset, encoding)
+    end
+
     # The byte offset from the beginning of the source where this location ends.
     def end_offset
       start_offset + length
@@ -148,6 +161,11 @@ module Prism
       source.character_offset(end_offset)
     end
 
+    # The offset from the start of the file in code units of the given encoding.
+    def end_code_units_offset(encoding = Encoding::UTF_16LE)
+      source.code_units_offset(end_offset, encoding)
+    end
+
     # The line number where this location starts.
     def start_line
       source.line(start_offset)
@@ -176,6 +194,12 @@ module Prism
       source.character_column(start_offset)
     end
 
+    # The column number in code units of the given encoding where this location
+    # starts from the start of the line.
+    def start_code_units_column(encoding = Encoding::UTF_16LE)
+      source.code_units_column(start_offset, encoding)
+    end
+
     # The column number in bytes where this location ends from the start of the
     # line.
     def end_column
@@ -188,6 +212,12 @@ module Prism
       source.character_column(end_offset)
     end
 
+    # The column number in code units of the given encoding where this location
+    # ends from the start of the line.
+    def end_code_units_column(encoding = Encoding::UTF_16LE)
+      source.code_units_column(end_offset, encoding)
+    end
+
     # Implement the hash pattern matching interface for Location.
     def deconstruct_keys(keys)
       { start_offset: start_offset, end_offset: end_offset }

data/lib/prism/serialize.rb

@@ -27,7 +27,7 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 21
+    MINOR_VERSION = 23
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
@@ -86,11 +86,15 @@ module Prism
       end
 
       def load_start_line
-        source.start_line = load_varsint
+        source.instance_variable_set :@start_line, load_varsint
+      end
+
+      def load_line_offsets
+        source.instance_variable_set :@offsets, Array.new(load_varuint) { load_varuint }
       end
 
       def load_comments
-        load_varuint.times.map do
+        Array.new(load_varuint) do
           case load_varuint
           when 0 then InlineComment.new(load_location)
           when 1 then EmbDocComment.new(load_location)
@@ -101,10 +105,10 @@ module Prism
 
       def load_metadata
         comments = load_comments
-        magic_comments = load_varuint.times.map { MagicComment.new(load_location, load_location) }
+        magic_comments = Array.new(load_varuint) { MagicComment.new(load_location, load_location) }
         data_loc = load_optional_location
-        errors = load_varuint.times.map { ParseError.new(load_embedded_string, load_location, load_error_level) }
-        warnings = load_varuint.times.map { ParseWarning.new(load_embedded_string, load_location, load_warning_level) }
+        errors = Array.new(load_varuint) { ParseError.new(load_embedded_string, load_location, load_error_level) }
+        warnings = Array.new(load_varuint) { ParseWarning.new(load_embedded_string, load_location, load_warning_level) }
         [comments, magic_comments, data_loc, errors, warnings]
       end
 
@@ -125,6 +129,7 @@ module Prism
         tokens = load_tokens
         encoding = load_encoding
         load_start_line
+        load_line_offsets
         comments, magic_comments, data_loc, errors, warnings = load_metadata
         tokens.each { |token,| token.value.force_encoding(encoding) }
 
@@ -136,6 +141,7 @@ module Prism
         load_header
         load_encoding
         load_start_line
+        load_line_offsets
 
         comments, magic_comments, data_loc, errors, warnings = load_metadata
 
@@ -244,6 +250,8 @@ module Prism
         case level
         when 0
           :fatal
+        when 1
+          :argument
         else
           raise "Unknown level: #{level}"
         end

data/lib/prism/translation/parser/compiler.rb

@@ -1062,12 +1062,22 @@ module Prism
 
         # foo in bar
         # ^^^^^^^^^^
-        def visit_match_predicate_node(node)
-          builder.match_pattern_p(
-            visit(node.value),
-            token(node.operator_loc),
-            within_pattern { |compiler| node.pattern.accept(compiler) }
-          )
+        if RUBY_VERSION >= "3.0"
+          def visit_match_predicate_node(node)
+            builder.match_pattern_p(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
+        else
+          def visit_match_predicate_node(node)
+            builder.match_pattern(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
         end
 
         # foo => bar

data/lib/prism/translation/parser.rb

@@ -68,17 +68,23 @@ module Prism
 
       # Parses a source buffer and returns the AST, the source code comments,
       # and the tokens emitted by the lexer.
-      def tokenize(source_buffer, _recover = false)
+      def tokenize(source_buffer, recover = false)
         @source_buffer = source_buffer
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache)
+        result =
+          begin
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache)
+          rescue ::Parser::SyntaxError
+            raise if !recover
+          end
 
         program, tokens = result.value
+        ast = build_ast(program, offset_cache) if result.success?
 
         [
-          build_ast(program, offset_cache),
+          ast,
           build_comments(result.comments, offset_cache),
           build_tokens(tokens, offset_cache)
         ]
@@ -118,20 +124,21 @@ module Prism
       # build the parser gem AST.
       #
       # If the bytesize of the source is the same as the length, then we can
-      # just use the offset directly. Otherwise, we build a hash that functions
-      # as a cache for the conversion.
-      #
-      # This is a good opportunity for some optimizations. If the source file
-      # has any multi-byte characters, this can tank the performance of the
-      # translator. We could make this significantly faster by using a
-      # different data structure for the cache.
+      # just use the offset directly. Otherwise, we build an array where the
+      # index is the byte offset and the value is the character offset.
       def build_offset_cache(source)
         if source.bytesize == source.length
           -> (offset) { offset }
         else
-          Hash.new do |hash, offset|
-            hash[offset] = source.byteslice(0, offset).length
+          offset_cache = []
+          offset = 0
+
+          source.each_char do |char|
+            char.bytesize.times { offset_cache << offset }
+            offset += 1
           end
+
+          offset_cache << offset
         end
       end