prism 1.5.1 → 1.9.0

data/lib/prism/parse_result.rb

@@ -76,6 +76,15 @@ module Prism
       source.byteslice(byte_offset, length) or raise
     end
 
+    # Converts the line number and column in bytes to a byte offset.
+    def byte_offset(line, column)
+      normal = line - @start_line
+      raise IndexError if normal < 0
+      offsets.fetch(normal) + column
+    rescue IndexError
+      raise ArgumentError, "line #{line} is out of range"
+    end
+
     # Binary search through the offsets to find the line number for the given
     # byte offset.
     def line(byte_offset)
@@ -155,21 +164,8 @@ module Prism
     # Binary search through the offsets to find the line number for the given
     # byte offset.
     def find_line(byte_offset)
-      left = 0
-      right = offsets.length - 1
-
-      while left <= right
-        mid = left + (right - left) / 2
-        return mid if (offset = offsets[mid]) == byte_offset
-
-        if offset < byte_offset
-          left = mid + 1
-        else
-          right = mid - 1
-        end
-      end
-
-      left - 1
+      index = offsets.bsearch_index { |offset| offset > byte_offset } || offsets.length
+      index - 1
     end
   end
 

data/lib/prism/polyfill/scan_byte.rb

@@ -3,7 +3,7 @@
 require "strscan"
 
 # Polyfill for StringScanner#scan_byte, which didn't exist until Ruby 3.4.
-if !(StringScanner.instance_methods.include?(:scan_byte))
+if !(StringScanner.method_defined?(:scan_byte))
   StringScanner.include(
     Module.new {
       def scan_byte # :nodoc:

data/lib/prism/polyfill/warn.rb

@@ -7,17 +7,14 @@ if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.pa
   Kernel.prepend(
     Module.new {
       def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
-        uplevel =
-          case uplevel
-          when nil
-            1
-          when Integer
-            uplevel + 1
-          else
-            uplevel.to_int + 1
-          end
-
-        super(*msgs, uplevel: uplevel)
+        case uplevel
+        when nil
+          super(*msgs)
+        when Integer
+          super(*msgs, uplevel: uplevel + 1)
+        else
+          super(*msgs, uplevel: uplevel.to_int + 1)
+        end
       end
     }
   )
@@ -25,17 +22,14 @@ if (method = Kernel.instance_method(:warn)).respond_to?(:parameters) ? method.pa
   Object.prepend(
     Module.new {
       def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
-        uplevel =
-          case uplevel
-          when nil
-            1
-          when Integer
-            uplevel + 1
-          else
-            uplevel.to_int + 1
-          end
-
-        super(*msgs, uplevel: uplevel)
+        case uplevel
+        when nil
+          super(*msgs)
+        when Integer
+          super(*msgs, uplevel: uplevel + 1)
+        else
+          super(*msgs, uplevel: uplevel.to_int + 1)
+        end
       end
     }
   )

data/lib/prism/reflection.rb

@@ -143,7 +143,7 @@ module Prism
       when :call_and_write_node
         [FlagsField.new(:flags, [:safe_navigation?, :variable_call?, :attribute_write?, :ignore_visibility?]), OptionalNodeField.new(:receiver), OptionalLocationField.new(:call_operator_loc), OptionalLocationField.new(:message_loc), ConstantField.new(:read_name), ConstantField.new(:write_name), LocationField.new(:operator_loc), NodeField.new(:value)]
       when :call_node
-        [FlagsField.new(:flags, [:safe_navigation?, :variable_call?, :attribute_write?, :ignore_visibility?]), OptionalNodeField.new(:receiver), OptionalLocationField.new(:call_operator_loc), ConstantField.new(:name), OptionalLocationField.new(:message_loc), OptionalLocationField.new(:opening_loc), OptionalNodeField.new(:arguments), OptionalLocationField.new(:closing_loc), OptionalNodeField.new(:block)]
+        [FlagsField.new(:flags, [:safe_navigation?, :variable_call?, :attribute_write?, :ignore_visibility?]), OptionalNodeField.new(:receiver), OptionalLocationField.new(:call_operator_loc), ConstantField.new(:name), OptionalLocationField.new(:message_loc), OptionalLocationField.new(:opening_loc), OptionalNodeField.new(:arguments), OptionalLocationField.new(:closing_loc), OptionalLocationField.new(:equal_loc), OptionalNodeField.new(:block)]
       when :call_operator_write_node
         [FlagsField.new(:flags, [:safe_navigation?, :variable_call?, :attribute_write?, :ignore_visibility?]), OptionalNodeField.new(:receiver), OptionalLocationField.new(:call_operator_loc), OptionalLocationField.new(:message_loc), ConstantField.new(:read_name), ConstantField.new(:write_name), ConstantField.new(:binary_operator), LocationField.new(:binary_operator_loc), NodeField.new(:value)]
       when :call_or_write_node

data/lib/prism/serialize.rb

@@ -21,11 +21,11 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 5
+    MINOR_VERSION = 9
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
-    PATCH_VERSION = 1
+    PATCH_VERSION = 0
 
     # Deserialize the dumped output from a request to parse or parse_file.
     #
@@ -397,6 +397,7 @@ module Prism
         :conditional_while_predicate,
         :constant_path_colon_colon_constant,
         :def_endless,
+        :def_endless_parameters,
         :def_endless_setter,
         :def_name,
         :def_params_term,
@@ -555,6 +556,7 @@ module Prism
         :parameter_wild_loose_comma,
         :pattern_array_multiple_rests,
         :pattern_capture_duplicate,
+        :pattern_capture_in_alternative,
         :pattern_expression_after_bracket,
         :pattern_expression_after_comma,
         :pattern_expression_after_hrocket,
@@ -616,6 +618,7 @@ module Prism
         :unexpected_index_keywords,
         :unexpected_label,
         :unexpected_multi_write,
+        :unexpected_parameter_default_value,
         :unexpected_range_operator,
         :unexpected_safe_navigation,
         :unexpected_token_close_context,
@@ -878,7 +881,7 @@ module Prism
           when 18 then
             CallAndWriteNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_constant(constant_pool, encoding), load_location(freeze), load_node(constant_pool, encoding, freeze))
           when 19 then
-            CallNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze))
+            CallNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze))
           when 20 then
             CallOperatorWriteNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_constant(constant_pool, encoding), load_constant(constant_pool, encoding), load_location(freeze), load_node(constant_pool, encoding, freeze))
           when 21 then
@@ -1286,7 +1289,7 @@ module Prism
             -> (constant_pool, encoding, freeze) {
               node_id = load_varuint
               location = load_location(freeze)
-              value = CallNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze))
+              value = CallNode.new(source, node_id, location, load_varuint, load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_constant(constant_pool, encoding), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze), load_optional_location(freeze), load_optional_location(freeze), load_optional_node(constant_pool, encoding, freeze))
               value.freeze if freeze
               value
             },
@@ -2238,6 +2241,7 @@ module Prism
       :KEYWORD_WHEN,
       :NEWLINE,
       :PARENTHESIS_RIGHT,
+      :PIPE,
       :SEMICOLON,
       :AMPERSAND,
       :AMPERSAND_AMPERSAND,
@@ -2354,7 +2358,6 @@ module Prism
       :PERCENT_LOWER_X,
       :PERCENT_UPPER_I,
       :PERCENT_UPPER_W,
-      :PIPE,
       :PIPE_EQUAL,
       :PIPE_PIPE,
       :PIPE_PIPE_EQUAL,

data/lib/prism/translation/parser/compiler.rb

@@ -217,7 +217,7 @@ module Prism
                 rescue_clause.exceptions.any? ? builder.array(nil, visit_all(rescue_clause.exceptions), nil) : nil,
                 token(rescue_clause.operator_loc),
                 visit(rescue_clause.reference),
-                srange_find(find_start_offset, find_end_offset, ";"),
+                srange_semicolon(find_start_offset, find_end_offset),
                 visit(rescue_clause.statements)
               )
             end until (rescue_clause = rescue_clause.subsequent).nil?
@@ -323,7 +323,7 @@ module Prism
                       visit_all(arguments),
                       token(node.closing_loc),
                     ),
-                    srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, "="),
+                    token(node.equal_loc),
                     visit(node.arguments.arguments.last)
                   ),
                   block
@@ -340,7 +340,7 @@ module Prism
             if name.end_with?("=") && !message_loc.slice.end_with?("=") && node.arguments && block.nil?
               builder.assign(
                 builder.attr_asgn(visit(node.receiver), call_operator, token(message_loc)),
-                srange_find(message_loc.end_offset, node.arguments.location.start_offset, "="),
+                token(node.equal_loc),
                 visit(node.arguments.arguments.last)
               )
             else
@@ -789,7 +789,7 @@ module Prism
             if (do_keyword_loc = node.do_keyword_loc)
               token(do_keyword_loc)
             else
-              srange_find(node.collection.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset, ";")
+              srange_semicolon(node.collection.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset)
             end,
             visit(node.statements),
             token(node.end_keyword_loc)
@@ -921,7 +921,7 @@ module Prism
               if (then_keyword_loc = node.then_keyword_loc)
                 token(then_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.subsequent&.location || node.end_keyword_loc).start_offset, ";")
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.subsequent&.location || node.end_keyword_loc).start_offset)
               end,
               visit(node.statements),
               case node.subsequent
@@ -987,7 +987,7 @@ module Prism
             if (then_loc = node.then_loc)
               token(then_loc)
             else
-              srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, ";")
+              srange_semicolon(node.pattern.location.end_offset, node.statements&.location&.start_offset)
             end,
             visit(node.statements)
           )
@@ -1808,7 +1808,7 @@ module Prism
               if (then_keyword_loc = node.then_keyword_loc)
                 token(then_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.else_clause&.location || node.end_keyword_loc).start_offset, ";")
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.else_clause&.location || node.end_keyword_loc).start_offset)
               end,
               visit(node.else_clause),
               token(node.else_clause&.else_keyword_loc),
@@ -1839,7 +1839,7 @@ module Prism
               if (do_keyword_loc = node.do_keyword_loc)
                 token(do_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, ";")
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset)
               end,
               visit(node.statements),
               token(node.closing_loc)
@@ -1863,7 +1863,7 @@ module Prism
             if (then_keyword_loc = node.then_keyword_loc)
               token(then_keyword_loc)
             else
-              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, ";")
+              srange_semicolon(node.conditions.last.location.end_offset, node.statements&.location&.start_offset)
             end,
             visit(node.statements)
           )
@@ -1883,7 +1883,7 @@ module Prism
               if (do_keyword_loc = node.do_keyword_loc)
                 token(do_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, ";")
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset)
               end,
               visit(node.statements),
               token(node.closing_loc)
@@ -2012,16 +2012,16 @@ module Prism
           Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])
         end
 
-        # Constructs a new source range by finding the given character between
-        # the given start offset and end offset. If the needle is not found, it
-        # returns nil. Importantly it does not search past newlines or comments.
+        # Constructs a new source range by finding a semicolon between the given
+        # start offset and end offset. If the semicolon is not found, it returns
+        # nil. Importantly it does not search past newlines or comments.
         #
         # Note that end_offset is allowed to be nil, in which case this will
         # search until the end of the string.
-        def srange_find(start_offset, end_offset, character)
-          if (match = source_buffer.source.byteslice(start_offset...end_offset)[/\A\s*#{character}/])
+        def srange_semicolon(start_offset, end_offset)
+          if (match = source_buffer.source.byteslice(start_offset...end_offset)[/\A\s*;/])
             final_offset = start_offset + match.bytesize
-            [character, Range.new(source_buffer, offset_cache[final_offset - character.bytesize], offset_cache[final_offset])]
+            [";", Range.new(source_buffer, offset_cache[final_offset - 1], offset_cache[final_offset])]
           end
         end
 

data/lib/prism/translation/parser.rb

@@ -19,6 +19,13 @@ module Prism
     # whitequark/parser gem's syntax tree. It inherits from the base parser for
     # the parser gem, and overrides the parse* methods to parse with prism and
     # then translate.
+    #
+    # Note that this version of the parser always parses using the latest
+    # version of Ruby syntax supported by Prism. If you want specific version
+    # support, use one of the version-specific subclasses, such as
+    # `Prism::Translation::Parser34`. If you want to parse using the same
+    # version of Ruby syntax as the currently running version of Ruby, use
+    # `Prism::Translation::ParserCurrent`.
     class Parser < ::Parser::Base
       Diagnostic = ::Parser::Diagnostic # :nodoc:
       private_constant :Diagnostic
@@ -77,7 +84,7 @@ module Prism
       end
 
       def version # :nodoc:
-        34
+        41
       end
 
       # The default encoding for Ruby files is UTF-8.
@@ -349,8 +356,10 @@ module Prism
           "3.3.1"
         when 34
           "3.4.0"
-        when 35
-          "3.5.0"
+        when 35, 40
+          "4.0.0"
+        when 41
+          "4.1.0"
         else
           "latest"
         end

data/lib/prism/translation/parser_current.rb

@@ -10,11 +10,13 @@ module Prism
       ParserCurrent = Parser33
     when /^3\.4\./
       ParserCurrent = Parser34
-    when /^3\.5\./
-      ParserCurrent = Parser35
+    when /^3\.5\./, /^4\.0\./
+      ParserCurrent = Parser40
+    when /^4\.1\./
+      ParserCurrent = Parser41
     else
       # Keep this in sync with released Ruby.
-      parser = Parser34
+      parser = Parser40
       major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
       warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
            "but you are running #{major}.#{minor}."

data/lib/prism/translation/parser_versions.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+module Prism
+  module Translation
+    # This class is the entry-point for Ruby 3.3 of `Prism::Translation::Parser`.
+    class Parser33 < Parser
+      def version # :nodoc:
+        33
+      end
+    end
+
+    # This class is the entry-point for Ruby 3.4 of `Prism::Translation::Parser`.
+    class Parser34 < Parser
+      def version # :nodoc:
+        34
+      end
+    end
+
+    # This class is the entry-point for Ruby 4.0 of `Prism::Translation::Parser`.
+    class Parser40 < Parser
+      def version # :nodoc:
+        40
+      end
+    end
+
+    Parser35 = Parser40 # :nodoc:
+
+    # This class is the entry-point for Ruby 4.1 of `Prism::Translation::Parser`.
+    class Parser41 < Parser
+      def version # :nodoc:
+        41
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/filter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Prism
+  module Translation
+    class Ripper
+      class Filter # :nodoc:
+        # :stopdoc:
+        def initialize(src, filename = '-', lineno = 1)
+          @__lexer = Lexer.new(src, filename, lineno)
+          @__line = nil
+          @__col = nil
+          @__state = nil
+        end
+
+        def filename
+          @__lexer.filename
+        end
+
+        def lineno
+          @__line
+        end
+
+        def column
+          @__col
+        end
+
+        def state
+          @__state
+        end
+
+        def parse(init = nil)
+          data = init
+          @__lexer.lex.each do |pos, event, tok, state|
+            @__line, @__col = *pos
+            @__state = state
+            data = if respond_to?(event, true)
+                  then __send__(event, tok, data)
+                  else on_default(event, tok, data)
+                  end
+          end
+          data
+        end
+
+        private
+
+        def on_default(event, token, data)
+          data
+        end
+        # :startdoc:
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/lexer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+require_relative "../ripper"
+
+module Prism
+  module Translation
+    class Ripper
+      class Lexer < Ripper # :nodoc:
+        # :stopdoc:
+        class State
+
+          attr_reader :to_int, :to_s
+
+          def initialize(i)
+            @to_int = i
+            @to_s = Ripper.lex_state_name(i)
+            freeze
+          end
+
+          def [](index)
+            case index
+            when 0, :to_int
+              @to_int
+            when 1, :to_s
+              @to_s
+            else
+              nil
+            end
+          end
+
+          alias to_i to_int
+          alias inspect to_s
+          def pretty_print(q) q.text(to_s) end
+          def ==(i) super or to_int == i end
+          def &(i) self.class.new(to_int & i) end
+          def |(i) self.class.new(to_int | i) end
+          def allbits?(i) to_int.allbits?(i) end
+          def anybits?(i) to_int.anybits?(i) end
+          def nobits?(i) to_int.nobits?(i) end
+
+          # Instances are frozen and there are only a handful of them so we cache them here.
+          STATES = Hash.new { |h,k| h[k] = State.new(k) }
+
+          def self.cached(i)
+            STATES[i]
+          end
+        end
+
+        class Elem
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State.cached(state)
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        # Pretty much just the same as Prism.lex_compat.
+        def lex(raise_errors: false)
+          Ripper.lex(@source, filename, lineno, raise_errors: raise_errors)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(...)
+          lex(...).map do |position, event, token, state|
+            Elem.new(position, event, token, state.to_int)
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
+        end
+
+        # :startdoc:
+      end
+    end
+  end
+end