prism 1.7.0 → 1.9.0

data/lib/prism/parse_result.rb

@@ -76,6 +76,15 @@ module Prism
       source.byteslice(byte_offset, length) or raise
     end
 
+    # Converts the line number and column in bytes to a byte offset.
+    def byte_offset(line, column)
+      normal = line - @start_line
+      raise IndexError if normal < 0
+      offsets.fetch(normal) + column
+    rescue IndexError
+      raise ArgumentError, "line #{line} is out of range"
+    end
+
     # Binary search through the offsets to find the line number for the given
     # byte offset.
     def line(byte_offset)

data/lib/prism/serialize.rb

@@ -21,7 +21,7 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 7
+    MINOR_VERSION = 9
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.

data/lib/prism/translation/parser_current.rb

@@ -16,7 +16,7 @@ module Prism
       ParserCurrent = Parser41
     else
       # Keep this in sync with released Ruby.
-      parser = Parser34
+      parser = Parser40
       major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
       warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
            "but you are running #{major}.#{minor}."

data/lib/prism/translation/parser_versions.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+module Prism
+  module Translation
+    # This class is the entry-point for Ruby 3.3 of `Prism::Translation::Parser`.
+    class Parser33 < Parser
+      def version # :nodoc:
+        33
+      end
+    end
+
+    # This class is the entry-point for Ruby 3.4 of `Prism::Translation::Parser`.
+    class Parser34 < Parser
+      def version # :nodoc:
+        34
+      end
+    end
+
+    # This class is the entry-point for Ruby 4.0 of `Prism::Translation::Parser`.
+    class Parser40 < Parser
+      def version # :nodoc:
+        40
+      end
+    end
+
+    Parser35 = Parser40 # :nodoc:
+
+    # This class is the entry-point for Ruby 4.1 of `Prism::Translation::Parser`.
+    class Parser41 < Parser
+      def version # :nodoc:
+        41
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/filter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Prism
+  module Translation
+    class Ripper
+      class Filter # :nodoc:
+        # :stopdoc:
+        def initialize(src, filename = '-', lineno = 1)
+          @__lexer = Lexer.new(src, filename, lineno)
+          @__line = nil
+          @__col = nil
+          @__state = nil
+        end
+
+        def filename
+          @__lexer.filename
+        end
+
+        def lineno
+          @__line
+        end
+
+        def column
+          @__col
+        end
+
+        def state
+          @__state
+        end
+
+        def parse(init = nil)
+          data = init
+          @__lexer.lex.each do |pos, event, tok, state|
+            @__line, @__col = *pos
+            @__state = state
+            data = if respond_to?(event, true)
+                  then __send__(event, tok, data)
+                  else on_default(event, tok, data)
+                  end
+          end
+          data
+        end
+
+        private
+
+        def on_default(event, token, data)
+          data
+        end
+        # :startdoc:
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/lexer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+require_relative "../ripper"
+
+module Prism
+  module Translation
+    class Ripper
+      class Lexer < Ripper # :nodoc:
+        # :stopdoc:
+        class State
+
+          attr_reader :to_int, :to_s
+
+          def initialize(i)
+            @to_int = i
+            @to_s = Ripper.lex_state_name(i)
+            freeze
+          end
+
+          def [](index)
+            case index
+            when 0, :to_int
+              @to_int
+            when 1, :to_s
+              @to_s
+            else
+              nil
+            end
+          end
+
+          alias to_i to_int
+          alias inspect to_s
+          def pretty_print(q) q.text(to_s) end
+          def ==(i) super or to_int == i end
+          def &(i) self.class.new(to_int & i) end
+          def |(i) self.class.new(to_int | i) end
+          def allbits?(i) to_int.allbits?(i) end
+          def anybits?(i) to_int.anybits?(i) end
+          def nobits?(i) to_int.nobits?(i) end
+
+          # Instances are frozen and there are only a handful of them so we cache them here.
+          STATES = Hash.new { |h,k| h[k] = State.new(k) }
+
+          def self.cached(i)
+            STATES[i]
+          end
+        end
+
+        class Elem
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State.cached(state)
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        # Pretty much just the same as Prism.lex_compat.
+        def lex(raise_errors: false)
+          Ripper.lex(@source, filename, lineno, raise_errors: raise_errors)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(...)
+          lex(...).map do |position, event, token, state|
+            Elem.new(position, event, token, state.to_int)
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
+        end
+
+        # :startdoc:
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 # :markup: markdown
 
-require "ripper"
-
 module Prism
   module Translation
     # This class provides a compatibility layer between prism and Ripper. It
@@ -80,6 +78,19 @@ module Prism
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {
@@ -426,9 +437,35 @@ module Prism
         end
       end
 
+      autoload :Filter, "prism/translation/ripper/filter"
+      autoload :Lexer, "prism/translation/ripper/lexer"
       autoload :SexpBuilder, "prism/translation/ripper/sexp"
       autoload :SexpBuilderPP, "prism/translation/ripper/sexp"
 
+      # :stopdoc:
+      # This is not part of the public API but used by some gems.
+
+      # Ripper-internal bitflags.
+      LEX_STATE_NAMES = %i[
+        BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
+      ].map.with_index.to_h { |name, i| [2 ** i, name] }.freeze
+      private_constant :LEX_STATE_NAMES
+
+      LEX_STATE_NAMES.each do |value, key|
+        const_set("EXPR_#{key}", value)
+      end
+      EXPR_NONE = 0
+      EXPR_VALUE = EXPR_BEG
+      EXPR_BEG_ANY = EXPR_BEG | EXPR_MID | EXPR_CLASS
+      EXPR_ARG_ANY = EXPR_ARG | EXPR_CMDARG
+      EXPR_END_ANY = EXPR_END | EXPR_ENDARG | EXPR_ENDFN
+
+      def self.lex_state_name(state)
+        LEX_STATE_NAMES.filter_map { |flag, name| name if state & flag != 0  }.join("|")
+      end
+
+      # :startdoc:
+
       # The source that is being parsed.
       attr_reader :source
 
@@ -795,7 +832,7 @@ module Prism
       # foo(bar)
       #     ^^^
       def visit_arguments_node(node)
-        arguments, _ = visit_call_node_arguments(node, nil, false)
+        arguments, _, _ = visit_call_node_arguments(node, nil, false)
         arguments
       end
 
@@ -1005,16 +1042,16 @@ module Prism
           case node.name
           when :[]
             receiver = visit(node.receiver)
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
             bounds(node.location)
             call = on_aref(receiver, arguments)
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           when :[]=
             receiver = visit(node.receiver)
@@ -1073,9 +1110,9 @@ module Prism
             if node.variable_call?
               on_vcall(message)
             else
-              arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+              arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
               call =
-                if node.opening_loc.nil? && arguments&.any?
+                if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any?
                   bounds(node.location)
                   on_command(message, arguments)
                 elsif !node.opening_loc.nil?
@@ -1086,11 +1123,11 @@ module Prism
                   on_method_add_arg(on_fcall(message), on_args_new)
                 end
 
-              if block.nil?
-                call
-              else
+              if has_ripper_block
                 bounds(node.block.location)
                 on_method_add_block(call, block)
+              else
+                call
               end
             end
           end
@@ -1114,7 +1151,7 @@ module Prism
             bounds(node.location)
             on_assign(on_field(receiver, call_operator, message), value)
           else
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
             call =
               if node.opening_loc.nil?
                 bounds(node.location)
@@ -1132,27 +1169,35 @@ module Prism
                 on_method_add_arg(on_call(receiver, call_operator, message), arguments)
               end
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.block.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           end
         end
       end
 
-      # Visit the arguments and block of a call node and return the arguments
-      # and block as they should be used.
-      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+      # Extract the arguments and block Ripper-style, which means if the block
+      # is like `&b` then it's moved to arguments.
+      private def get_arguments_and_block(arguments_node, block_node)
         arguments = arguments_node&.arguments || []
         block = block_node
 
         if block.is_a?(BlockArgumentNode)
-          arguments << block
+          arguments += [block]
           block = nil
         end
 
+        [arguments, block]
+      end
+
+      # Visit the arguments and block of a call node and return the arguments
+      # and block as they should be used.
+      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+        arguments, block = get_arguments_and_block(arguments_node, block_node)
+
         [
           if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode)
             visit(arguments.first)
@@ -1166,7 +1211,8 @@ module Prism
               on_args_add_block(args, false)
             end
           end,
-          visit(block)
+          visit(block),
+          block != nil,
         ]
       end
 
@@ -1603,10 +1649,10 @@ module Prism
           end
 
         bounds(node.location)
-        if receiver.nil?
-          on_def(name, parameters, bodystmt)
-        else
+        if receiver
           on_defs(receiver, operator, name, parameters, bodystmt)
+        else
+          on_def(name, parameters, bodystmt)
         end
       end
 
@@ -2004,7 +2050,7 @@ module Prism
       # ^^^^^^^^^^^^^^^
       def visit_index_operator_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2021,7 +2067,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_and_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2038,7 +2084,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_or_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2055,7 +2101,7 @@ module Prism
       # ^^^^^^^^
       def visit_index_target_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         on_aref_field(receiver, arguments)
@@ -3085,7 +3131,7 @@ module Prism
       # super(foo)
       # ^^^^^^^^^^
       def visit_super_node(node)
-        arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
+        arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
 
         if !node.lparen_loc.nil?
           bounds(node.lparen_loc)
@@ -3095,11 +3141,11 @@ module Prism
         bounds(node.location)
         call = on_super(arguments)
 
-        if block.nil?
-          call
-        else
+        if has_ripper_block
           bounds(node.block.location)
           on_method_add_block(call, block)
+        else
+          call
         end
       end
 
@@ -3409,12 +3455,12 @@ module Prism
 
       # :stopdoc:
       def _dispatch_0; end
-      def _dispatch_1(_); end
-      def _dispatch_2(_, _); end
-      def _dispatch_3(_, _, _); end
-      def _dispatch_4(_, _, _, _); end
-      def _dispatch_5(_, _, _, _, _); end
-      def _dispatch_7(_, _, _, _, _, _, _); end
+      def _dispatch_1(arg); arg end
+      def _dispatch_2(arg, _); arg end
+      def _dispatch_3(arg, _, _); arg end
+      def _dispatch_4(arg, _, _, _); arg end
+      def _dispatch_5(arg, _, _, _, _); arg end
+      def _dispatch_7(arg, _, _, _, _, _, _); arg end
       # :startdoc:
 
       #

data/lib/prism/translation/ruby_parser.rb

@@ -1422,7 +1422,7 @@ module Prism
         # ```
         def visit_parameters_node(node)
           children =
-            node.compact_child_nodes.map do |element|
+            node.each_child_node.map do |element|
               if element.is_a?(MultiTargetNode)
                 visit_destructured_parameter(element)
               else

data/lib/prism/translation.rb

@@ -7,11 +7,11 @@ module Prism
   module Translation # steep:ignore
     autoload :Parser, "prism/translation/parser"
     autoload :ParserCurrent, "prism/translation/parser_current"
-    autoload :Parser33, "prism/translation/parser33"
-    autoload :Parser34, "prism/translation/parser34"
-    autoload :Parser35, "prism/translation/parser35"
-    autoload :Parser40, "prism/translation/parser40"
-    autoload :Parser41, "prism/translation/parser41"
+    autoload :Parser33, "prism/translation/parser_versions"
+    autoload :Parser34, "prism/translation/parser_versions"
+    autoload :Parser35, "prism/translation/parser_versions"
+    autoload :Parser40, "prism/translation/parser_versions"
+    autoload :Parser41, "prism/translation/parser_versions"
     autoload :Ripper, "prism/translation/ripper"
     autoload :RubyParser, "prism/translation/ruby_parser"
   end