prism 1.8.0 → 1.9.0

data/lib/prism/translation/ripper/lexer.rb

@@ -6,7 +6,7 @@ require_relative "../ripper"
 module Prism
   module Translation
     class Ripper
-      class Lexer # :nodoc:
+      class Lexer < Ripper # :nodoc:
         # :stopdoc:
         class State
 
@@ -38,7 +38,96 @@ module Prism
           def allbits?(i) to_int.allbits?(i) end
           def anybits?(i) to_int.anybits?(i) end
           def nobits?(i) to_int.nobits?(i) end
+
+          # Instances are frozen and there are only a handful of them so we cache them here.
+          STATES = Hash.new { |h,k| h[k] = State.new(k) }
+
+          def self.cached(i)
+            STATES[i]
+          end
         end
+
+        class Elem
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State.cached(state)
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        # Pretty much just the same as Prism.lex_compat.
+        def lex(raise_errors: false)
+          Ripper.lex(@source, filename, lineno, raise_errors: raise_errors)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(...)
+          lex(...).map do |position, event, token, state|
+            Elem.new(position, event, token, state.to_int)
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
+        end
+
         # :startdoc:
       end
     end

data/lib/prism/translation/ripper.rb

@@ -78,6 +78,19 @@ module Prism
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {
@@ -424,6 +437,7 @@ module Prism
         end
       end
 
+      autoload :Filter, "prism/translation/ripper/filter"
       autoload :Lexer, "prism/translation/ripper/lexer"
       autoload :SexpBuilder, "prism/translation/ripper/sexp"
       autoload :SexpBuilderPP, "prism/translation/ripper/sexp"
@@ -818,7 +832,7 @@ module Prism
       # foo(bar)
       #     ^^^
       def visit_arguments_node(node)
-        arguments, _ = visit_call_node_arguments(node, nil, false)
+        arguments, _, _ = visit_call_node_arguments(node, nil, false)
         arguments
       end
 
@@ -1028,16 +1042,16 @@ module Prism
           case node.name
           when :[]
             receiver = visit(node.receiver)
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
             bounds(node.location)
             call = on_aref(receiver, arguments)
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           when :[]=
             receiver = visit(node.receiver)
@@ -1096,9 +1110,9 @@ module Prism
             if node.variable_call?
               on_vcall(message)
             else
-              arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+              arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
               call =
-                if node.opening_loc.nil? && arguments&.any?
+                if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any?
                   bounds(node.location)
                   on_command(message, arguments)
                 elsif !node.opening_loc.nil?
@@ -1109,11 +1123,11 @@ module Prism
                   on_method_add_arg(on_fcall(message), on_args_new)
                 end
 
-              if block.nil?
-                call
-              else
+              if has_ripper_block
                 bounds(node.block.location)
                 on_method_add_block(call, block)
+              else
+                call
               end
             end
           end
@@ -1137,7 +1151,7 @@ module Prism
             bounds(node.location)
             on_assign(on_field(receiver, call_operator, message), value)
           else
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
             call =
               if node.opening_loc.nil?
                 bounds(node.location)
@@ -1155,27 +1169,35 @@ module Prism
                 on_method_add_arg(on_call(receiver, call_operator, message), arguments)
               end
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.block.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           end
         end
       end
 
-      # Visit the arguments and block of a call node and return the arguments
-      # and block as they should be used.
-      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+      # Extract the arguments and block Ripper-style, which means if the block
+      # is like `&b` then it's moved to arguments.
+      private def get_arguments_and_block(arguments_node, block_node)
         arguments = arguments_node&.arguments || []
         block = block_node
 
         if block.is_a?(BlockArgumentNode)
-          arguments << block
+          arguments += [block]
           block = nil
         end
 
+        [arguments, block]
+      end
+
+      # Visit the arguments and block of a call node and return the arguments
+      # and block as they should be used.
+      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+        arguments, block = get_arguments_and_block(arguments_node, block_node)
+
         [
           if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode)
             visit(arguments.first)
@@ -1189,7 +1211,8 @@ module Prism
               on_args_add_block(args, false)
             end
           end,
-          visit(block)
+          visit(block),
+          block != nil,
         ]
       end
 
@@ -1626,10 +1649,10 @@ module Prism
           end
 
         bounds(node.location)
-        if receiver.nil?
-          on_def(name, parameters, bodystmt)
-        else
+        if receiver
           on_defs(receiver, operator, name, parameters, bodystmt)
+        else
+          on_def(name, parameters, bodystmt)
         end
       end
 
@@ -2027,7 +2050,7 @@ module Prism
       # ^^^^^^^^^^^^^^^
       def visit_index_operator_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2044,7 +2067,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_and_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2061,7 +2084,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_or_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2078,7 +2101,7 @@ module Prism
       # ^^^^^^^^
       def visit_index_target_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         on_aref_field(receiver, arguments)
@@ -3108,7 +3131,7 @@ module Prism
       # super(foo)
       # ^^^^^^^^^^
       def visit_super_node(node)
-        arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
+        arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
 
         if !node.lparen_loc.nil?
           bounds(node.lparen_loc)
@@ -3118,11 +3141,11 @@ module Prism
         bounds(node.location)
         call = on_super(arguments)
 
-        if block.nil?
-          call
-        else
+        if has_ripper_block
           bounds(node.block.location)
           on_method_add_block(call, block)
+        else
+          call
         end
       end
 
@@ -3432,12 +3455,12 @@ module Prism
 
       # :stopdoc:
       def _dispatch_0; end
-      def _dispatch_1(_); end
-      def _dispatch_2(_, _); end
-      def _dispatch_3(_, _, _); end
-      def _dispatch_4(_, _, _, _); end
-      def _dispatch_5(_, _, _, _, _); end
-      def _dispatch_7(_, _, _, _, _, _, _); end
+      def _dispatch_1(arg); arg end
+      def _dispatch_2(arg, _); arg end
+      def _dispatch_3(arg, _, _); arg end
+      def _dispatch_4(arg, _, _, _); arg end
+      def _dispatch_5(arg, _, _, _, _); arg end
+      def _dispatch_7(arg, _, _, _, _, _, _); arg end
       # :startdoc:
 
       #

data/lib/prism.rb

@@ -20,7 +20,6 @@ module Prism
   autoload :DSL, "prism/dsl"
   autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
-  autoload :LexRipper, "prism/lex_ripper"
   autoload :MutationCompiler, "prism/mutation_compiler"
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
@@ -35,7 +34,6 @@ module Prism
   # private here.
 
   private_constant :LexCompat
-  private_constant :LexRipper
 
   # Raised when requested to parse as the currently running Ruby version but Prism has no support for it.
   class CurrentVersionError < ArgumentError
@@ -61,24 +59,13 @@ module Prism
   #   Prism::lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
-  # resembles the return value of Ripper::lex. The main difference is that the
-  # `:on_sp` token is not emitted.
+  # resembles the return value of Ripper::lex.
   #
   # For supported options, see Prism::parse.
   def self.lex_compat(source, **options)
     LexCompat.new(source, **options).result # steep:ignore
   end
 
-  # :call-seq:
-  #   Prism::lex_ripper(source) -> Array
-  #
-  # This lexes with the Ripper lex. It drops any space events but otherwise
-  # returns the same tokens. Raises SyntaxError if the syntax in source is
-  # invalid.
-  def self.lex_ripper(source)
-    LexRipper.new(source).result # steep:ignore
-  end
-
   # :call-seq:
   #   Prism::load(source, serialized, freeze) -> ParseResult
   #

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.8.0"
+  spec.version = "1.9.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -77,7 +77,6 @@ Gem::Specification.new do |spec|
     "lib/prism/ffi.rb",
     "lib/prism/inspect_visitor.rb",
     "lib/prism/lex_compat.rb",
-    "lib/prism/lex_ripper.rb",
     "lib/prism/mutation_compiler.rb",
     "lib/prism/node_ext.rb",
     "lib/prism/node.rb",
@@ -104,6 +103,7 @@ Gem::Specification.new do |spec|
     "lib/prism/translation/parser/compiler.rb",
     "lib/prism/translation/parser/lexer.rb",
     "lib/prism/translation/ripper.rb",
+    "lib/prism/translation/ripper/filter.rb",
     "lib/prism/translation/ripper/lexer.rb",
     "lib/prism/translation/ripper/sexp.rb",
     "lib/prism/translation/ripper/shim.rb",

data/rbi/prism/node.rbi

@@ -57,6 +57,9 @@ class Prism::Node
   sig { params(block: T.proc.params(node: Prism::Node).returns(T::Boolean)).returns(T.nilable(Prism::Node)) }
   def breadth_first_search(&block); end
 
+  sig { params(block: T.proc.params(node: Prism::Node).returns(T::Boolean)).returns(T::Array[Prism::Node]) }
+  def breadth_first_search_all(&block); end
+
   sig { abstract.params(visitor: Prism::Visitor).returns(T.untyped) }
   def accept(visitor); end
 

data/rbi/prism.rbi

@@ -16,9 +16,6 @@ module Prism
   sig { params(source: String, options: T::Hash[Symbol, T.untyped]).returns(Prism::LexCompat::Result) }
   def self.lex_compat(source, **options); end
 
-  sig { params(source: String).returns(T::Array[T.untyped]) }
-  def self.lex_ripper(source); end
-
   sig { params(source: String, serialized: String, freeze: T.nilable(T::Boolean)).returns(Prism::ParseResult) }
   def self.load(source, serialized, freeze = false); end
 

data/sig/prism/node.rbs

@@ -29,6 +29,9 @@ module Prism
     def to_dot: () -> String
     def tunnel: (Integer line, Integer column) -> Array[Prism::node]
     def breadth_first_search: () { (Prism::node) -> bool } -> Prism::node?
+    alias find breadth_first_search
+    def breadth_first_search_all: () { (Prism::node) -> bool } -> Array[Prism::node]
+    alias find_all breadth_first_search_all
     def newline!: (Array[untyped]) -> void
 
     def save: (_Repository repository) -> void

data/sig/prism/parse_result.rbs

@@ -14,6 +14,7 @@ module Prism
     def encoding: () -> Encoding
     def lines: () -> Array[String]
     def slice: (Integer byte_offset, Integer length) -> String
+    def byte_offset: (Integer line, Integer column) -> Integer
     def line: (Integer byte_offset) -> Integer
     def line_start: (Integer byte_offset) -> Integer
     def line_end: (Integer byte_offset) -> Integer