prism 1.5.1 → 1.9.0

data/lib/prism/translation/ripper.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 # :markup: markdown
 
-require "ripper"
-
 module Prism
   module Translation
     # This class provides a compatibility layer between prism and Ripper. It
@@ -71,7 +69,7 @@ module Prism
       #          [[1, 13], :on_kw,     "end", END      ]]
       #
       def self.lex(src, filename = "-", lineno = 1, raise_errors: false)
-        result = Prism.lex_compat(src, filepath: filename, line: lineno)
+        result = Prism.lex_compat(src, filepath: filename, line: lineno, version: "current")
 
         if result.failure? && raise_errors
           raise SyntaxError, result.errors.first.message
@@ -80,6 +78,19 @@ module Prism
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {
@@ -426,9 +437,35 @@ module Prism
         end
       end
 
+      autoload :Filter, "prism/translation/ripper/filter"
+      autoload :Lexer, "prism/translation/ripper/lexer"
       autoload :SexpBuilder, "prism/translation/ripper/sexp"
       autoload :SexpBuilderPP, "prism/translation/ripper/sexp"
 
+      # :stopdoc:
+      # This is not part of the public API but used by some gems.
+
+      # Ripper-internal bitflags.
+      LEX_STATE_NAMES = %i[
+        BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
+      ].map.with_index.to_h { |name, i| [2 ** i, name] }.freeze
+      private_constant :LEX_STATE_NAMES
+
+      LEX_STATE_NAMES.each do |value, key|
+        const_set("EXPR_#{key}", value)
+      end
+      EXPR_NONE = 0
+      EXPR_VALUE = EXPR_BEG
+      EXPR_BEG_ANY = EXPR_BEG | EXPR_MID | EXPR_CLASS
+      EXPR_ARG_ANY = EXPR_ARG | EXPR_CMDARG
+      EXPR_END_ANY = EXPR_END | EXPR_ENDARG | EXPR_ENDFN
+
+      def self.lex_state_name(state)
+        LEX_STATE_NAMES.filter_map { |flag, name| name if state & flag != 0  }.join("|")
+      end
+
+      # :startdoc:
+
       # The source that is being parsed.
       attr_reader :source
 
@@ -795,7 +832,7 @@ module Prism
       # foo(bar)
       #     ^^^
       def visit_arguments_node(node)
-        arguments, _ = visit_call_node_arguments(node, nil, false)
+        arguments, _, _ = visit_call_node_arguments(node, nil, false)
         arguments
       end
 
@@ -1005,16 +1042,16 @@ module Prism
           case node.name
           when :[]
             receiver = visit(node.receiver)
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
             bounds(node.location)
             call = on_aref(receiver, arguments)
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           when :[]=
             receiver = visit(node.receiver)
@@ -1073,9 +1110,9 @@ module Prism
             if node.variable_call?
               on_vcall(message)
             else
-              arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+              arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
               call =
-                if node.opening_loc.nil? && arguments&.any?
+                if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any?
                   bounds(node.location)
                   on_command(message, arguments)
                 elsif !node.opening_loc.nil?
@@ -1086,11 +1123,11 @@ module Prism
                   on_method_add_arg(on_fcall(message), on_args_new)
                 end
 
-              if block.nil?
-                call
-              else
+              if has_ripper_block
                 bounds(node.block.location)
                 on_method_add_block(call, block)
+              else
+                call
               end
             end
           end
@@ -1114,7 +1151,7 @@ module Prism
             bounds(node.location)
             on_assign(on_field(receiver, call_operator, message), value)
           else
-            arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
+            arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location))
             call =
               if node.opening_loc.nil?
                 bounds(node.location)
@@ -1132,27 +1169,35 @@ module Prism
                 on_method_add_arg(on_call(receiver, call_operator, message), arguments)
               end
 
-            if block.nil?
-              call
-            else
+            if has_ripper_block
               bounds(node.block.location)
               on_method_add_block(call, block)
+            else
+              call
             end
           end
         end
       end
 
-      # Visit the arguments and block of a call node and return the arguments
-      # and block as they should be used.
-      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+      # Extract the arguments and block Ripper-style, which means if the block
+      # is like `&b` then it's moved to arguments.
+      private def get_arguments_and_block(arguments_node, block_node)
         arguments = arguments_node&.arguments || []
         block = block_node
 
         if block.is_a?(BlockArgumentNode)
-          arguments << block
+          arguments += [block]
           block = nil
         end
 
+        [arguments, block]
+      end
+
+      # Visit the arguments and block of a call node and return the arguments
+      # and block as they should be used.
+      private def visit_call_node_arguments(arguments_node, block_node, trailing_comma)
+        arguments, block = get_arguments_and_block(arguments_node, block_node)
+
         [
           if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode)
             visit(arguments.first)
@@ -1166,7 +1211,8 @@ module Prism
               on_args_add_block(args, false)
             end
           end,
-          visit(block)
+          visit(block),
+          block != nil,
         ]
       end
 
@@ -1603,10 +1649,10 @@ module Prism
           end
 
         bounds(node.location)
-        if receiver.nil?
-          on_def(name, parameters, bodystmt)
-        else
+        if receiver
           on_defs(receiver, operator, name, parameters, bodystmt)
+        else
+          on_def(name, parameters, bodystmt)
         end
       end
 
@@ -2004,7 +2050,7 @@ module Prism
       # ^^^^^^^^^^^^^^^
       def visit_index_operator_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2021,7 +2067,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_and_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2038,7 +2084,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^
       def visit_index_or_write_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         target = on_aref_field(receiver, arguments)
@@ -2055,7 +2101,7 @@ module Prism
       # ^^^^^^^^
       def visit_index_target_node(node)
         receiver = visit(node.receiver)
-        arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
+        arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc))
 
         bounds(node.location)
         on_aref_field(receiver, arguments)
@@ -3085,7 +3131,7 @@ module Prism
       # super(foo)
       # ^^^^^^^^^^
       def visit_super_node(node)
-        arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
+        arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location))
 
         if !node.lparen_loc.nil?
           bounds(node.lparen_loc)
@@ -3095,11 +3141,11 @@ module Prism
         bounds(node.location)
         call = on_super(arguments)
 
-        if block.nil?
-          call
-        else
+        if has_ripper_block
           bounds(node.block.location)
           on_method_add_block(call, block)
+        else
+          call
         end
       end
 
@@ -3295,7 +3341,7 @@ module Prism
 
       # Lazily initialize the parse result.
       def result
-        @result ||= Prism.parse(source, partial_script: true)
+        @result ||= Prism.parse(source, partial_script: true, version: "current")
       end
 
       ##########################################################################
@@ -3409,12 +3455,12 @@ module Prism
 
       # :stopdoc:
       def _dispatch_0; end
-      def _dispatch_1(_); end
-      def _dispatch_2(_, _); end
-      def _dispatch_3(_, _, _); end
-      def _dispatch_4(_, _, _, _); end
-      def _dispatch_5(_, _, _, _, _); end
-      def _dispatch_7(_, _, _, _, _, _, _); end
+      def _dispatch_1(arg); arg end
+      def _dispatch_2(arg, _); arg end
+      def _dispatch_3(arg, _, _); arg end
+      def _dispatch_4(arg, _, _, _); arg end
+      def _dispatch_5(arg, _, _, _, _); arg end
+      def _dispatch_7(arg, _, _, _, _, _, _); arg end
       # :startdoc:
 
       #

data/lib/prism/translation/ruby_parser.rb

@@ -2,12 +2,17 @@
 # :markup: markdown
 
 begin
-  require "ruby_parser"
+  require "sexp"
 rescue LoadError
-  warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.})
+  warn(%q{Error: Unable to load sexp. Add `gem "sexp_processor"` to your Gemfile.})
   exit(1)
 end
 
+class RubyParser # :nodoc:
+  class SyntaxError < RuntimeError # :nodoc:
+  end
+end
+
 module Prism
   module Translation
     # This module is the entry-point for converting a prism syntax tree into the
@@ -152,7 +157,7 @@ module Prism
         # ^^
         # ```
         def visit_back_reference_read_node(node)
-          s(node, :back_ref, node.name.name.delete_prefix("$").to_sym)
+          s(node, :back_ref, node.name.to_s.delete_prefix("$").to_sym)
         end
 
         # ```
@@ -415,14 +420,18 @@ module Prism
               visit(node.constant_path)
             end
 
-          if node.body.nil?
-            s(node, :class, name, visit(node.superclass))
-          elsif node.body.is_a?(StatementsNode)
-            compiler = copy_compiler(in_def: false)
-            s(node, :class, name, visit(node.superclass)).concat(node.body.body.map { |child| child.accept(compiler) })
-          else
-            s(node, :class, name, visit(node.superclass), node.body.accept(copy_compiler(in_def: false)))
-          end
+          result =
+            if node.body.nil?
+              s(node, :class, name, visit(node.superclass))
+            elsif node.body.is_a?(StatementsNode)
+              compiler = copy_compiler(in_def: false)
+              s(node, :class, name, visit(node.superclass)).concat(node.body.body.map { |child| child.accept(compiler) })
+            else
+              s(node, :class, name, visit(node.superclass), node.body.accept(copy_compiler(in_def: false)))
+            end
+
+          attach_comments(result, node)
+          result
         end
 
         # ```
@@ -611,7 +620,9 @@ module Prism
               s(node, :defs, visit(node.receiver), name)
             end
 
+          attach_comments(result, node)
           result.line(node.name_loc.start_line)
+
           if node.parameters.nil?
             result << s(node, :args).line(node.name_loc.start_line)
           else
@@ -1270,14 +1281,18 @@ module Prism
               visit(node.constant_path)
             end
 
-          if node.body.nil?
-            s(node, :module, name)
-          elsif node.body.is_a?(StatementsNode)
-            compiler = copy_compiler(in_def: false)
-            s(node, :module, name).concat(node.body.body.map { |child| child.accept(compiler) })
-          else
-            s(node, :module, name, node.body.accept(copy_compiler(in_def: false)))
-          end
+          result =
+            if node.body.nil?
+              s(node, :module, name)
+            elsif node.body.is_a?(StatementsNode)
+              compiler = copy_compiler(in_def: false)
+              s(node, :module, name).concat(node.body.body.map { |child| child.accept(compiler) })
+            else
+              s(node, :module, name, node.body.accept(copy_compiler(in_def: false)))
+            end
+
+          attach_comments(result, node)
+          result
         end
 
         # ```
@@ -1407,7 +1422,7 @@ module Prism
         # ```
         def visit_parameters_node(node)
           children =
-            node.compact_child_nodes.map do |element|
+            node.each_child_node.map do |element|
               if element.is_a?(MultiTargetNode)
                 visit_destructured_parameter(element)
               else
@@ -1820,6 +1835,17 @@ module Prism
 
         private
 
+        # Attach prism comments to the given sexp.
+        def attach_comments(sexp, node)
+          return unless node.comments
+          return if node.comments.empty?
+
+          extra = node.location.start_line - node.comments.last.location.start_line
+          comments = node.comments.map(&:slice)
+          comments.concat([nil] * [0, extra].max)
+          sexp.comments = comments.join("\n")
+        end
+
         # Create a new compiler with the given options.
         def copy_compiler(in_def: self.in_def, in_pattern: self.in_pattern)
           Compiler.new(file, in_def: in_def, in_pattern: in_pattern)
@@ -1898,6 +1924,14 @@ module Prism
         translate(Prism.parse_file(filepath, partial_script: true), filepath)
       end
 
+      # Parse the give file and translate it into the
+      # seattlerb/ruby_parser gem's Sexp format. This method is
+      # provided for API compatibility to RubyParser and takes an
+      # optional +timeout+ argument.
+      def process(ruby, file = "(string)", timeout = nil)
+        Timeout.timeout(timeout) { parse(ruby, file) }
+      end
+
       class << self
         # Parse the given source and translate it into the seattlerb/ruby_parser
         # gem's Sexp format.
@@ -1922,6 +1956,7 @@ module Prism
           raise ::RubyParser::SyntaxError, "#{filepath}:#{error.location.start_line} :: #{error.message}"
         end
 
+        result.attach_comments!
         result.value.accept(Compiler.new(filepath))
       end
     end

data/lib/prism/translation.rb

@@ -7,9 +7,11 @@ module Prism
   module Translation # steep:ignore
     autoload :Parser, "prism/translation/parser"
     autoload :ParserCurrent, "prism/translation/parser_current"
-    autoload :Parser33, "prism/translation/parser33"
-    autoload :Parser34, "prism/translation/parser34"
-    autoload :Parser35, "prism/translation/parser35"
+    autoload :Parser33, "prism/translation/parser_versions"
+    autoload :Parser34, "prism/translation/parser_versions"
+    autoload :Parser35, "prism/translation/parser_versions"
+    autoload :Parser40, "prism/translation/parser_versions"
+    autoload :Parser41, "prism/translation/parser_versions"
     autoload :Ripper, "prism/translation/ripper"
     autoload :RubyParser, "prism/translation/ruby_parser"
   end