prism 0.24.0 → 0.29.0

data/lib/prism/translation/ruby_parser.rb

@@ -6,7 +6,7 @@ module Prism
   module Translation
     # This module is the entry-point for converting a prism syntax tree into the
     # seattlerb/ruby_parser gem's syntax tree.
-    module RubyParser
+    class RubyParser
       # A prism visitor that builds Sexp objects.
       class Compiler < ::Prism::Compiler
         # This is the name of the file that we are compiling. We set it on every
@@ -192,19 +192,19 @@ module Prism
 
           if node.opening == "("
             result.line = node.opening_loc.start_line
-            result.max_line = node.closing_loc.end_line
+            result.line_max = node.closing_loc.end_line
             shadow_loc = false
           end
 
           if node.locals.any?
             shadow = s(node, :shadow).concat(visit_all(node.locals))
             shadow.line = node.locals.first.location.start_line
-            shadow.max_line = node.locals.last.location.end_line
+            shadow.line_max = node.locals.last.location.end_line
             result << shadow
 
             if shadow_loc
               result.line = shadow.line
-              result.max_line = shadow.max_line
+              result.line_max = shadow.line_max
             end
           end
 
@@ -271,9 +271,9 @@ module Prism
         # ^^^^^^^^^^^^^^^
         def visit_call_operator_write_node(node)
           if op_asgn?(node)
-            s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.operator)
+            s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator)
           else
-            s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.operator, visit_write_value(node.value))
+            s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value))
           end
         end
 
@@ -372,7 +372,7 @@ module Prism
         # @@foo += bar
         # ^^^^^^^^^^^^
         def visit_class_variable_operator_write_node(node)
-          s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.operator, visit_write_value(node.value)))
+          s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value)))
         end
 
         # @@foo &&= bar
@@ -417,7 +417,7 @@ module Prism
         # Foo += bar
         # ^^^^^^^^^^^
         def visit_constant_operator_write_node(node)
-          s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.operator, visit_write_value(node.value)))
+          s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value)))
         end
 
         # Foo &&= bar
@@ -442,9 +442,9 @@ module Prism
         # ^^^^^^^^
         def visit_constant_path_node(node)
           if node.parent.nil?
-            s(node, :colon3, node.child.name)
+            s(node, :colon3, node.name)
           else
-            s(node, :colon2, visit(node.parent), node.child.name)
+            s(node, :colon2, visit(node.parent), node.name)
           end
         end
 
@@ -460,7 +460,7 @@ module Prism
         # Foo::Bar += baz
         # ^^^^^^^^^^^^^^^
         def visit_constant_path_operator_write_node(node)
-          s(node, :op_asgn, visit(node.target), node.operator, visit_write_value(node.value))
+          s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value))
         end
 
         # Foo::Bar &&= baz
@@ -627,7 +627,7 @@ module Prism
         # $foo += bar
         # ^^^^^^^^^^^
         def visit_global_variable_operator_write_node(node)
-          s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.operator, visit(node.value)))
+          s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value)))
         end
 
         # $foo &&= bar
@@ -719,7 +719,7 @@ module Prism
             arglist << visit(node.block) if !node.block.nil?
           end
 
-          s(node, :op_asgn1, visit(node.receiver), arglist, node.operator, visit_write_value(node.value))
+          s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value))
         end
 
         # foo[bar] &&= baz
@@ -775,7 +775,7 @@ module Prism
         # @foo += bar
         # ^^^^^^^^^^^
         def visit_instance_variable_operator_write_node(node)
-          s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.operator, visit_write_value(node.value)))
+          s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value)))
         end
 
         # @foo &&= bar
@@ -805,17 +805,29 @@ module Prism
         # if /foo #{bar}/ then end
         #    ^^^^^^^^^^^^
         def visit_interpolated_match_last_line_node(node)
-          s(node, :match, s(node, :dregx).concat(visit_interpolated_parts(node.parts)))
+          parts = visit_interpolated_parts(node.parts)
+          regexp =
+            if parts.length == 1
+              s(node, :lit, Regexp.new(parts.first, node.options))
+            else
+              s(node, :dregx).concat(parts).tap do |result|
+                options = node.options
+                result << options if options != 0
+              end
+            end
+
+          s(node, :match, regexp)
         end
 
         # /foo #{bar}/
         # ^^^^^^^^^^^^
         def visit_interpolated_regular_expression_node(node)
-          if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }
-            unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :lit, Regexp.new(unescaped, node.options))
+          parts = visit_interpolated_parts(node.parts)
+
+          if parts.length == 1
+            s(node, :lit, Regexp.new(parts.first, node.options))
           else
-            s(node, :dregx).concat(visit_interpolated_parts(node.parts)).tap do |result|
+            s(node, :dregx).concat(parts).tap do |result|
               options = node.options
               result << options if options != 0
             end
@@ -825,45 +837,71 @@ module Prism
         # "foo #{bar}"
         # ^^^^^^^^^^^^
         def visit_interpolated_string_node(node)
-          if (node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }) ||
-             (node.opening.nil? && node.parts.all? { |part| part.is_a?(StringNode) && !part.opening_loc.nil? })
-             unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :str, unescaped)
-          else
-            s(node, :dstr).concat(visit_interpolated_parts(node.parts))
-          end
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(node, :str, parts.first) : s(node, :dstr).concat(parts)
         end
 
         # :"foo #{bar}"
         # ^^^^^^^^^^^^^
         def visit_interpolated_symbol_node(node)
-          if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }
-            unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :lit, unescaped.to_sym)
-          else
-            s(node, :dsym).concat(visit_interpolated_parts(node.parts))
-          end
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(node, :lit, parts.first.to_sym) : s(node, :dsym).concat(parts)
         end
 
         # `foo #{bar}`
         # ^^^^^^^^^^^^
         def visit_interpolated_x_string_node(node)
-          children = visit_interpolated_parts(node.parts)
-          s(node.heredoc? ? node.parts.first : node, :dxstr).concat(children)
+          source = node.heredoc? ? node.parts.first : node
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(source, :xstr, parts.first) : s(source, :dxstr).concat(parts)
         end
 
         # Visit the interpolated content of the string-like node.
         private def visit_interpolated_parts(parts)
-          parts.each_with_object([]).with_index do |(part, results), index|
-            if index == 0
-              if part.is_a?(StringNode)
-                results << part.unescaped
+          visited = []
+          parts.each do |part|
+            result = visit(part)
+
+            if result[0] == :evstr && result[1]
+              if result[1][0] == :str
+                visited << result[1]
+              elsif result[1][0] == :dstr
+                visited.concat(result[1][1..-1])
+              else
+                visited << result
+              end
+            else
+              visited << result
+            end
+          end
+
+          state = :beginning #: :beginning | :string_content | :interpolated_content
+
+          visited.each_with_object([]) do |result, results|
+            case state
+            when :beginning
+              if result.is_a?(String)
+                results << result
+                state = :string_content
+              elsif result.is_a?(Array) && result[0] == :str
+                results << result[1]
+                state = :string_content
               else
                 results << ""
-                results << visit(part)
+                results << result
+                state = :interpolated_content
+              end
+            when :string_content
+              if result.is_a?(String)
+                results[0] << result
+              elsif result.is_a?(Array) && result[0] == :str
+                results[0] << result[1]
+              else
+                results << result
+                state = :interpolated_content
               end
             else
-              results << visit(part)
+              results << result
             end
           end
         end
@@ -922,7 +960,7 @@ module Prism
         # foo += bar
         # ^^^^^^^^^^
         def visit_local_variable_operator_write_node(node)
-          s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.operator, visit_write_value(node.value)))
+          s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value)))
         end
 
         # foo &&= bar
@@ -1274,6 +1312,11 @@ module Prism
           s(node, :self)
         end
 
+        # A shareable constant.
+        def visit_shareable_constant_node(node)
+          visit(node.write)
+        end
+
         # class << self; end
         # ^^^^^^^^^^^^^^^^^^
         def visit_singleton_class_node(node)
@@ -1292,7 +1335,7 @@ module Prism
         # __FILE__
         # ^^^^^^^^
         def visit_source_file_node(node)
-          s(node, :str, file)
+          s(node, :str, node.filepath)
         end
 
         # __LINE__
@@ -1407,7 +1450,7 @@ module Prism
 
           if node.heredoc?
             result.line = node.content_loc.start_line
-            result.max_line = node.content_loc.end_line
+            result.line_max = node.content_loc.end_line
           end
 
           result
@@ -1434,7 +1477,7 @@ module Prism
           result = Sexp.new(*arguments)
           result.file = file
           result.line = node.location.start_line
-          result.max_line = node.location.end_line
+          result.line_max = node.location.end_line
           result
         end
 
@@ -1490,31 +1533,43 @@ module Prism
 
       private_constant :Compiler
 
+      # Parse the given source and translate it into the seattlerb/ruby_parser
+      # gem's Sexp format.
+      def parse(source, filepath = "(string)")
+        translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath)
+      end
+
+      # Parse the given file and translate it into the seattlerb/ruby_parser
+      # gem's Sexp format.
+      def parse_file(filepath)
+        translate(Prism.parse_file(filepath, scopes: [[]]), filepath)
+      end
+
       class << self
         # Parse the given source and translate it into the seattlerb/ruby_parser
         # gem's Sexp format.
         def parse(source, filepath = "(string)")
-          translate(Prism.parse(source), filepath)
+          new.parse(source, filepath)
         end
 
         # Parse the given file and translate it into the seattlerb/ruby_parser
         # gem's Sexp format.
         def parse_file(filepath)
-          translate(Prism.parse_file(filepath), filepath)
+          new.parse_file(filepath)
         end
+      end
 
-        private
-
-        # Translate the given parse result and filepath into the
-        # seattlerb/ruby_parser gem's Sexp format.
-        def translate(result, filepath)
-          if result.failure?
-            error = result.errors.first
-            raise ::RubyParser::SyntaxError, "#{filepath}:#{error.location.start_line} :: #{error.message}"
-          end
+      private
 
-          result.value.accept(Compiler.new(filepath))
+      # Translate the given parse result and filepath into the
+      # seattlerb/ruby_parser gem's Sexp format.
+      def translate(result, filepath)
+        if result.failure?
+          error = result.errors.first
+          raise ::RubyParser::SyntaxError, "#{filepath}:#{error.location.start_line} :: #{error.message}"
         end
+
+        result.value.accept(Compiler.new(filepath))
       end
     end
   end

data/lib/prism/translation.rb

@@ -3,8 +3,10 @@
 module Prism
   # This module is responsible for converting the prism syntax tree into other
   # syntax trees.
-  module Translation
+  module Translation # steep:ignore
     autoload :Parser, "prism/translation/parser"
+    autoload :Parser33, "prism/translation/parser33"
+    autoload :Parser34, "prism/translation/parser34"
     autoload :Ripper, "prism/translation/ripper"
     autoload :RubyParser, "prism/translation/ruby_parser"
   end

data/lib/prism/visitor.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/visitor.rb.erb
@@ -14,16 +15,19 @@ module Prism
     # Calls `accept` on the given node if it is not `nil`, which in turn should
     # call back into this visitor by calling the appropriate `visit_*` method.
     def visit(node)
+      # @type self: _Visitor
       node&.accept(self)
     end
 
     # Visits each node in `nodes` by calling `accept` on each one.
     def visit_all(nodes)
+      # @type self: _Visitor
       nodes.each { |node| node&.accept(self) }
     end
 
     # Visits the child nodes of `node` by calling `accept` on each one.
     def visit_child_nodes(node)
+      # @type self: _Visitor
       node.compact_child_nodes.each { |node| node.accept(self) }
     end
   end
@@ -309,6 +313,9 @@ module Prism
     # Visit a InterpolatedXStringNode node
     alias visit_interpolated_x_string_node visit_child_nodes
 
+    # Visit a ItParametersNode node
+    alias visit_it_parameters_node visit_child_nodes
+
     # Visit a KeywordHashNode node
     alias visit_keyword_hash_node visit_child_nodes
 
@@ -441,6 +448,9 @@ module Prism
     # Visit a SelfNode node
     alias visit_self_node visit_child_nodes
 
+    # Visit a ShareableConstantNode node
+    alias visit_shareable_constant_node visit_child_nodes
+
     # Visit a SingletonClassNode node
     alias visit_singleton_class_node visit_child_nodes
 

data/lib/prism.rb

@@ -18,12 +18,13 @@ module Prism
   autoload :Dispatcher, "prism/dispatcher"
   autoload :DotVisitor, "prism/dot_visitor"
   autoload :DSL, "prism/dsl"
+  autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
   autoload :LexRipper, "prism/lex_compat"
   autoload :MutationCompiler, "prism/mutation_compiler"
-  autoload :NodeInspector, "prism/node_inspector"
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
+  autoload :Reflection, "prism/reflection"
   autoload :Serialize, "prism/serialize"
   autoload :Translation, "prism/translation"
   autoload :Visitor, "prism/visitor"
@@ -36,7 +37,7 @@ module Prism
   private_constant :LexRipper
 
   # :call-seq:
-  #   Prism::lex_compat(source, **options) -> ParseResult
+  #   Prism::lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
   # resembles the return value of Ripper::lex. The main difference is that the
@@ -44,7 +45,7 @@ module Prism
   #
   # For supported options, see Prism::parse.
   def self.lex_compat(source, **options)
-    LexCompat.new(source, **options).result
+    LexCompat.new(source, **options).result # steep:ignore
   end
 
   # :call-seq:
@@ -54,7 +55,7 @@ module Prism
   # returns the same tokens. Raises SyntaxError if the syntax in source is
   # invalid.
   def self.lex_ripper(source)
-    LexRipper.new(source).result
+    LexRipper.new(source).result # steep:ignore
   end
 
   # :call-seq:
@@ -64,24 +65,9 @@ module Prism
   def self.load(source, serialized)
     Serialize.load(source, serialized)
   end
-
-  # :call-seq:
-  #   Prism::parse_failure?(source, **options) -> bool
-  #
-  # Returns true if the source parses with errors.
-  def self.parse_failure?(source, **options)
-    !parse_success?(source, **options)
-  end
-
-  # :call-seq:
-  #   Prism::parse_file_failure?(filepath, **options) -> bool
-  #
-  # Returns true if the file at filepath parses with errors.
-  def self.parse_file_failure?(filepath, **options)
-    !parse_file_success?(filepath, **options)
-  end
 end
 
+require_relative "prism/polyfill/byteindex"
 require_relative "prism/node"
 require_relative "prism/node_ext"
 require_relative "prism/parse_result"
@@ -94,6 +80,12 @@ require_relative "prism/parse_result/newlines"
 # module that uses FFI to call into the library.
 if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
   require "prism/prism"
+
+  # The C extension is the default backend on CRuby.
+  Prism::BACKEND = :CEXT
 else
   require_relative "prism/ffi"
+
+  # The FFI backend is used on other Ruby implementations.
+  Prism::BACKEND = :FFI
 end

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "0.24.0"
+  spec.version = "0.29.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
   spec.files = [
+    "BSDmakefile",
     "CHANGELOG.md",
     "CODE_OF_CONDUCT.md",
     "CONTRIBUTING.md",
@@ -34,7 +35,7 @@ Gem::Specification.new do |spec|
     "docs/parser_translation.md",
     "docs/parsing_rules.md",
     "docs/releasing.md",
-    "docs/ripper.md",
+    "docs/ripper_translation.md",
     "docs/ruby_api.md",
     "docs/ruby_parser_translation.md",
     "docs/serialization.md",
@@ -54,13 +55,14 @@ Gem::Specification.new do |spec|
     "include/prism/parser.h",
     "include/prism/prettyprint.h",
     "include/prism/regexp.h",
+    "include/prism/static_literals.h",
     "include/prism/util/pm_buffer.h",
     "include/prism/util/pm_char.h",
     "include/prism/util/pm_constant_pool.h",
+    "include/prism/util/pm_integer.h",
     "include/prism/util/pm_list.h",
     "include/prism/util/pm_memchr.h",
     "include/prism/util/pm_newline_list.h",
-    "include/prism/util/pm_state_stack.h",
     "include/prism/util/pm_strncasecmp.h",
     "include/prism/util/pm_string.h",
     "include/prism/util/pm_string_list.h",
@@ -74,16 +76,19 @@ Gem::Specification.new do |spec|
     "lib/prism/dot_visitor.rb",
     "lib/prism/dsl.rb",
     "lib/prism/ffi.rb",
+    "lib/prism/inspect_visitor.rb",
     "lib/prism/lex_compat.rb",
     "lib/prism/mutation_compiler.rb",
     "lib/prism/node_ext.rb",
-    "lib/prism/node_inspector.rb",
     "lib/prism/node.rb",
     "lib/prism/pack.rb",
     "lib/prism/parse_result.rb",
     "lib/prism/parse_result/comments.rb",
     "lib/prism/parse_result/newlines.rb",
     "lib/prism/pattern.rb",
+    "lib/prism/polyfill/byteindex.rb",
+    "lib/prism/polyfill/unpack1.rb",
+    "lib/prism/reflection.rb",
     "lib/prism/serialize.rb",
     "lib/prism/translation.rb",
     "lib/prism/translation/parser.rb",
@@ -93,34 +98,61 @@ Gem::Specification.new do |spec|
     "lib/prism/translation/parser/lexer.rb",
     "lib/prism/translation/parser/rubocop.rb",
     "lib/prism/translation/ripper.rb",
+    "lib/prism/translation/ripper/sexp.rb",
+    "lib/prism/translation/ripper/shim.rb",
     "lib/prism/translation/ruby_parser.rb",
     "lib/prism/visitor.rb",
+    "prism.gemspec",
+    "rbi/prism.rbi",
+    "rbi/prism/compiler.rbi",
+    "rbi/prism/inspect_visitor.rbi",
+    "rbi/prism/node_ext.rbi",
+    "rbi/prism/node.rbi",
+    "rbi/prism/parse_result.rbi",
+    "rbi/prism/reflection.rbi",
+    "rbi/prism/translation/parser.rbi",
+    "rbi/prism/translation/parser33.rbi",
+    "rbi/prism/translation/parser34.rbi",
+    "rbi/prism/translation/ripper.rbi",
+    "rbi/prism/visitor.rbi",
+    "sig/prism.rbs",
+    "sig/prism/compiler.rbs",
+    "sig/prism/dispatcher.rbs",
+    "sig/prism/dot_visitor.rbs",
+    "sig/prism/dsl.rbs",
+    "sig/prism/inspect_visitor.rbs",
+    "sig/prism/lex_compat.rbs",
+    "sig/prism/mutation_compiler.rbs",
+    "sig/prism/node_ext.rbs",
+    "sig/prism/node.rbs",
+    "sig/prism/pack.rbs",
+    "sig/prism/parse_result.rbs",
+    "sig/prism/pattern.rbs",
+    "sig/prism/reflection.rbs",
+    "sig/prism/serialize.rbs",
+    "sig/prism/visitor.rbs",
     "src/diagnostic.c",
     "src/encoding.c",
     "src/node.c",
+    "src/options.c",
     "src/pack.c",
     "src/prettyprint.c",
+    "src/prism.c",
     "src/regexp.c",
     "src/serialize.c",
+    "src/static_literals.c",
     "src/token_type.c",
     "src/util/pm_buffer.c",
     "src/util/pm_char.c",
     "src/util/pm_constant_pool.c",
+    "src/util/pm_integer.c",
     "src/util/pm_list.c",
     "src/util/pm_memchr.c",
     "src/util/pm_newline_list.c",
-    "src/util/pm_state_stack.c",
-    "src/util/pm_string.c",
     "src/util/pm_string_list.c",
+    "src/util/pm_string.c",
     "src/util/pm_strncasecmp.c",
-    "src/util/pm_strpbrk.c",
-    "src/options.c",
-    "src/prism.c",
-    "prism.gemspec",
-    "sig/prism.rbs",
-    "sig/prism_static.rbs",
-    "rbi/prism.rbi",
-    "rbi/prism_static.rbi"
+    "src/util/pm_strpbrk.c"
   ]
 
   spec.extensions = ["ext/prism/extconf.rb"]

data/rbi/prism/compiler.rbi

@@ -0,0 +1,12 @@
+# typed: strict
+
+class Prism::Compiler
+  sig { params(node: T.nilable(Prism::Node)).returns(T.untyped) }
+  def visit(node); end
+
+  sig { params(nodes: T::Array[T.nilable(Prism::Node)]).returns(T::Array[T.untyped]) }
+  def visit_all(nodes); end
+
+  sig { params(node: Prism::Node).returns(T::Array[T.untyped]) }
+  def visit_child_nodes(node); end
+end

data/rbi/prism/inspect_visitor.rbi

@@ -0,0 +1,12 @@
+# typed: strict
+
+class Prism::InspectVisitor < Prism::Visitor
+  sig { params(indent: String).void }
+  def initialize(indent = ""); end
+
+  sig { params(node: Prism::Node).returns(String) }
+  def self.compose(node); end
+
+  sig { returns(String) }
+  def compose; end
+end