prism 0.26.0 → 0.28.0

data/lib/prism/translation/parser/compiler.rb

@@ -483,13 +483,13 @@ module Prism
           if node.parent.nil?
             builder.const_global(
               token(node.delimiter_loc),
-              [node.child.name, srange(node.child.location)]
+              [node.name, srange(node.name_loc)]
             )
           else
             builder.const_fetch(
               visit(node.parent),
               token(node.delimiter_loc),
-              [node.child.name, srange(node.child.location)]
+              [node.name, srange(node.name_loc)]
             )
           end
         end
@@ -839,7 +839,7 @@ module Prism
             token(node.in_loc),
             pattern,
             guard,
-            srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset || node.location.end_offset, [";", "then"]),
+            srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, [";", "then"]),
             visit(node.statements)
           )
         end
@@ -1030,6 +1030,12 @@ module Prism
           end
         end
 
+        # -> { it }
+        # ^^^^^^^^^
+        def visit_it_parameters_node(node)
+          builder.args(nil, [], nil, false)
+        end
+
         # foo(bar: baz)
         #     ^^^^^^^^
         def visit_keyword_hash_node(node)
@@ -1052,13 +1058,14 @@ module Prism
         # ^^^^^
         def visit_lambda_node(node)
           parameters = node.parameters
+          implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode)
 
           builder.block(
             builder.call_lambda(token(node.operator_loc)),
             [node.opening, srange(node.opening_loc)],
             if parameters.nil?
               builder.args(nil, [], nil, false)
-            elsif node.parameters.is_a?(NumberedParametersNode)
+            elsif implicit_parameters
               visit(node.parameters)
             else
               builder.args(
@@ -1068,7 +1075,7 @@ module Prism
                 false
               )
             end,
-            node.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
+            node.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))),
             [node.closing, srange(node.closing_loc)]
           )
         end
@@ -1076,7 +1083,14 @@ module Prism
         # foo
         # ^^^
         def visit_local_variable_read_node(node)
-          builder.ident([node.name, srange(node.location)]).updated(:lvar)
+          name = node.name
+
+          # This is just a guess. parser doesn't have support for the implicit
+          # `it` variable yet, so we'll probably have to visit this once it
+          # does.
+          name = :it if name == :"0it"
+
+          builder.ident([name, srange(node.location)]).updated(:lvar)
         end
 
         # foo = 1
@@ -1665,7 +1679,7 @@ module Prism
         end
 
         # until foo; bar end
-        # ^^^^^^^^^^^^^^^^^
+        # ^^^^^^^^^^^^^^^^^^
         #
         # bar until foo
         # ^^^^^^^^^^^^^
@@ -1698,7 +1712,7 @@ module Prism
             if node.then_keyword_loc
               token(node.then_keyword_loc)
             else
-              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset || (node.conditions.last.location.end_offset + 1), [";"])
+              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, [";"])
             end,
             visit(node.statements)
           )
@@ -1857,12 +1871,16 @@ module Prism
 
         # Constructs a new source range by finding the given tokens between the
         # given start offset and end offset. If the needle is not found, it
-        # returns nil.
+        # returns nil. Importantly it does not search past newlines or comments.
+        #
+        # Note that end_offset is allowed to be nil, in which case this will
+        # search until the end of the string.
         def srange_find(start_offset, end_offset, tokens)
-          tokens.find do |token|
-            next unless (index = source_buffer.source.byteslice(start_offset...end_offset).index(token))
-            offset = start_offset + index
-            return [token, Range.new(source_buffer, offset_cache[offset], offset_cache[offset + token.length])]
+          if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/(\s*)(#{tokens.join("|")})/))
+            _, whitespace, token = *match
+            token_offset = start_offset + whitespace.bytesize
+
+            [token, Range.new(source_buffer, offset_cache[token_offset], offset_cache[token_offset + token.bytesize])]
           end
         end
 
@@ -1875,13 +1893,14 @@ module Prism
         def visit_block(call, block)
           if block
             parameters = block.parameters
+            implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode)
 
             builder.block(
               call,
               token(block.opening_loc),
               if parameters.nil?
                 builder.args(nil, [], nil, false)
-              elsif parameters.is_a?(NumberedParametersNode)
+              elsif implicit_parameters
                 visit(parameters)
               else
                 builder.args(
@@ -1896,7 +1915,7 @@ module Prism
                   false
                 )
               end,
-              block.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
+              block.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))),
               token(block.closing_loc)
             )
           else

data/lib/prism/translation/parser.rb

@@ -46,7 +46,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
 
         build_ast(result.value, offset_cache)
       ensure
@@ -59,7 +59,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
 
         [
           build_ast(result.value, offset_cache),
@@ -78,7 +78,7 @@ module Prism
         offset_cache = build_offset_cache(source)
         result =
           begin
-            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache)
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
           rescue ::Parser::SyntaxError
             raise if !recover
           end
@@ -149,17 +149,17 @@ module Prism
           Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, [])
         when :embdoc_term
           Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, [])
-        when :incomplete_variable_class, :incomplete_variable_class_3_3_0
+        when :incomplete_variable_class, :incomplete_variable_class_3_3
           location = location.copy(length: location.length + 1)
           diagnostic_location = build_range(location, offset_cache)
 
           Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, [])
-        when :incomplete_variable_instance, :incomplete_variable_instance_3_3_0
+        when :incomplete_variable_instance, :incomplete_variable_instance_3_3
           location = location.copy(length: location.length + 1)
           diagnostic_location = build_range(location, offset_cache)
 
           Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, [])
-        when :invalid_variable_global, :invalid_variable_global_3_3_0
+        when :invalid_variable_global, :invalid_variable_global_3_3
           Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, [])
         when :module_in_method
           Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, [])

data/lib/prism/translation/ripper.rb

@@ -19,31 +19,31 @@ module Prism
     # The main known difference is that we may omit dispatching some events in
     # some cases. This impacts the following events:
     #
-    # * on_assign_error
-    # * on_comma
-    # * on_ignored_nl
-    # * on_ignored_sp
-    # * on_kw
-    # * on_label_end
-    # * on_lbrace
-    # * on_lbracket
-    # * on_lparen
-    # * on_nl
-    # * on_op
-    # * on_operator_ambiguous
-    # * on_rbrace
-    # * on_rbracket
-    # * on_rparen
-    # * on_semicolon
-    # * on_sp
-    # * on_symbeg
-    # * on_tstring_beg
-    # * on_tstring_end
+    # - on_assign_error
+    # - on_comma
+    # - on_ignored_nl
+    # - on_ignored_sp
+    # - on_kw
+    # - on_label_end
+    # - on_lbrace
+    # - on_lbracket
+    # - on_lparen
+    # - on_nl
+    # - on_op
+    # - on_operator_ambiguous
+    # - on_rbrace
+    # - on_rbracket
+    # - on_rparen
+    # - on_semicolon
+    # - on_sp
+    # - on_symbeg
+    # - on_tstring_beg
+    # - on_tstring_end
     #
     class Ripper < Compiler
       # Parses the given Ruby program read from +src+.
       # +src+ must be a String or an IO or a object with a #gets method.
-      def Ripper.parse(src, filename = "(ripper)", lineno = 1)
+      def self.parse(src, filename = "(ripper)", lineno = 1)
         new(src, filename, lineno).parse
       end
 
@@ -54,22 +54,22 @@ module Prism
       # By default, this method does not handle syntax errors in +src+,
       # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
       #
-      #   require 'ripper'
-      #   require 'pp'
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.lex("def m(a) nil end")
-      #   #=> [[[1,  0], :on_kw,     "def", FNAME    ],
-      #        [[1,  3], :on_sp,     " ",   FNAME    ],
-      #        [[1,  4], :on_ident,  "m",   ENDFN    ],
-      #        [[1,  5], :on_lparen, "(",   BEG|LABEL],
-      #        [[1,  6], :on_ident,  "a",   ARG      ],
-      #        [[1,  7], :on_rparen, ")",   ENDFN    ],
-      #        [[1,  8], :on_sp,     " ",   BEG      ],
-      #        [[1,  9], :on_kw,     "nil", END      ],
-      #        [[1, 12], :on_sp,     " ",   END      ],
-      #        [[1, 13], :on_kw,     "end", END      ]]
+      #     pp Ripper.lex("def m(a) nil end")
+      #     #=> [[[1,  0], :on_kw,     "def", FNAME    ],
+      #          [[1,  3], :on_sp,     " ",   FNAME    ],
+      #          [[1,  4], :on_ident,  "m",   ENDFN    ],
+      #          [[1,  5], :on_lparen, "(",   BEG|LABEL],
+      #          [[1,  6], :on_ident,  "a",   ARG      ],
+      #          [[1,  7], :on_rparen, ")",   ENDFN    ],
+      #          [[1,  8], :on_sp,     " ",   BEG      ],
+      #          [[1,  9], :on_kw,     "nil", END      ],
+      #          [[1, 12], :on_sp,     " ",   END      ],
+      #          [[1, 13], :on_kw,     "end", END      ]]
       #
-      def Ripper.lex(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.lex(src, filename = "-", lineno = 1, raise_errors: false)
         result = Prism.lex_compat(src, filepath: filename, line: lineno)
 
         if result.failure? && raise_errors
@@ -368,17 +368,17 @@ module Prism
       # returning +nil+ in such cases. Use the +raise_errors+ keyword
       # to raise a SyntaxError for an error in +src+.
       #
-      #   require "ripper"
-      #   require "pp"
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.sexp("def m(a) nil end")
-      #     #=> [:program,
-      #          [[:def,
-      #           [:@ident, "m", [1, 4]],
-      #           [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil, nil, nil]],
-      #           [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
+      #     pp Ripper.sexp("def m(a) nil end")
+      #       #=> [:program,
+      #            [[:def,
+      #             [:@ident, "m", [1, 4]],
+      #             [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil, nil, nil]],
+      #             [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
       #
-      def Ripper.sexp(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.sexp(src, filename = "-", lineno = 1, raise_errors: false)
         builder = SexpBuilderPP.new(src, filename, lineno)
         sexp = builder.parse
         if builder.error?
@@ -397,23 +397,23 @@ module Prism
       # returning +nil+ in such cases. Use the +raise_errors+ keyword
       # to raise a SyntaxError for an error in +src+.
       #
-      #   require 'ripper'
-      #   require 'pp'
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.sexp_raw("def m(a) nil end")
-      #     #=> [:program,
-      #          [:stmts_add,
-      #           [:stmts_new],
-      #           [:def,
-      #            [:@ident, "m", [1, 4]],
-      #            [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
-      #            [:bodystmt,
-      #             [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
-      #             nil,
-      #             nil,
-      #             nil]]]]
+      #     pp Ripper.sexp_raw("def m(a) nil end")
+      #       #=> [:program,
+      #            [:stmts_add,
+      #             [:stmts_new],
+      #             [:def,
+      #              [:@ident, "m", [1, 4]],
+      #              [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+      #              [:bodystmt,
+      #               [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+      #               nil,
+      #               nil,
+      #               nil]]]]
       #
-      def Ripper.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false)
         builder = SexpBuilder.new(src, filename, lineno)
         sexp = builder.parse
         if builder.error?
@@ -1456,16 +1456,16 @@ module Prism
       # ^^^^^^^^
       def visit_constant_path_node(node)
         if node.parent.nil?
-          bounds(node.child.location)
-          child = on_const(node.child.name.to_s)
+          bounds(node.name_loc)
+          child = on_const(node.name.to_s)
 
           bounds(node.location)
           on_top_const_ref(child)
         else
           parent = visit(node.parent)
 
-          bounds(node.child.location)
-          child = on_const(node.child.name.to_s)
+          bounds(node.name_loc)
+          child = on_const(node.name.to_s)
 
           bounds(node.location)
           on_const_path_ref(parent, child)
@@ -1488,16 +1488,16 @@ module Prism
       # Visit a constant path that is part of a write node.
       private def visit_constant_path_write_node_target(node)
         if node.parent.nil?
-          bounds(node.child.location)
-          child = on_const(node.child.name.to_s)
+          bounds(node.name_loc)
+          child = on_const(node.name.to_s)
 
           bounds(node.location)
           on_top_const_field(child)
         else
           parent = visit(node.parent)
 
-          bounds(node.child.location)
-          child = on_const(node.child.name.to_s)
+          bounds(node.name_loc)
+          child = on_const(node.name.to_s)
 
           bounds(node.location)
           on_const_path_field(parent, child)
@@ -3267,7 +3267,11 @@ module Prism
 
       # Lazily initialize the parse result.
       def result
-        @result ||= Prism.parse(source)
+        @result ||=
+          begin
+            scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]]
+            Prism.parse(source, scopes: scopes)
+          end
       end
 
       ##########################################################################

data/lib/prism/translation/ruby_parser.rb

@@ -442,9 +442,9 @@ module Prism
         # ^^^^^^^^
         def visit_constant_path_node(node)
           if node.parent.nil?
-            s(node, :colon3, node.child.name)
+            s(node, :colon3, node.name)
           else
-            s(node, :colon2, visit(node.parent), node.child.name)
+            s(node, :colon2, visit(node.parent), node.name)
           end
         end
 
@@ -805,17 +805,29 @@ module Prism
         # if /foo #{bar}/ then end
         #    ^^^^^^^^^^^^
         def visit_interpolated_match_last_line_node(node)
-          s(node, :match, s(node, :dregx).concat(visit_interpolated_parts(node.parts)))
+          parts = visit_interpolated_parts(node.parts)
+          regexp =
+            if parts.length == 1
+              s(node, :lit, Regexp.new(parts.first, node.options))
+            else
+              s(node, :dregx).concat(parts).tap do |result|
+                options = node.options
+                result << options if options != 0
+              end
+            end
+
+          s(node, :match, regexp)
         end
 
         # /foo #{bar}/
         # ^^^^^^^^^^^^
         def visit_interpolated_regular_expression_node(node)
-          if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }
-            unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :lit, Regexp.new(unescaped, node.options))
+          parts = visit_interpolated_parts(node.parts)
+
+          if parts.length == 1
+            s(node, :lit, Regexp.new(parts.first, node.options))
           else
-            s(node, :dregx).concat(visit_interpolated_parts(node.parts)).tap do |result|
+            s(node, :dregx).concat(parts).tap do |result|
               options = node.options
               result << options if options != 0
             end
@@ -825,45 +837,71 @@ module Prism
         # "foo #{bar}"
         # ^^^^^^^^^^^^
         def visit_interpolated_string_node(node)
-          if (node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }) ||
-             (node.opening.nil? && node.parts.all? { |part| part.is_a?(StringNode) && !part.opening_loc.nil? })
-             unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :str, unescaped)
-          else
-            s(node, :dstr).concat(visit_interpolated_parts(node.parts))
-          end
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(node, :str, parts.first) : s(node, :dstr).concat(parts)
         end
 
         # :"foo #{bar}"
         # ^^^^^^^^^^^^^
         def visit_interpolated_symbol_node(node)
-          if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }
-            unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join
-            s(node, :lit, unescaped.to_sym)
-          else
-            s(node, :dsym).concat(visit_interpolated_parts(node.parts))
-          end
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(node, :lit, parts.first.to_sym) : s(node, :dsym).concat(parts)
         end
 
         # `foo #{bar}`
         # ^^^^^^^^^^^^
         def visit_interpolated_x_string_node(node)
-          children = visit_interpolated_parts(node.parts)
-          s(node.heredoc? ? node.parts.first : node, :dxstr).concat(children)
+          source = node.heredoc? ? node.parts.first : node
+          parts = visit_interpolated_parts(node.parts)
+          parts.length == 1 ? s(source, :xstr, parts.first) : s(source, :dxstr).concat(parts)
         end
 
         # Visit the interpolated content of the string-like node.
         private def visit_interpolated_parts(parts)
-          parts.each_with_object([]).with_index do |(part, results), index|
-            if index == 0
-              if part.is_a?(StringNode)
-                results << part.unescaped
+          visited = []
+          parts.each do |part|
+            result = visit(part)
+
+            if result[0] == :evstr && result[1]
+              if result[1][0] == :str
+                visited << result[1]
+              elsif result[1][0] == :dstr
+                visited.concat(result[1][1..-1])
+              else
+                visited << result
+              end
+            else
+              visited << result
+            end
+          end
+
+          state = :beginning #: :beginning | :string_content | :interpolated_content
+
+          visited.each_with_object([]) do |result, results|
+            case state
+            when :beginning
+              if result.is_a?(String)
+                results << result
+                state = :string_content
+              elsif result.is_a?(Array) && result[0] == :str
+                results << result[1]
+                state = :string_content
               else
                 results << ""
-                results << visit(part)
+                results << result
+                state = :interpolated_content
+              end
+            when :string_content
+              if result.is_a?(String)
+                results[0] << result
+              elsif result.is_a?(Array) && result[0] == :str
+                results[0] << result[1]
+              else
+                results << result
+                state = :interpolated_content
               end
             else
-              results << visit(part)
+              results << result
             end
           end
         end
@@ -1297,7 +1335,7 @@ module Prism
         # __FILE__
         # ^^^^^^^^
         def visit_source_file_node(node)
-          s(node, :str, file)
+          s(node, :str, node.filepath)
         end
 
         # __LINE__
@@ -1498,13 +1536,13 @@ module Prism
       # Parse the given source and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse(source, filepath = "(string)")
-        translate(Prism.parse(source), filepath)
+        translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath)
       end
 
       # Parse the given file and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse_file(filepath)
-        translate(Prism.parse_file(filepath), filepath)
+        translate(Prism.parse_file(filepath, scopes: [[]]), filepath)
       end
 
       class << self

data/lib/prism.rb

@@ -18,10 +18,10 @@ module Prism
   autoload :Dispatcher, "prism/dispatcher"
   autoload :DotVisitor, "prism/dot_visitor"
   autoload :DSL, "prism/dsl"
+  autoload :InspectVisitor, "prism/inspect_visitor"
   autoload :LexCompat, "prism/lex_compat"
   autoload :LexRipper, "prism/lex_compat"
   autoload :MutationCompiler, "prism/mutation_compiler"
-  autoload :NodeInspector, "prism/node_inspector"
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
   autoload :Reflection, "prism/reflection"
@@ -37,7 +37,7 @@ module Prism
   private_constant :LexRipper
 
   # :call-seq:
-  #   Prism::lex_compat(source, **options) -> ParseResult
+  #   Prism::lex_compat(source, **options) -> LexCompat::Result
   #
   # Returns a parse result whose value is an array of tokens that closely
   # resembles the return value of Ripper::lex. The main difference is that the
@@ -67,6 +67,7 @@ module Prism
   end
 end
 
+require_relative "prism/polyfill/byteindex"
 require_relative "prism/node"
 require_relative "prism/node_ext"
 require_relative "prism/parse_result"