prism 1.9.0 → 2.0.0.rc2

data/lib/prism/string_query.rb

@@ -1,31 +1,54 @@
 # frozen_string_literal: true
 # :markup: markdown
+#--
+# rbs_inline: enabled
 
 module Prism
   # Query methods that allow categorizing strings based on their context for
   # where they could be valid in a Ruby syntax tree.
   class StringQuery
+    # @rbs!
+    #    def self.local?: (String string) -> bool
+    #    def self.constant?: (String string) -> bool
+    #    def self.method_name?: (String string) -> bool
+    #    def self.keyword?: (String string) -> bool
+
     # The string that this query is wrapping.
-    attr_reader :string
+    attr_reader :string #: String
 
     # Initialize a new query with the given string.
+    #
+    #: (String string) -> void
     def initialize(string)
       @string = string
     end
 
     # Whether or not this string is a valid local variable name.
+    #
+    #: () -> bool
     def local?
       StringQuery.local?(string)
     end
 
     # Whether or not this string is a valid constant name.
+    #
+    #: () -> bool
     def constant?
       StringQuery.constant?(string)
     end
 
     # Whether or not this string is a valid method name.
+    #
+    #: () -> bool
     def method_name?
       StringQuery.method_name?(string)
     end
+
+    # Whether or not this string is a valid keyword.
+    #
+    #: () -> bool
+    def keyword?
+      StringQuery.keyword?(string)
+    end
   end
 end

data/lib/prism/translation/parser/builder.rb

@@ -7,12 +7,14 @@ module Prism
       # A builder that knows how to convert more modern Ruby syntax
       # into whitequark/parser gem's syntax tree.
       class Builder < ::Parser::Builders::Default
-        # It represents the `it` block argument, which is not yet implemented in the Parser gem.
+        # It represents the `it` block argument, which is not yet implemented in
+        # the Parser gem.
         def itarg
           n(:itarg, [:it], nil)
         end
 
-        # The following three lines have been added to support the `it` block parameter syntax in the source code below.
+        # The following three lines have been added to support the `it` block
+        # parameter syntax in the source code below.
         #
         #   if args.type == :itarg
         #     block_type = :itblock
@@ -56,6 +58,12 @@ module Prism
               method_call.loc.with_expression(join_exprs(method_call, block)))
           end
         end
+
+        # def foo(&nil); end
+        #         ^^^^
+        def blocknilarg(amper_t, nil_t)
+          n0(:blocknilarg, arg_prefix_map(amper_t, nil_t))
+        end
       end
     end
   end

data/lib/prism/translation/parser/compiler.rb

@@ -6,9 +6,9 @@ module Prism
     class Parser
       # A visitor that knows how to convert a prism syntax tree into the
       # whitequark/parser gem's syntax tree.
-      class Compiler < ::Prism::Compiler
+      class Compiler < ::Prism::Compiler # :nodoc:
         # Raised when the tree is malformed or there is a bug in the compiler.
-        class CompilationError < StandardError
+        class CompilationError < StandardError # :nodoc:
         end
 
         # The Parser::Base instance that is being used to build the AST.
@@ -140,7 +140,7 @@ module Prism
                 if key.opening.nil?
                   builder.match_hash_var([key.unescaped, srange(key.location)])
                 else
-                  builder.match_hash_var_from_str(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc))
+                  builder.match_hash_var_from_str(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.content_loc)])], token(key.closing_loc))
                 end
               else
                 builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc))
@@ -151,9 +151,9 @@ module Prism
               implicit_value = if value.is_a?(CallNode)
                 builder.call_method(nil, nil, [value.name, srange(value.message_loc)])
               elsif value.is_a?(ConstantReadNode)
-                builder.const([value.name, srange(key.value_loc)])
+                builder.const([value.name, srange(key.content_loc)])
               else
-                builder.ident([value.name, srange(key.value_loc)]).updated(:lvar)
+                builder.ident([value.name, srange(key.content_loc)]).updated(:lvar)
               end
 
               builder.pair_keyword([key.unescaped, srange(key)], implicit_value)
@@ -165,7 +165,14 @@ module Prism
           else
             parts =
               if key.is_a?(SymbolNode)
-                [builder.string_internal([key.unescaped, srange(key.value_loc)])]
+                content = key.content
+                if content == ""
+                  []
+                elsif content.include?("\n")
+                  string_nodes_from_line_continuations(key.unescaped, content, key.content_loc.start_offset, key.opening)
+                else
+                  [builder.string_internal([key.unescaped, srange(key.content_loc)])]
+                end
               else
                 visit_all(key.parts)
               end
@@ -297,11 +304,6 @@ module Prism
 
           if node.call_operator_loc.nil?
             case name
-            when :-@
-              case (receiver = node.receiver).type
-              when :integer_node, :float_node, :rational_node, :imaginary_node
-                return visit(numeric_negate(node.message_loc, receiver))
-              end
             when :!
               return visit_block(builder.not_op(token(node.message_loc), token(node.opening_loc), visit(node.receiver), token(node.closing_loc)), block)
             when :=~
@@ -975,17 +977,17 @@ module Prism
             guard = builder.if_guard(token(node.pattern.if_keyword_loc), visit(node.pattern.predicate))
           when UnlessNode
             pattern = within_pattern { |compiler| node.pattern.statements.accept(compiler) }
-            guard = builder.unless_guard(token(node.pattern.keyword_loc), visit(node.pattern.predicate))
+            guard = builder.unless_guard(token(node.pattern.unless_keyword_loc), visit(node.pattern.predicate))
           else
             pattern = within_pattern { |compiler| node.pattern.accept(compiler) }
           end
 
           builder.in_pattern(
-            token(node.in_loc),
+            token(node.in_keyword_loc),
             pattern,
             guard,
-            if (then_loc = node.then_loc)
-              token(then_loc)
+            if (then_keyword_loc = node.then_keyword_loc)
+              token(then_keyword_loc)
             else
               srange_semicolon(node.pattern.location.end_offset, node.statements&.location&.start_offset)
             end,
@@ -1296,7 +1298,7 @@ module Prism
         def visit_match_predicate_node(node)
           builder.match_pattern_p(
             visit(node.value),
-            token(node.operator_loc),
+            token(node.keyword_loc),
             within_pattern { |compiler| node.pattern.accept(compiler) }
           )
         end
@@ -1324,7 +1326,7 @@ module Prism
         # A node that is missing from the syntax tree. This is only used in the
         # case of a syntax error. The parser gem doesn't have such a concept, so
         # we invent our own here.
-        def visit_missing_node(node)
+        def visit_error_recovery_node(node)
           ::AST::Node.new(:missing, [], location: ::Parser::Source::Map.new(srange(node.location)))
         end
 
@@ -1390,6 +1392,12 @@ module Prism
           builder.nil(token(node.location))
         end
 
+        # def foo(&nil); end
+        #         ^^^^
+        def visit_no_block_parameter_node(node)
+          builder.blocknilarg(token(node.operator_loc), token(node.keyword_loc))
+        end
+
         # def foo(**nil); end
         #         ^^^^^
         def visit_no_keywords_parameter_node(node)
@@ -1767,12 +1775,12 @@ module Prism
             end
           else
             parts =
-              if node.value == ""
+              if node.content == ""
                 []
-              elsif node.value.include?("\n")
-                string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening)
+              elsif node.content.include?("\n")
+                string_nodes_from_line_continuations(node.unescaped, node.content, node.content_loc.start_offset, node.opening)
               else
-                [builder.string_internal([node.unescaped, srange(node.value_loc)])]
+                [builder.string_internal([node.unescaped, srange(node.content_loc)])]
               end
 
             builder.symbol_compose(
@@ -1801,9 +1809,9 @@ module Prism
         # bar unless foo
         # ^^^^^^^^^^^^^^
         def visit_unless_node(node)
-          if node.keyword_loc.start_offset == node.location.start_offset
+          if node.unless_keyword_loc.start_offset == node.location.start_offset
             builder.condition(
-              token(node.keyword_loc),
+              token(node.unless_keyword_loc),
               visit(node.predicate),
               if (then_keyword_loc = node.then_keyword_loc)
                 token(then_keyword_loc)
@@ -1819,7 +1827,7 @@ module Prism
             builder.condition_mod(
               visit(node.else_clause),
               visit(node.statements),
-              token(node.keyword_loc),
+              token(node.unless_keyword_loc),
               visit(node.predicate)
             )
           end
@@ -1831,24 +1839,24 @@ module Prism
         # bar until foo
         # ^^^^^^^^^^^^^
         def visit_until_node(node)
-          if node.location.start_offset == node.keyword_loc.start_offset
+          if node.location.start_offset == node.until_keyword_loc.start_offset
             builder.loop(
               :until,
-              token(node.keyword_loc),
+              token(node.until_keyword_loc),
               visit(node.predicate),
               if (do_keyword_loc = node.do_keyword_loc)
                 token(do_keyword_loc)
               else
-                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset)
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset)
               end,
               visit(node.statements),
-              token(node.closing_loc)
+              token(node.end_keyword_loc)
             )
           else
             builder.loop_mod(
               :until,
               visit(node.statements),
-              token(node.keyword_loc),
+              token(node.until_keyword_loc),
               visit(node.predicate)
             )
           end
@@ -1858,7 +1866,7 @@ module Prism
         #           ^^^^^^^^^^^^^
         def visit_when_node(node)
           builder.when(
-            token(node.keyword_loc),
+            token(node.when_keyword_loc),
             visit_all(node.conditions),
             if (then_keyword_loc = node.then_keyword_loc)
               token(then_keyword_loc)
@@ -1875,24 +1883,24 @@ module Prism
         # bar while foo
         # ^^^^^^^^^^^^^
         def visit_while_node(node)
-          if node.location.start_offset == node.keyword_loc.start_offset
+          if node.location.start_offset == node.while_keyword_loc.start_offset
             builder.loop(
               :while,
-              token(node.keyword_loc),
+              token(node.while_keyword_loc),
               visit(node.predicate),
               if (do_keyword_loc = node.do_keyword_loc)
                 token(do_keyword_loc)
               else
-                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset)
+                srange_semicolon(node.predicate.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset)
               end,
               visit(node.statements),
-              token(node.closing_loc)
+              token(node.end_keyword_loc)
             )
           else
             builder.loop_mod(
               :while,
               visit(node.statements),
-              token(node.keyword_loc),
+              token(node.while_keyword_loc),
               visit(node.predicate)
             )
           end
@@ -1967,22 +1975,6 @@ module Prism
           elements
         end
 
-        # Negate the value of a numeric node. This is a special case where you
-        # have a negative sign on one line and then a number on the next line.
-        # In normal Ruby, this will always be a method call. The parser gem,
-        # however, marks this as a numeric literal. We have to massage the tree
-        # here to get it into the correct form.
-        def numeric_negate(message_loc, receiver)
-          case receiver.type
-          when :integer_node, :float_node
-            receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location))
-          when :rational_node
-            receiver.copy(numerator: -receiver.numerator, location: message_loc.join(receiver.location))
-          when :imaginary_node
-            receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location))
-          end
-        end
-
         # Blocks can have a special set of parameters that automatically expand
         # when given arrays if they have a single required parameter and no
         # other parameters.
@@ -2193,7 +2185,7 @@ module Prism
                   else
                     lines.sum do |line|
                       count = line.scan(/(\\*)n/).count { |(backslashes)| backslashes&.length&.odd? }
-                      count -= 1 if !line.end_with?("\n") && count > 0
+                      count -= 1 if line.match?(/(?:\A|[^\\])(?:\\\\)*\\n\z/) && count > 0
                       count
                     end
                   end

data/lib/prism/translation/parser/lexer.rb

@@ -10,7 +10,7 @@ module Prism
     class Parser
       # Accepts a list of prism tokens and converts them into the expected
       # format for the parser gem.
-      class Lexer
+      class Lexer # :nodoc:
         # These tokens are always skipped
         TYPES_ALWAYS_SKIP = Set.new(%i[IGNORED_NEWLINE __END__ EOF])
         private_constant :TYPES_ALWAYS_SKIP
@@ -18,8 +18,6 @@ module Prism
         # The direct translating of types between the two lexers.
         TYPES = {
           # These tokens should never appear in the output of the lexer.
-          MISSING: nil,
-          NOT_PROVIDED: nil,
           EMBDOC_END: nil,
           EMBDOC_LINE: nil,
 
@@ -30,11 +28,13 @@ module Prism
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tXSTRING_BEG,
+          BACKTICK: :tBACK_REF2,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
           BRACE_LEFT: :tLCURLY,
+          BRACE_LEFT_ARGUMENT: :tLBRACE_ARG,
+          BRACE_LEFT_HASH: :tLBRACE,
           BRACE_RIGHT: :tRCURLY,
           BRACKET_LEFT: :tLBRACK2,
           BRACKET_LEFT_ARRAY: :tLBRACK,
@@ -89,6 +89,8 @@ module Prism
           KEYWORD_DEF: :kDEF,
           KEYWORD_DEFINED: :kDEFINED,
           KEYWORD_DO: :kDO,
+          KEYWORD_DO_BLOCK: :kDO_BLOCK,
+          KEYWORD_DO_LAMBDA: :kDO_LAMBDA,
           KEYWORD_DO_LOOP: :kDO_COND,
           KEYWORD_END: :kEND,
           KEYWORD_END_UPCASE: :klEND,
@@ -139,8 +141,10 @@ module Prism
           MINUS_EQUAL: :tOP_ASGN,
           MINUS_GREATER: :tLAMBDA,
           NEWLINE: :tNL,
+          NEWLINE_TERMINATOR: :tNL,
           NUMBERED_REFERENCE: :tNTH_REF,
           PARENTHESIS_LEFT: :tLPAREN2,
+          PARENTHESIS_LEFT_GROUPING: :tLPAREN,
           PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG,
           PARENTHESIS_RIGHT: :tRPAREN,
           PERCENT: :tPERCENT,
@@ -180,41 +184,15 @@ module Prism
           UPLUS: :tUPLUS,
           USTAR: :tSTAR,
           USTAR_STAR: :tDSTAR,
-          WORDS_SEP: :tSPACE
+          WORDS_SEP: :tSPACE,
+          WORDS_SEP_IMPLICIT: :tSPACE,
+          XSTRING_BEGIN: :tXSTRING_BEG
         }
 
-        # These constants represent flags in our lex state. We really, really
-        # don't want to be using them and we really, really don't want to be
-        # exposing them as part of our public API. Unfortunately, we don't have
-        # another way of matching the exact tokens that the parser gem expects
-        # without them. We should find another way to do this, but in the
-        # meantime we'll hide them from the documentation and mark them as
-        # private constants.
-        EXPR_BEG = 0x1 # :nodoc:
-        EXPR_LABEL = 0x400 # :nodoc:
-
-        # It is used to determine whether `do` is of the token type `kDO` or `kDO_LAMBDA`.
-        #
-        # NOTE: In edge cases like `-> (foo = -> (bar) {}) do end`, please note that `kDO` is still returned
-        # instead of `kDO_LAMBDA`, which is expected: https://github.com/ruby/prism/pull/3046
-        LAMBDA_TOKEN_TYPES = Set.new([:kDO_LAMBDA, :tLAMBDA, :tLAMBEG])
-
-        # The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
-        # The following token types are listed as those classified as `tLPAREN`.
-        LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
-          :kBREAK, :tCARET, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
-          :tEQL, :tLPAREN, :tLPAREN2, :tLPAREN_ARG, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS, :tLCURLY
-        ])
-
-        # Types of tokens that are allowed to continue a method call with comments in-between.
-        # For these, the parser gem doesn't emit a newline token after the last comment.
-        COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT])
-        private_constant :COMMENT_CONTINUATION_TYPES
-
         # Heredocs are complex and require us to keep track of a bit of info to refer to later
         HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true)
 
-        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :LAMBDA_TOKEN_TYPES, :LPAREN_CONVERSION_TOKEN_TYPES, :HeredocData
+        private_constant :TYPES, :HeredocData
 
         # The Parser::Source::Buffer that the tokens were lexed from.
         attr_reader :source_buffer
@@ -234,7 +212,7 @@ module Prism
           @offset_cache = offset_cache
         end
 
-        Range = ::Parser::Source::Range # :nodoc:
+        Range = ::Parser::Source::Range
         private_constant :Range
 
         # Convert the prism tokens into the expected format for the parser gem.
@@ -253,7 +231,7 @@ module Prism
           comment_newline_location = nil
 
           while index < length
-            token, state = lexed[index]
+            token = lexed[index]
             index += 1
             next if TYPES_ALWAYS_SKIP.include?(token.type)
 
@@ -261,15 +239,14 @@ module Prism
             value = token.value
             location = range(token.location.start_offset, token.location.end_offset)
 
-            case type
-            when :kDO
-              nearest_lambda_token = tokens.reverse_each.find do |token|
-                LAMBDA_TOKEN_TYPES.include?(token.first)
-              end
+            # A newline deferred past a run of comments is emitted before the
+            # token that follows the last comment.
+            if comment_newline_location && type != :tCOMMENT
+              tokens << [:tNL, [nil, comment_newline_location]]
+              comment_newline_location = nil
+            end
 
-              if nearest_lambda_token&.first == :tLAMBDA
-                type = :kDO_LAMBDA
-              end
+            case type
             when :tCHARACTER
               value.delete_prefix!("?")
               # Character literals behave similar to double-quoted strings. We can use the same escaping mechanism.
@@ -277,7 +254,7 @@ module Prism
             when :tCOMMENT
               if token.type == :EMBDOC_BEGIN
 
-                while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
+                while !((next_token = lexed[index]) && next_token.type == :EMBDOC_END) && (index < length - 1)
                   value += next_token.value
                   index += 1
                 end
@@ -286,30 +263,12 @@ module Prism
                 location = range(token.location.start_offset, next_token.location.end_offset)
                 index += 1
               else
-                is_at_eol = value.chomp!.nil?
-                location = range(token.location.start_offset, token.location.end_offset + (is_at_eol ? 0 : -1))
-
-                prev_token, _ = lexed[index - 2] if index - 2 >= 0
-                next_token, _ = lexed[index]
-
-                is_inline_comment = prev_token&.location&.start_line == token.location.start_line
-                if is_inline_comment && !is_at_eol && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
-                  tokens << [:tCOMMENT, [value, location]]
-
-                  nl_location = range(token.location.end_offset - 1, token.location.end_offset)
-                  tokens << [:tNL, [nil, nl_location]]
-                  next
-                elsif is_inline_comment && next_token&.type == :COMMENT
-                  comment_newline_location = range(token.location.end_offset - 1, token.location.end_offset)
-                elsif comment_newline_location && !COMMENT_CONTINUATION_TYPES.include?(next_token&.type)
-                  tokens << [:tCOMMENT, [value, location]]
-                  tokens << [:tNL, [nil, comment_newline_location]]
-                  comment_newline_location = nil
-                  next
-                end
+                # A carriage return before the terminating newline is part of
+                # the comment token but not of the comment's value.
+                location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp!
               end
             when :tNL
-              next_token, _ = lexed[index]
+              next_token = lexed[index]
               # Newlines after comments are emitted out of order.
               if next_token&.type == :COMMENT
                 comment_newline_location = location
@@ -332,10 +291,6 @@ module Prism
               value.chomp!(":")
             when :tLABEL_END
               value.chomp!(":")
-            when :tLCURLY
-              type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
-            when :tLPAREN2
-              type = :tLPAREN if tokens.empty? || LPAREN_CONVERSION_TOKEN_TYPES.include?(tokens.dig(-1, 0))
             when :tNTH_REF
               value = parse_integer(value.delete_prefix("$"))
             when :tOP_ASGN
@@ -346,8 +301,8 @@ module Prism
               location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
               value = nil
             when :tSTRING_BEG
-              next_token, _ = lexed[index]
-              next_next_token, _ = lexed[index + 1]
+              next_token = lexed[index]
+              next_next_token = lexed[index + 1]
               basic_quotes = value == '"' || value == "'"
 
               if basic_quotes && next_token&.type == :STRING_END
@@ -415,7 +370,7 @@ module Prism
                 while token.type == :STRING_CONTENT
                   current_length += token.value.bytesize
                   # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
-                  prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                  prev_token = lexed[index - 2] if index - 2 >= 0
                   is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
                   # The parser gem only removes indentation when the heredoc is not nested
                   not_nested = heredoc_stack.size == 1
@@ -435,7 +390,7 @@ module Prism
                     tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
                     break
                   end
-                  token, _ = lexed[index]
+                  token = lexed[index]
                   index += 1
                 end
               else
@@ -489,17 +444,9 @@ module Prism
                 location = range(token.location.start_offset, token.location.start_offset + 1)
               end
 
-              if percent_array?(quote_stack.pop)
-                prev_token, _ = lexed[index - 2] if index - 2 >= 0
-                empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
-                ends_with_whitespace = prev_token&.type == :WORDS_SEP
-                # parser always emits a space token after content in a percent array, even if no actual whitespace is present.
-                if !empty && !ends_with_whitespace
-                  tokens << [:tSPACE, [nil, range(token.location.start_offset, token.location.start_offset)]]
-                end
-              end
+              quote_stack.pop
             when :tSYMBEG
-              if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
+              if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
                 next_location = token.location.join(next_token.location)
                 type = :tSYMBOL
                 value = next_token.value
@@ -514,13 +461,11 @@ module Prism
                 type = :tIDENTIFIER
               end
             when :tXSTRING_BEG
-              if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
-                # self.`()
-                type = :tBACK_REF2
-              end
               quote_stack.push(value)
             when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
-              if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
+              # The separator that delimits the opener from the first word is
+              # part of the opener for parser, so it is not emitted separately.
+              if (next_token = lexed[index]) && %i[WORDS_SEP WORDS_SEP_IMPLICIT].include?(next_token.type)
                 index += 1
               end
 
@@ -536,6 +481,10 @@ module Prism
             end
           end
 
+          if comment_newline_location
+            tokens << [:tNL, [nil, comment_newline_location]]
+          end
+
           tokens
         end
 
@@ -596,9 +545,9 @@ module Prism
           previous_line = -1
           result = Float::MAX
 
-          while (next_token = lexed[next_token_index]&.first)
+          while (next_token = lexed[next_token_index])
             next_token_index += 1
-            next_next_token, _ = lexed[next_token_index]
+            next_next_token = lexed[next_token_index]
             first_token_on_line = next_token.location.start_column == 0
 
             # String content inside nested heredocs and interpolation is ignored

data/lib/prism/translation/parser.rb

@@ -33,7 +33,7 @@ module Prism
       # The parser gem has a list of diagnostics with a hard-coded set of error
       # messages. We create our own diagnostic class in order to set our own
       # error messages.
-      class PrismDiagnostic < Diagnostic
+      class PrismDiagnostic < Diagnostic # :nodoc:
         # This is the cached message coming from prism.
         attr_reader :message
 

data/lib/prism/translation/parser_current.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 # :markup: markdown
+#--
 # typed: ignore
 
-#
 module Prism
   module Translation
     case RUBY_VERSION