prism 0.24.0 → 0.26.0

data/lib/prism/translation/parser/compiler.rb

@@ -116,7 +116,14 @@ module Prism
               builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value))
             end
           elsif node.value.is_a?(ImplicitNode)
-            builder.pair_label([node.key.unescaped, srange(node.key.location)])
+            if (value = node.value.value).is_a?(LocalVariableReadNode)
+              builder.pair_keyword(
+                [node.key.unescaped, srange(node.key)],
+                builder.ident([value.name, srange(node.key.value_loc)]).updated(:lvar)
+              )
+            else
+              builder.pair_label([node.key.unescaped, srange(node.key.location)])
+            end
           elsif node.operator_loc
             builder.pair(visit(node.key), token(node.operator_loc), visit(node.value))
           elsif node.key.is_a?(SymbolNode) && node.key.opening_loc.nil?
@@ -247,18 +254,30 @@ module Prism
 
           if node.call_operator_loc.nil?
             case name
+            when :-@
+              case (receiver = node.receiver).type
+              when :integer_node, :float_node, :rational_node, :imaginary_node
+                return visit(numeric_negate(node.message_loc, receiver))
+              end
             when :!
               return visit_block(builder.not_op(token(node.message_loc), token(node.opening_loc), visit(node.receiver), token(node.closing_loc)), block)
+            when :=~
+              if (receiver = node.receiver).is_a?(RegularExpressionNode)
+                return builder.match_op(visit(receiver), token(node.message_loc), visit(node.arguments.arguments.first))
+              end
             when :[]
               return visit_block(builder.index(visit(node.receiver), token(node.opening_loc), visit_all(arguments), token(node.closing_loc)), block)
             when :[]=
               if node.message != "[]=" && node.arguments && block.nil? && !node.safe_navigation?
+                arguments = node.arguments.arguments[...-1]
+                arguments << node.block if node.block
+
                 return visit_block(
                   builder.assign(
                     builder.index_asgn(
                       visit(node.receiver),
                       token(node.opening_loc),
-                      visit_all(node.arguments.arguments[...-1]),
+                      visit_all(arguments),
                       token(node.closing_loc),
                     ),
                     srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, ["="]),
@@ -387,9 +406,6 @@ module Prism
 
         # @@foo = 1
         # ^^^^^^^^^
-        #
-        # @@foo, @@bar = 1
-        # ^^^^^  ^^^^^
         def visit_class_variable_write_node(node)
           builder.assign(
             builder.assignable(builder.cvar(token(node.name_loc))),
@@ -682,9 +698,6 @@ module Prism
 
         # $foo = 1
         # ^^^^^^^^
-        #
-        # $foo, $bar = 1
-        # ^^^^  ^^^^
         def visit_global_variable_write_node(node)
           builder.assign(
             builder.assignable(builder.gvar(token(node.name_loc))),
@@ -788,8 +801,9 @@ module Prism
         end
 
         # 1i
+        # ^^
         def visit_imaginary_node(node)
-          visit_numeric(node, builder.complex([node.value, srange(node.location)]))
+          visit_numeric(node, builder.complex([imaginary_value(node), srange(node.location)]))
         end
 
         # { foo: }
@@ -875,9 +889,6 @@ module Prism
 
         # @foo = 1
         # ^^^^^^^^
-        #
-        # @foo, @bar = 1
-        # ^^^^  ^^^^
         def visit_instance_variable_write_node(node)
           builder.assign(
             builder.assignable(builder.ivar(token(node.name_loc))),
@@ -934,16 +945,64 @@ module Prism
         # "foo #{bar}"
         # ^^^^^^^^^^^^
         def visit_interpolated_string_node(node)
-          if node.opening&.start_with?("<<")
+          if node.heredoc?
             children, closing = visit_heredoc(node)
-            builder.string_compose(token(node.opening_loc), children, closing)
+            opening = token(node.opening_loc)
+
+            start_offset = node.opening_loc.end_offset + 1
+            end_offset = node.parts.first.location.start_offset
+
+            # In the below case, the offsets should be the same:
+            #
+            # <<~HEREDOC
+            #   a #{b}
+            # HEREDOC
+            #
+            # But in this case, the end_offset would be greater than the start_offset:
+            #
+            # <<~HEREDOC
+            #   #{b}
+            # HEREDOC
+            #
+            # So we need to make sure the result node's heredoc range is correct, without updating the children
+            result = if start_offset < end_offset
+              # We need to add a padding string to ensure that the heredoc has correct range for its body
+              padding_string_node = builder.string_internal(["", srange_offsets(start_offset, end_offset)])
+              node_with_correct_location = builder.string_compose(opening, [padding_string_node, *children], closing)
+              # But the padding string should not be included in the final AST, so we need to update the result's children
+              node_with_correct_location.updated(:dstr, children)
+            else
+              builder.string_compose(opening, children, closing)
+            end
+
+            return result
+          end
+
+          parts = if node.parts.one? { |part| part.type == :string_node }
+            node.parts.flat_map do |node|
+              if node.type == :string_node && node.unescaped.lines.count >= 2
+                start_offset = node.content_loc.start_offset
+
+                node.unescaped.lines.map do |line|
+                  end_offset = start_offset + line.length
+                  offsets = srange_offsets(start_offset, end_offset)
+                  start_offset = end_offset
+
+                  builder.string_internal([line, offsets])
+                end
+              else
+                visit(node)
+              end
+            end
           else
-            builder.string_compose(
-              token(node.opening_loc),
-              visit_all(node.parts),
-              token(node.closing_loc)
-            )
+            visit_all(node.parts)
           end
+
+          builder.string_compose(
+            token(node.opening_loc),
+            parts,
+            token(node.closing_loc)
+          )
         end
 
         # :"foo #{bar}"
@@ -959,7 +1018,7 @@ module Prism
         # `foo #{bar}`
         # ^^^^^^^^^^^^
         def visit_interpolated_x_string_node(node)
-          if node.opening.start_with?("<<")
+          if node.heredoc?
             children, closing = visit_heredoc(node)
             builder.xstring_compose(token(node.opening_loc), children, closing)
           else
@@ -990,6 +1049,7 @@ module Prism
         end
 
         # -> {}
+        # ^^^^^
         def visit_lambda_node(node)
           parameters = node.parameters
 
@@ -1021,9 +1081,6 @@ module Prism
 
         # foo = 1
         # ^^^^^^^
-        #
-        # foo, bar = 1
-        # ^^^  ^^^
         def visit_local_variable_write_node(node)
           builder.assign(
             builder.assignable(builder.ident(token(node.name_loc))),
@@ -1062,22 +1119,12 @@ module Prism
 
         # foo in bar
         # ^^^^^^^^^^
-        if RUBY_VERSION >= "3.0"
-          def visit_match_predicate_node(node)
-            builder.match_pattern_p(
-              visit(node.value),
-              token(node.operator_loc),
-              within_pattern { |compiler| node.pattern.accept(compiler) }
-            )
-          end
-        else
-          def visit_match_predicate_node(node)
-            builder.match_pattern(
-              visit(node.value),
-              token(node.operator_loc),
-              within_pattern { |compiler| node.pattern.accept(compiler) }
-            )
-          end
+        def visit_match_predicate_node(node)
+          builder.match_pattern_p(
+            visit(node.value),
+            token(node.operator_loc),
+            within_pattern { |compiler| node.pattern.accept(compiler) }
+          )
         end
 
         # foo => bar
@@ -1263,7 +1310,8 @@ module Prism
         # foo => ^(bar)
         #        ^^^^^^
         def visit_pinned_expression_node(node)
-          builder.pin(token(node.operator_loc), visit(node.expression))
+          expression = builder.begin(token(node.lparen_loc), visit(node.expression), token(node.rparen_loc))
+          builder.pin(token(node.operator_loc), expression)
         end
 
         # foo = 1 and bar => ^foo
@@ -1322,7 +1370,7 @@ module Prism
         # 1r
         # ^^
         def visit_rational_node(node)
-          visit_numeric(node, builder.rational([node.value, srange(node.location)]))
+          visit_numeric(node, builder.rational([rational_value(node), srange(node.location)]))
         end
 
         # redo
@@ -1418,6 +1466,11 @@ module Prism
           builder.self(token(node.location))
         end
 
+        # A shareable constant.
+        def visit_shareable_constant_node(node)
+          visit(node.write)
+        end
+
         # class << self; end
         # ^^^^^^^^^^^^^^^^^^
         def visit_singleton_class_node(node)
@@ -1476,15 +1529,39 @@ module Prism
         # "foo"
         # ^^^^^
         def visit_string_node(node)
-          if node.opening&.start_with?("<<")
-            children, closing = visit_heredoc(InterpolatedStringNode.new(node.send(:source), node.opening_loc, [node.copy(opening_loc: nil, closing_loc: nil, location: node.content_loc)], node.closing_loc, node.location))
+          if node.heredoc?
+            children, closing = visit_heredoc(node.to_interpolated)
             builder.string_compose(token(node.opening_loc), children, closing)
           elsif node.opening == "?"
             builder.character([node.unescaped, srange(node.location)])
           else
+            content_lines = node.content.lines
+            unescaped_lines = node.unescaped.lines
+
+            parts =
+              if content_lines.length <= 1 || unescaped_lines.length <= 1
+                [builder.string_internal([node.unescaped, srange(node.content_loc)])]
+              elsif content_lines.length != unescaped_lines.length
+                # This occurs when we have line continuations in the string. We
+                # need to come back and fix this, but for now this stops the
+                # code from breaking when we encounter it because of trying to
+                # transpose arrays of different lengths.
+                [builder.string_internal([node.unescaped, srange(node.content_loc)])]
+              else
+                start_offset = node.content_loc.start_offset
+
+                [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line|
+                  end_offset = start_offset + content_line.length
+                  offsets = srange_offsets(start_offset, end_offset)
+                  start_offset = end_offset
+
+                  builder.string_internal([unescaped_line, offsets])
+                end
+              end
+
             builder.string_compose(
               token(node.opening_loc),
-              [builder.string_internal([node.unescaped, srange(node.content_loc)])],
+              parts,
               token(node.closing_loc)
             )
           end
@@ -1523,9 +1600,23 @@ module Prism
               builder.symbol([node.unescaped, srange(node.location)])
             end
           else
+            parts = if node.value.lines.one?
+              [builder.string_internal([node.unescaped, srange(node.value_loc)])]
+            else
+              start_offset = node.value_loc.start_offset
+
+              node.value.lines.map do |line|
+                end_offset = start_offset + line.length
+                offsets = srange_offsets(start_offset, end_offset)
+                start_offset = end_offset
+
+                builder.string_internal([line, offsets])
+              end
+            end
+
             builder.symbol_compose(
               token(node.opening_loc),
-              [builder.string_internal([node.unescaped, srange(node.value_loc)])],
+              parts,
               token(node.closing_loc)
             )
           end
@@ -1604,7 +1695,11 @@ module Prism
           builder.when(
             token(node.keyword_loc),
             visit_all(node.conditions),
-            srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset || (node.conditions.last.location.end_offset + 1), [";", "then"]),
+            if node.then_keyword_loc
+              token(node.then_keyword_loc)
+            else
+              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset || (node.conditions.last.location.end_offset + 1), [";"])
+            end,
             visit(node.statements)
           )
         end
@@ -1637,13 +1732,27 @@ module Prism
         # `foo`
         # ^^^^^
         def visit_x_string_node(node)
-          if node.opening&.start_with?("<<")
-            children, closing = visit_heredoc(InterpolatedXStringNode.new(node.opening_loc, [StringNode.new(0, nil, node.content_loc, nil, node.unescaped, node.content_loc)], node.closing_loc, node.location))
+          if node.heredoc?
+            children, closing = visit_heredoc(node.to_interpolated)
             builder.xstring_compose(token(node.opening_loc), children, closing)
           else
+            parts = if node.unescaped.lines.one?
+              [builder.string_internal([node.unescaped, srange(node.content_loc)])]
+            else
+              start_offset = node.content_loc.start_offset
+
+              node.unescaped.lines.map do |line|
+                end_offset = start_offset + line.length
+                offsets = srange_offsets(start_offset, end_offset)
+                start_offset = end_offset
+
+                builder.string_internal([line, offsets])
+              end
+            end
+
             builder.xstring_compose(
               token(node.opening_loc),
-              [builder.string_internal([node.unescaped, srange(node.content_loc)])],
+              parts,
               token(node.closing_loc)
             )
           end
@@ -1687,6 +1796,26 @@ module Prism
           forwarding
         end
 
+        # Because we have mutated the AST to allow for newlines in the middle of
+        # a rational, we need to manually handle the value here.
+        def imaginary_value(node)
+          Complex(0, node.numeric.is_a?(RationalNode) ? rational_value(node.numeric) : node.numeric.value)
+        end
+
+        # Negate the value of a numeric node. This is a special case where you
+        # have a negative sign on one line and then a number on the next line.
+        # In normal Ruby, this will always be a method call. The parser gem,
+        # however, marks this as a numeric literal. We have to massage the tree
+        # here to get it into the correct form.
+        def numeric_negate(message_loc, receiver)
+          case receiver.type
+          when :integer_node, :float_node
+            receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location))
+          when :rational_node, :imaginary_node
+            receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location))
+          end
+        end
+
         # Blocks can have a special set of parameters that automatically expand
         # when given arrays if they have a single required parameter and no
         # other parameters.
@@ -1701,6 +1830,16 @@ module Prism
             parameters.block.nil?
         end
 
+        # Because we have mutated the AST to allow for newlines in the middle of
+        # a rational, we need to manually handle the value here.
+        def rational_value(node)
+          if node.numeric.is_a?(IntegerNode)
+            Rational(node.numeric.value)
+          else
+            Rational(node.slice.gsub(/\s/, "").chomp("r"))
+          end
+        end
+
         # Locations in the parser gem AST are generated using this class. We
         # store a reference to its constant to make it slightly faster to look
         # up.
@@ -1767,7 +1906,7 @@ module Prism
 
         # Visit a heredoc that can be either a string or an xstring.
         def visit_heredoc(node)
-          children = []
+          children = Array.new
           node.parts.each do |part|
             pushing =
               if part.is_a?(StringNode) && part.unescaped.include?("\n")

data/lib/prism/translation/parser/lexer.rb

@@ -167,7 +167,7 @@ module Prism
           TILDE: :tTILDE,
           UAMPERSAND: :tAMPER,
           UCOLON_COLON: :tCOLON3,
-          UDOT_DOT: :tDOT2,
+          UDOT_DOT: :tBDOT2,
           UDOT_DOT_DOT: :tBDOT3,
           UMINUS: :tUMINUS,
           UMINUS_NUM: :tUNARY_NUM,
@@ -177,12 +177,23 @@ module Prism
           WORDS_SEP: :tSPACE
         }
 
-        private_constant :TYPES
+        # These constants represent flags in our lex state. We really, really
+        # don't want to be using them and we really, really don't want to be
+        # exposing them as part of our public API. Unfortunately, we don't have
+        # another way of matching the exact tokens that the parser gem expects
+        # without them. We should find another way to do this, but in the
+        # meantime we'll hide them from the documentation and mark them as
+        # private constants.
+        EXPR_BEG = 0x1 # :nodoc:
+        EXPR_LABEL = 0x400 # :nodoc:
+
+        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL
 
         # The Parser::Source::Buffer that the tokens were lexed from.
         attr_reader :source_buffer
 
-        # An array of prism tokens that we lexed.
+        # An array of tuples that contain prism tokens and their associated lex
+        # state when they were lexed.
         attr_reader :lexed
 
         # A hash that maps offsets in bytes to offsets in characters.
@@ -202,12 +213,16 @@ module Prism
         # Convert the prism tokens into the expected format for the parser gem.
         def to_a
           tokens = []
+
           index = 0
+          length = lexed.length
+
+          heredoc_identifier_stack = []
 
-          while index < lexed.length
-            token, = lexed[index]
+          while index < length
+            token, state = lexed[index]
             index += 1
-            next if token.type == :IGNORED_NEWLINE || token.type == :EOF
+            next if %i[IGNORED_NEWLINE __END__ EOF].include?(token.type)
 
             type = TYPES.fetch(token.type)
             value = token.value
@@ -218,14 +233,18 @@ module Prism
               value.delete_prefix!("?")
             when :tCOMMENT
               if token.type == :EMBDOC_BEGIN
-                until (next_token = lexed[index]) && next_token.type == :EMBDOC_END
+                start_index = index
+
+                while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1)
                   value += next_token.value
                   index += 1
                 end
 
-                value += next_token.value
-                location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index].location.end_offset])
-                index += 1
+                if start_index != index
+                  value += next_token.value
+                  location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index][0].location.end_offset])
+                  index += 1
+                end
               else
                 value.chomp!
                 location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - 1])
@@ -233,7 +252,7 @@ module Prism
             when :tNL
               value = nil
             when :tFLOAT
-              value = Float(value)
+              value = parse_float(value)
             when :tIMAGINARY
               value = parse_complex(value)
             when :tINTEGER
@@ -242,13 +261,15 @@ module Prism
                 location = Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset])
               end
 
-              value = Integer(value)
+              value = parse_integer(value)
             when :tLABEL
               value.chomp!(":")
             when :tLABEL_END
               value.chomp!(":")
+            when :tLCURLY
+              type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
             when :tNTH_REF
-              value = Integer(value.delete_prefix("$"))
+              value = parse_integer(value.delete_prefix("$"))
             when :tOP_ASGN
               value.chomp!("=")
             when :tRATIONAL
@@ -256,31 +277,69 @@ module Prism
             when :tSPACE
               value = nil
             when :tSTRING_BEG
-              if ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_END
+              if token.type == :HEREDOC_START
+                heredoc_identifier_stack.push(value.match(/<<[-~]?["'`]?(?<heredoc_identifier>.*?)["'`]?\z/)[:heredoc_identifier])
+              end
+              if ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_END
                 next_location = token.location.join(next_token.location)
                 type = :tSTRING
                 value = ""
                 location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
                 index += 1
-              elsif ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_CONTENT && (next_next_token = lexed[index + 1]) && next_next_token.type == :STRING_END
+              elsif ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_CONTENT && next_token.value.lines.count <= 1 && (next_next_token = lexed[index + 1][0]) && next_next_token.type == :STRING_END
                 next_location = token.location.join(next_next_token.location)
                 type = :tSTRING
-                value = next_token.value
+                value = next_token.value.gsub("\\\\", "\\")
                 location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
                 index += 2
               elsif value.start_with?("<<")
                 quote = value[2] == "-" || value[2] == "~" ? value[3] : value[2]
-                value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
+                if quote == "`"
+                  type = :tXSTRING_BEG
+                  value = "<<`"
+                else
+                  value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
+                end
+              end
+            when :tSTRING_CONTENT
+              unless (lines = token.value.lines).one?
+                start_offset = offset_cache[token.location.start_offset]
+                lines.map do |line|
+                  newline = line.end_with?("\r\n") ? "\r\n" : "\n"
+                  chomped_line = line.chomp
+                  if match = chomped_line.match(/(?<backslashes>\\+)\z/)
+                    adjustment = match[:backslashes].size / 2
+                    adjusted_line = chomped_line.delete_suffix("\\" * adjustment)
+                    if match[:backslashes].size.odd?
+                      adjusted_line.delete_suffix!("\\")
+                      adjustment += 2
+                    else
+                      adjusted_line << newline
+                    end
+                  else
+                    adjusted_line = line
+                    adjustment = 0
+                  end
+
+                  end_offset = start_offset + adjusted_line.length + adjustment
+                  tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
+                  start_offset = end_offset
+                end
+                next
               end
             when :tSTRING_DVAR
               value = nil
             when :tSTRING_END
-              if token.type == :REGEXP_END
+              if token.type == :HEREDOC_END && value.end_with?("\n")
+                newline_length = value.end_with?("\r\n") ? 2 : 1
+                value = heredoc_identifier_stack.pop
+                location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - newline_length])
+              elsif token.type == :REGEXP_END
                 value = value[0]
                 location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
               end
             when :tSYMBEG
-              if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
                 next_location = token.location.join(next_token.location)
                 type = :tSYMBOL
                 value = next_token.value
@@ -289,9 +348,13 @@ module Prism
                 index += 1
               end
             when :tFID
-              if tokens[-1][0] == :kDEF
+              if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
                 type = :tIDENTIFIER
               end
+            when :tXSTRING_BEG
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :STRING_END
+                type = :tBACK_REF2
+              end
             end
 
             tokens << [type, [value, location]]
@@ -306,6 +369,20 @@ module Prism
 
         private
 
+        # Parse an integer from the string representation.
+        def parse_integer(value)
+          Integer(value)
+        rescue ArgumentError
+          0
+        end
+
+        # Parse a float from the string representation.
+        def parse_float(value)
+          Float(value)
+        rescue ArgumentError
+          0.0
+        end
+
         # Parse a complex from the string representation.
         def parse_complex(value)
           value.chomp!("i")
@@ -313,10 +390,12 @@ module Prism
           if value.end_with?("r")
             Complex(0, parse_rational(value))
           elsif value.start_with?(/0[BbOoDdXx]/)
-            Complex(0, Integer(value))
+            Complex(0, parse_integer(value))
           else
             Complex(0, value)
           end
+        rescue ArgumentError
+          0i
         end
 
         # Parse a rational from the string representation.
@@ -324,10 +403,12 @@ module Prism
           value.chomp!("r")
 
           if value.start_with?(/0[BbOoDdXx]/)
-            Rational(Integer(value))
+            Rational(parse_integer(value))
           else
             Rational(value)
           end
+        rescue ArgumentError
+          0r
         end
       end
     end