prism 0.17.1 → 0.19.0

data/lib/prism/dsl.rb

@@ -63,13 +63,13 @@ module Prism
     end
 
     # Create a new ArgumentsNode node
-    def ArgumentsNode(arguments, flags, location = Location())
-      ArgumentsNode.new(arguments, flags, location)
+    def ArgumentsNode(flags, arguments, location = Location())
+      ArgumentsNode.new(flags, arguments, location)
     end
 
     # Create a new ArrayNode node
-    def ArrayNode(elements, opening_loc, closing_loc, location = Location())
-      ArrayNode.new(elements, opening_loc, closing_loc, location)
+    def ArrayNode(flags, elements, opening_loc, closing_loc, location = Location())
+      ArrayNode.new(flags, elements, opening_loc, closing_loc, location)
     end
 
     # Create a new ArrayPatternNode node
@@ -108,8 +108,8 @@ module Prism
     end
 
     # Create a new BlockNode node
-    def BlockNode(locals, parameters, body, opening_loc, closing_loc, location = Location())
-      BlockNode.new(locals, parameters, body, opening_loc, closing_loc, location)
+    def BlockNode(locals, locals_body_index, parameters, body, opening_loc, closing_loc, location = Location())
+      BlockNode.new(locals, locals_body_index, parameters, body, opening_loc, closing_loc, location)
     end
 
     # Create a new BlockParameterNode node
@@ -128,23 +128,28 @@ module Prism
     end
 
     # Create a new CallAndWriteNode node
-    def CallAndWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
-      CallAndWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
+    def CallAndWriteNode(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator_loc, value, location = Location())
+      CallAndWriteNode.new(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator_loc, value, location)
     end
 
     # Create a new CallNode node
-    def CallNode(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, block, flags, name, location = Location())
-      CallNode.new(receiver, call_operator_loc, message_loc, opening_loc, arguments, closing_loc, block, flags, name, location)
+    def CallNode(flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, block, location = Location())
+      CallNode.new(flags, receiver, call_operator_loc, name, message_loc, opening_loc, arguments, closing_loc, block, location)
     end
 
     # Create a new CallOperatorWriteNode node
-    def CallOperatorWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location = Location())
-      CallOperatorWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator, operator_loc, value, location)
+    def CallOperatorWriteNode(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator, operator_loc, value, location = Location())
+      CallOperatorWriteNode.new(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator, operator_loc, value, location)
     end
 
     # Create a new CallOrWriteNode node
-    def CallOrWriteNode(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location = Location())
-      CallOrWriteNode.new(receiver, call_operator_loc, message_loc, flags, read_name, write_name, operator_loc, value, location)
+    def CallOrWriteNode(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator_loc, value, location = Location())
+      CallOrWriteNode.new(flags, receiver, call_operator_loc, message_loc, read_name, write_name, operator_loc, value, location)
+    end
+
+    # Create a new CallTargetNode node
+    def CallTargetNode(flags, receiver, call_operator_loc, name, message_loc, location = Location())
+      CallTargetNode.new(flags, receiver, call_operator_loc, name, message_loc, location)
     end
 
     # Create a new CapturePatternNode node
@@ -152,6 +157,11 @@ module Prism
       CapturePatternNode.new(value, target, operator_loc, location)
     end
 
+    # Create a new CaseMatchNode node
+    def CaseMatchNode(predicate, conditions, consequent, case_keyword_loc, end_keyword_loc, location = Location())
+      CaseMatchNode.new(predicate, conditions, consequent, case_keyword_loc, end_keyword_loc, location)
+    end
+
     # Create a new CaseNode node
     def CaseNode(predicate, conditions, consequent, case_keyword_loc, end_keyword_loc, location = Location())
       CaseNode.new(predicate, conditions, consequent, case_keyword_loc, end_keyword_loc, location)
@@ -253,8 +263,8 @@ module Prism
     end
 
     # Create a new DefNode node
-    def DefNode(name, name_loc, receiver, parameters, body, locals, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location = Location())
-      DefNode.new(name, name_loc, receiver, parameters, body, locals, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location)
+    def DefNode(name, name_loc, receiver, parameters, body, locals, locals_body_index, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location = Location())
+      DefNode.new(name, name_loc, receiver, parameters, body, locals, locals_body_index, def_keyword_loc, operator_loc, lparen_loc, rparen_loc, equal_loc, end_keyword_loc, location)
     end
 
     # Create a new DefinedNode node
@@ -293,8 +303,8 @@ module Prism
     end
 
     # Create a new FlipFlopNode node
-    def FlipFlopNode(left, right, operator_loc, flags, location = Location())
-      FlipFlopNode.new(left, right, operator_loc, flags, location)
+    def FlipFlopNode(flags, left, right, operator_loc, location = Location())
+      FlipFlopNode.new(flags, left, right, operator_loc, location)
     end
 
     # Create a new FloatNode node
@@ -363,8 +373,8 @@ module Prism
     end
 
     # Create a new IfNode node
-    def IfNode(if_keyword_loc, predicate, statements, consequent, end_keyword_loc, location = Location())
-      IfNode.new(if_keyword_loc, predicate, statements, consequent, end_keyword_loc, location)
+    def IfNode(if_keyword_loc, predicate, then_keyword_loc, statements, consequent, end_keyword_loc, location = Location())
+      IfNode.new(if_keyword_loc, predicate, then_keyword_loc, statements, consequent, end_keyword_loc, location)
     end
 
     # Create a new ImaginaryNode node
@@ -377,24 +387,34 @@ module Prism
       ImplicitNode.new(value, location)
     end
 
+    # Create a new ImplicitRestNode node
+    def ImplicitRestNode(location = Location())
+      ImplicitRestNode.new(location)
+    end
+
     # Create a new InNode node
     def InNode(pattern, statements, in_loc, then_loc, location = Location())
       InNode.new(pattern, statements, in_loc, then_loc, location)
     end
 
     # Create a new IndexAndWriteNode node
-    def IndexAndWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
-      IndexAndWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
+    def IndexAndWriteNode(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator_loc, value, location = Location())
+      IndexAndWriteNode.new(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator_loc, value, location)
     end
 
     # Create a new IndexOperatorWriteNode node
-    def IndexOperatorWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location = Location())
-      IndexOperatorWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator, operator_loc, value, location)
+    def IndexOperatorWriteNode(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator, operator_loc, value, location = Location())
+      IndexOperatorWriteNode.new(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator, operator_loc, value, location)
     end
 
     # Create a new IndexOrWriteNode node
-    def IndexOrWriteNode(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location = Location())
-      IndexOrWriteNode.new(receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, flags, operator_loc, value, location)
+    def IndexOrWriteNode(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator_loc, value, location = Location())
+      IndexOrWriteNode.new(flags, receiver, call_operator_loc, opening_loc, arguments, closing_loc, block, operator_loc, value, location)
+    end
+
+    # Create a new IndexTargetNode node
+    def IndexTargetNode(flags, receiver, opening_loc, arguments, closing_loc, block, location = Location())
+      IndexTargetNode.new(flags, receiver, opening_loc, arguments, closing_loc, block, location)
     end
 
     # Create a new InstanceVariableAndWriteNode node
@@ -433,13 +453,13 @@ module Prism
     end
 
     # Create a new InterpolatedMatchLastLineNode node
-    def InterpolatedMatchLastLineNode(opening_loc, parts, closing_loc, flags, location = Location())
-      InterpolatedMatchLastLineNode.new(opening_loc, parts, closing_loc, flags, location)
+    def InterpolatedMatchLastLineNode(flags, opening_loc, parts, closing_loc, location = Location())
+      InterpolatedMatchLastLineNode.new(flags, opening_loc, parts, closing_loc, location)
     end
 
     # Create a new InterpolatedRegularExpressionNode node
-    def InterpolatedRegularExpressionNode(opening_loc, parts, closing_loc, flags, location = Location())
-      InterpolatedRegularExpressionNode.new(opening_loc, parts, closing_loc, flags, location)
+    def InterpolatedRegularExpressionNode(flags, opening_loc, parts, closing_loc, location = Location())
+      InterpolatedRegularExpressionNode.new(flags, opening_loc, parts, closing_loc, location)
     end
 
     # Create a new InterpolatedStringNode node
@@ -458,8 +478,8 @@ module Prism
     end
 
     # Create a new KeywordHashNode node
-    def KeywordHashNode(elements, location = Location())
-      KeywordHashNode.new(elements, location)
+    def KeywordHashNode(flags, elements, location = Location())
+      KeywordHashNode.new(flags, elements, location)
     end
 
     # Create a new KeywordRestParameterNode node
@@ -468,8 +488,8 @@ module Prism
     end
 
     # Create a new LambdaNode node
-    def LambdaNode(locals, operator_loc, opening_loc, closing_loc, parameters, body, location = Location())
-      LambdaNode.new(locals, operator_loc, opening_loc, closing_loc, parameters, body, location)
+    def LambdaNode(locals, locals_body_index, operator_loc, opening_loc, closing_loc, parameters, body, location = Location())
+      LambdaNode.new(locals, locals_body_index, operator_loc, opening_loc, closing_loc, parameters, body, location)
     end
 
     # Create a new LocalVariableAndWriteNode node
@@ -503,8 +523,8 @@ module Prism
     end
 
     # Create a new MatchLastLineNode node
-    def MatchLastLineNode(opening_loc, content_loc, closing_loc, unescaped, flags, location = Location())
-      MatchLastLineNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
+    def MatchLastLineNode(flags, opening_loc, content_loc, closing_loc, unescaped, location = Location())
+      MatchLastLineNode.new(flags, opening_loc, content_loc, closing_loc, unescaped, location)
     end
 
     # Create a new MatchPredicateNode node
@@ -518,8 +538,8 @@ module Prism
     end
 
     # Create a new MatchWriteNode node
-    def MatchWriteNode(call, locals, location = Location())
-      MatchWriteNode.new(call, locals, location)
+    def MatchWriteNode(call, targets, location = Location())
+      MatchWriteNode.new(call, targets, location)
     end
 
     # Create a new MissingNode node
@@ -557,6 +577,11 @@ module Prism
       NoKeywordsParameterNode.new(operator_loc, keyword_loc, location)
     end
 
+    # Create a new NumberedParametersNode node
+    def NumberedParametersNode(maximum, location = Location())
+      NumberedParametersNode.new(maximum, location)
+    end
+
     # Create a new NumberedReferenceReadNode node
     def NumberedReferenceReadNode(number, location = Location())
       NumberedReferenceReadNode.new(number, location)
@@ -613,8 +638,8 @@ module Prism
     end
 
     # Create a new RangeNode node
-    def RangeNode(left, right, operator_loc, flags, location = Location())
-      RangeNode.new(left, right, operator_loc, flags, location)
+    def RangeNode(flags, left, right, operator_loc, location = Location())
+      RangeNode.new(flags, left, right, operator_loc, location)
     end
 
     # Create a new RationalNode node
@@ -628,8 +653,8 @@ module Prism
     end
 
     # Create a new RegularExpressionNode node
-    def RegularExpressionNode(opening_loc, content_loc, closing_loc, unescaped, flags, location = Location())
-      RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
+    def RegularExpressionNode(flags, opening_loc, content_loc, closing_loc, unescaped, location = Location())
+      RegularExpressionNode.new(flags, opening_loc, content_loc, closing_loc, unescaped, location)
     end
 
     # Create a new RequiredKeywordParameterNode node
@@ -702,11 +727,6 @@ module Prism
       StatementsNode.new(body, location)
     end
 
-    # Create a new StringConcatNode node
-    def StringConcatNode(left, right, location = Location())
-      StringConcatNode.new(left, right, location)
-    end
-
     # Create a new StringNode node
     def StringNode(flags, opening_loc, content_loc, closing_loc, unescaped, location = Location())
       StringNode.new(flags, opening_loc, content_loc, closing_loc, unescaped, location)
@@ -718,8 +738,8 @@ module Prism
     end
 
     # Create a new SymbolNode node
-    def SymbolNode(opening_loc, value_loc, closing_loc, unescaped, location = Location())
-      SymbolNode.new(opening_loc, value_loc, closing_loc, unescaped, location)
+    def SymbolNode(flags, opening_loc, value_loc, closing_loc, unescaped, location = Location())
+      SymbolNode.new(flags, opening_loc, value_loc, closing_loc, unescaped, location)
     end
 
     # Create a new TrueNode node
@@ -733,13 +753,13 @@ module Prism
     end
 
     # Create a new UnlessNode node
-    def UnlessNode(keyword_loc, predicate, statements, consequent, end_keyword_loc, location = Location())
-      UnlessNode.new(keyword_loc, predicate, statements, consequent, end_keyword_loc, location)
+    def UnlessNode(keyword_loc, predicate, then_keyword_loc, statements, consequent, end_keyword_loc, location = Location())
+      UnlessNode.new(keyword_loc, predicate, then_keyword_loc, statements, consequent, end_keyword_loc, location)
     end
 
     # Create a new UntilNode node
-    def UntilNode(keyword_loc, closing_loc, predicate, statements, flags, location = Location())
-      UntilNode.new(keyword_loc, closing_loc, predicate, statements, flags, location)
+    def UntilNode(flags, keyword_loc, closing_loc, predicate, statements, location = Location())
+      UntilNode.new(flags, keyword_loc, closing_loc, predicate, statements, location)
     end
 
     # Create a new WhenNode node
@@ -748,13 +768,13 @@ module Prism
     end
 
     # Create a new WhileNode node
-    def WhileNode(keyword_loc, closing_loc, predicate, statements, flags, location = Location())
-      WhileNode.new(keyword_loc, closing_loc, predicate, statements, flags, location)
+    def WhileNode(flags, keyword_loc, closing_loc, predicate, statements, location = Location())
+      WhileNode.new(flags, keyword_loc, closing_loc, predicate, statements, location)
     end
 
     # Create a new XStringNode node
-    def XStringNode(opening_loc, content_loc, closing_loc, unescaped, location = Location())
-      XStringNode.new(opening_loc, content_loc, closing_loc, unescaped, location)
+    def XStringNode(flags, opening_loc, content_loc, closing_loc, unescaped, location = Location())
+      XStringNode.new(flags, opening_loc, content_loc, closing_loc, unescaped, location)
     end
 
     # Create a new YieldNode node

data/lib/prism/ffi.rb

@@ -14,7 +14,7 @@ module Prism
 
     # Define the library that we will be pulling functions from. Note that this
     # must align with the build shared library from make/rake.
-    ffi_lib File.expand_path("../../build/librubyparser.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
+    ffi_lib File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
 
     # Convert a native C type declaration into a symbol that FFI understands.
     # For example:
@@ -72,7 +72,8 @@ module Prism
       "pm_serialize_parse",
       "pm_serialize_parse_comments",
       "pm_serialize_lex",
-      "pm_serialize_parse_lex"
+      "pm_serialize_parse_lex",
+      "pm_parse_success_p"
     )
 
     load_exported_functions_from(
@@ -230,7 +231,7 @@ module Prism
         loader = Serialize::Loader.new(source, buffer.read)
 
         loader.load_header
-        loader.load_force_encoding
+        loader.load_encoding
         loader.load_start_line
         loader.load_comments
       end
@@ -254,10 +255,10 @@ module Prism
         loader = Serialize::Loader.new(source, buffer.read)
 
         tokens = loader.load_tokens
-        node, comments, magic_comments, errors, warnings = loader.load_nodes
+        node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
         tokens.each { |token,| token.value.force_encoding(loader.encoding) }
 
-        ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
+        ParseResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
       end
     end
 
@@ -268,6 +269,18 @@ module Prism
       end
     end
 
+    # Mirror the Prism.parse_success? API by using the serialization API.
+    def parse_success?(code, **options)
+      LibRubyParser.pm_parse_success_p(code, code.bytesize, dump_options(options))
+    end
+
+    # Mirror the Prism.parse_file_success? API by using the serialization API.
+    def parse_file_success?(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse_success?(string.read, **options, filepath: filepath)
+      end
+    end
+
     private
 
     # Convert the given options into a serialized options string.
@@ -299,7 +312,7 @@ module Prism
       values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
 
       template << "C"
-      values << (options[:verbose] ? 0 : 1)
+      values << (options.fetch(:verbose, true) ? 0 : 1)
 
       template << "L"
       if (scopes = options[:scopes])

data/lib/prism/lex_compat.rb

@@ -610,6 +610,7 @@ module Prism
       result = Prism.lex(source, **options)
       result_value = result.value
       previous_state = nil
+      last_heredoc_end = nil
 
       # In previous versions of Ruby, Ripper wouldn't flush the bom before the
       # first token, so we had to have a hack in place to account for that. This
@@ -664,6 +665,7 @@ module Prism
           when :on_heredoc_end
             # Heredoc end tokens can be emitted in an odd order, so we don't
             # want to bother comparing the state on them.
+            last_heredoc_end = token.location.end_offset
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
           when :on_ident
             if lex_state == Ripper::EXPR_END
@@ -729,16 +731,24 @@ module Prism
             # comment and there is still whitespace after the comment, then
             # Ripper will append a on_nl token (even though there isn't
             # necessarily a newline). We mirror that here.
-            start_offset = previous_token.location.end_offset
-            end_offset = token.location.start_offset
+            if previous_token.type == :COMMENT
+              # If the comment is at the start of a heredoc: <<HEREDOC # comment
+              # then the comment's end_offset is up near the heredoc_beg.
+              # This is not the correct offset to use for figuring out if
+              # there is trailing whitespace after the last token.
+              # Use the greater offset of the two to determine the start of
+              # the trailing whitespace.
+              start_offset = [previous_token.location.end_offset, last_heredoc_end].compact.max
+              end_offset = token.location.start_offset
+
+              if start_offset < end_offset
+                if bom
+                  start_offset += 3
+                  end_offset += 3
+                end
 
-            if previous_token.type == :COMMENT && start_offset < end_offset
-              if bom
-                start_offset += 3
-                end_offset += 3
+                tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state])
               end
-
-              tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state])
             end
 
             Token.new([[lineno, column], event, value, lex_state])
@@ -831,7 +841,7 @@ module Prism
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      ParseResult.new(tokens, result.comments, result.magic_comments, result.errors, result.warnings, [])
+      ParseResult.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, [])
     end
   end
 

data/lib/prism/mutation_compiler.rb

@@ -115,11 +115,21 @@ module Prism
       node.copy(receiver: visit(node.receiver), value: visit(node.value))
     end
 
+    # Copy a CallTargetNode node
+    def visit_call_target_node(node)
+      node.copy(receiver: visit(node.receiver))
+    end
+
     # Copy a CapturePatternNode node
     def visit_capture_pattern_node(node)
       node.copy(value: visit(node.value), target: visit(node.target))
     end
 
+    # Copy a CaseMatchNode node
+    def visit_case_match_node(node)
+      node.copy(predicate: visit(node.predicate), conditions: visit_all(node.conditions), consequent: visit(node.consequent))
+    end
+
     # Copy a CaseNode node
     def visit_case_node(node)
       node.copy(predicate: visit(node.predicate), conditions: visit_all(node.conditions), consequent: visit(node.consequent))
@@ -345,6 +355,11 @@ module Prism
       node.copy(value: visit(node.value))
     end
 
+    # Copy a ImplicitRestNode node
+    def visit_implicit_rest_node(node)
+      node.copy
+    end
+
     # Copy a InNode node
     def visit_in_node(node)
       node.copy(pattern: visit(node.pattern), statements: visit(node.statements))
@@ -365,6 +380,11 @@ module Prism
       node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block), value: visit(node.value))
     end
 
+    # Copy a IndexTargetNode node
+    def visit_index_target_node(node)
+      node.copy(receiver: visit(node.receiver), arguments: visit(node.arguments), block: visit(node.block))
+    end
+
     # Copy a InstanceVariableAndWriteNode node
     def visit_instance_variable_and_write_node(node)
       node.copy(value: visit(node.value))
@@ -487,7 +507,7 @@ module Prism
 
     # Copy a MatchWriteNode node
     def visit_match_write_node(node)
-      node.copy(call: visit(node.call))
+      node.copy(call: visit(node.call), targets: visit_all(node.targets))
     end
 
     # Copy a MissingNode node
@@ -525,6 +545,11 @@ module Prism
       node.copy
     end
 
+    # Copy a NumberedParametersNode node
+    def visit_numbered_parameters_node(node)
+      node.copy
+    end
+
     # Copy a NumberedReferenceReadNode node
     def visit_numbered_reference_read_node(node)
       node.copy
@@ -670,11 +695,6 @@ module Prism
       node.copy(body: visit_all(node.body))
     end
 
-    # Copy a StringConcatNode node
-    def visit_string_concat_node(node)
-      node.copy(left: visit(node.left), right: visit(node.right))
-    end
-
     # Copy a StringNode node
     def visit_string_node(node)
       node.copy