yarp 0.12.0 → 0.13.0

data/lib/{yarp → prism}/ffi.rb

@@ -6,7 +6,7 @@
 require "rbconfig"
 require "ffi"
 
-module YARP
+module Prism
   BACKEND = :FFI
 
   module LibRubyParser
@@ -35,13 +35,13 @@ module YARP
     def self.load_exported_functions_from(header, *functions)
       File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line|
         # We only want to attempt to load exported functions.
-        next unless line.start_with?("YP_EXPORTED_FUNCTION ")
+        next unless line.start_with?("PRISM_EXPORTED_FUNCTION ")
 
         # We only want to load the functions that we are interested in.
         next unless functions.any? { |function| line.include?(function) }
 
         # Parse the function declaration.
-        unless /^YP_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
+        unless /^PRISM_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
           raise "Could not parse #{line}"
         end
 
@@ -67,35 +67,35 @@ module YARP
     end
 
     load_exported_functions_from(
-      "yarp.h",
-      "yp_version",
-      "yp_parse_serialize",
-      "yp_lex_serialize",
-      "yp_parse_lex_serialize"
+      "prism.h",
+      "pm_version",
+      "pm_parse_serialize",
+      "pm_lex_serialize",
+      "pm_parse_lex_serialize"
     )
 
     load_exported_functions_from(
-      "yarp/util/yp_buffer.h",
-      "yp_buffer_sizeof",
-      "yp_buffer_init",
-      "yp_buffer_value",
-      "yp_buffer_length",
-      "yp_buffer_free"
+      "prism/util/pm_buffer.h",
+      "pm_buffer_sizeof",
+      "pm_buffer_init",
+      "pm_buffer_value",
+      "pm_buffer_length",
+      "pm_buffer_free"
     )
 
     load_exported_functions_from(
-      "yarp/util/yp_string.h",
-      "yp_string_mapped_init",
-      "yp_string_free",
-      "yp_string_source",
-      "yp_string_length",
-      "yp_string_sizeof"
+      "prism/util/pm_string.h",
+      "pm_string_mapped_init",
+      "pm_string_free",
+      "pm_string_source",
+      "pm_string_length",
+      "pm_string_sizeof"
     )
 
-    # This object represents a yp_buffer_t. We only use it as an opaque pointer,
-    # so it doesn't need to know the fields of yp_buffer_t.
-    class YPBuffer
-      SIZEOF = LibRubyParser.yp_buffer_sizeof
+    # This object represents a pm_buffer_t. We only use it as an opaque pointer,
+    # so it doesn't need to know the fields of pm_buffer_t.
+    class PrismBuffer
+      SIZEOF = LibRubyParser.pm_buffer_sizeof
 
       attr_reader :pointer
 
@@ -104,11 +104,11 @@ module YARP
       end
 
       def value
-        LibRubyParser.yp_buffer_value(pointer)
+        LibRubyParser.pm_buffer_value(pointer)
       end
 
       def length
-        LibRubyParser.yp_buffer_length(pointer)
+        LibRubyParser.pm_buffer_length(pointer)
       end
 
       def read
@@ -121,19 +121,19 @@ module YARP
         pointer = FFI::MemoryPointer.new(SIZEOF)
 
         begin
-          raise unless LibRubyParser.yp_buffer_init(pointer)
+          raise unless LibRubyParser.pm_buffer_init(pointer)
           yield new(pointer)
         ensure
-          LibRubyParser.yp_buffer_free(pointer)
+          LibRubyParser.pm_buffer_free(pointer)
           pointer.free
         end
       end
     end
 
-    # This object represents a yp_string_t. We only use it as an opaque pointer,
+    # This object represents a pm_string_t. We only use it as an opaque pointer,
     # so it doesn't have to be an FFI::Struct.
-    class YPString
-      SIZEOF = LibRubyParser.yp_string_sizeof
+    class PrismString
+      SIZEOF = LibRubyParser.pm_string_sizeof
 
       attr_reader :pointer
 
@@ -142,94 +142,93 @@ module YARP
       end
 
       def source
-        LibRubyParser.yp_string_source(pointer)
+        LibRubyParser.pm_string_source(pointer)
       end
 
       def length
-        LibRubyParser.yp_string_length(pointer)
+        LibRubyParser.pm_string_length(pointer)
       end
 
       def read
         source.read_string(length)
       end
 
-      # Yields a yp_string_t pointer to the given block.
+      # Yields a pm_string_t pointer to the given block.
       def self.with(filepath, &block)
         pointer = FFI::MemoryPointer.new(SIZEOF)
 
         begin
-          raise unless LibRubyParser.yp_string_mapped_init(pointer, filepath)
+          raise unless LibRubyParser.pm_string_mapped_init(pointer, filepath)
           yield new(pointer)
         ensure
-          LibRubyParser.yp_string_free(pointer)
+          LibRubyParser.pm_string_free(pointer)
           pointer.free
         end
       end
     end
+
+    def self.dump_internal(source, source_size, filepath)
+      PrismBuffer.with do |buffer|
+        metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+        pm_parse_serialize(source, source_size, buffer.pointer, metadata)
+        buffer.read
+      end
+    end
   end
 
   # Mark the LibRubyParser module as private as it should only be called through
-  # the YARP module.
+  # the prism module.
   private_constant :LibRubyParser
 
-  # The version constant is set by reading the result of calling yp_version.
-  VERSION = LibRubyParser.yp_version.read_string
-
-  def self.dump_internal(source, source_size, filepath)
-    LibRubyParser::YPBuffer.with do |buffer|
-      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.yp_parse_serialize(source, source_size, buffer.pointer, metadata)
-      buffer.read
-    end
-  end
-  private_class_method :dump_internal
+  # The version constant is set by reading the result of calling pm_version.
+  VERSION = LibRubyParser.pm_version.read_string
 
-  # Mirror the YARP.dump API by using the serialization API.
+  # Mirror the Prism.dump API by using the serialization API.
   def self.dump(code, filepath = nil)
-    dump_internal(code, code.bytesize, filepath)
+    LibRubyParser.dump_internal(code, code.bytesize, filepath)
   end
 
-  # Mirror the YARP.dump_file API by using the serialization API.
+  # Mirror the Prism.dump_file API by using the serialization API.
   def self.dump_file(filepath)
-    LibRubyParser::YPString.with(filepath) do |string|
-      dump_internal(string.source, string.length, filepath)
+    LibRubyParser::PrismString.with(filepath) do |string|
+      LibRubyParser.dump_internal(string.source, string.length, filepath)
     end
   end
 
-  # Mirror the YARP.lex API by using the serialization API.
+  # Mirror the Prism.lex API by using the serialization API.
   def self.lex(code, filepath = nil)
-    LibRubyParser::YPBuffer.with do |buffer|
-      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
+    LibRubyParser::PrismBuffer.with do |buffer|
+      LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
       Serialize.load_tokens(Source.new(code), buffer.read)
     end
   end
 
-  # Mirror the YARP.lex_file API by using the serialization API.
+  # Mirror the Prism.lex_file API by using the serialization API.
   def self.lex_file(filepath)
-    LibRubyParser::YPString.with(filepath) do |string|
+    LibRubyParser::PrismString.with(filepath) do |string|
       lex(string.read, filepath)
     end
   end
 
-  # Mirror the YARP.parse API by using the serialization API.
+  # Mirror the Prism.parse API by using the serialization API.
   def self.parse(code, filepath = nil)
-    YARP.load(code, dump(code, filepath))
+    Prism.load(code, dump(code, filepath))
   end
 
-  # Mirror the YARP.parse_file API by using the serialization API. This uses
+  # Mirror the Prism.parse_file API by using the serialization API. This uses
   # native strings instead of Ruby strings because it allows us to use mmap when
   # it is available.
   def self.parse_file(filepath)
-    LibRubyParser::YPString.with(filepath) do |string|
+    LibRubyParser::PrismString.with(filepath) do |string|
       parse(string.read, filepath)
     end
   end
 
-  # Mirror the YARP.parse_lex API by using the serialization API.
+  # Mirror the Prism.parse_lex API by using the serialization API.
   def self.parse_lex(code, filepath = nil)
-    LibRubyParser::YPBuffer.with do |buffer|
+    LibRubyParser::PrismBuffer.with do |buffer|
       metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.yp_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
+      LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
 
       source = Source.new(code)
       loader = Serialize::Loader.new(source, buffer.read)
@@ -243,9 +242,9 @@ module YARP
     end
   end
 
-  # Mirror the YARP.parse_lex_file API by using the serialization API.
+  # Mirror the Prism.parse_lex_file API by using the serialization API.
   def self.parse_lex_file(filepath)
-    LibRubyParser::YPString.with(filepath) do |string|
+    LibRubyParser::PrismString.with(filepath) do |string|
       parse_lex(string.read, filepath)
     end
   end

data/lib/{yarp → prism}/lex_compat.rb

@@ -2,14 +2,14 @@
 
 require "delegate"
 
-module YARP
-  # This class is responsible for lexing the source using YARP and then
+module Prism
+  # This class is responsible for lexing the source using prism and then
   # converting those tokens to be compatible with Ripper. In the vast majority
   # of cases, this is a one-to-one mapping of the token type. Everything else
   # generally lines up. However, there are a few cases that require special
   # handling.
   class LexCompat
-    # This is a mapping of YARP token types to Ripper token types. This is a
+    # This is a mapping of prism token types to Ripper token types. This is a
     # many-to-one mapping because we split up our token types, whereas Ripper
     # tends to group them.
     RIPPER = {
@@ -128,6 +128,7 @@ module YARP
       LESS_EQUAL_GREATER: :on_op,
       LESS_LESS: :on_op,
       LESS_LESS_EQUAL: :on_op,
+      METHOD_NAME: :on_ident,
       MINUS: :on_op,
       MINUS_EQUAL: :on_op,
       MINUS_GREATER: :on_tlambda,
@@ -338,8 +339,8 @@ module YARP
 
       # Heredocs that are dedenting heredocs are a little more complicated.
       # Ripper outputs on_ignored_sp tokens for the whitespace that is being
-      # removed from the output. YARP only modifies the node itself and keeps
-      # the token the same. This simplifies YARP, but makes comparing against
+      # removed from the output. prism only modifies the node itself and keeps
+      # the token the same. This simplifies prism, but makes comparing against
       # Ripper much harder because there is a length mismatch.
       #
       # Fortunately, we already have to pull out the heredoc tokens in order to
@@ -562,7 +563,7 @@ module YARP
       state = :default
       heredoc_stack = [[]]
 
-      result = YARP.lex(source, @filepath)
+      result = Prism.lex(source, @filepath)
       result_value = result.value
       previous_state = nil
 
@@ -649,7 +650,7 @@ module YARP
             IgnoredNewlineToken.new([[lineno, column], event, value, lex_state])
           when :on_regexp_end
             # On regex end, Ripper scans and then sets end state, so the ripper
-            # lexed output is begin, when it should be end. YARP sets lex state
+            # lexed output is begin, when it should be end. prism sets lex state
             # correctly to end state, but we want to be able to compare against
             # Ripper's lexed state. So here, if it's a regexp end token, we
             # output the state as the previous state, solely for the sake of
@@ -705,7 +706,7 @@ module YARP
 
         # The order in which tokens appear in our lexer is different from the
         # order that they appear in Ripper. When we hit the declaration of a
-        # heredoc in YARP, we skip forward and lex the rest of the content of
+        # heredoc in prism, we skip forward and lex the rest of the content of
         # the heredoc before going back and lexing at the end of the heredoc
         # identifier.
         #
@@ -794,48 +795,44 @@ module YARP
     end
   end
 
-  # The constant that wraps the behavior of the lexer to match Ripper's output
-  # is an implementation detail, so we don't want it to be public.
-  private_constant :LexCompat
+  # This is a class that wraps the Ripper lexer to produce almost exactly the
+  # same tokens.
+  class LexRipper
+    attr_reader :source
 
-  # Returns an array of tokens that closely resembles that of the Ripper lexer.
-  # The only difference is that since we don't keep track of lexer state in the
-  # same way, it's going to always return the NONE state.
-  def self.lex_compat(source, filepath = "")
-    LexCompat.new(source, filepath).result
-  end
+    def initialize(source)
+      @source = source
+    end
 
-  # This lexes with the Ripper lex. It drops any space events but otherwise
-  # returns the same tokens. Raises SyntaxError if the syntax in source is
-  # invalid.
-  def self.lex_ripper(source)
-    previous = []
-    results = []
-
-    Ripper.lex(source, raise_errors: true).each do |token|
-      case token[1]
-      when :on_sp
-        # skip
-      when :on_tstring_content
-        if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
-          previous[2] << token[2]
-        else
-          results << token
-          previous = token
-        end
-      when :on_words_sep
-        if previous[1] == :on_words_sep
-          previous[2] << token[2]
+    def result
+      previous = []
+      results = []
+
+      Ripper.lex(source, raise_errors: true).each do |token|
+        case token[1]
+        when :on_sp
+          # skip
+        when :on_tstring_content
+          if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
+            previous[2] << token[2]
+          else
+            results << token
+            previous = token
+          end
+        when :on_words_sep
+          if previous[1] == :on_words_sep
+            previous[2] << token[2]
+          else
+            results << token
+            previous = token
+          end
         else
           results << token
           previous = token
         end
-      else
-        results << token
-        previous = token
       end
-    end
 
-    results
+      results
+    end
   end
 end

data/lib/{yarp/mutation_visitor.rb → prism/mutation_compiler.rb}

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 =begin
 This file is generated by the templates/template.rb script and should not be
-modified manually. See templates/lib/yarp/mutation_visitor.rb.erb
+modified manually. See templates/lib/prism/mutation_compiler.rb.erb
 if you are looking to modify the template
 =end
 
-module YARP
+module Prism
   # This visitor walks through the tree and copies each node as it is being
   # visited. This is useful for consumers that want to mutate the tree, as you
   # can change subtrees in place without effecting the rest of the tree.
-  class MutationVisitor < BasicVisitor
+  class MutationCompiler < Compiler
     # Copy a AliasGlobalVariableNode node
     def visit_alias_global_variable_node(node)
       node.copy(new_name: visit(node.new_name), old_name: visit(node.old_name))