yarp 0.7.0 → 0.9.0

data/include/yarp/util/yp_buffer.h

@@ -12,16 +12,24 @@
 // A yp_buffer_t is a simple memory buffer that stores data in a contiguous
 // block of memory. It is used to store the serialized representation of a
 // YARP tree.
-// NOTE: keep in sync with YARP::LibRubyParser::Buffer in lib/yarp.rb
 typedef struct {
     char *value;
     size_t length;
     size_t capacity;
 } yp_buffer_t;
 
+// Return the size of the yp_buffer_t struct.
+YP_EXPORTED_FUNCTION size_t yp_buffer_sizeof(void);
+
 // Initialize a yp_buffer_t with its default values.
 YP_EXPORTED_FUNCTION bool yp_buffer_init(yp_buffer_t *buffer);
 
+// Return the value of the buffer.
+YP_EXPORTED_FUNCTION char * yp_buffer_value(yp_buffer_t *buffer);
+
+// Return the length of the buffer.
+YP_EXPORTED_FUNCTION size_t yp_buffer_length(yp_buffer_t *buffer);
+
 // Append the given amount of space as zeroes to the buffer.
 void yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length);
 

data/include/yarp/util/yp_constant_pool.h

@@ -51,6 +51,9 @@ typedef struct {
     size_t capacity;
 } yp_constant_pool_t;
 
+// Define an empty constant pool.
+#define YP_CONSTANT_POOL_EMPTY ((yp_constant_pool_t) { .constants = NULL, .size = 0, .capacity = 0 })
+
 // Initialize a new constant pool with a given capacity.
 bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity);
 

data/include/yarp/util/yp_list.h

@@ -15,9 +15,7 @@
 //       int value;
 //     } yp_int_node_t;
 //
-//     yp_list_t list;
-//     yp_list_init(&list);
-//
+//     yp_list_t list = YP_LIST_EMPTY;
 //     yp_int_node_t *node = malloc(sizeof(yp_int_node_t));
 //     node->value = 5;
 //
@@ -45,18 +43,20 @@ typedef struct yp_list_node {
 // This represents the overall linked list. It keeps a pointer to the head and
 // tail so that iteration is easy and pushing new nodes is easy.
 typedef struct {
+    size_t size;
     yp_list_node_t *head;
     yp_list_node_t *tail;
 } yp_list_t;
 
-// Initializes a new list.
-YP_EXPORTED_FUNCTION void yp_list_init(yp_list_t *list);
+// This represents an empty list. It's used to initialize a stack-allocated list
+// as opposed to a method call.
+#define YP_LIST_EMPTY ((yp_list_t) { .size = 0, .head = NULL, .tail = NULL })
 
 // Returns true if the given list is empty.
 YP_EXPORTED_FUNCTION bool yp_list_empty_p(yp_list_t *list);
 
-// Returns the size of the list in O(n) time.
-YP_EXPORTED_FUNCTION uint32_t yp_list_size(yp_list_t *list);
+// Returns the size of the list.
+YP_EXPORTED_FUNCTION size_t yp_list_size(yp_list_t *list);
 
 // Append a node to the given list.
 void yp_list_append(yp_list_t *list, yp_list_node_t *node);

data/include/yarp/util/yp_newline_list.h

@@ -35,6 +35,10 @@ typedef struct {
     size_t column;
 } yp_line_column_t;
 
+#define YP_NEWLINE_LIST_EMPTY ((yp_newline_list_t) { \
+    .start = NULL, .offsets = NULL, .size = 0, .capacity = 0, .last_offset = 0, .last_index = 0 \
+})
+
 // Initialize a new newline list with the given capacity. Returns true if the
 // allocation of the offsets succeeds, otherwise returns false.
 bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity);
@@ -43,6 +47,9 @@ bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t cap
 // the offsets succeeds (if one was necessary), otherwise returns false.
 bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor);
 
+// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+bool yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor);
+
 // Returns the line and column of the given offset. If the offset is not in the
 // list, the line and column of the closest offset less than the given offset
 // are returned.

data/include/yarp/util/yp_state_stack.h

@@ -10,7 +10,7 @@
 typedef uint32_t yp_state_stack_t;
 
 // Initializes the state stack to an empty stack.
-void yp_state_stack_init(yp_state_stack_t *stack);
+#define YP_STATE_STACK_EMPTY ((yp_state_stack_t) 0)
 
 // Pushes a value onto the stack.
 void yp_state_stack_push(yp_state_stack_t *stack, bool value);

data/include/yarp/version.h

@@ -1,4 +1,4 @@
 #define YP_VERSION_MAJOR 0
-#define YP_VERSION_MINOR 7
+#define YP_VERSION_MINOR 9
 #define YP_VERSION_PATCH 0
-#define YP_VERSION "0.7.0"
+#define YP_VERSION "0.9.0"

data/include/yarp.h

@@ -13,6 +13,7 @@
 #include "yarp/util/yp_char.h"
 #include "yarp/util/yp_memchr.h"
 #include "yarp/util/yp_strpbrk.h"
+#include "yarp/version.h"
 
 #include <assert.h>
 #include <stdarg.h>
@@ -30,6 +31,11 @@ void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buf
 
 void yp_print_node(yp_parser_t *parser, yp_node_t *node);
 
+void yp_parser_metadata(yp_parser_t *parser, const char *metadata);
+
+// Generate a scope node from the given node.
+void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest);
+
 // The YARP version and the serialization format.
 YP_EXPORTED_FUNCTION const char * yp_version(void);
 
@@ -65,6 +71,10 @@ YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp
 // Lex the given source and serialize to the given buffer.
 YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer);
 
+// Parse and serialize both the AST and the tokens represented by the given
+// source to the given buffer.
+YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
+
 // Returns a string representation of the given token type.
 YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);
 

data/lib/yarp/desugar_visitor.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+module YARP
+  class DesugarVisitor < MutationVisitor
+    # @@foo &&= bar
+    #
+    # becomes
+    #
+    # @@foo && @@foo = bar
+    def visit_class_variable_and_write_node(node)
+      AndNode.new(
+        ClassVariableReadNode.new(node.name_loc),
+        ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # @@foo ||= bar
+    #
+    # becomes
+    #
+    # @@foo || @@foo = bar
+    def visit_class_variable_or_write_node(node)
+      OrNode.new(
+        ClassVariableReadNode.new(node.name_loc),
+        ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # @@foo += bar
+    #
+    # becomes
+    #
+    # @@foo = @@foo + bar
+    def visit_class_variable_operator_write_node(node)
+      desugar_operator_write_node(node, ClassVariableWriteNode, ClassVariableReadNode)
+    end
+
+    # Foo &&= bar
+    #
+    # becomes
+    #
+    # Foo && Foo = bar
+    def visit_constant_and_write_node(node)
+      AndNode.new(
+        ConstantReadNode.new(node.name_loc),
+        ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # Foo ||= bar
+    #
+    # becomes
+    #
+    # Foo || Foo = bar
+    def visit_constant_or_write_node(node)
+      OrNode.new(
+        ConstantReadNode.new(node.name_loc),
+        ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # Foo += bar
+    #
+    # becomes
+    #
+    # Foo = Foo + bar
+    def visit_constant_operator_write_node(node)
+      desugar_operator_write_node(node, ConstantWriteNode, ConstantReadNode)
+    end
+
+    # Foo::Bar &&= baz
+    #
+    # becomes
+    #
+    # Foo::Bar && Foo::Bar = baz
+    def visit_constant_path_and_write_node(node)
+      AndNode.new(
+        node.target,
+        ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # Foo::Bar ||= baz
+    #
+    # becomes
+    #
+    # Foo::Bar || Foo::Bar = baz
+    def visit_constant_path_or_write_node(node)
+      OrNode.new(
+        node.target,
+        ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # Foo::Bar += baz
+    #
+    # becomes
+    #
+    # Foo::Bar = Foo::Bar + baz
+    def visit_constant_path_operator_write_node(node)
+      ConstantPathWriteNode.new(
+        node.target,
+        CallNode.new(
+          node.target,
+          nil,
+          node.operator_loc.copy(length: node.operator_loc.length - 1),
+          nil,
+          ArgumentsNode.new([node.value], node.value.location),
+          nil,
+          nil,
+          0,
+          node.operator_loc.slice.chomp("="),
+          node.location
+        ),
+        node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1),
+        node.location
+      )
+    end
+
+    # $foo &&= bar
+    #
+    # becomes
+    #
+    # $foo && $foo = bar
+    def visit_global_variable_and_write_node(node)
+      AndNode.new(
+        GlobalVariableReadNode.new(node.name_loc),
+        GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # $foo ||= bar
+    #
+    # becomes
+    #
+    # $foo || $foo = bar
+    def visit_global_variable_or_write_node(node)
+      OrNode.new(
+        GlobalVariableReadNode.new(node.name_loc),
+        GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # $foo += bar
+    #
+    # becomes
+    #
+    # $foo = $foo + bar
+    def visit_global_variable_operator_write_node(node)
+      desugar_operator_write_node(node, GlobalVariableWriteNode, GlobalVariableReadNode)
+    end
+
+    # @foo &&= bar
+    #
+    # becomes
+    #
+    # @foo && @foo = bar
+    def visit_instance_variable_and_write_node(node)
+      AndNode.new(
+        InstanceVariableReadNode.new(node.name_loc),
+        InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # @foo ||= bar
+    #
+    # becomes
+    #
+    # @foo || @foo = bar
+    def visit_instance_variable_or_write_node(node)
+      OrNode.new(
+        InstanceVariableReadNode.new(node.name_loc),
+        InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # @foo += bar
+    #
+    # becomes
+    #
+    # @foo = @foo + bar
+    def visit_instance_variable_operator_write_node(node)
+      desugar_operator_write_node(node, InstanceVariableWriteNode, InstanceVariableReadNode)
+    end
+
+    # foo &&= bar
+    #
+    # becomes
+    #
+    # foo && foo = bar
+    def visit_local_variable_and_write_node(node)
+      AndNode.new(
+        LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc),
+        LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # foo ||= bar
+    #
+    # becomes
+    #
+    # foo || foo = bar
+    def visit_local_variable_or_write_node(node)
+      OrNode.new(
+        LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc),
+        LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+
+    # foo += bar
+    #
+    # becomes
+    #
+    # foo = foo + bar
+    def visit_local_variable_operator_write_node(node)
+      desugar_operator_write_node(node, LocalVariableWriteNode, LocalVariableReadNode, arguments: [node.constant_id, node.depth])
+    end
+
+    private
+
+    # Desugar `x += y` to `x = x + y`
+    def desugar_operator_write_node(node, write_class, read_class, arguments: [])
+      write_class.new(
+        *arguments,
+        node.name_loc,
+        CallNode.new(
+          read_class.new(*arguments, node.name_loc),
+          nil,
+          node.operator_loc.copy(length: node.operator_loc.length - 1),
+          nil,
+          ArgumentsNode.new([node.value], node.value.location),
+          nil,
+          nil,
+          0,
+          node.operator_loc.slice.chomp("="),
+          node.location
+        ),
+        node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1),
+        node.location
+      )
+    end
+  end
+end

data/lib/yarp/ffi.rb

@@ -70,12 +70,16 @@ module YARP
       "yarp.h",
       "yp_version",
       "yp_parse_serialize",
-      "yp_lex_serialize"
+      "yp_lex_serialize",
+      "yp_parse_lex_serialize"
     )
 
     load_exported_functions_from(
       "yarp/util/yp_buffer.h",
+      "yp_buffer_sizeof",
       "yp_buffer_init",
+      "yp_buffer_value",
+      "yp_buffer_length",
       "yp_buffer_free"
     )
 
@@ -88,34 +92,49 @@ module YARP
       "yp_string_sizeof"
     )
 
-    # This object represents a yp_buffer_t. Its structure must be kept in sync
-    # with the C version.
-    class YPBuffer < FFI::Struct
-      layout value: :pointer, length: :size_t, capacity: :size_t
+    # This object represents a yp_buffer_t. We only use it as an opaque pointer,
+    # so it doesn't need to know the fields of yp_buffer_t.
+    class YPBuffer
+      SIZEOF = LibRubyParser.yp_buffer_sizeof
 
-      # Read the contents of the buffer into a String object and return it.
-      def to_ruby_string
-        self[:value].read_string(self[:length])
+      attr_reader :pointer
+
+      def initialize(pointer)
+        @pointer = pointer
       end
-    end
 
-    # Initialize a new buffer and yield it to the block. The buffer will be
-    # automatically freed when the block returns.
-    def self.with_buffer(&block)
-      buffer = YPBuffer.new
-
-      begin
-        raise unless yp_buffer_init(buffer)
-        yield buffer
-      ensure
-        yp_buffer_free(buffer)
-        buffer.pointer.free
+      def value
+        LibRubyParser.yp_buffer_value(pointer)
+      end
+
+      def length
+        LibRubyParser.yp_buffer_length(pointer)
+      end
+
+      def read
+        value.read_string(length)
+      end
+
+      # Initialize a new buffer and yield it to the block. The buffer will be
+      # automatically freed when the block returns.
+      def self.with(&block)
+        pointer = FFI::MemoryPointer.new(SIZEOF)
+
+        begin
+          raise unless LibRubyParser.yp_buffer_init(pointer)
+          yield new(pointer)
+        ensure
+          LibRubyParser.yp_buffer_free(pointer)
+          pointer.free
+        end
       end
     end
 
     # This object represents a yp_string_t. We only use it as an opaque pointer,
     # so it doesn't have to be an FFI::Struct.
     class YPString
+      SIZEOF = LibRubyParser.yp_string_sizeof
+
       attr_reader :pointer
 
       def initialize(pointer)
@@ -133,23 +152,18 @@ module YARP
       def read
         source.read_string(length)
       end
-    end
 
-    # This is the size of a yp_string_t. It is returned by the yp_string_sizeof
-    # function which we call once to ensure we have sufficient space for the
-    # yp_string_t FFI pointer.
-    SIZEOF_YP_STRING = yp_string_sizeof
-
-    # Yields a yp_string_t pointer to the given block.
-    def self.with_string(filepath, &block)
-      string = FFI::MemoryPointer.new(SIZEOF_YP_STRING)
-
-      begin
-        raise unless yp_string_mapped_init(string, filepath)
-        yield YPString.new(string)
-      ensure
-        yp_string_free(string)
-        string.free
+      # Yields a yp_string_t pointer to the given block.
+      def self.with(filepath, &block)
+        pointer = FFI::MemoryPointer.new(SIZEOF)
+
+        begin
+          raise unless LibRubyParser.yp_string_mapped_init(pointer, filepath)
+          yield new(pointer)
+        ensure
+          LibRubyParser.yp_string_free(pointer)
+          pointer.free
+        end
       end
     end
   end
@@ -162,10 +176,10 @@ module YARP
   VERSION = LibRubyParser.yp_version.read_string
 
   def self.dump_internal(source, source_size, filepath)
-    LibRubyParser.with_buffer do |buffer|
+    LibRubyParser::YPBuffer.with do |buffer|
       metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.yp_parse_serialize(source, source_size, buffer, metadata)
-      buffer.to_ruby_string
+      LibRubyParser.yp_parse_serialize(source, source_size, buffer.pointer, metadata)
+      buffer.read
     end
   end
   private_class_method :dump_internal
@@ -177,35 +191,62 @@ module YARP
 
   # Mirror the YARP.dump_file API by using the serialization API.
   def self.dump_file(filepath)
-    LibRubyParser.with_string(filepath) do |string|
+    LibRubyParser::YPString.with(filepath) do |string|
       dump_internal(string.source, string.length, filepath)
     end
   end
 
   # Mirror the YARP.lex API by using the serialization API.
   def self.lex(code, filepath = nil)
-    LibRubyParser.with_buffer do |buffer|
-      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer)
-
-      source = Source.new(code)
-      Serialize.load_tokens(source, buffer.to_ruby_string).with_source(source)
+    LibRubyParser::YPBuffer.with do |buffer|
+      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
+      Serialize.load_tokens(Source.new(code), buffer.read)
     end
   end
 
   # Mirror the YARP.lex_file API by using the serialization API.
   def self.lex_file(filepath)
-    LibRubyParser.with_string(filepath) { |string| lex(string.read, filepath) }
+    LibRubyParser::YPString.with(filepath) do |string|
+      lex(string.read, filepath)
+    end
   end
 
   # Mirror the YARP.parse API by using the serialization API.
   def self.parse(code, filepath = nil)
-    YARP.load(code, dump(code, filepath)).with_source(Source.new(code))
+    YARP.load(code, dump(code, filepath))
   end
 
   # Mirror the YARP.parse_file API by using the serialization API. This uses
   # native strings instead of Ruby strings because it allows us to use mmap when
   # it is available.
   def self.parse_file(filepath)
-    LibRubyParser.with_string(filepath) { |string| parse(string.read, filepath) }
+    LibRubyParser::YPString.with(filepath) do |string|
+      parse(string.read, filepath)
+    end
+  end
+
+  # Mirror the YARP.parse_lex API by using the serialization API.
+  def self.parse_lex(code, filepath = nil)
+    LibRubyParser::YPBuffer.with do |buffer|
+      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+      LibRubyParser.yp_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
+
+      source = Source.new(code)
+      loader = Serialize::Loader.new(source, buffer.read)
+
+      tokens = loader.load_tokens
+      node, comments, errors, warnings = loader.load_nodes
+
+      tokens.each { |token,| token.value.force_encoding(loader.encoding) }
+
+      ParseResult.new([node, tokens], comments, errors, warnings, source)
+    end
+  end
+
+  # Mirror the YARP.parse_lex_file API by using the serialization API.
+  def self.parse_lex_file(filepath)
+    LibRubyParser::YPString.with(filepath) do |string|
+      parse_lex(string.read, filepath)
+    end
   end
 end