yarp 0.6.0 → 0.8.0

data/include/yarp/unescape.h

@@ -31,7 +31,7 @@ typedef enum {
 
 // Unescape the contents of the given token into the given string using the
 // given unescape mode.
-YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, const char *value, size_t length, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list);
+YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list);
 
 // Accepts a source string and a type of unescaping and returns the unescaped version.
 // The caller must yp_string_free(result); after calling this function.

data/include/yarp/util/yp_buffer.h

@@ -18,9 +18,18 @@ typedef struct {
     size_t capacity;
 } yp_buffer_t;
 
+// Return the size of the yp_buffer_t struct.
+YP_EXPORTED_FUNCTION size_t yp_buffer_sizeof(void);
+
 // Initialize a yp_buffer_t with its default values.
 YP_EXPORTED_FUNCTION bool yp_buffer_init(yp_buffer_t *buffer);
 
+// Return the value of the buffer.
+YP_EXPORTED_FUNCTION char * yp_buffer_value(yp_buffer_t *buffer);
+
+// Return the length of the buffer.
+YP_EXPORTED_FUNCTION size_t yp_buffer_length(yp_buffer_t *buffer);
+
 // Append the given amount of space as zeroes to the buffer.
 void yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length);
 

data/include/yarp/util/yp_constant_pool.h

@@ -51,6 +51,9 @@ typedef struct {
     size_t capacity;
 } yp_constant_pool_t;
 
+// Define an empty constant pool.
+#define YP_CONSTANT_POOL_EMPTY ((yp_constant_pool_t) { .constants = NULL, .size = 0, .capacity = 0 })
+
 // Initialize a new constant pool with a given capacity.
 bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity);
 

data/include/yarp/util/yp_list.h

@@ -15,9 +15,7 @@
 //       int value;
 //     } yp_int_node_t;
 //
-//     yp_list_t list;
-//     yp_list_init(&list);
-//
+//     yp_list_t list = YP_LIST_EMPTY;
 //     yp_int_node_t *node = malloc(sizeof(yp_int_node_t));
 //     node->value = 5;
 //
@@ -45,18 +43,20 @@ typedef struct yp_list_node {
 // This represents the overall linked list. It keeps a pointer to the head and
 // tail so that iteration is easy and pushing new nodes is easy.
 typedef struct {
+    size_t size;
     yp_list_node_t *head;
     yp_list_node_t *tail;
 } yp_list_t;
 
-// Initializes a new list.
-YP_EXPORTED_FUNCTION void yp_list_init(yp_list_t *list);
+// This represents an empty list. It's used to initialize a stack-allocated list
+// as opposed to a method call.
+#define YP_LIST_EMPTY ((yp_list_t) { .size = 0, .head = NULL, .tail = NULL })
 
 // Returns true if the given list is empty.
 YP_EXPORTED_FUNCTION bool yp_list_empty_p(yp_list_t *list);
 
-// Returns the size of the list in O(n) time.
-YP_EXPORTED_FUNCTION uint32_t yp_list_size(yp_list_t *list);
+// Returns the size of the list.
+YP_EXPORTED_FUNCTION size_t yp_list_size(yp_list_t *list);
 
 // Append a node to the given list.
 void yp_list_append(yp_list_t *list, yp_list_node_t *node);

data/include/yarp/util/yp_newline_list.h

@@ -35,6 +35,10 @@ typedef struct {
     size_t column;
 } yp_line_column_t;
 
+#define YP_NEWLINE_LIST_EMPTY ((yp_newline_list_t) { \
+    .start = NULL, .offsets = NULL, .size = 0, .capacity = 0, .last_offset = 0, .last_index = 0 \
+})
+
 // Initialize a new newline list with the given capacity. Returns true if the
 // allocation of the offsets succeeds, otherwise returns false.
 bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity);

data/include/yarp/util/yp_state_stack.h

@@ -10,7 +10,7 @@
 typedef uint32_t yp_state_stack_t;
 
 // Initializes the state stack to an empty stack.
-void yp_state_stack_init(yp_state_stack_t *stack);
+#define YP_STATE_STACK_EMPTY ((yp_state_stack_t) 0)
 
 // Pushes a value onto the stack.
 void yp_state_stack_push(yp_state_stack_t *stack, bool value);

data/include/yarp/util/yp_string.h

@@ -36,7 +36,7 @@ void yp_string_constant_init(yp_string_t *string, const char *source, size_t len
 // for large files). This means that if we're on windows we'll use
 // `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
 // `mmap`, and on other POSIX systems we'll use `read`.
-bool yp_string_mapped_init(yp_string_t *string, const char *filepath);
+YP_EXPORTED_FUNCTION bool yp_string_mapped_init(yp_string_t *string, const char *filepath);
 
 // Returns the memory size associated with the string.
 size_t yp_string_memsize(const yp_string_t *string);
@@ -54,4 +54,8 @@ YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string);
 // Free the associated memory of the given string.
 YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string);
 
+// Returns the size of the yp_string_t struct. This is necessary to allocate the
+// correct amount of memory in the FFI backend.
+YP_EXPORTED_FUNCTION size_t yp_string_sizeof(void);
+
 #endif // YARP_STRING_H

data/include/yarp/version.h

@@ -1,5 +1,4 @@
 #define YP_VERSION_MAJOR 0
-#define YP_VERSION_MINOR 6
+#define YP_VERSION_MINOR 8
 #define YP_VERSION_PATCH 0
-
-#define YP_VERSION "0.6.0"
+#define YP_VERSION "0.8.0"

data/include/yarp.h

@@ -59,10 +59,12 @@ YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, y
 // Serialize the AST represented by the given node to the given buffer.
 YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
 
-// Parse and serialize the AST represented by the given source to the given
-// buffer.
+// Parse the given source to the AST and serialize the AST to the given buffer.
 YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
 
+// Lex the given source and serialize to the given buffer.
+YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer);
+
 // Returns a string representation of the given token type.
 YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);
 

data/lib/yarp/ffi.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+# This file is responsible for mirroring the API provided by the C extension by
+# using FFI to call into the shared library.
+
+require "rbconfig"
+require "ffi"
+
+module YARP
+  BACKEND = :FFI
+
+  module LibRubyParser
+    extend FFI::Library
+
+    # Define the library that we will be pulling functions from. Note that this
+    # must align with the build shared library from make/rake.
+    ffi_lib File.expand_path("../../build/librubyparser.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
+
+    # Convert a native C type declaration into a symbol that FFI understands.
+    # For example:
+    #
+    #     const char * -> :pointer
+    #     bool         -> :bool
+    #     size_t       -> :size_t
+    #     void         -> :void
+    #
+    def self.resolve_type(type)
+      type = type.strip.delete_prefix("const ")
+      type.end_with?("*") ? :pointer : type.to_sym
+    end
+
+    # Read through the given header file and find the declaration of each of the
+    # given functions. For each one, define a function with the same name and
+    # signature as the C function.
+    def self.load_exported_functions_from(header, *functions)
+      File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line|
+        # We only want to attempt to load exported functions.
+        next unless line.start_with?("YP_EXPORTED_FUNCTION ")
+
+        # We only want to load the functions that we are interested in.
+        next unless functions.any? { |function| line.include?(function) }
+
+        # Parse the function declaration.
+        unless /^YP_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
+          raise "Could not parse #{line}"
+        end
+
+        # Delete the function from the list of functions we are looking for to
+        # mark it as having been found.
+        functions.delete(name)
+
+        # Split up the argument types into an array, ensure we handle the case
+        # where there are no arguments (by explicit void).
+        arg_types = arg_types.split(",").map(&:strip)
+        arg_types = [] if arg_types == %w[void]
+
+        # Resolve the type of the argument by dropping the name of the argument
+        # first if it is present.
+        arg_types.map! { |type| resolve_type(type.sub(/\w+$/, "")) }
+
+        # Attach the function using the FFI library.
+        attach_function name, arg_types, resolve_type(return_type)
+      end
+
+      # If we didn't find all of the functions, raise an error.
+      raise "Could not find functions #{functions.inspect}" unless functions.empty?
+    end
+
+    load_exported_functions_from(
+      "yarp.h",
+      "yp_version",
+      "yp_parse_serialize",
+      "yp_lex_serialize"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_buffer.h",
+      "yp_buffer_sizeof",
+      "yp_buffer_init",
+      "yp_buffer_value",
+      "yp_buffer_length",
+      "yp_buffer_free"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_string.h",
+      "yp_string_mapped_init",
+      "yp_string_free",
+      "yp_string_source",
+      "yp_string_length",
+      "yp_string_sizeof"
+    )
+
+    # This object represents a yp_buffer_t. We only use it as an opaque pointer,
+    # so it doesn't need to know the fields of yp_buffer_t.
+    class YPBuffer
+      SIZEOF = LibRubyParser.yp_buffer_sizeof
+
+      attr_reader :pointer
+
+      def initialize(pointer)
+        @pointer = pointer
+      end
+
+      def value
+        LibRubyParser.yp_buffer_value(pointer)
+      end
+
+      def length
+        LibRubyParser.yp_buffer_length(pointer)
+      end
+
+      def read
+        value.read_string(length)
+      end
+
+      # Initialize a new buffer and yield it to the block. The buffer will be
+      # automatically freed when the block returns.
+      def self.with(&block)
+        pointer = FFI::MemoryPointer.new(SIZEOF)
+
+        begin
+          raise unless LibRubyParser.yp_buffer_init(pointer)
+          yield new(pointer)
+        ensure
+          LibRubyParser.yp_buffer_free(pointer)
+          pointer.free
+        end
+      end
+    end
+
+    # This object represents a yp_string_t. We only use it as an opaque pointer,
+    # so it doesn't have to be an FFI::Struct.
+    class YPString
+      SIZEOF = LibRubyParser.yp_string_sizeof
+
+      attr_reader :pointer
+
+      def initialize(pointer)
+        @pointer = pointer
+      end
+
+      def source
+        LibRubyParser.yp_string_source(pointer)
+      end
+
+      def length
+        LibRubyParser.yp_string_length(pointer)
+      end
+
+      def read
+        source.read_string(length)
+      end
+
+      # Yields a yp_string_t pointer to the given block.
+      def self.with(filepath, &block)
+        pointer = FFI::MemoryPointer.new(SIZEOF)
+
+        begin
+          raise unless LibRubyParser.yp_string_mapped_init(pointer, filepath)
+          yield new(pointer)
+        ensure
+          LibRubyParser.yp_string_free(pointer)
+          pointer.free
+        end
+      end
+    end
+  end
+
+  # Mark the LibRubyParser module as private as it should only be called through
+  # the YARP module.
+  private_constant :LibRubyParser
+
+  # The version constant is set by reading the result of calling yp_version.
+  VERSION = LibRubyParser.yp_version.read_string
+
+  def self.dump_internal(source, source_size, filepath)
+    LibRubyParser::YPBuffer.with do |buffer|
+      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+      LibRubyParser.yp_parse_serialize(source, source_size, buffer.pointer, metadata)
+      buffer.read
+    end
+  end
+  private_class_method :dump_internal
+
+  # Mirror the YARP.dump API by using the serialization API.
+  def self.dump(code, filepath = nil)
+    dump_internal(code, code.bytesize, filepath)
+  end
+
+  # Mirror the YARP.dump_file API by using the serialization API.
+  def self.dump_file(filepath)
+    LibRubyParser::YPString.with(filepath) do |string|
+      dump_internal(string.source, string.length, filepath)
+    end
+  end
+
+  # Mirror the YARP.lex API by using the serialization API.
+  def self.lex(code, filepath = nil)
+    LibRubyParser::YPBuffer.with do |buffer|
+      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
+      Serialize.load_tokens(Source.new(code), buffer.read)
+    end
+  end
+
+  # Mirror the YARP.lex_file API by using the serialization API.
+  def self.lex_file(filepath)
+    LibRubyParser::YPString.with(filepath) do |string|
+      lex(string.read, filepath)
+    end
+  end
+
+  # Mirror the YARP.parse API by using the serialization API.
+  def self.parse(code, filepath = nil)
+    YARP.load(code, dump(code, filepath))
+  end
+
+  # Mirror the YARP.parse_file API by using the serialization API. This uses
+  # native strings instead of Ruby strings because it allows us to use mmap when
+  # it is available.
+  def self.parse_file(filepath)
+    LibRubyParser::YPString.with(filepath) do |string|
+      parse(string.read, filepath)
+    end
+  end
+end

data/lib/yarp/lex_compat.rb

@@ -647,19 +647,34 @@ module YARP
         # can shuffle around the token to match Ripper's output.
         case state
         when :default
+          # The default state is when there are no heredocs at all. In this
+          # state we can append the token to the list of tokens and move on.
           tokens << token
 
+          # If we get the declaration of a heredoc, then we open a new heredoc
+          # and move into the heredoc_opened state.
           if event == :on_heredoc_beg
             state = :heredoc_opened
             heredoc_stack.last << Heredoc.build(token)
           end
         when :heredoc_opened
+          # The heredoc_opened state is when we've seen the declaration of a
+          # heredoc and are now lexing the body of the heredoc. In this state we
+          # push tokens onto the most recently created heredoc.
           heredoc_stack.last.last << token
 
           case event
           when :on_heredoc_beg
+            # If we receive a heredoc declaration while lexing the body of a
+            # heredoc, this means we have nested heredocs. In this case we'll
+            # push a new heredoc onto the stack and stay in the heredoc_opened
+            # state since we're now lexing the body of the new heredoc.
             heredoc_stack << [Heredoc.build(token)]
           when :on_heredoc_end
+            # If we receive the end of a heredoc, then we're done lexing the
+            # body of the heredoc. In this case we now have a completed heredoc
+            # but need to wait for the next newline to push it into the token
+            # stream.
             state = :heredoc_closed
           end
         when :heredoc_closed
@@ -734,8 +749,7 @@ module YARP
       when :on_sp
         # skip
       when :on_tstring_content
-        if previous[1] == :on_tstring_content &&
-            (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
+        if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
           previous[2] << token[2]
         else
           results << token