yarp 0.6.0 → 0.7.0

data/include/yarp/ast.h

@@ -259,95 +259,97 @@ enum yp_node_type {
     YP_NODE_ENSURE_NODE = 43,
     YP_NODE_FALSE_NODE = 44,
     YP_NODE_FIND_PATTERN_NODE = 45,
-    YP_NODE_FLOAT_NODE = 46,
-    YP_NODE_FOR_NODE = 47,
-    YP_NODE_FORWARDING_ARGUMENTS_NODE = 48,
-    YP_NODE_FORWARDING_PARAMETER_NODE = 49,
-    YP_NODE_FORWARDING_SUPER_NODE = 50,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 51,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 52,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE = 53,
-    YP_NODE_GLOBAL_VARIABLE_READ_NODE = 54,
-    YP_NODE_GLOBAL_VARIABLE_WRITE_NODE = 55,
-    YP_NODE_HASH_NODE = 56,
-    YP_NODE_HASH_PATTERN_NODE = 57,
-    YP_NODE_IF_NODE = 58,
-    YP_NODE_IMAGINARY_NODE = 59,
-    YP_NODE_IN_NODE = 60,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE = 61,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE = 62,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 63,
-    YP_NODE_INSTANCE_VARIABLE_READ_NODE = 64,
-    YP_NODE_INSTANCE_VARIABLE_WRITE_NODE = 65,
-    YP_NODE_INTEGER_NODE = 66,
-    YP_NODE_INTERPOLATED_REGULAR_EXPRESSION_NODE = 67,
-    YP_NODE_INTERPOLATED_STRING_NODE = 68,
-    YP_NODE_INTERPOLATED_SYMBOL_NODE = 69,
-    YP_NODE_INTERPOLATED_X_STRING_NODE = 70,
-    YP_NODE_KEYWORD_HASH_NODE = 71,
-    YP_NODE_KEYWORD_PARAMETER_NODE = 72,
-    YP_NODE_KEYWORD_REST_PARAMETER_NODE = 73,
-    YP_NODE_LAMBDA_NODE = 74,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 75,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 76,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 77,
-    YP_NODE_LOCAL_VARIABLE_READ_NODE = 78,
-    YP_NODE_LOCAL_VARIABLE_WRITE_NODE = 79,
-    YP_NODE_MATCH_PREDICATE_NODE = 80,
-    YP_NODE_MATCH_REQUIRED_NODE = 81,
-    YP_NODE_MISSING_NODE = 82,
-    YP_NODE_MODULE_NODE = 83,
-    YP_NODE_MULTI_WRITE_NODE = 84,
-    YP_NODE_NEXT_NODE = 85,
-    YP_NODE_NIL_NODE = 86,
-    YP_NODE_NO_KEYWORDS_PARAMETER_NODE = 87,
-    YP_NODE_NUMBERED_REFERENCE_READ_NODE = 88,
-    YP_NODE_OPTIONAL_PARAMETER_NODE = 89,
-    YP_NODE_OR_NODE = 90,
-    YP_NODE_PARAMETERS_NODE = 91,
-    YP_NODE_PARENTHESES_NODE = 92,
-    YP_NODE_PINNED_EXPRESSION_NODE = 93,
-    YP_NODE_PINNED_VARIABLE_NODE = 94,
-    YP_NODE_POST_EXECUTION_NODE = 95,
-    YP_NODE_PRE_EXECUTION_NODE = 96,
-    YP_NODE_PROGRAM_NODE = 97,
-    YP_NODE_RANGE_NODE = 98,
-    YP_NODE_RATIONAL_NODE = 99,
-    YP_NODE_REDO_NODE = 100,
-    YP_NODE_REGULAR_EXPRESSION_NODE = 101,
-    YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 102,
-    YP_NODE_REQUIRED_PARAMETER_NODE = 103,
-    YP_NODE_RESCUE_MODIFIER_NODE = 104,
-    YP_NODE_RESCUE_NODE = 105,
-    YP_NODE_REST_PARAMETER_NODE = 106,
-    YP_NODE_RETRY_NODE = 107,
-    YP_NODE_RETURN_NODE = 108,
-    YP_NODE_SELF_NODE = 109,
-    YP_NODE_SINGLETON_CLASS_NODE = 110,
-    YP_NODE_SOURCE_ENCODING_NODE = 111,
-    YP_NODE_SOURCE_FILE_NODE = 112,
-    YP_NODE_SOURCE_LINE_NODE = 113,
-    YP_NODE_SPLAT_NODE = 114,
-    YP_NODE_STATEMENTS_NODE = 115,
-    YP_NODE_STRING_CONCAT_NODE = 116,
-    YP_NODE_STRING_NODE = 117,
-    YP_NODE_SUPER_NODE = 118,
-    YP_NODE_SYMBOL_NODE = 119,
-    YP_NODE_TRUE_NODE = 120,
-    YP_NODE_UNDEF_NODE = 121,
-    YP_NODE_UNLESS_NODE = 122,
-    YP_NODE_UNTIL_NODE = 123,
-    YP_NODE_WHEN_NODE = 124,
-    YP_NODE_WHILE_NODE = 125,
-    YP_NODE_X_STRING_NODE = 126,
-    YP_NODE_YIELD_NODE = 127,
+    YP_NODE_FLIP_FLOP_NODE = 46,
+    YP_NODE_FLOAT_NODE = 47,
+    YP_NODE_FOR_NODE = 48,
+    YP_NODE_FORWARDING_ARGUMENTS_NODE = 49,
+    YP_NODE_FORWARDING_PARAMETER_NODE = 50,
+    YP_NODE_FORWARDING_SUPER_NODE = 51,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 52,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 53,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE = 54,
+    YP_NODE_GLOBAL_VARIABLE_READ_NODE = 55,
+    YP_NODE_GLOBAL_VARIABLE_WRITE_NODE = 56,
+    YP_NODE_HASH_NODE = 57,
+    YP_NODE_HASH_PATTERN_NODE = 58,
+    YP_NODE_IF_NODE = 59,
+    YP_NODE_IMAGINARY_NODE = 60,
+    YP_NODE_IN_NODE = 61,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE = 62,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE = 63,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 64,
+    YP_NODE_INSTANCE_VARIABLE_READ_NODE = 65,
+    YP_NODE_INSTANCE_VARIABLE_WRITE_NODE = 66,
+    YP_NODE_INTEGER_NODE = 67,
+    YP_NODE_INTERPOLATED_REGULAR_EXPRESSION_NODE = 68,
+    YP_NODE_INTERPOLATED_STRING_NODE = 69,
+    YP_NODE_INTERPOLATED_SYMBOL_NODE = 70,
+    YP_NODE_INTERPOLATED_X_STRING_NODE = 71,
+    YP_NODE_KEYWORD_HASH_NODE = 72,
+    YP_NODE_KEYWORD_PARAMETER_NODE = 73,
+    YP_NODE_KEYWORD_REST_PARAMETER_NODE = 74,
+    YP_NODE_LAMBDA_NODE = 75,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 76,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 77,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 78,
+    YP_NODE_LOCAL_VARIABLE_READ_NODE = 79,
+    YP_NODE_LOCAL_VARIABLE_WRITE_NODE = 80,
+    YP_NODE_MATCH_PREDICATE_NODE = 81,
+    YP_NODE_MATCH_REQUIRED_NODE = 82,
+    YP_NODE_MISSING_NODE = 83,
+    YP_NODE_MODULE_NODE = 84,
+    YP_NODE_MULTI_WRITE_NODE = 85,
+    YP_NODE_NEXT_NODE = 86,
+    YP_NODE_NIL_NODE = 87,
+    YP_NODE_NO_KEYWORDS_PARAMETER_NODE = 88,
+    YP_NODE_NUMBERED_REFERENCE_READ_NODE = 89,
+    YP_NODE_OPTIONAL_PARAMETER_NODE = 90,
+    YP_NODE_OR_NODE = 91,
+    YP_NODE_PARAMETERS_NODE = 92,
+    YP_NODE_PARENTHESES_NODE = 93,
+    YP_NODE_PINNED_EXPRESSION_NODE = 94,
+    YP_NODE_PINNED_VARIABLE_NODE = 95,
+    YP_NODE_POST_EXECUTION_NODE = 96,
+    YP_NODE_PRE_EXECUTION_NODE = 97,
+    YP_NODE_PROGRAM_NODE = 98,
+    YP_NODE_RANGE_NODE = 99,
+    YP_NODE_RATIONAL_NODE = 100,
+    YP_NODE_REDO_NODE = 101,
+    YP_NODE_REGULAR_EXPRESSION_NODE = 102,
+    YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 103,
+    YP_NODE_REQUIRED_PARAMETER_NODE = 104,
+    YP_NODE_RESCUE_MODIFIER_NODE = 105,
+    YP_NODE_RESCUE_NODE = 106,
+    YP_NODE_REST_PARAMETER_NODE = 107,
+    YP_NODE_RETRY_NODE = 108,
+    YP_NODE_RETURN_NODE = 109,
+    YP_NODE_SELF_NODE = 110,
+    YP_NODE_SINGLETON_CLASS_NODE = 111,
+    YP_NODE_SOURCE_ENCODING_NODE = 112,
+    YP_NODE_SOURCE_FILE_NODE = 113,
+    YP_NODE_SOURCE_LINE_NODE = 114,
+    YP_NODE_SPLAT_NODE = 115,
+    YP_NODE_STATEMENTS_NODE = 116,
+    YP_NODE_STRING_CONCAT_NODE = 117,
+    YP_NODE_STRING_NODE = 118,
+    YP_NODE_SUPER_NODE = 119,
+    YP_NODE_SYMBOL_NODE = 120,
+    YP_NODE_TRUE_NODE = 121,
+    YP_NODE_UNDEF_NODE = 122,
+    YP_NODE_UNLESS_NODE = 123,
+    YP_NODE_UNTIL_NODE = 124,
+    YP_NODE_WHEN_NODE = 125,
+    YP_NODE_WHILE_NODE = 126,
+    YP_NODE_X_STRING_NODE = 127,
+    YP_NODE_YIELD_NODE = 128,
 };
 
 typedef uint16_t yp_node_type_t;
 typedef uint16_t yp_node_flags_t;
 
-// We store the flags enum in every node in the tree
-static const uint16_t YP_NODE_FLAG_NEWLINE = 0x1;
+// We store the flags enum in every node in the tree. Some flags are common to
+// all nodes (the ones listed below). Others are specific to certain node types.
+static const yp_node_flags_t YP_NODE_FLAG_NEWLINE = 0x1;
 
 // For easy access, we define some macros to check node type
 #define YP_NODE_TYPE(node) ((enum yp_node_type)node->type)
@@ -498,7 +500,6 @@ typedef struct yp_call_node {
     struct yp_arguments_node *arguments;
     yp_location_t closing_loc;
     struct yp_block_node *block;
-    uint32_t flags;
     yp_string_t name;
 } yp_call_node_t;
 
@@ -746,6 +747,14 @@ typedef struct yp_find_pattern_node {
     yp_location_t closing_loc;
 } yp_find_pattern_node_t;
 
+// FlipFlopNode
+typedef struct yp_flip_flop_node {
+    yp_node_t base;
+    struct yp_node *left;
+    struct yp_node *right;
+    yp_location_t operator_loc;
+} yp_flip_flop_node_t;
+
 // FloatNode
 typedef struct yp_float_node {
     yp_node_t base;
@@ -909,7 +918,6 @@ typedef struct yp_interpolated_regular_expression_node {
     yp_location_t opening_loc;
     struct yp_node_list parts;
     yp_location_t closing_loc;
-    uint32_t flags;
 } yp_interpolated_regular_expression_node_t;
 
 // InterpolatedStringNode
@@ -1159,7 +1167,6 @@ typedef struct yp_range_node {
     struct yp_node *left;
     struct yp_node *right;
     yp_location_t operator_loc;
-    uint32_t flags;
 } yp_range_node_t;
 
 // RationalNode
@@ -1180,7 +1187,6 @@ typedef struct yp_regular_expression_node {
     yp_location_t content_loc;
     yp_location_t closing_loc;
     yp_string_t unescaped;
-    uint32_t flags;
 } yp_regular_expression_node_t;
 
 // RequiredDestructuredParameterNode
@@ -1343,7 +1349,6 @@ typedef struct yp_until_node {
     yp_location_t keyword_loc;
     struct yp_node *predicate;
     struct yp_statements_node *statements;
-    uint32_t flags;
 } yp_until_node_t;
 
 // WhenNode
@@ -1360,7 +1365,6 @@ typedef struct yp_while_node {
     yp_location_t keyword_loc;
     struct yp_node *predicate;
     struct yp_statements_node *statements;
-    uint32_t flags;
 } yp_while_node_t;
 
 // XStringNode
@@ -1383,30 +1387,30 @@ typedef struct yp_yield_node {
 
 // CallNodeFlags
 typedef enum {
-    YP_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 0,
-    YP_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 1,
+    YP_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 1,
+    YP_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 2,
 } yp_call_node_flags_t;
 
 // LoopFlags
 typedef enum {
-    YP_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 0,
+    YP_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 1,
 } yp_loop_flags_t;
 
-// RangeNodeFlags
+// RangeFlags
 typedef enum {
-    YP_RANGE_NODE_FLAGS_EXCLUDE_END = 1 << 0,
-} yp_range_node_flags_t;
+    YP_RANGE_FLAGS_EXCLUDE_END = 1 << 1,
+} yp_range_flags_t;
 
 // RegularExpressionFlags
 typedef enum {
-    YP_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 0,
-    YP_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 1,
-    YP_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 2,
-    YP_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 3,
-    YP_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 4,
-    YP_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 5,
-    YP_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 6,
-    YP_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 7,
+    YP_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 1,
+    YP_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 2,
+    YP_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 3,
+    YP_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 4,
+    YP_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 5,
+    YP_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 6,
+    YP_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 7,
+    YP_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 8,
 } yp_regular_expression_flags_t;
 
 #endif // YARP_AST_H

data/include/yarp/defines.h

@@ -3,8 +3,6 @@
 
 // This file should be included first by any *.h or *.c in YARP
 
-#include "yarp/config.h"
-
 #include <ctype.h>
 #include <stdarg.h>
 #include <stddef.h>
@@ -38,17 +36,4 @@
 
 int yp_strncasecmp(const char *string1, const char *string2, size_t length);
 
-int yp_snprintf(char *dest, YP_ATTRIBUTE_UNUSED size_t size, const char *format, ...);
-
-#if defined(HAVE_SNPRINTF)
-    // We use snprintf if it's available
-#   define yp_snprintf snprintf
-
-#else
-    // In case snprintf isn't present on the system, we provide our own that simply
-    // forwards to the less-safe sprintf.
-#   define yp_snprintf(dest, size, ...) sprintf((dest), __VA_ARGS__)
-
-#endif
-
 #endif

data/include/yarp/enc/yp_encoding.h

@@ -87,6 +87,7 @@ extern yp_encoding_t yp_encoding_iso_8859_16;
 extern yp_encoding_t yp_encoding_koi8_r;
 extern yp_encoding_t yp_encoding_shift_jis;
 extern yp_encoding_t yp_encoding_utf_8;
+extern yp_encoding_t yp_encoding_utf8_mac;
 extern yp_encoding_t yp_encoding_windows_31j;
 extern yp_encoding_t yp_encoding_windows_1251;
 extern yp_encoding_t yp_encoding_windows_1252;

data/include/yarp/util/yp_buffer.h

@@ -12,6 +12,7 @@
 // A yp_buffer_t is a simple memory buffer that stores data in a contiguous
 // block of memory. It is used to store the serialized representation of a
 // YARP tree.
+// NOTE: keep in sync with YARP::LibRubyParser::Buffer in lib/yarp.rb
 typedef struct {
     char *value;
     size_t length;

data/include/yarp/util/yp_string.h

@@ -36,7 +36,7 @@ void yp_string_constant_init(yp_string_t *string, const char *source, size_t len
 // for large files). This means that if we're on windows we'll use
 // `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
 // `mmap`, and on other POSIX systems we'll use `read`.
-bool yp_string_mapped_init(yp_string_t *string, const char *filepath);
+YP_EXPORTED_FUNCTION bool yp_string_mapped_init(yp_string_t *string, const char *filepath);
 
 // Returns the memory size associated with the string.
 size_t yp_string_memsize(const yp_string_t *string);
@@ -54,4 +54,8 @@ YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string);
 // Free the associated memory of the given string.
 YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string);
 
+// Returns the size of the yp_string_t struct. This is necessary to allocate the
+// correct amount of memory in the FFI backend.
+YP_EXPORTED_FUNCTION size_t yp_string_sizeof(void);
+
 #endif // YARP_STRING_H

data/include/yarp/version.h

@@ -1,5 +1,4 @@
 #define YP_VERSION_MAJOR 0
-#define YP_VERSION_MINOR 6
+#define YP_VERSION_MINOR 7
 #define YP_VERSION_PATCH 0
-
-#define YP_VERSION "0.6.0"
+#define YP_VERSION "0.7.0"

data/include/yarp.h

@@ -59,10 +59,12 @@ YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, y
 // Serialize the AST represented by the given node to the given buffer.
 YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
 
-// Parse and serialize the AST represented by the given source to the given
-// buffer.
+// Parse the given source to the AST and serialize the AST to the given buffer.
 YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
 
+// Lex the given source and serialize to the given buffer.
+YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer);
+
 // Returns a string representation of the given token type.
 YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);
 

data/lib/yarp/ffi.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+# This file is responsible for mirroring the API provided by the C extension by
+# using FFI to call into the shared library.
+
+require "rbconfig"
+require "ffi"
+
+module YARP
+  BACKEND = :FFI
+
+  module LibRubyParser
+    extend FFI::Library
+
+    # Define the library that we will be pulling functions from. Note that this
+    # must align with the build shared library from make/rake.
+    ffi_lib File.expand_path("../../build/librubyparser.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
+
+    # Convert a native C type declaration into a symbol that FFI understands.
+    # For example:
+    #
+    #     const char * -> :pointer
+    #     bool         -> :bool
+    #     size_t       -> :size_t
+    #     void         -> :void
+    #
+    def self.resolve_type(type)
+      type = type.strip.delete_prefix("const ")
+      type.end_with?("*") ? :pointer : type.to_sym
+    end
+
+    # Read through the given header file and find the declaration of each of the
+    # given functions. For each one, define a function with the same name and
+    # signature as the C function.
+    def self.load_exported_functions_from(header, *functions)
+      File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line|
+        # We only want to attempt to load exported functions.
+        next unless line.start_with?("YP_EXPORTED_FUNCTION ")
+
+        # We only want to load the functions that we are interested in.
+        next unless functions.any? { |function| line.include?(function) }
+
+        # Parse the function declaration.
+        unless /^YP_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
+          raise "Could not parse #{line}"
+        end
+
+        # Delete the function from the list of functions we are looking for to
+        # mark it as having been found.
+        functions.delete(name)
+
+        # Split up the argument types into an array, ensure we handle the case
+        # where there are no arguments (by explicit void).
+        arg_types = arg_types.split(",").map(&:strip)
+        arg_types = [] if arg_types == %w[void]
+
+        # Resolve the type of the argument by dropping the name of the argument
+        # first if it is present.
+        arg_types.map! { |type| resolve_type(type.sub(/\w+$/, "")) }
+
+        # Attach the function using the FFI library.
+        attach_function name, arg_types, resolve_type(return_type)
+      end
+
+      # If we didn't find all of the functions, raise an error.
+      raise "Could not find functions #{functions.inspect}" unless functions.empty?
+    end
+
+    load_exported_functions_from(
+      "yarp.h",
+      "yp_version",
+      "yp_parse_serialize",
+      "yp_lex_serialize"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_buffer.h",
+      "yp_buffer_init",
+      "yp_buffer_free"
+    )
+
+    load_exported_functions_from(
+      "yarp/util/yp_string.h",
+      "yp_string_mapped_init",
+      "yp_string_free",
+      "yp_string_source",
+      "yp_string_length",
+      "yp_string_sizeof"
+    )
+
+    # This object represents a yp_buffer_t. Its structure must be kept in sync
+    # with the C version.
+    class YPBuffer < FFI::Struct
+      layout value: :pointer, length: :size_t, capacity: :size_t
+
+      # Read the contents of the buffer into a String object and return it.
+      def to_ruby_string
+        self[:value].read_string(self[:length])
+      end
+    end
+
+    # Initialize a new buffer and yield it to the block. The buffer will be
+    # automatically freed when the block returns.
+    def self.with_buffer(&block)
+      buffer = YPBuffer.new
+
+      begin
+        raise unless yp_buffer_init(buffer)
+        yield buffer
+      ensure
+        yp_buffer_free(buffer)
+        buffer.pointer.free
+      end
+    end
+
+    # This object represents a yp_string_t. We only use it as an opaque pointer,
+    # so it doesn't have to be an FFI::Struct.
+    class YPString
+      attr_reader :pointer
+
+      def initialize(pointer)
+        @pointer = pointer
+      end
+
+      def source
+        LibRubyParser.yp_string_source(pointer)
+      end
+
+      def length
+        LibRubyParser.yp_string_length(pointer)
+      end
+
+      def read
+        source.read_string(length)
+      end
+    end
+
+    # This is the size of a yp_string_t. It is returned by the yp_string_sizeof
+    # function which we call once to ensure we have sufficient space for the
+    # yp_string_t FFI pointer.
+    SIZEOF_YP_STRING = yp_string_sizeof
+
+    # Yields a yp_string_t pointer to the given block.
+    def self.with_string(filepath, &block)
+      string = FFI::MemoryPointer.new(SIZEOF_YP_STRING)
+
+      begin
+        raise unless yp_string_mapped_init(string, filepath)
+        yield YPString.new(string)
+      ensure
+        yp_string_free(string)
+        string.free
+      end
+    end
+  end
+
+  # Mark the LibRubyParser module as private as it should only be called through
+  # the YARP module.
+  private_constant :LibRubyParser
+
+  # The version constant is set by reading the result of calling yp_version.
+  VERSION = LibRubyParser.yp_version.read_string
+
+  def self.dump_internal(source, source_size, filepath)
+    LibRubyParser.with_buffer do |buffer|
+      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+      LibRubyParser.yp_parse_serialize(source, source_size, buffer, metadata)
+      buffer.to_ruby_string
+    end
+  end
+  private_class_method :dump_internal
+
+  # Mirror the YARP.dump API by using the serialization API.
+  def self.dump(code, filepath = nil)
+    dump_internal(code, code.bytesize, filepath)
+  end
+
+  # Mirror the YARP.dump_file API by using the serialization API.
+  def self.dump_file(filepath)
+    LibRubyParser.with_string(filepath) do |string|
+      dump_internal(string.source, string.length, filepath)
+    end
+  end
+
+  # Mirror the YARP.lex API by using the serialization API.
+  def self.lex(code, filepath = nil)
+    LibRubyParser.with_buffer do |buffer|
+      LibRubyParser.yp_lex_serialize(code, code.bytesize, filepath, buffer)
+
+      source = Source.new(code)
+      Serialize.load_tokens(source, buffer.to_ruby_string).with_source(source)
+    end
+  end
+
+  # Mirror the YARP.lex_file API by using the serialization API.
+  def self.lex_file(filepath)
+    LibRubyParser.with_string(filepath) { |string| lex(string.read, filepath) }
+  end
+
+  # Mirror the YARP.parse API by using the serialization API.
+  def self.parse(code, filepath = nil)
+    YARP.load(code, dump(code, filepath)).with_source(Source.new(code))
+  end
+
+  # Mirror the YARP.parse_file API by using the serialization API. This uses
+  # native strings instead of Ruby strings because it allows us to use mmap when
+  # it is available.
+  def self.parse_file(filepath)
+    LibRubyParser.with_string(filepath) { |string| parse(string.read, filepath) }
+  end
+end

data/lib/yarp/lex_compat.rb

@@ -647,19 +647,34 @@ module YARP
         # can shuffle around the token to match Ripper's output.
         case state
         when :default
+          # The default state is when there are no heredocs at all. In this
+          # state we can append the token to the list of tokens and move on.
           tokens << token
 
+          # If we get the declaration of a heredoc, then we open a new heredoc
+          # and move into the heredoc_opened state.
           if event == :on_heredoc_beg
             state = :heredoc_opened
             heredoc_stack.last << Heredoc.build(token)
           end
         when :heredoc_opened
+          # The heredoc_opened state is when we've seen the declaration of a
+          # heredoc and are now lexing the body of the heredoc. In this state we
+          # push tokens onto the most recently created heredoc.
           heredoc_stack.last.last << token
 
           case event
           when :on_heredoc_beg
+            # If we receive a heredoc declaration while lexing the body of a
+            # heredoc, this means we have nested heredocs. In this case we'll
+            # push a new heredoc onto the stack and stay in the heredoc_opened
+            # state since we're now lexing the body of the new heredoc.
             heredoc_stack << [Heredoc.build(token)]
           when :on_heredoc_end
+            # If we receive the end of a heredoc, then we're done lexing the
+            # body of the heredoc. In this case we now have a completed heredoc
+            # but need to wait for the next newline to push it into the token
+            # stream.
             state = :heredoc_closed
           end
         when :heredoc_closed
@@ -734,8 +749,7 @@ module YARP
       when :on_sp
         # skip
       when :on_tstring_content
-        if previous[1] == :on_tstring_content &&
-            (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
+        if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
           previous[2] << token[2]
         else
           results << token