herb 0.11.1 → 0.11.2

data/vendor/prism/templates/src/serialize.c.erb

@@ -1,57 +1,62 @@
-#include "prism.h"
+#include "prism/excludes.h"
+
+/* We optionally support serializing to a binary string. For systems that do not
+ * want or need this functionality, it can be turned off with the
+ * PRISM_EXCLUDE_SERIALIZATION define. */
+#ifdef PRISM_EXCLUDE_SERIALIZATION
+
+/* Ensure this translation unit is never empty, even when serialization is
+ * excluded. */
+typedef int pm_serialize_unused_t;
+
+#else
+
+#include "prism/compiler/branch.h"
+#include "prism/compiler/inline.h"
+
+#include "prism/internal/allocator.h"
+#include "prism/internal/buffer.h"
+#include "prism/internal/comments.h"
+#include "prism/internal/diagnostic.h"
+#include "prism/internal/encoding.h"
+#include "prism/internal/list.h"
+#include "prism/internal/magic_comments.h"
+#include "prism/internal/options.h"
+#include "prism/internal/parser.h"
 
-// We optionally support serializing to a binary string. For systems that don't
-// want or need this functionality, it can be turned off with the
-// PRISM_EXCLUDE_SERIALIZATION define.
-#ifndef PRISM_EXCLUDE_SERIALIZATION
+#include "prism.h"
+#include "prism/ast.h"
+#include "prism/line_offset_list.h"
 
+#include <assert.h>
+#include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
-static inline uint32_t
+static PRISM_INLINE uint32_t
 pm_ptrdifft_to_u32(ptrdiff_t value) {
     assert(value >= 0 && ((unsigned long) value) < UINT32_MAX);
     return (uint32_t) value;
 }
 
-static inline uint32_t
+static PRISM_INLINE uint32_t
 pm_sizet_to_u32(size_t value) {
     assert(value < UINT32_MAX);
     return (uint32_t) value;
 }
 
 static void
-pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location, pm_buffer_t *buffer) {
-    assert(location->start);
-    assert(location->end);
-    assert(location->start <= location->end);
-
-    pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->start - parser->start));
-    pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->end - location->start));
+pm_serialize_location(const pm_location_t *location, pm_buffer_t *buffer) {
+    pm_buffer_append_varuint(buffer, location->start);
+    pm_buffer_append_varuint(buffer, location->length);
 }
 
 static void
-pm_serialize_string(const pm_parser_t *parser, const pm_string_t *string, pm_buffer_t *buffer) {
-    switch (string->type) {
-        case PM_STRING_SHARED: {
-            pm_buffer_append_byte(buffer, 1);
-            pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(pm_string_source(string) - parser->start));
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_string_length(string)));
-            break;
-        }
-        case PM_STRING_OWNED:
-        case PM_STRING_CONSTANT: {
-            uint32_t length = pm_sizet_to_u32(pm_string_length(string));
-            pm_buffer_append_byte(buffer, 2);
-            pm_buffer_append_varuint(buffer, length);
-            pm_buffer_append_bytes(buffer, pm_string_source(string), length);
-            break;
-        }
-#ifdef PRISM_HAS_MMAP
-        case PM_STRING_MAPPED:
-            assert(false && "Cannot serialize mapped strings.");
-            break;
-#endif
-    }
+pm_serialize_string(const pm_string_t *string, pm_buffer_t *buffer) {
+    uint32_t length = pm_sizet_to_u32(pm_string_length(string));
+    pm_buffer_append_varuint(buffer, length);
+    pm_buffer_append_bytes(buffer, pm_string_source(string), length);
 }
 
 static void
@@ -68,97 +73,247 @@ pm_serialize_integer(const pm_integer_t *integer, pm_buffer_t *buffer) {
     }
 }
 
+/**
+ * The kinds of entries on the serialization stack.
+ */
+typedef enum {
+    /** A node to serialize, or NULL for an absent optional node. */
+    PM_SERIALIZE_ENTRY_NODE,
+
+    /** Write the lazy marker and placeholders for its skip and length. */
+    PM_SERIALIZE_ENTRY_LAZY_HEADER,
+
+    /** Patch the skip, now that the lazy subtree is about to be written. */
+    PM_SERIALIZE_ENTRY_LAZY_START,
+
+    /** Patch the length, now that the lazy subtree has been written. */
+    PM_SERIALIZE_ENTRY_LAZY_END
+} pm_serialize_entry_kind_t;
+
+/**
+ * An entry on the serialization stack.
+ */
+typedef struct {
+    /** The node to serialize, for NODE entries. */
+    const pm_node_t *node;
+
+    /**
+     * For NODE entries, the offset of the placeholders that follow the node's
+     * lazy marker. For LAZY_* entries, the index of the NODE entry that owns
+     * them, which stays valid because that entry is beneath them.
+     */
+    size_t offset;
+
+    /** The kind of this entry. */
+    pm_serialize_entry_kind_t kind;
+
+    /** Whether the children of this node have already been pushed. */
+    bool visited;
+} pm_serialize_entry_t;
+
+/**
+ * The explicit stack used to walk the tree, so that serializing deeply nested
+ * trees does not depend on the size of the C stack.
+ */
+typedef struct {
+    /** The entries on the stack. */
+    pm_serialize_entry_t *entries;
+
+    /** The number of entries on the stack. */
+    size_t size;
+
+    /** The number of entries that fit in the allocated memory. */
+    size_t capacity;
+} pm_serialize_stack_t;
+
+/**
+ * Push an entry onto the serialization stack. This may reallocate the stack,
+ * which invalidates any pointers into it.
+ */
 static void
-pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
-    pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node));
+pm_serialize_stack_push(pm_serialize_stack_t *stack, pm_serialize_entry_kind_t kind, const pm_node_t *node, size_t offset) {
+    if (PRISM_UNLIKELY(stack->size == stack->capacity)) {
+        size_t capacity = stack->capacity == 0 ? 64 : stack->capacity * 2;
+        pm_serialize_entry_t *entries = (pm_serialize_entry_t *) xrealloc_sized(stack->entries, capacity * sizeof(pm_serialize_entry_t), stack->capacity * sizeof(pm_serialize_entry_t));
+
+        if (PRISM_UNLIKELY(entries == NULL)) {
+            fprintf(stderr, "prism: out of memory; aborting\n");
+            abort();
+        }
 
-    size_t offset = buffer->length;
+        stack->entries = entries;
+        stack->capacity = capacity;
+    }
 
-    <%- if Prism::Template::INCLUDE_NODE_ID -%>
-    pm_buffer_append_varuint(buffer, node->node_id);
-    <%- end -%>
-    pm_serialize_location(parser, &node->location, buffer);
-
-    switch (PM_NODE_TYPE(node)) {
-        // We do not need to serialize a ScopeNode ever as
-        // it is not part of the AST
-        case PM_SCOPE_NODE:
-            return;
-        <%- nodes.each do |node| -%>
-        case <%= node.type %>: {
-            <%- if node.needs_serialized_length? -%>
-            // serialize length
-            // encoding of location u32s make us need to save this offset.
-            size_t length_offset = buffer->length;
-            pm_buffer_append_string(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
-            <%- end -%>
-            <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS && !node.flags -%>
-            pm_buffer_append_varuint(buffer, (uint32_t) node->flags);
-            <%- end -%>
-            <%- node.fields.each do |field| -%>
-            <%- case field -%>
-            <%- when Prism::Template::NodeField -%>
-            pm_serialize_node(parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
-            <%- when Prism::Template::OptionalNodeField -%>
-            if (((pm_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
-                pm_buffer_append_byte(buffer, 0);
-            } else {
-                pm_serialize_node(parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+    stack->entries[stack->size++] = (pm_serialize_entry_t) { .node = node, .offset = offset, .kind = kind, .visited = false };
+}
+
+/**
+ * Serialize the tree rooted at the given node in post-order: the subtrees of a
+ * node's child fields are written in field order, followed by the node's own
+ * record. A loader can therefore build every node from a stack of already
+ * built children without recursing.
+ *
+ * The child subtrees of a node with a lazy field are preceded by the lazy
+ * marker, the byte length of the subtrees before the lazy field's subtree, and
+ * the byte length of the lazy field's subtree, so that a loader can skip it.
+ */
+static void
+pm_serialize_node(pm_node_t *root, pm_buffer_t *buffer) {
+    pm_serialize_stack_t stack = { 0 };
+    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_NODE, root, 0);
+
+    while (stack.size > 0) {
+        pm_serialize_entry_t *entry = &stack.entries[--stack.size];
+
+        switch (entry->kind) {
+            case PM_SERIALIZE_ENTRY_LAZY_HEADER: {
+                pm_buffer_append_byte(buffer, <%= Prism::Template::LAZY_MARKER %>);
+                stack.entries[entry->offset].offset = buffer->length;
+                pm_buffer_append_zeroes(buffer, 8);
+                continue;
             }
-            <%- when Prism::Template::StringField -%>
-            pm_serialize_string(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
-            <%- when Prism::Template::NodeListField -%>
-            uint32_t <%= field.name %>_size = pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.size);
-            pm_buffer_append_varuint(buffer, <%= field.name %>_size);
-            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
-                pm_serialize_node(parser, (pm_node_t *) ((pm_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer);
+            case PM_SERIALIZE_ENTRY_LAZY_START: {
+                size_t offset = stack.entries[entry->offset].offset;
+                uint32_t skip = pm_sizet_to_u32(buffer->length - offset - 8);
+                memcpy(buffer->value + offset, &skip, sizeof(uint32_t));
+                continue;
             }
-            <%- when Prism::Template::ConstantField, Prism::Template::OptionalConstantField -%>
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>));
-            <%- when Prism::Template::ConstantListField -%>
-            uint32_t <%= field.name %>_size = pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.size);
-            pm_buffer_append_varuint(buffer, <%= field.name %>_size);
-            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
-                pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
+            case PM_SERIALIZE_ENTRY_LAZY_END: {
+                size_t offset = stack.entries[entry->offset].offset;
+                uint32_t skip;
+                memcpy(&skip, buffer->value + offset, sizeof(uint32_t));
+
+                uint32_t length = pm_sizet_to_u32(buffer->length - offset - 8 - skip);
+                memcpy(buffer->value + offset + 4, &length, sizeof(uint32_t));
+                continue;
             }
-            <%- when Prism::Template::LocationField -%>
-            <%- if field.should_be_serialized? -%>
-            pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
-            <%- end -%>
-            <%- when Prism::Template::OptionalLocationField -%>
-            <%- if field.should_be_serialized? -%>
-            if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
-                pm_buffer_append_byte(buffer, 0);
-            } else {
-                pm_buffer_append_byte(buffer, 1);
-                pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            case PM_SERIALIZE_ENTRY_NODE:
+                break;
+        }
+
+        const pm_node_t *node = entry->node;
+
+        if (node == NULL) {
+            pm_buffer_append_byte(buffer, 0);
+            continue;
+        }
+
+        if (!entry->visited) {
+            /* Leave the node on the stack beneath its children, which are
+             * pushed in reverse field order so that they are popped, and
+             * therefore written, in field order. A node without children is
+             * written straight away instead. */
+            entry->visited = true;
+            size_t index = stack.size++;
+
+            switch (PM_NODE_TYPE(node)) {
+                <%- nodes.each do |node| -%>
+                <%- next if node.child_fields.empty? -%>
+                case <%= node.type %>: {
+                    const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node;
+                    <%- node.child_fields.reverse_each do |field| -%>
+                    <%- case field -%>
+                    <%- when Prism::Template::NodeListField -%>
+                    for (size_t list_index = cast-><%= field.name %>.size; list_index > 0; list_index--) {
+                        pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_NODE, cast-><%= field.name %>.nodes[list_index - 1], 0);
+                    }
+                    <%- else -%>
+                    <%- if field == node.lazy_field -%>
+                    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_LAZY_END, NULL, index);
+                    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_NODE, (const pm_node_t *) cast-><%= field.name %>, 0);
+                    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_LAZY_START, NULL, index);
+                    <%- else -%>
+                    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_NODE, (const pm_node_t *) cast-><%= field.name %>, 0);
+                    <%- end -%>
+                    <%- end -%>
+                    <%- end -%>
+                    <%- if node.lazy_field -%>
+                    pm_serialize_stack_push(&stack, PM_SERIALIZE_ENTRY_LAZY_HEADER, NULL, index);
+                    <%- end -%>
+                    break;
+                }
+                <%- end -%>
+                default:
+                    break;
             }
-            <%- end -%>
-            <%- when Prism::Template::UInt8Field -%>
-            pm_buffer_append_byte(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
-            <%- when Prism::Template::UInt32Field -%>
-            pm_buffer_append_varuint(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
-            <%- when Prism::Template::IntegerField -%>
-            pm_serialize_integer(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
-            <%- when Prism::Template::DoubleField -%>
-            pm_buffer_append_double(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
-            <%- else -%>
-            <%- raise -%>
-            <%- end -%>
-            <%- end -%>
-            <%- if node.needs_serialized_length? -%>
-            // serialize length
-            uint32_t length = pm_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
-            memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
-            <%- end -%>
-            break;
+
+            if (stack.size != index + 1) continue;
+            stack.size--;
         }
+
+        pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node));
+        <%- if Prism::Template::INCLUDE_NODE_ID -%>
+        pm_buffer_append_varuint(buffer, node->node_id);
         <%- end -%>
+        pm_serialize_location(&node->location, buffer);
+
+        switch (PM_NODE_TYPE(node)) {
+            /* A ScopeNode is never part of the tree, so it has no fields to
+             * serialize. */
+            case PM_SCOPE_NODE:
+                break;
+            <%- nodes.each do |node| -%>
+            case <%= node.type %>: {
+                <%- if node.fields.any? { |field| !field.is_a?(Prism::Template::NodeField) && !field.is_a?(Prism::Template::OptionalNodeField) && field.should_be_serialized? } -%>
+                const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node;
+                <%- end -%>
+                <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS && !node.flags -%>
+                pm_buffer_append_varuint(buffer, (uint32_t) node->flags);
+                <%- end -%>
+                <%- node.fields.each do |field| -%>
+                <%- case field -%>
+                <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
+                <%- when Prism::Template::NodeListField -%>
+                pm_buffer_append_varuint(buffer, pm_sizet_to_u32(cast-><%= field.name %>.size));
+                <%- when Prism::Template::StringField -%>
+                pm_serialize_string(&cast-><%= field.name %>, buffer);
+                <%- when Prism::Template::ConstantField, Prism::Template::OptionalConstantField -%>
+                pm_buffer_append_varuint(buffer, pm_sizet_to_u32(cast-><%= field.name %>));
+                <%- when Prism::Template::ConstantListField -%>
+                pm_buffer_append_varuint(buffer, pm_sizet_to_u32(cast-><%= field.name %>.size));
+                for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
+                    pm_buffer_append_varuint(buffer, pm_sizet_to_u32(cast-><%= field.name %>.ids[index]));
+                }
+                <%- when Prism::Template::LocationField -%>
+                <%- if field.should_be_serialized? -%>
+                pm_serialize_location(&cast-><%= field.name %>, buffer);
+                <%- end -%>
+                <%- when Prism::Template::OptionalLocationField -%>
+                <%- if field.should_be_serialized? -%>
+                if (cast-><%= field.name %>.length == 0) {
+                    pm_buffer_append_byte(buffer, 0);
+                } else {
+                    pm_buffer_append_byte(buffer, 1);
+                    pm_serialize_location(&cast-><%= field.name %>, buffer);
+                }
+                <%- end -%>
+                <%- when Prism::Template::UInt8Field -%>
+                pm_buffer_append_byte(buffer, cast-><%= field.name %>);
+                <%- when Prism::Template::UInt32Field -%>
+                pm_buffer_append_varuint(buffer, cast-><%= field.name %>);
+                <%- when Prism::Template::IntegerField -%>
+                pm_serialize_integer(&cast-><%= field.name %>, buffer);
+                <%- when Prism::Template::DoubleField -%>
+                pm_buffer_append_double(buffer, cast-><%= field.name %>);
+                <%- else -%>
+                <%- raise -%>
+                <%- end -%>
+                <%- end -%>
+                break;
+            }
+            <%- end -%>
+        }
     }
+
+    xfree_sized(stack.entries, stack.capacity * sizeof(pm_serialize_entry_t));
 }
 
-static void
-pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) {
+/*
+ * Serialize the given list of offsets to the given buffer.
+ */
+void
+pm_serialize_line_offset_list(pm_line_offset_list_t *list, pm_buffer_t *buffer) {
     uint32_t size = pm_sizet_to_u32(list->size);
     pm_buffer_append_varuint(buffer, size);
 
@@ -169,60 +324,90 @@ pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) {
 }
 
 static void
-pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *buffer) {
+pm_serialize_comment(pm_comment_t *comment, pm_buffer_t *buffer) {
     // serialize type
     pm_buffer_append_byte(buffer, (uint8_t) comment->type);
 
     // serialize location
-    pm_serialize_location(parser, &comment->location, buffer);
+    pm_serialize_location(&comment->location, buffer);
 }
 
 /**
  * Serialize the given list of comments to the given buffer.
  */
 void
-pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
+pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer) {
     pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list)));
 
     pm_comment_t *comment;
     for (comment = (pm_comment_t *) list->head; comment != NULL; comment = (pm_comment_t *) comment->node.next) {
-        pm_serialize_comment(parser, comment, buffer);
+        pm_serialize_comment(comment, buffer);
     }
 }
 
+<%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%>
+/**
+ * Serialize the comments attached to each node. This is the number of nodes
+ * that own comments, followed by each of those nodes' id, the number of
+ * comments it owns, and the index of each of those comments in the comment
+ * list.
+ */
 static void
-pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) {
+pm_serialize_comment_attachments(const pm_comment_attachments_t *attachments, pm_buffer_t *buffer) {
+    uint32_t count = 0;
+    for (uint32_t node_id = 0; node_id < attachments->ranges_size; node_id++) {
+        if (attachments->ranges[node_id].length > 0) count++;
+    }
+
+    pm_buffer_append_varuint(buffer, count);
+
+    for (uint32_t node_id = 0; node_id < attachments->ranges_size; node_id++) {
+        pm_comment_range_t range = attachments->ranges[node_id];
+        if (range.length == 0) continue;
+
+        pm_buffer_append_varuint(buffer, node_id);
+        pm_buffer_append_varuint(buffer, range.length);
+
+        for (uint32_t index = 0; index < range.length; index++) {
+            pm_buffer_append_varuint(buffer, attachments->attachments[range.index + index]);
+        }
+    }
+}
+<%- end -%>
+
+static void
+pm_serialize_magic_comment(pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) {
     // serialize key location
-    pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start));
-    pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->key_length));
+    pm_buffer_append_varuint(buffer, magic_comment->key.start);
+    pm_buffer_append_varuint(buffer, magic_comment->key.length);
 
     // serialize value location
-    pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start));
-    pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->value_length));
+    pm_buffer_append_varuint(buffer, magic_comment->value.start);
+    pm_buffer_append_varuint(buffer, magic_comment->value.length);
 }
 
 static void
-pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
+pm_serialize_magic_comment_list(pm_list_t *list, pm_buffer_t *buffer) {
     pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list)));
 
     pm_magic_comment_t *magic_comment;
     for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) {
-        pm_serialize_magic_comment(parser, magic_comment, buffer);
+        pm_serialize_magic_comment(magic_comment, buffer);
     }
 }
 
 static void
 pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) {
-    if (parser->data_loc.end == NULL) {
+    if (parser->data_loc.length == 0) {
         pm_buffer_append_byte(buffer, 0);
     } else {
         pm_buffer_append_byte(buffer, 1);
-        pm_serialize_location(parser, &parser->data_loc, buffer);
+        pm_serialize_location(&parser->data_loc, buffer);
     }
 }
 
 static void
-pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) {
+pm_serialize_diagnostic(pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) {
     // serialize the type
     pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id);
 
@@ -232,18 +417,18 @@ pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buf
     pm_buffer_append_string(buffer, diagnostic->message, message_length);
 
     // serialize location
-    pm_serialize_location(parser, &diagnostic->location, buffer);
+    pm_serialize_location(&diagnostic->location, buffer);
 
     pm_buffer_append_byte(buffer, diagnostic->level);
 }
 
 static void
-pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
+pm_serialize_diagnostic_list(pm_list_t *list, pm_buffer_t *buffer) {
     pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list)));
 
     pm_diagnostic_t *diagnostic;
     for (diagnostic = (pm_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (pm_diagnostic_t *) diagnostic->node.next) {
-        pm_serialize_diagnostic(parser, diagnostic, buffer);
+        pm_serialize_diagnostic(diagnostic, buffer);
     }
 }
 
@@ -261,14 +446,17 @@ static void
 pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) {
     pm_serialize_encoding(parser->encoding, buffer);
     pm_buffer_append_varsint(buffer, parser->start_line);
-    pm_serialize_newline_list(&parser->newline_list, buffer);
+    pm_buffer_append_uint64(buffer, pm_parser_checksum(parser));
+    pm_serialize_line_offset_list(&parser->line_offsets, buffer);
 <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%>
-    pm_serialize_comment_list(parser, &parser->comment_list, buffer);
+    pm_serialize_comment_list(&parser->comment_list, buffer);
+    pm_serialize_comment_attachments(&parser->comment_attachments, buffer);
 <%- end -%>
-    pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer);
+    pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer);
     pm_serialize_data_loc(parser, buffer);
-    pm_serialize_diagnostic_list(parser, &parser->error_list, buffer);
-    pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer);
+    pm_serialize_diagnostic_list(&parser->error_list, buffer);
+    pm_serialize_diagnostic_list(&parser->warning_list, buffer);
+    pm_buffer_append_byte(buffer, (uint8_t) parser->continuable);
 }
 
 #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>"
@@ -288,7 +476,7 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer)
     pm_buffer_append_varuint(buffer, parser->constant_pool.size);
 
     // Now we're going to serialize the content of the node.
-    pm_serialize_node(parser, node, buffer);
+    pm_serialize_node(node, buffer);
 
     // Now we're going to serialize the offset of the constant pool back where
     // we left space for it.
@@ -308,28 +496,12 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer)
             pm_constant_t *constant = &parser->constant_pool.constants[bucket->id - 1];
             size_t buffer_offset = offset + ((((size_t)bucket->id) - 1) * 8);
 
-            if (bucket->type == PM_CONSTANT_POOL_BUCKET_OWNED || bucket->type == PM_CONSTANT_POOL_BUCKET_CONSTANT) {
-                // Since this is an owned or constant constant, we are going to
-                // write its contents into the buffer after the constant pool.
-                // So effectively in place of the source offset, we have a
-                // buffer offset. We will add a leading 1 to indicate that this
-                // is a buffer offset.
-                uint32_t content_offset = pm_sizet_to_u32(buffer->length);
-                uint32_t owned_mask = 1U << 31;
-
-                assert(content_offset < owned_mask);
-                content_offset |= owned_mask;
-
-                memcpy(buffer->value + buffer_offset, &content_offset, 4);
-                pm_buffer_append_bytes(buffer, constant->start, constant->length);
-            } else {
-                // Since this is a shared constant, we are going to write its
-                // source offset directly into the buffer.
-                uint32_t source_offset = pm_ptrdifft_to_u32(constant->start - parser->start);
-                memcpy(buffer->value + buffer_offset, &source_offset, 4);
-            }
+            // Write the constant contents into the buffer after the constant
+            // pool. In place of the source offset, we store a buffer offset.
+            uint32_t content_offset = pm_sizet_to_u32(buffer->length);
+            memcpy(buffer->value + buffer_offset, &content_offset, 4);
+            pm_buffer_append_bytes(buffer, constant->start, constant->length);
 
-            // Now we can write the length of the constant into the buffer.
             uint32_t constant_length = pm_sizet_to_u32(constant->length);
             memcpy(buffer->value + buffer_offset + 4, &constant_length, 4);
         }
@@ -337,7 +509,7 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer)
 }
 
 static void
-serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
+serialize_token(pm_parser_t *parser, pm_token_t *token, void *data) {
     pm_buffer_t *buffer = (pm_buffer_t *) data;
 
     pm_buffer_append_varuint(buffer, token->type);
@@ -349,58 +521,72 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
 /**
  * Lex the given source and serialize to the given buffer.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_serialize_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
     pm_options_t options = { 0 };
     pm_options_read(&options, data);
 
+    pm_arena_t arena = { 0 };
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, &options);
-
-    pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
-        .data = (void *) buffer,
-        .callback = serialize_token,
-    };
+    pm_parser_init(&arena, &parser, source, size, &options);
 
-    parser.lex_callback = &lex_callback;
-    pm_node_t *node = pm_parse(&parser);
+    pm_parser_lex_callback_set(&parser, serialize_token, buffer);
+    pm_parse(&parser);
 
     // Append 0 to mark end of tokens.
     pm_buffer_append_byte(buffer, 0);
 
     pm_serialize_metadata(&parser, buffer);
 
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-    pm_options_free(&options);
+    pm_parser_cleanup(&parser);
+    pm_arena_cleanup(&arena);
+    pm_options_cleanup(&options);
 }
 
 /**
  * Parse and serialize both the AST and the tokens represented by the given
  * source to the given buffer.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
     pm_options_t options = { 0 };
     pm_options_read(&options, data);
 
+    pm_arena_t arena = { 0 };
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, &options);
-
-    pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
-        .data = (void *) buffer,
-        .callback = serialize_token,
-    };
+    pm_parser_init(&arena, &parser, source, size, &options);
 
-    parser.lex_callback = &lex_callback;
+    pm_parser_lex_callback_set(&parser, serialize_token, buffer);
     pm_node_t *node = pm_parse(&parser);
 
     pm_buffer_append_byte(buffer, 0);
     pm_serialize(&parser, node, buffer);
 
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-    pm_options_free(&options);
+    pm_parser_cleanup(&parser);
+    pm_arena_cleanup(&arena);
+    pm_options_cleanup(&options);
+}
+
+/**
+ * Parse the source and return true if it parses without errors or warnings.
+ */
+bool
+pm_serialize_parse_success_p(const uint8_t *source, size_t size, const char *data) {
+    pm_options_t options = { 0 };
+    pm_options_read(&options, data);
+
+    pm_arena_t arena = { 0 };
+    pm_parser_t parser;
+    pm_parser_init(&arena, &parser, source, size, &options);
+
+    pm_parse(&parser);
+
+    bool result = parser.error_list.size == 0;
+    pm_parser_cleanup(&parser);
+    pm_arena_cleanup(&arena);
+    pm_options_cleanup(&options);
+
+    return result;
 }
 
 #endif