herb 0.11.1 → 0.11.2

data/vendor/prism/src/xxh3.c

@@ -0,0 +1,292 @@
+/**
+ * An implementation of the XXH3 hash function.
+ *
+ * The reason some of these names do not look quite like the style of the rest
+ * of the code in Prism is that this was adapted straight from the XXH3
+ * specification here:
+ *
+ * https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
+ *
+ * To better understand this code it is recommended to read the XXH3
+ * specification linked above, where you should be able to follow along with
+ * this implementation more easily. Note this only implements the XXH3_64
+ * variant with the default secret and a seed of 0. Additionally it does not
+ * include a lot of the various CPU-specific optimizations that exist in the
+ * reference implementation.
+ */
+
+#include "prism/internal/xxh3.h"
+
+#include "prism/compiler/accel.h"
+#include "prism/compiler/bswap.h"
+#include "prism/compiler/endianness.h"
+
+#include <string.h>
+
+#if defined(PRISM_HAS_NEON) && defined(PRISM_LITTLE_ENDIAN_COMPTIME)
+    #include <arm_neon.h>
+    #define XXH3_NEON
+#elif defined(PRISM_HAS_SSE2)
+    #include <emmintrin.h>
+    #define XXH3_SSE2
+#endif
+
+#if defined(__SIZEOF_INT128__)
+static uint64_t
+XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
+    __uint128_t x = (__uint128_t) lhs * (__uint128_t) rhs;
+    return ((uint64_t) (x & 0xFFFFFFFFFFFFFFFFULL)) ^ ((uint64_t) (x >> 64));
+}
+#else
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
+#include <intrin.h>
+#endif
+
+static uint64_t
+XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
+    uint64_t lo, hi;
+#if defined(_MSC_VER) && defined(_M_X64)
+    lo = _umul128(lhs, rhs, &hi);
+#elif defined(_MSC_VER) && defined(_M_ARM64)
+    hi = __umulh(lhs, rhs);
+    lo = lhs * rhs;
+#else
+    uint64_t lo_lo = (lhs & 0xFFFFFFFF) * (rhs & 0xFFFFFFFF);
+    uint64_t hi_lo = (lhs >> 32) * (rhs & 0xFFFFFFFF);
+    uint64_t lo_hi = (lhs & 0xFFFFFFFF) * (rhs >> 32);
+    uint64_t hi_hi = (lhs >> 32) * (rhs >> 32);
+
+    uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+    hi = hi_hi + (hi_lo >> 32) + (cross >> 32);
+    lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+#endif
+    return lo ^ hi;
+}
+#endif
+
+static uint32_t
+read32(const void* ptr) {
+    uint32_t val;
+    memcpy(&val, ptr, sizeof(val));
+    if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
+        val = pm_bswap32(val);
+    }
+    return val;
+}
+
+static uint64_t
+read64(const void* ptr) {
+    uint64_t val;
+    memcpy(&val, ptr, sizeof(val));
+    if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
+        val = pm_bswap64(val);
+    }
+    return val;
+}
+
+static const uint64_t PRIME32_1 = 0x9E3779B1U;
+static const uint64_t PRIME32_2 = 0x85EBCA77U;
+static const uint64_t PRIME32_3 = 0xC2B2AE3DU;
+static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
+static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
+static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
+static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
+static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;
+static const uint64_t PRIME_MX1 = 0x165667919E3779F9ULL;
+static const uint64_t PRIME_MX2 = 0x9FB21C651E98DF25ULL;
+
+static const uint8_t secret[192] = {
+    0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
+    0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
+    0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
+    0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
+    0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
+    0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
+    0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
+    0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
+    0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
+    0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
+    0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
+    0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
+};
+
+static uint64_t
+avalanche(uint64_t x) {
+    x ^= (x >> 37);
+    x *= PRIME_MX1;
+    x ^= (x >> 32);
+    return x;
+}
+
+static uint64_t
+avalanche_XXH64(uint64_t x) {
+    x ^= (x >> 33);
+    x *= PRIME64_2;
+    x ^= (x >> 29);
+    x *= PRIME64_3;
+    x ^= (x >> 32);
+    return x;
+}
+
+static uint64_t
+XXH3_64_empty(void) {
+    return avalanche_XXH64(read64(secret + 56) ^ read64(secret + 64));
+}
+
+static uint64_t
+XXH3_64_1to3(const uint8_t *input, size_t length) {
+    uint32_t combined = (uint32_t) input[length - 1] | ((uint32_t) length << 8) | ((uint32_t) input[0] << 16) | ((uint32_t) input[length >> 1] << 24);
+    return avalanche_XXH64((uint64_t) (read32(secret) ^ read32(secret + 4)) ^ (uint64_t) combined);
+}
+
+static uint64_t
+XXH3_64_4to8(const uint8_t *input, size_t length) {
+#define rot(x_, r_) (((x_) << (r_)) | ((x_) >> (64 - (r_))))
+
+    uint64_t combined = (uint64_t) read32(input + length - 4) | ((uint64_t) read32(input) << 32);
+    uint64_t value = (read64(secret + 8) ^ read64(secret + 16)) ^ combined;
+    value ^= rot(value, 49) ^ rot(value, 24);
+    value *= PRIME_MX2;
+    value ^= ((value >> 35) + length);
+    value *= PRIME_MX2;
+    value ^= (value >> 28);
+    return value;
+
+#undef rot
+}
+
+static uint64_t
+XXH3_64_9to16(const uint8_t *input, size_t length) {
+    uint64_t lo = (read64(secret + 24) ^ read64(secret + 32)) ^ read64(input);
+    uint64_t hi = (read64(secret + 40) ^ read64(secret + 48)) ^ read64(input + length - 8);
+    return avalanche(length + pm_bswap64(lo) + hi + XXH3_mul128_fold64(lo, hi));
+}
+
+static uint64_t
+XXH3_64_17to240(const uint8_t *input, size_t length) {
+#define mix(data_, offset_) XXH3_mul128_fold64(read64((data_)) ^ read64(secret + (offset_)), read64((data_) + 8) ^ read64(secret + (offset_) + 8))
+
+    uint64_t acc = length * PRIME64_1;
+
+    if (length <= 128) {
+        size_t idx = (length - 1) / 32;
+        do {
+            acc += mix(input + idx * 16, idx * 32) + mix(input + length - idx * 16 - 16, idx * 32 + 16);
+        } while (idx--);
+    } else {
+        for (size_t idx = 0; idx < 8; idx++) {
+            acc += mix(input + idx * 16, idx * 16);
+        }
+        acc = avalanche(acc);
+        for (size_t idx = 8; idx < length >> 4; idx++) {
+            acc += mix(input + idx * 16, (idx - 8) * 16 + 3);
+        }
+        acc += mix(input + length - 16, 119);
+    }
+
+    return avalanche(acc);
+
+#undef mix
+}
+
+static void
+accumulate(uint64_t acc[8], const uint8_t *stripe, size_t secret_offset) {
+#if defined(XXH3_NEON)
+    for (int idx = 0; idx < 8; idx += 2) {
+        uint64x2_t data = vreinterpretq_u64_u8(vld1q_u8(stripe + 8 * idx));
+        uint64x2_t value = veorq_u64(data, vreinterpretq_u64_u8(vld1q_u8(secret + secret_offset + 8 * idx)));
+        uint64x2_t product = vmull_u32(vmovn_u64(value), vshrn_n_u64(value, 32));
+        vst1q_u64(acc + idx, vaddq_u64(vld1q_u64(acc + idx), vaddq_u64(vextq_u64(data, data, 1), product)));
+    }
+#elif defined(XXH3_SSE2)
+    for (int idx = 0; idx < 8; idx += 2) {
+        __m128i data = _mm_loadu_si128((const __m128i *) (stripe + 8 * idx));
+        __m128i value = _mm_xor_si128(data, _mm_loadu_si128((const __m128i *) (secret + secret_offset + 8 * idx)));
+        __m128i product = _mm_mul_epu32(value, _mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)));
+        _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_loadu_si128((const __m128i *) (acc + idx)), _mm_add_epi64(_mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)), product)));
+    }
+#else
+    for (int idx = 0; idx < 8; idx++) {
+        uint64_t data = read64(stripe + 8 * idx);
+        uint64_t value = data ^ read64(secret + secret_offset + 8 * idx);
+        acc[idx ^ 1] = acc[idx ^ 1] + data;
+        acc[idx] += (value & 0xFFFFFFFF) * (value >> 32);
+    }
+#endif
+}
+
+static uint64_t
+XXH3_64_large(const uint8_t *input, size_t length) {
+    const size_t secret_length = sizeof(secret);
+    const size_t stripes_per_block = (secret_length - 64) / 8;
+    const size_t block_size = 64 * stripes_per_block;
+    const size_t block_count = (length - 1) / block_size;
+    const size_t last_offset = block_count * block_size;
+
+    uint64_t acc[8] = {
+        PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
+        PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1
+    };
+
+    for (size_t block_idx = 0; block_idx < block_count; block_idx++) {
+        const uint8_t *block = input + block_idx * block_size;
+
+        for (size_t stripe_idx = 0; stripe_idx < stripes_per_block; stripe_idx++) {
+            accumulate(acc, block + stripe_idx * 64, stripe_idx * 8);
+        }
+
+#if defined(XXH3_NEON)
+        const uint32x2_t prime = vdup_n_u32((uint32_t) PRIME32_1);
+        for (size_t idx = 0; idx < 8; idx += 2) {
+            uint64x2_t value = vld1q_u64(acc + idx);
+            value = veorq_u64(value, vshrq_n_u64(value, 47));
+            value = veorq_u64(value, vreinterpretq_u64_u8(vld1q_u8(secret + secret_length - 64 + idx * 8)));
+            vst1q_u64(acc + idx, vaddq_u64(vmull_u32(vmovn_u64(value), prime), vshlq_n_u64(vmull_u32(vshrn_n_u64(value, 32), prime), 32)));
+        }
+#elif defined(XXH3_SSE2)
+        const __m128i prime = _mm_set1_epi32((int) (uint32_t) PRIME32_1);
+        for (size_t idx = 0; idx < 8; idx += 2) {
+            __m128i value = _mm_loadu_si128((const __m128i *) (acc + idx));
+            value = _mm_xor_si128(value, _mm_srli_epi64(value, 47));
+            value = _mm_xor_si128(value, _mm_loadu_si128((const __m128i *) (secret + secret_length - 64 + idx * 8)));
+            _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_mul_epu32(value, prime), _mm_slli_epi64(_mm_mul_epu32(_mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)), prime), 32)));
+        }
+#else
+        for (size_t idx = 0; idx < 8; idx++) {
+            acc[idx] ^= (acc[idx] >> 47);
+            acc[idx] ^= read64(secret + secret_length - 64 + idx * 8);
+            acc[idx] *= PRIME32_1;
+        }
+#endif
+    }
+
+    for (size_t stripe_idx = 0; stripe_idx < (length - last_offset - 1) / 64; stripe_idx++) {
+        accumulate(acc, input + last_offset + stripe_idx * 64, stripe_idx * 8);
+    }
+
+    accumulate(acc, input + length - 64, secret_length - 71);
+
+    uint64_t result = (uint64_t) length * PRIME64_1;
+    for (size_t idx = 0; idx < 4; idx++) {
+        result += XXH3_mul128_fold64(acc[idx * 2] ^ read64(secret + 11 + 16 * idx), acc[idx * 2 + 1] ^ read64(secret + 11 + 16 * idx + 8));
+    }
+
+    return avalanche(result);
+}
+
+uint64_t
+pm_xxh3(const uint8_t *input, size_t length) {
+    if (length == 0) {
+        return XXH3_64_empty();
+    } else if (length <= 3) {
+        return XXH3_64_1to3(input, length);
+    } else if (length <= 8) {
+        return XXH3_64_4to8(input, length);
+    } else if (length <= 16) {
+        return XXH3_64_9to16(input, length);
+    } else if (length <= 240) {
+        return XXH3_64_17to240(input, length);
+    } else {
+        return XXH3_64_large(input, length);
+    }
+}

data/vendor/prism/templates/ext/prism/api_node.c.erb

@@ -1,5 +1,9 @@
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
 #include "prism/extension.h"
+#include "prism/internal/allocator.h"
+#include "prism/internal/arena.h"
+
+#include <assert.h>
 
 extern VALUE rb_cPrism;
 extern VALUE rb_cPrismNode;
@@ -12,31 +16,26 @@ static VALUE rb_cPrism<%= node.name %>;
 <%- end -%>
 
 static VALUE
-pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) {
+pm_location_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) {
     if (freeze) {
-        VALUE location_argv[] = {
-            source,
-            LONG2FIX(start - parser->start),
-            LONG2FIX(end - start)
-        };
-
+        VALUE location_argv[] = { source, LONG2FIX(start), LONG2FIX(length) };
         return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation));
     } else {
-        uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start)));
+        uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length));
         return ULL2NUM(value);
     }
 }
 
 VALUE
 pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) {
-    ID type = rb_intern(pm_token_type_name(token->type));
-    VALUE location = pm_location_new(parser, token->start, token->end, source, freeze);
+    ID type = rb_intern(pm_token_type(token->type));
+    VALUE location = pm_location_new((uint32_t) (token->start - pm_parser_start(parser)), (uint32_t) (token->end - token->start), source, freeze);
 
     VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding);
     if (freeze) rb_obj_freeze(slice);
 
-    VALUE argv[] = { source, ID2SYM(type), slice, location };
-    VALUE value = rb_class_new_instance(4, argv, rb_cPrismToken);
+    VALUE argv[] = { source, ID2SYM(type), slice, location, INT2FIX(pm_parser_lex_state(parser)) };
+    VALUE value = rb_class_new_instance(5, argv, rb_cPrismToken);
     if (freeze) rb_obj_freeze(value);
 
     return value;
@@ -78,20 +77,26 @@ pm_integer_new(const pm_integer_t *integer) {
 
 // Create a Prism::Source object from the given parser, after pm_parse() was called.
 VALUE
-pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze) {
-    VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding);
+pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, VALUE comments, VALUE comment_attachments, VALUE comment_ranges, bool freeze) {
+    const uint8_t *start = pm_parser_start(parser);
+    VALUE source_string = rb_enc_str_new((const char *) start, pm_parser_end(parser) - start, encoding);
 
-    VALUE offsets = rb_ary_new_capa(parser->newline_list.size);
-    for (size_t index = 0; index < parser->newline_list.size; index++) {
-        rb_ary_push(offsets, ULONG2NUM(parser->newline_list.offsets[index]));
-    }
+    const pm_line_offset_list_t *line_offsets = pm_parser_line_offsets(parser);
+    VALUE offsets;
 
     if (freeze) {
+        offsets = rb_ary_new_capa(line_offsets->size);
+        for (size_t index = 0; index < line_offsets->size; index++) {
+            rb_ary_push(offsets, ULONG2NUM(line_offsets->offsets[index]));
+        }
+
         rb_obj_freeze(source_string);
         rb_obj_freeze(offsets);
+    } else {
+        offsets = rb_str_new((const char *) line_offsets->offsets, line_offsets->size * sizeof(uint32_t));
     }
 
-    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets);
+    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 7, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, comment_attachments, comment_ranges, ULL2NUM(pm_parser_checksum(parser)));
     if (freeze) rb_obj_freeze(source);
 
     return source;
@@ -104,8 +109,8 @@ typedef struct pm_node_stack_node {
 } pm_node_stack_node_t;
 
 static void
-pm_node_stack_push(pm_node_stack_node_t **stack, const pm_node_t *visit) {
-    pm_node_stack_node_t *node = xmalloc(sizeof(pm_node_stack_node_t));
+pm_node_stack_push(pm_arena_t *arena, pm_node_stack_node_t **stack, const pm_node_t *visit) {
+    pm_node_stack_node_t *node = (pm_node_stack_node_t *) pm_arena_alloc(arena, sizeof(pm_node_stack_node_t), PRISM_ALIGNOF(pm_node_stack_node_t));
     node->prev = *stack;
     node->visit = visit;
     node->visited = false;
@@ -118,32 +123,40 @@ pm_node_stack_pop(pm_node_stack_node_t **stack) {
     const pm_node_t *visit = current->visit;
 
     *stack = current->prev;
-    xfree(current);
 
     return visit;
 }
 
-VALUE
-pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) {
-    VALUE constants = rb_ary_new_capa(parser->constant_pool.size);
+/**
+ * Build the Symbol for the given constant.
+ */
+static VALUE
+pm_ast_constant(const pm_constant_t *constant, rb_encoding *encoding) {
+    int state = 0;
 
-    for (uint32_t index = 0; index < parser->constant_pool.size; index++) {
-        pm_constant_t *constant = &parser->constant_pool.constants[index];
-        int state = 0;
+    VALUE string = rb_enc_str_new((const char *) pm_constant_start(constant), pm_constant_length(constant), encoding);
+    VALUE value = rb_protect(rb_str_intern, string, &state);
 
-        VALUE string = rb_enc_str_new((const char *) constant->start, constant->length, encoding);
-        VALUE value = rb_protect(rb_str_intern, string, &state);
+    if (state != 0) {
+        value = ID2SYM(rb_intern_const("?"));
+        rb_set_errinfo(Qnil);
+    }
 
-        if (state != 0) {
-            value = ID2SYM(rb_intern_const("?"));
-            rb_set_errinfo(Qnil);
-        }
+    return value;
+}
+
+VALUE
+pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) {
+    size_t constants_size = pm_parser_constants_size(parser);
+    VALUE constants = rb_ary_new_capa((long) constants_size);
 
-        rb_ary_push(constants, value);
+    for (size_t index = 0; index < constants_size; index++) {
+        rb_ary_push(constants, pm_ast_constant(pm_parser_constant_at(parser, index), encoding));
     }
 
+    pm_arena_t *node_arena = pm_arena_new();
     pm_node_stack_node_t *node_stack = NULL;
-    pm_node_stack_push(&node_stack, node);
+    pm_node_stack_push(node_arena, &node_stack, node);
     VALUE value_stack = rb_ary_new();
 
     while (node_stack != NULL) {
@@ -166,10 +179,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     <%- node.fields.each do |field| -%>
                     <%- case field -%>
                     <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
-                    pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>);
+                    pm_node_stack_push(node_arena, &node_stack, (pm_node_t *) cast-><%= field.name %>);
                     <%- when Prism::Template::NodeListField -%>
                     for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
-                        pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>.nodes[index]);
+                        pm_node_stack_push(node_arena, &node_stack, (pm_node_t *) cast-><%= field.name %>.nodes[index]);
                     }
                     <%- end -%>
                     <%- end -%>
@@ -200,7 +213,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     argv[1] = ULONG2NUM(node->node_id);
 
                     // location
-                    argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze);
+                    argv[2] = pm_location_new(node->location.start, node->location.length, source, freeze);
 
                     // flags
                     argv[3] = ULONG2NUM(node->flags);
@@ -224,23 +237,23 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     <%- when Prism::Template::ConstantField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
                     assert(cast-><%= field.name %> != 0);
-                    argv[<%= index %>] = RARRAY_AREF(constants, cast-><%= field.name %> - 1);
+                    argv[<%= index %>] = RARRAY_AREF(constants, pm_constant_id_index(cast-><%= field.name %>));
                     <%- when Prism::Template::OptionalConstantField -%>
-                    argv[<%= index %>] = cast-><%= field.name %> == 0 ? Qnil : RARRAY_AREF(constants, cast-><%= field.name %> - 1);
+                    argv[<%= index %>] = cast-><%= field.name %> == 0 ? Qnil : RARRAY_AREF(constants, pm_constant_id_index(cast-><%= field.name %>));
                     <%- when Prism::Template::ConstantListField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
                     argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size);
                     for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
                         assert(cast-><%= field.name %>.ids[index] != 0);
-                        rb_ary_push(argv[<%= index %>], RARRAY_AREF(constants, cast-><%= field.name %>.ids[index] - 1));
+                        rb_ary_push(argv[<%= index %>], RARRAY_AREF(constants, pm_constant_id_index(cast-><%= field.name %>.ids[index])));
                     }
                     if (freeze) rb_obj_freeze(argv[<%= index %>]);
                     <%- when Prism::Template::LocationField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
-                    argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
+                    argv[<%= index %>] = pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze);
                     <%- when Prism::Template::OptionalLocationField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
-                    argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
+                    argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze);
                     <%- when Prism::Template::UInt8Field -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
                     argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>);
@@ -271,6 +284,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
         }
     }
 
+    pm_arena_free(node_arena);
     return rb_ary_pop(value_stack);
 }
 

data/vendor/prism/templates/include/prism/ast.h.erb

@@ -8,21 +8,27 @@
 #ifndef PRISM_AST_H
 #define PRISM_AST_H
 
-#include "prism/defines.h"
-#include "prism/util/pm_constant_pool.h"
-#include "prism/util/pm_integer.h"
-#include "prism/util/pm_string.h"
+#include "prism/compiler/align.h"
+#include "prism/compiler/exported.h"
+
+#include "prism/arena.h"
+#include "prism/constant_pool.h"
+#include "prism/integer.h"
+#include "prism/stringy.h"
 
-#include <assert.h>
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * This enum represents every type of token in the Ruby source.
  */
 typedef enum pm_token_type {
 <%- tokens.each do |token| -%>
-    /** <%= token.comment %> */
+    /** <%= Prism::Template::Doxygen.verbatim(token.comment) %> */
     PM_TOKEN_<%= token.name %><%= " = #{token.value}" if token.value %>,
 
 <%- end -%>
@@ -46,15 +52,28 @@ typedef struct {
 } pm_token_t;
 
 /**
- * This represents a range of bytes in the source string to which a node or
- * token corresponds.
+ * Returns a string representation of the given token type.
+ *
+ * @param token_type The type of the token to get the string representation of.
+ * @returns A string representation of the given token type. This is meant for
+ *     debugging purposes and is not guaranteed to be stable across versions.
+ */
+PRISM_EXPORTED_FUNCTION const char * pm_token_type(pm_token_type_t token_type);
+
+/**
+ * This struct represents a slice in the source code, defined by an offset and
+ * a length. Note that we have confirmation that we can represent all locations
+ * within Ruby source files using 32-bit integers per:
+ *
+ *     https://bugs.ruby-lang.org/issues/20488#note-1
+ *
  */
 typedef struct {
-    /** A pointer to the start location of the range in the source. */
-    const uint8_t *start;
+    /** The offset of the location from the start of the source. */
+    uint32_t start;
 
-    /** A pointer to the end location of the range in the source. */
-    const uint8_t *end;
+    /** The length of the location. */
+    uint32_t length;
 } pm_location_t;
 
 struct pm_node;
@@ -112,7 +131,7 @@ static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2;
 typedef struct pm_node {
     /**
      * This represents the type of the node. It somewhat maps to the nodes that
-     * existed in the original grammar and ripper, but it's not a 1:1 mapping.
+     * existed in the original grammar and ripper, but it is not a 1:1 mapping.
      */
     pm_node_type_t type;
 
@@ -124,12 +143,14 @@ typedef struct pm_node {
 
     /**
      * The unique identifier for this node, which is deterministic based on the
-     * source. It is used to identify unique nodes across parses.
+     * source. It is used to identify unique nodes across parses. Nodes created
+     * outside of parsing have no position in the source and can use 0, which
+     * no parsed node uses.
      */
     uint32_t node_id;
 
     /**
-     * This is the location of the node in the source. It's a range of bytes
+     * This is the location of the node in the source. It is a range of bytes
      * containing a start and an end.
      */
     pm_location_t location;
@@ -160,6 +181,15 @@ typedef struct pm_node {
  * Return true if the given flag is set on the given node.
  */
 #define PM_NODE_FLAG_P(node_, flag_) ((PM_NODE_FLAGS(node_) & (flag_)) != 0)
+
+/**
+ * The alignment required for a child node within a parent node.
+ */
+#ifdef _MSC_VER
+#define PM_NODE_ALIGNAS __declspec(align(8))
+#else
+#define PM_NODE_ALIGNAS PRISM_ALIGNAS(PRISM_ALIGNOF(void *))
+#endif
 <%- nodes.each do |node| -%>
 
 /**
@@ -182,7 +212,6 @@ typedef struct pm_node {
 typedef struct pm_<%= node.human %> {
     /** The embedded base node. */
     pm_node_t base;
-
 <%- node.fields.each do |field| -%>
 
     /**
@@ -195,7 +224,7 @@ typedef struct pm_<%= node.human %> {
     <%- end -%>
      */
     <%= case field
-    when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "struct #{field.c_type} *#{field.name}"
+    when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "PM_NODE_ALIGNAS struct #{field.c_type} *#{field.name}"
     when Prism::Template::NodeListField then "struct pm_node_list #{field.name}"
     when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}"
     when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}"
@@ -226,6 +255,23 @@ typedef enum pm_<%= flag.human %> {
     PM_<%= flag.human.upcase %>_LAST,
 } pm_<%= flag.human %>_t;
 <%- end -%>
+<%- nodes.each do |node| -%>
+
+<%- params = node.fields.map(&:c_param) -%>
+/**
+ * Allocate and initialize a new <%= node.name %> node.
+ *
+ * @param arena The arena to allocate from.
+ * @param node_id The unique identifier for this node.
+ * @param flags The flags for this node.
+ * @param location The location of this node in the source.
+<%- node.fields.each do |field| -%>
+ * @param <%= field.name %> <%= field.comment ? Prism::Template::Doxygen.verbatim(field.comment.lines.first.strip) : "The #{field.name} field." %>
+<%- end -%>
+ * @returns The newly allocated and initialized node.
+ */
+PRISM_EXPORTED_FUNCTION pm_<%= node.human %>_t * pm_<%= node.human %>_new(pm_arena_t *arena, uint32_t node_id, pm_node_flags_t flags, pm_location_t location<%= params.empty? ? "" : ", #{params.join(", ")}" %>);
+<%- end -%>
 
 /**
  * When we're serializing to Java, we want to skip serializing the location
@@ -235,4 +281,8 @@ typedef enum pm_<%= flag.human %> {
  */
 #define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS <%= Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0 %>
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif