prism 2.0.0-arm64-darwin

data/include/prism/internal/buffer.h

@@ -0,0 +1,94 @@
+#ifndef PRISM_INTERNAL_BUFFER_H
+#define PRISM_INTERNAL_BUFFER_H
+
+#include "prism/compiler/format.h"
+
+#include "prism/buffer.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/*
+ * A simple memory buffer that stores data in a contiguous block of memory.
+ */
+struct pm_buffer_t {
+    /* The length of the buffer in bytes. */
+    size_t length;
+
+    /* The capacity of the buffer in bytes that has been allocated. */
+    size_t capacity;
+
+    /* A pointer to the start of the buffer. */
+    char *value;
+};
+
+/* Initialize a pm_buffer_t with the given capacity. */
+void pm_buffer_init(pm_buffer_t *buffer, size_t capacity);
+
+/* Free the memory held by the buffer. */
+void pm_buffer_cleanup(pm_buffer_t *buffer);
+
+/* Append the given amount of space as zeroes to the buffer. */
+void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
+
+/* Append a formatted string to the buffer. */
+void pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) PRISM_ATTRIBUTE_FORMAT(2, 3);
+
+/* Append a string to the buffer. */
+void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
+
+/* Append a list of bytes to the buffer. */
+void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
+
+/* Append a single byte to the buffer. */
+void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
+
+/* Append a 32-bit unsigned integer to the buffer as a variable-length integer. */
+void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
+
+/* Append a 32-bit signed integer to the buffer as a variable-length integer. */
+void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
+
+/* Append a double to the buffer. */
+void pm_buffer_append_double(pm_buffer_t *buffer, double value);
+
+/* Append a 64-bit unsigned integer to the buffer in native endian order. */
+void pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value);
+
+/* Append a unicode codepoint to the buffer. */
+bool pm_buffer_append_unicode_codepoint(pm_buffer_t *buffer, uint32_t value);
+
+/*
+ * The different types of escaping that can be performed by the buffer when
+ * appending a slice of Ruby source code.
+ */
+typedef enum {
+    PM_BUFFER_ESCAPING_RUBY,
+    PM_BUFFER_ESCAPING_JSON
+} pm_buffer_escaping_t;
+
+/* Append a slice of source code to the buffer. */
+void pm_buffer_append_source(pm_buffer_t *buffer, const uint8_t *source, size_t length, pm_buffer_escaping_t escaping);
+
+/* Prepend the given string to the buffer. */
+void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
+
+/* Concatenate one buffer onto another. */
+void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
+
+/*
+ * Clear the buffer by reducing its size to 0. This does not free the allocated
+ * memory, but it does allow the buffer to be reused.
+ */
+void pm_buffer_clear(pm_buffer_t *buffer);
+
+/* Strip the whitespace from the end of the buffer. */
+void pm_buffer_rstrip(pm_buffer_t *buffer);
+
+/* Checks if the buffer includes the given value. */
+size_t pm_buffer_index(const pm_buffer_t *buffer, char value);
+
+/* Insert the given string into the buffer at the given index. */
+void pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size_t length);
+
+#endif

data/include/prism/internal/char.h

@@ -0,0 +1,145 @@
+#ifndef PRISM_INTERNAL_CHAR_H
+#define PRISM_INTERNAL_CHAR_H
+
+#include "prism/compiler/force_inline.h"
+
+#include "prism/arena.h"
+#include "prism/line_offset_list.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* Bit flag for whitespace characters in pm_byte_table. */
+#define PRISM_CHAR_BIT_WHITESPACE (1 << 0)
+
+/* Bit flag for inline whitespace characters in pm_byte_table. */
+#define PRISM_CHAR_BIT_INLINE_WHITESPACE (1 << 1)
+
+/*
+ * A lookup table for classifying bytes. Each entry is a bitfield of
+ * PRISM_CHAR_BIT_* flags. Defined in char.c.
+ */
+extern const uint8_t pm_byte_table[256];
+
+/* Returns true if the given character is a whitespace character. */
+static PRISM_FORCE_INLINE bool
+pm_char_is_whitespace(const uint8_t b) {
+    return (pm_byte_table[b] & PRISM_CHAR_BIT_WHITESPACE) != 0;
+}
+
+/* Returns true if the given character is an inline whitespace character. */
+static PRISM_FORCE_INLINE bool
+pm_char_is_inline_whitespace(const uint8_t b) {
+    return (pm_byte_table[b] & PRISM_CHAR_BIT_INLINE_WHITESPACE) != 0;
+}
+
+/*
+ * Returns the number of characters at the start of the string that are inline
+ * whitespace (space/tab). Scans the byte table directly for use in hot paths.
+ */
+static PRISM_FORCE_INLINE size_t
+pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) {
+    if (length <= 0) return 0;
+    size_t size = 0;
+    size_t maximum = (size_t) length;
+    while (size < maximum && (pm_byte_table[string[size]] & PRISM_CHAR_BIT_INLINE_WHITESPACE)) size++;
+    return size;
+}
+
+/*
+ * Returns the number of characters at the start of the string that are
+ * whitespace. Disallows searching past the given maximum number of characters.
+ */
+size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
+
+/*
+ * Returns the number of characters at the end of the string that are
+ * whitespace. Disallows searching past the given maximum number of characters.
+ */
+size_t pm_strrspn_whitespace(const uint8_t *string, ptrdiff_t length);
+
+/*
+ * Returns the number of characters at the start of the string that are
+ * whitespace while also tracking the location of each newline. Disallows
+ * searching past the given maximum number of characters.
+ */
+size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_arena_t *arena, pm_line_offset_list_t *line_offsets, uint32_t start_offset);
+
+/*
+ * Returns the number of characters at the start of the string that are decimal
+ * digits. Disallows searching past the given maximum number of characters.
+ */
+size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
+
+/*
+ * Returns the number of characters at the start of the string that are
+ * hexadecimal digits. Disallows searching past the given maximum number of
+ * characters.
+ */
+size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
+
+/*
+ * Returns the number of characters at the start of the string that are octal
+ * digits or underscores. Disallows searching past the given maximum number of
+ * characters.
+ *
+ * If multiple underscores are found in a row or if an underscore is
+ * found at the end of the number, then the invalid pointer is set to the index
+ * of the first invalid underscore.
+ */
+size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+/*
+ * Returns the number of characters at the start of the string that are decimal
+ * digits or underscores. Disallows searching past the given maximum number of
+ * characters.
+ *
+ * If multiple underscores are found in a row or if an underscore is
+ * found at the end of the number, then the invalid pointer is set to the index
+ * of the first invalid underscore.
+ */
+size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+/*
+ * Returns the number of characters at the start of the string that are
+ * hexadecimal digits or underscores. Disallows searching past the given maximum
+ * number of characters.
+ *
+ * If multiple underscores are found in a row or if an underscore is
+ * found at the end of the number, then the invalid pointer is set to the index
+ * of the first invalid underscore.
+ */
+size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+/*
+ * Returns the number of characters at the start of the string that are regexp
+ * options. Disallows searching past the given maximum number of characters.
+ */
+size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
+
+/*
+ * Returns the number of characters at the start of the string that are binary
+ * digits or underscores. Disallows searching past the given maximum number of
+ * characters.
+ *
+ * If multiple underscores are found in a row or if an underscore is
+ * found at the end of the number, then the invalid pointer is set to the index
+ * of the first invalid underscore.
+ */
+size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+
+/* Returns true if the given character is a binary digit. */
+bool pm_char_is_binary_digit(const uint8_t b);
+
+/* Returns true if the given character is an octal digit. */
+bool pm_char_is_octal_digit(const uint8_t b);
+
+/* Returns true if the given character is a decimal digit. */
+bool pm_char_is_decimal_digit(const uint8_t b);
+
+/* Returns true if the given character is a hexadecimal digit. */
+bool pm_char_is_hexadecimal_digit(const uint8_t b);
+
+#endif

data/include/prism/internal/comments.h

@@ -0,0 +1,53 @@
+#ifndef PRISM_INTERNAL_COMMENTS_H
+#define PRISM_INTERNAL_COMMENTS_H
+
+#include "prism/comments.h"
+#include "prism/parser.h"
+
+#include "prism/internal/list.h"
+
+#include <stdint.h>
+
+/* A comment found while parsing. */
+struct pm_comment_t {
+    /* The embedded base node. */
+    pm_list_node_t node;
+
+    /* The location of the comment in the source. */
+    pm_location_t location;
+
+    /* The type of the comment. */
+    pm_comment_type_t type;
+};
+
+/*
+ * The result of attaching comments to the nodes that own them. All of the
+ * arrays are allocated in the parser's metadata arena.
+ */
+typedef struct {
+    /* The comments found while parsing, sorted by start offset. */
+    const pm_comment_t **comments;
+
+    /*
+     * Indices into comments, grouped by owning node in order of node id and
+     * sorted by start offset within each group. A node's comments are not
+     * necessarily contiguous in comments, because a child can own a comment
+     * that falls between two comments owned by its parent.
+     */
+    uint32_t *attachments;
+
+    /* The slice of attachments for each node, indexed by node id. */
+    pm_comment_range_t *ranges;
+
+    /* The number of entries in ranges. */
+    uint32_t ranges_size;
+} pm_comment_attachments_t;
+
+/*
+ * Sort the parser's comment list in place by start offset, then assign every
+ * comment to the node that owns it. This is called after parsing when the
+ * attach_comments option is set.
+ */
+void pm_comments_attach(pm_parser_t *parser, const pm_node_t *root);
+
+#endif

data/include/prism/internal/constant_pool.h

@@ -0,0 +1,152 @@
+#ifndef PRISM_INTERNAL_CONSTANT_POOL_H
+#define PRISM_INTERNAL_CONSTANT_POOL_H
+
+#include "prism/constant_pool.h"
+
+#include "prism/arena.h"
+#include "prism/compiler/inline.h"
+
+#include <stdbool.h>
+
+/* A constant in the pool which effectively stores a string. */
+struct pm_constant_t {
+    /* A pointer to the start of the string. */
+    const uint8_t *start;
+
+    /* The length of the string. */
+    size_t length;
+};
+
+/*
+ * The type of bucket in the constant pool hash map. This determines how the
+ * bucket should be freed.
+ */
+typedef unsigned int pm_constant_pool_bucket_type_t;
+
+/* By default, each constant is a slice of the source. */
+static const pm_constant_pool_bucket_type_t PM_CONSTANT_POOL_BUCKET_DEFAULT = 0;
+
+/* An owned constant is one for which memory has been allocated. */
+static const pm_constant_pool_bucket_type_t PM_CONSTANT_POOL_BUCKET_OWNED = 1;
+
+/* A constant constant is known at compile time. */
+static const pm_constant_pool_bucket_type_t PM_CONSTANT_POOL_BUCKET_CONSTANT = 2;
+
+/* A bucket in the hash map. */
+typedef struct {
+    /* The incremental ID used for indexing back into the pool. */
+    unsigned int id: 30;
+
+    /* The type of the bucket, which determines how to free it. */
+    pm_constant_pool_bucket_type_t type: 2;
+
+    /* The hash of the bucket. */
+    uint32_t hash;
+
+    /*
+     * A pointer to the start of the string, stored directly in the bucket to
+     * avoid a pointer chase to the constants array during probing.
+     */
+    const uint8_t *start;
+
+    /* The length of the string. */
+    size_t length;
+} pm_constant_pool_bucket_t;
+
+/* The overall constant pool, which stores constants found while parsing. */
+struct pm_constant_pool_t {
+    /* The buckets in the hash map. */
+    pm_constant_pool_bucket_t *buckets;
+
+    /* The constants that are stored in the buckets. */
+    pm_constant_t *constants;
+
+    /* The number of buckets in the hash map. */
+    uint32_t size;
+
+    /* The number of buckets that have been allocated in the hash map. */
+    uint32_t capacity;
+};
+
+/*
+ * When we allocate constants into the pool, we reserve 0 to mean that the slot
+ * is not yet filled. This constant is reused in other places to indicate the
+ * lack of a constant id.
+ */
+#define PM_CONSTANT_ID_UNSET 0
+
+/* Initialize a list of constant ids with a given capacity. */
+void pm_constant_id_list_init_capacity(pm_arena_t *arena, pm_constant_id_list_t *list, size_t capacity);
+
+/* Insert a constant id into a list of constant ids at the specified index. */
+void pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_constant_id_t id);
+
+/*
+ * Mix the bits of a constant id so that ids which are close together, as
+ * sequentially assigned ids are, land in unrelated slots of a hash table.
+ */
+static PRISM_INLINE uint32_t
+pm_constant_id_hash(pm_constant_id_t id) {
+    id = ((id >> 16) ^ id) * 0x45d9f3b;
+    id = ((id >> 16) ^ id) * 0x45d9f3b;
+    id = (id >> 16) ^ id;
+    return id;
+}
+
+/*
+ * A set of constant ids, for where the parser only needs to know whether it has
+ * already seen a name, as with the variables a pattern captures or the named
+ * capture groups of a regular expression.
+ */
+typedef struct {
+    /* The number of ids in the set. */
+    size_t size;
+
+    /* The number of slots in `ids`, either zero or a power of two. */
+    size_t capacity;
+
+    /*
+     * An open-addressed table of the ids in the set, where PM_CONSTANT_ID_UNSET
+     * marks an empty slot. It is NULL until the first id is added.
+     */
+    pm_constant_id_t *ids;
+} pm_constant_id_set_t;
+
+/*
+ * Add a constant id, which must not be PM_CONSTANT_ID_UNSET, to the set.
+ * Returns whether the set did not already contain it.
+ */
+bool pm_constant_id_set_insert(pm_arena_t *arena, pm_constant_id_set_t *set, pm_constant_id_t id);
+
+/* Initialize a new constant pool with a given capacity. */
+void pm_constant_pool_init(pm_arena_t *arena, pm_constant_pool_t *pool, uint32_t capacity);
+
+/* Return a pointer to the constant indicated by the given constant id. */
+pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
+
+/*
+ * Find a constant in a constant pool. Returns the id of the constant, or 0 if
+ * the constant is not found.
+ */
+pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+/*
+ * Insert a constant into a constant pool that is a slice of a source string.
+ * Returns the id of the constant, or 0 if any potential calls to resize fail.
+ */
+pm_constant_id_t pm_constant_pool_insert_shared(pm_arena_t *arena, pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+/*
+ * Insert a constant into a constant pool from memory that is now owned by the
+ * constant pool. Returns the id of the constant, or 0 if any potential calls to
+ * resize fail.
+ */
+pm_constant_id_t pm_constant_pool_insert_owned(pm_arena_t *arena, pm_constant_pool_t *pool, uint8_t *start, size_t length);
+
+/*
+ * Insert a constant into a constant pool from memory that is constant. Returns
+ * the id of the constant, or 0 if any potential calls to resize fail.
+ */
+pm_constant_id_t pm_constant_pool_insert_constant(pm_arena_t *arena, pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+#endif