prism 1.9.0 → 2.0.0.rc2

data/include/prism/internal/regexp.h

@@ -0,0 +1,26 @@
+#ifndef PRISM_INTERNAL_REGEXP_H
+#define PRISM_INTERNAL_REGEXP_H
+
+#include "prism/ast.h"
+#include "prism/parser.h"
+
+/*
+ * Callback invoked by pm_regexp_parse() for each named capture group found.
+ * The data pointer is whatever the caller passed in alongside the callback; the
+ * regexp parser passes it through without inspecting it.
+ */
+typedef void (*pm_regexp_name_callback_t)(pm_parser_t *parser, const pm_string_t *name, void *data);
+
+/*
+ * Parse a regular expression, validate its encoding, and optionally extract
+ * named capture groups. Returns the encoding flags to set on the node.
+ */
+PRISM_EXPORTED_FUNCTION pm_node_flags_t pm_regexp_parse(pm_parser_t *parser, pm_regular_expression_node_t *node, pm_regexp_name_callback_t name_callback, void *name_data);
+
+/*
+ * Parse an interpolated regular expression for named capture groups only.
+ * No encoding validation is performed.
+ */
+void pm_regexp_parse_named_captures(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data);
+
+#endif

data/include/prism/internal/serialize.h

@@ -0,0 +1,39 @@
+#ifndef PRISM_INTERNAL_SERIALIZE_H
+#define PRISM_INTERNAL_SERIALIZE_H
+
+#include "prism/internal/encoding.h"
+#include "prism/internal/list.h"
+
+#include "prism/ast.h"
+#include "prism/buffer.h"
+#include "prism/excludes.h"
+#include "prism/parser.h"
+
+/* We optionally support serializing to a binary string. For systems that do not
+ * want or need this functionality, it can be turned off with the
+ * PRISM_EXCLUDE_SERIALIZATION define. */
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
+/*
+ * Serialize the given list of offsets to the given buffer.
+ */
+void pm_serialize_line_offset_list(pm_line_offset_list_t *list, pm_buffer_t *buffer);
+
+/*
+ * Serialize the given list of comments to the given buffer.
+ */
+void pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer);
+
+/*
+ * Serialize the name of the encoding to the buffer.
+ */
+void pm_serialize_encoding(const pm_encoding_t *encoding, pm_buffer_t *buffer);
+
+/*
+ * Serialize the encoding, metadata, nodes, and constant pool.
+ */
+void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+#endif
+
+#endif

data/include/prism/internal/source.h

@@ -0,0 +1,72 @@
+#ifndef PRISM_INTERNAL_SOURCE_H
+#define PRISM_INTERNAL_SOURCE_H
+
+#include "prism/source.h"
+#include "prism/buffer.h"
+
+#include <stdbool.h>
+
+/*
+ * The type of source, which determines cleanup behavior.
+ */
+typedef enum {
+    /* Wraps existing constant memory, no cleanup. */
+    PM_SOURCE_CONSTANT,
+
+    /* Wraps existing shared memory (non-owning slice), no cleanup. */
+    PM_SOURCE_SHARED,
+
+    /* Owns a heap-allocated buffer, freed on cleanup. */
+    PM_SOURCE_OWNED,
+
+    /* Memory-mapped file, unmapped on cleanup. */
+    PM_SOURCE_MAPPED,
+
+    /* Stream source backed by a pm_buffer_t. */
+    PM_SOURCE_STREAM
+} pm_source_type_t;
+
+/*
+ * The internal representation of a source.
+ */
+struct pm_source_t {
+    /* A pointer to the start of the source data. */
+    const uint8_t *source;
+
+    /* The length of the source data in bytes. */
+    size_t length;
+
+    /* The type of the source. */
+    pm_source_type_t type;
+
+    /* Stream-specific data, only used for PM_SOURCE_STREAM sources. */
+    struct {
+        /* The buffer that holds the accumulated stream data. */
+        pm_buffer_t *buffer;
+
+        /* The stream object to read from. */
+        void *stream;
+
+        /* The function to use to read from the stream. */
+        pm_source_stream_fgets_t *fgets;
+
+        /* The function to use to check if the stream is at EOF. */
+        pm_source_stream_feof_t *feof;
+
+        /* Whether the stream has reached EOF. */
+        bool eof;
+    } stream;
+};
+
+/*
+ * Read from a stream into the source's internal buffer. This is used by
+ * pm_parse_stream to incrementally read the source.
+ */
+bool pm_source_stream_read(pm_source_t *source);
+
+/*
+ * Returns whether the stream source has reached EOF.
+ */
+bool pm_source_stream_eof(const pm_source_t *source);
+
+#endif

data/include/prism/{static_literals.h → internal/static_literals.h}

@@ -1,33 +1,27 @@
-/**
- * @file static_literals.h
- *
- * A set of static literal nodes that can be checked for duplicates.
- */
-#ifndef PRISM_STATIC_LITERALS_H
-#define PRISM_STATIC_LITERALS_H
+#ifndef PRISM_INTERNAL_STATIC_LITERALS_H
+#define PRISM_INTERNAL_STATIC_LITERALS_H
 
-#include "prism/defines.h"
-#include "prism/ast.h"
-#include "prism/util/pm_newline_list.h"
+#include "prism/internal/encoding.h"
 
-#include <assert.h>
-#include <stdbool.h>
+#include "prism/ast.h"
+#include "prism/buffer.h"
+#include "prism/line_offset_list.h"
 
-/**
+/*
  * An internal hash table for a set of nodes.
  */
 typedef struct {
-    /** The array of nodes in the hash table. */
+    /* The array of nodes in the hash table. */
     pm_node_t **nodes;
 
-    /** The size of the hash table. */
+    /* The size of the hash table. */
     uint32_t size;
 
-    /** The space that has been allocated in the hash table. */
+    /* The space that has been allocated in the hash table. */
     uint32_t capacity;
 } pm_node_hash_t;
 
-/**
+/*
  * Certain sets of nodes (hash keys and when clauses) check for duplicate nodes
  * to alert the user of potential issues. To do this, we keep a set of the nodes
  * that have been seen so far, and compare whenever we find a new node.
@@ -36,86 +30,71 @@ typedef struct {
  * that need to be performed.
  */
 typedef struct {
-    /**
+    /*
      * This is the set of IntegerNode and SourceLineNode instances.
      */
     pm_node_hash_t integer_nodes;
 
-    /**
+    /*
      * This is the set of FloatNode instances.
      */
     pm_node_hash_t float_nodes;
 
-    /**
+    /*
      * This is the set of RationalNode and ImaginaryNode instances.
      */
     pm_node_hash_t number_nodes;
 
-    /**
+    /*
      * This is the set of StringNode and SourceFileNode instances.
      */
     pm_node_hash_t string_nodes;
 
-    /**
+    /*
      * This is the set of RegularExpressionNode instances.
      */
     pm_node_hash_t regexp_nodes;
 
-    /**
+    /*
      * This is the set of SymbolNode instances.
      */
     pm_node_hash_t symbol_nodes;
 
-    /**
+    /*
      * A pointer to the last TrueNode instance that was inserted, or NULL.
      */
     pm_node_t *true_node;
 
-    /**
+    /*
      * A pointer to the last FalseNode instance that was inserted, or NULL.
      */
     pm_node_t *false_node;
 
-    /**
+    /*
      * A pointer to the last NilNode instance that was inserted, or NULL.
      */
     pm_node_t *nil_node;
 
-    /**
+    /*
      * A pointer to the last SourceEncodingNode instance that was inserted, or
      * NULL.
      */
     pm_node_t *source_encoding_node;
 } pm_static_literals_t;
 
-/**
+/*
  * Add a node to the set of static literals.
- *
- * @param newline_list The list of newline offsets to use to calculate lines.
- * @param start_line The line number that the parser starts on.
- * @param literals The set of static literals to add the node to.
- * @param node The node to add to the set.
- * @param replace Whether to replace the previous node if one already exists.
- * @return A pointer to the node that is being overwritten, if there is one.
  */
-pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace);
+pm_node_t * pm_static_literals_add(const pm_line_offset_list_t *line_offsets, const uint8_t *start, int32_t start_line, const pm_encoding_t *encoding, pm_static_literals_t *literals, pm_node_t *node, bool replace);
 
-/**
+/*
  * Free the internal memory associated with the given static literals set.
- *
- * @param literals The set of static literals to free.
  */
 void pm_static_literals_free(pm_static_literals_t *literals);
 
-/**
+/*
  * Create a string-based representation of the given static literal.
- *
- * @param buffer The buffer to write the string to.
- * @param newline_list The list of newline offsets to use to calculate lines.
- * @param start_line The line number that the parser starts on.
- * @param encoding_name The name of the encoding of the source being parsed.
- * @param node The node to create a string representation of.
  */
-void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node);
+void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_line_offset_list_t *line_offsets, const uint8_t *start, int32_t start_line, const pm_encoding_t *encoding, const pm_node_t *node);
 
 #endif

data/include/prism/internal/stringy.h

@@ -0,0 +1,35 @@
+#ifndef PRISM_INTERNAL_STRINGY_H
+#define PRISM_INTERNAL_STRINGY_H
+
+#include "prism/stringy.h"
+
+/*
+ * Backing storage for an empty string.
+ */
+static const uint8_t empty_source[] = "";
+
+/*
+ * Defines an empty string. This is useful for initializing a string that will
+ * be filled in later.
+ */
+#define PM_STRING_EMPTY ((pm_string_t) { .type = PM_STRING_CONSTANT, .source = empty_source, .length = 0 })
+
+/*
+ * Initialize a shared string that is based on initial input.
+ */
+void pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end);
+
+/*
+ * Compare the underlying lengths and bytes of two strings. Returns 0 if the
+ * strings are equal, a negative number if the left string is less than the
+ * right string, and a positive number if the left string is greater than the
+ * right string.
+ */
+int pm_string_compare(const pm_string_t *left, const pm_string_t *right);
+
+/*
+ * Free the associated memory of the given string.
+ */
+void pm_string_cleanup(pm_string_t *string);
+
+#endif

data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h}

@@ -1,18 +1,10 @@
-/**
- * @file pm_strncasecmp.h
- *
- * A custom strncasecmp implementation.
- */
-#ifndef PRISM_STRNCASECMP_H
-#define PRISM_STRNCASECMP_H
+#ifndef PRISM_INTERNAL_STRNCASECMP_H
+#define PRISM_INTERNAL_STRNCASECMP_H
 
-#include "prism/defines.h"
-
-#include <ctype.h>
 #include <stddef.h>
 #include <stdint.h>
 
-/**
+/*
  * Compare two strings, ignoring case, up to the given length. Returns 0 if the
  * strings are equal, a negative number if string1 is less than string2, or a
  * positive number if string1 is greater than string2.
@@ -20,12 +12,6 @@
  * Note that this is effectively our own implementation of strncasecmp, but it's
  * not available on all of the platforms we want to support so we're rolling it
  * here.
- *
- * @param string1 The first string to compare.
- * @param string2 The second string to compare
- * @param length The maximum number of characters to compare.
- * @return 0 if the strings are equal, a negative number if string1 is less than
- *     string2, or a positive number if string1 is greater than string2.
  */
 int pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length);
 

data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h}

@@ -1,19 +1,15 @@
-/**
- * @file pm_strpbrk.h
- *
- * A custom strpbrk implementation.
- */
-#ifndef PRISM_STRPBRK_H
-#define PRISM_STRPBRK_H
+#ifndef PRISM_INTERNAL_STRPBRK_H
+#define PRISM_INTERNAL_STRPBRK_H
 
-#include "prism/defines.h"
-#include "prism/diagnostic.h"
 #include "prism/parser.h"
 
+/* The maximum number of bytes in a strpbrk charset. */
+#define PM_STRPBRK_CACHE_SIZE 16
+
 #include <stddef.h>
-#include <string.h>
+#include <stdint.h>
 
-/**
+/*
  * Here we have rolled our own version of strpbrk. The standard library strpbrk
  * has undefined behavior when the source string is not null-terminated. We want
  * to support strings that are not null-terminated because pm_parse does not
@@ -31,15 +27,6 @@
  * characters that are trailing bytes of multi-byte characters. For example, in
  * Shift-JIS, the backslash character can be a trailing byte. In that case we
  * need to take a slower path and iterate one multi-byte character at a time.
- *
- * @param parser The parser.
- * @param source The source to search.
- * @param charset The charset to search for.
- * @param length The maximum number of bytes to search.
- * @param validate Whether to validate that the source string is valid in the
- *     current encoding of the parser.
- * @return A pointer to the first character in the source string that is in the
- *     charset, or NULL if no such character exists.
  */
 const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
 

data/include/prism/internal/tokens.h

@@ -0,0 +1,11 @@
+#ifndef PRISM_INTERNAL_TOKENS_H
+#define PRISM_INTERNAL_TOKENS_H
+
+#include "prism/ast.h"
+
+/*
+ * Returns the human name of the given token type.
+ */
+const char * pm_token_str(pm_token_type_t token_type);
+
+#endif

data/include/prism/internal/xxh3.h

@@ -0,0 +1,16 @@
+#ifndef PRISM_INTERNAL_XXH3_H
+#define PRISM_INTERNAL_XXH3_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * Computes the XXH3 digest of the given input.
+ *
+ * @param input The input data to hash.
+ * @param length The length of the input data.
+ * @return The 64-bit XXH3 hash of the input data.
+ */
+uint64_t pm_xxh3(const uint8_t *input, size_t length);
+
+#endif

data/include/prism/json.h

@@ -0,0 +1,32 @@
+/**
+ * @file json.h
+ */
+#ifndef PRISM_JSON_H
+#define PRISM_JSON_H
+
+#include "prism/excludes.h"
+
+/* We optionally support dumping to JSON. For systems that don't want or need
+ * this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define.
+ */
+#ifndef PRISM_EXCLUDE_JSON
+
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include "prism/ast.h"
+#include "prism/buffer.h"
+#include "prism/parser.h"
+
+/**
+ * Dump JSON to the given buffer.
+ *
+ * @param buffer The buffer to serialize to.
+ * @param parser The parser that parsed the node.
+ * @param node The node to serialize.
+ */
+PRISM_EXPORTED_FUNCTION void pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node) PRISM_NONNULL(1, 2, 3);
+
+#endif
+
+#endif

data/include/prism/line_offset_list.h

@@ -0,0 +1,61 @@
+/**
+ * @file line_offset_list.h
+ *
+ * A list of byte offsets of newlines in a string.
+ *
+ * When compiling the syntax tree, it's necessary to know the line and column
+ * of many nodes. This is necessary to support things like error messages,
+ * tracepoints, etc.
+ *
+ * It's possible that we could store the start line, start column, end line, and
+ * end column on every node in addition to the offsets that we already store,
+ * but that would be quite a lot of memory overhead.
+ */
+#ifndef PRISM_LINE_OFFSET_LIST_H
+#define PRISM_LINE_OFFSET_LIST_H
+
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * A list of offsets of the start of lines in a string. The offsets are assumed
+ * to be sorted/inserted in ascending order.
+ */
+typedef struct {
+    /** The number of offsets in the list. */
+    size_t size;
+
+    /** The capacity of the list that has been allocated. */
+    size_t capacity;
+
+    /** The list of offsets. */
+    uint32_t *offsets;
+} pm_line_offset_list_t;
+
+/**
+ * A line and column in a string.
+ */
+typedef struct {
+    /** The line number. */
+    int32_t line;
+
+    /** The column in bytes. */
+    uint32_t column;
+} pm_line_column_t;
+
+/**
+ * Returns the line and column of the given offset. If the offset is not in the
+ * list, the line and column of the closest offset less than the given offset
+ * are returned.
+ *
+ * @param list The list to search.
+ * @param cursor The offset to search for.
+ * @param start_line The line to start counting from.
+ * @returns The line and column of the given offset.
+ */
+PRISM_EXPORTED_FUNCTION pm_line_column_t pm_line_offset_list_line_column(const pm_line_offset_list_t *list, uint32_t cursor, int32_t start_line) PRISM_NONNULL(1);
+
+#endif

data/include/prism/magic_comments.h

@@ -0,0 +1,35 @@
+/**
+ * @file magic_comments.h
+ *
+ * Types and functions related to magic comments found during parsing.
+ */
+#ifndef PRISM_MAGIC_COMMENTS_H
+#define PRISM_MAGIC_COMMENTS_H
+
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include "prism/ast.h"
+
+#include <stddef.h>
+
+/** An opaque pointer to a magic comment found while parsing. */
+typedef struct pm_magic_comment_t pm_magic_comment_t;
+
+/**
+ * Returns the location of the key associated with the given magic comment.
+ *
+ * @param magic_comment the magic comment whose key location we want to get
+ * @returns the location of the key associated with the given magic comment
+ */
+PRISM_EXPORTED_FUNCTION pm_location_t pm_magic_comment_key(const pm_magic_comment_t *magic_comment) PRISM_NONNULL(1);
+
+/**
+ * Returns the location of the value associated with the given magic comment.
+ *
+ * @param magic_comment the magic comment whose value location we want to get
+ * @returns the location of the value associated with the given magic comment
+ */
+PRISM_EXPORTED_FUNCTION pm_location_t pm_magic_comment_value(const pm_magic_comment_t *magic_comment) PRISM_NONNULL(1);
+
+#endif

data/include/prism/node.h

@@ -6,9 +6,10 @@
 #ifndef PRISM_NODE_H
 #define PRISM_NODE_H
 
-#include "prism/defines.h"
-#include "prism/parser.h"
-#include "prism/util/pm_buffer.h"
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include "prism/ast.h"
 
 /**
  * Loop through each node in the node list, writing each node to the given
@@ -17,52 +18,13 @@
 #define PM_NODE_LIST_FOREACH(list, index, node) \
     for (size_t index = 0; index < (list)->size && ((node) = (list)->nodes[index]); index++)
 
-/**
- * Append a new node onto the end of the node list.
- *
- * @param list The list to append to.
- * @param node The node to append.
- */
-void pm_node_list_append(pm_node_list_t *list, pm_node_t *node);
-
-/**
- * Prepend a new node onto the beginning of the node list.
- *
- * @param list The list to prepend to.
- * @param node The node to prepend.
- */
-void pm_node_list_prepend(pm_node_list_t *list, pm_node_t *node);
-
-/**
- * Concatenate the given node list onto the end of the other node list.
- *
- * @param list The list to concatenate onto.
- * @param other The list to concatenate.
- */
-void pm_node_list_concat(pm_node_list_t *list, pm_node_list_t *other);
-
-/**
- * Free the internal memory associated with the given node list.
- *
- * @param list The list to free.
- */
-void pm_node_list_free(pm_node_list_t *list);
-
-/**
- * Deallocate a node and all of its children.
- *
- * @param parser The parser that owns the node.
- * @param node The node to deallocate.
- */
-PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
-
 /**
  * Returns a string representation of the given node type.
  *
  * @param node_type The node type to convert to a string.
- * @return A string representation of the given node type.
+ * @returns A string representation of the given node type.
  */
-PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_type);
+PRISM_EXPORTED_FUNCTION const char * pm_node_type(pm_node_type_t node_type);
 
 /**
  * Visit each of the nodes in this subtree using the given visitor callback. The
@@ -80,7 +42,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
  * bool visit(const pm_node_t *node, void *data) {
  *     size_t *indent = (size_t *) data;
  *     for (size_t i = 0; i < *indent * 2; i++) putc(' ', stdout);
- *     printf("%s\n", pm_node_type_to_str(node->type));
+ *     printf("%s\n", pm_node_type(node->type));
  *
  *     size_t next_indent = *indent + 1;
  *     size_t *next_data = &next_indent;
@@ -93,18 +55,21 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
  *     const char *source = "1 + 2; 3 + 4";
  *     size_t size = strlen(source);
  *
- *     pm_parser_t parser;
- *     pm_options_t options = { 0 };
- *     pm_parser_init(&parser, (const uint8_t *) source, size, &options);
+ *     pm_arena_t *arena = pm_arena_new();
+ *     pm_options_t *options = pm_options_new();
+ *
+ *     pm_parser_t *parser = pm_parser_new(arena, (const uint8_t *) source, size, options);
  *
  *     size_t indent = 0;
- *     pm_node_t *node = pm_parse(&parser);
+ *     pm_node_t *node = pm_parse(parser);
  *
  *     size_t *data = &indent;
  *     pm_visit_node(node, visit, data);
  *
- *     pm_node_destroy(&parser, node);
- *     pm_parser_free(&parser);
+ *     pm_parser_free(parser);
+ *     pm_options_free(options);
+ *     pm_arena_free(arena);
+ *
  *     return EXIT_SUCCESS;
  * }
  * ```
@@ -113,7 +78,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
  * @param visitor The callback to call for each node in the subtree.
  * @param data An opaque pointer that is passed to the visitor callback.
  */
-PRISM_EXPORTED_FUNCTION void pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+PRISM_EXPORTED_FUNCTION void pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data) PRISM_NONNULL(1);
 
 /**
  * Visit the children of the given node with the given callback. This is the
@@ -124,6 +89,6 @@ PRISM_EXPORTED_FUNCTION void pm_visit_node(const pm_node_t *node, bool (*visitor
  * @param visitor The callback to call for each child node.
  * @param data An opaque pointer that is passed to the visitor callback.
  */
-PRISM_EXPORTED_FUNCTION void pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+PRISM_EXPORTED_FUNCTION void pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data) PRISM_NONNULL(1);
 
 #endif