prism 1.9.0 → 2.0.0.rc2

data/include/prism/internal/line_offset_list.h

@@ -0,0 +1,34 @@
+#ifndef PRISM_INTERNAL_LINE_OFFSET_LIST_H
+#define PRISM_INTERNAL_LINE_OFFSET_LIST_H
+
+#include "prism/compiler/force_inline.h"
+
+#include "prism/arena.h"
+#include "prism/line_offset_list.h"
+
+/* Initialize a new line offset list with the given capacity. */
+void pm_line_offset_list_init(pm_arena_t *arena, pm_line_offset_list_t *list, size_t capacity);
+
+/* Clear out the offsets that have been appended to the list. */
+void pm_line_offset_list_clear(pm_line_offset_list_t *list);
+
+/* Append a new offset to the list (slow path with resize). */
+void pm_line_offset_list_append_slow(pm_arena_t *arena, pm_line_offset_list_t *list, uint32_t cursor);
+
+/* Append a new offset to the list. */
+static PRISM_FORCE_INLINE void
+pm_line_offset_list_append(pm_arena_t *arena, pm_line_offset_list_t *list, uint32_t cursor) {
+    if (list->size < list->capacity) {
+        list->offsets[list->size++] = cursor;
+    } else {
+        pm_line_offset_list_append_slow(arena, list, cursor);
+    }
+}
+
+/*
+ * Returns the line of the given offset. If the offset is not in the list, the
+ * line of the closest offset less than the given offset is returned.
+ */
+int32_t pm_line_offset_list_line(const pm_line_offset_list_t *list, uint32_t cursor, int32_t start_line);
+
+#endif

data/include/prism/{util/pm_list.h → internal/list.h}

@@ -1,19 +1,9 @@
-/**
- * @file pm_list.h
- *
- * An abstract linked list.
- */
-#ifndef PRISM_LIST_H
-#define PRISM_LIST_H
+#ifndef PRISM_INTERNAL_LIST_H
+#define PRISM_INTERNAL_LIST_H
 
-#include "prism/defines.h"
-
-#include <stdbool.h>
 #include <stddef.h>
-#include <stdint.h>
-#include <stdlib.h>
 
-/**
+/*
  * This struct represents an abstract linked list that provides common
  * functionality. It is meant to be used any time a linked list is necessary to
  * store data.
@@ -44,60 +34,29 @@
  * iteration and appending of new nodes.
  */
 typedef struct pm_list_node {
-    /** A pointer to the next node in the list. */
+    /* A pointer to the next node in the list. */
     struct pm_list_node *next;
 } pm_list_node_t;
 
-/**
+/*
  * This represents the overall linked list. It keeps a pointer to the head and
  * tail so that iteration is easy and pushing new nodes is easy.
  */
 typedef struct {
-    /** The size of the list. */
+    /* The size of the list. */
     size_t size;
 
-    /** A pointer to the head of the list. */
+    /* A pointer to the head of the list. */
     pm_list_node_t *head;
 
-    /** A pointer to the tail of the list. */
+    /* A pointer to the tail of the list. */
     pm_list_node_t *tail;
 } pm_list_t;
 
-/**
- * Returns true if the given list is empty.
- *
- * @param list The list to check.
- * @return True if the given list is empty, otherwise false.
- *
- * \public \memberof pm_list_t
- */
-PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list);
-
-/**
- * Returns the size of the list.
- *
- * @param list The list to check.
- * @return The size of the list.
- *
- * \public \memberof pm_list_t
- */
-PRISM_EXPORTED_FUNCTION size_t pm_list_size(pm_list_t *list);
+/* Returns the size of the list. */
+size_t pm_list_size(pm_list_t *list);
 
-/**
- * Append a node to the given list.
- *
- * @param list The list to append to.
- * @param node The node to append.
- */
+/* Append a node to the given list. */
 void pm_list_append(pm_list_t *list, pm_list_node_t *node);
 
-/**
- * Deallocate the internal state of the given list.
- *
- * @param list The list to free.
- *
- * \public \memberof pm_list_t
- */
-PRISM_EXPORTED_FUNCTION void pm_list_free(pm_list_t *list);
-
 #endif

data/include/prism/internal/magic_comments.h

@@ -0,0 +1,23 @@
+#ifndef PRISM_INTERNAL_MAGIC_COMMENTS_H
+#define PRISM_INTERNAL_MAGIC_COMMENTS_H
+
+#include "prism/magic_comments.h"
+
+#include "prism/internal/list.h"
+
+/*
+ * This is a node in the linked list of magic comments that we've found while
+ * parsing.
+ */
+struct pm_magic_comment_t {
+    /* The embedded base node. */
+    pm_list_node_t node;
+
+    /* The key of the magic comment. */
+    pm_location_t key;
+
+    /* The value of the magic comment. */
+    pm_location_t value;
+};
+
+#endif

data/include/prism/internal/memchr.h

@@ -0,0 +1,15 @@
+#ifndef PRISM_INTERNAL_MEMCHR_H
+#define PRISM_INTERNAL_MEMCHR_H
+
+#include "prism/internal/encoding.h"
+
+#include <stddef.h>
+
+/*
+ * We need to roll our own memchr to handle cases where the encoding changes and
+ * we need to search for a character in a buffer that could be the trailing byte
+ * of a multibyte character.
+ */
+const void * pm_memchr(const void *source, int character, size_t number, bool encoding_changed, const pm_encoding_t *encoding);
+
+#endif

data/include/prism/internal/node.h

@@ -0,0 +1,38 @@
+#ifndef PRISM_INTERNAL_NODE_H
+#define PRISM_INTERNAL_NODE_H
+
+#include "prism/node.h"
+
+#include "prism/compiler/force_inline.h"
+
+#include "prism/arena.h"
+
+/* The offset of the start of the given location. */
+#define PM_LOCATION_START(location_) ((location_)->start)
+
+/* The offset just past the end of the given location. */
+#define PM_LOCATION_END(location_) ((location_)->start + (location_)->length)
+
+/*
+ * Slow path for pm_node_list_append: grow the list and append the node.
+ * Do not call directly — use pm_node_list_append instead.
+ */
+void pm_node_list_append_slow(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node);
+
+/* Append a new node onto the end of the node list. */
+static PRISM_FORCE_INLINE void
+pm_node_list_append(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node) {
+    if (list->size < list->capacity) {
+        list->nodes[list->size++] = node;
+    } else {
+        pm_node_list_append_slow(arena, list, node);
+    }
+}
+
+/* Prepend a new node onto the beginning of the node list. */
+void pm_node_list_prepend(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node);
+
+/* Concatenate the given node list onto the end of the other node list. */
+void pm_node_list_concat(pm_arena_t *arena, pm_node_list_t *list, pm_node_list_t *other);
+
+#endif

data/include/prism/internal/options.h

@@ -0,0 +1,232 @@
+#ifndef PRISM_INTERNAL_OPTIONS_H
+#define PRISM_INTERNAL_OPTIONS_H
+
+#include "prism/options.h"
+
+/* A scope of locals surrounding the code that is being parsed. */
+struct pm_options_scope_t {
+    /* The number of locals in the scope. */
+    size_t locals_count;
+
+    /* The names of the locals in the scope. */
+    pm_string_t *locals;
+
+    /* Flags for the set of forwarding parameters in this scope. */
+    uint8_t forwarding;
+};
+
+/*
+ * The version of Ruby syntax that we should be parsing with. This is used to
+ * allow consumers to specify which behavior they want in case they need to
+ * parse in the same way as a specific version of CRuby would have.
+ */
+typedef enum {
+    /*
+     * If an explicit version is not provided, the current version of prism will
+     * be used.
+     */
+    PM_OPTIONS_VERSION_UNSET = 0,
+
+    /* The vendored version of prism in CRuby 3.3.x. */
+    PM_OPTIONS_VERSION_CRUBY_3_3 = 1,
+
+    /* The vendored version of prism in CRuby 3.4.x. */
+    PM_OPTIONS_VERSION_CRUBY_3_4 = 2,
+
+    /* The vendored version of prism in CRuby 4.0.x. */
+    PM_OPTIONS_VERSION_CRUBY_3_5 = 3,
+
+    /* The vendored version of prism in CRuby 4.0.x. */
+    PM_OPTIONS_VERSION_CRUBY_4_0 = 3,
+
+    /* The vendored version of prism in CRuby 4.1.x. */
+    PM_OPTIONS_VERSION_CRUBY_4_1 = 4,
+
+    /* The current version of prism. */
+    PM_OPTIONS_VERSION_LATEST = PM_OPTIONS_VERSION_CRUBY_4_1
+} pm_options_version_t;
+
+/* The options that can be passed to the parser. */
+struct pm_options_t {
+    /*
+     * The callback to call when additional switches are found in a shebang
+     * comment.
+     */
+    pm_options_shebang_callback_t shebang_callback;
+
+    /*
+     * Any additional data that should be passed along to the shebang callback
+     * if one was set.
+     */
+    void *shebang_callback_data;
+
+    /* The name of the file that is currently being parsed. */
+    pm_string_t filepath;
+
+    /*
+     * The line within the file that the parse starts on. This value is
+     * 1-indexed.
+     */
+    int32_t line;
+
+    /*
+     * The name of the encoding that the source file is in. Note that this must
+     * correspond to a name that can be found with Encoding.find in Ruby.
+     */
+    pm_string_t encoding;
+
+    /* The number of scopes surrounding the code that is being parsed. */
+    size_t scopes_count;
+
+    /*
+     * The scopes surrounding the code that is being parsed. For most parses
+     * this will be NULL, but for evals it will be the locals that are in scope
+     * surrounding the eval. Scopes are ordered from the outermost scope to the
+     * innermost one.
+     */
+    pm_options_scope_t *scopes;
+
+    /*
+     * The version of prism that we should be parsing with. This is used to
+     * allow consumers to specify which behavior they want in case they need to
+     * parse exactly as a specific version of CRuby.
+     */
+    pm_options_version_t version;
+
+    /* A bitset of the various options that were set on the command line. */
+    uint8_t command_line;
+
+    /*
+    * Whether or not the frozen string literal option has been set.
+    * May be:
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
+    */
+    int8_t frozen_string_literal;
+
+    /*
+     * Whether or not the parser should raise an error when it encounters an
+     * error. By default this is off, but can be turned on in the case that you
+     * want nicely formatted syntax errors. The values that this can be set to
+     * are:
+     *  - 0 - do not raise an error, just return the errors in the result
+     *  - PM_ERRORS_FORMAT_PLAIN - raise a plain error with no formatting
+     *  - PM_ERRORS_FORMAT_STYLE - raise an error with bold formatting
+     *  - PM_ERRORS_FORMAT_COLOR - raise an error with color formatting
+     */
+    uint8_t raise_error;
+
+    /*
+     * Whether or not the encoding magic comments should be respected. This is a
+     * niche use-case where you want to parse a file with a specific encoding
+     * but ignore any encoding magic comments at the top of the file.
+     */
+    bool encoding_locked;
+
+    /*
+     * When the file being parsed is the main script, the shebang will be
+     * considered for command-line flags (or for implicit -x). The caller needs
+     * to pass this information to the parser so that it can behave correctly.
+     */
+    bool main_script;
+
+    /*
+     * When the file being parsed is considered a "partial" script, jumps will
+     * not be marked as errors if they are not contained within loops/blocks.
+     * This is used in the case that you're parsing a script that you know will
+     * be embedded inside another script later, but you do not have that context
+     * yet. For example, when parsing an ERB template that will be evaluated
+     * inside another script.
+     */
+    bool partial_script;
+
+    /*
+     * Whether or not the parser should freeze the nodes that it creates. This
+     * makes it possible to have a deeply frozen AST that is safe to share
+     * between concurrency primitives.
+     */
+    bool freeze;
+
+    /*
+     * Whether or not the parser should attach comments to the nodes that own
+     * them after parsing. When this is set, the parser's comment list is
+     * sorted by start offset and every comment is assigned an owning node.
+     */
+    bool attach_comments;
+};
+
+/* Free the internal memory associated with the options. */
+void pm_options_cleanup(pm_options_t *options);
+
+/*
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ *
+ * `data` is assumed to be a valid pointer pointing to well-formed data. The
+ * layout of this data should be the same every time, and is described below:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the length of the filepath |
+ * | ...     | the filepath bytes         |
+ * | `4`     | the line number            |
+ * | `4`     | the length the encoding    |
+ * | ...     | the encoding bytes         |
+ * | `1`     | frozen string literal      |
+ * | `1`     | -p command line option     |
+ * | `1`     | -n command line option     |
+ * | `1`     | -l command line option     |
+ * | `1`     | -a command line option     |
+ * | `1`     | the version                |
+ * | `1`     | encoding locked            |
+ * | `1`     | main script                |
+ * | `1`     | partial script             |
+ * | `1`     | freeze                     |
+ * | `1`     | attach comments            |
+ * | `4`     | the number of scopes       |
+ * | ...     | the scopes                 |
+ *
+ * The version field is an enum, so it should be one of the following values:
+ *
+ * | value | version                   |
+ * | ----- | ------------------------- |
+ * | `0`   | use the latest version of prism |
+ * | `1`   | use the version of prism that is vendored in CRuby 3.3.0 |
+ * | `2`   | use the version of prism that is vendored in CRuby 3.4.0 |
+ * | `3`   | use the version of prism that is vendored in CRuby 4.0.0 |
+ * | `4`   | use the version of prism that is vendored in CRuby 4.1.0 |
+ *
+ * Each scope is laid out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the number of locals       |
+ * | `1`     | the forwarding flags       |
+ * | ...     | the locals                 |
+ *
+ * Each local is laid out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the length of the local    |
+ * | ...     | the local bytes            |
+ *
+ * Some additional things to note about this layout:
+ *
+ * * The filepath can have a length of 0, in which case we'll consider it an
+ *   empty string.
+ * * The line number should be 0-indexed.
+ * * The encoding can have a length of 0, in which case we'll use the default
+ *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
+ *   encoding that can be passed to `Encoding.find` in Ruby.
+ * * The frozen string literal, encoding locked, main script, and partial script
+ *   fields are booleans, so their values should be either 0 or 1.
+ * * The number of scopes can be 0.
+ */
+void pm_options_read(pm_options_t *options, const char *data);
+
+#endif