jruby-prism-parser 0.24.0-java → 1.4.0-java

data/include/prism/node.h

@@ -8,6 +8,14 @@
 
 #include "prism/defines.h"
 #include "prism/parser.h"
+#include "prism/util/pm_buffer.h"
+
+/**
+ * Loop through each node in the node list, writing each node to the given
+ * pm_node_t pointer.
+ */
+#define PM_NODE_LIST_FOREACH(list, index, node) \
+    for (size_t index = 0; index < (list)->size && ((node) = (list)->nodes[index]); index++)
 
 /**
  * Append a new node onto the end of the node list.
@@ -18,33 +26,35 @@
 void pm_node_list_append(pm_node_list_t *list, pm_node_t *node);
 
 /**
- * Deallocate a node and all of its children.
+ * Prepend a new node onto the beginning of the node list.
  *
- * @param parser The parser that owns the node.
- * @param node The node to deallocate.
+ * @param list The list to prepend to.
+ * @param node The node to prepend.
  */
-PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
+void pm_node_list_prepend(pm_node_list_t *list, pm_node_t *node);
 
 /**
- * This struct stores the information gathered by the pm_node_memsize function.
- * It contains both the memory footprint and additionally metadata about the
- * shape of the tree.
+ * Concatenate the given node list onto the end of the other node list.
+ *
+ * @param list The list to concatenate onto.
+ * @param other The list to concatenate.
  */
-typedef struct {
-    /** The total memory footprint of the node and all of its children. */
-    size_t memsize;
+void pm_node_list_concat(pm_node_list_t *list, pm_node_list_t *other);
 
-    /** The number of children the node has. */
-    size_t node_count;
-} pm_memsize_t;
+/**
+ * Free the internal memory associated with the given node list.
+ *
+ * @param list The list to free.
+ */
+void pm_node_list_free(pm_node_list_t *list);
 
 /**
- * Calculates the memory footprint of a given node.
+ * Deallocate a node and all of its children.
  *
- * @param node The node to calculate the memory footprint of.
- * @param memsize The memory footprint of the node and all of its children.
+ * @param parser The parser that owns the node.
+ * @param node The node to deallocate.
  */
-PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize);
+PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
 
 /**
  * Returns a string representation of the given node type.
@@ -54,4 +64,66 @@ PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *mems
  */
 PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_type);
 
+/**
+ * Visit each of the nodes in this subtree using the given visitor callback. The
+ * callback function will be called for each node in the subtree. If it returns
+ * false, then that node's children will not be visited. If it returns true,
+ * then the children will be visited. The data parameter is treated as an opaque
+ * pointer and is passed to the visitor callback for consumers to use as they
+ * see fit.
+ *
+ * As an example:
+ *
+ * ```c
+ * #include "prism.h"
+ *
+ * bool visit(const pm_node_t *node, void *data) {
+ *     size_t *indent = (size_t *) data;
+ *     for (size_t i = 0; i < *indent * 2; i++) putc(' ', stdout);
+ *     printf("%s\n", pm_node_type_to_str(node->type));
+ *
+ *     size_t next_indent = *indent + 1;
+ *     size_t *next_data = &next_indent;
+ *     pm_visit_child_nodes(node, visit, next_data);
+ *
+ *     return false;
+ * }
+ *
+ * int main(void) {
+ *     const char *source = "1 + 2; 3 + 4";
+ *     size_t size = strlen(source);
+ *
+ *     pm_parser_t parser;
+ *     pm_options_t options = { 0 };
+ *     pm_parser_init(&parser, (const uint8_t *) source, size, &options);
+ *
+ *     size_t indent = 0;
+ *     pm_node_t *node = pm_parse(&parser);
+ *
+ *     size_t *data = &indent;
+ *     pm_visit_node(node, visit, data);
+ *
+ *     pm_node_destroy(&parser, node);
+ *     pm_parser_free(&parser);
+ *     return EXIT_SUCCESS;
+ * }
+ * ```
+ *
+ * @param node The root node to start visiting from.
+ * @param visitor The callback to call for each node in the subtree.
+ * @param data An opaque pointer that is passed to the visitor callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+
+/**
+ * Visit the children of the given node with the given callback. This is the
+ * default behavior for walking the tree that is called from pm_visit_node if
+ * the callback returns true.
+ *
+ * @param node The node to visit the children of.
+ * @param visitor The callback to call for each child node.
+ * @param data An opaque pointer that is passed to the visitor callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+
 #endif

data/include/prism/options.h

@@ -7,12 +7,29 @@
 #define PRISM_OPTIONS_H
 
 #include "prism/defines.h"
+#include "prism/util/pm_char.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 
+/**
+ * String literals should be made frozen.
+ */
+#define PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED   ((int8_t) -1)
+
+/**
+ * String literals may be frozen or mutable depending on the implementation
+ * default.
+ */
+#define PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET      ((int8_t)  0)
+
+/**
+ * String literals should be made mutable.
+ */
+#define PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED    ((int8_t)  1)
+
 /**
  * A scope of locals surrounding the code that is being parsed.
  */
@@ -22,8 +39,43 @@ typedef struct pm_options_scope {
 
     /** The names of the locals in the scope. */
     pm_string_t *locals;
+
+    /** Flags for the set of forwarding parameters in this scope. */
+    uint8_t forwarding;
 } pm_options_scope_t;
 
+/** The default value for parameters. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_NONE = 0x0;
+
+/** When the scope is fowarding with the * parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_POSITIONALS = 0x1;
+
+/** When the scope is fowarding with the ** parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_KEYWORDS = 0x2;
+
+/** When the scope is fowarding with the & parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_BLOCK = 0x4;
+
+/** When the scope is fowarding with the ... parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_ALL = 0x8;
+
+// Forward declaration needed by the callback typedef.
+struct pm_options;
+
+/**
+ * The callback called when additional switches are found in a shebang comment
+ * that need to be processed by the runtime.
+ *
+ * @param options The options struct that may be updated by this callback.
+ *   Certain fields will be checked for changes, specifically encoding,
+ *   command_line, and frozen_string_literal.
+ * @param source The source of the shebang comment.
+ * @param length The length of the source.
+ * @param shebang_callback_data Any additional data that should be passed along
+ *   to the callback.
+ */
+typedef void (*pm_options_shebang_callback_t)(struct pm_options *options, const uint8_t *source, size_t length, void *shebang_callback_data);
+
 /**
  * The version of Ruby syntax that we should be parsing with. This is used to
  * allow consumers to specify which behavior they want in case they need to
@@ -33,14 +85,29 @@ typedef enum {
     /** The current version of prism. */
     PM_OPTIONS_VERSION_LATEST = 0,
 
-    /** The vendored version of prism in CRuby 3.3.0. */
-    PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
+    /** The vendored version of prism in CRuby 3.3.x. */
+    PM_OPTIONS_VERSION_CRUBY_3_3 = 1,
+
+    /** The vendored version of prism in CRuby 3.4.x. */
+    PM_OPTIONS_VERSION_CRUBY_3_4 = 2
 } pm_options_version_t;
 
 /**
  * The options that can be passed to the parser.
  */
-typedef struct {
+typedef struct pm_options {
+    /**
+     * The callback to call when additional switches are found in a shebang
+     * comment.
+     */
+    pm_options_shebang_callback_t shebang_callback;
+
+    /**
+     * Any additional data that should be passed along to the shebang callback
+     * if one was set.
+     */
+    void *shebang_callback_data;
+
     /** The name of the file that is currently being parsed. */
     pm_string_t filepath;
 
@@ -76,10 +143,97 @@ typedef struct {
      */
     pm_options_version_t version;
 
-    /** Whether or not the frozen string literal option has been set. */
-    bool frozen_string_literal;
+    /** A bitset of the various options that were set on the command line. */
+    uint8_t command_line;
+
+    /**
+    * Whether or not the frozen string literal option has been set.
+    * May be:
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
+    *  - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
+    */
+    int8_t frozen_string_literal;
+
+    /**
+     * Whether or not the encoding magic comments should be respected. This is a
+     * niche use-case where you want to parse a file with a specific encoding
+     * but ignore any encoding magic comments at the top of the file.
+     */
+    bool encoding_locked;
+
+    /**
+     * When the file being parsed is the main script, the shebang will be
+     * considered for command-line flags (or for implicit -x). The caller needs
+     * to pass this information to the parser so that it can behave correctly.
+     */
+    bool main_script;
+
+    /**
+     * When the file being parsed is considered a "partial" script, jumps will
+     * not be marked as errors if they are not contained within loops/blocks.
+     * This is used in the case that you're parsing a script that you know will
+     * be embedded inside another script later, but you do not have that context
+     * yet. For example, when parsing an ERB template that will be evaluated
+     * inside another script.
+     */
+    bool partial_script;
+
+    /**
+     * Whether or not the parser should freeze the nodes that it creates. This
+     * makes it possible to have a deeply frozen AST that is safe to share
+     * between concurrency primitives.
+     */
+    bool freeze;
 } pm_options_t;
 
+/**
+ * A bit representing whether or not the command line -a option was set. -a
+ * splits the input line $_ into $F.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_A = 0x1;
+
+/**
+ * A bit representing whether or not the command line -e option was set. -e
+ * allow the user to specify a script to be executed. This is necessary for
+ * prism to know because certain warnings are not generated when -e is used.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_E = 0x2;
+
+/**
+ * A bit representing whether or not the command line -l option was set. -l
+ * chomps the input line by default.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_L = 0x4;
+
+/**
+ * A bit representing whether or not the command line -n option was set. -n
+ * wraps the script in a while gets loop.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_N = 0x8;
+
+/**
+ * A bit representing whether or not the command line -p option was set. -p
+ * prints the value of $_ at the end of each loop.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_P = 0x10;
+
+/**
+ * A bit representing whether or not the command line -x option was set. -x
+ * searches the input file for a shebang that matches the current Ruby engine.
+ */
+static const uint8_t PM_OPTIONS_COMMAND_LINE_X = 0x20;
+
+/**
+ * Set the shebang callback option on the given options struct.
+ *
+ * @param options The options struct to set the shebang callback on.
+ * @param shebang_callback The shebang callback to set.
+ * @param shebang_callback_data Any additional data that should be passed along
+ *   to the callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *options, pm_options_shebang_callback_t shebang_callback, void *shebang_callback_data);
+
 /**
  * Set the filepath option on the given options struct.
  *
@@ -104,6 +258,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t
  */
 PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
 
+/**
+ * Set the encoding_locked option on the given options struct.
+ *
+ * @param options The options struct to set the encoding_locked value on.
+ * @param encoding_locked The encoding_locked value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked);
+
 /**
  * Set the frozen string literal option on the given options struct.
  *
@@ -112,6 +274,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
  */
 PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
 
+/**
+ * Sets the command line option on the given options struct.
+ *
+ * @param options The options struct to set the command line option on.
+ * @param command_line The command_line value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options, uint8_t command_line);
+
 /**
  * Set the version option on the given options struct by parsing the given
  * string. If the string contains an invalid option, this returns false.
@@ -124,13 +294,38 @@ PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *
  */
 PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
 
+/**
+ * Set the main script option on the given options struct.
+ *
+ * @param options The options struct to set the main script value on.
+ * @param main_script The main script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script);
+
+/**
+ * Set the partial script option on the given options struct.
+ *
+ * @param options The options struct to set the partial script value on.
+ * @param partial_script The partial script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
+
+/**
+ * Set the freeze option on the given options struct.
+ *
+ * @param options The options struct to set the freeze value on.
+ * @param freeze The freeze value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze);
+
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
  * @param options The options struct to initialize the scopes array on.
  * @param scopes_count The number of scopes to allocate.
+ * @return Whether or not the scopes array was initialized successfully.
  */
-PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
+PRISM_EXPORTED_FUNCTION bool pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
 
 /**
  * Return a pointer to the scope at the given index within the given options.
@@ -147,8 +342,9 @@ PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm
  *
  * @param scope The scope struct to initialize.
  * @param locals_count The number of locals to allocate.
+ * @return Whether or not the scope was initialized successfully.
  */
-PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
+PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
 
 /**
  * Return a pointer to the local at the given index within the given scope.
@@ -159,6 +355,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, si
  */
 PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
 
+/**
+ * Set the forwarding option on the given scope struct.
+ *
+ * @param scope The scope struct to set the forwarding on.
+ * @param forwarding The forwarding value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding);
+
 /**
  * Free the internal memory associated with the options.
  *
@@ -184,8 +388,15 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `4`     | the length the encoding    |
  * | ...     | the encoding bytes         |
  * | `1`     | frozen string literal      |
- * | `1`     | suppress warnings          |
+ * | `1`     | -p command line option     |
+ * | `1`     | -n command line option     |
+ * | `1`     | -l command line option     |
+ * | `1`     | -a command line option     |
  * | `1`     | the version                |
+ * | `1`     | encoding locked            |
+ * | `1`     | main script                |
+ * | `1`     | partial script             |
+ * | `1`     | freeze                     |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *
@@ -196,14 +407,15 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `0`   | use the latest version of prism |
  * | `1`   | use the version of prism that is vendored in CRuby 3.3.0 |
  *
- * Each scope is layed out as follows:
+ * Each scope is laid out as follows:
  *
  * | # bytes | field                      |
  * | ------- | -------------------------- |
  * | `4`     | the number of locals       |
+ * | `1`     | the forwarding flags       |
  * | ...     | the locals                 |
  *
- * Each local is layed out as follows:
+ * Each local is laid out as follows:
  *
  * | # bytes | field                      |
  * | ------- | -------------------------- |
@@ -218,8 +430,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * * The encoding can have a length of 0, in which case we'll use the default
  *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
  *   encoding that can be passed to `Encoding.find` in Ruby.
- * * The frozen string literal and suppress warnings fields are booleans, so
- *   their values should be either 0 or 1.
+ * * The frozen string literal, encoding locked, main script, and partial script
+ *   fields are booleans, so their values should be either 0 or 1.
  * * The number of scopes can be 0.
  *
  * @param options The options struct to deserialize into.

data/include/prism/pack.h

@@ -8,6 +8,15 @@
 
 #include "prism/defines.h"
 
+// We optionally support parsing String#pack templates. For systems that don't
+// want or need this functionality, it can be turned off with the
+// PRISM_EXCLUDE_PACK define.
+#ifdef PRISM_EXCLUDE_PACK
+
+void pm_pack_parse(void);
+
+#else
+
 #include <stdint.h>
 #include <stdlib.h>
 
@@ -150,3 +159,5 @@ pm_pack_parse(
 PRISM_EXPORTED_FUNCTION size_t pm_size_to_native(pm_pack_size size);
 
 #endif
+
+#endif