prism 0.29.0 → 1.1.0

data/include/prism/defines.h

@@ -25,6 +25,15 @@
 #define __STDC_FORMAT_MACROS
 #include <inttypes.h>
 
+/**
+ * When we are parsing using recursive descent, we want to protect against
+ * malicious payloads that could attempt to crash our parser. We do this by
+ * specifying a maximum depth to which we are allowed to recurse.
+ */
+#ifndef PRISM_DEPTH_MAXIMUM
+    #define PRISM_DEPTH_MAXIMUM 1000
+#endif
+
 /**
  * By default, we compile with -fvisibility=hidden. When this is enabled, we
  * need to mark certain functions as being publically-visible. This macro does
@@ -118,6 +127,15 @@
 #   endif
 #endif
 
+/**
+ * If PRISM_HAS_NO_FILESYSTEM is defined, then we want to exclude all filesystem
+ * related code from the library. All filesystem related code should be guarded
+ * by PRISM_HAS_FILESYSTEM.
+ */
+#ifndef PRISM_HAS_NO_FILESYSTEM
+#   define PRISM_HAS_FILESYSTEM
+#endif
+
 /**
  * isinf on Windows is defined as accepting a float, but on POSIX systems it
  * accepts a float, a double, or a long double. We want to mirror this behavior
@@ -203,4 +221,22 @@
     #define PRISM_ENCODING_EXCLUDE_FULL
 #endif
 
+/**
+ * Support PRISM_LIKELY and PRISM_UNLIKELY to help the compiler optimize its
+ * branch predication.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+    /** The compiler should predicate that this branch will be taken. */
+    #define PRISM_LIKELY(x) __builtin_expect(!!(x), 1)
+
+    /** The compiler should predicate that this branch will not be taken. */
+    #define PRISM_UNLIKELY(x) __builtin_expect(!!(x), 0)
+#else
+    /** Void because this platform does not support branch prediction hints. */
+    #define PRISM_LIKELY(x)   (x)
+
+    /** Void because this platform does not support branch prediction hints. */
+    #define PRISM_UNLIKELY(x) (x)
+#endif
+
 #endif

data/include/prism/diagnostic.h

@@ -1,10 +1,10 @@
-/******************************************************************************/
+/*----------------------------------------------------------------------------*/
 /* This file is generated by the templates/template.rb script and should not  */
 /* be modified manually. See                                                  */
 /* templates/include/prism/diagnostic.h.erb                                   */
 /* if you are looking to modify the                                           */
 /* template                                                                   */
-/******************************************************************************/
+/*----------------------------------------------------------------------------*/
 
 /**
  * @file diagnostic.h
@@ -44,7 +44,6 @@ typedef enum {
     PM_ERR_ARGUMENT_FORMAL_GLOBAL,
     PM_ERR_ARGUMENT_FORMAL_IVAR,
     PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
-    PM_ERR_ARGUMENT_IN,
     PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND,
     PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
     PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -110,8 +109,10 @@ typedef enum {
     PM_ERR_ESCAPE_INVALID_META_REPEAT,
     PM_ERR_ESCAPE_INVALID_UNICODE,
     PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS,
+    PM_ERR_ESCAPE_INVALID_UNICODE_LIST,
     PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL,
     PM_ERR_ESCAPE_INVALID_UNICODE_LONG,
+    PM_ERR_ESCAPE_INVALID_UNICODE_SHORT,
     PM_ERR_ESCAPE_INVALID_UNICODE_TERM,
     PM_ERR_EXPECT_ARGUMENT,
     PM_ERR_EXPECT_EOL_AFTER_STATEMENT,
@@ -126,6 +127,7 @@ typedef enum {
     PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT,
     PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH,
     PM_ERR_EXPECT_EXPRESSION_AFTER_STAR,
+    PM_ERR_EXPECT_FOR_DELIMITER,
     PM_ERR_EXPECT_IDENT_REQ_PARAMETER,
     PM_ERR_EXPECT_IN_DELIMITER,
     PM_ERR_EXPECT_LPAREN_REQ_PARAMETER,
@@ -134,6 +136,7 @@ typedef enum {
     PM_ERR_EXPECT_RPAREN,
     PM_ERR_EXPECT_RPAREN_AFTER_MULTI,
     PM_ERR_EXPECT_RPAREN_REQ_PARAMETER,
+    PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER,
     PM_ERR_EXPECT_STRING_CONTENT,
     PM_ERR_EXPECT_WHEN_DELIMITER,
     PM_ERR_EXPRESSION_BARE_HASH,
@@ -143,6 +146,7 @@ typedef enum {
     PM_ERR_EXPRESSION_NOT_WRITABLE_FILE,
     PM_ERR_EXPRESSION_NOT_WRITABLE_LINE,
     PM_ERR_EXPRESSION_NOT_WRITABLE_NIL,
+    PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED,
     PM_ERR_EXPRESSION_NOT_WRITABLE_SELF,
     PM_ERR_EXPRESSION_NOT_WRITABLE_TRUE,
     PM_ERR_FLOAT_PARSE,
@@ -182,6 +186,7 @@ typedef enum {
     PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER,
     PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING,
     PM_ERR_INVALID_PERCENT,
+    PM_ERR_INVALID_PERCENT_EOF,
     PM_ERR_INVALID_PRINTABLE_CHARACTER,
     PM_ERR_INVALID_RETRY_AFTER_ELSE,
     PM_ERR_INVALID_RETRY_AFTER_ENSURE,
@@ -210,12 +215,15 @@ typedef enum {
     PM_ERR_MODULE_TERM,
     PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
     PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
+    PM_ERR_NESTING_TOO_DEEP,
     PM_ERR_NO_LOCAL_VARIABLE,
+    PM_ERR_NON_ASSOCIATIVE_OPERATOR,
     PM_ERR_NOT_EXPRESSION,
     PM_ERR_NUMBER_LITERAL_UNDERSCORE,
+    PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK,
     PM_ERR_NUMBERED_PARAMETER_IT,
     PM_ERR_NUMBERED_PARAMETER_ORDINARY,
-    PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
+    PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK,
     PM_ERR_OPERATOR_MULTI_ASSIGN,
     PM_ERR_OPERATOR_WRITE_ARGUMENTS,
     PM_ERR_OPERATOR_WRITE_BLOCK,
@@ -232,8 +240,9 @@ typedef enum {
     PM_ERR_PARAMETER_SPLAT_MULTI,
     PM_ERR_PARAMETER_STAR,
     PM_ERR_PARAMETER_UNEXPECTED_FWD,
-    PM_ERR_PARAMETER_WILD_LOOSE_COMMA,
     PM_ERR_PARAMETER_UNEXPECTED_NO_KW,
+    PM_ERR_PARAMETER_WILD_LOOSE_COMMA,
+    PM_ERR_PATTERN_ARRAY_MULTIPLE_RESTS,
     PM_ERR_PATTERN_CAPTURE_DUPLICATE,
     PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET,
     PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA,
@@ -245,6 +254,7 @@ typedef enum {
     PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE,
     PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE,
     PM_ERR_PATTERN_EXPRESSION_AFTER_REST,
+    PM_ERR_PATTERN_FIND_MISSING_INNER,
     PM_ERR_PATTERN_HASH_IMPLICIT,
     PM_ERR_PATTERN_HASH_KEY,
     PM_ERR_PATTERN_HASH_KEY_DUPLICATE,
@@ -262,6 +272,7 @@ typedef enum {
     PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING,
     PM_ERR_REGEXP_INVALID_UNICODE_RANGE,
     PM_ERR_REGEXP_NON_ESCAPED_MBC,
+    PM_ERR_REGEXP_PARSE_ERROR,
     PM_ERR_REGEXP_TERM,
     PM_ERR_REGEXP_UNKNOWN_OPTIONS,
     PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP,
@@ -286,11 +297,15 @@ typedef enum {
     PM_ERR_TERNARY_COLON,
     PM_ERR_TERNARY_EXPRESSION_FALSE,
     PM_ERR_TERNARY_EXPRESSION_TRUE,
+    PM_ERR_UNARY_DISALLOWED,
     PM_ERR_UNARY_RECEIVER,
     PM_ERR_UNDEF_ARGUMENT,
     PM_ERR_UNEXPECTED_BLOCK_ARGUMENT,
     PM_ERR_UNEXPECTED_INDEX_BLOCK,
     PM_ERR_UNEXPECTED_INDEX_KEYWORDS,
+    PM_ERR_UNEXPECTED_LABEL,
+    PM_ERR_UNEXPECTED_MULTI_WRITE,
+    PM_ERR_UNEXPECTED_RANGE_OPERATOR,
     PM_ERR_UNEXPECTED_SAFE_NAVIGATION,
     PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
     PM_ERR_UNEXPECTED_TOKEN_IGNORE,
@@ -303,6 +318,7 @@ typedef enum {
     PM_ERR_XSTRING_TERM,
 
     // These are the warning diagnostics.
+    PM_WARN_AMBIGUOUS_BINARY_OPERATOR,
     PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
     PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
     PM_WARN_AMBIGUOUS_PREFIX_AMPERSAND,
@@ -318,10 +334,11 @@ typedef enum {
     PM_WARN_DUPLICATED_WHEN_CLAUSE,
     PM_WARN_FLOAT_OUT_OF_RANGE,
     PM_WARN_IGNORED_FROZEN_STRING_LITERAL,
+    PM_WARN_INDENTATION_MISMATCH,
     PM_WARN_INTEGER_IN_FLIP_FLOP,
     PM_WARN_INVALID_CHARACTER,
+    PM_WARN_INVALID_MAGIC_COMMENT_VALUE,
     PM_WARN_INVALID_NUMBERED_REFERENCE,
-    PM_WARN_INVALID_SHAREABLE_CONSTANT_VALUE,
     PM_WARN_KEYWORD_EOL,
     PM_WARN_LITERAL_IN_CONDITION_DEFAULT,
     PM_WARN_LITERAL_IN_CONDITION_VERBOSE,

data/include/prism/node.h

@@ -56,27 +56,6 @@ void pm_node_list_free(pm_node_list_t *list);
  */
 PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
 
-/**
- * This struct stores the information gathered by the pm_node_memsize function.
- * It contains both the memory footprint and additionally metadata about the
- * shape of the tree.
- */
-typedef struct {
-    /** The total memory footprint of the node and all of its children. */
-    size_t memsize;
-
-    /** The number of children the node has. */
-    size_t node_count;
-} pm_memsize_t;
-
-/**
- * Calculates the memory footprint of a given node.
- *
- * @param node The node to calculate the memory footprint of.
- * @param memsize The memory footprint of the node and all of its children.
- */
-PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize);
-
 /**
  * Returns a string representation of the given node type.
  *

data/include/prism/options.h

@@ -7,6 +7,7 @@
 #define PRISM_OPTIONS_H
 
 #include "prism/defines.h"
+#include "prism/util/pm_char.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
@@ -40,6 +41,23 @@ typedef struct pm_options_scope {
     pm_string_t *locals;
 } pm_options_scope_t;
 
+// Forward declaration needed by the callback typedef.
+struct pm_options;
+
+/**
+ * The callback called when additional switches are found in a shebang comment
+ * that need to be processed by the runtime.
+ *
+ * @param options The options struct that may be updated by this callback.
+ *   Certain fields will be checked for changes, specifically encoding,
+ *   command_line, and frozen_string_literal.
+ * @param source The source of the shebang comment.
+ * @param length The length of the source.
+ * @param shebang_callback_data Any additional data that should be passed along
+ *   to the callback.
+ */
+typedef void (*pm_options_shebang_callback_t)(struct pm_options *options, const uint8_t *source, size_t length, void *shebang_callback_data);
+
 /**
  * The version of Ruby syntax that we should be parsing with. This is used to
  * allow consumers to specify which behavior they want in case they need to
@@ -56,7 +74,19 @@ typedef enum {
 /**
  * The options that can be passed to the parser.
  */
-typedef struct {
+typedef struct pm_options {
+    /**
+     * The callback to call when additional switches are found in a shebang
+     * comment.
+     */
+    pm_options_shebang_callback_t shebang_callback;
+
+    /**
+     * Any additional data that should be passed along to the shebang callback
+     * if one was set.
+     */
+    void *shebang_callback_data;
+
     /** The name of the file that is currently being parsed. */
     pm_string_t filepath;
 
@@ -103,6 +133,30 @@ typedef struct {
     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
     */
     int8_t frozen_string_literal;
+
+    /**
+     * Whether or not the encoding magic comments should be respected. This is a
+     * niche use-case where you want to parse a file with a specific encoding
+     * but ignore any encoding magic comments at the top of the file.
+     */
+    bool encoding_locked;
+
+    /**
+     * When the file being parsed is the main script, the shebang will be
+     * considered for command-line flags (or for implicit -x). The caller needs
+     * to pass this information to the parser so that it can behave correctly.
+     */
+    bool main_script;
+
+    /**
+     * When the file being parsed is considered a "partial" script, jumps will
+     * not be marked as errors if they are not contained within loops/blocks.
+     * This is used in the case that you're parsing a script that you know will
+     * be embedded inside another script later, but you do not have that context
+     * yet. For example, when parsing an ERB template that will be evaluated
+     * inside another script.
+     */
+    bool partial_script;
 } pm_options_t;
 
 /**
@@ -142,6 +196,16 @@ static const uint8_t PM_OPTIONS_COMMAND_LINE_P = 0x10;
  */
 static const uint8_t PM_OPTIONS_COMMAND_LINE_X = 0x20;
 
+/**
+ * Set the shebang callback option on the given options struct.
+ *
+ * @param options The options struct to set the shebang callback on.
+ * @param shebang_callback The shebang callback to set.
+ * @param shebang_callback_data Any additional data that should be passed along
+ *   to the callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_shebang_callback_set(pm_options_t *options, pm_options_shebang_callback_t shebang_callback, void *shebang_callback_data);
+
 /**
  * Set the filepath option on the given options struct.
  *
@@ -166,6 +230,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t
  */
 PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
 
+/**
+ * Set the encoding_locked option on the given options struct.
+ *
+ * @param options The options struct to set the encoding_locked value on.
+ * @param encoding_locked The encoding_locked value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked);
+
 /**
  * Set the frozen string literal option on the given options struct.
  *
@@ -194,6 +266,22 @@ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options,
  */
 PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
 
+/**
+ * Set the main script option on the given options struct.
+ *
+ * @param options The options struct to set the main script value on.
+ * @param main_script The main script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script);
+
+/**
+ * Set the partial script option on the given options struct.
+ *
+ * @param options The options struct to set the partial script value on.
+ * @param partial_script The partial script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
+
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
@@ -261,6 +349,9 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `1`     | -l command line option     |
  * | `1`     | -a command line option     |
  * | `1`     | the version                |
+ * | `1`     | encoding locked            |
+ * | `1`     | main script                |
+ * | `1`     | partial script             |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *
@@ -293,8 +384,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * * The encoding can have a length of 0, in which case we'll use the default
  *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
  *   encoding that can be passed to `Encoding.find` in Ruby.
- * * The frozen string literal and suppress warnings fields are booleans, so
- *   their values should be either 0 or 1.
+ * * The frozen string literal, encoding locked, main script, and partial script
+ *   fields are booleans, so their values should be either 0 or 1.
  * * The number of scopes can be 0.
  *
  * @param options The options struct to deserialize into.

data/include/prism/parser.h

@@ -364,6 +364,9 @@ typedef enum {
     /** a rescue statement within a lambda expression */
     PM_CONTEXT_LAMBDA_RESCUE,
 
+    /** the predicate clause of a loop statement */
+    PM_CONTEXT_LOOP_PREDICATE,
+
     /** the top level context */
     PM_CONTEXT_MAIN,
 
@@ -505,9 +508,9 @@ typedef struct {
 /** The type of shareable constant value that can be set. */
 typedef uint8_t pm_shareable_constant_value_t;
 static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
-static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = 0x1;
-static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = 0x2;
-static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = 0x4;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY;
 
 /**
  * This tracks an individual local variable in a certain lexical context, as
@@ -546,6 +549,17 @@ typedef struct pm_locals {
     pm_local_t *locals;
 } pm_locals_t;
 
+/** The flags about scope parameters that can be set. */
+typedef uint8_t pm_scope_parameters_t;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40;
+
 /**
  * This struct represents a node in a linked list of scopes. Some scopes can see
  * into their parent scopes, while others cannot.
@@ -557,10 +571,19 @@ typedef struct pm_scope {
     /** The IDs of the locals in the given scope. */
     pm_locals_t locals;
 
+    /**
+     * This is a list of the implicit parameters contained within the block.
+     * These will be processed after the block is parsed to determine the kind
+     * of parameters node that should be used and to check if any errors need to
+     * be added.
+     */
+    pm_node_list_t implicit_parameters;
+
     /**
      * This is a bitfield that indicates the parameters that are being used in
-     * this scope. It is a combination of the PM_SCOPE_PARAMS_* constants. There
-     * are three different kinds of parameters that can be used in a scope:
+     * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants.
+     * There are three different kinds of parameters that can be used in a
+     * scope:
      *
      * - Ordinary parameters (e.g., def foo(bar); end)
      * - Numbered parameters (e.g., def foo; _1; end)
@@ -575,15 +598,7 @@ typedef struct pm_scope {
      * - def foo(&); end
      * - def foo(...); end
      */
-    uint8_t parameters;
-
-    /**
-     * An integer indicating the number of numbered parameters on this scope.
-     * This is necessary to determine if child blocks are allowed to use
-     * numbered parameters, and to pass information to consumers of the AST
-     * about how many numbered parameters exist.
-     */
-    int8_t numbered_parameters;
+    pm_scope_parameters_t parameters;
 
     /**
      * The current state of constant shareability for this scope. This is
@@ -598,20 +613,6 @@ typedef struct pm_scope {
     bool closed;
 } pm_scope_t;
 
-static const uint8_t PM_SCOPE_PARAMETERS_NONE = 0x0;
-static const uint8_t PM_SCOPE_PARAMETERS_ORDINARY = 0x1;
-static const uint8_t PM_SCOPE_PARAMETERS_NUMBERED = 0x2;
-static const uint8_t PM_SCOPE_PARAMETERS_IT = 0x4;
-static const uint8_t PM_SCOPE_PARAMETERS_TYPE_MASK = 0x7;
-
-static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x8;
-static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x10;
-static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x20;
-static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x40;
-
-static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED = -1;
-static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_NONE = 0;
-
 /**
  * A struct that represents a stack of boolean values.
  */
@@ -624,6 +625,13 @@ typedef uint32_t pm_state_stack_t;
  * it's considering.
  */
 struct pm_parser {
+    /**
+     * The next node identifier that will be assigned. This is a unique
+     * identifier used to track nodes such that the syntax tree can be dropped
+     * but the node can be found through another parse.
+     */
+    uint32_t node_id;
+
     /** The current state of the lexer. */
     pm_lex_state_t lex_state;
 
@@ -853,12 +861,27 @@ struct pm_parser {
      */
     bool parsing_eval;
 
+    /**
+     * Whether or not we are parsing a "partial" script, which is a script that
+     * will be evaluated in the context of another script, so we should not
+     * check jumps (next/break/etc.) for validity.
+     */
+    bool partial_script;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 
     /** Whether or not we're currently recovering from a syntax error. */
     bool recovering;
 
+    /**
+     * This is very specialized behavior for when you want to parse in a context
+     * that does not respect encoding comments. Its main use case is translating
+     * into the whitequark/parser AST which re-encodes source files in UTF-8
+     * before they are parsed and ignores encoding comments.
+     */
+    bool encoding_locked;
+
     /**
      * Whether or not the encoding has been changed by a magic comment. We use
      * this to provide a fast path for the lexer instead of going through the
@@ -886,6 +909,12 @@ struct pm_parser {
      * characters.
      */
     bool current_regular_expression_ascii_only;
+
+    /**
+     * By default, Ruby always warns about mismatched indentation. This can be
+     * toggled with a magic comment.
+     */
+    bool warn_mismatched_indentation;
 };
 
 #endif

data/include/prism/regexp.h

@@ -10,7 +10,6 @@
 #include "prism/parser.h"
 #include "prism/encoding.h"
 #include "prism/util/pm_memchr.h"
-#include "prism/util/pm_string_list.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
@@ -18,16 +17,27 @@
 #include <string.h>
 
 /**
- * Parse a regular expression and extract the names of all of the named capture
- * groups.
+ * This callback is called when a named capture group is found.
+ */
+typedef void (*pm_regexp_name_callback_t)(const pm_string_t *name, void *data);
+
+/**
+ * This callback is called when a parse error is found.
+ */
+typedef void (*pm_regexp_error_callback_t)(const uint8_t *start, const uint8_t *end, const char *message, void *data);
+
+/**
+ * Parse a regular expression.
  *
+ * @param parser The parser that is currently being used.
  * @param source The source code to parse.
  * @param size The size of the source code.
- * @param named_captures The list to add the names of the named capture groups.
- * @param encoding_changed Whether or not the encoding changed from the default.
- * @param encoding The encoding of the source code.
- * @return Whether or not the parsing was successful.
+ * @param extended_mode Whether to parse the regular expression in extended mode.
+ * @param name_callback The optional callback to call when a named capture group is found.
+ * @param name_data The optional data to pass to the name callback.
+ * @param error_callback The callback to call when a parse error is found.
+ * @param error_data The data to pass to the error callback.
  */
-PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, const pm_encoding_t *encoding);
+PRISM_EXPORTED_FUNCTION void pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data, pm_regexp_error_callback_t error_callback, void *error_data);
 
 #endif

data/include/prism/static_literals.h

@@ -95,9 +95,10 @@ typedef struct {
  * @param start_line The line number that the parser starts on.
  * @param literals The set of static literals to add the node to.
  * @param node The node to add to the set.
+ * @param replace Whether to replace the previous node if one already exists.
  * @return A pointer to the node that is being overwritten, if there is one.
  */
-pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node);
+pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace);
 
 /**
  * Free the internal memory associated with the given static literals set.
@@ -115,6 +116,6 @@ void pm_static_literals_free(pm_static_literals_t *literals);
  * @param encoding_name The name of the encoding of the source being parsed.
  * @param node The node to create a string representation of.
  */
-PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node);
+void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node);
 
 #endif

data/include/prism/util/pm_char.h

@@ -34,8 +34,7 @@ size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
  * @return The number of characters at the start of the string that are
  *     whitespace.
  */
-size_t
-pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
+size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
 
 /**
  * Returns the number of characters at the start of the string that are inline

data/include/prism/util/pm_constant_pool.h

@@ -87,14 +87,6 @@ void pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_co
  */
 bool pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id);
 
-/**
- * Get the memory size of a list of constant ids.
- *
- * @param list The list to get the memory size of.
- * @return The memory size of the list.
- */
-size_t pm_constant_id_list_memsize(pm_constant_id_list_t *list);
-
 /**
  * Free the memory associated with a list of constant ids.
  *