prism 0.19.0 → 0.20.0

data/include/prism/defines.h

@@ -16,6 +16,14 @@
 #include <stdio.h>
 #include <string.h>
 
+/**
+ * We want to be able to use the PRI* macros for printing out integers, but on
+ * some platforms they aren't included unless this is already defined.
+ */
+#define __STDC_FORMAT_MACROS
+
+#include <inttypes.h>
+
 /**
  * By default, we compile with -fvisibility=hidden. When this is enabled, we
  * need to mark certain functions as being publically-visible. This macro does

data/include/prism/diagnostic.h

@@ -14,6 +14,24 @@
 #include <stdlib.h>
 #include <assert.h>
 
+/**
+ * The levels of errors generated during parsing.
+ */
+typedef enum {
+    /** For errors that cannot be recovered from. */
+    PM_ERROR_LEVEL_FATAL = 0
+} pm_error_level_t;
+
+/**
+ * The levels of warnings generated during parsing.
+ */
+typedef enum {
+    /** For warnings which should be emitted if $VERBOSE != nil. */
+    PM_WARNING_LEVEL_DEFAULT = 0,
+    /** For warnings which should be emitted if $VERBOSE == true. */
+    PM_WARNING_LEVEL_VERBOSE = 1
+} pm_warning_level_t;
+
 /**
  * This struct represents a diagnostic generated during parsing.
  *
@@ -35,6 +53,12 @@ typedef struct {
      * diagnostic is freed.
      */
     bool owned;
+
+    /**
+     * The level of the diagnostic, see `pm_error_level_t` and
+     * `pm_warning_level_t` for possible values.
+     */
+    uint8_t level;
 } pm_diagnostic_t;
 
 /**
@@ -42,17 +66,24 @@ typedef struct {
  * of errors between the parser and the user.
  */
 typedef enum {
+    // This is a special error that we can potentially replace by others. For
+    // an example of how this is used, see parse_expression_prefix.
+    PM_ERR_CANNOT_PARSE_EXPRESSION,
+
+    // These are the error codes.
     PM_ERR_ALIAS_ARGUMENT,
     PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
     PM_ERR_ARGUMENT_AFTER_BLOCK,
     PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES,
     PM_ERR_ARGUMENT_BARE_HASH,
+    PM_ERR_ARGUMENT_BLOCK_FORWARDING,
     PM_ERR_ARGUMENT_BLOCK_MULTI,
     PM_ERR_ARGUMENT_FORMAL_CLASS,
     PM_ERR_ARGUMENT_FORMAL_CONSTANT,
     PM_ERR_ARGUMENT_FORMAL_GLOBAL,
     PM_ERR_ARGUMENT_FORMAL_IVAR,
     PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
+    PM_ERR_ARGUMENT_IN,
     PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
     PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
     PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -74,7 +105,6 @@ typedef enum {
     PM_ERR_BLOCK_PARAM_PIPE_TERM,
     PM_ERR_BLOCK_TERM_BRACE,
     PM_ERR_BLOCK_TERM_END,
-    PM_ERR_CANNOT_PARSE_EXPRESSION,
     PM_ERR_CANNOT_PARSE_STRING_PART,
     PM_ERR_CASE_EXPRESSION_AFTER_CASE,
     PM_ERR_CASE_EXPRESSION_AFTER_WHEN,
@@ -165,6 +195,7 @@ typedef enum {
     PM_ERR_INVALID_PERCENT,
     PM_ERR_INVALID_TOKEN,
     PM_ERR_INVALID_VARIABLE_GLOBAL,
+    PM_ERR_IT_NOT_ALLOWED,
     PM_ERR_LAMBDA_OPEN,
     PM_ERR_LAMBDA_TERM_BRACE,
     PM_ERR_LAMBDA_TERM_END,
@@ -183,6 +214,7 @@ typedef enum {
     PM_ERR_MODULE_TERM,
     PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
     PM_ERR_NOT_EXPRESSION,
+    PM_ERR_NO_LOCAL_VARIABLE,
     PM_ERR_NUMBER_LITERAL_UNDERSCORE,
     PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
     PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
@@ -227,6 +259,7 @@ typedef enum {
     PM_ERR_RESCUE_TERM,
     PM_ERR_RESCUE_VARIABLE,
     PM_ERR_RETURN_INVALID,
+    PM_ERR_SINGLETON_FOR_LITERALS,
     PM_ERR_STATEMENT_ALIAS,
     PM_ERR_STATEMENT_POSTEXE_END,
     PM_ERR_STATEMENT_PREEXE_BEGIN,
@@ -244,6 +277,8 @@ typedef enum {
     PM_ERR_UNARY_RECEIVER_MINUS,
     PM_ERR_UNARY_RECEIVER_PLUS,
     PM_ERR_UNARY_RECEIVER_TILDE,
+    PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
+    PM_ERR_UNEXPECTED_TOKEN_IGNORE,
     PM_ERR_UNDEF_ARGUMENT,
     PM_ERR_UNTIL_TERM,
     PM_ERR_VOID_EXPRESSION,
@@ -252,13 +287,15 @@ typedef enum {
     PM_ERR_WRITE_TARGET_READONLY,
     PM_ERR_WRITE_TARGET_UNEXPECTED,
     PM_ERR_XSTRING_TERM,
+
+    // These are the warning codes.
     PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
     PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
     PM_WARN_AMBIGUOUS_PREFIX_STAR,
     PM_WARN_AMBIGUOUS_SLASH,
     PM_WARN_END_IN_METHOD,
 
-    /* This must be the last member. */
+    // This is the number of diagnostic codes.
     PM_DIAGNOSTIC_ID_LEN,
 } pm_diagnostic_id_t;
 

data/include/prism/encoding.h

@@ -79,6 +79,16 @@ typedef struct {
  */
 #define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
 
+/**
+ * Return the size of the next character in the UTF-8 encoding.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
+
 /**
  * Return the size of the next character in the UTF-8 encoding if it is an
  * alphabetical character.

data/include/prism/options.h

@@ -24,6 +24,19 @@ typedef struct pm_options_scope {
     pm_string_t *locals;
 } pm_options_scope_t;
 
+/**
+ * The version of Ruby syntax that we should be parsing with. This is used to
+ * allow consumers to specify which behavior they want in case they need to
+ * parse in the same way as a specific version of CRuby would have.
+ */
+typedef enum {
+    /** The current version of prism. */
+    PM_OPTIONS_VERSION_LATEST = 0,
+
+    /** The vendored version of prism in CRuby 3.3.0. */
+    PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
+} pm_options_version_t;
+
 /**
  * The options that can be passed to the parser.
  */
@@ -33,7 +46,7 @@ typedef struct {
 
     /**
      * The line within the file that the parse starts on. This value is
-     * 0-indexed.
+     * 1-indexed.
      */
     int32_t line;
 
@@ -51,19 +64,20 @@ typedef struct {
     /**
      * The scopes surrounding the code that is being parsed. For most parses
      * this will be NULL, but for evals it will be the locals that are in scope
-     * surrounding the eval.
+     * surrounding the eval. Scopes are ordered from the outermost scope to the
+     * innermost one.
      */
     pm_options_scope_t *scopes;
 
-    /** Whether or not the frozen string literal option has been set. */
-    bool frozen_string_literal;
-
     /**
-     * Whether or not we should suppress warnings. This is purposefully negated
-     * so that the default is to not suppress warnings, which allows us to still
-     * create an options struct with zeroed memory.
+     * The version of prism that we should be parsing with. This is used to
+     * allow consumers to specify which behavior they want in case they need to
+     * parse exactly as a specific version of CRuby.
      */
-    bool suppress_warnings;
+    pm_options_version_t version;
+
+    /** Whether or not the frozen string literal option has been set. */
+    bool frozen_string_literal;
 } pm_options_t;
 
 /**
@@ -99,12 +113,16 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
 PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
 
 /**
- * Set the suppress warnings option on the given options struct.
- *
- * @param options The options struct to set the suppress warnings value on.
- * @param suppress_warnings The suppress warnings value to set.
+ * Set the version option on the given options struct by parsing the given
+ * string. If the string contains an invalid option, this returns false.
+ * Otherwise, it returns true.
+ *
+ * @param options The options struct to set the version on.
+ * @param version The version to set.
+ * @param length The length of the version string.
+ * @return Whether or not the version was parsed successfully.
  */
-PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
+PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
 
 /**
  * Allocate and zero out the scopes array on the given options struct.
@@ -167,9 +185,17 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | ...     | the encoding bytes         |
  * | `1`     | frozen string literal      |
  * | `1`     | suppress warnings          |
+ * | `1`     | the version                |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *
+ * The version field is an enum, so it should be one of the following values:
+ *
+ * | value | version                   |
+ * | ----- | ------------------------- |
+ * | `0`   | use the latest version of prism |
+ * | `1`   | use the version of prism that is vendored in CRuby 3.3.0 |
+ *
  * Each scope is layed out as follows:
  *
  * | # bytes | field                      |

data/include/prism/parser.h

@@ -9,6 +9,7 @@
 #include "prism/ast.h"
 #include "prism/defines.h"
 #include "prism/encoding.h"
+#include "prism/options.h"
 #include "prism/util/pm_constant_pool.h"
 #include "prism/util/pm_list.h"
 #include "prism/util/pm_newline_list.h"
@@ -17,12 +18,6 @@
 
 #include <stdbool.h>
 
-// TODO: remove this by renaming the original flag
-/**
- * Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
- */
-#define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
-
 /**
  * This enum provides various bits that represent different kinds of states that
  * the lexer can track. This is used to determine which kind of token to return
@@ -264,6 +259,9 @@ typedef struct pm_parser pm_parser_t;
  * token that is understood by a parent context but not by the current context.
  */
 typedef enum {
+    /** a null context, used for returning a value from a function */
+    PM_CONTEXT_NONE = 0,
+
     /** a begin statement */
     PM_CONTEXT_BEGIN,
 
@@ -471,6 +469,19 @@ typedef struct pm_scope {
      */
     bool explicit_params;
 
+    /**
+     * Booleans indicating whether the parameters for this scope have declared
+     * forwarding parameters.
+     *
+     * For example, some combinations of:
+     *   def foo(*); end
+     *   def foo(**); end
+     *   def foo(&); end
+     *   def foo(...); end
+     */
+
+    uint8_t forwarding_params;
+
     /**
      * An integer indicating the number of numbered parameters on this scope.
      * This is necessary to determine if child blocks are allowed to use
@@ -480,6 +491,11 @@ typedef struct pm_scope {
     uint8_t numbered_parameters;
 } pm_scope_t;
 
+static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
+static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
+static const uint8_t PM_FORWARDING_BLOCK = 0x4;
+static const uint8_t PM_FORWARDING_ALL = 0x8;
+
 /**
  * This struct represents the overall parser. It contains a reference to the
  * source file, as well as pointers that indicate where in the source it's
@@ -562,7 +578,11 @@ struct pm_parser {
     /** The list of magic comments that have been found while parsing. */
     pm_list_t magic_comment_list;
 
-    /** The optional location of the __END__ keyword and its contents. */
+    /**
+     * An optional location that represents the location of the __END__ marker
+     * and the rest of the content of the file. This content is loaded into the
+     * DATA constant when the file being parsed is the main file being executed.
+     */
     pm_location_t data_loc;
 
     /** The list of warnings that have been found while parsing. */
@@ -668,6 +688,12 @@ struct pm_parser {
      */
     const pm_encoding_t *explicit_encoding;
 
+    /** The current parameter name id on parsing its default value. */
+    pm_constant_id_t current_param_name;
+
+    /** The version of prism that we should use to parse. */
+    pm_options_version_t version;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 
@@ -690,9 +716,6 @@ struct pm_parser {
     /** This flag indicates that we are currently parsing a keyword argument. */
     bool in_keyword_arg;
 
-    /** The current parameter name id on parsing its default value. */
-    pm_constant_id_t current_param_name;
-
     /**
      * Whether or not the parser has seen a token that has semantic meaning
      * (i.e., a token that is not a comment or whitespace).
@@ -704,13 +727,6 @@ struct pm_parser {
      * a true value.
      */
     bool frozen_string_literal;
-
-    /**
-     * Whether or not we should emit warnings. This will be set to false if the
-     * consumer of the library specified it, usually because they are parsing
-     * when $VERBOSE is nil.
-     */
-    bool suppress_warnings;
 };
 
 #endif

data/include/prism/util/pm_buffer.h

@@ -128,6 +128,15 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
  */
 void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
 
+/**
+ * Prepend the given string to the buffer.
+ *
+ * @param buffer The buffer to prepend to.
+ * @param value The string to prepend.
+ * @param length The length of the string to prepend.
+ */
+void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
+
 /**
  * Concatenate one buffer onto another.
  *

data/include/prism/util/pm_constant_pool.h

@@ -18,6 +18,13 @@
 #include <stdlib.h>
 #include <string.h>
 
+/**
+ * When we allocate constants into the pool, we reserve 0 to mean that the slot
+ * is not yet filled. This constant is reused in other places to indicate the
+ * lack of a constant id.
+ */
+#define PM_CONSTANT_ID_UNSET 0
+
 /**
  * A constant id is a unique identifier for a constant in the constant pool.
  */

data/include/prism/util/pm_newline_list.h

@@ -72,17 +72,6 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t
  */
 bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
 
-/**
- * Conditionally append a new offset to the newline list, if the value passed in
- * is a newline.
- *
- * @param list The list to append to.
- * @param cursor A pointer to the offset to append.
- * @return True if the reallocation of the offsets succeeds (if one was
- *     necessary), otherwise false.
- */
-bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
-
 /**
  * Returns the line and column of the given offset. If the offset is not in the
  * list, the line and column of the closest offset less than the given offset

data/include/prism/version.h

@@ -14,7 +14,7 @@
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 19
+#define PRISM_VERSION_MINOR 20
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "0.19.0"
+#define PRISM_VERSION "0.20.0"
 
 #endif

data/include/prism.h

@@ -168,7 +168,24 @@ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t si
  * @param token_type The token type to convert to a string.
  * @return A string representation of the given token type.
  */
-PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
+PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
+
+/**
+ * Returns the human name of the given token type.
+ *
+ * @param token_type The token type to convert to a human name.
+ * @return The human name of the given token type.
+ */
+const char * pm_token_type_human(pm_token_type_t token_type);
+
+/**
+ * Format the errors on the parser into the given buffer.
+ *
+ * @param parser The parser to format the errors for.
+ * @param buffer The buffer to format the errors into.
+ * @param colorize Whether or not to colorize the errors with ANSI escape sequences.
+ */
+PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
 
 /**
  * @mainpage
@@ -260,7 +277,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
  *     pm_buffer_t buffer = { 0 };
  *
  *     pm_prettyprint(&buffer, &parser, root);
- *     printf("*.s%\n", (int) buffer.length, buffer.value);
+ *     printf("%*.s\n", (int) buffer.length, buffer.value);
  *
  *     pm_buffer_free(&buffer);
  *     pm_node_destroy(&parser, root);

data/lib/prism/debug.rb

@@ -138,12 +138,14 @@ module Prism
               *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
             ]
 
+            sorted << AnonymousLocal if params.keywords.any?
+
             if params.keyword_rest.is_a?(ForwardingParameterNode)
-              sorted.push(:*, :&, :"...")
+              sorted.push(:*, :**, :&, :"...")
+            elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
+              sorted << (params.keyword_rest.name || :**)
             end
 
-            sorted << AnonymousLocal if params.keywords.any?
-
             # Recurse down the parameter tree to find any destructured
             # parameters and add them after the other parameters.
             param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
@@ -151,15 +153,19 @@ module Prism
               case param
               when MultiTargetNode
                 param_stack.concat(param.rights.reverse)
-                param_stack << param.rest
+                param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
                 param_stack.concat(param.lefts.reverse)
               when RequiredParameterNode
                 sorted << param.name
               when SplatNode
-                sorted << param.expression.name if param.expression
+                sorted << param.expression.name
               end
             end
 
+            if params.block
+              sorted << (params.block.name || :&)
+            end
+
             names = sorted.concat(names - sorted)
           end
 

data/lib/prism/dot_visitor.rb

@@ -353,10 +353,8 @@ module Prism
       digraph.edge("#{id}:key -> #{node_id(node.key)};")
 
       # value
-      unless (value = node.value).nil?
-        table.field("value", port: true)
-        digraph.edge("#{id}:value -> #{node_id(value)};")
-      end
+      table.field("value", port: true)
+      digraph.edge("#{id}:value -> #{node_id(node.value)};")
 
       # operator_loc
       unless (operator_loc = node.operator_loc).nil?
@@ -488,6 +486,9 @@ module Prism
       table = Table.new("BlockLocalVariableNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -508,9 +509,6 @@ module Prism
       # locals
       table.field("locals", node.locals.inspect)
 
-      # locals_body_index
-      table.field("locals_body_index", node.locals_body_index.inspect)
-
       # parameters
       unless (parameters = node.parameters).nil?
         table.field("parameters", port: true)
@@ -543,6 +541,9 @@ module Prism
       table = Table.new("BlockParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -1501,9 +1502,6 @@ module Prism
       # locals
       table.field("locals", node.locals.inspect)
 
-      # locals_body_index
-      table.field("locals_body_index", node.locals_body_index.inspect)
-
       # def_keyword_loc
       table.field("def_keyword_loc", location_inspect(node.def_keyword_loc))
 
@@ -2805,6 +2803,9 @@ module Prism
       table = Table.new("KeywordRestParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -2833,9 +2834,6 @@ module Prism
       # locals
       table.field("locals", node.locals.inspect)
 
-      # locals_body_index
-      table.field("locals_body_index", node.locals_body_index.inspect)
-
       # operator_loc
       table.field("operator_loc", location_inspect(node.operator_loc))
 
@@ -3404,6 +3402,9 @@ module Prism
       table = Table.new("OptionalKeywordParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -3428,6 +3429,9 @@ module Prism
       table = Table.new("OptionalParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -3810,6 +3814,9 @@ module Prism
       table = Table.new("RequiredKeywordParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -3830,6 +3837,9 @@ module Prism
       table = Table.new("RequiredParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -3925,6 +3935,9 @@ module Prism
       table = Table.new("RestParameterNode")
       id = node_id(node)
 
+      # flags
+      table.field("flags", parameter_flags_inspect(node))
+
       # name
       table.field("name", node.name.inspect)
 
@@ -4524,6 +4537,7 @@ module Prism
       flags << "safe_navigation" if node.safe_navigation?
       flags << "variable_call" if node.variable_call?
       flags << "attribute_write" if node.attribute_write?
+      flags << "ignore_visibility" if node.ignore_visibility?
       flags.join(", ")
     end
 
@@ -4551,7 +4565,7 @@ module Prism
     # comma-separated list.
     def keyword_hash_node_flags_inspect(node)
       flags = []
-      flags << "static_keys" if node.static_keys?
+      flags << "symbol_keys" if node.symbol_keys?
       flags.join(", ")
     end
 
@@ -4563,6 +4577,14 @@ module Prism
       flags.join(", ")
     end
 
+    # Inspect a node that has parameter_flags flags to display the flags as a
+    # comma-separated list.
+    def parameter_flags_inspect(node)
+      flags = []
+      flags << "repeated_parameter" if node.repeated_parameter?
+      flags.join(", ")
+    end
+
     # Inspect a node that has range_flags flags to display the flags as a
     # comma-separated list.
     def range_flags_inspect(node)