prism 0.23.0 → 0.25.0

data/include/prism/parser.h

@@ -6,8 +6,8 @@
 #ifndef PRISM_PARSER_H
 #define PRISM_PARSER_H
 
-#include "prism/ast.h"
 #include "prism/defines.h"
+#include "prism/ast.h"
 #include "prism/encoding.h"
 #include "prism/options.h"
 #include "prism/util/pm_constant_pool.h"
@@ -173,7 +173,7 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the regular expression.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } regexp;
 
         struct {
@@ -206,7 +206,7 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the string.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } string;
 
         struct {
@@ -234,6 +234,9 @@ typedef struct pm_lex_mode {
              * a tilde heredoc.
              */
             size_t common_whitespace;
+
+            /** True if the previous token ended with a line continuation. */
+            bool line_continuation;
         } heredoc;
     } as;
 
@@ -265,12 +268,30 @@ typedef enum {
     /** a begin statement */
     PM_CONTEXT_BEGIN,
 
+    /** an ensure statement with an explicit begin */
+    PM_CONTEXT_BEGIN_ENSURE,
+
+    /** a rescue else statement with an explicit begin */
+    PM_CONTEXT_BEGIN_ELSE,
+
+    /** a rescue statement with an explicit begin */
+    PM_CONTEXT_BEGIN_RESCUE,
+
     /** expressions in block arguments using braces */
     PM_CONTEXT_BLOCK_BRACES,
 
     /** expressions in block arguments using do..end */
     PM_CONTEXT_BLOCK_KEYWORDS,
 
+    /** an ensure statement within a do..end block */
+    PM_CONTEXT_BLOCK_ENSURE,
+
+    /** a rescue else statement within a do..end block */
+    PM_CONTEXT_BLOCK_ELSE,
+
+    /** a rescue statement within a do..end block */
+    PM_CONTEXT_BLOCK_RESCUE,
+
     /** a case when statements */
     PM_CONTEXT_CASE_WHEN,
 
@@ -280,12 +301,33 @@ typedef enum {
     /** a class declaration */
     PM_CONTEXT_CLASS,
 
+    /** an ensure statement within a class statement */
+    PM_CONTEXT_CLASS_ENSURE,
+
+    /** a rescue else statement within a class statement */
+    PM_CONTEXT_CLASS_ELSE,
+
+    /** a rescue statement within a class statement */
+    PM_CONTEXT_CLASS_RESCUE,
+
     /** a method definition */
     PM_CONTEXT_DEF,
 
+    /** an ensure statement within a method definition */
+    PM_CONTEXT_DEF_ENSURE,
+
+    /** a rescue else statement within a method definition */
+    PM_CONTEXT_DEF_ELSE,
+
+    /** a rescue statement within a method definition */
+    PM_CONTEXT_DEF_RESCUE,
+
     /** a method definition's parameters */
     PM_CONTEXT_DEF_PARAMS,
 
+    /** a defined? expression */
+    PM_CONTEXT_DEFINED,
+
     /** a method definition's default parameter */
     PM_CONTEXT_DEFAULT_PARAMS,
 
@@ -298,12 +340,6 @@ typedef enum {
     /** an interpolated expression */
     PM_CONTEXT_EMBEXPR,
 
-    /** an ensure statement */
-    PM_CONTEXT_ENSURE,
-
-    /** an ensure statement within a method definition */
-    PM_CONTEXT_ENSURE_DEF,
-
     /** a for loop */
     PM_CONTEXT_FOR,
 
@@ -319,12 +355,30 @@ typedef enum {
     /** a lambda expression with do..end */
     PM_CONTEXT_LAMBDA_DO_END,
 
+    /** an ensure statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_ENSURE,
+
+    /** a rescue else statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_ELSE,
+
+    /** a rescue statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_RESCUE,
+
     /** the top level context */
     PM_CONTEXT_MAIN,
 
     /** a module declaration */
     PM_CONTEXT_MODULE,
 
+    /** an ensure statement within a module statement */
+    PM_CONTEXT_MODULE_ENSURE,
+
+    /** a rescue else statement within a module statement */
+    PM_CONTEXT_MODULE_ELSE,
+
+    /** a rescue statement within a module statement */
+    PM_CONTEXT_MODULE_RESCUE,
+
     /** a parenthesized expression */
     PM_CONTEXT_PARENS,
 
@@ -337,20 +391,23 @@ typedef enum {
     /** a BEGIN block */
     PM_CONTEXT_PREEXE,
 
-    /** a rescue else statement */
-    PM_CONTEXT_RESCUE_ELSE,
+    /** a modifier rescue clause */
+    PM_CONTEXT_RESCUE_MODIFIER,
 
-    /** a rescue else statement within a method definition */
-    PM_CONTEXT_RESCUE_ELSE_DEF,
+    /** a singleton class definition */
+    PM_CONTEXT_SCLASS,
 
-    /** a rescue statement */
-    PM_CONTEXT_RESCUE,
+    /** an ensure statement with a singleton class */
+    PM_CONTEXT_SCLASS_ENSURE,
 
-    /** a rescue statement within a method definition */
-    PM_CONTEXT_RESCUE_DEF,
+    /** a rescue else statement with a singleton class */
+    PM_CONTEXT_SCLASS_ELSE,
 
-    /** a singleton class definition */
-    PM_CONTEXT_SCLASS,
+    /** a rescue statement with a singleton class */
+    PM_CONTEXT_SCLASS_RESCUE,
+
+    /** a ternary expression */
+    PM_CONTEXT_TERNARY,
 
     /** an unless statement */
     PM_CONTEXT_UNLESS,
@@ -445,42 +502,80 @@ typedef struct {
     void (*callback)(void *data, pm_parser_t *parser, pm_token_t *token);
 } pm_lex_callback_t;
 
+/** The type of shareable constant value that can be set. */
+typedef uint8_t pm_shareable_constant_value_t;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = 0x1;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = 0x2;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = 0x4;
+
+/**
+ * This tracks an individual local variable in a certain lexical context, as
+ * well as the number of times is it read.
+ */
+typedef struct {
+    /** The name of the local variable. */
+    pm_constant_id_t name;
+
+    /** The location of the local variable in the source. */
+    pm_location_t location;
+
+    /** The index of the local variable in the local table. */
+    uint32_t index;
+
+    /** The number of times the local variable is read. */
+    uint32_t reads;
+
+    /** The hash of the local variable. */
+    uint32_t hash;
+} pm_local_t;
+
+/**
+ * This is a set of local variables in a certain lexical context (method, class,
+ * module, etc.). We need to track how many times these variables are read in
+ * order to warn if they only get written.
+ */
+typedef struct pm_locals {
+    /** The number of local variables in the set. */
+    uint32_t size;
+
+    /** The capacity of the local variables set. */
+    uint32_t capacity;
+
+    /** The nullable allocated memory for the local variables in the set. */
+    pm_local_t *locals;
+} pm_locals_t;
+
 /**
  * This struct represents a node in a linked list of scopes. Some scopes can see
  * into their parent scopes, while others cannot.
  */
 typedef struct pm_scope {
-    /** The IDs of the locals in the given scope. */
-    pm_constant_id_list_t locals;
-
     /** A pointer to the previous scope in the linked list. */
     struct pm_scope *previous;
 
-    /**
-     * A boolean indicating whether or not this scope can see into its parent.
-     * If closed is true, then the scope cannot see into its parent.
-     */
-    bool closed;
-
-    /**
-     * A boolean indicating whether or not this scope has explicit parameters.
-     * This is necessary to determine whether or not numbered parameters are
-     * allowed.
-     */
-    bool explicit_params;
+    /** The IDs of the locals in the given scope. */
+    pm_locals_t locals;
 
     /**
-     * Booleans indicating whether the parameters for this scope have declared
-     * forwarding parameters.
+     * This is a bitfield that indicates the parameters that are being used in
+     * this scope. It is a combination of the PM_SCOPE_PARAMS_* constants. There
+     * are three different kinds of parameters that can be used in a scope:
      *
-     * For example, some combinations of:
-     *   def foo(*); end
-     *   def foo(**); end
-     *   def foo(&); end
-     *   def foo(...); end
+     * - Ordinary parameters (e.g., def foo(bar); end)
+     * - Numbered parameters (e.g., def foo; _1; end)
+     * - The it parameter (e.g., def foo; it; end)
+     *
+     * If ordinary parameters are being used, then certain parameters can be
+     * forwarded to another method/structure. Those are indicated by four
+     * additional bits in the params field. For example, some combinations of:
+     *
+     * - def foo(*); end
+     * - def foo(**); end
+     * - def foo(&); end
+     * - def foo(...); end
      */
-
-    uint8_t forwarding_params;
+    uint8_t parameters;
 
     /**
      * An integer indicating the number of numbered parameters on this scope.
@@ -488,13 +583,34 @@ typedef struct pm_scope {
      * numbered parameters, and to pass information to consumers of the AST
      * about how many numbered parameters exist.
      */
-    uint8_t numbered_parameters;
+    int8_t numbered_parameters;
+
+    /**
+     * The current state of constant shareability for this scope. This is
+     * changed by magic shareable_constant_value comments.
+     */
+    pm_shareable_constant_value_t shareable_constant;
+
+    /**
+     * A boolean indicating whether or not this scope can see into its parent.
+     * If closed is true, then the scope cannot see into its parent.
+     */
+    bool closed;
 } pm_scope_t;
 
-static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
-static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
-static const uint8_t PM_FORWARDING_BLOCK = 0x4;
-static const uint8_t PM_FORWARDING_ALL = 0x8;
+static const uint8_t PM_SCOPE_PARAMETERS_NONE = 0x0;
+static const uint8_t PM_SCOPE_PARAMETERS_ORDINARY = 0x1;
+static const uint8_t PM_SCOPE_PARAMETERS_NUMBERED = 0x2;
+static const uint8_t PM_SCOPE_PARAMETERS_IT = 0x4;
+static const uint8_t PM_SCOPE_PARAMETERS_TYPE_MASK = 0x7;
+
+static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x8;
+static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x10;
+static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x20;
+static const uint8_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x40;
+
+static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_DISALLOWED = -1;
+static const int8_t PM_SCOPE_NUMBERED_PARAMETERS_NONE = 0;
 
 /**
  * This struct represents the overall parser. It contains a reference to the
@@ -688,12 +804,41 @@ struct pm_parser {
      */
     const pm_encoding_t *explicit_encoding;
 
-    /** The current parameter name id on parsing its default value. */
-    pm_constant_id_t current_param_name;
+    /**
+     * When parsing block exits (e.g., break, next, redo), we need to validate
+     * that they are in correct contexts. For the most part we can do this by
+     * looking at our parent contexts. However, modifier while and until
+     * expressions can change that context to make block exits valid. In these
+     * cases, we need to keep track of the block exits and then validate them
+     * after the expression has been parsed.
+     *
+     * We use a pointer here because we don't want to keep a whole list attached
+     * since this will only be used in the context of begin/end expressions.
+     */
+    pm_node_list_t *current_block_exits;
 
     /** The version of prism that we should use to parse. */
     pm_options_version_t version;
 
+    /** The command line flags given from the options. */
+    uint8_t command_line;
+
+    /**
+     * Whether or not we have found a frozen_string_literal magic comment with
+     * a true or false value.
+     * May be:
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
+     */
+    int8_t frozen_string_literal;
+
+    /**
+     * Whether or not we are parsing an eval string. This impacts whether or not
+     * we should evaluate if block exits/yields are valid.
+     */
+    bool parsing_eval;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 
@@ -723,10 +868,10 @@ struct pm_parser {
     bool semantic_token_seen;
 
     /**
-     * Whether or not we have found a frozen_string_literal magic comment with
-     * a true value.
+     * True if the current regular expression being lexed contains only ASCII
+     * characters.
      */
-    bool frozen_string_literal;
+    bool current_regular_expression_ascii_only;
 };
 
 #endif

data/include/prism/prettyprint.h

@@ -8,6 +8,12 @@
 
 #include "prism/defines.h"
 
+#ifdef PRISM_EXCLUDE_PRETTYPRINT
+
+void pm_prettyprint(void);
+
+#else
+
 #include <stdio.h>
 
 #include "prism/ast.h"
@@ -24,3 +30,5 @@
 PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
 
 #endif
+
+#endif

data/include/prism/static_literals.h

@@ -0,0 +1,118 @@
+/**
+ * @file static_literals.h
+ *
+ * A set of static literal nodes that can be checked for duplicates.
+ */
+#ifndef PRISM_STATIC_LITERALS_H
+#define PRISM_STATIC_LITERALS_H
+
+#include "prism/defines.h"
+#include "prism/ast.h"
+#include "prism/node.h"
+#include "prism/parser.h"
+
+#include <assert.h>
+#include <stdbool.h>
+
+/**
+ * An internal hash table for a set of nodes.
+ */
+typedef struct {
+    /** The array of nodes in the hash table. */
+    pm_node_t **nodes;
+
+    /** The size of the hash table. */
+    uint32_t size;
+
+    /** The space that has been allocated in the hash table. */
+    uint32_t capacity;
+} pm_node_hash_t;
+
+/**
+ * Certain sets of nodes (hash keys and when clauses) check for duplicate nodes
+ * to alert the user of potential issues. To do this, we keep a set of the nodes
+ * that have been seen so far, and compare whenever we find a new node.
+ *
+ * We bucket the nodes based on their type to minimize the number of comparisons
+ * that need to be performed.
+ */
+typedef struct {
+    /**
+     * This is the set of IntegerNode and SourceLineNode instances.
+     */
+    pm_node_hash_t integer_nodes;
+
+    /**
+     * This is the set of FloatNode instances.
+     */
+    pm_node_hash_t float_nodes;
+
+    /**
+     * This is the set of RationalNode and ImaginaryNode instances.
+     */
+    pm_node_hash_t number_nodes;
+
+    /**
+     * This is the set of StringNode and SourceFileNode instances.
+     */
+    pm_node_hash_t string_nodes;
+
+    /**
+     * This is the set of RegularExpressionNode instances.
+     */
+    pm_node_hash_t regexp_nodes;
+
+    /**
+     * This is the set of SymbolNode instances.
+     */
+    pm_node_hash_t symbol_nodes;
+
+    /**
+     * A pointer to the last TrueNode instance that was inserted, or NULL.
+     */
+    pm_node_t *true_node;
+
+    /**
+     * A pointer to the last FalseNode instance that was inserted, or NULL.
+     */
+    pm_node_t *false_node;
+
+    /**
+     * A pointer to the last NilNode instance that was inserted, or NULL.
+     */
+    pm_node_t *nil_node;
+
+    /**
+     * A pointer to the last SourceEncodingNode instance that was inserted, or
+     * NULL.
+     */
+    pm_node_t *source_encoding_node;
+} pm_static_literals_t;
+
+/**
+ * Add a node to the set of static literals.
+ *
+ * @param parser The parser that created the node.
+ * @param literals The set of static literals to add the node to.
+ * @param node The node to add to the set.
+ * @return A pointer to the node that is being overwritten, if there is one.
+ */
+pm_node_t * pm_static_literals_add(const pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node);
+
+/**
+ * Free the internal memory associated with the given static literals set.
+ *
+ * @param literals The set of static literals to free.
+ */
+void pm_static_literals_free(pm_static_literals_t *literals);
+
+/**
+ * Create a string-based representation of the given static literal.
+ *
+ * @param buffer The buffer to write the string to.
+ * @param parser The parser that created the node.
+ * @param node The node to create a string representation of.
+ */
+PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
+
+#endif

data/include/prism/util/pm_buffer.h

@@ -7,6 +7,7 @@
 #define PRISM_BUFFER_H
 
 #include "prism/defines.h"
+#include "prism/util/pm_char.h"
 
 #include <assert.h>
 #include <stdbool.h>
@@ -59,7 +60,7 @@ PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
  * @param buffer The buffer to get the value of.
  * @returns The value of the buffer.
  */
-PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
+PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer);
 
 /**
  * Return the length of the buffer.
@@ -67,7 +68,7 @@ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
  * @param buffer The buffer to get the length of.
  * @returns The length of the buffer.
  */
-PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
+PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(const pm_buffer_t *buffer);
 
 /**
  * Append the given amount of space as zeroes to the buffer.
@@ -128,6 +129,33 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
  */
 void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
 
+/**
+ * Append a double to the buffer.
+ *
+ * @param buffer The buffer to append to.
+ * @param value The double to append.
+ */
+void pm_buffer_append_double(pm_buffer_t *buffer, double value);
+
+/**
+ * The different types of escaping that can be performed by the buffer when
+ * appending a slice of Ruby source code.
+ */
+typedef enum {
+    PM_BUFFER_ESCAPING_RUBY,
+    PM_BUFFER_ESCAPING_JSON
+} pm_buffer_escaping_t;
+
+/**
+ * Append a slice of source code to the buffer.
+ *
+ * @param buffer The buffer to append to.
+ * @param source The source code to append.
+ * @param length The length of the source code to append.
+ * @param escaping The type of escaping to perform.
+ */
+void pm_buffer_append_source(pm_buffer_t *buffer, const uint8_t *source, size_t length, pm_buffer_escaping_t escaping);
+
 /**
  * Prepend the given string to the buffer.
  *
@@ -145,6 +173,41 @@ void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t len
  */
 void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
 
+/**
+ * Clear the buffer by reducing its size to 0. This does not free the allocated
+ * memory, but it does allow the buffer to be reused.
+ *
+ * @param buffer The buffer to clear.
+ */
+void pm_buffer_clear(pm_buffer_t *buffer);
+
+/**
+ * Strip the whitespace from the end of the buffer.
+ *
+ * @param buffer The buffer to strip.
+ */
+void pm_buffer_rstrip(pm_buffer_t *buffer);
+
+/**
+ * Checks if the buffer includes the given value.
+ *
+ * @param buffer The buffer to check.
+ * @param value The value to check for.
+ * @returns The index of the first occurrence of the value in the buffer, or
+ *   SIZE_MAX if the value is not found.
+ */
+size_t pm_buffer_index(const pm_buffer_t *buffer, char value);
+
+/**
+ * Insert the given string into the buffer at the given index.
+ *
+ * @param buffer The buffer to insert into.
+ * @param index The index to insert at.
+ * @param value The string to insert.
+ * @param length The length of the string to insert.
+ */
+void pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size_t length);
+
 /**
  * Free the memory associated with the buffer.
  *

data/include/prism/util/pm_constant_pool.h

@@ -51,6 +51,14 @@ typedef struct {
  */
 void pm_constant_id_list_init(pm_constant_id_list_t *list);
 
+/**
+ * Initialize a list of constant ids with a given capacity.
+ *
+ * @param list The list to initialize.
+ * @param capacity The initial capacity of the list.
+ */
+void pm_constant_id_list_init_capacity(pm_constant_id_list_t *list, size_t capacity);
+
 /**
  * Append a constant id to a list of constant ids. Returns false if any
  * potential reallocations fail.
@@ -61,6 +69,15 @@ void pm_constant_id_list_init(pm_constant_id_list_t *list);
  */
 bool pm_constant_id_list_append(pm_constant_id_list_t *list, pm_constant_id_t id);
 
+/**
+ * Insert a constant id into a list of constant ids at the specified index.
+ *
+ * @param list The list to insert into.
+ * @param index The index at which to insert.
+ * @param id The id to insert.
+ */
+void pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_constant_id_t id);
+
 /**
  * Checks if the current constant id list includes the given constant id.
  *
@@ -186,7 +203,7 @@ pm_constant_id_t pm_constant_pool_insert_shared(pm_constant_pool_t *pool, const
  * @param length The length of the constant.
  * @return The id of the constant.
  */
-pm_constant_id_t pm_constant_pool_insert_owned(pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+pm_constant_id_t pm_constant_pool_insert_owned(pm_constant_pool_t *pool, uint8_t *start, size_t length);
 
 /**
  * Insert a constant into a constant pool from memory that is constant. Returns