jruby-prism-parser 0.24.0-java → 1.4.0-java

data/include/prism/parser.h

@@ -6,14 +6,14 @@
 #ifndef PRISM_PARSER_H
 #define PRISM_PARSER_H
 
-#include "prism/ast.h"
 #include "prism/defines.h"
+#include "prism/ast.h"
 #include "prism/encoding.h"
 #include "prism/options.h"
+#include "prism/static_literals.h"
 #include "prism/util/pm_constant_pool.h"
 #include "prism/util/pm_list.h"
 #include "prism/util/pm_newline_list.h"
-#include "prism/util/pm_state_stack.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
@@ -82,6 +82,23 @@ typedef enum {
     PM_HEREDOC_INDENT_TILDE,
 } pm_heredoc_indent_t;
 
+/**
+ * All of the information necessary to store to lexing a heredoc.
+ */
+typedef struct {
+    /** A pointer to the start of the heredoc identifier. */
+    const uint8_t *ident_start;
+
+    /** The length of the heredoc identifier. */
+    size_t ident_length;
+
+    /** The type of quote that the heredoc uses. */
+    pm_heredoc_quote_t quote;
+
+    /** The type of indentation that the heredoc uses. */
+    pm_heredoc_indent_t indent;
+} pm_heredoc_lex_mode_t;
+
 /**
  * When lexing Ruby source, the lexer has a small amount of state to tell which
  * kind of token it is currently lexing. For example, when we find the start of
@@ -173,7 +190,7 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the regular expression.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } regexp;
 
         struct {
@@ -206,21 +223,14 @@ typedef struct pm_lex_mode {
              * This is the character set that should be used to delimit the
              * tokens within the string.
              */
-            uint8_t breakpoints[6];
+            uint8_t breakpoints[7];
         } string;
 
         struct {
-            /** A pointer to the start of the heredoc identifier. */
-            const uint8_t *ident_start;
-
-            /** The length of the heredoc identifier. */
-            size_t ident_length;
-
-            /** The type of quote that the heredoc uses. */
-            pm_heredoc_quote_t quote;
-
-            /** The type of indentation that the heredoc uses. */
-            pm_heredoc_indent_t indent;
+            /**
+             * All of the data necessary to lex a heredoc.
+             */
+            pm_heredoc_lex_mode_t base;
 
             /**
              * This is the pointer to the character where lexing should resume
@@ -233,7 +243,10 @@ typedef struct pm_lex_mode {
              * line so that we know how much to dedent each line in the case of
              * a tilde heredoc.
              */
-            size_t common_whitespace;
+            size_t *common_whitespace;
+
+            /** True if the previous token ended with a line continuation. */
+            bool line_continuation;
         } heredoc;
     } as;
 
@@ -265,12 +278,30 @@ typedef enum {
     /** a begin statement */
     PM_CONTEXT_BEGIN,
 
+    /** an ensure statement with an explicit begin */
+    PM_CONTEXT_BEGIN_ENSURE,
+
+    /** a rescue else statement with an explicit begin */
+    PM_CONTEXT_BEGIN_ELSE,
+
+    /** a rescue statement with an explicit begin */
+    PM_CONTEXT_BEGIN_RESCUE,
+
     /** expressions in block arguments using braces */
     PM_CONTEXT_BLOCK_BRACES,
 
     /** expressions in block arguments using do..end */
     PM_CONTEXT_BLOCK_KEYWORDS,
 
+    /** an ensure statement within a do..end block */
+    PM_CONTEXT_BLOCK_ENSURE,
+
+    /** a rescue else statement within a do..end block */
+    PM_CONTEXT_BLOCK_ELSE,
+
+    /** a rescue statement within a do..end block */
+    PM_CONTEXT_BLOCK_RESCUE,
+
     /** a case when statements */
     PM_CONTEXT_CASE_WHEN,
 
@@ -280,12 +311,33 @@ typedef enum {
     /** a class declaration */
     PM_CONTEXT_CLASS,
 
+    /** an ensure statement within a class statement */
+    PM_CONTEXT_CLASS_ENSURE,
+
+    /** a rescue else statement within a class statement */
+    PM_CONTEXT_CLASS_ELSE,
+
+    /** a rescue statement within a class statement */
+    PM_CONTEXT_CLASS_RESCUE,
+
     /** a method definition */
     PM_CONTEXT_DEF,
 
+    /** an ensure statement within a method definition */
+    PM_CONTEXT_DEF_ENSURE,
+
+    /** a rescue else statement within a method definition */
+    PM_CONTEXT_DEF_ELSE,
+
+    /** a rescue statement within a method definition */
+    PM_CONTEXT_DEF_RESCUE,
+
     /** a method definition's parameters */
     PM_CONTEXT_DEF_PARAMS,
 
+    /** a defined? expression */
+    PM_CONTEXT_DEFINED,
+
     /** a method definition's default parameter */
     PM_CONTEXT_DEFAULT_PARAMS,
 
@@ -298,12 +350,6 @@ typedef enum {
     /** an interpolated expression */
     PM_CONTEXT_EMBEXPR,
 
-    /** an ensure statement */
-    PM_CONTEXT_ENSURE,
-
-    /** an ensure statement within a method definition */
-    PM_CONTEXT_ENSURE_DEF,
-
     /** a for loop */
     PM_CONTEXT_FOR,
 
@@ -319,12 +365,36 @@ typedef enum {
     /** a lambda expression with do..end */
     PM_CONTEXT_LAMBDA_DO_END,
 
+    /** an ensure statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_ENSURE,
+
+    /** a rescue else statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_ELSE,
+
+    /** a rescue statement within a lambda expression */
+    PM_CONTEXT_LAMBDA_RESCUE,
+
+    /** the predicate clause of a loop statement */
+    PM_CONTEXT_LOOP_PREDICATE,
+
     /** the top level context */
     PM_CONTEXT_MAIN,
 
     /** a module declaration */
     PM_CONTEXT_MODULE,
 
+    /** an ensure statement within a module statement */
+    PM_CONTEXT_MODULE_ENSURE,
+
+    /** a rescue else statement within a module statement */
+    PM_CONTEXT_MODULE_ELSE,
+
+    /** a rescue statement within a module statement */
+    PM_CONTEXT_MODULE_RESCUE,
+
+    /** a multiple target expression */
+    PM_CONTEXT_MULTI_TARGET,
+
     /** a parenthesized expression */
     PM_CONTEXT_PARENS,
 
@@ -337,20 +407,23 @@ typedef enum {
     /** a BEGIN block */
     PM_CONTEXT_PREEXE,
 
-    /** a rescue else statement */
-    PM_CONTEXT_RESCUE_ELSE,
+    /** a modifier rescue clause */
+    PM_CONTEXT_RESCUE_MODIFIER,
 
-    /** a rescue else statement within a method definition */
-    PM_CONTEXT_RESCUE_ELSE_DEF,
+    /** a singleton class definition */
+    PM_CONTEXT_SCLASS,
 
-    /** a rescue statement */
-    PM_CONTEXT_RESCUE,
+    /** an ensure statement with a singleton class */
+    PM_CONTEXT_SCLASS_ENSURE,
 
-    /** a rescue statement within a method definition */
-    PM_CONTEXT_RESCUE_DEF,
+    /** a rescue else statement with a singleton class */
+    PM_CONTEXT_SCLASS_ELSE,
 
-    /** a singleton class definition */
-    PM_CONTEXT_SCLASS,
+    /** a rescue statement with a singleton class */
+    PM_CONTEXT_SCLASS_RESCUE,
+
+    /** a ternary expression */
+    PM_CONTEXT_TERNARY,
 
     /** an unless statement */
     PM_CONTEXT_UNLESS,
@@ -445,56 +518,118 @@ typedef struct {
     void (*callback)(void *data, pm_parser_t *parser, pm_token_t *token);
 } pm_lex_callback_t;
 
+/** The type of shareable constant value that can be set. */
+typedef uint8_t pm_shareable_constant_value_t;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_NONE = 0x0;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_LITERAL = PM_SHAREABLE_CONSTANT_NODE_FLAGS_LITERAL;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_EVERYTHING = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_EVERYTHING;
+static const pm_shareable_constant_value_t PM_SCOPE_SHAREABLE_CONSTANT_EXPERIMENTAL_COPY = PM_SHAREABLE_CONSTANT_NODE_FLAGS_EXPERIMENTAL_COPY;
+
+/**
+ * This tracks an individual local variable in a certain lexical context, as
+ * well as the number of times is it read.
+ */
+typedef struct {
+    /** The name of the local variable. */
+    pm_constant_id_t name;
+
+    /** The location of the local variable in the source. */
+    pm_location_t location;
+
+    /** The index of the local variable in the local table. */
+    uint32_t index;
+
+    /** The number of times the local variable is read. */
+    uint32_t reads;
+
+    /** The hash of the local variable. */
+    uint32_t hash;
+} pm_local_t;
+
+/**
+ * This is a set of local variables in a certain lexical context (method, class,
+ * module, etc.). We need to track how many times these variables are read in
+ * order to warn if they only get written.
+ */
+typedef struct pm_locals {
+    /** The number of local variables in the set. */
+    uint32_t size;
+
+    /** The capacity of the local variables set. */
+    uint32_t capacity;
+
+    /** The nullable allocated memory for the local variables in the set. */
+    pm_local_t *locals;
+} pm_locals_t;
+
+/** The flags about scope parameters that can be set. */
+typedef uint8_t pm_scope_parameters_t;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NONE = 0x0;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS = 0x1;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS = 0x2;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_BLOCK = 0x4;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_FORWARDING_ALL = 0x8;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_IMPLICIT_DISALLOWED = 0x10;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_INNER = 0x20;
+static const pm_scope_parameters_t PM_SCOPE_PARAMETERS_NUMBERED_FOUND = 0x40;
+
 /**
  * This struct represents a node in a linked list of scopes. Some scopes can see
  * into their parent scopes, while others cannot.
  */
 typedef struct pm_scope {
-    /** The IDs of the locals in the given scope. */
-    pm_constant_id_list_t locals;
-
     /** A pointer to the previous scope in the linked list. */
     struct pm_scope *previous;
 
-    /**
-     * A boolean indicating whether or not this scope can see into its parent.
-     * If closed is true, then the scope cannot see into its parent.
-     */
-    bool closed;
+    /** The IDs of the locals in the given scope. */
+    pm_locals_t locals;
 
     /**
-     * A boolean indicating whether or not this scope has explicit parameters.
-     * This is necessary to determine whether or not numbered parameters are
-     * allowed.
+     * This is a list of the implicit parameters contained within the block.
+     * These will be processed after the block is parsed to determine the kind
+     * of parameters node that should be used and to check if any errors need to
+     * be added.
      */
-    bool explicit_params;
+    pm_node_list_t implicit_parameters;
 
     /**
-     * Booleans indicating whether the parameters for this scope have declared
-     * forwarding parameters.
+     * This is a bitfield that indicates the parameters that are being used in
+     * this scope. It is a combination of the PM_SCOPE_PARAMETERS_* constants.
+     * There are three different kinds of parameters that can be used in a
+     * scope:
+     *
+     * - Ordinary parameters (e.g., def foo(bar); end)
+     * - Numbered parameters (e.g., def foo; _1; end)
+     * - The it parameter (e.g., def foo; it; end)
      *
-     * For example, some combinations of:
-     *   def foo(*); end
-     *   def foo(**); end
-     *   def foo(&); end
-     *   def foo(...); end
+     * If ordinary parameters are being used, then certain parameters can be
+     * forwarded to another method/structure. Those are indicated by four
+     * additional bits in the params field. For example, some combinations of:
+     *
+     * - def foo(*); end
+     * - def foo(**); end
+     * - def foo(&); end
+     * - def foo(...); end
      */
+    pm_scope_parameters_t parameters;
 
-    uint8_t forwarding_params;
+    /**
+     * The current state of constant shareability for this scope. This is
+     * changed by magic shareable_constant_value comments.
+     */
+    pm_shareable_constant_value_t shareable_constant;
 
     /**
-     * An integer indicating the number of numbered parameters on this scope.
-     * This is necessary to determine if child blocks are allowed to use
-     * numbered parameters, and to pass information to consumers of the AST
-     * about how many numbered parameters exist.
+     * A boolean indicating whether or not this scope can see into its parent.
+     * If closed is true, then the scope cannot see into its parent.
      */
-    uint8_t numbered_parameters;
+    bool closed;
 } pm_scope_t;
 
-static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
-static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
-static const uint8_t PM_FORWARDING_BLOCK = 0x4;
-static const uint8_t PM_FORWARDING_ALL = 0x8;
+/**
+ * A struct that represents a stack of boolean values.
+ */
+typedef uint32_t pm_state_stack_t;
 
 /**
  * This struct represents the overall parser. It contains a reference to the
@@ -503,6 +638,13 @@ static const uint8_t PM_FORWARDING_ALL = 0x8;
  * it's considering.
  */
 struct pm_parser {
+    /**
+     * The next node identifier that will be assigned. This is a unique
+     * identifier used to track nodes such that the syntax tree can be dropped
+     * but the node can be found through another parse.
+     */
+    uint32_t node_id;
+
     /** The current state of the lexer. */
     pm_lex_state_t lex_state;
 
@@ -597,6 +739,15 @@ struct pm_parser {
     /** The current parsing context. */
     pm_context_node_t *current_context;
 
+    /**
+     * The hash keys for the hash that is currently being parsed. This is not
+     * usually necessary because it can pass it down the various call chains,
+     * but in the event that you're parsing a hash that is being directly
+     * pushed into another hash with **, we need to share the hash keys so that
+     * we can warn for the nested hash as well.
+     */
+    pm_static_literals_t *current_hash_keys;
+
     /**
      * The encoding functions for the current file is attached to the parser as
      * it's parsing so that it can change with a magic comment.
@@ -688,18 +839,62 @@ struct pm_parser {
      */
     const pm_encoding_t *explicit_encoding;
 
-    /** The current parameter name id on parsing its default value. */
-    pm_constant_id_t current_param_name;
+    /**
+     * When parsing block exits (e.g., break, next, redo), we need to validate
+     * that they are in correct contexts. For the most part we can do this by
+     * looking at our parent contexts. However, modifier while and until
+     * expressions can change that context to make block exits valid. In these
+     * cases, we need to keep track of the block exits and then validate them
+     * after the expression has been parsed.
+     *
+     * We use a pointer here because we don't want to keep a whole list attached
+     * since this will only be used in the context of begin/end expressions.
+     */
+    pm_node_list_t *current_block_exits;
 
     /** The version of prism that we should use to parse. */
     pm_options_version_t version;
 
+    /** The command line flags given from the options. */
+    uint8_t command_line;
+
+    /**
+     * Whether or not we have found a frozen_string_literal magic comment with
+     * a true or false value.
+     * May be:
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED
+     *  - PM_OPTIONS_FROZEN_STRING_LITERAL_UNSET
+     */
+    int8_t frozen_string_literal;
+
+    /**
+     * Whether or not we are parsing an eval string. This impacts whether or not
+     * we should evaluate if block exits/yields are valid.
+     */
+    bool parsing_eval;
+
+    /**
+     * Whether or not we are parsing a "partial" script, which is a script that
+     * will be evaluated in the context of another script, so we should not
+     * check jumps (next/break/etc.) for validity.
+     */
+    bool partial_script;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 
     /** Whether or not we're currently recovering from a syntax error. */
     bool recovering;
 
+    /**
+     * This is very specialized behavior for when you want to parse in a context
+     * that does not respect encoding comments. Its main use case is translating
+     * into the whitequark/parser AST which re-encodes source files in UTF-8
+     * before they are parsed and ignores encoding comments.
+     */
+    bool encoding_locked;
+
     /**
      * Whether or not the encoding has been changed by a magic comment. We use
      * this to provide a fast path for the lexer instead of going through the
@@ -723,10 +918,16 @@ struct pm_parser {
     bool semantic_token_seen;
 
     /**
-     * Whether or not we have found a frozen_string_literal magic comment with
-     * a true value.
+     * True if the current regular expression being lexed contains only ASCII
+     * characters.
+     */
+    bool current_regular_expression_ascii_only;
+
+    /**
+     * By default, Ruby always warns about mismatched indentation. This can be
+     * toggled with a magic comment.
      */
-    bool frozen_string_literal;
+    bool warn_mismatched_indentation;
 };
 
 #endif

data/include/prism/prettyprint.h

@@ -8,6 +8,12 @@
 
 #include "prism/defines.h"
 
+#ifdef PRISM_EXCLUDE_PRETTYPRINT
+
+void pm_prettyprint(void);
+
+#else
+
 #include <stdio.h>
 
 #include "prism/ast.h"
@@ -24,3 +30,5 @@
 PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
 
 #endif
+
+#endif

data/include/prism/regexp.h

@@ -10,7 +10,6 @@
 #include "prism/parser.h"
 #include "prism/encoding.h"
 #include "prism/util/pm_memchr.h"
-#include "prism/util/pm_string_list.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
@@ -18,16 +17,27 @@
 #include <string.h>
 
 /**
- * Parse a regular expression and extract the names of all of the named capture
- * groups.
+ * This callback is called when a named capture group is found.
+ */
+typedef void (*pm_regexp_name_callback_t)(const pm_string_t *name, void *data);
+
+/**
+ * This callback is called when a parse error is found.
+ */
+typedef void (*pm_regexp_error_callback_t)(const uint8_t *start, const uint8_t *end, const char *message, void *data);
+
+/**
+ * Parse a regular expression.
  *
+ * @param parser The parser that is currently being used.
  * @param source The source code to parse.
  * @param size The size of the source code.
- * @param named_captures The list to add the names of the named capture groups.
- * @param encoding_changed Whether or not the encoding changed from the default.
- * @param encoding The encoding of the source code.
- * @return Whether or not the parsing was successful.
+ * @param extended_mode Whether to parse the regular expression in extended mode.
+ * @param name_callback The optional callback to call when a named capture group is found.
+ * @param name_data The optional data to pass to the name callback.
+ * @param error_callback The callback to call when a parse error is found.
+ * @param error_data The data to pass to the error callback.
  */
-PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, const pm_encoding_t *encoding);
+PRISM_EXPORTED_FUNCTION void pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data, pm_regexp_error_callback_t error_callback, void *error_data);
 
 #endif

data/include/prism/static_literals.h

@@ -0,0 +1,121 @@
+/**
+ * @file static_literals.h
+ *
+ * A set of static literal nodes that can be checked for duplicates.
+ */
+#ifndef PRISM_STATIC_LITERALS_H
+#define PRISM_STATIC_LITERALS_H
+
+#include "prism/defines.h"
+#include "prism/ast.h"
+#include "prism/util/pm_newline_list.h"
+
+#include <assert.h>
+#include <stdbool.h>
+
+/**
+ * An internal hash table for a set of nodes.
+ */
+typedef struct {
+    /** The array of nodes in the hash table. */
+    pm_node_t **nodes;
+
+    /** The size of the hash table. */
+    uint32_t size;
+
+    /** The space that has been allocated in the hash table. */
+    uint32_t capacity;
+} pm_node_hash_t;
+
+/**
+ * Certain sets of nodes (hash keys and when clauses) check for duplicate nodes
+ * to alert the user of potential issues. To do this, we keep a set of the nodes
+ * that have been seen so far, and compare whenever we find a new node.
+ *
+ * We bucket the nodes based on their type to minimize the number of comparisons
+ * that need to be performed.
+ */
+typedef struct {
+    /**
+     * This is the set of IntegerNode and SourceLineNode instances.
+     */
+    pm_node_hash_t integer_nodes;
+
+    /**
+     * This is the set of FloatNode instances.
+     */
+    pm_node_hash_t float_nodes;
+
+    /**
+     * This is the set of RationalNode and ImaginaryNode instances.
+     */
+    pm_node_hash_t number_nodes;
+
+    /**
+     * This is the set of StringNode and SourceFileNode instances.
+     */
+    pm_node_hash_t string_nodes;
+
+    /**
+     * This is the set of RegularExpressionNode instances.
+     */
+    pm_node_hash_t regexp_nodes;
+
+    /**
+     * This is the set of SymbolNode instances.
+     */
+    pm_node_hash_t symbol_nodes;
+
+    /**
+     * A pointer to the last TrueNode instance that was inserted, or NULL.
+     */
+    pm_node_t *true_node;
+
+    /**
+     * A pointer to the last FalseNode instance that was inserted, or NULL.
+     */
+    pm_node_t *false_node;
+
+    /**
+     * A pointer to the last NilNode instance that was inserted, or NULL.
+     */
+    pm_node_t *nil_node;
+
+    /**
+     * A pointer to the last SourceEncodingNode instance that was inserted, or
+     * NULL.
+     */
+    pm_node_t *source_encoding_node;
+} pm_static_literals_t;
+
+/**
+ * Add a node to the set of static literals.
+ *
+ * @param newline_list The list of newline offsets to use to calculate lines.
+ * @param start_line The line number that the parser starts on.
+ * @param literals The set of static literals to add the node to.
+ * @param node The node to add to the set.
+ * @param replace Whether to replace the previous node if one already exists.
+ * @return A pointer to the node that is being overwritten, if there is one.
+ */
+pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace);
+
+/**
+ * Free the internal memory associated with the given static literals set.
+ *
+ * @param literals The set of static literals to free.
+ */
+void pm_static_literals_free(pm_static_literals_t *literals);
+
+/**
+ * Create a string-based representation of the given static literal.
+ *
+ * @param buffer The buffer to write the string to.
+ * @param newline_list The list of newline offsets to use to calculate lines.
+ * @param start_line The line number that the parser starts on.
+ * @param encoding_name The name of the encoding of the source being parsed.
+ * @param node The node to create a string representation of.
+ */
+void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node);
+
+#endif