prism 0.13.0 → 0.14.0

data/include/prism/ast.h

@@ -358,8 +358,10 @@ typedef uint16_t pm_node_flags_t;
 
 // We store the flags enum in every node in the tree. Some flags are common to
 // all nodes (the ones listed below). Others are specific to certain node types.
-static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = 0x1;
-static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2;
+#define PM_NODE_FLAG_BITS (sizeof(pm_node_flags_t) * 8)
+static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = (1 << (PM_NODE_FLAG_BITS - 1));
+static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = (1 << (PM_NODE_FLAG_BITS - 2));
+static const pm_node_flags_t PM_NODE_FLAG_COMMON_MASK = PM_NODE_FLAG_NEWLINE | PM_NODE_FLAG_STATIC_LITERAL;
 
 // For easy access, we define some macros to check node type
 #define PM_NODE_TYPE(node) ((enum pm_node_type)node->type)
@@ -562,8 +564,8 @@ typedef struct pm_call_and_write_node {
     pm_location_t opening_loc;
     struct pm_arguments_node *arguments;
     pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_location_t operator_loc;
     struct pm_node *value;
 } pm_call_and_write_node_t;
@@ -583,7 +585,7 @@ typedef struct pm_call_node {
     struct pm_arguments_node *arguments;
     pm_location_t closing_loc;
     struct pm_node *block;
-    pm_string_t name;
+    pm_constant_id_t name;
 } pm_call_node_t;
 
 // CallOperatorWriteNode
@@ -600,8 +602,8 @@ typedef struct pm_call_operator_write_node {
     pm_location_t opening_loc;
     struct pm_arguments_node *arguments;
     pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_constant_id_t operator;
     pm_location_t operator_loc;
     struct pm_node *value;
@@ -621,8 +623,8 @@ typedef struct pm_call_or_write_node {
     pm_location_t opening_loc;
     struct pm_arguments_node *arguments;
     pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_location_t operator_loc;
     struct pm_node *value;
 } pm_call_or_write_node_t;
@@ -1193,11 +1195,11 @@ typedef struct pm_integer_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1212,11 +1214,11 @@ typedef struct pm_interpolated_match_last_line_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1369,11 +1371,11 @@ typedef struct pm_local_variable_write_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1616,11 +1618,11 @@ typedef struct pm_redo_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1887,44 +1889,44 @@ typedef struct pm_yield_node {
 } pm_yield_node_t;
 
 // CallNodeFlags
-typedef enum {
-    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 2,
-    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 3,
+typedef enum pm_call_node_flags {
+    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 0,
+    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 1,
 } pm_call_node_flags_t;
 
 // IntegerBaseFlags
-typedef enum {
-    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 2,
-    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 3,
-    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 4,
-    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 5,
+typedef enum pm_integer_base_flags {
+    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 0,
+    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 1,
+    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 2,
+    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 3,
 } pm_integer_base_flags_t;
 
 // LoopFlags
-typedef enum {
-    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 2,
+typedef enum pm_loop_flags {
+    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 0,
 } pm_loop_flags_t;
 
 // RangeFlags
-typedef enum {
-    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 2,
+typedef enum pm_range_flags {
+    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 0,
 } pm_range_flags_t;
 
 // RegularExpressionFlags
-typedef enum {
-    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 2,
-    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 3,
-    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 4,
-    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 5,
-    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 6,
-    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 7,
-    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 8,
-    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 9,
+typedef enum pm_regular_expression_flags {
+    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 0,
+    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 1,
+    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 2,
+    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 3,
+    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 4,
+    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 5,
+    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 6,
+    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 7,
 } pm_regular_expression_flags_t;
 
 // StringFlags
-typedef enum {
-    PM_STRING_FLAGS_FROZEN = 1 << 2,
+typedef enum pm_string_flags {
+    PM_STRING_FLAGS_FROZEN = 1 << 0,
 } pm_string_flags_t;
 
 #define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS false

data/include/prism/node.h

@@ -33,6 +33,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
 // declare them here to avoid generating them.
 typedef struct pm_scope_node {
     pm_node_t base;
+    pm_node_t *ast_node;
     struct pm_parameters_node *parameters;
     pm_node_t *body;
     pm_constant_id_list_t locals;

data/include/prism/parser.h

@@ -8,6 +8,7 @@
 #include "prism/util/pm_list.h"
 #include "prism/util/pm_newline_list.h"
 #include "prism/util/pm_state_stack.h"
+#include "prism/util/pm_string.h"
 
 #include <stdbool.h>
 
@@ -172,6 +173,11 @@ typedef struct pm_lex_mode {
             // This is the pointer to the character where lexing should resume
             // once the heredoc has been completely processed.
             const uint8_t *next_start;
+
+            // This is used to track the amount of common whitespace on each
+            // line so that we know how much to dedent each line in the case of
+            // a tilde heredoc.
+            size_t common_whitespace;
         } heredoc;
     } as;
 
@@ -244,6 +250,16 @@ typedef struct pm_comment {
     pm_comment_type_t type;
 } pm_comment_t;
 
+// This is a node in the linked list of magic comments that we've found while
+// parsing.
+typedef struct {
+    pm_list_node_t node;
+    const uint8_t *key_start;
+    const uint8_t *value_start;
+    uint32_t key_length;
+    uint32_t value_length;
+} pm_magic_comment_t;
+
 // When the encoding that is being used to parse the source is changed by prism,
 // we provide the ability here to call out to a user-defined function.
 typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
@@ -293,6 +309,11 @@ typedef struct pm_scope {
     // This is necessary to determine if child blocks are allowed to use
     // numbered parameters.
     bool numbered_params;
+
+    // A transparent scope is a scope that cannot have locals set on itself.
+    // When a local is set on this scope, it will instead be set on the parent
+    // scope's local table.
+    bool transparent;
 } pm_scope_t;
 
 // This struct represents the overall parser. It contains a reference to the
@@ -342,6 +363,7 @@ struct pm_parser {
     const uint8_t *heredoc_end;
 
     pm_list_t comment_list;             // the list of comments that have been found while parsing
+    pm_list_t magic_comment_list;       // the list of magic comments that have been found while parsing.
     pm_list_t warning_list;             // the list of warnings that have been found while parsing
     pm_list_t error_list;               // the list of errors that have been found while parsing
     pm_scope_t *current_scope;          // the current local scope
@@ -388,6 +410,10 @@ struct pm_parser {
     // when we find tokens that we need it for.
     pm_node_flags_t integer_base;
 
+    // This string is used to pass information from the lexer to the parser. It
+    // is particularly necessary because of escape sequences.
+    pm_string_t current_string;
+
     // Whether or not we're at the beginning of a command
     bool command_start;
 

data/include/prism/util/pm_buffer.h

@@ -21,6 +21,9 @@ typedef struct {
 // Return the size of the pm_buffer_t struct.
 PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
 
+// Initialize a pm_buffer_t with the given capacity.
+bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
+
 // Initialize a pm_buffer_t with its default values.
 PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
 

data/include/prism/util/pm_constant_pool.h

@@ -63,6 +63,11 @@ typedef struct {
 // Initialize a new constant pool with a given capacity.
 bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
 
+static inline pm_constant_t* pm_constant_pool_id_to_constant(pm_constant_pool_t *pool, pm_constant_id_t constant_id) {
+    assert(constant_id > 0 && constant_id <= pool->size);
+    return &pool->constants[constant_id - 1];
+}
+
 // Insert a constant into a constant pool that is a slice of a source string.
 // Returns the id of the constant, or 0 if any potential calls to resize fail.
 pm_constant_id_t pm_constant_pool_insert_shared(pm_constant_pool_t *pool, const uint8_t *start, size_t length);

data/include/prism/util/pm_string.h

@@ -11,9 +11,10 @@
 
 // This struct represents a string value.
 typedef struct {
-    enum { PM_STRING_SHARED, PM_STRING_OWNED, PM_STRING_CONSTANT, PM_STRING_MAPPED } type;
     const uint8_t *source;
     size_t length;
+    // This field is not the first one, because otherwise things like .pm_string_t_field = 123/pm_constant_id_t does not warn or error
+    enum { PM_STRING_SHARED, PM_STRING_OWNED, PM_STRING_CONSTANT, PM_STRING_MAPPED } type;
 } pm_string_t;
 
 #define PM_EMPTY_STRING ((pm_string_t) { .type = PM_STRING_CONSTANT, .source = NULL, .length = 0 })

data/include/prism/version.h

@@ -1,4 +1,4 @@
 #define PRISM_VERSION_MAJOR 0
-#define PRISM_VERSION_MINOR 13
+#define PRISM_VERSION_MINOR 14
 #define PRISM_VERSION_PATCH 0
-#define PRISM_VERSION "0.13.0"
+#define PRISM_VERSION "0.14.0"

data/include/prism.h

@@ -8,7 +8,6 @@
 #include "prism/pack.h"
 #include "prism/parser.h"
 #include "prism/regexp.h"
-#include "prism/unescape.h"
 #include "prism/util/pm_buffer.h"
 #include "prism/util/pm_char.h"
 #include "prism/util/pm_memchr.h"