prism 0.13.0 → 0.15.0

data/ext/prism/extension.c

@@ -10,6 +10,7 @@ VALUE rb_cPrismToken;
 VALUE rb_cPrismLocation;
 
 VALUE rb_cPrismComment;
+VALUE rb_cPrismMagicComment;
 VALUE rb_cPrismParseError;
 VALUE rb_cPrismParseWarning;
 VALUE rb_cPrismParseResult;
@@ -153,6 +154,35 @@ parser_comments(pm_parser_t *parser, VALUE source) {
     return comments;
 }
 
+// Extract the magic comments out of the parser into an array.
+static VALUE
+parser_magic_comments(pm_parser_t *parser, VALUE source) {
+    VALUE magic_comments = rb_ary_new();
+
+    for (pm_magic_comment_t *magic_comment = (pm_magic_comment_t *) parser->magic_comment_list.head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) {
+        VALUE key_loc_argv[] = {
+            source,
+            LONG2FIX(magic_comment->key_start - parser->start),
+            LONG2FIX(magic_comment->key_length)
+        };
+
+        VALUE value_loc_argv[] = {
+            source,
+            LONG2FIX(magic_comment->value_start - parser->start),
+            LONG2FIX(magic_comment->value_length)
+        };
+
+        VALUE magic_comment_argv[] = {
+            rb_class_new_instance(3, key_loc_argv, rb_cPrismLocation),
+            rb_class_new_instance(3, value_loc_argv, rb_cPrismLocation)
+        };
+
+        rb_ary_push(magic_comments, rb_class_new_instance(2, magic_comment_argv, rb_cPrismMagicComment));
+    }
+
+    return magic_comments;
+}
+
 // Extract the errors out of the parser into an array.
 static VALUE
 parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
@@ -297,6 +327,7 @@ parse_lex_input(pm_string_t *input, const char *filepath, bool return_nodes) {
     VALUE result_argv[] = {
         value,
         parser_comments(&parser, source),
+        parser_magic_comments(&parser, source),
         parser_errors(&parser, parse_lex_data.encoding, source),
         parser_warnings(&parser, parse_lex_data.encoding, source),
         source
@@ -304,7 +335,7 @@ parse_lex_input(pm_string_t *input, const char *filepath, bool return_nodes) {
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
-    return rb_class_new_instance(5, result_argv, rb_cPrismParseResult);
+    return rb_class_new_instance(6, result_argv, rb_cPrismParseResult);
 }
 
 // Return an array of tokens corresponding to the given string.
@@ -351,12 +382,13 @@ parse_input(pm_string_t *input, const char *filepath) {
     VALUE result_argv[] = {
         pm_ast_new(&parser, node, encoding),
         parser_comments(&parser, source),
+        parser_magic_comments(&parser, source),
         parser_errors(&parser, encoding, source),
         parser_warnings(&parser, encoding, source),
         source
     };
 
-    VALUE result = rb_class_new_instance(5, result_argv, rb_cPrismParseResult);
+    VALUE result = rb_class_new_instance(6, result_argv, rb_cPrismParseResult);
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -461,48 +493,6 @@ named_captures(VALUE self, VALUE source) {
     return names;
 }
 
-// Accepts a source string and a type of unescaping and returns the unescaped
-// version.
-static VALUE
-unescape(VALUE source, pm_unescape_type_t unescape_type) {
-    pm_string_t result;
-
-    if (pm_unescape_string((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), unescape_type, &result)) {
-        VALUE str = rb_str_new((const char *) pm_string_source(&result), pm_string_length(&result));
-        pm_string_free(&result);
-        return str;
-    } else {
-        pm_string_free(&result);
-        return Qnil;
-    }
-}
-
-// Do not unescape anything in the given string. This is here to provide a
-// consistent API.
-static VALUE
-unescape_none(VALUE self, VALUE source) {
-    return unescape(source, PM_UNESCAPE_NONE);
-}
-
-// Minimally unescape the given string. This means effectively unescaping just
-// the quotes of a string. Returns the unescaped string.
-static VALUE
-unescape_minimal(VALUE self, VALUE source) {
-    return unescape(source, PM_UNESCAPE_MINIMAL);
-}
-
-// Escape the given string minimally plus whitespace. Returns the unescaped string.
-static VALUE
-unescape_whitespace(VALUE self, VALUE source) {
-    return unescape(source, PM_UNESCAPE_WHITESPACE);
-}
-
-// Unescape everything in the given string. Return the unescaped string.
-static VALUE
-unescape_all(VALUE self, VALUE source) {
-    return unescape(source, PM_UNESCAPE_ALL);
-}
-
 // Return a hash of information about the given source string's memory usage.
 static VALUE
 memsize(VALUE self, VALUE string) {
@@ -589,6 +579,7 @@ Init_prism(void) {
     rb_cPrismToken = rb_define_class_under(rb_cPrism, "Token", rb_cObject);
     rb_cPrismLocation = rb_define_class_under(rb_cPrism, "Location", rb_cObject);
     rb_cPrismComment = rb_define_class_under(rb_cPrism, "Comment", rb_cObject);
+    rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject);
     rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject);
     rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject);
     rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cObject);
@@ -612,10 +603,6 @@ Init_prism(void) {
     // internal tasks. We expose these to make them easier to test.
     VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug");
     rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "unescape_none", unescape_none, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "unescape_minimal", unescape_minimal, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "unescape_whitespace", unescape_whitespace, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "unescape_all", unescape_all, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
     rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2);

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "0.13.0"
+#define EXPECTED_PRISM_VERSION "0.15.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>

data/include/prism/ast.h

@@ -277,79 +277,82 @@ enum pm_node_type {
     PM_IMAGINARY_NODE = 66,
     PM_IMPLICIT_NODE = 67,
     PM_IN_NODE = 68,
-    PM_INSTANCE_VARIABLE_AND_WRITE_NODE = 69,
-    PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 70,
-    PM_INSTANCE_VARIABLE_OR_WRITE_NODE = 71,
-    PM_INSTANCE_VARIABLE_READ_NODE = 72,
-    PM_INSTANCE_VARIABLE_TARGET_NODE = 73,
-    PM_INSTANCE_VARIABLE_WRITE_NODE = 74,
-    PM_INTEGER_NODE = 75,
-    PM_INTERPOLATED_MATCH_LAST_LINE_NODE = 76,
-    PM_INTERPOLATED_REGULAR_EXPRESSION_NODE = 77,
-    PM_INTERPOLATED_STRING_NODE = 78,
-    PM_INTERPOLATED_SYMBOL_NODE = 79,
-    PM_INTERPOLATED_X_STRING_NODE = 80,
-    PM_KEYWORD_HASH_NODE = 81,
-    PM_KEYWORD_PARAMETER_NODE = 82,
-    PM_KEYWORD_REST_PARAMETER_NODE = 83,
-    PM_LAMBDA_NODE = 84,
-    PM_LOCAL_VARIABLE_AND_WRITE_NODE = 85,
-    PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 86,
-    PM_LOCAL_VARIABLE_OR_WRITE_NODE = 87,
-    PM_LOCAL_VARIABLE_READ_NODE = 88,
-    PM_LOCAL_VARIABLE_TARGET_NODE = 89,
-    PM_LOCAL_VARIABLE_WRITE_NODE = 90,
-    PM_MATCH_LAST_LINE_NODE = 91,
-    PM_MATCH_PREDICATE_NODE = 92,
-    PM_MATCH_REQUIRED_NODE = 93,
-    PM_MATCH_WRITE_NODE = 94,
-    PM_MISSING_NODE = 95,
-    PM_MODULE_NODE = 96,
-    PM_MULTI_TARGET_NODE = 97,
-    PM_MULTI_WRITE_NODE = 98,
-    PM_NEXT_NODE = 99,
-    PM_NIL_NODE = 100,
-    PM_NO_KEYWORDS_PARAMETER_NODE = 101,
-    PM_NUMBERED_REFERENCE_READ_NODE = 102,
-    PM_OPTIONAL_PARAMETER_NODE = 103,
-    PM_OR_NODE = 104,
-    PM_PARAMETERS_NODE = 105,
-    PM_PARENTHESES_NODE = 106,
-    PM_PINNED_EXPRESSION_NODE = 107,
-    PM_PINNED_VARIABLE_NODE = 108,
-    PM_POST_EXECUTION_NODE = 109,
-    PM_PRE_EXECUTION_NODE = 110,
-    PM_PROGRAM_NODE = 111,
-    PM_RANGE_NODE = 112,
-    PM_RATIONAL_NODE = 113,
-    PM_REDO_NODE = 114,
-    PM_REGULAR_EXPRESSION_NODE = 115,
-    PM_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 116,
-    PM_REQUIRED_PARAMETER_NODE = 117,
-    PM_RESCUE_MODIFIER_NODE = 118,
-    PM_RESCUE_NODE = 119,
-    PM_REST_PARAMETER_NODE = 120,
-    PM_RETRY_NODE = 121,
-    PM_RETURN_NODE = 122,
-    PM_SELF_NODE = 123,
-    PM_SINGLETON_CLASS_NODE = 124,
-    PM_SOURCE_ENCODING_NODE = 125,
-    PM_SOURCE_FILE_NODE = 126,
-    PM_SOURCE_LINE_NODE = 127,
-    PM_SPLAT_NODE = 128,
-    PM_STATEMENTS_NODE = 129,
-    PM_STRING_CONCAT_NODE = 130,
-    PM_STRING_NODE = 131,
-    PM_SUPER_NODE = 132,
-    PM_SYMBOL_NODE = 133,
-    PM_TRUE_NODE = 134,
-    PM_UNDEF_NODE = 135,
-    PM_UNLESS_NODE = 136,
-    PM_UNTIL_NODE = 137,
-    PM_WHEN_NODE = 138,
-    PM_WHILE_NODE = 139,
-    PM_X_STRING_NODE = 140,
-    PM_YIELD_NODE = 141,
+    PM_INDEX_AND_WRITE_NODE = 69,
+    PM_INDEX_OPERATOR_WRITE_NODE = 70,
+    PM_INDEX_OR_WRITE_NODE = 71,
+    PM_INSTANCE_VARIABLE_AND_WRITE_NODE = 72,
+    PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 73,
+    PM_INSTANCE_VARIABLE_OR_WRITE_NODE = 74,
+    PM_INSTANCE_VARIABLE_READ_NODE = 75,
+    PM_INSTANCE_VARIABLE_TARGET_NODE = 76,
+    PM_INSTANCE_VARIABLE_WRITE_NODE = 77,
+    PM_INTEGER_NODE = 78,
+    PM_INTERPOLATED_MATCH_LAST_LINE_NODE = 79,
+    PM_INTERPOLATED_REGULAR_EXPRESSION_NODE = 80,
+    PM_INTERPOLATED_STRING_NODE = 81,
+    PM_INTERPOLATED_SYMBOL_NODE = 82,
+    PM_INTERPOLATED_X_STRING_NODE = 83,
+    PM_KEYWORD_HASH_NODE = 84,
+    PM_KEYWORD_PARAMETER_NODE = 85,
+    PM_KEYWORD_REST_PARAMETER_NODE = 86,
+    PM_LAMBDA_NODE = 87,
+    PM_LOCAL_VARIABLE_AND_WRITE_NODE = 88,
+    PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 89,
+    PM_LOCAL_VARIABLE_OR_WRITE_NODE = 90,
+    PM_LOCAL_VARIABLE_READ_NODE = 91,
+    PM_LOCAL_VARIABLE_TARGET_NODE = 92,
+    PM_LOCAL_VARIABLE_WRITE_NODE = 93,
+    PM_MATCH_LAST_LINE_NODE = 94,
+    PM_MATCH_PREDICATE_NODE = 95,
+    PM_MATCH_REQUIRED_NODE = 96,
+    PM_MATCH_WRITE_NODE = 97,
+    PM_MISSING_NODE = 98,
+    PM_MODULE_NODE = 99,
+    PM_MULTI_TARGET_NODE = 100,
+    PM_MULTI_WRITE_NODE = 101,
+    PM_NEXT_NODE = 102,
+    PM_NIL_NODE = 103,
+    PM_NO_KEYWORDS_PARAMETER_NODE = 104,
+    PM_NUMBERED_REFERENCE_READ_NODE = 105,
+    PM_OPTIONAL_PARAMETER_NODE = 106,
+    PM_OR_NODE = 107,
+    PM_PARAMETERS_NODE = 108,
+    PM_PARENTHESES_NODE = 109,
+    PM_PINNED_EXPRESSION_NODE = 110,
+    PM_PINNED_VARIABLE_NODE = 111,
+    PM_POST_EXECUTION_NODE = 112,
+    PM_PRE_EXECUTION_NODE = 113,
+    PM_PROGRAM_NODE = 114,
+    PM_RANGE_NODE = 115,
+    PM_RATIONAL_NODE = 116,
+    PM_REDO_NODE = 117,
+    PM_REGULAR_EXPRESSION_NODE = 118,
+    PM_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 119,
+    PM_REQUIRED_PARAMETER_NODE = 120,
+    PM_RESCUE_MODIFIER_NODE = 121,
+    PM_RESCUE_NODE = 122,
+    PM_REST_PARAMETER_NODE = 123,
+    PM_RETRY_NODE = 124,
+    PM_RETURN_NODE = 125,
+    PM_SELF_NODE = 126,
+    PM_SINGLETON_CLASS_NODE = 127,
+    PM_SOURCE_ENCODING_NODE = 128,
+    PM_SOURCE_FILE_NODE = 129,
+    PM_SOURCE_LINE_NODE = 130,
+    PM_SPLAT_NODE = 131,
+    PM_STATEMENTS_NODE = 132,
+    PM_STRING_CONCAT_NODE = 133,
+    PM_STRING_NODE = 134,
+    PM_SUPER_NODE = 135,
+    PM_SYMBOL_NODE = 136,
+    PM_TRUE_NODE = 137,
+    PM_UNDEF_NODE = 138,
+    PM_UNLESS_NODE = 139,
+    PM_UNTIL_NODE = 140,
+    PM_WHEN_NODE = 141,
+    PM_WHILE_NODE = 142,
+    PM_X_STRING_NODE = 143,
+    PM_YIELD_NODE = 144,
     PM_SCOPE_NODE
 };
 
@@ -358,8 +361,10 @@ typedef uint16_t pm_node_flags_t;
 
 // We store the flags enum in every node in the tree. Some flags are common to
 // all nodes (the ones listed below). Others are specific to certain node types.
-static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = 0x1;
-static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2;
+#define PM_NODE_FLAG_BITS (sizeof(pm_node_flags_t) * 8)
+static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = (1 << (PM_NODE_FLAG_BITS - 1));
+static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = (1 << (PM_NODE_FLAG_BITS - 2));
+static const pm_node_flags_t PM_NODE_FLAG_COMMON_MASK = (1 << (PM_NODE_FLAG_BITS - 1)) | (1 << (PM_NODE_FLAG_BITS - 2));
 
 // For easy access, we define some macros to check node type
 #define PM_NODE_TYPE(node) ((enum pm_node_type)node->type)
@@ -474,6 +479,7 @@ typedef struct pm_assoc_splat_node {
 // Type: PM_BACK_REFERENCE_READ_NODE
 typedef struct pm_back_reference_read_node {
     pm_node_t base;
+    pm_constant_id_t name;
 } pm_back_reference_read_node_t;
 
 // BeginNode
@@ -559,11 +565,8 @@ typedef struct pm_call_and_write_node {
     struct pm_node *receiver;
     pm_location_t call_operator_loc;
     pm_location_t message_loc;
-    pm_location_t opening_loc;
-    struct pm_arguments_node *arguments;
-    pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_location_t operator_loc;
     struct pm_node *value;
 } pm_call_and_write_node_t;
@@ -583,7 +586,7 @@ typedef struct pm_call_node {
     struct pm_arguments_node *arguments;
     pm_location_t closing_loc;
     struct pm_node *block;
-    pm_string_t name;
+    pm_constant_id_t name;
 } pm_call_node_t;
 
 // CallOperatorWriteNode
@@ -597,11 +600,8 @@ typedef struct pm_call_operator_write_node {
     struct pm_node *receiver;
     pm_location_t call_operator_loc;
     pm_location_t message_loc;
-    pm_location_t opening_loc;
-    struct pm_arguments_node *arguments;
-    pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_constant_id_t operator;
     pm_location_t operator_loc;
     struct pm_node *value;
@@ -618,11 +618,8 @@ typedef struct pm_call_or_write_node {
     struct pm_node *receiver;
     pm_location_t call_operator_loc;
     pm_location_t message_loc;
-    pm_location_t opening_loc;
-    struct pm_arguments_node *arguments;
-    pm_location_t closing_loc;
-    pm_string_t read_name;
-    pm_string_t write_name;
+    pm_constant_id_t read_name;
+    pm_constant_id_t write_name;
     pm_location_t operator_loc;
     struct pm_node *value;
 } pm_call_or_write_node_t;
@@ -1113,6 +1110,61 @@ typedef struct pm_in_node {
     pm_location_t then_loc;
 } pm_in_node_t;
 
+// IndexAndWriteNode
+//
+// Type: PM_INDEX_AND_WRITE_NODE
+// Flags:
+//    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION
+//    PM_CALL_NODE_FLAGS_VARIABLE_CALL
+typedef struct pm_index_and_write_node {
+    pm_node_t base;
+    struct pm_node *receiver;
+    pm_location_t call_operator_loc;
+    pm_location_t opening_loc;
+    struct pm_arguments_node *arguments;
+    pm_location_t closing_loc;
+    struct pm_node *block;
+    pm_location_t operator_loc;
+    struct pm_node *value;
+} pm_index_and_write_node_t;
+
+// IndexOperatorWriteNode
+//
+// Type: PM_INDEX_OPERATOR_WRITE_NODE
+// Flags:
+//    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION
+//    PM_CALL_NODE_FLAGS_VARIABLE_CALL
+typedef struct pm_index_operator_write_node {
+    pm_node_t base;
+    struct pm_node *receiver;
+    pm_location_t call_operator_loc;
+    pm_location_t opening_loc;
+    struct pm_arguments_node *arguments;
+    pm_location_t closing_loc;
+    struct pm_node *block;
+    pm_constant_id_t operator;
+    pm_location_t operator_loc;
+    struct pm_node *value;
+} pm_index_operator_write_node_t;
+
+// IndexOrWriteNode
+//
+// Type: PM_INDEX_OR_WRITE_NODE
+// Flags:
+//    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION
+//    PM_CALL_NODE_FLAGS_VARIABLE_CALL
+typedef struct pm_index_or_write_node {
+    pm_node_t base;
+    struct pm_node *receiver;
+    pm_location_t call_operator_loc;
+    pm_location_t opening_loc;
+    struct pm_arguments_node *arguments;
+    pm_location_t closing_loc;
+    struct pm_node *block;
+    pm_location_t operator_loc;
+    struct pm_node *value;
+} pm_index_or_write_node_t;
+
 // InstanceVariableAndWriteNode
 //
 // Type: PM_INSTANCE_VARIABLE_AND_WRITE_NODE
@@ -1193,11 +1245,11 @@ typedef struct pm_integer_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1212,11 +1264,11 @@ typedef struct pm_interpolated_match_last_line_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1369,11 +1421,11 @@ typedef struct pm_local_variable_write_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1616,11 +1668,11 @@ typedef struct pm_redo_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1887,44 +1939,44 @@ typedef struct pm_yield_node {
 } pm_yield_node_t;
 
 // CallNodeFlags
-typedef enum {
-    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 2,
-    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 3,
+typedef enum pm_call_node_flags {
+    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 0,
+    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 1,
 } pm_call_node_flags_t;
 
 // IntegerBaseFlags
-typedef enum {
-    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 2,
-    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 3,
-    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 4,
-    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 5,
+typedef enum pm_integer_base_flags {
+    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 0,
+    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 1,
+    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 2,
+    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 3,
 } pm_integer_base_flags_t;
 
 // LoopFlags
-typedef enum {
-    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 2,
+typedef enum pm_loop_flags {
+    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 0,
 } pm_loop_flags_t;
 
 // RangeFlags
-typedef enum {
-    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 2,
+typedef enum pm_range_flags {
+    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 0,
 } pm_range_flags_t;
 
 // RegularExpressionFlags
-typedef enum {
-    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 2,
-    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 3,
-    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 4,
-    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 5,
-    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 6,
-    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 7,
-    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 8,
-    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 9,
+typedef enum pm_regular_expression_flags {
+    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 0,
+    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 1,
+    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 2,
+    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 3,
+    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 4,
+    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 5,
+    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 6,
+    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 7,
 } pm_regular_expression_flags_t;
 
 // StringFlags
-typedef enum {
-    PM_STRING_FLAGS_FROZEN = 1 << 2,
+typedef enum pm_string_flags {
+    PM_STRING_FLAGS_FROZEN = 1 << 0,
 } pm_string_flags_t;
 
 #define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS false

data/include/prism/diagnostic.h

@@ -158,6 +158,7 @@ typedef enum {
     PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
     PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
     PM_ERR_OPERATOR_MULTI_ASSIGN,
+    PM_ERR_OPERATOR_WRITE_ARGUMENTS,
     PM_ERR_OPERATOR_WRITE_BLOCK,
     PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI,
     PM_ERR_PARAMETER_BLOCK_MULTI,

data/include/prism/node.h

@@ -33,9 +33,17 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
 // declare them here to avoid generating them.
 typedef struct pm_scope_node {
     pm_node_t base;
+    struct pm_scope_node *previous;
+    pm_node_t *ast_node;
     struct pm_parameters_node *parameters;
     pm_node_t *body;
     pm_constant_id_list_t locals;
+    pm_parser_t *parser;
+
+    // We don't have the CRuby types ID and st_table within Prism
+    // so we use void *
+    void *constants; // ID *constants
+    void *index_lookup_table; // st_table *index_lookup_table
 } pm_scope_node_t;
 
 #endif // PRISM_NODE_H

data/include/prism/parser.h

@@ -8,6 +8,7 @@
 #include "prism/util/pm_list.h"
 #include "prism/util/pm_newline_list.h"
 #include "prism/util/pm_state_stack.h"
+#include "prism/util/pm_string.h"
 
 #include <stdbool.h>
 
@@ -172,6 +173,11 @@ typedef struct pm_lex_mode {
             // This is the pointer to the character where lexing should resume
             // once the heredoc has been completely processed.
             const uint8_t *next_start;
+
+            // This is used to track the amount of common whitespace on each
+            // line so that we know how much to dedent each line in the case of
+            // a tilde heredoc.
+            size_t common_whitespace;
         } heredoc;
     } as;
 
@@ -244,6 +250,16 @@ typedef struct pm_comment {
     pm_comment_type_t type;
 } pm_comment_t;
 
+// This is a node in the linked list of magic comments that we've found while
+// parsing.
+typedef struct {
+    pm_list_node_t node;
+    const uint8_t *key_start;
+    const uint8_t *value_start;
+    uint32_t key_length;
+    uint32_t value_length;
+} pm_magic_comment_t;
+
 // When the encoding that is being used to parse the source is changed by prism,
 // we provide the ability here to call out to a user-defined function.
 typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
@@ -293,6 +309,11 @@ typedef struct pm_scope {
     // This is necessary to determine if child blocks are allowed to use
     // numbered parameters.
     bool numbered_params;
+
+    // A transparent scope is a scope that cannot have locals set on itself.
+    // When a local is set on this scope, it will instead be set on the parent
+    // scope's local table.
+    bool transparent;
 } pm_scope_t;
 
 // This struct represents the overall parser. It contains a reference to the
@@ -342,6 +363,7 @@ struct pm_parser {
     const uint8_t *heredoc_end;
 
     pm_list_t comment_list;             // the list of comments that have been found while parsing
+    pm_list_t magic_comment_list;       // the list of magic comments that have been found while parsing.
     pm_list_t warning_list;             // the list of warnings that have been found while parsing
     pm_list_t error_list;               // the list of errors that have been found while parsing
     pm_scope_t *current_scope;          // the current local scope
@@ -388,6 +410,10 @@ struct pm_parser {
     // when we find tokens that we need it for.
     pm_node_flags_t integer_base;
 
+    // This string is used to pass information from the lexer to the parser. It
+    // is particularly necessary because of escape sequences.
+    pm_string_t current_string;
+
     // Whether or not we're at the beginning of a command
     bool command_start;
 

data/include/prism/util/pm_buffer.h

@@ -21,6 +21,9 @@ typedef struct {
 // Return the size of the pm_buffer_t struct.
 PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
 
+// Initialize a pm_buffer_t with the given capacity.
+bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
+
 // Initialize a pm_buffer_t with its default values.
 PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);