prism 0.13.0 → 0.14.0

data/src/serialize.c

@@ -329,9 +329,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_buffer_append_u8(buffer, 1);
                 pm_serialize_location(parser, &((pm_call_and_write_node_t *)node)->closing_loc, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
-            pm_serialize_string(parser, &((pm_call_and_write_node_t *)node)->read_name, buffer);
-            pm_serialize_string(parser, &((pm_call_and_write_node_t *)node)->write_name, buffer);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_and_write_node_t *)node)->read_name));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_and_write_node_t *)node)->write_name));
             pm_serialize_location(parser, &((pm_call_and_write_node_t *)node)->operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_call_and_write_node_t *)node)->value, buffer);
             break;
@@ -376,8 +376,8 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_call_node_t *)node)->block, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
-            pm_serialize_string(parser, &((pm_call_node_t *)node)->name, buffer);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_node_t *)node)->name));
             break;
         }
         case PM_CALL_OPERATOR_WRITE_NODE: {
@@ -415,9 +415,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_buffer_append_u8(buffer, 1);
                 pm_serialize_location(parser, &((pm_call_operator_write_node_t *)node)->closing_loc, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
-            pm_serialize_string(parser, &((pm_call_operator_write_node_t *)node)->read_name, buffer);
-            pm_serialize_string(parser, &((pm_call_operator_write_node_t *)node)->write_name, buffer);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->read_name));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->write_name));
             pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->operator));
             pm_serialize_location(parser, &((pm_call_operator_write_node_t *)node)->operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_call_operator_write_node_t *)node)->value, buffer);
@@ -458,9 +458,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_buffer_append_u8(buffer, 1);
                 pm_serialize_location(parser, &((pm_call_or_write_node_t *)node)->closing_loc, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
-            pm_serialize_string(parser, &((pm_call_or_write_node_t *)node)->read_name, buffer);
-            pm_serialize_string(parser, &((pm_call_or_write_node_t *)node)->write_name, buffer);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_or_write_node_t *)node)->read_name));
+            pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_call_or_write_node_t *)node)->write_name));
             pm_serialize_location(parser, &((pm_call_or_write_node_t *)node)->operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_call_or_write_node_t *)node)->value, buffer);
             break;
@@ -805,7 +805,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_serialize_node(parser, (pm_node_t *)((pm_flip_flop_node_t *)node)->right, buffer);
             }
             pm_serialize_location(parser, &((pm_flip_flop_node_t *)node)->operator_loc, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_FLOAT_NODE: {
@@ -1009,7 +1009,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_INTEGER_NODE: {
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: {
@@ -1020,7 +1020,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_serialize_node(parser, (pm_node_t *) ((pm_interpolated_match_last_line_node_t *)node)->parts.nodes[index], buffer);
             }
             pm_serialize_location(parser, &((pm_interpolated_match_last_line_node_t *)node)->closing_loc, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: {
@@ -1031,7 +1031,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_serialize_node(parser, (pm_node_t *) ((pm_interpolated_regular_expression_node_t *)node)->parts.nodes[index], buffer);
             }
             pm_serialize_location(parser, &((pm_interpolated_regular_expression_node_t *)node)->closing_loc, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_INTERPOLATED_STRING_NODE: {
@@ -1182,7 +1182,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_serialize_location(parser, &((pm_match_last_line_node_t *)node)->content_loc, buffer);
             pm_serialize_location(parser, &((pm_match_last_line_node_t *)node)->closing_loc, buffer);
             pm_serialize_string(parser, &((pm_match_last_line_node_t *)node)->unescaped, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_MATCH_PREDICATE_NODE: {
@@ -1405,7 +1405,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
                 pm_serialize_node(parser, (pm_node_t *)((pm_range_node_t *)node)->right, buffer);
             }
             pm_serialize_location(parser, &((pm_range_node_t *)node)->operator_loc, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_RATIONAL_NODE: {
@@ -1420,7 +1420,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_serialize_location(parser, &((pm_regular_expression_node_t *)node)->content_loc, buffer);
             pm_serialize_location(parser, &((pm_regular_expression_node_t *)node)->closing_loc, buffer);
             pm_serialize_string(parser, &((pm_regular_expression_node_t *)node)->unescaped, buffer);
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_REQUIRED_DESTRUCTURED_PARAMETER_NODE: {
@@ -1549,7 +1549,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_STRING_NODE: {
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             if (((pm_string_node_t *)node)->opening_loc.start == NULL) {
                 pm_buffer_append_u8(buffer, 0);
             } else {
@@ -1661,7 +1661,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_until_node_t *)node)->statements, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_WHEN_NODE: {
@@ -1692,7 +1692,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_while_node_t *)node)->statements, buffer);
             }
-            pm_buffer_append_u32(buffer, node->flags >> 2);
+            pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
         case PM_X_STRING_NODE: {
@@ -1746,6 +1746,27 @@ pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buf
     }
 }
 
+static void
+pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) {
+    // serialize key location
+    pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start));
+    pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->key_length));
+
+    // serialize value location
+    pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start));
+    pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->value_length));
+}
+
+static void
+pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
+    pm_buffer_append_u32(buffer, pm_sizet_to_u32(pm_list_size(list)));
+
+    pm_magic_comment_t *magic_comment;
+    for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) {
+        pm_serialize_magic_comment(parser, magic_comment, buffer);
+    }
+}
+
 static void
 pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) {
     // serialize message
@@ -1775,11 +1796,12 @@ pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer) {
     pm_buffer_append_str(buffer, encoding->name, encoding_length);
 }
 
-#line 179 "serialize.c.erb"
+#line 200 "serialize.c.erb"
 void
 pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
     pm_serialize_encoding(&parser->encoding, buffer);
     pm_serialize_comment_list(parser, &parser->comment_list, buffer);
+    pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer);
     pm_serialize_diagnostic_list(parser, &parser->error_list, buffer);
     pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer);
 
@@ -1868,6 +1890,7 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
 
     pm_serialize_encoding(&parser.encoding, buffer);
     pm_serialize_comment_list(&parser, &parser.comment_list, buffer);
+    pm_serialize_magic_comment_list(&parser, &parser.magic_comment_list, buffer);
     pm_serialize_diagnostic_list(&parser, &parser.error_list, buffer);
     pm_serialize_diagnostic_list(&parser, &parser.warning_list, buffer);
 

data/src/util/pm_buffer.c

@@ -1,24 +1,26 @@
 #include "prism/util/pm_buffer.h"
 
-#define PRISM_BUFFER_INITIAL_SIZE 1024
-
 // Return the size of the pm_buffer_t struct.
 size_t
 pm_buffer_sizeof(void) {
     return sizeof(pm_buffer_t);
 }
 
-// Initialize a pm_buffer_t with its default values.
+// Initialize a pm_buffer_t with the given capacity.
 bool
-pm_buffer_init(pm_buffer_t *buffer) {
+pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity) {
     buffer->length = 0;
-    buffer->capacity = PRISM_BUFFER_INITIAL_SIZE;
+    buffer->capacity = capacity;
 
-    buffer->value = (char *) malloc(PRISM_BUFFER_INITIAL_SIZE);
+    buffer->value = (char *) malloc(capacity);
     return buffer->value != NULL;
 }
 
-#undef PRISM_BUFFER_INITIAL_SIZE
+// Initialize a pm_buffer_t with its default values.
+bool
+pm_buffer_init(pm_buffer_t *buffer) {
+    return pm_buffer_init_capacity(buffer, 1024);
+}
 
 // Return the value of the buffer.
 char *

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-29 00:00:00.000000000 Z
+date: 2023-10-13 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -33,6 +33,7 @@ files:
 - docs/fuzzing.md
 - docs/heredocs.md
 - docs/mapping.md
+- docs/prism.png
 - docs/ripper.md
 - docs/ruby_api.md
 - docs/serialization.md
@@ -51,7 +52,6 @@ files:
 - include/prism/pack.h
 - include/prism/parser.h
 - include/prism/regexp.h
-- include/prism/unescape.h
 - include/prism/util/pm_buffer.h
 - include/prism/util/pm_char.h
 - include/prism/util/pm_constant_pool.h
@@ -99,7 +99,6 @@ files:
 - src/regexp.c
 - src/serialize.c
 - src/token_type.c
-- src/unescape.c
 - src/util/pm_buffer.c
 - src/util/pm_char.c
 - src/util/pm_constant_pool.c

data/include/prism/unescape.h

@@ -1,48 +0,0 @@
-#ifndef PRISM_UNESCAPE_H
-#define PRISM_UNESCAPE_H
-
-#include "prism/defines.h"
-#include "prism/diagnostic.h"
-#include "prism/parser.h"
-#include "prism/util/pm_char.h"
-#include "prism/util/pm_list.h"
-#include "prism/util/pm_memchr.h"
-#include "prism/util/pm_string.h"
-
-#include <assert.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-
-// The type of unescape we are performing.
-typedef enum {
-    // When we're creating a string inside of a list literal like %w, we
-    // shouldn't escape anything.
-    PM_UNESCAPE_NONE,
-
-    // When we're unescaping a single-quoted string, we only need to unescape
-    // single quotes and backslashes.
-    PM_UNESCAPE_MINIMAL,
-
-    // When we're unescaping a string list, in addition to MINIMAL, we need to
-    // unescape whitespace.
-    PM_UNESCAPE_WHITESPACE,
-
-    // When we're unescaping a double-quoted string, we need to unescape all
-    // escapes.
-    PM_UNESCAPE_ALL,
-} pm_unescape_type_t;
-
-// Unescape the contents of the given token into the given string using the given unescape mode.
-PRISM_EXPORTED_FUNCTION void pm_unescape_manipulate_string(pm_parser_t *parser, pm_string_t *string, pm_unescape_type_t unescape_type);
-void pm_unescape_manipulate_char_literal(pm_parser_t *parser, pm_string_t *string, pm_unescape_type_t unescape_type);
-
-// Accepts a source string and a type of unescaping and returns the unescaped version.
-// The caller must pm_string_free(result); after calling this function.
-PRISM_EXPORTED_FUNCTION bool pm_unescape_string(const uint8_t *start, size_t length, pm_unescape_type_t unescape_type, pm_string_t *result);
-
-// Returns the number of bytes that encompass the first escape sequence in the
-// given string.
-size_t pm_unescape_calculate_difference(pm_parser_t *parser, const uint8_t *value, pm_unescape_type_t unescape_type, bool expect_single_codepoint);
-
-#endif