yarp 0.6.0 → 0.7.0

data/src/serialize.c

@@ -5,10 +5,7 @@
 /* if you are looking to modify the                                           */
 /* template                                                                   */
 /******************************************************************************/
-#include "yarp/ast.h"
-#include "yarp/diagnostic.h"
-#include "yarp/parser.h"
-#include "yarp/util/yp_buffer.h"
+#include "yarp.h"
 
 #include <stdio.h>
 
@@ -299,7 +296,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             } else {
                 yp_serialize_node(parser, (yp_node_t *)((yp_call_node_t *)node)->block, buffer);
             }
-            yp_buffer_append_u32(buffer, ((yp_call_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             uint32_t name_length = yp_sizet_to_u32(yp_string_length(&((yp_call_node_t *)node)->name));
             yp_buffer_append_u32(buffer, name_length);
             yp_buffer_append_str(buffer, yp_string_source(&((yp_call_node_t *)node)->name), name_length);
@@ -496,10 +493,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             break;
         }
         case YP_NODE_DEF_NODE: {
-                // serialize length
-                // encoding of location u32s make us need to save this offset.
-                size_t length_offset = buffer->length;
-                yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
+            // serialize length
+            // encoding of location u32s make us need to save this offset.
+            size_t length_offset = buffer->length;
+            yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
             serialize_location(parser, &((yp_def_node_t *)node)->name_loc, buffer);
             if (((yp_def_node_t *)node)->receiver == NULL) {
                 yp_buffer_append_u8(buffer, 0);
@@ -552,9 +549,9 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
                 yp_buffer_append_u8(buffer, 1);
                 serialize_location(parser, &((yp_def_node_t *)node)->end_keyword_loc, buffer);
             }
-                // serialize length
-                uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
-                memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
+            // serialize length
+            uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
+            memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
             break;
         }
         case YP_NODE_DEFINED_NODE: {
@@ -644,6 +641,21 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             }
             break;
         }
+        case YP_NODE_FLIP_FLOP_NODE: {
+            if (((yp_flip_flop_node_t *)node)->left == NULL) {
+                yp_buffer_append_u8(buffer, 0);
+            } else {
+                yp_serialize_node(parser, (yp_node_t *)((yp_flip_flop_node_t *)node)->left, buffer);
+            }
+            if (((yp_flip_flop_node_t *)node)->right == NULL) {
+                yp_buffer_append_u8(buffer, 0);
+            } else {
+                yp_serialize_node(parser, (yp_node_t *)((yp_flip_flop_node_t *)node)->right, buffer);
+            }
+            serialize_location(parser, &((yp_flip_flop_node_t *)node)->operator_loc, buffer);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
+            break;
+        }
         case YP_NODE_FLOAT_NODE: {
             break;
         }
@@ -851,7 +863,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
                 yp_serialize_node(parser, (yp_node_t *) ((yp_interpolated_regular_expression_node_t *)node)->parts.nodes[index], buffer);
             }
             serialize_location(parser, &((yp_interpolated_regular_expression_node_t *)node)->closing_loc, buffer);
-            yp_buffer_append_u32(buffer, ((yp_interpolated_regular_expression_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             break;
         }
         case YP_NODE_INTERPOLATED_STRING_NODE: {
@@ -1192,7 +1204,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
                 yp_serialize_node(parser, (yp_node_t *)((yp_range_node_t *)node)->right, buffer);
             }
             serialize_location(parser, &((yp_range_node_t *)node)->operator_loc, buffer);
-            yp_buffer_append_u32(buffer, ((yp_range_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             break;
         }
         case YP_NODE_RATIONAL_NODE: {
@@ -1209,7 +1221,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             uint32_t unescaped_length = yp_sizet_to_u32(yp_string_length(&((yp_regular_expression_node_t *)node)->unescaped));
             yp_buffer_append_u32(buffer, unescaped_length);
             yp_buffer_append_str(buffer, yp_string_source(&((yp_regular_expression_node_t *)node)->unescaped), unescaped_length);
-            yp_buffer_append_u32(buffer, ((yp_regular_expression_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             break;
         }
         case YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE: {
@@ -1443,7 +1455,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             } else {
                 yp_serialize_node(parser, (yp_node_t *)((yp_until_node_t *)node)->statements, buffer);
             }
-            yp_buffer_append_u32(buffer, ((yp_until_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             break;
         }
         case YP_NODE_WHEN_NODE: {
@@ -1468,7 +1480,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             } else {
                 yp_serialize_node(parser, (yp_node_t *)((yp_while_node_t *)node)->statements, buffer);
             }
-            yp_buffer_append_u32(buffer, ((yp_while_node_t *)node)->flags);
+            yp_buffer_append_u32(buffer, node->flags >> 1);
             break;
         }
         case YP_NODE_X_STRING_NODE: {
@@ -1504,6 +1516,24 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
     }
 }
 
+void yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) {
+    // serialize type
+    yp_buffer_append_u8(buffer, (uint8_t) comment->type);
+
+    // serialize location
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start));
+}
+
+void yp_serialize_comment_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) {
+    yp_buffer_append_u32(buffer, yp_list_size(&list));
+
+    yp_comment_t *comment;
+    for (comment = (yp_comment_t *) list.head; comment != NULL; comment = (yp_comment_t *) comment->node.next) {
+        yp_serialize_comment(parser, comment, buffer);
+    }
+}
+
 void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) {
     // serialize message
     size_t message_length = strlen(diagnostic->message);
@@ -1524,6 +1554,7 @@ void yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t list, yp_buffer
     }
 }
 
+#line 145 "serialize.c.erb"
 void
 yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
     // First, serialize the encoding of the parser.
@@ -1531,6 +1562,9 @@ yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer)
     yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length));
     yp_buffer_append_str(buffer, parser->encoding.name, encoding_length);
 
+    // Serialize the comments
+    yp_serialize_comment_list(parser, parser->comment_list, buffer);
+
     // Serialize the errors
     yp_serialize_diagnostic_list(parser, parser->error_list, buffer);
 
@@ -1574,3 +1608,42 @@ yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer)
         }
     }
 }
+
+static void
+serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) {
+    yp_buffer_t *buffer = (yp_buffer_t *) data;
+
+    yp_buffer_append_u32(buffer, token->type);
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->end - token->start));
+    yp_buffer_append_u32(buffer, parser->lex_state);
+}
+
+YP_EXPORTED_FUNCTION void
+yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer) {
+    yp_parser_t parser;
+    yp_parser_init(&parser, source, size, filepath);
+
+    yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
+        .data = (void *) buffer,
+        .callback = serialize_token,
+    };
+
+    parser.lex_callback = &lex_callback;
+    yp_node_t *node = yp_parse(&parser);
+
+    // Append 0 to mark end of tokens
+    yp_buffer_append_u32(buffer, 0);
+
+    // Serialize the comments
+    yp_serialize_comment_list(&parser, parser.comment_list, buffer);
+
+    // Serialize the errors
+    yp_serialize_diagnostic_list(&parser, parser.error_list, buffer);
+
+    // Serialize the warnings
+    yp_serialize_diagnostic_list(&parser, parser.warning_list, buffer);
+
+    yp_node_destroy(&parser, node);
+    yp_parser_free(&parser);
+}

data/src/util/yp_string.c

@@ -97,10 +97,8 @@ yp_string_free(yp_string_t *string) {
         void *memory = (void *) string->source;
         #if defined(_WIN32)
         UnmapViewOfFile(memory);
-        #elif defined(HAVE_MMAP)
-        munmap(memory, string->length);
         #else
-        free(memory);
+        munmap(memory, string->length);
         #endif
     }
 }
@@ -180,28 +178,21 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) {
         return true;
     }
 
-#ifdef HAVE_MMAP
     source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
     if (source == MAP_FAILED) {
         perror("Map failed");
         return false;
     }
-#else
-    source = malloc(size);
-    if (source == NULL) {
-        return false;
-    }
-
-    ssize_t read_size = read(fd, (void *) source, size);
-    if (read_size < 0 || (size_t)read_size != size) {
-        perror("Read size is incorrect");
-        free((void *) source);
-        return false;
-    }
-#endif
 
     close(fd);
     yp_string_mapped_init_internal(string, source, size);
     return true;
 #endif
 }
+
+// Returns the size of the yp_string_t struct. This is necessary to allocate the
+// correct amount of memory in the FFI backend.
+YP_EXPORTED_FUNCTION size_t
+yp_string_sizeof(void) {
+    return sizeof(yp_string_t);
+}