prism 0.24.0 → 0.29.0

data/src/regexp.c

@@ -565,21 +565,36 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) {
  */
 static bool
 pm_regexp_parse_item(pm_regexp_parser_t *parser) {
-    switch (*parser->cursor++) {
+    switch (*parser->cursor) {
         case '^':
         case '$':
+            parser->cursor++;
             return true;
         case '\\':
+            parser->cursor++;
             if (!pm_regexp_char_is_eof(parser)) {
                 parser->cursor++;
             }
             return pm_regexp_parse_quantifier(parser);
         case '(':
+            parser->cursor++;
             return pm_regexp_parse_group(parser) && pm_regexp_parse_quantifier(parser);
         case '[':
+            parser->cursor++;
             return pm_regexp_parse_lbracket(parser) && pm_regexp_parse_quantifier(parser);
-        default:
+        default: {
+            size_t width;
+            if (!parser->encoding_changed) {
+                width = pm_encoding_utf_8_char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
+            } else {
+                width = parser->encoding->char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
+            }
+
+            if (width == 0) return false; // TODO: add appropriate error
+            parser->cursor += width;
+
             return pm_regexp_parse_quantifier(parser);
+        }
     }
 }
 

data/src/serialize.c

@@ -5,8 +5,14 @@
 /* if you are looking to modify the                                           */
 /* template                                                                   */
 /******************************************************************************/
+
 #include "prism.h"
 
+// We optionally support serializing to a binary string. For systems that don't
+// want or need this functionality, it can be turned off with the
+// PRISM_EXCLUDE_SERIALIZATION define.
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
 #include <stdio.h>
 
 static inline uint32_t
@@ -32,7 +38,7 @@ pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location,
 }
 
 static void
-pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffer) {
+pm_serialize_string(const pm_parser_t *parser, const pm_string_t *string, pm_buffer_t *buffer) {
     switch (string->type) {
         case PM_STRING_SHARED: {
             pm_buffer_append_byte(buffer, 1);
@@ -48,9 +54,25 @@ pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffe
             pm_buffer_append_bytes(buffer, pm_string_source(string), length);
             break;
         }
+#ifdef PRISM_HAS_MMAP
         case PM_STRING_MAPPED:
             assert(false && "Cannot serialize mapped strings.");
             break;
+#endif
+    }
+}
+
+static void
+pm_serialize_integer(const pm_integer_t *integer, pm_buffer_t *buffer) {
+    pm_buffer_append_byte(buffer, integer->negative ? 1 : 0);
+    if (integer->values == NULL) {
+        pm_buffer_append_varuint(buffer, pm_sizet_to_u32(1));
+        pm_buffer_append_varuint(buffer, integer->value);
+    } else {
+        pm_buffer_append_varuint(buffer, pm_sizet_to_u32(integer->length));
+        for (size_t i = 0; i < integer->length; i++) {
+            pm_buffer_append_varuint(buffer, integer->values[i]);
+        }
     }
 }
 
@@ -385,8 +407,8 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             }
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->read_name));
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->write_name));
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->operator));
-            pm_serialize_location(parser, &((pm_call_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_call_operator_write_node_t *)node)->binary_operator));
+            pm_serialize_location(parser, &((pm_call_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_call_operator_write_node_t *)node)->value, buffer);
             break;
         }
@@ -507,9 +529,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         case PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE: {
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_class_variable_operator_write_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_class_variable_operator_write_node_t *)node)->name_loc, buffer);
-            pm_serialize_location(parser, &((pm_class_variable_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_class_variable_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_class_variable_operator_write_node_t *)node)->value, buffer);
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_class_variable_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_class_variable_operator_write_node_t *)node)->binary_operator));
             break;
         }
         case PM_CLASS_VARIABLE_OR_WRITE_NODE: {
@@ -531,12 +553,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_class_variable_write_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_class_variable_write_node_t *)node)->name_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_class_variable_write_node_t *)node)->value, buffer);
-            if (((pm_class_variable_write_node_t *)node)->operator_loc.start == NULL) {
-                pm_buffer_append_byte(buffer, 0);
-            } else {
-                pm_buffer_append_byte(buffer, 1);
-                pm_serialize_location(parser, &((pm_class_variable_write_node_t *)node)->operator_loc, buffer);
-            }
+            pm_serialize_location(parser, &((pm_class_variable_write_node_t *)node)->operator_loc, buffer);
             break;
         }
         case PM_CONSTANT_AND_WRITE_NODE: {
@@ -549,9 +566,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         case PM_CONSTANT_OPERATOR_WRITE_NODE: {
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_operator_write_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_constant_operator_write_node_t *)node)->name_loc, buffer);
-            pm_serialize_location(parser, &((pm_constant_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_constant_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_constant_operator_write_node_t *)node)->value, buffer);
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_operator_write_node_t *)node)->binary_operator));
             break;
         }
         case PM_CONSTANT_OR_WRITE_NODE: {
@@ -573,15 +590,16 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_node_t *)node)->parent, buffer);
             }
-            pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_node_t *)node)->child, buffer);
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_path_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_constant_path_node_t *)node)->delimiter_loc, buffer);
+            pm_serialize_location(parser, &((pm_constant_path_node_t *)node)->name_loc, buffer);
             break;
         }
         case PM_CONSTANT_PATH_OPERATOR_WRITE_NODE: {
             pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_operator_write_node_t *)node)->target, buffer);
-            pm_serialize_location(parser, &((pm_constant_path_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_constant_path_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_operator_write_node_t *)node)->value, buffer);
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_path_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_path_operator_write_node_t *)node)->binary_operator));
             break;
         }
         case PM_CONSTANT_PATH_OR_WRITE_NODE: {
@@ -596,8 +614,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_target_node_t *)node)->parent, buffer);
             }
-            pm_serialize_node(parser, (pm_node_t *)((pm_constant_path_target_node_t *)node)->child, buffer);
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_constant_path_target_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_constant_path_target_node_t *)node)->delimiter_loc, buffer);
+            pm_serialize_location(parser, &((pm_constant_path_target_node_t *)node)->name_loc, buffer);
             break;
         }
         case PM_CONSTANT_PATH_WRITE_NODE: {
@@ -787,6 +806,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_FLOAT_NODE: {
+            pm_buffer_append_double(buffer, ((pm_float_node_t *)node)->value);
             break;
         }
         case PM_FOR_NODE: {
@@ -832,9 +852,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         case PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE: {
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_global_variable_operator_write_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_global_variable_operator_write_node_t *)node)->name_loc, buffer);
-            pm_serialize_location(parser, &((pm_global_variable_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_global_variable_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_global_variable_operator_write_node_t *)node)->value, buffer);
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_global_variable_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_global_variable_operator_write_node_t *)node)->binary_operator));
             break;
         }
         case PM_GLOBAL_VARIABLE_OR_WRITE_NODE: {
@@ -1012,8 +1032,8 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             } else {
                 pm_serialize_node(parser, (pm_node_t *)((pm_index_operator_write_node_t *)node)->block, buffer);
             }
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_index_operator_write_node_t *)node)->operator));
-            pm_serialize_location(parser, &((pm_index_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_index_operator_write_node_t *)node)->binary_operator));
+            pm_serialize_location(parser, &((pm_index_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_index_operator_write_node_t *)node)->value, buffer);
             break;
         }
@@ -1073,9 +1093,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         case PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE: {
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_instance_variable_operator_write_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_instance_variable_operator_write_node_t *)node)->name_loc, buffer);
-            pm_serialize_location(parser, &((pm_instance_variable_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_instance_variable_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_instance_variable_operator_write_node_t *)node)->value, buffer);
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_instance_variable_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_instance_variable_operator_write_node_t *)node)->binary_operator));
             break;
         }
         case PM_INSTANCE_VARIABLE_OR_WRITE_NODE: {
@@ -1102,6 +1122,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         }
         case PM_INTEGER_NODE: {
             pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_serialize_integer(&((pm_integer_node_t *)node)->value, buffer);
             break;
         }
         case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: {
@@ -1127,6 +1148,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_INTERPOLATED_STRING_NODE: {
+            pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             if (((pm_interpolated_string_node_t *)node)->opening_loc.start == NULL) {
                 pm_buffer_append_byte(buffer, 0);
             } else {
@@ -1176,6 +1198,9 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_serialize_location(parser, &((pm_interpolated_x_string_node_t *)node)->closing_loc, buffer);
             break;
         }
+        case PM_IT_PARAMETERS_NODE: {
+            break;
+        }
         case PM_KEYWORD_HASH_NODE: {
             pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             uint32_t elements_size = pm_sizet_to_u32(((pm_keyword_hash_node_t *)node)->elements.size);
@@ -1228,10 +1253,10 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         }
         case PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE: {
             pm_serialize_location(parser, &((pm_local_variable_operator_write_node_t *)node)->name_loc, buffer);
-            pm_serialize_location(parser, &((pm_local_variable_operator_write_node_t *)node)->operator_loc, buffer);
+            pm_serialize_location(parser, &((pm_local_variable_operator_write_node_t *)node)->binary_operator_loc, buffer);
             pm_serialize_node(parser, (pm_node_t *)((pm_local_variable_operator_write_node_t *)node)->value, buffer);
             pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_local_variable_operator_write_node_t *)node)->name));
-            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_local_variable_operator_write_node_t *)node)->operator));
+            pm_buffer_append_varuint(buffer, pm_sizet_to_u32(((pm_local_variable_operator_write_node_t *)node)->binary_operator));
             pm_buffer_append_varuint(buffer, ((pm_local_variable_operator_write_node_t *)node)->depth);
             break;
         }
@@ -1602,6 +1627,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_RETURN_NODE: {
+            pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             pm_serialize_location(parser, &((pm_return_node_t *)node)->keyword_loc, buffer);
             if (((pm_return_node_t *)node)->arguments == NULL) {
                 pm_buffer_append_byte(buffer, 0);
@@ -1613,6 +1639,11 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
         case PM_SELF_NODE: {
             break;
         }
+        case PM_SHAREABLE_CONSTANT_NODE: {
+            pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
+            pm_serialize_node(parser, (pm_node_t *)((pm_shareable_constant_node_t *)node)->write, buffer);
+            break;
+        }
         case PM_SINGLETON_CLASS_NODE: {
             uint32_t locals_size = pm_sizet_to_u32(((pm_singleton_class_node_t *)node)->locals.size);
             pm_buffer_append_varuint(buffer, locals_size);
@@ -1634,6 +1665,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             break;
         }
         case PM_SOURCE_FILE_NODE: {
+            pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             pm_serialize_string(parser, &((pm_source_file_node_t *)node)->filepath, buffer);
             break;
         }
@@ -1787,6 +1819,12 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             for (uint32_t index = 0; index < conditions_size; index++) {
                 pm_serialize_node(parser, (pm_node_t *) ((pm_when_node_t *)node)->conditions.nodes[index], buffer);
             }
+            if (((pm_when_node_t *)node)->then_keyword_loc.start == NULL) {
+                pm_buffer_append_byte(buffer, 0);
+            } else {
+                pm_buffer_append_byte(buffer, 1);
+                pm_serialize_location(parser, &((pm_when_node_t *)node)->then_keyword_loc, buffer);
+            }
             if (((pm_when_node_t *)node)->statements == NULL) {
                 pm_buffer_append_byte(buffer, 0);
             } else {
@@ -1909,6 +1947,9 @@ pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) {
 
 static void
 pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) {
+    // serialize the type
+    pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id);
+
     // serialize message
     size_t message_length = strlen(diagnostic->message);
     pm_buffer_append_varuint(buffer, pm_sizet_to_u32(message_length));
@@ -1952,7 +1993,7 @@ pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) {
     pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer);
 }
 
-#line 243 "serialize.c.erb"
+#line 271 "serialize.c.erb"
 /**
  * Serialize the metadata, nodes, and constant pool.
  */
@@ -2084,23 +2125,4 @@ pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size,
     pm_options_free(&options);
 }
 
-/**
- * Parse the source and return true if it parses without errors or warnings.
- */
-PRISM_EXPORTED_FUNCTION bool
-pm_parse_success_p(const uint8_t *source, size_t size, const char *data) {
-    pm_options_t options = { 0 };
-    pm_options_read(&options, data);
-
-    pm_parser_t parser;
-    pm_parser_init(&parser, source, size, &options);
-
-    pm_node_t *node = pm_parse(&parser);
-    pm_node_destroy(&parser, node);
-
-    bool result = parser.error_list.size == 0 && parser.warning_list.size == 0;
-    pm_parser_free(&parser);
-    pm_options_free(&options);
-
-    return result;
-}
+#endif