yarp 0.10.0 → 0.12.0

data/src/unescape.c

@@ -94,7 +94,7 @@ static inline size_t
 unescape_hexadecimal(const uint8_t *backslash, uint8_t *value, const uint8_t *end, yp_list_t *error_list) {
     *value = 0;
     if (backslash + 2 >= end || !yp_char_is_hexadecimal_digit(backslash[2])) {
-        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid hex escape.");
+        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_HEXADECIMAL);
         return 2;
     }
     *value = unescape_hexadecimal_digit(backslash[2]);
@@ -157,7 +157,7 @@ unescape_unicode_write(uint8_t *dest, uint32_t value, const uint8_t *start, cons
     // If we get here, then the value is too big. This is an error, but we don't
     // want to just crash, so instead we'll add an error to the error list and put
     // in a replacement character instead.
-    if (error_list) yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence.");
+    if (error_list) yp_diagnostic_list_append(error_list, start, end, YP_ERR_ESCAPE_INVALID_UNICODE);
     dest[0] = 0xEF;
     dest[1] = 0xBF;
     dest[2] = 0xBD;
@@ -235,7 +235,7 @@ unescape(
         // \unnnn       Unicode character, where nnnn is exactly 4 hexadecimal digits ([0-9a-fA-F])
         case 'u': {
             if ((flags & YP_UNESCAPE_FLAG_CONTROL) | (flags & YP_UNESCAPE_FLAG_META)) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags.");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS);
                 return backslash + 2;
             }
 
@@ -252,11 +252,11 @@ unescape(
 
                     // \u{nnnn} character literal allows only 1-6 hexadecimal digits
                     if (hexadecimal_length > 6) {
-                        if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape.");
+                        if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, YP_ERR_ESCAPE_INVALID_UNICODE_LONG);
                     }
                     // there are not hexadecimal characters
                     else if (hexadecimal_length == 0) {
-                        if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape");
+                        if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, YP_ERR_ESCAPE_INVALID_UNICODE);
                         return unicode_cursor;
                     }
 
@@ -277,13 +277,13 @@ unescape(
 
                 // ?\u{nnnn} character literal should contain only one codepoint and cannot be like ?\u{nnnn mmmm}
                 if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) {
-                    if (error_list) yp_diagnostic_list_append(error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal");
+                    if (error_list) yp_diagnostic_list_append(error_list, extra_codepoints_start, unicode_cursor - 1, YP_ERR_ESCAPE_INVALID_UNICODE_LITERAL);
                 }
 
                 if (unicode_cursor < end && *unicode_cursor == '}') {
                     unicode_cursor++;
                 } else {
-                    if (error_list) yp_diagnostic_list_append(error_list, backslash, unicode_cursor, "invalid Unicode escape.");
+                    if (error_list) yp_diagnostic_list_append(error_list, backslash, unicode_cursor, YP_ERR_ESCAPE_INVALID_UNICODE_TERM);
                 }
 
                 return unicode_cursor;
@@ -298,7 +298,7 @@ unescape(
                 return backslash + 6;
             }
 
-            if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid Unicode escape sequence");
+            if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_UNICODE);
             return backslash + 2;
         }
         // \c\M-x       meta control character, where x is an ASCII printable character
@@ -306,12 +306,12 @@ unescape(
         // \cx          control character, where x is an ASCII printable character
         case 'c':
             if (backslash + 2 >= end) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL);
                 return end;
             }
 
             if (flags & YP_UNESCAPE_FLAG_CONTROL) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled.");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL_REPEAT);
                 return backslash + 2;
             }
 
@@ -325,7 +325,7 @@ unescape(
                     return backslash + 3;
                 default: {
                     if (!char_is_ascii_printable(backslash[2])) {
-                        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence");
+                        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL);
                         return backslash + 2;
                     }
 
@@ -339,17 +339,17 @@ unescape(
         // \C-?         delete, ASCII 7Fh (DEL)
         case 'C':
             if (backslash + 3 >= end) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL);
                 return end;
             }
 
             if (flags & YP_UNESCAPE_FLAG_CONTROL) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled.");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL_REPEAT);
                 return backslash + 2;
             }
 
             if (backslash[2] != '-') {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_CONTROL);
                 return backslash + 2;
             }
 
@@ -363,7 +363,7 @@ unescape(
                     return backslash + 4;
                 default:
                     if (!char_is_ascii_printable(backslash[3])) {
-                        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid control escape sequence");
+                        if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_CONTROL);
                         return backslash + 2;
                     }
 
@@ -377,17 +377,17 @@ unescape(
         // \M-x         meta character, where x is an ASCII printable character
         case 'M': {
             if (backslash + 3 >= end) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, YP_ERR_ESCAPE_INVALID_META);
                 return end;
             }
 
             if (flags & YP_UNESCAPE_FLAG_META) {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled.");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_META_REPEAT);
                 return backslash + 2;
             }
 
             if (backslash[2] != '-') {
-                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence");
+                if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_META);
                 return backslash + 2;
             }
 
@@ -402,7 +402,7 @@ unescape(
                 return backslash + 4;
             }
 
-            if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence");
+            if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, YP_ERR_ESCAPE_INVALID_META);
             return backslash + 3;
         }
         // \n
@@ -474,7 +474,7 @@ yp_unescape_manipulate_string_or_char_literal(yp_parser_t *parser, yp_string_t *
     // within the string.
     uint8_t *allocated = malloc(string->length);
     if (allocated == NULL) {
-        yp_diagnostic_list_append(&parser->error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping.");
+        yp_diagnostic_list_append(&parser->error_list, string->source, string->source + string->length, YP_ERR_MALLOC_FAILED);
         return;
     }
 

data/src/util/yp_char.c

@@ -75,7 +75,7 @@ yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length) {
 // whitespace while also tracking the location of each newline. Disallows
 // searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) {
+yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list) {
     if (length <= 0) return 0;
 
     size_t size = 0;
@@ -83,12 +83,7 @@ yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newlin
 
     while (size < maximum && (yp_byte_table[string[size]] & YP_CHAR_BIT_WHITESPACE)) {
         if (string[size] == '\n') {
-            if (stop_at_newline) {
-                return size + 1;
-            }
-            else {
-                yp_newline_list_append(newline_list, string + size);
-            }
+            yp_newline_list_append(newline_list, string + size);
         }
 
         size++;
@@ -128,6 +123,9 @@ yp_char_is_inline_whitespace(const uint8_t b) {
     return yp_char_is_char_kind(b, YP_CHAR_BIT_INLINE_WHITESPACE);
 }
 
+// Scan through the string and return the number of characters at the start of
+// the string that match the given kind. Disallows searching past the given
+// maximum number of characters.
 static inline size_t
 yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     if (length <= 0) return 0;
@@ -139,20 +137,57 @@ yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     return size;
 }
 
+// Scan through the string and return the number of characters at the start of
+// the string that match the given kind. Disallows searching past the given
+// maximum number of characters.
+//
+// Additionally, report the location of the last invalid underscore character
+// found in the string through the out invalid parameter.
+static inline size_t
+yp_strspn_number_kind_underscores(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid, uint8_t kind) {
+    if (length <= 0) return 0;
+
+    size_t size = 0;
+    size_t maximum = (size_t) length;
+
+    bool underscore = false;
+    while (size < maximum && (yp_number_table[string[size]] & kind)) {
+        if (string[size] == '_') {
+            if (underscore) *invalid = string + size;
+            underscore = true;
+        } else {
+            underscore = false;
+        }
+
+        size++;
+    }
+
+    if (string[size - 1] == '_') *invalid = string + size - 1;
+    return size;
+}
+
 // Returns the number of characters at the start of the string that are binary
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
 size_t
-yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length) {
-    return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_BINARY_NUMBER);
+yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
+    return yp_strspn_number_kind_underscores(string, length, invalid, YP_NUMBER_BIT_BINARY_NUMBER);
 }
 
 // Returns the number of characters at the start of the string that are octal
-// digits or underscores.  Disallows searching past the given maximum number of
+// digits or underscores. Disallows searching past the given maximum number of
 // characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
 size_t
-yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length) {
-    return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_OCTAL_NUMBER);
+yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
+    return yp_strspn_number_kind_underscores(string, length, invalid, YP_NUMBER_BIT_OCTAL_NUMBER);
 }
 
 // Returns the number of characters at the start of the string that are decimal
@@ -165,9 +200,13 @@ yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) {
 // Returns the number of characters at the start of the string that are decimal
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
 size_t
-yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length) {
-    return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_NUMBER);
+yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
+    return yp_strspn_number_kind_underscores(string, length, invalid, YP_NUMBER_BIT_DECIMAL_NUMBER);
 }
 
 // Returns the number of characters at the start of the string that are
@@ -181,9 +220,13 @@ yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) {
 // Returns the number of characters at the start of the string that are
 // hexadecimal digits or underscores. Disallows searching past the given maximum
 // number of characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
 size_t
-yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length) {
-    return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_NUMBER);
+yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
+    return yp_strspn_number_kind_underscores(string, length, invalid, YP_NUMBER_BIT_HEXADECIMAL_NUMBER);
 }
 
 static inline bool

data/src/util/yp_constant_pool.c

@@ -59,10 +59,42 @@ yp_constant_pool_hash(const uint8_t *start, size_t length) {
     return value;
 }
 
+// https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+static size_t
+next_power_of_two(size_t v) {
+    // Avoid underflow in subtraction on next line.
+    if (v == 0) {
+        // 1 is the nearest power of 2 to 0 (2^0)
+        return 1;
+    }
+    v--;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+#if defined(__LP64__) || defined(_WIN64)
+    v |= v >> 32;
+#endif
+    v++;
+    return v;
+}
+
+#ifndef NDEBUG
+static bool
+is_power_of_two(size_t size) {
+    return (size & (size - 1)) == 0;
+}
+#endif
+
 // Resize a constant pool to a given capacity.
 static inline bool
 yp_constant_pool_resize(yp_constant_pool_t *pool) {
+    assert(is_power_of_two(pool->capacity));
     size_t next_capacity = pool->capacity * 2;
+    if (next_capacity < pool->capacity) return false;
+
+    const size_t mask = next_capacity - 1;
     yp_constant_t *next_constants = calloc(next_capacity, sizeof(yp_constant_t));
     if (next_constants == NULL) return false;
 
@@ -74,13 +106,13 @@ yp_constant_pool_resize(yp_constant_pool_t *pool) {
         // If an id is set on this constant, then we know we have content here.
         // In this case we need to insert it into the next constant pool.
         if (constant->id != 0) {
-            size_t next_index = constant->hash % next_capacity;
+            size_t next_index = constant->hash & mask;
 
             // This implements linear scanning to find the next available slot
             // in case this index is already taken. We don't need to bother
             // comparing the values since we know that the hash is unique.
             while (next_constants[next_index].id != 0) {
-                next_index = (next_index + 1) % next_capacity;
+                next_index = (next_index + 1) & mask;
             }
 
             // Here we copy over the entire constant, which includes the id so
@@ -98,6 +130,10 @@ yp_constant_pool_resize(yp_constant_pool_t *pool) {
 // Initialize a new constant pool with a given capacity.
 bool
 yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity) {
+    const size_t size_t_max = (~((size_t) 0));
+    if (capacity >= ((size_t_max / 2) + 1)) return false;
+
+    capacity = next_power_of_two(capacity);
     pool->constants = calloc(capacity, sizeof(yp_constant_t));
     if (pool->constants == NULL) return false;
 
@@ -106,16 +142,17 @@ yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity) {
     return true;
 }
 
-// Insert a constant into a constant pool. Returns the id of the constant, or 0
-// if any potential calls to resize fail.
-yp_constant_id_t
-yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length) {
+// Insert a constant into a constant pool and return its index in the pool.
+static inline yp_constant_id_t
+yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length, bool owned) {
     if (pool->size >= (pool->capacity / 4 * 3)) {
         if (!yp_constant_pool_resize(pool)) return 0;
     }
 
+    assert(is_power_of_two(pool->capacity));
+    const size_t mask = pool->capacity - 1;
     size_t hash = yp_constant_pool_hash(start, length);
-    size_t index = hash % pool->capacity;
+    size_t index = hash & mask;
     yp_constant_t *constant;
 
     while (constant = &pool->constants[index], constant->id != 0) {
@@ -123,25 +160,72 @@ yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t l
         // same as the content we are trying to insert. If it is, then we can
         // return the id of the existing constant.
         if ((constant->length == length) && memcmp(constant->start, start, length) == 0) {
-            return pool->constants[index].id;
+            // Since we have found a match, we need to check if this is
+            // attempting to insert a shared or an owned constant. We want to
+            // prefer shared constants since they don't require allocations.
+            if (owned) {
+                // If we're attempting to insert an owned constant and we have
+                // an existing constant, then either way we don't want the given
+                // memory. Either it's duplicated with the existing constant or
+                // it's not necessary because we have a shared version.
+                free((void *) start);
+            } else if (constant->owned) {
+                // If we're attempting to insert a shared constant and the
+                // existing constant is owned, then we can free the owned
+                // constant and replace it with the shared constant.
+                free((void *) constant->start);
+                constant->start = start;
+                constant->owned = false;
+            }
+
+            return constant->id;
         }
 
-        index = (index + 1) % pool->capacity;
+        index = (index + 1) & mask;
     }
 
-    yp_constant_id_t id = (yp_constant_id_t)++pool->size;
-    pool->constants[index] = (yp_constant_t) {
-        .id = id,
+    pool->size++;
+    assert(pool->size < ((size_t) (1 << 31)));
+
+    *constant = (yp_constant_t) {
+        .id = (unsigned int) (pool->size & 0x7FFFFFFF),
+        .owned = owned,
         .start = start,
         .length = length,
         .hash = hash
     };
 
-    return id;
+    return constant->id;
+}
+
+// Insert a constant into a constant pool. Returns the id of the constant, or 0
+// if any potential calls to resize fail.
+yp_constant_id_t
+yp_constant_pool_insert_shared(yp_constant_pool_t *pool, const uint8_t *start, size_t length) {
+    return yp_constant_pool_insert(pool, start, length, false);
+}
+
+// Insert a constant into a constant pool from memory that is now owned by the
+// constant pool. Returns the id of the constant, or 0 if any potential calls to
+// resize fail.
+yp_constant_id_t
+yp_constant_pool_insert_owned(yp_constant_pool_t *pool, const uint8_t *start, size_t length) {
+    return yp_constant_pool_insert(pool, start, length, true);
 }
 
 // Free the memory associated with a constant pool.
 void
 yp_constant_pool_free(yp_constant_pool_t *pool) {
+    // For each constant in the current constant pool, free the contents if the
+    // contents are owned.
+    for (uint32_t index = 0; index < pool->capacity; index++) {
+        yp_constant_t *constant = &pool->constants[index];
+
+        // If an id is set on this constant, then we know we have content here.
+        if (constant->id != 0 && constant->owned) {
+            free((void *) constant->start);
+        }
+    }
+
     free(pool->constants);
 }

data/src/util/yp_newline_list.c

@@ -25,8 +25,12 @@ yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capac
 bool
 yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor) {
     if (list->size == list->capacity) {
+        size_t *original_offsets = list->offsets;
+
         list->capacity = (list->capacity * 3) / 2;
-        list->offsets = (size_t *) realloc(list->offsets, list->capacity * sizeof(size_t));
+        list->offsets = (size_t *) calloc(list->capacity, sizeof(size_t));
+        memcpy(list->offsets, original_offsets, list->size * sizeof(size_t));
+        free(original_offsets);
         if (list->offsets == NULL) return false;
     }
 

data/src/util/yp_string_list.c

@@ -12,8 +12,11 @@ yp_string_list_init(yp_string_list_t *string_list) {
 void
 yp_string_list_append(yp_string_list_t *string_list, yp_string_t *string) {
     if (string_list->length + 1 > string_list->capacity) {
+        yp_string_t *original_string = string_list->strings;
         string_list->capacity *= 2;
-        string_list->strings = (yp_string_t *) realloc(string_list->strings, string_list->capacity * sizeof(yp_string_t));
+        string_list->strings = (yp_string_t *) malloc(string_list->capacity * sizeof(yp_string_t));
+        memcpy(string_list->strings, original_string, (string_list->length) * sizeof(yp_string_t));
+        free(original_string);
     }
 
     string_list->strings[string_list->length++] = *string;