yarp 0.8.0 → 0.10.0

data/src/util/yp_buffer.c

@@ -63,8 +63,13 @@ yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length) {
 // Append a string to the buffer.
 void
 yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length) {
-    const void *source = value;
-    yp_buffer_append(buffer, source, length);
+    yp_buffer_append(buffer, value, length);
+}
+
+// Append a list of bytes to the buffer.
+void
+yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length) {
+    yp_buffer_append(buffer, (const char *) value, length);
 }
 
 // Append a single byte to the buffer.

data/src/util/yp_char.c

@@ -13,8 +13,8 @@
 #define YP_NUMBER_BIT_HEXADECIMAL_DIGIT (1 << 6)
 #define YP_NUMBER_BIT_HEXADECIMAL_NUMBER (1 << 7)
 
-static const unsigned char yp_char_table[256] = {
-//0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
+static const uint8_t yp_byte_table[256] = {
+//  0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
     0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 0, 0, // 0x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
     3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x
@@ -33,7 +33,7 @@ static const unsigned char yp_char_table[256] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx
 };
 
-static const unsigned char yp_number_table[256] = {
+static const uint8_t yp_number_table[256] = {
     // 0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 1x
@@ -54,20 +54,20 @@ static const unsigned char yp_number_table[256] = {
 };
 
 static inline size_t
-yp_strspn_char_kind(const char *string, ptrdiff_t length, unsigned char kind) {
+yp_strspn_char_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     if (length <= 0) return 0;
 
     size_t size = 0;
     size_t maximum = (size_t) length;
 
-    while (size < maximum && (yp_char_table[(unsigned char) string[size]] & kind)) size++;
+    while (size < maximum && (yp_byte_table[string[size]] & kind)) size++;
     return size;
 }
 
 // Returns the number of characters at the start of the string that are
 // whitespace. Disallows searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace(const char *string, ptrdiff_t length) {
+yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_char_kind(string, length, YP_CHAR_BIT_WHITESPACE);
 }
 
@@ -75,13 +75,13 @@ yp_strspn_whitespace(const char *string, ptrdiff_t length) {
 // whitespace while also tracking the location of each newline. Disallows
 // searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) {
+yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) {
     if (length <= 0) return 0;
 
     size_t size = 0;
     size_t maximum = (size_t) length;
 
-    while (size < maximum && (yp_char_table[(unsigned char) string[size]] & YP_CHAR_BIT_WHITESPACE)) {
+    while (size < maximum && (yp_byte_table[string[size]] & YP_CHAR_BIT_WHITESPACE)) {
         if (string[size] == '\n') {
             if (stop_at_newline) {
                 return size + 1;
@@ -100,42 +100,42 @@ yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_l
 // Returns the number of characters at the start of the string that are inline
 // whitespace. Disallows searching past the given maximum number of characters.
 size_t
-yp_strspn_inline_whitespace(const char *string, ptrdiff_t length) {
+yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_char_kind(string, length, YP_CHAR_BIT_INLINE_WHITESPACE);
 }
 
 // Returns the number of characters at the start of the string that are regexp
 // options. Disallows searching past the given maximum number of characters.
 size_t
-yp_strspn_regexp_option(const char *string, ptrdiff_t length) {
+yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_char_kind(string, length, YP_CHAR_BIT_REGEXP_OPTION);
 }
 
 static inline bool
-yp_char_is_char_kind(const char c, unsigned char kind) {
-    return (yp_char_table[(unsigned char) c] & kind) != 0;
+yp_char_is_char_kind(const uint8_t b, uint8_t kind) {
+    return (yp_byte_table[b] & kind) != 0;
 }
 
 // Returns true if the given character is a whitespace character.
 bool
-yp_char_is_whitespace(const char c) {
-    return yp_char_is_char_kind(c, YP_CHAR_BIT_WHITESPACE);
+yp_char_is_whitespace(const uint8_t b) {
+    return yp_char_is_char_kind(b, YP_CHAR_BIT_WHITESPACE);
 }
 
 // Returns true if the given character is an inline whitespace character.
 bool
-yp_char_is_inline_whitespace(const char c) {
-    return yp_char_is_char_kind(c, YP_CHAR_BIT_INLINE_WHITESPACE);
+yp_char_is_inline_whitespace(const uint8_t b) {
+    return yp_char_is_char_kind(b, YP_CHAR_BIT_INLINE_WHITESPACE);
 }
 
 static inline size_t
-yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind) {
+yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     if (length <= 0) return 0;
 
     size_t size = 0;
     size_t maximum = (size_t) length;
 
-    while (size < maximum && (yp_number_table[(unsigned char) string[size]] & kind)) size++;
+    while (size < maximum && (yp_number_table[string[size]] & kind)) size++;
     return size;
 }
 
@@ -143,7 +143,7 @@ yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind)
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
 size_t
-yp_strspn_binary_number(const char *string, ptrdiff_t length) {
+yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_BINARY_NUMBER);
 }
 
@@ -151,14 +151,14 @@ yp_strspn_binary_number(const char *string, ptrdiff_t length) {
 // digits or underscores.  Disallows searching past the given maximum number of
 // characters.
 size_t
-yp_strspn_octal_number(const char *string, ptrdiff_t length) {
+yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_OCTAL_NUMBER);
 }
 
 // Returns the number of characters at the start of the string that are decimal
 // digits. Disallows searching past the given maximum number of characters.
 size_t
-yp_strspn_decimal_digit(const char *string, ptrdiff_t length) {
+yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_DIGIT);
 }
 
@@ -166,7 +166,7 @@ yp_strspn_decimal_digit(const char *string, ptrdiff_t length) {
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
 size_t
-yp_strspn_decimal_number(const char *string, ptrdiff_t length) {
+yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_NUMBER);
 }
 
@@ -174,7 +174,7 @@ yp_strspn_decimal_number(const char *string, ptrdiff_t length) {
 // hexadecimal digits. Disallows searching past the given maximum number of
 // characters.
 size_t
-yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) {
+yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_DIGIT);
 }
 
@@ -182,37 +182,37 @@ yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) {
 // hexadecimal digits or underscores. Disallows searching past the given maximum
 // number of characters.
 size_t
-yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length) {
+yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length) {
     return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_NUMBER);
 }
 
 static inline bool
-yp_char_is_number_kind(const char c, unsigned char kind) {
-    return (yp_number_table[(unsigned char) c] & kind) != 0;
+yp_char_is_number_kind(const uint8_t b, uint8_t kind) {
+    return (yp_number_table[b] & kind) != 0;
 }
 
 // Returns true if the given character is a binary digit.
 bool
-yp_char_is_binary_digit(const char c) {
-    return yp_char_is_number_kind(c, YP_NUMBER_BIT_BINARY_DIGIT);
+yp_char_is_binary_digit(const uint8_t b) {
+    return yp_char_is_number_kind(b, YP_NUMBER_BIT_BINARY_DIGIT);
 }
 
 // Returns true if the given character is an octal digit.
 bool
-yp_char_is_octal_digit(const char c) {
-    return yp_char_is_number_kind(c, YP_NUMBER_BIT_OCTAL_DIGIT);
+yp_char_is_octal_digit(const uint8_t b) {
+    return yp_char_is_number_kind(b, YP_NUMBER_BIT_OCTAL_DIGIT);
 }
 
 // Returns true if the given character is a decimal digit.
 bool
-yp_char_is_decimal_digit(const char c) {
-    return yp_char_is_number_kind(c, YP_NUMBER_BIT_DECIMAL_DIGIT);
+yp_char_is_decimal_digit(const uint8_t b) {
+    return yp_char_is_number_kind(b, YP_NUMBER_BIT_DECIMAL_DIGIT);
 }
 
 // Returns true if the given character is a hexadecimal digit.
 bool
-yp_char_is_hexadecimal_digit(const char c) {
-    return yp_char_is_number_kind(c, YP_NUMBER_BIT_HEXADECIMAL_DIGIT);
+yp_char_is_hexadecimal_digit(const uint8_t b) {
+    return yp_char_is_number_kind(b, YP_NUMBER_BIT_HEXADECIMAL_DIGIT);
 }
 
 #undef YP_CHAR_BIT_WHITESPACE

data/src/util/yp_constant_pool.c

@@ -48,12 +48,12 @@ yp_constant_id_list_free(yp_constant_id_list_t *list) {
 // A relatively simple hash function (djb2) that is used to hash strings. We are
 // optimizing here for simplicity and speed.
 static inline size_t
-yp_constant_pool_hash(const char *start, size_t length) {
+yp_constant_pool_hash(const uint8_t *start, size_t length) {
     // This is a prime number used as the initial value for the hash function.
     size_t value = 5381;
 
     for (size_t index = 0; index < length; index++) {
-        value = ((value << 5) + value) + ((unsigned char) start[index]);
+        value = ((value << 5) + value) + start[index];
     }
 
     return value;
@@ -109,7 +109,7 @@ yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity) {
 // Insert a constant into a constant pool. Returns the id of the constant, or 0
 // if any potential calls to resize fail.
 yp_constant_id_t
-yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length) {
+yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length) {
     if (pool->size >= (pool->capacity / 4 * 3)) {
         if (!yp_constant_pool_resize(pool)) return 0;
     }
@@ -122,7 +122,7 @@ yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t leng
         // If there is a collision, then we need to check if the content is the
         // same as the content we are trying to insert. If it is, then we can
         // return the id of the existing constant.
-        if ((constant->length == length) && strncmp(constant->start, start, length) == 0) {
+        if ((constant->length == length) && memcmp(constant->start, start, length) == 0) {
             return pool->constants[index].id;
         }
 

data/src/util/yp_memchr.c

@@ -8,7 +8,7 @@
 void *
 yp_memchr(const void *memory, int character, size_t number, bool encoding_changed, yp_encoding_t *encoding) {
     if (encoding_changed && encoding->multibyte && character >= YP_MEMCHR_TRAILING_BYTE_MINIMUM) {
-        const char *source = (const char *) memory;
+        const uint8_t *source = (const uint8_t *) memory;
         size_t index = 0;
 
         while (index < number) {

data/src/util/yp_newline_list.c

@@ -3,7 +3,7 @@
 // Initialize a new newline list with the given capacity. Returns true if the
 // allocation of the offsets succeeds, otherwise returns false.
 bool
-yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity) {
+yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity) {
     list->offsets = (size_t *) calloc(capacity, sizeof(size_t));
     if (list->offsets == NULL) return false;
 
@@ -23,21 +23,32 @@ yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity
 // Append a new offset to the newline list. Returns true if the reallocation of
 // the offsets succeeds (if one was necessary), otherwise returns false.
 bool
-yp_newline_list_append(yp_newline_list_t *list, const char *cursor) {
+yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor) {
     if (list->size == list->capacity) {
         list->capacity = (list->capacity * 3) / 2;
         list->offsets = (size_t *) realloc(list->offsets, list->capacity * sizeof(size_t));
         if (list->offsets == NULL) return false;
     }
 
+    assert(*cursor == '\n');
     assert(cursor >= list->start);
     size_t newline_offset = (size_t) (cursor - list->start + 1);
+
     assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]);
     list->offsets[list->size++] = newline_offset;
 
     return true;
 }
 
+// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+bool
+yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor) {
+    if (*cursor != '\n') {
+        return true;
+    }
+    return yp_newline_list_append(list, cursor);
+}
+
 // Returns the line and column of the given offset, assuming we don't have any
 // information about the previous index that we found.
 static yp_line_column_t
@@ -95,7 +106,7 @@ yp_newline_list_line_column_scan(yp_newline_list_t *list, size_t offset) {
 // list, the line and column of the closest offset less than the given offset
 // are returned.
 yp_line_column_t
-yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor) {
+yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor) {
     assert(cursor >= list->start);
     size_t offset = (size_t) (cursor - list->start);
     yp_line_column_t result;

data/src/util/yp_string.c

@@ -12,18 +12,19 @@
 
 // Initialize a shared string that is based on initial input.
 void
-yp_string_shared_init(yp_string_t *string, const char *start, const char *end) {
+yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end) {
     assert(start <= end);
+
     *string = (yp_string_t) {
         .type = YP_STRING_SHARED,
-        .source = (char*) start,
+        .source = start,
         .length = (size_t) (end - start)
     };
 }
 
 // Initialize an owned string that is responsible for freeing allocated memory.
 void
-yp_string_owned_init(yp_string_t *string, char *source, size_t length) {
+yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length) {
     *string = (yp_string_t) {
         .type = YP_STRING_OWNED,
         .source = source,
@@ -36,13 +37,13 @@ void
 yp_string_constant_init(yp_string_t *string, const char *source, size_t length) {
     *string = (yp_string_t) {
         .type = YP_STRING_CONSTANT,
-        .source = (char*) source,
+        .source = (const uint8_t *) source,
         .length = length
     };
 }
 
 static void
-yp_string_mapped_init_internal(yp_string_t *string, char *source, size_t length) {
+yp_string_mapped_init_internal(yp_string_t *string, uint8_t *source, size_t length) {
     *string = (yp_string_t) {
         .type = YP_STRING_MAPPED,
         .source = source,
@@ -67,13 +68,13 @@ yp_string_ensure_owned(yp_string_t *string) {
     if (string->type == YP_STRING_OWNED) return;
 
     size_t length = yp_string_length(string);
-    const char *source = yp_string_source(string);
+    const uint8_t *source = yp_string_source(string);
 
-    char *memory = malloc(length);
+    uint8_t *memory = malloc(length);
     if (!memory) return;
 
     yp_string_owned_init(string, memory, length);
-    memcpy(string->source, source, length);
+    memcpy((void *) string->source, source, length);
 }
 
 // Returns the length associated with the string.
@@ -83,7 +84,7 @@ yp_string_length(const yp_string_t *string) {
 }
 
 // Returns the start pointer associated with the string.
-YP_EXPORTED_FUNCTION const char *
+YP_EXPORTED_FUNCTION const uint8_t *
 yp_string_source(const yp_string_t *string) {
     return string->source;
 }
@@ -91,15 +92,16 @@ yp_string_source(const yp_string_t *string) {
 // Free the associated memory of the given string.
 YP_EXPORTED_FUNCTION void
 yp_string_free(yp_string_t *string) {
+    void *memory = (void *) string->source;
+
     if (string->type == YP_STRING_OWNED) {
-        free(string->source);
+        free(memory);
     } else if (string->type == YP_STRING_MAPPED && string->length) {
-        void *memory = (void *) string->source;
-        #if defined(_WIN32)
+#if defined(_WIN32)
         UnmapViewOfFile(memory);
-        #else
+#else
         munmap(memory, string->length);
-        #endif
+#endif
     }
 }
 
@@ -126,8 +128,8 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) {
     // the source to a constant empty string and return.
     if (file_size == 0) {
         CloseHandle(file);
-        char empty_string[] = "";
-        yp_string_mapped_init_internal(string, empty_string, 0);
+        uint8_t empty[] = "";
+        yp_string_mapped_init_internal(string, empty, 0);
         return true;
     }
 
@@ -140,7 +142,7 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) {
     }
 
     // Map the file into memory.
-    char *source = (char *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+    uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
     CloseHandle(mapping);
     CloseHandle(file);
 
@@ -169,12 +171,12 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) {
 
     // mmap the file descriptor to virtually get the contents
     size_t size = (size_t) sb.st_size;
-    char *source = NULL;
+    uint8_t *source = NULL;
 
     if (size == 0) {
         close(fd);
-        char empty_string[] = "";
-        yp_string_mapped_init_internal(string, empty_string, 0);
+        uint8_t empty[] = "";
+        yp_string_mapped_init_internal(string, empty, 0);
         return true;
     }
 

data/src/util/yp_string_list.c

@@ -1,11 +1,5 @@
 #include "yarp/util/yp_string_list.h"
 
-// Allocate a new yp_string_list_t.
-yp_string_list_t *
-yp_string_list_alloc(void) {
-    return (yp_string_list_t *) malloc(sizeof(yp_string_list_t));
-}
-
 // Initialize a yp_string_list_t with its default values.
 void
 yp_string_list_init(yp_string_list_t *string_list) {

data/src/util/yp_strncasecmp.c

@@ -1,18 +1,15 @@
 #include <ctype.h>
 #include <stddef.h>
+#include <stdint.h>
 
 int
-yp_strncasecmp(const char *string1, const char *string2, size_t length) {
+yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) {
     size_t offset = 0;
     int difference = 0;
 
     while (offset < length && string1[offset] != '\0') {
         if (string2[offset] == '\0') return string1[offset];
-
-        unsigned char left = (unsigned char) string1[offset];
-        unsigned char right = (unsigned char) string2[offset];
-
-        if ((difference = tolower(left) - tolower(right)) != 0) return difference;
+        if ((difference = tolower(string1[offset]) - tolower(string2[offset])) != 0) return difference;
         offset++;
     }
 

data/src/util/yp_strpbrk.c

@@ -1,12 +1,12 @@
 #include "yarp/util/yp_strpbrk.h"
 
 // This is the slow path that does care about the encoding.
-static inline const char *
-yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *charset, size_t maximum) {
+static inline const uint8_t *
+yp_strpbrk_multi_byte(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) {
     size_t index = 0;
 
     while (index < maximum) {
-        if (strchr(charset, source[index]) != NULL) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
             return source + index;
         }
 
@@ -22,12 +22,12 @@ yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *chars
 }
 
 // This is the fast path that does not care about the encoding.
-static inline const char *
-yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum) {
+static inline const uint8_t *
+yp_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t maximum) {
     size_t index = 0;
 
     while (index < maximum) {
-        if (strchr(charset, source[index]) != NULL) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
             return source + index;
         }
 
@@ -54,8 +54,8 @@ yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum)
 // characters that are trailing bytes of multi-byte characters. For example, in
 // Shift-JIS, the backslash character can be a trailing byte. In that case we
 // need to take a slower path and iterate one multi-byte character at a time.
-const char *
-yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length) {
+const uint8_t *
+yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) {
     if (length <= 0) {
         return NULL;
     } else if (parser->encoding_changed && parser->encoding.multibyte) {