jruby-prism-parser 0.24.0-java → 1.4.0-java

data/src/util/pm_newline_list.c

@@ -6,7 +6,7 @@
  */
 bool
 pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity) {
-    list->offsets = (size_t *) calloc(capacity, sizeof(size_t));
+    list->offsets = (size_t *) xcalloc(capacity, sizeof(size_t));
     if (list->offsets == NULL) return false;
 
     list->start = start;
@@ -19,6 +19,14 @@ pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capac
     return true;
 }
 
+/**
+ * Clear out the newlines that have been appended to the list.
+ */
+void
+pm_newline_list_clear(pm_newline_list_t *list) {
+    list->size = 1;
+}
+
 /**
  * Append a new offset to the newline list. Returns true if the reallocation of
  * the offsets succeeds (if one was necessary), otherwise returns false.
@@ -29,10 +37,11 @@ pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) {
         size_t *original_offsets = list->offsets;
 
         list->capacity = (list->capacity * 3) / 2;
-        list->offsets = (size_t *) calloc(list->capacity, sizeof(size_t));
-        memcpy(list->offsets, original_offsets, list->size * sizeof(size_t));
-        free(original_offsets);
+        list->offsets = (size_t *) xcalloc(list->capacity, sizeof(size_t));
         if (list->offsets == NULL) return false;
+
+        memcpy(list->offsets, original_offsets, list->size * sizeof(size_t));
+        xfree(original_offsets);
     }
 
     assert(*cursor == '\n');
@@ -45,6 +54,35 @@ pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) {
     return true;
 }
 
+/**
+ * Returns the line of the given offset. If the offset is not in the list, the
+ * line of the closest offset less than the given offset is returned.
+ */
+int32_t
+pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line) {
+    assert(cursor >= list->start);
+    size_t offset = (size_t) (cursor - list->start);
+
+    size_t left = 0;
+    size_t right = list->size - 1;
+
+    while (left <= right) {
+        size_t mid = left + (right - left) / 2;
+
+        if (list->offsets[mid] == offset) {
+            return ((int32_t) mid) + start_line;
+        }
+
+        if (list->offsets[mid] < offset) {
+            left = mid + 1;
+        } else {
+            right = mid - 1;
+        }
+    }
+
+    return ((int32_t) left) + start_line - 1;
+}
+
 /**
  * Returns the line and column of the given offset. If the offset is not in the
  * list, the line and column of the closest offset less than the given offset
@@ -83,5 +121,5 @@ pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor
  */
 void
 pm_newline_list_free(pm_newline_list_t *list) {
-    free(list->offsets);
+    xfree(list->offsets);
 }

data/src/util/pm_string.c

@@ -47,6 +47,62 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
     };
 }
 
+#ifdef _WIN32
+/**
+ * Represents a file handle on Windows, where the path will need to be freed
+ * when the file is closed.
+ */
+typedef struct {
+    /** The path to the file, which will become allocated memory. */
+    WCHAR *path;
+
+    /** The handle to the file, which will start as uninitialized memory. */
+    HANDLE file;
+} pm_string_file_handle_t;
+
+/**
+ * Open the file indicated by the filepath parameter for reading on Windows.
+ * Perform any kind of normalization that needs to happen on the filepath.
+ */
+static pm_string_init_result_t
+pm_string_file_handle_open(pm_string_file_handle_t *handle, const char *filepath) {
+    int length = MultiByteToWideChar(CP_UTF8, 0, filepath, -1, NULL, 0);
+    if (length == 0) return PM_STRING_INIT_ERROR_GENERIC;
+
+    handle->path = xmalloc(sizeof(WCHAR) * ((size_t) length));
+    if ((handle->path == NULL) || (MultiByteToWideChar(CP_UTF8, 0, filepath, -1, handle->path, length) == 0)) {
+        xfree(handle->path);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    handle->file = CreateFileW(handle->path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);
+    if (handle->file == INVALID_HANDLE_VALUE) {
+        pm_string_init_result_t result = PM_STRING_INIT_ERROR_GENERIC;
+
+        if (GetLastError() == ERROR_ACCESS_DENIED) {
+            DWORD attributes = GetFileAttributesW(handle->path);
+            if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+                result = PM_STRING_INIT_ERROR_DIRECTORY;
+            }
+        }
+
+        xfree(handle->path);
+        return result;
+    }
+
+    return PM_STRING_INIT_SUCCESS;
+}
+
+/**
+ * Close the file handle and free the path.
+ */
+static void
+pm_string_file_handle_close(pm_string_file_handle_t *handle) {
+    xfree(handle->path);
+    CloseHandle(handle->file);
+}
+#endif
+
 /**
  * Read the file indicated by the filepath parameter into source and load its
  * contents and size into the given `pm_string_t`. The given `pm_string_t`
@@ -58,62 +114,66 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
  * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
  * `mmap`, and on other POSIX systems we'll use `read`.
  */
-bool
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t
 pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
     // Open the file for reading.
-    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-
-    if (file == INVALID_HANDLE_VALUE) {
-        return false;
-    }
+    pm_string_file_handle_t handle;
+    pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
+    if (result != PM_STRING_INIT_SUCCESS) return result;
 
     // Get the file size.
-    DWORD file_size = GetFileSize(file, NULL);
+    DWORD file_size = GetFileSize(handle.file, NULL);
     if (file_size == INVALID_FILE_SIZE) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // If the file is empty, then we don't need to do anything else, we'll set
     // the source to a constant empty string and return.
     if (file_size == 0) {
-        CloseHandle(file);
+        pm_string_file_handle_close(&handle);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
     // Create a mapping of the file.
-    HANDLE mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
+    HANDLE mapping = CreateFileMapping(handle.file, NULL, PAGE_READONLY, 0, 0, NULL);
     if (mapping == NULL) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Map the file into memory.
     uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
     CloseHandle(mapping);
-    CloseHandle(file);
+    pm_string_file_handle_close(&handle);
 
     if (source == NULL) {
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
-    return true;
-#else
+    return PM_STRING_INIT_SUCCESS;
+#elif defined(_POSIX_MAPPED_FILES)
     // Open the file for reading
     int fd = open(filepath, O_RDONLY);
     if (fd == -1) {
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Stat the file to get the file size
     struct stat sb;
     if (fstat(fd, &sb) == -1) {
         close(fd);
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Ensure it is a file and not a directory
+    if (S_ISDIR(sb.st_mode)) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_DIRECTORY;
     }
 
     // mmap the file descriptor to virtually get the contents
@@ -124,30 +184,128 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
         close(fd);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
     source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
     if (source == MAP_FAILED) {
-        return false;
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     close(fd);
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
-    return true;
+    return PM_STRING_INIT_SUCCESS;
+#else
+    return pm_string_file_init(string, filepath);
 #endif
 }
 
 /**
- * Returns the memory size associated with the string.
+ * Read the file indicated by the filepath parameter into source and load its
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
+ * should be freed using `pm_string_free` when it is no longer used.
  */
-size_t
-pm_string_memsize(const pm_string_t *string) {
-    size_t size = sizeof(pm_string_t);
-    if (string->type == PM_STRING_OWNED) {
-        size += string->length;
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t
+pm_string_file_init(pm_string_t *string, const char *filepath) {
+#ifdef _WIN32
+    // Open the file for reading.
+    pm_string_file_handle_t handle;
+    pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
+    if (result != PM_STRING_INIT_SUCCESS) return result;
+
+    // Get the file size.
+    DWORD file_size = GetFileSize(handle.file, NULL);
+    if (file_size == INVALID_FILE_SIZE) {
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // If the file is empty, then we don't need to do anything else, we'll set
+    // the source to a constant empty string and return.
+    if (file_size == 0) {
+        pm_string_file_handle_close(&handle);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return PM_STRING_INIT_SUCCESS;
+    }
+
+    // Create a buffer to read the file into.
+    uint8_t *source = xmalloc(file_size);
+    if (source == NULL) {
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Read the contents of the file
+    DWORD bytes_read;
+    if (!ReadFile(handle.file, source, file_size, &bytes_read, NULL)) {
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Check the number of bytes read
+    if (bytes_read != file_size) {
+        xfree(source);
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    pm_string_file_handle_close(&handle);
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = (size_t) file_size };
+    return PM_STRING_INIT_SUCCESS;
+#elif defined(PRISM_HAS_FILESYSTEM)
+    // Open the file for reading
+    int fd = open(filepath, O_RDONLY);
+    if (fd == -1) {
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Stat the file to get the file size
+    struct stat sb;
+    if (fstat(fd, &sb) == -1) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Ensure it is a file and not a directory
+    if (S_ISDIR(sb.st_mode)) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_DIRECTORY;
+    }
+
+    // Check the size to see if it's empty
+    size_t size = (size_t) sb.st_size;
+    if (size == 0) {
+        close(fd);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return PM_STRING_INIT_SUCCESS;
+    }
+
+    size_t length = (size_t) size;
+    uint8_t *source = xmalloc(length);
+    if (source == NULL) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    long bytes_read = (long) read(fd, source, length);
+    close(fd);
+
+    if (bytes_read == -1) {
+        xfree(source);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
-    return size;
+
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = length };
+    return PM_STRING_INIT_SUCCESS;
+#else
+    (void) string;
+    (void) filepath;
+    perror("pm_string_file_init is not implemented for this platform");
+    return PM_STRING_INIT_ERROR_GENERIC;
+#endif
 }
 
 /**
@@ -161,13 +319,33 @@ pm_string_ensure_owned(pm_string_t *string) {
     size_t length = pm_string_length(string);
     const uint8_t *source = pm_string_source(string);
 
-    uint8_t *memory = malloc(length);
+    uint8_t *memory = xmalloc(length);
     if (!memory) return;
 
     pm_string_owned_init(string, memory, length);
     memcpy((void *) string->source, source, length);
 }
 
+/**
+ * Compare the underlying lengths and bytes of two strings. Returns 0 if the
+ * strings are equal, a negative number if the left string is less than the
+ * right string, and a positive number if the left string is greater than the
+ * right string.
+ */
+int
+pm_string_compare(const pm_string_t *left, const pm_string_t *right) {
+    size_t left_length = pm_string_length(left);
+    size_t right_length = pm_string_length(right);
+
+    if (left_length < right_length) {
+        return -1;
+    } else if (left_length > right_length) {
+        return 1;
+    }
+
+    return memcmp(pm_string_source(left), pm_string_source(right), left_length);
+}
+
 /**
  * Returns the length associated with the string.
  */
@@ -192,12 +370,14 @@ pm_string_free(pm_string_t *string) {
     void *memory = (void *) string->source;
 
     if (string->type == PM_STRING_OWNED) {
-        free(memory);
+        xfree(memory);
+#ifdef PRISM_HAS_MMAP
     } else if (string->type == PM_STRING_MAPPED && string->length) {
 #if defined(_WIN32)
         UnmapViewOfFile(memory);
-#else
+#elif defined(_POSIX_MAPPED_FILES)
         munmap(memory, string->length);
 #endif
+#endif /* PRISM_HAS_MMAP */
     }
 }

data/src/util/pm_strncasecmp.c

@@ -1,5 +1,17 @@
 #include "prism/util/pm_strncasecmp.h"
 
+/**
+ * A locale-insensitive version of `tolower(3)`
+ */
+static inline int
+pm_tolower(int c)
+{
+    if ('A' <= c && c <= 'Z') {
+        return c | 0x20;
+    }
+    return c;
+}
+
 /**
  * Compare two strings, ignoring case, up to the given length. Returns 0 if the
  * strings are equal, a negative number if string1 is less than string2, or a
@@ -16,7 +28,7 @@ pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) {
 
     while (offset < length && string1[offset] != '\0') {
         if (string2[offset] == '\0') return string1[offset];
-        if ((difference = tolower(string1[offset]) - tolower(string2[offset])) != 0) return difference;
+        if ((difference = pm_tolower(string1[offset]) - pm_tolower(string2[offset])) != 0) return difference;
         offset++;
     }
 

data/src/util/pm_strpbrk.c

@@ -8,6 +8,27 @@ pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start
     pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
 }
 
+/**
+ * Set the explicit encoding for the parser to the current encoding.
+ */
+static inline void
+pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
+    if (parser->explicit_encoding != NULL) {
+        if (parser->explicit_encoding == parser->encoding) {
+            // Okay, we already locked to this encoding.
+        } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
+            // Not okay, we already found a Unicode escape sequence and this
+            // conflicts.
+            pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
+        } else {
+            // Should not be anything else.
+            assert(false && "unreachable");
+        }
+    }
+
+    parser->explicit_encoding = parser->encoding;
+}
+
 /**
  * This is the default path.
  */
@@ -52,7 +73,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars
  * This is the path when the encoding is ASCII-8BIT.
  */
 static inline const uint8_t *
-pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) {
+pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
 
     while (index < maximum) {
@@ -60,6 +81,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
             return source + index;
         }
 
+        if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
         index++;
     }
 
@@ -72,6 +94,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
 static inline const uint8_t *
 pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
 
     while (index < maximum) {
         if (strchr((const char *) charset, source[index]) != NULL) {
@@ -81,7 +104,8 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
         if (source[index] < 0x80) {
             index++;
         } else {
-            size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
 
             if (width > 0) {
                 index += width;
@@ -96,7 +120,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 
                 do {
                     index++;
-                } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
 
                 pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
             }
@@ -113,6 +137,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 static inline const uint8_t *
 pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
 
     while (index < maximum) {
         if (strchr((const char *) charset, source[index]) != NULL) {
@@ -122,7 +147,8 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
         if (source[index] < 0x80 || !validate) {
             index++;
         } else {
-            size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            pm_strpbrk_explicit_encoding_set(parser, source, width);
 
             if (width > 0) {
                 index += width;
@@ -135,7 +161,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 
                 do {
                     index++;
-                } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
 
                 pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
             }
@@ -171,7 +197,7 @@ pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, p
     } else if (!parser->encoding_changed) {
         return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
     } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
-        return pm_strpbrk_ascii_8bit(source, charset, (size_t) length);
+        return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
     } else if (parser->encoding->multibyte) {
         return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
     } else {