prism 0.29.0 → 1.3.0

data/src/util/pm_string.c

@@ -47,6 +47,62 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
     };
 }
 
+#ifdef _WIN32
+/**
+ * Represents a file handle on Windows, where the path will need to be freed
+ * when the file is closed.
+ */
+typedef struct {
+    /** The path to the file, which will become allocated memory. */
+    WCHAR *path;
+
+    /** The handle to the file, which will start as uninitialized memory. */
+    HANDLE file;
+} pm_string_file_handle_t;
+
+/**
+ * Open the file indicated by the filepath parameter for reading on Windows.
+ * Perform any kind of normalization that needs to happen on the filepath.
+ */
+static pm_string_init_result_t
+pm_string_file_handle_open(pm_string_file_handle_t *handle, const char *filepath) {
+    int length = MultiByteToWideChar(CP_UTF8, 0, filepath, -1, NULL, 0);
+    if (length == 0) return PM_STRING_INIT_ERROR_GENERIC;
+
+    handle->path = xmalloc(sizeof(WCHAR) * ((size_t) length));
+    if ((handle->path == NULL) || (MultiByteToWideChar(CP_UTF8, 0, filepath, -1, handle->path, length) == 0)) {
+        xfree(handle->path);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    handle->file = CreateFileW(handle->path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);
+    if (handle->file == INVALID_HANDLE_VALUE) {
+        pm_string_init_result_t result = PM_STRING_INIT_ERROR_GENERIC;
+
+        if (GetLastError() == ERROR_ACCESS_DENIED) {
+            DWORD attributes = GetFileAttributesW(handle->path);
+            if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+                result = PM_STRING_INIT_ERROR_DIRECTORY;
+            }
+        }
+
+        xfree(handle->path);
+        return result;
+    }
+
+    return PM_STRING_INIT_SUCCESS;
+}
+
+/**
+ * Close the file handle and free the path.
+ */
+static void
+pm_string_file_handle_close(pm_string_file_handle_t *handle) {
+    xfree(handle->path);
+    CloseHandle(handle->file);
+}
+#endif
+
 /**
  * Read the file indicated by the filepath parameter into source and load its
  * contents and size into the given `pm_string_t`. The given `pm_string_t`
@@ -58,62 +114,66 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
  * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
  * `mmap`, and on other POSIX systems we'll use `read`.
  */
-PRISM_EXPORTED_FUNCTION bool
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t
 pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
     // Open the file for reading.
-    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-
-    if (file == INVALID_HANDLE_VALUE) {
-        return false;
-    }
+    pm_string_file_handle_t handle;
+    pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
+    if (result != PM_STRING_INIT_SUCCESS) return result;
 
     // Get the file size.
-    DWORD file_size = GetFileSize(file, NULL);
+    DWORD file_size = GetFileSize(handle.file, NULL);
     if (file_size == INVALID_FILE_SIZE) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // If the file is empty, then we don't need to do anything else, we'll set
     // the source to a constant empty string and return.
     if (file_size == 0) {
-        CloseHandle(file);
+        pm_string_file_handle_close(&handle);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
     // Create a mapping of the file.
-    HANDLE mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
+    HANDLE mapping = CreateFileMapping(handle.file, NULL, PAGE_READONLY, 0, 0, NULL);
     if (mapping == NULL) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Map the file into memory.
     uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
     CloseHandle(mapping);
-    CloseHandle(file);
+    pm_string_file_handle_close(&handle);
 
     if (source == NULL) {
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
-    return true;
+    return PM_STRING_INIT_SUCCESS;
 #elif defined(_POSIX_MAPPED_FILES)
     // Open the file for reading
     int fd = open(filepath, O_RDONLY);
     if (fd == -1) {
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Stat the file to get the file size
     struct stat sb;
     if (fstat(fd, &sb) == -1) {
         close(fd);
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
+
+    // Ensure it is a file and not a directory
+    if (S_ISDIR(sb.st_mode)) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_DIRECTORY;
     }
 
     // mmap the file descriptor to virtually get the contents
@@ -124,22 +184,20 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
         close(fd);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
     source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
     if (source == MAP_FAILED) {
-        return false;
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     close(fd);
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
-    return true;
+    return PM_STRING_INIT_SUCCESS;
 #else
-    (void) string;
-    (void) filepath;
-    perror("pm_string_mapped_init is not implemented for this platform");
-    return false;
+    return pm_string_file_init(string, filepath);
 #endif
 }
 
@@ -148,115 +206,108 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
  * contents and size into the given `pm_string_t`. The given `pm_string_t`
  * should be freed using `pm_string_free` when it is no longer used.
  */
-PRISM_EXPORTED_FUNCTION bool
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t
 pm_string_file_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
     // Open the file for reading.
-    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-
-    if (file == INVALID_HANDLE_VALUE) {
-        return false;
-    }
+    pm_string_file_handle_t handle;
+    pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
+    if (result != PM_STRING_INIT_SUCCESS) return result;
 
     // Get the file size.
-    DWORD file_size = GetFileSize(file, NULL);
+    DWORD file_size = GetFileSize(handle.file, NULL);
     if (file_size == INVALID_FILE_SIZE) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // If the file is empty, then we don't need to do anything else, we'll set
     // the source to a constant empty string and return.
     if (file_size == 0) {
-        CloseHandle(file);
+        pm_string_file_handle_close(&handle);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
     // Create a buffer to read the file into.
     uint8_t *source = xmalloc(file_size);
     if (source == NULL) {
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Read the contents of the file
     DWORD bytes_read;
-    if (!ReadFile(file, source, file_size, &bytes_read, NULL)) {
-        CloseHandle(file);
-        return false;
+    if (!ReadFile(handle.file, source, file_size, &bytes_read, NULL)) {
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     // Check the number of bytes read
     if (bytes_read != file_size) {
         xfree(source);
-        CloseHandle(file);
-        return false;
+        pm_string_file_handle_close(&handle);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
-    CloseHandle(file);
+    pm_string_file_handle_close(&handle);
     *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = (size_t) file_size };
-    return true;
-#elif defined(_POSIX_MAPPED_FILES)
-    FILE *file = fopen(filepath, "rb");
-    if (file == NULL) {
-        return false;
+    return PM_STRING_INIT_SUCCESS;
+#elif defined(PRISM_HAS_FILESYSTEM)
+    // Open the file for reading
+    int fd = open(filepath, O_RDONLY);
+    if (fd == -1) {
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
-    fseek(file, 0, SEEK_END);
-    long file_size = ftell(file);
+    // Stat the file to get the file size
+    struct stat sb;
+    if (fstat(fd, &sb) == -1) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
+    }
 
-    if (file_size == -1) {
-        fclose(file);
-        return false;
+    // Ensure it is a file and not a directory
+    if (S_ISDIR(sb.st_mode)) {
+        close(fd);
+        return PM_STRING_INIT_ERROR_DIRECTORY;
     }
 
-    if (file_size == 0) {
-        fclose(file);
+    // Check the size to see if it's empty
+    size_t size = (size_t) sb.st_size;
+    if (size == 0) {
+        close(fd);
         const uint8_t source[] = "";
         *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
-        return true;
+        return PM_STRING_INIT_SUCCESS;
     }
 
-    size_t length = (size_t) file_size;
+    size_t length = (size_t) size;
     uint8_t *source = xmalloc(length);
     if (source == NULL) {
-        fclose(file);
-        return false;
+        close(fd);
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
-    fseek(file, 0, SEEK_SET);
-    size_t bytes_read = fread(source, length, 1, file);
-    fclose(file);
+    long bytes_read = (long) read(fd, source, length);
+    close(fd);
 
-    if (bytes_read != 1) {
+    if (bytes_read == -1) {
         xfree(source);
-        return false;
+        return PM_STRING_INIT_ERROR_GENERIC;
     }
 
     *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = length };
-    return true;
+    return PM_STRING_INIT_SUCCESS;
 #else
     (void) string;
     (void) filepath;
     perror("pm_string_file_init is not implemented for this platform");
-    return false;
+    return PM_STRING_INIT_ERROR_GENERIC;
 #endif
 }
 
-/**
- * Returns the memory size associated with the string.
- */
-size_t
-pm_string_memsize(const pm_string_t *string) {
-    size_t size = sizeof(pm_string_t);
-    if (string->type == PM_STRING_OWNED) {
-        size += string->length;
-    }
-    return size;
-}
-
 /**
  * Ensure the string is owned. If it is not, then reinitialize it as owned and
  * copy over the previous source.

data/src/util/pm_strpbrk.c

@@ -8,6 +8,27 @@ pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start
     pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
 }
 
+/**
+ * Set the explicit encoding for the parser to the current encoding.
+ */
+static inline void
+pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
+    if (parser->explicit_encoding != NULL) {
+        if (parser->explicit_encoding == parser->encoding) {
+            // Okay, we already locked to this encoding.
+        } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
+            // Not okay, we already found a Unicode escape sequence and this
+            // conflicts.
+            pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
+        } else {
+            // Should not be anything else.
+            assert(false && "unreachable");
+        }
+    }
+
+    parser->explicit_encoding = parser->encoding;
+}
+
 /**
  * This is the default path.
  */
@@ -52,7 +73,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars
  * This is the path when the encoding is ASCII-8BIT.
  */
 static inline const uint8_t *
-pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) {
+pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
 
     while (index < maximum) {
@@ -60,6 +81,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
             return source + index;
         }
 
+        if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
         index++;
     }
 
@@ -72,6 +94,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
 static inline const uint8_t *
 pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
 
     while (index < maximum) {
         if (strchr((const char *) charset, source[index]) != NULL) {
@@ -81,7 +104,8 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
         if (source[index] < 0x80) {
             index++;
         } else {
-            size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
 
             if (width > 0) {
                 index += width;
@@ -96,7 +120,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 
                 do {
                     index++;
-                } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
 
                 pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
             }
@@ -113,6 +137,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 static inline const uint8_t *
 pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
     size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
 
     while (index < maximum) {
         if (strchr((const char *) charset, source[index]) != NULL) {
@@ -122,7 +147,8 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
         if (source[index] < 0x80 || !validate) {
             index++;
         } else {
-            size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            pm_strpbrk_explicit_encoding_set(parser, source, width);
 
             if (width > 0) {
                 index += width;
@@ -135,7 +161,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
 
                 do {
                     index++;
-                } while (index < maximum && parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
 
                 pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
             }
@@ -171,7 +197,7 @@ pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, p
     } else if (!parser->encoding_changed) {
         return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
     } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
-        return pm_strpbrk_ascii_8bit(source, charset, (size_t) length);
+        return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
     } else if (parser->encoding->multibyte) {
         return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
     } else {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.29.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-10 00:00:00.000000000 Z
+date: 2024-12-21 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -39,6 +39,7 @@ files:
 - docs/parser_translation.md
 - docs/parsing_rules.md
 - docs/releasing.md
+- docs/relocation.md
 - docs/ripper_translation.md
 - docs/ruby_api.md
 - docs/ruby_parser_translation.md
@@ -69,13 +70,11 @@ files:
 - include/prism/util/pm_memchr.h
 - include/prism/util/pm_newline_list.h
 - include/prism/util/pm_string.h
-- include/prism/util/pm_string_list.h
 - include/prism/util/pm_strncasecmp.h
 - include/prism/util/pm_strpbrk.h
 - include/prism/version.h
 - lib/prism.rb
 - lib/prism/compiler.rb
-- lib/prism/debug.rb
 - lib/prism/desugar_compiler.rb
 - lib/prism/dispatcher.rb
 - lib/prism/dot_visitor.rb
@@ -89,17 +88,19 @@ files:
 - lib/prism/pack.rb
 - lib/prism/parse_result.rb
 - lib/prism/parse_result/comments.rb
+- lib/prism/parse_result/errors.rb
 - lib/prism/parse_result/newlines.rb
 - lib/prism/pattern.rb
 - lib/prism/polyfill/byteindex.rb
 - lib/prism/polyfill/unpack1.rb
 - lib/prism/reflection.rb
+- lib/prism/relocation.rb
 - lib/prism/serialize.rb
+- lib/prism/string_query.rb
 - lib/prism/translation.rb
 - lib/prism/translation/parser.rb
 - lib/prism/translation/parser/compiler.rb
 - lib/prism/translation/parser/lexer.rb
-- lib/prism/translation/parser/rubocop.rb
 - lib/prism/translation/parser33.rb
 - lib/prism/translation/parser34.rb
 - lib/prism/translation/ripper.rb
@@ -110,11 +111,13 @@ files:
 - prism.gemspec
 - rbi/prism.rbi
 - rbi/prism/compiler.rbi
+- rbi/prism/dsl.rbi
 - rbi/prism/inspect_visitor.rbi
 - rbi/prism/node.rbi
 - rbi/prism/node_ext.rbi
 - rbi/prism/parse_result.rbi
 - rbi/prism/reflection.rbi
+- rbi/prism/string_query.rbi
 - rbi/prism/translation/parser.rbi
 - rbi/prism/translation/parser33.rbi
 - rbi/prism/translation/parser34.rbi
@@ -134,7 +137,9 @@ files:
 - sig/prism/parse_result.rbs
 - sig/prism/pattern.rbs
 - sig/prism/reflection.rbs
+- sig/prism/relocation.rbs
 - sig/prism/serialize.rbs
+- sig/prism/string_query.rbs
 - sig/prism/visitor.rbs
 - src/diagnostic.c
 - src/encoding.c
@@ -155,7 +160,6 @@ files:
 - src/util/pm_memchr.c
 - src/util/pm_newline_list.c
 - src/util/pm_string.c
-- src/util/pm_string_list.c
 - src/util/pm_strncasecmp.c
 - src/util/pm_strpbrk.c
 homepage: https://github.com/ruby/prism
@@ -180,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.0.dev
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Prism Ruby parser

data/include/prism/util/pm_string_list.h

@@ -1,44 +0,0 @@
-/**
- * @file pm_string_list.h
- *
- * A list of strings.
- */
-#ifndef PRISM_STRING_LIST_H
-#define PRISM_STRING_LIST_H
-
-#include "prism/defines.h"
-#include "prism/util/pm_string.h"
-
-#include <stddef.h>
-#include <stdlib.h>
-
-/**
- * A list of strings.
- */
-typedef struct {
-    /** The length of the string list. */
-    size_t length;
-
-    /** The capacity of the string list that has been allocated. */
-    size_t capacity;
-
-    /** A pointer to the start of the string list. */
-    pm_string_t *strings;
-} pm_string_list_t;
-
-/**
- * Append a pm_string_t to the given string list.
- *
- * @param string_list The string list to append to.
- * @param string The string to append.
- */
-void pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string);
-
-/**
- * Free the memory associated with the string list.
- *
- * @param string_list The string list to free.
- */
-PRISM_EXPORTED_FUNCTION void pm_string_list_free(pm_string_list_t *string_list);
-
-#endif