prism 0.15.1 → 0.17.0

data/src/util/pm_list.c

@@ -1,18 +1,24 @@
 #include "prism/util/pm_list.h"
 
-// Returns true if the given list is empty.
+/**
+ * Returns true if the given list is empty.
+ */
 PRISM_EXPORTED_FUNCTION bool
 pm_list_empty_p(pm_list_t *list) {
     return list->head == NULL;
 }
 
-// Returns the size of the list.
+/**
+ * Returns the size of the list.
+ */
 PRISM_EXPORTED_FUNCTION size_t
 pm_list_size(pm_list_t *list) {
     return list->size;
 }
 
-// Append a node to the given list.
+/**
+ * Append a node to the given list.
+ */
 void
 pm_list_append(pm_list_t *list, pm_list_node_t *node) {
     if (list->head == NULL) {
@@ -25,7 +31,9 @@ pm_list_append(pm_list_t *list, pm_list_node_t *node) {
     list->size++;
 }
 
-// Deallocate the internal state of the given list.
+/**
+ * Deallocate the internal state of the given list.
+ */
 PRISM_EXPORTED_FUNCTION void
 pm_list_free(pm_list_t *list) {
     pm_list_node_t *node = list->head;

data/src/util/pm_memchr.c

@@ -2,9 +2,11 @@
 
 #define PRISM_MEMCHR_TRAILING_BYTE_MINIMUM 0x40
 
-// We need to roll our own memchr to handle cases where the encoding changes and
-// we need to search for a character in a buffer that could be the trailing byte
-// of a multibyte character.
+/**
+ * We need to roll our own memchr to handle cases where the encoding changes and
+ * we need to search for a character in a buffer that could be the trailing byte
+ * of a multibyte character.
+ */
 void *
 pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, pm_encoding_t *encoding) {
     if (encoding_changed && encoding->multibyte && character >= PRISM_MEMCHR_TRAILING_BYTE_MINIMUM) {

data/src/util/pm_newline_list.c

@@ -1,7 +1,9 @@
 #include "prism/util/pm_newline_list.h"
 
-// Initialize a new newline list with the given capacity. Returns true if the
-// allocation of the offsets succeeds, otherwise returns false.
+/**
+ * Initialize a new newline list with the given capacity. Returns true if the
+ * allocation of the offsets succeeds, otherwise returns false.
+ */
 bool
 pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity) {
     list->offsets = (size_t *) calloc(capacity, sizeof(size_t));
@@ -14,14 +16,13 @@ pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capac
     list->size = 1;
     list->capacity = capacity;
 
-    list->last_index = 0;
-    list->last_offset = 0;
-
     return true;
 }
 
-// Append a new offset to the newline list. Returns true if the reallocation of
-// the offsets succeeds (if one was necessary), otherwise returns false.
+/**
+ * Append a new offset to the newline list. Returns true if the reallocation of
+ * the offsets succeeds (if one was necessary), otherwise returns false.
+ */
 bool
 pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) {
     if (list->size == list->capacity) {
@@ -44,7 +45,10 @@ pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) {
     return true;
 }
 
-// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+/**
+ * Conditionally append a new offset to the newline list, if the value passed in
+ * is a newline.
+ */
 bool
 pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor) {
     if (*cursor != '\n') {
@@ -53,10 +57,16 @@ pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor) {
     return pm_newline_list_append(list, cursor);
 }
 
-// Returns the line and column of the given offset, assuming we don't have any
-// information about the previous index that we found.
-static pm_line_column_t
-pm_newline_list_line_column_search(pm_newline_list_t *list, size_t offset) {
+/**
+ * Returns the line and column of the given offset. If the offset is not in the
+ * list, the line and column of the closest offset less than the given offset
+ * are returned.
+ */
+pm_line_column_t
+pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor) {
+    assert(cursor >= list->start);
+    size_t offset = (size_t) (cursor - list->start);
+
     size_t left = 0;
     size_t right = list->size - 1;
 
@@ -77,57 +87,9 @@ pm_newline_list_line_column_search(pm_newline_list_t *list, size_t offset) {
     return ((pm_line_column_t) { left - 1, offset - list->offsets[left - 1] });
 }
 
-// Returns the line and column of the given offset, assuming we know the last
-// index that we found.
-static pm_line_column_t
-pm_newline_list_line_column_scan(pm_newline_list_t *list, size_t offset) {
-    if (offset > list->last_offset) {
-        size_t index = list->last_index;
-        while (index < list->size && list->offsets[index] < offset) {
-            index++;
-        }
-
-        if (index == list->size) {
-            return ((pm_line_column_t) { index - 1, offset - list->offsets[index - 1] });
-        }
-
-        return ((pm_line_column_t) { index, 0 });
-    } else {
-        size_t index = list->last_index;
-        while (index > 0 && list->offsets[index] > offset) {
-            index--;
-        }
-
-        if (index == 0) {
-            return ((pm_line_column_t) { 0, offset });
-        }
-
-        return ((pm_line_column_t) { index, offset - list->offsets[index - 1] });
-    }
-}
-
-// Returns the line and column of the given offset. If the offset is not in the
-// list, the line and column of the closest offset less than the given offset
-// are returned.
-pm_line_column_t
-pm_newline_list_line_column(pm_newline_list_t *list, const uint8_t *cursor) {
-    assert(cursor >= list->start);
-    size_t offset = (size_t) (cursor - list->start);
-    pm_line_column_t result;
-
-    if (list->last_offset == 0) {
-        result = pm_newline_list_line_column_search(list, offset);
-    } else {
-        result = pm_newline_list_line_column_scan(list, offset);
-    }
-
-    list->last_index = result.line;
-    list->last_offset = offset;
-
-    return result;
-}
-
-// Free the internal memory allocated for the newline list.
+/**
+ * Free the internal memory allocated for the newline list.
+ */
 void
 pm_newline_list_free(pm_newline_list_t *list) {
     free(list->offsets);

data/src/util/pm_state_stack.c

@@ -1,18 +1,24 @@
 #include "prism/util/pm_state_stack.h"
 
-// Pushes a value onto the stack.
+/**
+ * Pushes a value onto the stack.
+ */
 void
 pm_state_stack_push(pm_state_stack_t *stack, bool value) {
     *stack = (*stack << 1) | (value & 1);
 }
 
-// Pops a value off the stack.
+/**
+ * Pops a value off the stack.
+ */
 void
 pm_state_stack_pop(pm_state_stack_t *stack) {
     *stack >>= 1;
 }
 
-// Returns the value at the top of the stack.
+/**
+ * Returns the value at the top of the stack.
+ */
 bool
 pm_state_stack_p(pm_state_stack_t *stack) {
     return *stack & 1;

data/src/util/pm_string.c

@@ -1,16 +1,17 @@
 #include "prism/util/pm_string.h"
 
-// The following headers are necessary to read files using demand paging.
-#ifdef _WIN32
-#include <windows.h>
-#else
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#endif
+/**
+ * Returns the size of the pm_string_t struct. This is necessary to allocate the
+ * correct amount of memory in the FFI backend.
+ */
+PRISM_EXPORTED_FUNCTION size_t
+pm_string_sizeof(void) {
+    return sizeof(pm_string_t);
+}
 
-// Initialize a shared string that is based on initial input.
+/**
+ * Initialize a shared string that is based on initial input.
+ */
 void
 pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end) {
     assert(start <= end);
@@ -22,7 +23,9 @@ pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *
     };
 }
 
-// Initialize an owned string that is responsible for freeing allocated memory.
+/**
+ * Initialize an owned string that is responsible for freeing allocated memory.
+ */
 void
 pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length) {
     *string = (pm_string_t) {
@@ -32,7 +35,9 @@ pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length) {
     };
 }
 
-// Initialize a constant string that doesn't own its memory source.
+/**
+ * Initialize a constant string that doesn't own its memory source.
+ */
 void
 pm_string_constant_init(pm_string_t *string, const char *source, size_t length) {
     *string = (pm_string_t) {
@@ -42,69 +47,17 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
     };
 }
 
-static void
-pm_string_mapped_init_internal(pm_string_t *string, uint8_t *source, size_t length) {
-    *string = (pm_string_t) {
-        .type = PM_STRING_MAPPED,
-        .source = source,
-        .length = length
-    };
-}
-
-// Returns the memory size associated with the string.
-size_t
-pm_string_memsize(const pm_string_t *string) {
-    size_t size = sizeof(pm_string_t);
-    if (string->type == PM_STRING_OWNED) {
-        size += string->length;
-    }
-    return size;
-}
-
-// Ensure the string is owned. If it is not, then reinitialize it as owned and
-// copy over the previous source.
-void
-pm_string_ensure_owned(pm_string_t *string) {
-    if (string->type == PM_STRING_OWNED) return;
-
-    size_t length = pm_string_length(string);
-    const uint8_t *source = pm_string_source(string);
-
-    uint8_t *memory = malloc(length);
-    if (!memory) return;
-
-    pm_string_owned_init(string, memory, length);
-    memcpy((void *) string->source, source, length);
-}
-
-// Returns the length associated with the string.
-PRISM_EXPORTED_FUNCTION size_t
-pm_string_length(const pm_string_t *string) {
-    return string->length;
-}
-
-// Returns the start pointer associated with the string.
-PRISM_EXPORTED_FUNCTION const uint8_t *
-pm_string_source(const pm_string_t *string) {
-    return string->source;
-}
-
-// Free the associated memory of the given string.
-PRISM_EXPORTED_FUNCTION void
-pm_string_free(pm_string_t *string) {
-    void *memory = (void *) string->source;
-
-    if (string->type == PM_STRING_OWNED) {
-        free(memory);
-    } else if (string->type == PM_STRING_MAPPED && string->length) {
-#if defined(_WIN32)
-        UnmapViewOfFile(memory);
-#else
-        munmap(memory, string->length);
-#endif
-    }
-}
-
+/**
+ * Read the file indicated by the filepath parameter into source and load its
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
+ * should be freed using `pm_string_free` when it is no longer used.
+ *
+ * We want to use demand paging as much as possible in order to avoid having to
+ * read the entire file into memory (which could be detrimental to performance
+ * for large files). This means that if we're on windows we'll use
+ * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
+ * `mmap`, and on other POSIX systems we'll use `read`.
+ */
 bool
 pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
@@ -128,8 +81,8 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     // the source to a constant empty string and return.
     if (file_size == 0) {
         CloseHandle(file);
-        uint8_t empty[] = "";
-        pm_string_mapped_init_internal(string, empty, 0);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
         return true;
     }
 
@@ -151,7 +104,7 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
         return false;
     }
 
-    pm_string_mapped_init_internal(string, source, (size_t) file_size);
+    *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
     return true;
 #else
     // Open the file for reading
@@ -175,8 +128,8 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 
     if (size == 0) {
         close(fd);
-        uint8_t empty[] = "";
-        pm_string_mapped_init_internal(string, empty, 0);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
         return true;
     }
 
@@ -187,14 +140,71 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     }
 
     close(fd);
-    pm_string_mapped_init_internal(string, source, size);
+    *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
     return true;
 #endif
 }
 
-// Returns the size of the pm_string_t struct. This is necessary to allocate the
-// correct amount of memory in the FFI backend.
+/**
+ * Returns the memory size associated with the string.
+ */
+size_t
+pm_string_memsize(const pm_string_t *string) {
+    size_t size = sizeof(pm_string_t);
+    if (string->type == PM_STRING_OWNED) {
+        size += string->length;
+    }
+    return size;
+}
+
+/**
+ * Ensure the string is owned. If it is not, then reinitialize it as owned and
+ * copy over the previous source.
+ */
+void
+pm_string_ensure_owned(pm_string_t *string) {
+    if (string->type == PM_STRING_OWNED) return;
+
+    size_t length = pm_string_length(string);
+    const uint8_t *source = pm_string_source(string);
+
+    uint8_t *memory = malloc(length);
+    if (!memory) return;
+
+    pm_string_owned_init(string, memory, length);
+    memcpy((void *) string->source, source, length);
+}
+
+/**
+ * Returns the length associated with the string.
+ */
 PRISM_EXPORTED_FUNCTION size_t
-pm_string_sizeof(void) {
-    return sizeof(pm_string_t);
+pm_string_length(const pm_string_t *string) {
+    return string->length;
+}
+
+/**
+ * Returns the start pointer associated with the string.
+ */
+PRISM_EXPORTED_FUNCTION const uint8_t *
+pm_string_source(const pm_string_t *string) {
+    return string->source;
+}
+
+/**
+ * Free the associated memory of the given string.
+ */
+PRISM_EXPORTED_FUNCTION void
+pm_string_free(pm_string_t *string) {
+    void *memory = (void *) string->source;
+
+    if (string->type == PM_STRING_OWNED) {
+        free(memory);
+    } else if (string->type == PM_STRING_MAPPED && string->length) {
+#if defined(_WIN32)
+        UnmapViewOfFile(memory);
+#else
+        munmap(memory, string->length);
+#endif
+    }
 }

data/src/util/pm_string_list.c

@@ -1,28 +1,27 @@
 #include "prism/util/pm_string_list.h"
 
-// Initialize a pm_string_list_t with its default values.
-void
-pm_string_list_init(pm_string_list_t *string_list) {
-    string_list->strings = (pm_string_t *) malloc(sizeof(pm_string_t));
-    string_list->length = 0;
-    string_list->capacity = 1;
-}
-
-// Append a pm_string_t to the given string list.
+/**
+ * Append a pm_string_t to the given string list.
+ */
 void
 pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
     if (string_list->length + 1 > string_list->capacity) {
-        pm_string_t *original_string = string_list->strings;
-        string_list->capacity *= 2;
-        string_list->strings = (pm_string_t *) malloc(string_list->capacity * sizeof(pm_string_t));
-        memcpy(string_list->strings, original_string, (string_list->length) * sizeof(pm_string_t));
-        free(original_string);
+        if (string_list->capacity == 0) {
+            string_list->capacity = 1;
+        } else {
+            string_list->capacity *= 2;
+        }
+
+        string_list->strings = realloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
+        if (string_list->strings == NULL) abort();
     }
 
     string_list->strings[string_list->length++] = *string;
 }
 
-// Free the memory associated with the string list.
+/**
+ * Free the memory associated with the string list
+ */
 void
 pm_string_list_free(pm_string_list_t *string_list) {
     free(string_list->strings);

data/src/util/pm_strncasecmp.c

@@ -1,7 +1,14 @@
-#include <ctype.h>
-#include <stddef.h>
-#include <stdint.h>
+#include "prism/util/pm_strncasecmp.h"
 
+/**
+ * Compare two strings, ignoring case, up to the given length. Returns 0 if the
+ * strings are equal, a negative number if string1 is less than string2, or a
+ * positive number if string1 is greater than string2.
+ *
+ * Note that this is effectively our own implementation of strncasecmp, but it's
+ * not available on all of the platforms we want to support so we're rolling it
+ * here.
+ */
 int
 pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) {
     size_t offset = 0;

data/src/util/pm_strpbrk.c

@@ -1,6 +1,8 @@
 #include "prism/util/pm_strpbrk.h"
 
-// This is the slow path that does care about the encoding.
+/**
+ * This is the slow path that does care about the encoding.
+ */
 static inline const uint8_t *
 pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) {
     size_t index = 0;
@@ -21,7 +23,9 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
     return NULL;
 }
 
-// This is the fast path that does not care about the encoding.
+/**
+ * This is the fast path that does not care about the encoding.
+ */
 static inline const uint8_t *
 pm_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t maximum) {
     size_t index = 0;
@@ -37,23 +41,25 @@ pm_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t max
     return NULL;
 }
 
-// Here we have rolled our own version of strpbrk. The standard library strpbrk
-// has undefined behavior when the source string is not null-terminated. We want
-// to support strings that are not null-terminated because pm_parse does not
-// have the contract that the string is null-terminated. (This is desirable
-// because it means the extension can call pm_parse with the result of a call to
-// mmap).
-//
-// The standard library strpbrk also does not support passing a maximum length
-// to search. We want to support this for the reason mentioned above, but we
-// also don't want it to stop on null bytes. Ruby actually allows null bytes
-// within strings, comments, regular expressions, etc. So we need to be able to
-// skip past them.
-//
-// Finally, we want to support encodings wherein the charset could contain
-// characters that are trailing bytes of multi-byte characters. For example, in
-// Shift-JIS, the backslash character can be a trailing byte. In that case we
-// need to take a slower path and iterate one multi-byte character at a time.
+/**
+ * Here we have rolled our own version of strpbrk. The standard library strpbrk
+ * has undefined behavior when the source string is not null-terminated. We want
+ * to support strings that are not null-terminated because pm_parse does not
+ * have the contract that the string is null-terminated. (This is desirable
+ * because it means the extension can call pm_parse with the result of a call to
+ * mmap).
+ *
+ * The standard library strpbrk also does not support passing a maximum length
+ * to search. We want to support this for the reason mentioned above, but we
+ * also don't want it to stop on null bytes. Ruby actually allows null bytes
+ * within strings, comments, regular expressions, etc. So we need to be able to
+ * skip past them.
+ *
+ * Finally, we want to support encodings wherein the charset could contain
+ * characters that are trailing bytes of multi-byte characters. For example, in
+ * Shift-JIS, the backslash character can be a trailing byte. In that case we
+ * need to take a slower path and iterate one multi-byte character at a time.
+ */
 const uint8_t *
 pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) {
     if (length <= 0) {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.17.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-18 00:00:00.000000000 Z
+date: 2023-11-03 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -32,8 +32,9 @@ files:
 - docs/encoding.md
 - docs/fuzzing.md
 - docs/heredocs.md
+- docs/javascript.md
 - docs/mapping.md
-- docs/prism.png
+- docs/releasing.md
 - docs/ripper.md
 - docs/ruby_api.md
 - docs/serialization.md
@@ -49,8 +50,10 @@ files:
 - include/prism/diagnostic.h
 - include/prism/enc/pm_encoding.h
 - include/prism/node.h
+- include/prism/options.h
 - include/prism/pack.h
 - include/prism/parser.h
+- include/prism/prettyprint.h
 - include/prism/regexp.h
 - include/prism/util/pm_buffer.h
 - include/prism/util/pm_char.h
@@ -61,6 +64,7 @@ files:
 - include/prism/util/pm_state_stack.h
 - include/prism/util/pm_string.h
 - include/prism/util/pm_string_list.h
+- include/prism/util/pm_strncasecmp.h
 - include/prism/util/pm_strpbrk.h
 - include/prism/version.h
 - lib/prism.rb
@@ -84,6 +88,10 @@ files:
 - lib/prism/serialize.rb
 - lib/prism/visitor.rb
 - prism.gemspec
+- rbi/prism.rbi
+- rbi/prism_static.rbi
+- sig/prism.rbs
+- sig/prism_static.rbs
 - src/diagnostic.c
 - src/enc/pm_big5.c
 - src/enc/pm_euc_jp.c
@@ -93,6 +101,7 @@ files:
 - src/enc/pm_unicode.c
 - src/enc/pm_windows_31j.c
 - src/node.c
+- src/options.c
 - src/pack.c
 - src/prettyprint.c
 - src/prism.c