prism 0.13.0

data/src/util/pm_memchr.c

@@ -0,0 +1,33 @@
+#include "prism/util/pm_memchr.h"
+
+#define PRISM_MEMCHR_TRAILING_BYTE_MINIMUM 0x40
+
+// We need to roll our own memchr to handle cases where the encoding changes and
+// we need to search for a character in a buffer that could be the trailing byte
+// of a multibyte character.
+void *
+pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, pm_encoding_t *encoding) {
+    if (encoding_changed && encoding->multibyte && character >= PRISM_MEMCHR_TRAILING_BYTE_MINIMUM) {
+        const uint8_t *source = (const uint8_t *) memory;
+        size_t index = 0;
+
+        while (index < number) {
+            if (source[index] == character) {
+                return (void *) (source + index);
+            }
+
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (number - index));
+            if (width == 0) {
+                return NULL;
+            }
+
+            index += width;
+        }
+
+        return NULL;
+    } else {
+        return memchr(memory, character, number);
+    }
+}
+
+#undef PRISM_MEMCHR_TRAILING_BYTE_MINIMUM

data/src/util/pm_newline_list.c

@@ -0,0 +1,134 @@
+#include "prism/util/pm_newline_list.h"
+
+// Initialize a new newline list with the given capacity. Returns true if the
+// allocation of the offsets succeeds, otherwise returns false.
+bool
+pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity) {
+    list->offsets = (size_t *) calloc(capacity, sizeof(size_t));
+    if (list->offsets == NULL) return false;
+
+    list->start = start;
+
+    // This is 1 instead of 0 because we want to include the first line of the
+    // file as having offset 0, which is set because of calloc.
+    list->size = 1;
+    list->capacity = capacity;
+
+    list->last_index = 0;
+    list->last_offset = 0;
+
+    return true;
+}
+
+// Append a new offset to the newline list. Returns true if the reallocation of
+// the offsets succeeds (if one was necessary), otherwise returns false.
+bool
+pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) {
+    if (list->size == list->capacity) {
+        size_t *original_offsets = list->offsets;
+
+        list->capacity = (list->capacity * 3) / 2;
+        list->offsets = (size_t *) calloc(list->capacity, sizeof(size_t));
+        memcpy(list->offsets, original_offsets, list->size * sizeof(size_t));
+        free(original_offsets);
+        if (list->offsets == NULL) return false;
+    }
+
+    assert(*cursor == '\n');
+    assert(cursor >= list->start);
+    size_t newline_offset = (size_t) (cursor - list->start + 1);
+
+    assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]);
+    list->offsets[list->size++] = newline_offset;
+
+    return true;
+}
+
+// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+bool
+pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor) {
+    if (*cursor != '\n') {
+        return true;
+    }
+    return pm_newline_list_append(list, cursor);
+}
+
+// Returns the line and column of the given offset, assuming we don't have any
+// information about the previous index that we found.
+static pm_line_column_t
+pm_newline_list_line_column_search(pm_newline_list_t *list, size_t offset) {
+    size_t left = 0;
+    size_t right = list->size - 1;
+
+    while (left <= right) {
+        size_t mid = left + (right - left) / 2;
+
+        if (list->offsets[mid] == offset) {
+            return ((pm_line_column_t) { mid, 0 });
+        }
+
+        if (list->offsets[mid] < offset) {
+            left = mid + 1;
+        } else {
+            right = mid - 1;
+        }
+    }
+
+    return ((pm_line_column_t) { left - 1, offset - list->offsets[left - 1] });
+}
+
+// Returns the line and column of the given offset, assuming we know the last
+// index that we found.
+static pm_line_column_t
+pm_newline_list_line_column_scan(pm_newline_list_t *list, size_t offset) {
+    if (offset > list->last_offset) {
+        size_t index = list->last_index;
+        while (index < list->size && list->offsets[index] < offset) {
+            index++;
+        }
+
+        if (index == list->size) {
+            return ((pm_line_column_t) { index - 1, offset - list->offsets[index - 1] });
+        }
+
+        return ((pm_line_column_t) { index, 0 });
+    } else {
+        size_t index = list->last_index;
+        while (index > 0 && list->offsets[index] > offset) {
+            index--;
+        }
+
+        if (index == 0) {
+            return ((pm_line_column_t) { 0, offset });
+        }
+
+        return ((pm_line_column_t) { index, offset - list->offsets[index - 1] });
+    }
+}
+
+// Returns the line and column of the given offset. If the offset is not in the
+// list, the line and column of the closest offset less than the given offset
+// are returned.
+pm_line_column_t
+pm_newline_list_line_column(pm_newline_list_t *list, const uint8_t *cursor) {
+    assert(cursor >= list->start);
+    size_t offset = (size_t) (cursor - list->start);
+    pm_line_column_t result;
+
+    if (list->last_offset == 0) {
+        result = pm_newline_list_line_column_search(list, offset);
+    } else {
+        result = pm_newline_list_line_column_scan(list, offset);
+    }
+
+    list->last_index = result.line;
+    list->last_offset = offset;
+
+    return result;
+}
+
+// Free the internal memory allocated for the newline list.
+void
+pm_newline_list_free(pm_newline_list_t *list) {
+    free(list->offsets);
+}

data/src/util/pm_state_stack.c

@@ -0,0 +1,19 @@
+#include "prism/util/pm_state_stack.h"
+
+// Pushes a value onto the stack.
+void
+pm_state_stack_push(pm_state_stack_t *stack, bool value) {
+    *stack = (*stack << 1) | (value & 1);
+}
+
+// Pops a value off the stack.
+void
+pm_state_stack_pop(pm_state_stack_t *stack) {
+    *stack >>= 1;
+}
+
+// Returns the value at the top of the stack.
+bool
+pm_state_stack_p(pm_state_stack_t *stack) {
+    return *stack & 1;
+}

data/src/util/pm_string.c

@@ -0,0 +1,200 @@
+#include "prism/util/pm_string.h"
+
+// The following headers are necessary to read files using demand paging.
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
+// Initialize a shared string that is based on initial input.
+void
+pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end) {
+    assert(start <= end);
+
+    *string = (pm_string_t) {
+        .type = PM_STRING_SHARED,
+        .source = start,
+        .length = (size_t) (end - start)
+    };
+}
+
+// Initialize an owned string that is responsible for freeing allocated memory.
+void
+pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length) {
+    *string = (pm_string_t) {
+        .type = PM_STRING_OWNED,
+        .source = source,
+        .length = length
+    };
+}
+
+// Initialize a constant string that doesn't own its memory source.
+void
+pm_string_constant_init(pm_string_t *string, const char *source, size_t length) {
+    *string = (pm_string_t) {
+        .type = PM_STRING_CONSTANT,
+        .source = (const uint8_t *) source,
+        .length = length
+    };
+}
+
+static void
+pm_string_mapped_init_internal(pm_string_t *string, uint8_t *source, size_t length) {
+    *string = (pm_string_t) {
+        .type = PM_STRING_MAPPED,
+        .source = source,
+        .length = length
+    };
+}
+
+// Returns the memory size associated with the string.
+size_t
+pm_string_memsize(const pm_string_t *string) {
+    size_t size = sizeof(pm_string_t);
+    if (string->type == PM_STRING_OWNED) {
+        size += string->length;
+    }
+    return size;
+}
+
+// Ensure the string is owned. If it is not, then reinitialize it as owned and
+// copy over the previous source.
+void
+pm_string_ensure_owned(pm_string_t *string) {
+    if (string->type == PM_STRING_OWNED) return;
+
+    size_t length = pm_string_length(string);
+    const uint8_t *source = pm_string_source(string);
+
+    uint8_t *memory = malloc(length);
+    if (!memory) return;
+
+    pm_string_owned_init(string, memory, length);
+    memcpy((void *) string->source, source, length);
+}
+
+// Returns the length associated with the string.
+PRISM_EXPORTED_FUNCTION size_t
+pm_string_length(const pm_string_t *string) {
+    return string->length;
+}
+
+// Returns the start pointer associated with the string.
+PRISM_EXPORTED_FUNCTION const uint8_t *
+pm_string_source(const pm_string_t *string) {
+    return string->source;
+}
+
+// Free the associated memory of the given string.
+PRISM_EXPORTED_FUNCTION void
+pm_string_free(pm_string_t *string) {
+    void *memory = (void *) string->source;
+
+    if (string->type == PM_STRING_OWNED) {
+        free(memory);
+    } else if (string->type == PM_STRING_MAPPED && string->length) {
+#if defined(_WIN32)
+        UnmapViewOfFile(memory);
+#else
+        munmap(memory, string->length);
+#endif
+    }
+}
+
+bool
+pm_string_mapped_init(pm_string_t *string, const char *filepath) {
+#ifdef _WIN32
+    // Open the file for reading.
+    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+
+    if (file == INVALID_HANDLE_VALUE) {
+        perror("CreateFile failed");
+        return false;
+    }
+
+    // Get the file size.
+    DWORD file_size = GetFileSize(file, NULL);
+    if (file_size == INVALID_FILE_SIZE) {
+        CloseHandle(file);
+        perror("GetFileSize failed");
+        return false;
+    }
+
+    // If the file is empty, then we don't need to do anything else, we'll set
+    // the source to a constant empty string and return.
+    if (file_size == 0) {
+        CloseHandle(file);
+        uint8_t empty[] = "";
+        pm_string_mapped_init_internal(string, empty, 0);
+        return true;
+    }
+
+    // Create a mapping of the file.
+    HANDLE mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
+    if (mapping == NULL) {
+        CloseHandle(file);
+        perror("CreateFileMapping failed");
+        return false;
+    }
+
+    // Map the file into memory.
+    uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+    CloseHandle(mapping);
+    CloseHandle(file);
+
+    if (source == NULL) {
+        perror("MapViewOfFile failed");
+        return false;
+    }
+
+    pm_string_mapped_init_internal(string, source, (size_t) file_size);
+    return true;
+#else
+    // Open the file for reading
+    int fd = open(filepath, O_RDONLY);
+    if (fd == -1) {
+        perror("open");
+        return false;
+    }
+
+    // Stat the file to get the file size
+    struct stat sb;
+    if (fstat(fd, &sb) == -1) {
+        close(fd);
+        perror("fstat");
+        return false;
+    }
+
+    // mmap the file descriptor to virtually get the contents
+    size_t size = (size_t) sb.st_size;
+    uint8_t *source = NULL;
+
+    if (size == 0) {
+        close(fd);
+        uint8_t empty[] = "";
+        pm_string_mapped_init_internal(string, empty, 0);
+        return true;
+    }
+
+    source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+    if (source == MAP_FAILED) {
+        perror("Map failed");
+        return false;
+    }
+
+    close(fd);
+    pm_string_mapped_init_internal(string, source, size);
+    return true;
+#endif
+}
+
+// Returns the size of the pm_string_t struct. This is necessary to allocate the
+// correct amount of memory in the FFI backend.
+PRISM_EXPORTED_FUNCTION size_t
+pm_string_sizeof(void) {
+    return sizeof(pm_string_t);
+}

data/src/util/pm_string_list.c

@@ -0,0 +1,29 @@
+#include "prism/util/pm_string_list.h"
+
+// Initialize a pm_string_list_t with its default values.
+void
+pm_string_list_init(pm_string_list_t *string_list) {
+    string_list->strings = (pm_string_t *) malloc(sizeof(pm_string_t));
+    string_list->length = 0;
+    string_list->capacity = 1;
+}
+
+// Append a pm_string_t to the given string list.
+void
+pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
+    if (string_list->length + 1 > string_list->capacity) {
+        pm_string_t *original_string = string_list->strings;
+        string_list->capacity *= 2;
+        string_list->strings = (pm_string_t *) malloc(string_list->capacity * sizeof(pm_string_t));
+        memcpy(string_list->strings, original_string, (string_list->length) * sizeof(pm_string_t));
+        free(original_string);
+    }
+
+    string_list->strings[string_list->length++] = *string;
+}
+
+// Free the memory associated with the string list.
+void
+pm_string_list_free(pm_string_list_t *string_list) {
+    free(string_list->strings);
+}

data/src/util/pm_strncasecmp.c

@@ -0,0 +1,17 @@
+#include <ctype.h>
+#include <stddef.h>
+#include <stdint.h>
+
+int
+pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) {
+    size_t offset = 0;
+    int difference = 0;
+
+    while (offset < length && string1[offset] != '\0') {
+        if (string2[offset] == '\0') return string1[offset];
+        if ((difference = tolower(string1[offset]) - tolower(string2[offset])) != 0) return difference;
+        offset++;
+    }
+
+    return difference;
+}

data/src/util/pm_strpbrk.c

@@ -0,0 +1,66 @@
+#include "prism/util/pm_strpbrk.h"
+
+// This is the slow path that does care about the encoding.
+static inline const uint8_t *
+pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        size_t width = parser->encoding.char_width(source + index, (ptrdiff_t) (maximum - index));
+        if (width == 0) {
+            return NULL;
+        }
+
+        index += width;
+    }
+
+    return NULL;
+}
+
+// This is the fast path that does not care about the encoding.
+static inline const uint8_t *
+pm_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t maximum) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        index++;
+    }
+
+    return NULL;
+}
+
+// Here we have rolled our own version of strpbrk. The standard library strpbrk
+// has undefined behavior when the source string is not null-terminated. We want
+// to support strings that are not null-terminated because pm_parse does not
+// have the contract that the string is null-terminated. (This is desirable
+// because it means the extension can call pm_parse with the result of a call to
+// mmap).
+//
+// The standard library strpbrk also does not support passing a maximum length
+// to search. We want to support this for the reason mentioned above, but we
+// also don't want it to stop on null bytes. Ruby actually allows null bytes
+// within strings, comments, regular expressions, etc. So we need to be able to
+// skip past them.
+//
+// Finally, we want to support encodings wherein the charset could contain
+// characters that are trailing bytes of multi-byte characters. For example, in
+// Shift-JIS, the backslash character can be a trailing byte. In that case we
+// need to take a slower path and iterate one multi-byte character at a time.
+const uint8_t *
+pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) {
+    if (length <= 0) {
+        return NULL;
+    } else if (parser->encoding_changed && parser->encoding.multibyte) {
+        return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length);
+    } else {
+        return pm_strpbrk_single_byte(source, charset, (size_t) length);
+    }
+}

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: prism
+version: !ruby/object:Gem::Version
+  version: 0.13.0
+platform: ruby
+authors:
+- Shopify
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-09-29 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- ruby@shopify.com
+executables: []
+extensions:
+- ext/prism/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- LICENSE.md
+- Makefile
+- README.md
+- config.yml
+- docs/build_system.md
+- docs/building.md
+- docs/configuration.md
+- docs/design.md
+- docs/encoding.md
+- docs/fuzzing.md
+- docs/heredocs.md
+- docs/mapping.md
+- docs/ripper.md
+- docs/ruby_api.md
+- docs/serialization.md
+- docs/testing.md
+- ext/prism/api_node.c
+- ext/prism/api_pack.c
+- ext/prism/extconf.rb
+- ext/prism/extension.c
+- ext/prism/extension.h
+- include/prism.h
+- include/prism/ast.h
+- include/prism/defines.h
+- include/prism/diagnostic.h
+- include/prism/enc/pm_encoding.h
+- include/prism/node.h
+- include/prism/pack.h
+- include/prism/parser.h
+- include/prism/regexp.h
+- include/prism/unescape.h
+- include/prism/util/pm_buffer.h
+- include/prism/util/pm_char.h
+- include/prism/util/pm_constant_pool.h
+- include/prism/util/pm_list.h
+- include/prism/util/pm_memchr.h
+- include/prism/util/pm_newline_list.h
+- include/prism/util/pm_state_stack.h
+- include/prism/util/pm_string.h
+- include/prism/util/pm_string_list.h
+- include/prism/util/pm_strpbrk.h
+- include/prism/version.h
+- lib/prism.rb
+- lib/prism/compiler.rb
+- lib/prism/debug.rb
+- lib/prism/desugar_compiler.rb
+- lib/prism/dispatcher.rb
+- lib/prism/dsl.rb
+- lib/prism/ffi.rb
+- lib/prism/lex_compat.rb
+- lib/prism/mutation_compiler.rb
+- lib/prism/node.rb
+- lib/prism/node_ext.rb
+- lib/prism/node_inspector.rb
+- lib/prism/pack.rb
+- lib/prism/parse_result.rb
+- lib/prism/parse_result/comments.rb
+- lib/prism/parse_result/newlines.rb
+- lib/prism/pattern.rb
+- lib/prism/ripper_compat.rb
+- lib/prism/serialize.rb
+- lib/prism/visitor.rb
+- prism.gemspec
+- src/diagnostic.c
+- src/enc/pm_big5.c
+- src/enc/pm_euc_jp.c
+- src/enc/pm_gbk.c
+- src/enc/pm_shift_jis.c
+- src/enc/pm_tables.c
+- src/enc/pm_unicode.c
+- src/enc/pm_windows_31j.c
+- src/node.c
+- src/pack.c
+- src/prettyprint.c
+- src/prism.c
+- src/regexp.c
+- src/serialize.c
+- src/token_type.c
+- src/unescape.c
+- src/util/pm_buffer.c
+- src/util/pm_char.c
+- src/util/pm_constant_pool.c
+- src/util/pm_list.c
+- src/util/pm_memchr.c
+- src/util/pm_newline_list.c
+- src/util/pm_state_stack.c
+- src/util/pm_string.c
+- src/util/pm_string_list.c
+- src/util/pm_strncasecmp.c
+- src/util/pm_strpbrk.c
+homepage: https://github.com/ruby/prism
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: Prism Ruby parser
+test_files: []