prism 0.13.0

data/include/prism/util/pm_constant_pool.h

@@ -0,0 +1,78 @@
+// The constant pool is a data structure that stores a set of strings. Each
+// string is assigned a unique id, which can be used to compare strings for
+// equality. This comparison ends up being much faster than strcmp, since it
+// only requires a single integer comparison.
+
+#ifndef PRISM_CONSTANT_POOL_H
+#define PRISM_CONSTANT_POOL_H
+
+#include "prism/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef uint32_t pm_constant_id_t;
+
+typedef struct {
+    pm_constant_id_t *ids;
+    size_t size;
+    size_t capacity;
+} pm_constant_id_list_t;
+
+// Initialize a list of constant ids.
+void pm_constant_id_list_init(pm_constant_id_list_t *list);
+
+// Append a constant id to a list of constant ids. Returns false if any
+// potential reallocations fail.
+bool pm_constant_id_list_append(pm_constant_id_list_t *list, pm_constant_id_t id);
+
+// Checks if the current constant id list includes the given constant id.
+bool
+pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id);
+
+// Get the memory size of a list of constant ids.
+size_t pm_constant_id_list_memsize(pm_constant_id_list_t *list);
+
+// Free the memory associated with a list of constant ids.
+void pm_constant_id_list_free(pm_constant_id_list_t *list);
+
+typedef struct {
+    unsigned int id: 31;
+    bool owned: 1;
+    uint32_t hash;
+} pm_constant_pool_bucket_t;
+
+typedef struct {
+    const uint8_t *start;
+    size_t length;
+} pm_constant_t;
+
+typedef struct {
+    pm_constant_pool_bucket_t *buckets;
+    pm_constant_t *constants;
+    uint32_t size;
+    uint32_t capacity;
+} pm_constant_pool_t;
+
+// Define an empty constant pool.
+#define PM_CONSTANT_POOL_EMPTY ((pm_constant_pool_t) { .buckets = NULL, .constants = NULL, .size = 0, .capacity = 0 })
+
+// Initialize a new constant pool with a given capacity.
+bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
+
+// Insert a constant into a constant pool that is a slice of a source string.
+// Returns the id of the constant, or 0 if any potential calls to resize fail.
+pm_constant_id_t pm_constant_pool_insert_shared(pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+// Insert a constant into a constant pool from memory that is now owned by the
+// constant pool. Returns the id of the constant, or 0 if any potential calls to
+// resize fail.
+pm_constant_id_t pm_constant_pool_insert_owned(pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+
+// Free the memory associated with a constant pool.
+void pm_constant_pool_free(pm_constant_pool_t *pool);
+
+#endif

data/include/prism/util/pm_list.h

@@ -0,0 +1,67 @@
+// This struct represents an abstract linked list that provides common
+// functionality. It is meant to be used any time a linked list is necessary to
+// store data.
+//
+// The linked list itself operates off a set of pointers. Because the pointers
+// are not necessarily sequential, they can be of any size. We use this fact to
+// allow the consumer of this linked list to extend the node struct to include
+// any data they want. This is done by using the pm_list_node_t as the first
+// member of the struct.
+//
+// For example, if we want to store a list of integers, we can do the following:
+//
+//     typedef struct {
+//       pm_list_node_t node;
+//       int value;
+//     } pm_int_node_t;
+//
+//     pm_list_t list = PM_LIST_EMPTY;
+//     pm_int_node_t *node = malloc(sizeof(pm_int_node_t));
+//     node->value = 5;
+//
+//     pm_list_append(&list, &node->node);
+//
+// The pm_list_t struct is used to represent the overall linked list. It
+// contains a pointer to the head and tail of the list. This allows for easy
+// iteration and appending of new nodes.
+
+#ifndef PRISM_LIST_H
+#define PRISM_LIST_H
+
+#include "prism/defines.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+// This represents a node in the linked list.
+typedef struct pm_list_node {
+    struct pm_list_node *next;
+} pm_list_node_t;
+
+// This represents the overall linked list. It keeps a pointer to the head and
+// tail so that iteration is easy and pushing new nodes is easy.
+typedef struct {
+    size_t size;
+    pm_list_node_t *head;
+    pm_list_node_t *tail;
+} pm_list_t;
+
+// This represents an empty list. It's used to initialize a stack-allocated list
+// as opposed to a method call.
+#define PM_LIST_EMPTY ((pm_list_t) { .size = 0, .head = NULL, .tail = NULL })
+
+// Returns true if the given list is empty.
+PRISM_EXPORTED_FUNCTION bool pm_list_empty_p(pm_list_t *list);
+
+// Returns the size of the list.
+PRISM_EXPORTED_FUNCTION size_t pm_list_size(pm_list_t *list);
+
+// Append a node to the given list.
+void pm_list_append(pm_list_t *list, pm_list_node_t *node);
+
+// Deallocate the internal state of the given list.
+PRISM_EXPORTED_FUNCTION void pm_list_free(pm_list_t *list);
+
+#endif

data/include/prism/util/pm_memchr.h

@@ -0,0 +1,14 @@
+#ifndef PRISM_MEMCHR_H
+#define PRISM_MEMCHR_H
+
+#include "prism/defines.h"
+#include "prism/enc/pm_encoding.h"
+
+#include <stddef.h>
+
+// We need to roll our own memchr to handle cases where the encoding changes and
+// we need to search for a character in a buffer that could be the trailing byte
+// of a multibyte character.
+void * pm_memchr(const void *source, int character, size_t number, bool encoding_changed, pm_encoding_t *encoding);
+
+#endif

data/include/prism/util/pm_newline_list.h

@@ -0,0 +1,61 @@
+// When compiling the syntax tree, it's necessary to know the line and column
+// of many nodes. This is necessary to support things like error messages,
+// tracepoints, etc.
+//
+// It's possible that we could store the start line, start column, end line, and
+// end column on every node in addition to the offsets that we already store,
+// but that would be quite a lot of memory overhead.
+
+#ifndef PRISM_NEWLINE_LIST_H
+#define PRISM_NEWLINE_LIST_H
+
+#include "prism/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+// A list of offsets of newlines in a string. The offsets are assumed to be
+// sorted/inserted in ascending order.
+typedef struct {
+    const uint8_t *start;
+
+    size_t *offsets;
+    size_t size;
+    size_t capacity;
+
+    size_t last_offset;
+    size_t last_index;
+} pm_newline_list_t;
+
+// A line and column in a string.
+typedef struct {
+    size_t line;
+    size_t column;
+} pm_line_column_t;
+
+#define PM_NEWLINE_LIST_EMPTY ((pm_newline_list_t) { \
+    .start = NULL, .offsets = NULL, .size = 0, .capacity = 0, .last_offset = 0, .last_index = 0 \
+})
+
+// Initialize a new newline list with the given capacity. Returns true if the
+// allocation of the offsets succeeds, otherwise returns false.
+bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity);
+
+// Append a new offset to the newline list. Returns true if the reallocation of
+// the offsets succeeds (if one was necessary), otherwise returns false.
+bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
+
+// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
+
+// Returns the line and column of the given offset. If the offset is not in the
+// list, the line and column of the closest offset less than the given offset
+// are returned.
+pm_line_column_t pm_newline_list_line_column(pm_newline_list_t *list, const uint8_t *cursor);
+
+// Free the internal memory allocated for the newline list.
+void pm_newline_list_free(pm_newline_list_t *list);
+
+#endif

data/include/prism/util/pm_state_stack.h

@@ -0,0 +1,24 @@
+#ifndef PRISM_STATE_STACK_H
+#define PRISM_STATE_STACK_H
+
+#include "prism/defines.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// A struct that represents a stack of bools.
+typedef uint32_t pm_state_stack_t;
+
+// Initializes the state stack to an empty stack.
+#define PM_STATE_STACK_EMPTY ((pm_state_stack_t) 0)
+
+// Pushes a value onto the stack.
+void pm_state_stack_push(pm_state_stack_t *stack, bool value);
+
+// Pops a value off the stack.
+void pm_state_stack_pop(pm_state_stack_t *stack);
+
+// Returns the value at the top of the stack.
+bool pm_state_stack_p(pm_state_stack_t *stack);
+
+#endif

data/include/prism/util/pm_string.h

@@ -0,0 +1,61 @@
+#ifndef PRISM_STRING_H
+#define PRISM_STRING_H
+
+#include "prism/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+// This struct represents a string value.
+typedef struct {
+    enum { PM_STRING_SHARED, PM_STRING_OWNED, PM_STRING_CONSTANT, PM_STRING_MAPPED } type;
+    const uint8_t *source;
+    size_t length;
+} pm_string_t;
+
+#define PM_EMPTY_STRING ((pm_string_t) { .type = PM_STRING_CONSTANT, .source = NULL, .length = 0 })
+
+// Initialize a shared string that is based on initial input.
+void pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end);
+
+// Initialize an owned string that is responsible for freeing allocated memory.
+void pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length);
+
+// Initialize a constant string that doesn't own its memory source.
+void pm_string_constant_init(pm_string_t *string, const char *source, size_t length);
+
+// Read the file indicated by the filepath parameter into source and load its
+// contents and size into the given pm_string_t.
+// The given pm_string_t should be freed using pm_string_free() when it is no longer used.
+//
+// We want to use demand paging as much as possible in order to avoid having to
+// read the entire file into memory (which could be detrimental to performance
+// for large files). This means that if we're on windows we'll use
+// `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
+// `mmap`, and on other POSIX systems we'll use `read`.
+PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const char *filepath);
+
+// Returns the memory size associated with the string.
+size_t pm_string_memsize(const pm_string_t *string);
+
+// Ensure the string is owned. If it is not, then reinitialize it as owned and
+// copy over the previous source.
+void pm_string_ensure_owned(pm_string_t *string);
+
+// Returns the length associated with the string.
+PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string);
+
+// Returns the start pointer associated with the string.
+PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *string);
+
+// Free the associated memory of the given string.
+PRISM_EXPORTED_FUNCTION void pm_string_free(pm_string_t *string);
+
+// Returns the size of the pm_string_t struct. This is necessary to allocate the
+// correct amount of memory in the FFI backend.
+PRISM_EXPORTED_FUNCTION size_t pm_string_sizeof(void);
+
+#endif // PRISM_STRING_H

data/include/prism/util/pm_string_list.h

@@ -0,0 +1,25 @@
+#ifndef PRISM_STRING_LIST_H
+#define PRISM_STRING_LIST_H
+
+#include "prism/defines.h"
+#include "prism/util/pm_string.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+
+typedef struct {
+    pm_string_t *strings;
+    size_t length;
+    size_t capacity;
+} pm_string_list_t;
+
+// Initialize a pm_string_list_t with its default values.
+PRISM_EXPORTED_FUNCTION void pm_string_list_init(pm_string_list_t *string_list);
+
+// Append a pm_string_t to the given string list.
+void pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string);
+
+// Free the memory associated with the string list.
+PRISM_EXPORTED_FUNCTION void pm_string_list_free(pm_string_list_t *string_list);
+
+#endif

data/include/prism/util/pm_strpbrk.h

@@ -0,0 +1,29 @@
+#ifndef PRISM_STRPBRK_H
+#define PRISM_STRPBRK_H
+
+#include "prism/defines.h"
+#include "prism/parser.h"
+
+#include <stddef.h>
+#include <string.h>
+
+// Here we have rolled our own version of strpbrk. The standard library strpbrk
+// has undefined behavior when the source string is not null-terminated. We want
+// to support strings that are not null-terminated because pm_parse does not
+// have the contract that the string is null-terminated. (This is desirable
+// because it means the extension can call pm_parse with the result of a call to
+// mmap).
+//
+// The standard library strpbrk also does not support passing a maximum length
+// to search. We want to support this for the reason mentioned above, but we
+// also don't want it to stop on null bytes. Ruby actually allows null bytes
+// within strings, comments, regular expressions, etc. So we need to be able to
+// skip past them.
+//
+// Finally, we want to support encodings wherein the charset could contain
+// characters that are trailing bytes of multi-byte characters. For example, in
+// Shift-JIS, the backslash character can be a trailing byte. In that case we
+// need to take a slower path and iterate one multi-byte character at a time.
+const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
+
+#endif

data/include/prism/version.h

@@ -0,0 +1,4 @@
+#define PRISM_VERSION_MAJOR 0
+#define PRISM_VERSION_MINOR 13
+#define PRISM_VERSION_PATCH 0
+#define PRISM_VERSION "0.13.0"

data/include/prism.h

@@ -0,0 +1,82 @@
+#ifndef PRISM_H
+#define PRISM_H
+
+#include "prism/defines.h"
+#include "prism/ast.h"
+#include "prism/diagnostic.h"
+#include "prism/node.h"
+#include "prism/pack.h"
+#include "prism/parser.h"
+#include "prism/regexp.h"
+#include "prism/unescape.h"
+#include "prism/util/pm_buffer.h"
+#include "prism/util/pm_char.h"
+#include "prism/util/pm_memchr.h"
+#include "prism/util/pm_strpbrk.h"
+#include "prism/version.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <strings.h>
+#endif
+
+void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+void pm_print_node(pm_parser_t *parser, pm_node_t *node);
+
+void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
+
+// Generate a scope node from the given node.
+void pm_scope_node_init(pm_node_t *node, pm_scope_node_t *dest);
+
+// The prism version and the serialization format.
+PRISM_EXPORTED_FUNCTION const char * pm_version(void);
+
+// Initialize a parser with the given start and end pointers.
+PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const char *filepath);
+
+// Register a callback that will be called whenever prism changes the encoding it
+// is using to parse based on the magic comment.
+PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
+
+// Register a callback that will be called when prism encounters a magic comment
+// with an encoding referenced that it doesn't understand. The callback should
+// return NULL if it also doesn't understand the encoding or it should return a
+// pointer to a pm_encoding_t struct that contains the functions necessary to
+// parse identifiers.
+PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_decode_callback(pm_parser_t *parser, pm_encoding_decode_callback_t callback);
+
+// Free any memory associated with the given parser.
+PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser);
+
+// Parse the Ruby source associated with the given parser and return the tree.
+PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser);
+
+// Pretty-prints the AST represented by the given node to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+// Serialize the AST represented by the given node to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+// Parse the given source to the AST and serialize the AST to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+
+// Lex the given source and serialize to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer);
+
+// Parse and serialize both the AST and the tokens represented by the given
+// source to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+
+// Returns a string representation of the given token type.
+PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
+
+#endif