yarp 0.6.0

data/include/yarp/util/yp_list.h

@@ -0,0 +1,67 @@
+// This struct represents an abstract linked list that provides common
+// functionality. It is meant to be used any time a linked list is necessary to
+// store data.
+//
+// The linked list itself operates off a set of pointers. Because the pointers
+// are not necessarily sequential, they can be of any size. We use this fact to
+// allow the consumer of this linked list to extend the node struct to include
+// any data they want. This is done by using the yp_list_node_t as the first
+// member of the struct.
+//
+// For example, if we want to store a list of integers, we can do the following:
+//
+//     typedef struct {
+//       yp_list_node_t node;
+//       int value;
+//     } yp_int_node_t;
+//
+//     yp_list_t list;
+//     yp_list_init(&list);
+//
+//     yp_int_node_t *node = malloc(sizeof(yp_int_node_t));
+//     node->value = 5;
+//
+//     yp_list_append(&list, &node->node);
+//
+// The yp_list_t struct is used to represent the overall linked list. It
+// contains a pointer to the head and tail of the list. This allows for easy
+// iteration and appending of new nodes.
+
+#ifndef YARP_LIST_H
+#define YARP_LIST_H
+
+#include "yarp/defines.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+// This represents a node in the linked list.
+typedef struct yp_list_node {
+    struct yp_list_node *next;
+} yp_list_node_t;
+
+// This represents the overall linked list. It keeps a pointer to the head and
+// tail so that iteration is easy and pushing new nodes is easy.
+typedef struct {
+    yp_list_node_t *head;
+    yp_list_node_t *tail;
+} yp_list_t;
+
+// Initializes a new list.
+YP_EXPORTED_FUNCTION void yp_list_init(yp_list_t *list);
+
+// Returns true if the given list is empty.
+YP_EXPORTED_FUNCTION bool yp_list_empty_p(yp_list_t *list);
+
+// Returns the size of the list in O(n) time.
+YP_EXPORTED_FUNCTION uint32_t yp_list_size(yp_list_t *list);
+
+// Append a node to the given list.
+void yp_list_append(yp_list_t *list, yp_list_node_t *node);
+
+// Deallocate the internal state of the given list.
+YP_EXPORTED_FUNCTION void yp_list_free(yp_list_t *list);
+
+#endif

data/include/yarp/util/yp_memchr.h

@@ -0,0 +1,14 @@
+#ifndef YP_MEMCHR_H
+#define YP_MEMCHR_H
+
+#include "yarp/defines.h"
+#include "yarp/enc/yp_encoding.h"
+
+#include <stddef.h>
+
+// We need to roll our own memchr to handle cases where the encoding changes and
+// we need to search for a character in a buffer that could be the trailing byte
+// of a multibyte character.
+void * yp_memchr(const void *source, int character, size_t number, bool encoding_changed, yp_encoding_t *encoding);
+
+#endif

data/include/yarp/util/yp_newline_list.h

@@ -0,0 +1,54 @@
+// When compiling the syntax tree, it's necessary to know the line and column
+// of many nodes. This is necessary to support things like error messages,
+// tracepoints, etc.
+//
+// It's possible that we could store the start line, start column, end line, and
+// end column on every node in addition to the offsets that we already store,
+// but that would be quite a lot of memory overhead.
+
+#ifndef YP_NEWLINE_LIST_H
+#define YP_NEWLINE_LIST_H
+
+#include "yarp/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+// A list of offsets of newlines in a string. The offsets are assumed to be
+// sorted/inserted in ascending order.
+typedef struct {
+    const char *start;
+
+    size_t *offsets;
+    size_t size;
+    size_t capacity;
+
+    size_t last_offset;
+    size_t last_index;
+} yp_newline_list_t;
+
+// A line and column in a string.
+typedef struct {
+    size_t line;
+    size_t column;
+} yp_line_column_t;
+
+// Initialize a new newline list with the given capacity. Returns true if the
+// allocation of the offsets succeeds, otherwise returns false.
+bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity);
+
+// Append a new offset to the newline list. Returns true if the reallocation of
+// the offsets succeeds (if one was necessary), otherwise returns false.
+bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor);
+
+// Returns the line and column of the given offset. If the offset is not in the
+// list, the line and column of the closest offset less than the given offset
+// are returned.
+yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor);
+
+// Free the internal memory allocated for the newline list.
+void yp_newline_list_free(yp_newline_list_t *list);
+
+#endif

data/include/yarp/util/yp_state_stack.h

@@ -0,0 +1,24 @@
+#ifndef YP_STATE_STACK_H
+#define YP_STATE_STACK_H
+
+#include "yarp/defines.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// A struct that represents a stack of bools.
+typedef uint32_t yp_state_stack_t;
+
+// Initializes the state stack to an empty stack.
+void yp_state_stack_init(yp_state_stack_t *stack);
+
+// Pushes a value onto the stack.
+void yp_state_stack_push(yp_state_stack_t *stack, bool value);
+
+// Pops a value off the stack.
+void yp_state_stack_pop(yp_state_stack_t *stack);
+
+// Returns the value at the top of the stack.
+bool yp_state_stack_p(yp_state_stack_t *stack);
+
+#endif

data/include/yarp/util/yp_string.h

@@ -0,0 +1,57 @@
+#ifndef YARP_STRING_H
+#define YARP_STRING_H
+
+#include "yarp/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+// This struct represents a string value.
+typedef struct {
+    enum { YP_STRING_SHARED, YP_STRING_OWNED, YP_STRING_CONSTANT, YP_STRING_MAPPED } type;
+    char *source;
+    size_t length;
+} yp_string_t;
+
+#define YP_EMPTY_STRING ((yp_string_t) { .type = YP_STRING_CONSTANT, .source = NULL, .length = 0 })
+
+// Initialize a shared string that is based on initial input.
+void yp_string_shared_init(yp_string_t *string, const char *start, const char *end);
+
+// Initialize an owned string that is responsible for freeing allocated memory.
+void yp_string_owned_init(yp_string_t *string, char *source, size_t length);
+
+// Initialize a constant string that doesn't own its memory source.
+void yp_string_constant_init(yp_string_t *string, const char *source, size_t length);
+
+// Read the file indicated by the filepath parameter into source and load its
+// contents and size into the given yp_string_t.
+// The given yp_string_t should be freed using yp_string_free() when it is no longer used.
+//
+// We want to use demand paging as much as possible in order to avoid having to
+// read the entire file into memory (which could be detrimental to performance
+// for large files). This means that if we're on windows we'll use
+// `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
+// `mmap`, and on other POSIX systems we'll use `read`.
+bool yp_string_mapped_init(yp_string_t *string, const char *filepath);
+
+// Returns the memory size associated with the string.
+size_t yp_string_memsize(const yp_string_t *string);
+
+// Ensure the string is owned. If it is not, then reinitialize it as owned and
+// copy over the previous source.
+void yp_string_ensure_owned(yp_string_t *string);
+
+// Returns the length associated with the string.
+YP_EXPORTED_FUNCTION size_t yp_string_length(const yp_string_t *string);
+
+// Returns the start pointer associated with the string.
+YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string);
+
+// Free the associated memory of the given string.
+YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string);
+
+#endif // YARP_STRING_H

data/include/yarp/util/yp_string_list.h

@@ -0,0 +1,28 @@
+#ifndef YARP_STRING_LIST_H
+#define YARP_STRING_LIST_H
+
+#include "yarp/defines.h"
+#include "yarp/util/yp_string.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+
+typedef struct {
+    yp_string_t *strings;
+    size_t length;
+    size_t capacity;
+} yp_string_list_t;
+
+// Allocate a new yp_string_list_t.
+yp_string_list_t * yp_string_list_alloc(void);
+
+// Initialize a yp_string_list_t with its default values.
+YP_EXPORTED_FUNCTION void yp_string_list_init(yp_string_list_t *string_list);
+
+// Append a yp_string_t to the given string list.
+void yp_string_list_append(yp_string_list_t *string_list, yp_string_t *string);
+
+// Free the memory associated with the string list.
+YP_EXPORTED_FUNCTION void yp_string_list_free(yp_string_list_t *string_list);
+
+#endif

data/include/yarp/util/yp_strpbrk.h

@@ -0,0 +1,29 @@
+#ifndef YP_STRPBRK_H
+#define YP_STRPBRK_H
+
+#include "yarp/defines.h"
+#include "yarp/parser.h"
+
+#include <stddef.h>
+#include <string.h>
+
+// Here we have rolled our own version of strpbrk. The standard library strpbrk
+// has undefined behavior when the source string is not null-terminated. We want
+// to support strings that are not null-terminated because yp_parse does not
+// have the contract that the string is null-terminated. (This is desirable
+// because it means the extension can call yp_parse with the result of a call to
+// mmap).
+//
+// The standard library strpbrk also does not support passing a maximum length
+// to search. We want to support this for the reason mentioned above, but we
+// also don't want it to stop on null bytes. Ruby actually allows null bytes
+// within strings, comments, regular expressions, etc. So we need to be able to
+// skip past them.
+//
+// Finally, we want to support encodings wherein the charset could contain
+// characters that are trailing bytes of multi-byte characters. For example, in
+// Shift-JIS, the backslash character can be a trailing byte. In that case we
+// need to take a slower path and iterate one multi-byte character at a time.
+const char * yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length);
+
+#endif

data/include/yarp/version.h

@@ -0,0 +1,5 @@
+#define YP_VERSION_MAJOR 0
+#define YP_VERSION_MINOR 6
+#define YP_VERSION_PATCH 0
+
+#define YP_VERSION "0.6.0"

data/include/yarp.h

@@ -0,0 +1,69 @@
+#ifndef YARP_H
+#define YARP_H
+
+#include "yarp/defines.h"
+#include "yarp/ast.h"
+#include "yarp/diagnostic.h"
+#include "yarp/node.h"
+#include "yarp/pack.h"
+#include "yarp/parser.h"
+#include "yarp/regexp.h"
+#include "yarp/unescape.h"
+#include "yarp/util/yp_buffer.h"
+#include "yarp/util/yp_char.h"
+#include "yarp/util/yp_memchr.h"
+#include "yarp/util/yp_strpbrk.h"
+
+#include <assert.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <strings.h>
+#endif
+
+void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
+
+void yp_print_node(yp_parser_t *parser, yp_node_t *node);
+
+// The YARP version and the serialization format.
+YP_EXPORTED_FUNCTION const char * yp_version(void);
+
+// Initialize a parser with the given start and end pointers.
+YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath);
+
+// Register a callback that will be called whenever YARP changes the encoding it
+// is using to parse based on the magic comment.
+YP_EXPORTED_FUNCTION void yp_parser_register_encoding_changed_callback(yp_parser_t *parser, yp_encoding_changed_callback_t callback);
+
+// Register a callback that will be called when YARP encounters a magic comment
+// with an encoding referenced that it doesn't understand. The callback should
+// return NULL if it also doesn't understand the encoding or it should return a
+// pointer to a yp_encoding_t struct that contains the functions necessary to
+// parse identifiers.
+YP_EXPORTED_FUNCTION void yp_parser_register_encoding_decode_callback(yp_parser_t *parser, yp_encoding_decode_callback_t callback);
+
+// Free any memory associated with the given parser.
+YP_EXPORTED_FUNCTION void yp_parser_free(yp_parser_t *parser);
+
+// Parse the Ruby source associated with the given parser and return the tree.
+YP_EXPORTED_FUNCTION yp_node_t * yp_parse(yp_parser_t *parser);
+
+// Pretty-prints the AST represented by the given node to the given buffer.
+YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
+
+// Serialize the AST represented by the given node to the given buffer.
+YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
+
+// Parse and serialize the AST represented by the given source to the given
+// buffer.
+YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
+
+// Returns a string representation of the given token type.
+YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);
+
+#endif