herb 0.11.1 → 0.11.2

data/vendor/prism/include/prism/string_query.h

@@ -0,0 +1,82 @@
+/**
+ * @file string_query.h
+ *
+ * Functions for querying properties of strings, such as whether they are valid
+ * local variable names, constant names, or method names.
+ */
+#ifndef PRISM_STRING_QUERY_H
+#define PRISM_STRING_QUERY_H
+
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Represents the results of a slice query.
+ */
+typedef enum {
+    /** Returned if the encoding given to a slice query was invalid. */
+    PM_STRING_QUERY_ERROR = -1,
+
+    /** Returned if the result of the slice query is false. */
+    PM_STRING_QUERY_FALSE,
+
+    /** Returned if the result of the slice query is true. */
+    PM_STRING_QUERY_TRUE
+} pm_string_query_t;
+
+/**
+ * Check that the slice is a valid local variable name.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_local(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
+/**
+ * Check that the slice is a valid constant name.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_constant(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
+/**
+ * Check that the slice is a valid method name.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
+/**
+ * Check that the slice is a keyword.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_keyword(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/prism/include/prism/stringy.h

@@ -0,0 +1,61 @@
+/**
+ * @file stringy.h
+ *
+ * A view of a sequence of bytes owned by someone else.
+ */
+#ifndef PRISM_STRINGY_H
+#define PRISM_STRINGY_H
+
+#include "prism/compiler/exported.h"
+#include "prism/compiler/nonnull.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * A view of a sequence of bytes. The string does not own the bytes, so they
+ * must remain valid for as long as the string is in use.
+ */
+typedef struct {
+    /** A pointer to the start of the string. */
+    const uint8_t *source;
+
+    /** The length of the string in bytes of memory. */
+    size_t length;
+} pm_string_t;
+
+/**
+ * Initialize a string that views the given bytes. The bytes are not copied, so
+ * they must remain valid for as long as the string is in use.
+ *
+ * @param string The string to initialize.
+ * @param source The start of the bytes.
+ * @param length The number of bytes.
+ */
+PRISM_EXPORTED_FUNCTION void pm_string_init(pm_string_t *string, const uint8_t *source, size_t length) PRISM_NONNULL(1);
+
+/**
+ * Returns the length associated with the string.
+ *
+ * @param string The string to get the length of.
+ * @returns The length of the string.
+ */
+PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string) PRISM_NONNULL(1);
+
+/**
+ * Returns the start pointer associated with the string.
+ *
+ * @param string The string to get the start pointer of.
+ * @returns The start pointer of the string.
+ */
+PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *string) PRISM_NONNULL(1);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/prism/include/prism/stringy.hpp

@@ -0,0 +1,31 @@
+/**
+ * @file stringy.hpp
+ *
+ * Access to the contents of strings from C++.
+ */
+#ifndef PRISM_STRINGY_HPP
+#define PRISM_STRINGY_HPP
+
+#include "prism/compiler/cplusplus.hpp"
+#include "prism/compiler/lifetimebound.hpp"
+
+#include "prism/stringy.h"
+
+#include <string_view>
+
+namespace prism {
+
+/**
+ * The bytes of the given string.
+ *
+ * @param string The string to view.
+ * @return A view of the bytes of the string.
+ */
+[[nodiscard]] inline std::string_view
+view(const pm_string_t &string PRISM_LIFETIMEBOUND) noexcept {
+    return std::string_view(reinterpret_cast<const char *>(pm_string_source(&string)), pm_string_length(&string));
+}
+
+}
+
+#endif

data/vendor/prism/include/prism/version.h

@@ -6,15 +6,21 @@
 #ifndef PRISM_VERSION_H
 #define PRISM_VERSION_H
 
+#include "prism/compiler/exported.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * The major version of the Prism library as an int.
  */
-#define PRISM_VERSION_MAJOR 1
+#define PRISM_VERSION_MAJOR 2
 
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 9
+#define PRISM_VERSION_MINOR 1
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +30,17 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "1.9.0"
+#define PRISM_VERSION "2.1.0"
+
+/**
+ * The prism version and the serialization format.
+ *
+ * @returns The prism version as a constant string.
+ */
+PRISM_EXPORTED_FUNCTION const char * pm_version(void);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

data/vendor/prism/include/prism.h

@@ -6,306 +6,22 @@
 #ifndef PRISM_H
 #define PRISM_H
 
-#include "prism/defines.h"
-#include "prism/util/pm_buffer.h"
-#include "prism/util/pm_char.h"
-#include "prism/util/pm_integer.h"
-#include "prism/util/pm_memchr.h"
-#include "prism/util/pm_strncasecmp.h"
-#include "prism/util/pm_strpbrk.h"
+#include "prism/arena.h"
 #include "prism/ast.h"
+#include "prism/buffer.h"
 #include "prism/diagnostic.h"
+#include "prism/errors_format.h"
+#include "prism/json.h"
 #include "prism/node.h"
 #include "prism/options.h"
-#include "prism/pack.h"
 #include "prism/parser.h"
 #include "prism/prettyprint.h"
-#include "prism/regexp.h"
-#include "prism/static_literals.h"
+#include "prism/serialize.h"
+#include "prism/source.h"
+#include "prism/stream.h"
+#include "prism/string_query.h"
 #include "prism/version.h"
 
-#include <assert.h>
-#include <errno.h>
-#include <locale.h>
-#include <math.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifndef _WIN32
-#include <strings.h>
-#endif
-
-/**
- * The prism version and the serialization format.
- *
- * @returns The prism version as a constant string.
- */
-PRISM_EXPORTED_FUNCTION const char * pm_version(void);
-
-/**
- * Initialize a parser with the given start and end pointers.
- *
- * The resulting parser must eventually be freed with `pm_parser_free()`.
- *
- * @param parser The parser to initialize.
- * @param source The source to parse.
- * @param size The size of the source.
- * @param options The optional options to use when parsing. These options must
- *   live for the whole lifetime of this parser.
- *
- * \public \memberof pm_parser
- */
-PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options);
-
-/**
- * Register a callback that will be called whenever prism changes the encoding
- * it is using to parse based on the magic comment.
- *
- * @param parser The parser to register the callback with.
- * @param callback The callback to register.
- *
- * \public \memberof pm_parser
- */
-PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
-
-/**
- * Free any memory associated with the given parser.
- *
- * This does not free the `pm_options_t` object that was used to initialize the
- * parser.
- *
- * @param parser The parser to free.
- *
- * \public \memberof pm_parser
- */
-PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser);
-
-/**
- * Initiate the parser with the given parser.
- *
- * @param parser The parser to use.
- * @return The AST representing the source.
- *
- * \public \memberof pm_parser
- */
-PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser);
-
-/**
- * This function is used in pm_parse_stream() to retrieve a line of input from a
- * stream. It closely mirrors that of fgets so that fgets can be used as the
- * default implementation.
- */
-typedef char * (pm_parse_stream_fgets_t)(char *string, int size, void *stream);
-
-/**
- * This function is used in pm_parse_stream to check whether a stream is EOF.
- * It closely mirrors that of feof so that feof can be used as the
- * default implementation.
- */
-typedef int (pm_parse_stream_feof_t)(void *stream);
-
-/**
- * Parse a stream of Ruby source and return the tree.
- *
- * @param parser The parser to use.
- * @param buffer The buffer to use.
- * @param stream The stream to parse.
- * @param stream_fgets The function to use to read from the stream.
- * @param stream_feof The function to use to determine if the stream has hit eof.
- * @param options The optional options to use when parsing.
- * @return The AST representing the source.
- *
- * \public \memberof pm_parser
- */
-PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t *parser, pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *stream_fgets, pm_parse_stream_feof_t *stream_feof, const pm_options_t *options);
-
-// We optionally support serializing to a binary string. For systems that don't
-// want or need this functionality, it can be turned off with the
-// PRISM_EXCLUDE_SERIALIZATION define.
-#ifndef PRISM_EXCLUDE_SERIALIZATION
-
-/**
- * Parse and serialize the AST represented by the source that is read out of the
- * given stream into to the given buffer.
- *
- * @param buffer The buffer to serialize to.
- * @param stream The stream to parse.
- * @param stream_fgets The function to use to read from the stream.
- * @param stream_feof The function to use to tell if the stream has hit eof.
- * @param data The optional data to pass to the parser.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize_parse_stream(pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *stream_fgets, pm_parse_stream_feof_t *stream_feof, const char *data);
-
-/**
- * Serialize the given list of comments to the given buffer.
- *
- * @param parser The parser to serialize.
- * @param list The list of comments to serialize.
- * @param buffer The buffer to serialize to.
- */
-void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer);
-
-/**
- * Serialize the name of the encoding to the buffer.
- *
- * @param encoding The encoding to serialize.
- * @param buffer The buffer to serialize to.
- */
-void pm_serialize_encoding(const pm_encoding_t *encoding, pm_buffer_t *buffer);
-
-/**
- * Serialize the encoding, metadata, nodes, and constant pool.
- *
- * @param parser The parser to serialize.
- * @param node The node to serialize.
- * @param buffer The buffer to serialize to.
- */
-void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
-
-/**
- * Serialize the AST represented by the given node to the given buffer.
- *
- * @param parser The parser to serialize.
- * @param node The node to serialize.
- * @param buffer The buffer to serialize to.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
-
-/**
- * Parse the given source to the AST and dump the AST to the given buffer.
- *
- * @param buffer The buffer to serialize to.
- * @param source The source to parse.
- * @param size The size of the source.
- * @param data The optional data to pass to the parser.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
-
-/**
- * Parse and serialize the comments in the given source to the given buffer.
- *
- * @param buffer The buffer to serialize to.
- * @param source The source to parse.
- * @param size The size of the source.
- * @param data The optional data to pass to the parser.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
-
-/**
- * Lex the given source and serialize to the given buffer.
- *
- * @param source The source to lex.
- * @param size The size of the source.
- * @param buffer The buffer to serialize to.
- * @param data The optional data to pass to the lexer.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
-
-/**
- * Parse and serialize both the AST and the tokens represented by the given
- * source to the given buffer.
- *
- * @param buffer The buffer to serialize to.
- * @param source The source to parse.
- * @param size The size of the source.
- * @param data The optional data to pass to the parser.
- */
-PRISM_EXPORTED_FUNCTION void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
-
-#endif
-
-/**
- * Parse the source and return true if it parses without errors or warnings.
- *
- * @param source The source to parse.
- * @param size The size of the source.
- * @param data The optional data to pass to the parser.
- * @return True if the source parses without errors or warnings.
- */
-PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t size, const char *data);
-
-/**
- * Returns a string representation of the given token type.
- *
- * @param token_type The token type to convert to a string.
- * @return A string representation of the given token type.
- */
-PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
-
-/**
- * Returns the human name of the given token type.
- *
- * @param token_type The token type to convert to a human name.
- * @return The human name of the given token type.
- */
-const char * pm_token_type_human(pm_token_type_t token_type);
-
-// We optionally support dumping to JSON. For systems that don't want or need
-// this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define.
-#ifndef PRISM_EXCLUDE_JSON
-
-/**
- * Dump JSON to the given buffer.
- *
- * @param buffer The buffer to serialize to.
- * @param parser The parser that parsed the node.
- * @param node The node to serialize.
- */
-PRISM_EXPORTED_FUNCTION void pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
-
-#endif
-
-/**
- * Represents the results of a slice query.
- */
-typedef enum {
-    /** Returned if the encoding given to a slice query was invalid. */
-    PM_STRING_QUERY_ERROR = -1,
-
-    /** Returned if the result of the slice query is false. */
-    PM_STRING_QUERY_FALSE,
-
-    /** Returned if the result of the slice query is true. */
-    PM_STRING_QUERY_TRUE
-} pm_string_query_t;
-
-/**
- * Check that the slice is a valid local variable name.
- *
- * @param source The source to check.
- * @param length The length of the source.
- * @param encoding_name The name of the encoding of the source.
- * @return PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
- *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
- */
-PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_local(const uint8_t *source, size_t length, const char *encoding_name);
-
-/**
- * Check that the slice is a valid constant name.
- *
- * @param source The source to check.
- * @param length The length of the source.
- * @param encoding_name The name of the encoding of the source.
- * @return PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
- *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
- */
-PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_constant(const uint8_t *source, size_t length, const char *encoding_name);
-
-/**
- * Check that the slice is a valid method name.
- *
- * @param source The source to check.
- * @param length The length of the source.
- * @param encoding_name The name of the encoding of the source.
- * @return PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
- *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
- */
-PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint8_t *source, size_t length, const char *encoding_name);
-
 /**
  * @mainpage
  *
@@ -328,27 +44,27 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint
  *
  * @section parsing Parsing
  *
- * In order to parse Ruby code, the structures and functions that you're going
- * to want to use and be aware of are:
+ * In order to parse Ruby code, the functions that you are going to want to use
+ * and be aware of are:
  *
- * * `pm_parser_t` - the main parser structure
- * * `pm_parser_init()` - initialize a parser
+ * * `pm_arena_new()` - create a new arena to hold all AST-lifetime allocations
+ * * `pm_parser_new()` - allocate and initialize a new parser
  * * `pm_parse()` - parse and return the root node
- * * `pm_node_destroy()` - deallocate the root node returned by `pm_parse()`
- * * `pm_parser_free()` - free the internal memory of the parser
+ * * `pm_parser_free()` - free the parser and its internal memory
+ * * `pm_arena_free()` - free all AST-lifetime memory
  *
  * Putting all of this together would look something like:
  *
  * ```c
  * void parse(const uint8_t *source, size_t length) {
- *     pm_parser_t parser;
- *     pm_parser_init(&parser, source, length, NULL);
+ *     pm_arena_t *arena = pm_arena_new();
+ *     pm_parser_t *parser = pm_parser_new(arena, source, length, NULL);
  *
- *     pm_node_t *root = pm_parse(&parser);
+ *     pm_node_t *root = pm_parse(parser);
  *     printf("PARSED!\n");
  *
- *     pm_node_destroy(&parser, root);
- *     pm_parser_free(&parser);
+ *     pm_parser_free(parser);
+ *     pm_arena_free(arena);
  * }
  * ```
  *
@@ -361,24 +77,23 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint
  * Prism provides the ability to serialize the AST and its related metadata into
  * a binary format. This format is designed to be portable to different
  * languages and runtimes so that you only need to make one FFI call in order to
- * parse Ruby code. The structures and functions that you're going to want to
- * use and be aware of are:
+ * parse Ruby code. The functions that you are going to want to use and be
+ * aware of are:
  *
- * * `pm_buffer_t` - a small buffer object that will hold the serialized AST
- * * `pm_buffer_free()` - free the memory associated with the buffer
- * * `pm_serialize()` - serialize the AST into a buffer
+ * * `pm_buffer_new()` - create a new buffer
+ * * `pm_buffer_free()` - free the buffer and its internal memory
  * * `pm_serialize_parse()` - parse and serialize the AST into a buffer
  *
  * Putting all of this together would look something like:
  *
  * ```c
  * void serialize(const uint8_t *source, size_t length) {
- *     pm_buffer_t buffer = { 0 };
+ *     pm_buffer_t *buffer = pm_buffer_new();
  *
- *     pm_serialize_parse(&buffer, source, length, NULL);
+ *     pm_serialize_parse(buffer, source, length, NULL);
  *     printf("SERIALIZED!\n");
  *
- *     pm_buffer_free(&buffer);
+ *     pm_buffer_free(buffer);
  * }
  * ```
  *
@@ -389,18 +104,18 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint
  *
  * ```c
  * void prettyprint(const uint8_t *source, size_t length) {
- *     pm_parser_t parser;
- *     pm_parser_init(&parser, source, length, NULL);
+ *     pm_arena_t *arena = pm_arena_new();
+ *     pm_parser_t *parser = pm_parser_new(arena, source, length, NULL);
  *
- *     pm_node_t *root = pm_parse(&parser);
- *     pm_buffer_t buffer = { 0 };
+ *     pm_node_t *root = pm_parse(parser);
+ *     pm_buffer_t *buffer = pm_buffer_new();
  *
- *     pm_prettyprint(&buffer, &parser, root);
- *     printf("%*.s\n", (int) buffer.length, buffer.value);
+ *     pm_prettyprint(buffer, parser, root);
+ *     printf("%*.s\n", (int) pm_buffer_length(buffer), pm_buffer_value(buffer));
  *
- *     pm_buffer_free(&buffer);
- *     pm_node_destroy(&parser, root);
- *     pm_parser_free(&parser);
+ *     pm_buffer_free(buffer);
+ *     pm_parser_free(parser);
+ *     pm_arena_free(arena);
  * }
  * ```
  */

data/vendor/prism/include/prism.hpp

@@ -0,0 +1,24 @@
+/**
+ * @file prism.hpp
+ *
+ * The main header file for the C++ interface to the prism parser. It adds
+ * ownership, checked casts between node types, and callable-based iteration on
+ * top of the C API. It never wraps the C structs: every function in it takes
+ * and returns the same pm_*_t types that prism.h declares, so the two can be
+ * mixed freely. It requires C++17 or later.
+ */
+#ifndef PRISM_HPP
+#define PRISM_HPP
+
+#include "prism.h"
+
+#include "prism/arena.hpp"
+#include "prism/ast.hpp"
+#include "prism/buffer.hpp"
+#include "prism/node.hpp"
+#include "prism/node_type.hpp"
+#include "prism/options.hpp"
+#include "prism/parser.hpp"
+#include "prism/stringy.hpp"
+
+#endif