debase-ruby_core_source 3.3.1 → 3.3.5

data/lib/debase/ruby_core_source/ruby-3.4.0-preview1/prism/encoding.h

@@ -0,0 +1,283 @@
+/**
+ * @file encoding.h
+ *
+ * The encoding interface and implementations used by the parser.
+ */
+#ifndef PRISM_ENCODING_H
+#define PRISM_ENCODING_H
+
+#include "prism/defines.h"
+#include "prism/util/pm_strncasecmp.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * This struct defines the functions necessary to implement the encoding
+ * interface so we can determine how many bytes the subsequent character takes.
+ * Each callback should return the number of bytes, or 0 if the next bytes are
+ * invalid for the encoding and type.
+ */
+typedef struct {
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding. Does not read more than n bytes. It is assumed that n is
+     * at least 1.
+     */
+    size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphabetical. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphanumeric. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
+    size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * Return true if the next character is valid in the encoding and is an
+     * uppercase character. Does not read more than n bytes. It is assumed that
+     * n is at least 1.
+     */
+    bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
+
+    /**
+     * The name of the encoding. This should correspond to a value that can be
+     * passed to Encoding.find in Ruby.
+     */
+    const char *name;
+
+    /**
+     * Return true if the encoding is a multibyte encoding.
+     */
+    bool multibyte;
+} pm_encoding_t;
+
+/**
+ * All of the lookup tables use the first bit of each embedded byte to indicate
+ * whether the codepoint is alphabetical.
+ */
+#define PRISM_ENCODING_ALPHABETIC_BIT 1 << 0
+
+/**
+ * All of the lookup tables use the second bit of each embedded byte to indicate
+ * whether the codepoint is alphanumeric.
+ */
+#define PRISM_ENCODING_ALPHANUMERIC_BIT 1 << 1
+
+/**
+ * All of the lookup tables use the third bit of each embedded byte to indicate
+ * whether the codepoint is uppercase.
+ */
+#define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
+
+/**
+ * Return the size of the next character in the UTF-8 encoding.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphabetical character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t pm_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphanumeric character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
+size_t pm_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return true if the next character in the UTF-8 encoding if it is an uppercase
+ * character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns True if the next character is valid in the encoding and is an
+ *     uppercase character, or false if it is not.
+ */
+bool pm_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * This lookup table is referenced in both the UTF-8 encoding file and the
+ * parser directly in order to speed up the default encoding processing. It is
+ * used to indicate whether a character is alphabetical, alphanumeric, or
+ * uppercase in unicode mappings.
+ */
+extern const uint8_t pm_encoding_unicode_table[256];
+
+/**
+ * These are all of the encodings that prism supports.
+ */
+typedef enum {
+    PM_ENCODING_UTF_8 = 0,
+    PM_ENCODING_US_ASCII,
+    PM_ENCODING_ASCII_8BIT,
+    PM_ENCODING_EUC_JP,
+    PM_ENCODING_WINDOWS_31J,
+
+// We optionally support excluding the full set of encodings to only support the
+// minimum necessary to process Ruby code without encoding comments.
+#ifndef PRISM_ENCODING_EXCLUDE_FULL
+    PM_ENCODING_BIG5,
+    PM_ENCODING_BIG5_HKSCS,
+    PM_ENCODING_BIG5_UAO,
+    PM_ENCODING_CESU_8,
+    PM_ENCODING_CP51932,
+    PM_ENCODING_CP850,
+    PM_ENCODING_CP852,
+    PM_ENCODING_CP855,
+    PM_ENCODING_CP949,
+    PM_ENCODING_CP950,
+    PM_ENCODING_CP951,
+    PM_ENCODING_EMACS_MULE,
+    PM_ENCODING_EUC_JP_MS,
+    PM_ENCODING_EUC_JIS_2004,
+    PM_ENCODING_EUC_KR,
+    PM_ENCODING_EUC_TW,
+    PM_ENCODING_GB12345,
+    PM_ENCODING_GB18030,
+    PM_ENCODING_GB1988,
+    PM_ENCODING_GB2312,
+    PM_ENCODING_GBK,
+    PM_ENCODING_IBM437,
+    PM_ENCODING_IBM720,
+    PM_ENCODING_IBM737,
+    PM_ENCODING_IBM775,
+    PM_ENCODING_IBM852,
+    PM_ENCODING_IBM855,
+    PM_ENCODING_IBM857,
+    PM_ENCODING_IBM860,
+    PM_ENCODING_IBM861,
+    PM_ENCODING_IBM862,
+    PM_ENCODING_IBM863,
+    PM_ENCODING_IBM864,
+    PM_ENCODING_IBM865,
+    PM_ENCODING_IBM866,
+    PM_ENCODING_IBM869,
+    PM_ENCODING_ISO_8859_1,
+    PM_ENCODING_ISO_8859_2,
+    PM_ENCODING_ISO_8859_3,
+    PM_ENCODING_ISO_8859_4,
+    PM_ENCODING_ISO_8859_5,
+    PM_ENCODING_ISO_8859_6,
+    PM_ENCODING_ISO_8859_7,
+    PM_ENCODING_ISO_8859_8,
+    PM_ENCODING_ISO_8859_9,
+    PM_ENCODING_ISO_8859_10,
+    PM_ENCODING_ISO_8859_11,
+    PM_ENCODING_ISO_8859_13,
+    PM_ENCODING_ISO_8859_14,
+    PM_ENCODING_ISO_8859_15,
+    PM_ENCODING_ISO_8859_16,
+    PM_ENCODING_KOI8_R,
+    PM_ENCODING_KOI8_U,
+    PM_ENCODING_MAC_CENT_EURO,
+    PM_ENCODING_MAC_CROATIAN,
+    PM_ENCODING_MAC_CYRILLIC,
+    PM_ENCODING_MAC_GREEK,
+    PM_ENCODING_MAC_ICELAND,
+    PM_ENCODING_MAC_JAPANESE,
+    PM_ENCODING_MAC_ROMAN,
+    PM_ENCODING_MAC_ROMANIA,
+    PM_ENCODING_MAC_THAI,
+    PM_ENCODING_MAC_TURKISH,
+    PM_ENCODING_MAC_UKRAINE,
+    PM_ENCODING_SHIFT_JIS,
+    PM_ENCODING_SJIS_DOCOMO,
+    PM_ENCODING_SJIS_KDDI,
+    PM_ENCODING_SJIS_SOFTBANK,
+    PM_ENCODING_STATELESS_ISO_2022_JP,
+    PM_ENCODING_STATELESS_ISO_2022_JP_KDDI,
+    PM_ENCODING_TIS_620,
+    PM_ENCODING_UTF8_MAC,
+    PM_ENCODING_UTF8_DOCOMO,
+    PM_ENCODING_UTF8_KDDI,
+    PM_ENCODING_UTF8_SOFTBANK,
+    PM_ENCODING_WINDOWS_1250,
+    PM_ENCODING_WINDOWS_1251,
+    PM_ENCODING_WINDOWS_1252,
+    PM_ENCODING_WINDOWS_1253,
+    PM_ENCODING_WINDOWS_1254,
+    PM_ENCODING_WINDOWS_1255,
+    PM_ENCODING_WINDOWS_1256,
+    PM_ENCODING_WINDOWS_1257,
+    PM_ENCODING_WINDOWS_1258,
+    PM_ENCODING_WINDOWS_874,
+#endif
+
+    PM_ENCODING_MAXIMUM
+} pm_encoding_type_t;
+
+/**
+ * This is the table of all of the encodings that prism supports.
+ */
+extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
+
+/**
+ * This is the default UTF-8 encoding. We need a reference to it to quickly
+ * create parsers.
+ */
+#define PM_ENCODING_UTF_8_ENTRY (&pm_encodings[PM_ENCODING_UTF_8])
+
+/**
+ * This is the US-ASCII encoding. We need a reference to it to be able to
+ * compare against it when a string is being created because it could possibly
+ * need to fall back to ASCII-8BIT.
+ */
+#define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
+
+/**
+ * This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
+ * can compare against it because invalid multibyte characters are not a thing
+ * in this encoding. It is also needed for handling Regexp encoding flags.
+ */
+#define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
+
+/**
+ * This is the EUC-JP encoding. We need a reference to it to quickly process
+ * regular expression modifiers.
+ */
+#define PM_ENCODING_EUC_JP_ENTRY (&pm_encodings[PM_ENCODING_EUC_JP])
+
+/**
+ * This is the Windows-31J encoding. We need a reference to it to quickly
+ * process regular expression modifiers.
+ */
+#define PM_ENCODING_WINDOWS_31J_ENTRY (&pm_encodings[PM_ENCODING_WINDOWS_31J])
+
+/**
+ * Parse the given name of an encoding and return a pointer to the corresponding
+ * encoding struct if one can be found, otherwise return NULL.
+ *
+ * @param start A pointer to the first byte of the name.
+ * @param end A pointer to the last byte of the name.
+ * @returns A pointer to the encoding struct if one is found, otherwise NULL.
+ */
+const pm_encoding_t * pm_encoding_find(const uint8_t *start, const uint8_t *end);
+
+#endif

data/lib/debase/ruby_core_source/ruby-3.4.0-preview1/prism/extension.h

@@ -0,0 +1,19 @@
+#ifndef PRISM_EXT_NODE_H
+#define PRISM_EXT_NODE_H
+
+#define EXPECTED_PRISM_VERSION "0.29.0"
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include "prism.h"
+
+VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding);
+VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source);
+VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source);
+VALUE pm_integer_new(const pm_integer_t *integer);
+
+void Init_prism_api_node(void);
+void Init_prism_pack(void);
+PRISM_EXPORTED_FUNCTION void Init_prism(void);
+
+#endif

data/lib/debase/ruby_core_source/ruby-3.4.0-preview1/prism/node.h

@@ -0,0 +1,150 @@
+/**
+ * @file node.h
+ *
+ * Functions related to nodes in the AST.
+ */
+#ifndef PRISM_NODE_H
+#define PRISM_NODE_H
+
+#include "prism/defines.h"
+#include "prism/parser.h"
+#include "prism/util/pm_buffer.h"
+
+/**
+ * Loop through each node in the node list, writing each node to the given
+ * pm_node_t pointer.
+ */
+#define PM_NODE_LIST_FOREACH(list, index, node) \
+    for (size_t index = 0; index < (list)->size && ((node) = (list)->nodes[index]); index++)
+
+/**
+ * Append a new node onto the end of the node list.
+ *
+ * @param list The list to append to.
+ * @param node The node to append.
+ */
+void pm_node_list_append(pm_node_list_t *list, pm_node_t *node);
+
+/**
+ * Prepend a new node onto the beginning of the node list.
+ *
+ * @param list The list to prepend to.
+ * @param node The node to prepend.
+ */
+void pm_node_list_prepend(pm_node_list_t *list, pm_node_t *node);
+
+/**
+ * Concatenate the given node list onto the end of the other node list.
+ *
+ * @param list The list to concatenate onto.
+ * @param other The list to concatenate.
+ */
+void pm_node_list_concat(pm_node_list_t *list, pm_node_list_t *other);
+
+/**
+ * Free the internal memory associated with the given node list.
+ *
+ * @param list The list to free.
+ */
+void pm_node_list_free(pm_node_list_t *list);
+
+/**
+ * Deallocate a node and all of its children.
+ *
+ * @param parser The parser that owns the node.
+ * @param node The node to deallocate.
+ */
+PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
+
+/**
+ * This struct stores the information gathered by the pm_node_memsize function.
+ * It contains both the memory footprint and additionally metadata about the
+ * shape of the tree.
+ */
+typedef struct {
+    /** The total memory footprint of the node and all of its children. */
+    size_t memsize;
+
+    /** The number of children the node has. */
+    size_t node_count;
+} pm_memsize_t;
+
+/**
+ * Calculates the memory footprint of a given node.
+ *
+ * @param node The node to calculate the memory footprint of.
+ * @param memsize The memory footprint of the node and all of its children.
+ */
+PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize);
+
+/**
+ * Returns a string representation of the given node type.
+ *
+ * @param node_type The node type to convert to a string.
+ * @return A string representation of the given node type.
+ */
+PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_type);
+
+/**
+ * Visit each of the nodes in this subtree using the given visitor callback. The
+ * callback function will be called for each node in the subtree. If it returns
+ * false, then that node's children will not be visited. If it returns true,
+ * then the children will be visited. The data parameter is treated as an opaque
+ * pointer and is passed to the visitor callback for consumers to use as they
+ * see fit.
+ *
+ * As an example:
+ *
+ * ```c
+ * #include "prism.h"
+ *
+ * bool visit(const pm_node_t *node, void *data) {
+ *     size_t *indent = (size_t *) data;
+ *     for (size_t i = 0; i < *indent * 2; i++) putc(' ', stdout);
+ *     printf("%s\n", pm_node_type_to_str(node->type));
+ *
+ *     size_t next_indent = *indent + 1;
+ *     size_t *next_data = &next_indent;
+ *     pm_visit_child_nodes(node, visit, next_data);
+ *
+ *     return false;
+ * }
+ *
+ * int main(void) {
+ *     const char *source = "1 + 2; 3 + 4";
+ *     size_t size = strlen(source);
+ *
+ *     pm_parser_t parser;
+ *     pm_options_t options = { 0 };
+ *     pm_parser_init(&parser, (const uint8_t *) source, size, &options);
+ *
+ *     size_t indent = 0;
+ *     pm_node_t *node = pm_parse(&parser);
+ *
+ *     size_t *data = &indent;
+ *     pm_visit_node(node, visit, data);
+ *
+ *     pm_node_destroy(&parser, node);
+ *     pm_parser_free(&parser);
+ *     return EXIT_SUCCESS;
+ * }
+ * ```
+ *
+ * @param node The root node to start visiting from.
+ * @param visitor The callback to call for each node in the subtree.
+ * @param data An opaque pointer that is passed to the visitor callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_visit_node(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+
+/**
+ * Visit the children of the given node with the given callback. This is the
+ * default behavior for walking the tree that is called from pm_visit_node if
+ * the callback returns true.
+ *
+ * @param node The node to visit the children of.
+ * @param visitor The callback to call for each child node.
+ * @param data An opaque pointer that is passed to the visitor callback.
+ */
+PRISM_EXPORTED_FUNCTION void pm_visit_child_nodes(const pm_node_t *node, bool (*visitor)(const pm_node_t *node, void *data), void *data);
+
+#endif