yarp 0.8.0 → 0.10.0

data/include/yarp/defines.h

@@ -6,6 +6,7 @@
 #include <ctype.h>
 #include <stdarg.h>
 #include <stddef.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -39,6 +40,6 @@
 #   define snprintf _snprintf
 #endif
 
-int yp_strncasecmp(const char *string1, const char *string2, size_t length);
+int yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length);
 
 #endif

data/include/yarp/diagnostic.h

@@ -10,13 +10,13 @@
 // This struct represents a diagnostic found during parsing.
 typedef struct {
     yp_list_node_t node;
-    const char *start;
-    const char *end;
+    const uint8_t *start;
+    const uint8_t *end;
     const char *message;
 } yp_diagnostic_t;
 
 // Append a diagnostic to the given list of diagnostics.
-bool yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message);
+bool yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message);
 
 // Deallocate the internal state of the given diagnostic list.
 void yp_diagnostic_list_free(yp_list_t *list);

data/include/yarp/enc/yp_encoding.h

@@ -16,22 +16,22 @@ typedef struct {
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding. Does not read more than n bytes. It is assumed that n is
     // at least 1.
-    size_t (*char_width)(const char *c, ptrdiff_t n);
+    size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
 
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding and is alphabetical. Does not read more than n bytes. It
     // is assumed that n is at least 1.
-    size_t (*alpha_char)(const char *c, ptrdiff_t n);
+    size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
 
     // Return the number of bytes that the next character takes if it is valid
     // in the encoding and is alphanumeric. Does not read more than n bytes. It
     // is assumed that n is at least 1.
-    size_t (*alnum_char)(const char *c, ptrdiff_t n);
+    size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
 
     // Return true if the next character is valid in the encoding and is an
     // uppercase character. Does not read more than n bytes. It is assumed that
     // n is at least 1.
-    bool (*isupper_char)(const char *c, ptrdiff_t n);
+    bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
 
     // The name of the encoding. This should correspond to a value that can be
     // passed to Encoding.find in Ruby.
@@ -49,18 +49,18 @@ typedef struct {
 
 // These functions are reused by some other encodings, so they are defined here
 // so they can be shared.
-size_t yp_encoding_ascii_alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
-size_t yp_encoding_ascii_alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
-bool yp_encoding_ascii_isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
+size_t yp_encoding_ascii_alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
+size_t yp_encoding_ascii_alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
+bool yp_encoding_ascii_isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n);
 
 // These functions are shared between the actual encoding and the fast path in
 // the parser so they need to be internally visible.
-size_t yp_encoding_utf_8_alpha_char(const char *c, ptrdiff_t n);
-size_t yp_encoding_utf_8_alnum_char(const char *c, ptrdiff_t n);
+size_t yp_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n);
+size_t yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
 
 // This lookup table is referenced in both the UTF-8 encoding file and the
 // parser directly in order to speed up the default encoding processing.
-extern unsigned char yp_encoding_unicode_table[256];
+extern uint8_t yp_encoding_unicode_table[256];
 
 // These are the encodings that are supported by the parser. They are defined in
 // their own files in the src/enc directory.

data/include/yarp/node.h

@@ -34,3 +34,13 @@ YP_EXPORTED_FUNCTION const char * yp_node_type_to_str(yp_node_type_t node_type);
 #define YP_EMPTY_LOCATION_LIST ((yp_location_list_t) { .locations = NULL, .size = 0, .capacity = 0 })
 
 #endif // YARP_NODE_H
+
+// ScopeNodes are helper nodes, and will never
+// be part of the AST. We manually declare them
+// here to avoid generating them
+typedef struct yp_scope_node {
+    yp_node_t base;
+    struct yp_parameters_node *parameters;
+    yp_node_t *body;
+    yp_constant_id_list_t locals;
+} yp_scope_node_t;

data/include/yarp/parser.h

@@ -109,14 +109,14 @@ typedef struct yp_lex_mode {
 
             // When lexing a list, it takes into account balancing the
             // terminator if the terminator is one of (), [], {}, or <>.
-            char incrementor;
+            uint8_t incrementor;
 
             // This is the terminator of the list literal.
-            char terminator;
+            uint8_t terminator;
 
             // This is the character set that should be used to delimit the
             // tokens within the list.
-            char breakpoints[11];
+            uint8_t breakpoints[11];
         } list;
 
         struct {
@@ -125,14 +125,14 @@ typedef struct yp_lex_mode {
 
             // When lexing a regular expression, it takes into account balancing
             // the terminator if the terminator is one of (), [], {}, or <>.
-            char incrementor;
+            uint8_t incrementor;
 
             // This is the terminator of the regular expression.
-            char terminator;
+            uint8_t terminator;
 
             // This is the character set that should be used to delimit the
             // tokens within the regular expression.
-            char breakpoints[6];
+            uint8_t breakpoints[6];
         } regexp;
 
         struct {
@@ -149,21 +149,21 @@ typedef struct yp_lex_mode {
 
             // When lexing a string, it takes into account balancing the
             // terminator if the terminator is one of (), [], {}, or <>.
-            char incrementor;
+            uint8_t incrementor;
 
             // This is the terminator of the string. It is typically either a
             // single or double quote.
-            char terminator;
+            uint8_t terminator;
 
             // This is the character set that should be used to delimit the
             // tokens within the string.
-            char breakpoints[6];
+            uint8_t breakpoints[6];
         } string;
 
         struct {
             // These pointers point to the beginning and end of the heredoc
             // identifier.
-            const char *ident_start;
+            const uint8_t *ident_start;
             size_t ident_length;
 
             yp_heredoc_quote_t quote;
@@ -171,7 +171,7 @@ typedef struct yp_lex_mode {
 
             // This is the pointer to the character where lexing should resume
             // once the heredoc has been completely processed.
-            const char *next_start;
+            const uint8_t *next_start;
         } heredoc;
     } as;
 
@@ -239,8 +239,8 @@ typedef enum {
 // This is a node in the linked list of comments that we've found while parsing.
 typedef struct yp_comment {
     yp_list_node_t node;
-    const char *start;
-    const char *end;
+    const uint8_t *start;
+    const uint8_t *end;
     yp_comment_type_t type;
 } yp_comment_t;
 
@@ -252,7 +252,7 @@ typedef void (*yp_encoding_changed_callback_t)(yp_parser_t *parser);
 // the ability here to call out to a user-defined function to get an encoding
 // struct. If the function returns something that isn't NULL, we set that to
 // our encoding and use it to parse identifiers.
-typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const char *name, size_t width);
+typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const uint8_t *name, size_t width);
 
 // When you are lexing through a file, the lexer needs all of the information
 // that the parser additionally provides (for example, the local table). So if
@@ -316,21 +316,21 @@ struct yp_parser {
         size_t index;                           // the current index into the lexer mode stack
     } lex_modes;
 
-    const char *start;   // the pointer to the start of the source
-    const char *end;     // the pointer to the end of the source
+    const uint8_t *start;   // the pointer to the start of the source
+    const uint8_t *end;     // the pointer to the end of the source
     yp_token_t previous; // the previous token we were considering
     yp_token_t current;  // the current token we're considering
 
     // This is a special field set on the parser when we need the parser to jump
     // to a specific location when lexing the next token, as opposed to just
     // using the end of the previous token. Normally this is NULL.
-    const char *next_start;
+    const uint8_t *next_start;
 
     // This field indicates the end of a heredoc whose identifier was found on
     // the current line. If another heredoc is found on the same line, then this
     // will be moved forward to the end of that heredoc. If no heredocs are
     // found on a line then this is NULL.
-    const char *heredoc_end;
+    const uint8_t *heredoc_end;
 
     yp_list_t comment_list;             // the list of comments that have been found while parsing
     yp_list_t warning_list;             // the list of warnings that have been found while parsing
@@ -361,7 +361,7 @@ struct yp_parser {
 
     // This pointer indicates where a comment must start if it is to be
     // considered an encoding comment.
-    const char *encoding_comment_start;
+    const uint8_t *encoding_comment_start;
 
     // This is an optional callback that can be attached to the parser that will
     // be called whenever a new token is lexed by the parser.

data/include/yarp/regexp.h

@@ -14,6 +14,6 @@
 
 // Parse a regular expression and extract the names of all of the named capture
 // groups.
-YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const char *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding);
+YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const uint8_t *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding);
 
 #endif

data/include/yarp/unescape.h

@@ -29,14 +29,16 @@ typedef enum {
     YP_UNESCAPE_ALL
 } yp_unescape_type_t;
 
-// Unescape the contents of the given token into the given string using the
-// given unescape mode.
-YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list);
+// Unescape the contents of the given token into the given string using the given unescape mode.
+YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type);
+void yp_unescape_manipulate_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type);
 
 // Accepts a source string and a type of unescaping and returns the unescaped version.
 // The caller must yp_string_free(result); after calling this function.
-YP_EXPORTED_FUNCTION bool yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result);
+YP_EXPORTED_FUNCTION bool yp_unescape_string(const uint8_t *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result);
 
-YP_EXPORTED_FUNCTION size_t yp_unescape_calculate_difference(const char *value, const char *end, yp_unescape_type_t unescape_type, bool expect_single_codepoint, yp_list_t *error_list);
+// Returns the number of bytes that encompass the first escape sequence in the
+// given string.
+size_t yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint);
 
 #endif

data/include/yarp/util/yp_buffer.h

@@ -36,6 +36,9 @@ void yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length);
 // Append a string to the buffer.
 void yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length);
 
+// Append a list of bytes to the buffer.
+void yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length);
+
 // Append a single byte to the buffer.
 void yp_buffer_append_u8(yp_buffer_t *buffer, uint8_t value);
 

data/include/yarp/util/yp_char.h

@@ -9,67 +9,67 @@
 
 // Returns the number of characters at the start of the string that are
 // whitespace. Disallows searching past the given maximum number of characters.
-size_t yp_strspn_whitespace(const char *string, ptrdiff_t length);
+size_t yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are
 // whitespace while also tracking the location of each newline. Disallows
 // searching past the given maximum number of characters.
 size_t
-yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool);
+yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline);
 
 // Returns the number of characters at the start of the string that are inline
 // whitespace. Disallows searching past the given maximum number of characters.
-size_t yp_strspn_inline_whitespace(const char *string, ptrdiff_t length);
+size_t yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are decimal
 // digits. Disallows searching past the given maximum number of characters.
-size_t yp_strspn_decimal_digit(const char *string, ptrdiff_t length);
+size_t yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are
 // hexadecimal digits. Disallows searching past the given maximum number of
 // characters.
-size_t yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length);
+size_t yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are octal
 // digits or underscores.  Disallows searching past the given maximum number of
 // characters.
-size_t yp_strspn_octal_number(const char *string, ptrdiff_t length);
+size_t yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are decimal
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
-size_t yp_strspn_decimal_number(const char *string, ptrdiff_t length);
+size_t yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are
 // hexadecimal digits or underscores. Disallows searching past the given maximum
 // number of characters.
-size_t yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length);
+size_t yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are regexp
 // options. Disallows searching past the given maximum number of characters.
-size_t yp_strspn_regexp_option(const char *string, ptrdiff_t length);
+size_t yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
 
 // Returns the number of characters at the start of the string that are binary
 // digits or underscores. Disallows searching past the given maximum number of
 // characters.
-size_t yp_strspn_binary_number(const char *string, ptrdiff_t length);
+size_t yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length);
 
 // Returns true if the given character is a whitespace character.
-bool yp_char_is_whitespace(const char c);
+bool yp_char_is_whitespace(const uint8_t b);
 
 // Returns true if the given character is an inline whitespace character.
-bool yp_char_is_inline_whitespace(const char c);
+bool yp_char_is_inline_whitespace(const uint8_t b);
 
 // Returns true if the given character is a binary digit.
-bool yp_char_is_binary_digit(const char c);
+bool yp_char_is_binary_digit(const uint8_t b);
 
 // Returns true if the given character is an octal digit.
-bool yp_char_is_octal_digit(const char c);
+bool yp_char_is_octal_digit(const uint8_t b);
 
 // Returns true if the given character is a decimal digit.
-bool yp_char_is_decimal_digit(const char c);
+bool yp_char_is_decimal_digit(const uint8_t b);
 
 // Returns true if the given character is a hexadecimal digit.
-bool yp_char_is_hexadecimal_digit(const char c);
+bool yp_char_is_hexadecimal_digit(const uint8_t b);
 
 #endif

data/include/yarp/util/yp_constant_pool.h

@@ -40,7 +40,7 @@ void yp_constant_id_list_free(yp_constant_id_list_t *list);
 
 typedef struct {
     yp_constant_id_t id;
-    const char *start;
+    const uint8_t *start;
     size_t length;
     size_t hash;
 } yp_constant_t;
@@ -59,7 +59,7 @@ bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity);
 
 // Insert a constant into a constant pool. Returns the id of the constant, or 0
 // if any potential calls to resize fail.
-yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length);
+yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length);
 
 // Free the memory associated with a constant pool.
 void yp_constant_pool_free(yp_constant_pool_t *pool);

data/include/yarp/util/yp_newline_list.h

@@ -19,7 +19,7 @@
 // A list of offsets of newlines in a string. The offsets are assumed to be
 // sorted/inserted in ascending order.
 typedef struct {
-    const char *start;
+    const uint8_t *start;
 
     size_t *offsets;
     size_t size;
@@ -41,16 +41,19 @@ typedef struct {
 
 // Initialize a new newline list with the given capacity. Returns true if the
 // allocation of the offsets succeeds, otherwise returns false.
-bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity);
+bool yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity);
 
 // Append a new offset to the newline list. Returns true if the reallocation of
 // the offsets succeeds (if one was necessary), otherwise returns false.
-bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor);
+bool yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor);
+
+// Conditionally append a new offset to the newline list, if the value passed in is a newline.
+bool yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor);
 
 // Returns the line and column of the given offset. If the offset is not in the
 // list, the line and column of the closest offset less than the given offset
 // are returned.
-yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor);
+yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor);
 
 // Free the internal memory allocated for the newline list.
 void yp_newline_list_free(yp_newline_list_t *list);

data/include/yarp/util/yp_string.h

@@ -12,17 +12,17 @@
 // This struct represents a string value.
 typedef struct {
     enum { YP_STRING_SHARED, YP_STRING_OWNED, YP_STRING_CONSTANT, YP_STRING_MAPPED } type;
-    char *source;
+    const uint8_t *source;
     size_t length;
 } yp_string_t;
 
 #define YP_EMPTY_STRING ((yp_string_t) { .type = YP_STRING_CONSTANT, .source = NULL, .length = 0 })
 
 // Initialize a shared string that is based on initial input.
-void yp_string_shared_init(yp_string_t *string, const char *start, const char *end);
+void yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end);
 
 // Initialize an owned string that is responsible for freeing allocated memory.
-void yp_string_owned_init(yp_string_t *string, char *source, size_t length);
+void yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length);
 
 // Initialize a constant string that doesn't own its memory source.
 void yp_string_constant_init(yp_string_t *string, const char *source, size_t length);
@@ -49,7 +49,7 @@ void yp_string_ensure_owned(yp_string_t *string);
 YP_EXPORTED_FUNCTION size_t yp_string_length(const yp_string_t *string);
 
 // Returns the start pointer associated with the string.
-YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string);
+YP_EXPORTED_FUNCTION const uint8_t * yp_string_source(const yp_string_t *string);
 
 // Free the associated memory of the given string.
 YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string);

data/include/yarp/util/yp_string_list.h

@@ -13,9 +13,6 @@ typedef struct {
     size_t capacity;
 } yp_string_list_t;
 
-// Allocate a new yp_string_list_t.
-yp_string_list_t * yp_string_list_alloc(void);
-
 // Initialize a yp_string_list_t with its default values.
 YP_EXPORTED_FUNCTION void yp_string_list_init(yp_string_list_t *string_list);
 

data/include/yarp/util/yp_strpbrk.h

@@ -24,6 +24,6 @@
 // characters that are trailing bytes of multi-byte characters. For example, in
 // Shift-JIS, the backslash character can be a trailing byte. In that case we
 // need to take a slower path and iterate one multi-byte character at a time.
-const char * yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length);
+const uint8_t * yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
 
 #endif

data/include/yarp/version.h

@@ -1,4 +1,4 @@
 #define YP_VERSION_MAJOR 0
-#define YP_VERSION_MINOR 8
+#define YP_VERSION_MINOR 10
 #define YP_VERSION_PATCH 0
-#define YP_VERSION "0.8.0"
+#define YP_VERSION "0.10.0"

data/include/yarp.h

@@ -13,8 +13,10 @@
 #include "yarp/util/yp_char.h"
 #include "yarp/util/yp_memchr.h"
 #include "yarp/util/yp_strpbrk.h"
+#include "yarp/version.h"
 
 #include <assert.h>
+#include <errno.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -30,11 +32,16 @@ void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buf
 
 void yp_print_node(yp_parser_t *parser, yp_node_t *node);
 
+void yp_parser_metadata(yp_parser_t *parser, const char *metadata);
+
+// Generate a scope node from the given node.
+void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest);
+
 // The YARP version and the serialization format.
 YP_EXPORTED_FUNCTION const char * yp_version(void);
 
 // Initialize a parser with the given start and end pointers.
-YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath);
+YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const char *filepath);
 
 // Register a callback that will be called whenever YARP changes the encoding it
 // is using to parse based on the magic comment.
@@ -60,10 +67,14 @@ YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, y
 YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer);
 
 // Parse the given source to the AST and serialize the AST to the given buffer.
-YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
+YP_EXPORTED_FUNCTION void yp_parse_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata);
 
 // Lex the given source and serialize to the given buffer.
-YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer);
+YP_EXPORTED_FUNCTION void yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer);
+
+// Parse and serialize both the AST and the tokens represented by the given
+// source to the given buffer.
+YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata);
 
 // Returns a string representation of the given token type.
 YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);