prism 0.13.0

data/include/prism/parser.h

@@ -0,0 +1,418 @@
+#ifndef PRISM_PARSER_H
+#define PRISM_PARSER_H
+
+#include "prism/ast.h"
+#include "prism/defines.h"
+#include "prism/enc/pm_encoding.h"
+#include "prism/util/pm_constant_pool.h"
+#include "prism/util/pm_list.h"
+#include "prism/util/pm_newline_list.h"
+#include "prism/util/pm_state_stack.h"
+
+#include <stdbool.h>
+
+// This enum provides various bits that represent different kinds of states that
+// the lexer can track. This is used to determine which kind of token to return
+// based on the context of the parser.
+typedef enum {
+    PM_LEX_STATE_BIT_BEG,
+    PM_LEX_STATE_BIT_END,
+    PM_LEX_STATE_BIT_ENDARG,
+    PM_LEX_STATE_BIT_ENDFN,
+    PM_LEX_STATE_BIT_ARG,
+    PM_LEX_STATE_BIT_CMDARG,
+    PM_LEX_STATE_BIT_MID,
+    PM_LEX_STATE_BIT_FNAME,
+    PM_LEX_STATE_BIT_DOT,
+    PM_LEX_STATE_BIT_CLASS,
+    PM_LEX_STATE_BIT_LABEL,
+    PM_LEX_STATE_BIT_LABELED,
+    PM_LEX_STATE_BIT_FITEM
+} pm_lex_state_bit_t;
+
+// This enum combines the various bits from the above enum into individual
+// values that represent the various states of the lexer.
+typedef enum {
+    PM_LEX_STATE_NONE = 0,
+    PM_LEX_STATE_BEG = (1 << PM_LEX_STATE_BIT_BEG),
+    PM_LEX_STATE_END = (1 << PM_LEX_STATE_BIT_END),
+    PM_LEX_STATE_ENDARG = (1 << PM_LEX_STATE_BIT_ENDARG),
+    PM_LEX_STATE_ENDFN = (1 << PM_LEX_STATE_BIT_ENDFN),
+    PM_LEX_STATE_ARG = (1 << PM_LEX_STATE_BIT_ARG),
+    PM_LEX_STATE_CMDARG = (1 << PM_LEX_STATE_BIT_CMDARG),
+    PM_LEX_STATE_MID = (1 << PM_LEX_STATE_BIT_MID),
+    PM_LEX_STATE_FNAME = (1 << PM_LEX_STATE_BIT_FNAME),
+    PM_LEX_STATE_DOT = (1 << PM_LEX_STATE_BIT_DOT),
+    PM_LEX_STATE_CLASS = (1 << PM_LEX_STATE_BIT_CLASS),
+    PM_LEX_STATE_LABEL = (1 << PM_LEX_STATE_BIT_LABEL),
+    PM_LEX_STATE_LABELED = (1 << PM_LEX_STATE_BIT_LABELED),
+    PM_LEX_STATE_FITEM = (1 << PM_LEX_STATE_BIT_FITEM),
+    PM_LEX_STATE_BEG_ANY = PM_LEX_STATE_BEG | PM_LEX_STATE_MID | PM_LEX_STATE_CLASS,
+    PM_LEX_STATE_ARG_ANY = PM_LEX_STATE_ARG | PM_LEX_STATE_CMDARG,
+    PM_LEX_STATE_END_ANY = PM_LEX_STATE_END | PM_LEX_STATE_ENDARG | PM_LEX_STATE_ENDFN
+} pm_lex_state_t;
+
+typedef enum {
+    PM_HEREDOC_QUOTE_NONE,
+    PM_HEREDOC_QUOTE_SINGLE = '\'',
+    PM_HEREDOC_QUOTE_DOUBLE = '"',
+    PM_HEREDOC_QUOTE_BACKTICK = '`',
+} pm_heredoc_quote_t;
+
+typedef enum {
+    PM_HEREDOC_INDENT_NONE,
+    PM_HEREDOC_INDENT_DASH,
+    PM_HEREDOC_INDENT_TILDE,
+} pm_heredoc_indent_t;
+
+// When lexing Ruby source, the lexer has a small amount of state to tell which
+// kind of token it is currently lexing. For example, when we find the start of
+// a string, the first token that we return is a TOKEN_STRING_BEGIN token. After
+// that the lexer is now in the PM_LEX_STRING mode, and will return tokens that
+// are found as part of a string.
+typedef struct pm_lex_mode {
+    enum {
+        // This state is used when any given token is being lexed.
+        PM_LEX_DEFAULT,
+
+        // This state is used when we're lexing as normal but inside an embedded
+        // expression of a string.
+        PM_LEX_EMBEXPR,
+
+        // This state is used when we're lexing a variable that is embedded
+        // directly inside of a string with the # shorthand.
+        PM_LEX_EMBVAR,
+
+        // This state is used when you are inside the content of a heredoc.
+        PM_LEX_HEREDOC,
+
+        // This state is used when we are lexing a list of tokens, as in a %w
+        // word list literal or a %i symbol list literal.
+        PM_LEX_LIST,
+
+        // This state is used when a regular expression has been begun and we
+        // are looking for the terminator.
+        PM_LEX_REGEXP,
+
+        // This state is used when we are lexing a string or a string-like
+        // token, as in string content with either quote or an xstring.
+        PM_LEX_STRING
+    } mode;
+
+    union {
+        struct {
+            // This keeps track of the nesting level of the list.
+            size_t nesting;
+
+            // Whether or not interpolation is allowed in this list.
+            bool interpolation;
+
+            // When lexing a list, it takes into account balancing the
+            // terminator if the terminator is one of (), [], {}, or <>.
+            uint8_t incrementor;
+
+            // This is the terminator of the list literal.
+            uint8_t terminator;
+
+            // This is the character set that should be used to delimit the
+            // tokens within the list.
+            uint8_t breakpoints[11];
+        } list;
+
+        struct {
+            // This keeps track of the nesting level of the regular expression.
+            size_t nesting;
+
+            // When lexing a regular expression, it takes into account balancing
+            // the terminator if the terminator is one of (), [], {}, or <>.
+            uint8_t incrementor;
+
+            // This is the terminator of the regular expression.
+            uint8_t terminator;
+
+            // This is the character set that should be used to delimit the
+            // tokens within the regular expression.
+            uint8_t breakpoints[6];
+        } regexp;
+
+        struct {
+            // This keeps track of the nesting level of the string.
+            size_t nesting;
+
+            // Whether or not interpolation is allowed in this string.
+            bool interpolation;
+
+            // Whether or not at the end of the string we should allow a :,
+            // which would indicate this was a dynamic symbol instead of a
+            // string.
+            bool label_allowed;
+
+            // When lexing a string, it takes into account balancing the
+            // terminator if the terminator is one of (), [], {}, or <>.
+            uint8_t incrementor;
+
+            // This is the terminator of the string. It is typically either a
+            // single or double quote.
+            uint8_t terminator;
+
+            // This is the character set that should be used to delimit the
+            // tokens within the string.
+            uint8_t breakpoints[6];
+        } string;
+
+        struct {
+            // These pointers point to the beginning and end of the heredoc
+            // identifier.
+            const uint8_t *ident_start;
+            size_t ident_length;
+
+            pm_heredoc_quote_t quote;
+            pm_heredoc_indent_t indent;
+
+            // This is the pointer to the character where lexing should resume
+            // once the heredoc has been completely processed.
+            const uint8_t *next_start;
+        } heredoc;
+    } as;
+
+    // The previous lex state so that it knows how to pop.
+    struct pm_lex_mode *prev;
+} pm_lex_mode_t;
+
+// We pre-allocate a certain number of lex states in order to avoid having to
+// call malloc too many times while parsing. You really shouldn't need more than
+// this because you only really nest deeply when doing string interpolation.
+#define PM_LEX_STACK_SIZE 4
+
+// A forward declaration since our error handler struct accepts a parser for
+// each of its function calls.
+typedef struct pm_parser pm_parser_t;
+
+// While parsing, we keep track of a stack of contexts. This is helpful for
+// error recovery so that we can pop back to a previous context when we hit a
+// token that is understood by a parent context but not by the current context.
+typedef enum {
+    PM_CONTEXT_BEGIN,          // a begin statement
+    PM_CONTEXT_BLOCK_BRACES,   // expressions in block arguments using braces
+    PM_CONTEXT_BLOCK_KEYWORDS, // expressions in block arguments using do..end
+    PM_CONTEXT_CASE_WHEN,      // a case when statements
+    PM_CONTEXT_CASE_IN,        // a case in statements
+    PM_CONTEXT_CLASS,          // a class declaration
+    PM_CONTEXT_DEF,            // a method definition
+    PM_CONTEXT_DEF_PARAMS,     // a method definition's parameters
+    PM_CONTEXT_DEFAULT_PARAMS, // a method definition's default parameter
+    PM_CONTEXT_ELSE,           // an else clause
+    PM_CONTEXT_ELSIF,          // an elsif clause
+    PM_CONTEXT_EMBEXPR,        // an interpolated expression
+    PM_CONTEXT_ENSURE,         // an ensure statement
+    PM_CONTEXT_FOR,            // a for loop
+    PM_CONTEXT_IF,             // an if statement
+    PM_CONTEXT_LAMBDA_BRACES,  // a lambda expression with braces
+    PM_CONTEXT_LAMBDA_DO_END,  // a lambda expression with do..end
+    PM_CONTEXT_MAIN,           // the top level context
+    PM_CONTEXT_MODULE,         // a module declaration
+    PM_CONTEXT_PARENS,         // a parenthesized expression
+    PM_CONTEXT_POSTEXE,        // an END block
+    PM_CONTEXT_PREDICATE,      // a predicate inside an if/elsif/unless statement
+    PM_CONTEXT_PREEXE,         // a BEGIN block
+    PM_CONTEXT_RESCUE_ELSE,    // a rescue else statement
+    PM_CONTEXT_RESCUE,         // a rescue statement
+    PM_CONTEXT_SCLASS,         // a singleton class definition
+    PM_CONTEXT_UNLESS,         // an unless statement
+    PM_CONTEXT_UNTIL,          // an until statement
+    PM_CONTEXT_WHILE,          // a while statement
+} pm_context_t;
+
+// This is a node in a linked list of contexts.
+typedef struct pm_context_node {
+    pm_context_t context;
+    struct pm_context_node *prev;
+} pm_context_node_t;
+
+// This is the type of a comment that we've found while parsing.
+typedef enum {
+    PM_COMMENT_INLINE,
+    PM_COMMENT_EMBDOC,
+    PM_COMMENT___END__
+} pm_comment_type_t;
+
+// This is a node in the linked list of comments that we've found while parsing.
+typedef struct pm_comment {
+    pm_list_node_t node;
+    const uint8_t *start;
+    const uint8_t *end;
+    pm_comment_type_t type;
+} pm_comment_t;
+
+// When the encoding that is being used to parse the source is changed by prism,
+// we provide the ability here to call out to a user-defined function.
+typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
+
+// When an encoding is encountered that isn't understood by prism, we provide
+// the ability here to call out to a user-defined function to get an encoding
+// struct. If the function returns something that isn't NULL, we set that to
+// our encoding and use it to parse identifiers.
+typedef pm_encoding_t *(*pm_encoding_decode_callback_t)(pm_parser_t *parser, const uint8_t *name, size_t width);
+
+// When you are lexing through a file, the lexer needs all of the information
+// that the parser additionally provides (for example, the local table). So if
+// you want to properly lex Ruby, you need to actually lex it in the context of
+// the parser. In order to provide this functionality, we optionally allow a
+// struct to be attached to the parser that calls back out to a user-provided
+// callback when each token is lexed.
+typedef struct {
+    // This opaque pointer is used to provide whatever information the user
+    // deemed necessary to the callback. In our case we use it to pass the array
+    // that the tokens get appended into.
+    void *data;
+
+    // This is the callback that is called when a token is lexed. It is passed
+    // the opaque data pointer, the parser, and the token that was lexed.
+    void (*callback)(void *data, pm_parser_t *parser, pm_token_t *token);
+} pm_lex_callback_t;
+
+// This struct represents a node in a linked list of scopes. Some scopes can see
+// into their parent scopes, while others cannot.
+typedef struct pm_scope {
+    // The IDs of the locals in the given scope.
+    pm_constant_id_list_t locals;
+
+    // A pointer to the previous scope in the linked list.
+    struct pm_scope *previous;
+
+    // A boolean indicating whether or not this scope can see into its parent.
+    // If closed is true, then the scope cannot see into its parent.
+    bool closed;
+
+    // A boolean indicating whether or not this scope has explicit parameters.
+    // This is necessary to determine whether or not numbered parameters are
+    // allowed.
+    bool explicit_params;
+
+    // A boolean indicating whether or not this scope has numbered parameters.
+    // This is necessary to determine if child blocks are allowed to use
+    // numbered parameters.
+    bool numbered_params;
+} pm_scope_t;
+
+// This struct represents the overall parser. It contains a reference to the
+// source file, as well as pointers that indicate where in the source it's
+// currently parsing. It also contains the most recent and current token that
+// it's considering.
+struct pm_parser {
+    pm_lex_state_t lex_state; // the current state of the lexer
+    int enclosure_nesting;    // tracks the current nesting of (), [], and {}
+
+    // Used to temporarily track the nesting of enclosures to determine if a {
+    // is the beginning of a lambda following the parameters of a lambda.
+    int lambda_enclosure_nesting;
+
+    // Used to track the nesting of braces to ensure we get the correct value
+    // when we are interpolating blocks with braces.
+    int brace_nesting;
+
+    // the stack used to determine if a do keyword belongs to the predicate of a
+    // while, until, or for loop
+    pm_state_stack_t do_loop_stack;
+
+    // the stack used to determine if a do keyword belongs to the beginning of a
+    // block
+    pm_state_stack_t accepts_block_stack;
+
+    struct {
+        pm_lex_mode_t *current;                 // the current mode of the lexer
+        pm_lex_mode_t stack[PM_LEX_STACK_SIZE]; // the stack of lexer modes
+        size_t index;                           // the current index into the lexer mode stack
+    } lex_modes;
+
+    const uint8_t *start;   // the pointer to the start of the source
+    const uint8_t *end;     // the pointer to the end of the source
+    pm_token_t previous; // the previous token we were considering
+    pm_token_t current;  // the current token we're considering
+
+    // This is a special field set on the parser when we need the parser to jump
+    // to a specific location when lexing the next token, as opposed to just
+    // using the end of the previous token. Normally this is NULL.
+    const uint8_t *next_start;
+
+    // This field indicates the end of a heredoc whose identifier was found on
+    // the current line. If another heredoc is found on the same line, then this
+    // will be moved forward to the end of that heredoc. If no heredocs are
+    // found on a line then this is NULL.
+    const uint8_t *heredoc_end;
+
+    pm_list_t comment_list;             // the list of comments that have been found while parsing
+    pm_list_t warning_list;             // the list of warnings that have been found while parsing
+    pm_list_t error_list;               // the list of errors that have been found while parsing
+    pm_scope_t *current_scope;          // the current local scope
+
+    pm_context_node_t *current_context; // the current parsing context
+
+    // The encoding functions for the current file is attached to the parser as
+    // it's parsing so that it can change with a magic comment.
+    pm_encoding_t encoding;
+
+    // When the encoding that is being used to parse the source is changed by
+    // prism, we provide the ability here to call out to a user-defined
+    // function.
+    pm_encoding_changed_callback_t encoding_changed_callback;
+
+    // When an encoding is encountered that isn't understood by prism, we
+    // provide the ability here to call out to a user-defined function to get an
+    // encoding struct. If the function returns something that isn't NULL, we
+    // set that to our encoding and use it to parse identifiers.
+    pm_encoding_decode_callback_t encoding_decode_callback;
+
+    // This pointer indicates where a comment must start if it is to be
+    // considered an encoding comment.
+    const uint8_t *encoding_comment_start;
+
+    // This is an optional callback that can be attached to the parser that will
+    // be called whenever a new token is lexed by the parser.
+    pm_lex_callback_t *lex_callback;
+
+    // This is the path of the file being parsed
+    // We use the filepath when constructing SourceFileNodes
+    pm_string_t filepath_string;
+
+    // This constant pool keeps all of the constants defined throughout the file
+    // so that we can reference them later.
+    pm_constant_pool_t constant_pool;
+
+    // This is the list of newline offsets in the source file.
+    pm_newline_list_t newline_list;
+
+    // We want to add a flag to integer nodes that indicates their base. We only
+    // want to parse these once, but we don't have space on the token itself to
+    // communicate this information. So we store it here and pass it through
+    // when we find tokens that we need it for.
+    pm_node_flags_t integer_base;
+
+    // Whether or not we're at the beginning of a command
+    bool command_start;
+
+    // Whether or not we're currently recovering from a syntax error
+    bool recovering;
+
+    // Whether or not the encoding has been changed by a magic comment. We use
+    // this to provide a fast path for the lexer instead of going through the
+    // function pointer.
+    bool encoding_changed;
+
+    // This flag indicates that we are currently parsing a pattern matching
+    // expression and impacts that calculation of newlines.
+    bool pattern_matching_newlines;
+
+    // This flag indicates that we are currently parsing a keyword argument.
+    bool in_keyword_arg;
+
+    // Whether or not the parser has seen a token that has semantic meaning
+    // (i.e., a token that is not a comment or whitespace).
+    bool semantic_token_seen;
+
+    // Whether or not we have found a frozen_string_literal magic comment with
+    // a true value.
+    bool frozen_string_literal;
+};
+
+#endif // PRISM_PARSER_H

data/include/prism/regexp.h

@@ -0,0 +1,19 @@
+#ifndef PRISM_REGEXP_H
+#define PRISM_REGEXP_H
+
+#include "prism/defines.h"
+#include "prism/parser.h"
+#include "prism/enc/pm_encoding.h"
+#include "prism/util/pm_memchr.h"
+#include "prism/util/pm_string_list.h"
+#include "prism/util/pm_string.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+
+// Parse a regular expression and extract the names of all of the named capture
+// groups.
+PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding);
+
+#endif

data/include/prism/unescape.h

@@ -0,0 +1,48 @@
+#ifndef PRISM_UNESCAPE_H
+#define PRISM_UNESCAPE_H
+
+#include "prism/defines.h"
+#include "prism/diagnostic.h"
+#include "prism/parser.h"
+#include "prism/util/pm_char.h"
+#include "prism/util/pm_list.h"
+#include "prism/util/pm_memchr.h"
+#include "prism/util/pm_string.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+// The type of unescape we are performing.
+typedef enum {
+    // When we're creating a string inside of a list literal like %w, we
+    // shouldn't escape anything.
+    PM_UNESCAPE_NONE,
+
+    // When we're unescaping a single-quoted string, we only need to unescape
+    // single quotes and backslashes.
+    PM_UNESCAPE_MINIMAL,
+
+    // When we're unescaping a string list, in addition to MINIMAL, we need to
+    // unescape whitespace.
+    PM_UNESCAPE_WHITESPACE,
+
+    // When we're unescaping a double-quoted string, we need to unescape all
+    // escapes.
+    PM_UNESCAPE_ALL,
+} pm_unescape_type_t;
+
+// Unescape the contents of the given token into the given string using the given unescape mode.
+PRISM_EXPORTED_FUNCTION void pm_unescape_manipulate_string(pm_parser_t *parser, pm_string_t *string, pm_unescape_type_t unescape_type);
+void pm_unescape_manipulate_char_literal(pm_parser_t *parser, pm_string_t *string, pm_unescape_type_t unescape_type);
+
+// Accepts a source string and a type of unescaping and returns the unescaped version.
+// The caller must pm_string_free(result); after calling this function.
+PRISM_EXPORTED_FUNCTION bool pm_unescape_string(const uint8_t *start, size_t length, pm_unescape_type_t unescape_type, pm_string_t *result);
+
+// Returns the number of bytes that encompass the first escape sequence in the
+// given string.
+size_t pm_unescape_calculate_difference(pm_parser_t *parser, const uint8_t *value, pm_unescape_type_t unescape_type, bool expect_single_codepoint);
+
+#endif

data/include/prism/util/pm_buffer.h

@@ -0,0 +1,51 @@
+#ifndef PRISM_BUFFER_H
+#define PRISM_BUFFER_H
+
+#include "prism/defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+// A pm_buffer_t is a simple memory buffer that stores data in a contiguous
+// block of memory. It is used to store the serialized representation of a
+// prism tree.
+typedef struct {
+    char *value;
+    size_t length;
+    size_t capacity;
+} pm_buffer_t;
+
+// Return the size of the pm_buffer_t struct.
+PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
+
+// Initialize a pm_buffer_t with its default values.
+PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
+
+// Return the value of the buffer.
+PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
+
+// Return the length of the buffer.
+PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
+
+// Append the given amount of space as zeroes to the buffer.
+void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
+
+// Append a string to the buffer.
+void pm_buffer_append_str(pm_buffer_t *buffer, const char *value, size_t length);
+
+// Append a list of bytes to the buffer.
+void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
+
+// Append a single byte to the buffer.
+void pm_buffer_append_u8(pm_buffer_t *buffer, uint8_t value);
+
+// Append a 32-bit unsigned integer to the buffer.
+void pm_buffer_append_u32(pm_buffer_t *buffer, uint32_t value);
+
+// Free the memory associated with the buffer.
+PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);
+
+#endif

data/include/prism/util/pm_char.h

@@ -0,0 +1,91 @@
+#ifndef PRISM_CHAR_H
+#define PRISM_CHAR_H
+
+#include "prism/defines.h"
+#include "prism/util/pm_newline_list.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+// Returns the number of characters at the start of the string that are
+// whitespace. Disallows searching past the given maximum number of characters.
+size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
+
+// Returns the number of characters at the start of the string that are
+// whitespace while also tracking the location of each newline. Disallows
+// searching past the given maximum number of characters.
+size_t
+pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
+
+// Returns the number of characters at the start of the string that are inline
+// whitespace. Disallows searching past the given maximum number of characters.
+size_t pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
+
+// Returns the number of characters at the start of the string that are decimal
+// digits. Disallows searching past the given maximum number of characters.
+size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
+
+// Returns the number of characters at the start of the string that are
+// hexadecimal digits. Disallows searching past the given maximum number of
+// characters.
+size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
+
+// Returns the number of characters at the start of the string that are octal
+// digits or underscores. Disallows searching past the given maximum number of
+// characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
+size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+// Returns the number of characters at the start of the string that are decimal
+// digits or underscores. Disallows searching past the given maximum number of
+// characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
+size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+// Returns the number of characters at the start of the string that are
+// hexadecimal digits or underscores. Disallows searching past the given maximum
+// number of characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
+size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+// Returns the number of characters at the start of the string that are regexp
+// options. Disallows searching past the given maximum number of characters.
+size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
+
+// Returns the number of characters at the start of the string that are binary
+// digits or underscores. Disallows searching past the given maximum number of
+// characters.
+//
+// If multiple underscores are found in a row or if an underscore is
+// found at the end of the number, then the invalid pointer is set to the index
+// of the first invalid underscore.
+size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
+
+// Returns true if the given character is a whitespace character.
+bool pm_char_is_whitespace(const uint8_t b);
+
+// Returns true if the given character is an inline whitespace character.
+bool pm_char_is_inline_whitespace(const uint8_t b);
+
+// Returns true if the given character is a binary digit.
+bool pm_char_is_binary_digit(const uint8_t b);
+
+// Returns true if the given character is an octal digit.
+bool pm_char_is_octal_digit(const uint8_t b);
+
+// Returns true if the given character is a decimal digit.
+bool pm_char_is_decimal_digit(const uint8_t b);
+
+// Returns true if the given character is a hexadecimal digit.
+bool pm_char_is_hexadecimal_digit(const uint8_t b);
+
+#endif