prism 0.16.0 → 0.17.0

data/include/prism/defines.h

@@ -1,8 +1,14 @@
+/**
+ * @file defines.h
+ *
+ * Macro definitions used throughout the prism library.
+ *
+ * This file should be included first by any *.h or *.c in prism for consistency
+ * and to ensure that the macros are defined before they are used.
+ */
 #ifndef PRISM_DEFINES_H
 #define PRISM_DEFINES_H
 
-// This file should be included first by any *.h or *.c in prism
-
 #include <ctype.h>
 #include <stdarg.h>
 #include <stddef.h>
@@ -10,7 +16,11 @@
 #include <stdio.h>
 #include <string.h>
 
-// PRISM_EXPORTED_FUNCTION
+/**
+ * By default, we compile with -fvisibility=hidden. When this is enabled, we
+ * need to mark certain functions as being publically-visible. This macro does
+ * that in a compiler-agnostic way.
+ */
 #ifndef PRISM_EXPORTED_FUNCTION
 #   ifdef PRISM_EXPORT_SYMBOLS
 #       ifdef _WIN32
@@ -23,7 +33,12 @@
 #   endif
 #endif
 
-// PRISM_ATTRIBUTE_FORMAT
+/**
+ * Certain compilers support specifying that a function accepts variadic
+ * parameters that look like printf format strings to provide a better developer
+ * experience when someone is using the function. This macro does that in a
+ * compiler-agnostic way.
+ */
 #if defined(__GNUC__)
 #   define PRISM_ATTRIBUTE_FORMAT(string_index, argument_index) __attribute__((format(printf, string_index, argument_index)))
 #elif defined(__clang__)
@@ -32,23 +47,31 @@
 #   define PRISM_ATTRIBUTE_FORMAT(string_index, argument_index)
 #endif
 
-// PRISM_ATTRIBUTE_UNUSED
+/**
+ * GCC will warn if you specify a function or parameter that is unused at
+ * runtime. This macro allows you to mark a function or parameter as unused in a
+ * compiler-agnostic way.
+ */
 #if defined(__GNUC__)
 #   define PRISM_ATTRIBUTE_UNUSED __attribute__((unused))
 #else
 #   define PRISM_ATTRIBUTE_UNUSED
 #endif
 
-// inline
+/**
+ * Old Visual Studio versions do not support the inline keyword, so we need to
+ * define it to be __inline.
+ */
 #if defined(_MSC_VER) && !defined(inline)
 #   define inline __inline
 #endif
 
-// Windows versions before 2015 use _snprintf
+/**
+ * Old Visual Studio versions before 2015 do not implement sprintf, but instead
+ * implement _snprintf. We standard that here.
+ */
 #if !defined(snprintf) && defined(_MSC_VER) && (_MSC_VER < 1900)
 #   define snprintf _snprintf
 #endif
 
-int pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length);
-
 #endif

data/include/prism/diagnostic.h

@@ -1,3 +1,8 @@
+/**
+ * @file diagnostic.h
+ *
+ * A list of diagnostics generated during parsing.
+ */
 #ifndef PRISM_DIAGNOSTIC_H
 #define PRISM_DIAGNOSTIC_H
 
@@ -8,14 +13,29 @@
 #include <stdlib.h>
 #include <assert.h>
 
-// This struct represents a diagnostic found during parsing.
+/**
+ * This struct represents a diagnostic generated during parsing.
+ *
+ * @extends pm_list_node_t
+ */
 typedef struct {
+    /** The embedded base node. */
     pm_list_node_t node;
+
+    /** A pointer to the start of the source that generated the diagnostic. */
     const uint8_t *start;
+
+    /** A pointer to the end of the source that generated the diagnostic. */
     const uint8_t *end;
+
+    /** The message associated with the diagnostic. */
     const char *message;
 } pm_diagnostic_t;
 
+/**
+ * The diagnostic IDs of all of the diagnostics, used to communicate the types
+ * of errors between the parser and the user.
+ */
 typedef enum {
     PM_ERR_ALIAS_ARGUMENT,
     PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
@@ -219,14 +239,27 @@ typedef enum {
     PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
     PM_WARN_AMBIGUOUS_PREFIX_STAR,
     PM_WARN_AMBIGUOUS_SLASH,
+
     /* This must be the last member. */
     PM_DIAGNOSTIC_ID_LEN,
 } pm_diagnostic_id_t;
 
-// Append a diagnostic to the given list of diagnostics.
+/**
+ * Append a diagnostic to the given list of diagnostics.
+ *
+ * @param list The list to append to.
+ * @param start The start of the diagnostic.
+ * @param end The end of the diagnostic.
+ * @param diag_id The diagnostic ID.
+ * @return Whether the diagnostic was successfully appended.
+ */
 bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id);
 
-// Deallocate the internal state of the given diagnostic list.
+/**
+ * Deallocate the internal state of the given diagnostic list.
+ *
+ * @param list The list to deallocate.
+ */
 void pm_diagnostic_list_free(pm_list_t *list);
 
 #endif

data/include/prism/enc/pm_encoding.h

@@ -1,3 +1,8 @@
+/**
+ * @file pm_encoding.h
+ *
+ * The encoding interface and implementations used by the parser.
+ */
 #ifndef PRISM_ENCODING_H
 #define PRISM_ENCODING_H
 
@@ -8,63 +13,148 @@
 #include <stddef.h>
 #include <stdint.h>
 
-// This struct defines the functions necessary to implement the encoding
-// interface so we can determine how many bytes the subsequent character takes.
-// Each callback should return the number of bytes, or 0 if the next bytes are
-// invalid for the encoding and type.
+/**
+ * This struct defines the functions necessary to implement the encoding
+ * interface so we can determine how many bytes the subsequent character takes.
+ * Each callback should return the number of bytes, or 0 if the next bytes are
+ * invalid for the encoding and type.
+ */
 typedef struct {
-    // Return the number of bytes that the next character takes if it is valid
-    // in the encoding. Does not read more than n bytes. It is assumed that n is
-    // at least 1.
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding. Does not read more than n bytes. It is assumed that n is
+     * at least 1.
+     */
     size_t (*char_width)(const uint8_t *b, ptrdiff_t n);
 
-    // Return the number of bytes that the next character takes if it is valid
-    // in the encoding and is alphabetical. Does not read more than n bytes. It
-    // is assumed that n is at least 1.
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphabetical. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
     size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n);
 
-    // Return the number of bytes that the next character takes if it is valid
-    // in the encoding and is alphanumeric. Does not read more than n bytes. It
-    // is assumed that n is at least 1.
+    /**
+     * Return the number of bytes that the next character takes if it is valid
+     * in the encoding and is alphanumeric. Does not read more than n bytes. It
+     * is assumed that n is at least 1.
+     */
     size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n);
 
-    // Return true if the next character is valid in the encoding and is an
-    // uppercase character. Does not read more than n bytes. It is assumed that
-    // n is at least 1.
+    /**
+     * Return true if the next character is valid in the encoding and is an
+     * uppercase character. Does not read more than n bytes. It is assumed that
+     * n is at least 1.
+     */
     bool (*isupper_char)(const uint8_t *b, ptrdiff_t n);
 
-    // The name of the encoding. This should correspond to a value that can be
-    // passed to Encoding.find in Ruby.
+    /**
+     * The name of the encoding. This should correspond to a value that can be
+     * passed to Encoding.find in Ruby.
+     */
     const char *name;
 
-    // Return true if the encoding is a multibyte encoding.
+    /**
+     * Return true if the encoding is a multibyte encoding.
+     */
     bool multibyte;
 } pm_encoding_t;
 
-// These bits define the location of each bit of metadata within the various
-// lookup tables that are used to determine the properties of a character.
+/**
+ * All of the lookup tables use the first bit of each embedded byte to indicate
+ * whether the codepoint is alphabetical.
+ */
 #define PRISM_ENCODING_ALPHABETIC_BIT 1 << 0
+
+/**
+ * All of the lookup tables use the second bit of each embedded byte to indicate
+ * whether the codepoint is alphanumeric.
+ */
 #define PRISM_ENCODING_ALPHANUMERIC_BIT 1 << 1
+
+/**
+ * All of the lookup tables use the third bit of each embedded byte to indicate
+ * whether the codepoint is uppercase.
+ */
 #define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
 
-// These functions are reused by some other encodings, so they are defined here
-// so they can be shared.
+/**
+ * Return the size of the next character in the ASCII encoding if it is an
+ * alphabetical character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
 size_t pm_encoding_ascii_alpha_char(const uint8_t *b, PRISM_ATTRIBUTE_UNUSED ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the ASCII encoding if it is an
+ * alphanumeric character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
 size_t pm_encoding_ascii_alnum_char(const uint8_t *b, PRISM_ATTRIBUTE_UNUSED ptrdiff_t n);
+
+/**
+ * Return true if the next character in the ASCII encoding if it is an uppercase
+ * character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns True if the next character is valid in the encoding and is an
+ *     uppercase character, or false if it is not.
+ */
 bool pm_encoding_ascii_isupper_char(const uint8_t *b, PRISM_ATTRIBUTE_UNUSED ptrdiff_t n);
 
-// These functions are shared between the actual encoding and the fast path in
-// the parser so they need to be internally visible.
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphabetical character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
 size_t pm_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return the size of the next character in the UTF-8 encoding if it is an
+ * alphanumeric character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns The number of bytes that the next character takes if it is valid in
+ *     the encoding, or 0 if it is not.
+ */
 size_t pm_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
+
+/**
+ * Return true if the next character in the UTF-8 encoding if it is an uppercase
+ * character.
+ *
+ * @param b The bytes to read.
+ * @param n The number of bytes that can be read.
+ * @returns True if the next character is valid in the encoding and is an
+ *     uppercase character, or false if it is not.
+ */
 bool pm_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n);
 
-// This lookup table is referenced in both the UTF-8 encoding file and the
-// parser directly in order to speed up the default encoding processing.
+/**
+ * This lookup table is referenced in both the UTF-8 encoding file and the
+ * parser directly in order to speed up the default encoding processing. It is
+ * used to indicate whether a character is alphabetical, alphanumeric, or
+ * uppercase in unicode mappings.
+ */
 extern const uint8_t pm_encoding_unicode_table[256];
 
-// These are the encodings that are supported by the parser. They are defined in
+// Below are the encodings that are supported by the parser. They are defined in
 // their own files in the src/enc directory.
+
 extern pm_encoding_t pm_encoding_ascii;
 extern pm_encoding_t pm_encoding_ascii_8bit;
 extern pm_encoding_t pm_encoding_big5;

data/include/prism/node.h

@@ -1,32 +1,57 @@
+/**
+ * @file node.h
+ *
+ * Functions related to nodes in the AST.
+ */
 #ifndef PRISM_NODE_H
 #define PRISM_NODE_H
 
 #include "prism/defines.h"
 #include "prism/parser.h"
 
-// Append a new node onto the end of the node list.
+/**
+ * Append a new node onto the end of the node list.
+ *
+ * @param list The list to append to.
+ * @param node The node to append.
+ */
 void pm_node_list_append(pm_node_list_t *list, pm_node_t *node);
 
-// Clear the node but preserves the location.
-void pm_node_clear(pm_node_t *node);
-
-// Deallocate a node and all of its children.
+/**
+ * Deallocate a node and all of its children.
+ *
+ * @param parser The parser that owns the node.
+ * @param node The node to deallocate.
+ */
 PRISM_EXPORTED_FUNCTION void pm_node_destroy(pm_parser_t *parser, struct pm_node *node);
 
-// This struct stores the information gathered by the pm_node_memsize function.
-// It contains both the memory footprint and additionally metadata about the
-// shape of the tree.
+/**
+ * This struct stores the information gathered by the pm_node_memsize function.
+ * It contains both the memory footprint and additionally metadata about the
+ * shape of the tree.
+ */
 typedef struct {
+    /** The total memory footprint of the node and all of its children. */
     size_t memsize;
+
+    /** The number of children the node has. */
     size_t node_count;
 } pm_memsize_t;
 
-// Calculates the memory footprint of a given node.
+/**
+ * Calculates the memory footprint of a given node.
+ *
+ * @param node The node to calculate the memory footprint of.
+ * @param memsize The memory footprint of the node and all of its children.
+ */
 PRISM_EXPORTED_FUNCTION void pm_node_memsize(pm_node_t *node, pm_memsize_t *memsize);
 
-// Returns a string representation of the given node type.
+/**
+ * Returns a string representation of the given node type.
+ *
+ * @param node_type The node type to convert to a string.
+ * @return A string representation of the given node type.
+ */
 PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_type);
 
-#define PM_EMPTY_NODE_LIST ((pm_node_list_t) { .nodes = NULL, .size = 0, .capacity = 0 })
-
-#endif // PRISM_NODE_H
+#endif

data/include/prism/options.h

@@ -0,0 +1,204 @@
+/**
+ * @file options.h
+ *
+ * The options that can be passed to parsing.
+ */
+#ifndef PRISM_OPTIONS_H
+#define PRISM_OPTIONS_H
+
+#include "prism/defines.h"
+#include "prism/util/pm_string.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * A scope of locals surrounding the code that is being parsed.
+ */
+typedef struct pm_options_scope {
+    /** The number of locals in the scope. */
+    size_t locals_count;
+
+    /** The names of the locals in the scope. */
+    pm_string_t *locals;
+} pm_options_scope_t;
+
+/**
+ * The options that can be passed to the parser.
+ */
+typedef struct {
+    /** The name of the file that is currently being parsed. */
+    pm_string_t filepath;
+
+    /**
+     * The line within the file that the parse starts on. This value is
+     * 0-indexed.
+     */
+    uint32_t line;
+
+    /**
+     * The name of the encoding that the source file is in. Note that this must
+     * correspond to a name that can be found with Encoding.find in Ruby.
+     */
+    pm_string_t encoding;
+
+    /**
+     * The number of scopes surrounding the code that is being parsed.
+     */
+    size_t scopes_count;
+
+    /**
+     * The scopes surrounding the code that is being parsed. For most parses
+     * this will be NULL, but for evals it will be the locals that are in scope
+     * surrounding the eval.
+     */
+    pm_options_scope_t *scopes;
+
+    /** Whether or not the frozen string literal option has been set. */
+    bool frozen_string_literal;
+
+    /**
+     * Whether or not we should suppress warnings. This is purposefully negated
+     * so that the default is to not suppress warnings, which allows us to still
+     * create an options struct with zeroed memory.
+     */
+    bool suppress_warnings;
+} pm_options_t;
+
+/**
+ * Set the filepath option on the given options struct.
+ *
+ * @param options The options struct to set the filepath on.
+ * @param filepath The filepath to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath);
+
+/**
+ * Set the line option on the given options struct.
+ *
+ * @param options The options struct to set the line on.
+ * @param line The line to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line);
+
+/**
+ * Set the encoding option on the given options struct.
+ *
+ * @param options The options struct to set the encoding on.
+ * @param encoding The encoding to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
+
+/**
+ * Set the frozen string literal option on the given options struct.
+ *
+ * @param options The options struct to set the frozen string literal value on.
+ * @param frozen_string_literal The frozen string literal value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
+
+/**
+ * Set the suppress warnings option on the given options struct.
+ *
+ * @param options The options struct to set the suppress warnings value on.
+ * @param suppress_warnings The suppress warnings value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
+
+/**
+ * Allocate and zero out the scopes array on the given options struct.
+ *
+ * @param options The options struct to initialize the scopes array on.
+ * @param scopes_count The number of scopes to allocate.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
+
+/**
+ * Return a pointer to the scope at the given index within the given options.
+ *
+ * @param options The options struct to get the scope from.
+ * @param index The index of the scope to get.
+ * @return A pointer to the scope at the given index.
+ */
+PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index);
+
+/**
+ * Create a new options scope struct. This will hold a set of locals that are in
+ * scope surrounding the code that is being parsed.
+ *
+ * @param scope The scope struct to initialize.
+ * @param locals_count The number of locals to allocate.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
+
+/**
+ * Return a pointer to the local at the given index within the given scope.
+ *
+ * @param scope The scope struct to get the local from.
+ * @param index The index of the local to get.
+ * @return A pointer to the local at the given index.
+ */
+PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
+
+/**
+ * Free the internal memory associated with the options.
+ *
+ * @param options The options struct whose internal memory should be freed.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ *
+ * `data` is assumed to be a valid pointer pointing to well-formed data. The
+ * layout of this data should be the same every time, and is described below:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the length of the filepath |
+ * | ...     | the filepath bytes         |
+ * | `4`     | the line number            |
+ * | `4`     | the length the encoding    |
+ * | ...     | the encoding bytes         |
+ * | `1`     | frozen string literal      |
+ * | `1`     | suppress warnings          |
+ * | `4`     | the number of scopes       |
+ * | ...     | the scopes                 |
+ *
+ * Each scope is layed out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the number of locals       |
+ * | ...     | the locals                 |
+ *
+ * Each local is layed out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | `4`     | the length of the local    |
+ * | ...     | the local bytes            |
+ *
+ * Some additional things to note about this layout:
+ *
+ * * The filepath can have a length of 0, in which case we'll consider it an
+ *   empty string.
+ * * The line number should be 0-indexed.
+ * * The encoding can have a length of 0, in which case we'll use the default
+ *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
+ *   encoding that can be passed to `Encoding.find` in Ruby.
+ * * The frozen string literal and suppress warnings fields are booleans, so
+ *   their values should be either 0 or 1.
+ * * The number of scopes can be 0.
+ *
+ * @param options The options struct to deserialize into.
+ * @param data The binary string to deserialize from.
+ */
+void pm_options_read(pm_options_t *options, const char *data);
+
+#endif