prism 0.16.0 → 0.17.0

data/src/regexp.c

@@ -1,16 +1,31 @@
 #include "prism/regexp.h"
 
-// This is the parser that is going to handle parsing regular expressions.
+/**
+ * This is the parser that is going to handle parsing regular expressions.
+ */
 typedef struct {
+    /** A pointer to the start of the source that we are parsing. */
     const uint8_t *start;
+
+    /** A pointer to the current position in the source. */
     const uint8_t *cursor;
+
+    /** A pointer to the end of the source that we are parsing. */
     const uint8_t *end;
+
+    /** A list of named captures that we've found. */
     pm_string_list_t *named_captures;
+
+    /** Whether the encoding has changed from the default. */
     bool encoding_changed;
+
+    /** The encoding of the source. */
     pm_encoding_t *encoding;
 } pm_regexp_parser_t;
 
-// This initializes a new parser with the given source.
+/**
+ * This initializes a new parser with the given source.
+ */
 static void
 pm_regexp_parser_init(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding) {
     *parser = (pm_regexp_parser_t) {
@@ -23,7 +38,9 @@ pm_regexp_parser_init(pm_regexp_parser_t *parser, const uint8_t *start, const ui
     };
 }
 
-// This appends a new string to the list of named captures.
+/**
+ * This appends a new string to the list of named captures.
+ */
 static void
 pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) {
     pm_string_t string;
@@ -32,13 +49,17 @@ pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start,
     pm_string_free(&string);
 }
 
-// Returns true if the next character is the end of the source.
+/**
+ * Returns true if the next character is the end of the source.
+ */
 static inline bool
 pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
     return parser->cursor >= parser->end;
 }
 
-// Optionally accept a char and consume it if it exists.
+/**
+ * Optionally accept a char and consume it if it exists.
+ */
 static inline bool
 pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
     if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
@@ -48,7 +69,9 @@ pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
     return false;
 }
 
-// Expect a character to be present and consume it.
+/**
+ * Expect a character to be present and consume it.
+ */
 static inline bool
 pm_regexp_char_expect(pm_regexp_parser_t *parser, uint8_t value) {
     if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
@@ -58,7 +81,9 @@ pm_regexp_char_expect(pm_regexp_parser_t *parser, uint8_t value) {
     return false;
 }
 
-// This advances the current token to the next instance of the given character.
+/**
+ * This advances the current token to the next instance of the given character.
+ */
 static bool
 pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
     if (pm_regexp_char_is_eof(parser)) {
@@ -74,37 +99,39 @@ pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
     return true;
 }
 
-// Range quantifiers are a special class of quantifiers that look like
-//
-// * {digit}
-// * {digit,}
-// * {digit,digit}
-// * {,digit}
-//
-// Unfortunately, if there are any spaces in between, then this just becomes a
-// regular character match expression and we have to backtrack. So when this
-// function first starts running, we'll create a "save" point and then attempt
-// to parse the quantifier. If it fails, we'll restore the save point and
-// return.
-//
-// The properly track everything, we're going to build a little state machine.
-// It looks something like the following:
-//
-//                  ┌───────┐                 ┌─────────┐ ────────────┐
-// ──── lbrace ───> │ start │ ──── digit ───> │ minimum │             │
-//                  └───────┘                 └─────────┘ <─── digit ─┘
-//                      │                       │    │
-//   ┌───────┐          │                       │  rbrace
-//   │ comma │ <───── comma  ┌──── comma ───────┘    │
-//   └───────┘               V                       V
-//      │             ┌─────────┐               ┌─────────┐
-//      └── digit ──> │ maximum │ ── rbrace ──> │| final |│
-//                    └─────────┘               └─────────┘
-//                    │         ^
-//                    └─ digit ─┘
-//
-// Note that by the time we've hit this function, the lbrace has already been
-// consumed so we're in the start state.
+/**
+ * Range quantifiers are a special class of quantifiers that look like
+ *
+ * * {digit}
+ * * {digit,}
+ * * {digit,digit}
+ * * {,digit}
+ *
+ * Unfortunately, if there are any spaces in between, then this just becomes a
+ * regular character match expression and we have to backtrack. So when this
+ * function first starts running, we'll create a "save" point and then attempt
+ * to parse the quantifier. If it fails, we'll restore the save point and
+ * return.
+ *
+ * The properly track everything, we're going to build a little state machine.
+ * It looks something like the following:
+ *
+ *                  ┌───────┐                 ┌─────────┐ ────────────┐
+ * ──── lbrace ───> │ start │ ──── digit ───> │ minimum │             │
+ *                  └───────┘                 └─────────┘ <─── digit ─┘
+ *                      │                       │    │
+ *   ┌───────┐          │                       │  rbrace
+ *   │ comma │ <───── comma  ┌──── comma ───────┘    │
+ *   └───────┘               V                       V
+ *      │             ┌─────────┐               ┌─────────┐
+ *      └── digit ──> │ maximum │ ── rbrace ──> │| final |│
+ *                    └─────────┘               └─────────┘
+ *                    │         ^
+ *                    └─ digit ─┘
+ *
+ * Note that by the time we've hit this function, the lbrace has already been
+ * consumed so we're in the start state.
+ */
 static bool
 pm_regexp_parse_range_quantifier(pm_regexp_parser_t *parser) {
     const uint8_t *savepoint = parser->cursor;
@@ -180,12 +207,14 @@ pm_regexp_parse_range_quantifier(pm_regexp_parser_t *parser) {
     return true;
 }
 
-// quantifier : star-quantifier
-//            | plus-quantifier
-//            | optional-quantifier
-//            | range-quantifier
-//            | <empty>
-//            ;
+/**
+ * quantifier : star-quantifier
+ *            | plus-quantifier
+ *            | optional-quantifier
+ *            | range-quantifier
+ *            | <empty>
+ *            ;
+ */
 static bool
 pm_regexp_parse_quantifier(pm_regexp_parser_t *parser) {
     if (pm_regexp_char_is_eof(parser)) return true;
@@ -205,8 +234,10 @@ pm_regexp_parse_quantifier(pm_regexp_parser_t *parser) {
     }
 }
 
-// match-posix-class : '[' '[' ':' '^'? CHAR+ ':' ']' ']'
-//                   ;
+/**
+ * match-posix-class : '[' '[' ':' '^'? CHAR+ ':' ']' ']'
+ *                   ;
+ */
 static bool
 pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) {
     if (!pm_regexp_char_expect(parser, ':')) {
@@ -226,8 +257,10 @@ pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) {
 static bool
 pm_regexp_parse_lbracket(pm_regexp_parser_t *parser);
 
-// match-char-set : '[' '^'? (match-range | match-char)* ']'
-//                ;
+/**
+ * match-char-set : '[' '^'? (match-range | match-char)* ']'
+ *                ;
+ */
 static bool
 pm_regexp_parse_character_set(pm_regexp_parser_t *parser) {
     pm_regexp_char_accept(parser, '^');
@@ -251,7 +284,9 @@ pm_regexp_parse_character_set(pm_regexp_parser_t *parser) {
     return pm_regexp_char_expect(parser, ']');
 }
 
-// A left bracket can either mean a POSIX class or a character set.
+/**
+ * A left bracket can either mean a POSIX class or a character set.
+ */
 static bool
 pm_regexp_parse_lbracket(pm_regexp_parser_t *parser) {
     const uint8_t *reset = parser->cursor;
@@ -271,8 +306,10 @@ pm_regexp_parse_lbracket(pm_regexp_parser_t *parser) {
 static bool
 pm_regexp_parse_expression(pm_regexp_parser_t *parser);
 
-// These are the states of the options that are configurable on the regular
-// expression (or from within a group).
+/**
+ * These are the states of the options that are configurable on the regular
+ * expression (or from within a group).
+ */
 typedef enum {
     PM_REGEXP_OPTION_STATE_INVALID,
     PM_REGEXP_OPTION_STATE_TOGGLEABLE,
@@ -283,16 +320,22 @@ typedef enum {
 
 // These are the options that are configurable on the regular expression (or
 // from within a group).
+
 #define PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM 'a'
 #define PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM 'x'
 #define PRISM_REGEXP_OPTION_STATE_SLOTS (PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM + 1)
 
-// This is the set of options that are configurable on the regular expression.
+/**
+ * This is the set of options that are configurable on the regular expression.
+ */
 typedef struct {
+    /** The current state of each option. */
     uint8_t values[PRISM_REGEXP_OPTION_STATE_SLOTS];
 } pm_regexp_options_t;
 
-// Initialize a new set of options to their default values.
+/**
+ * Initialize a new set of options to their default values.
+ */
 static void
 pm_regexp_options_init(pm_regexp_options_t *options) {
     memset(options, PM_REGEXP_OPTION_STATE_INVALID, sizeof(uint8_t) * PRISM_REGEXP_OPTION_STATE_SLOTS);
@@ -304,8 +347,10 @@ pm_regexp_options_init(pm_regexp_options_t *options) {
     options->values['u' - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM] = PM_REGEXP_OPTION_STATE_ADDABLE;
 }
 
-// Attempt to add the given option to the set of options. Returns true if it was
-// added, false if it was already present.
+/**
+ * Attempt to add the given option to the set of options. Returns true if it was
+ * added, false if it was already present.
+ */
 static bool
 pm_regexp_options_add(pm_regexp_options_t *options, uint8_t key) {
     if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
@@ -327,8 +372,10 @@ pm_regexp_options_add(pm_regexp_options_t *options, uint8_t key) {
     return false;
 }
 
-// Attempt to remove the given option from the set of options. Returns true if
-// it was removed, false if it was already absent.
+/**
+ * Attempt to remove the given option from the set of options. Returns true if
+ * it was removed, false if it was already absent.
+ */
 static bool
 pm_regexp_options_remove(pm_regexp_options_t *options, uint8_t key) {
     if (key >= PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM) {
@@ -349,26 +396,27 @@ pm_regexp_options_remove(pm_regexp_options_t *options, uint8_t key) {
     return false;
 }
 
-// Groups can have quite a few different patterns for syntax. They basically
-// just wrap a set of expressions, but they can potentially have options after a
-// question mark. If there _isn't_ a question mark, then it's just a set of
-// expressions. If there _is_, then here are the options:
-//
-// * (?#...)                       - inline comments
-// * (?:subexp)                    - non-capturing group
-// * (?=subexp)                    - positive lookahead
-// * (?!subexp)                    - negative lookahead
-// * (?>subexp)                    - atomic group
-// * (?~subexp)                    - absence operator
-// * (?<=subexp)                   - positive lookbehind
-// * (?<!subexp)                   - negative lookbehind
-// * (?<name>subexp)               - named capturing group
-// * (?'name'subexp)               - named capturing group
-// * (?(cond)yes-subexp)           - conditional expression
-// * (?(cond)yes-subexp|no-subexp) - conditional expression
-// * (?imxdau-imx)                 - turn on and off configuration
-// * (?imxdau-imx:subexp)          - turn on and off configuration for an expression
-//
+/**
+ * Groups can have quite a few different patterns for syntax. They basically
+ * just wrap a set of expressions, but they can potentially have options after a
+ * question mark. If there _isn't_ a question mark, then it's just a set of
+ * expressions. If there _is_, then here are the options:
+ *
+ * * (?#...)                       - inline comments
+ * * (?:subexp)                    - non-capturing group
+ * * (?=subexp)                    - positive lookahead
+ * * (?!subexp)                    - negative lookahead
+ * * (?>subexp)                    - atomic group
+ * * (?~subexp)                    - absence operator
+ * * (?<=subexp)                   - positive lookbehind
+ * * (?<!subexp)                   - negative lookbehind
+ * * (?<name>subexp)               - named capturing group
+ * * (?'name'subexp)               - named capturing group
+ * * (?(cond)yes-subexp)           - conditional expression
+ * * (?(cond)yes-subexp|no-subexp) - conditional expression
+ * * (?imxdau-imx)                 - turn on and off configuration
+ * * (?imxdau-imx:subexp)          - turn on and off configuration for an expression
+ */
 static bool
 pm_regexp_parse_group(pm_regexp_parser_t *parser) {
     // First, parse any options for the group.
@@ -503,16 +551,18 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser) {
     return pm_regexp_char_expect(parser, ')');
 }
 
-// item : anchor
-//      | match-posix-class
-//      | match-char-set
-//      | match-char-class
-//      | match-char-prop
-//      | match-char
-//      | match-any
-//      | group
-//      | quantified
-//      ;
+/**
+ * item : anchor
+ *      | match-posix-class
+ *      | match-char-set
+ *      | match-char-class
+ *      | match-char-prop
+ *      | match-char
+ *      | match-any
+ *      | group
+ *      | quantified
+ *      ;
+ */
 static bool
 pm_regexp_parse_item(pm_regexp_parser_t *parser) {
     switch (*parser->cursor++) {
@@ -533,8 +583,10 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser) {
     }
 }
 
-// expression : item+
-//            ;
+/**
+ * expression : item+
+ *            ;
+ */
 static bool
 pm_regexp_parse_expression(pm_regexp_parser_t *parser) {
     if (!pm_regexp_parse_item(parser)) {
@@ -550,10 +602,12 @@ pm_regexp_parse_expression(pm_regexp_parser_t *parser) {
     return true;
 }
 
-// pattern : EOF
-//         | expression EOF
-//         | expression '|' pattern
-//         ;
+/**
+ * pattern : EOF
+ *         | expression EOF
+ *         | expression '|' pattern
+ *         ;
+ */
 static bool
 pm_regexp_parse_pattern(pm_regexp_parser_t *parser) {
     return (
@@ -572,8 +626,10 @@ pm_regexp_parse_pattern(pm_regexp_parser_t *parser) {
     );
 }
 
-// Parse a regular expression and extract the names of all of the named capture
-// groups.
+/**
+ * Parse a regular expression and extract the names of all of the named capture
+ * groups.
+ */
 PRISM_EXPORTED_FUNCTION bool
 pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding) {
     pm_regexp_parser_t parser;

data/src/serialize.c

@@ -54,7 +54,7 @@ pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffe
     }
 }
 
-void
+static void
 pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
     pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node));
 
@@ -1131,16 +1131,6 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             }
             break;
         }
-        case PM_KEYWORD_PARAMETER_NODE: {
-            pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_keyword_parameter_node_t *)node)->name));
-            pm_serialize_location(parser, &((pm_keyword_parameter_node_t *)node)->name_loc, buffer);
-            if (((pm_keyword_parameter_node_t *)node)->value == NULL) {
-                pm_buffer_append_byte(buffer, 0);
-            } else {
-                pm_serialize_node(parser, (pm_node_t *)((pm_keyword_parameter_node_t *)node)->value, buffer);
-            }
-            break;
-        }
         case PM_KEYWORD_REST_PARAMETER_NODE: {
             pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_keyword_rest_parameter_node_t *)node)->name));
             if (((pm_keyword_rest_parameter_node_t *)node)->name_loc.start == NULL) {
@@ -1348,6 +1338,12 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_buffer_append_varint(buffer, ((pm_numbered_reference_read_node_t *)node)->number);
             break;
         }
+        case PM_OPTIONAL_KEYWORD_PARAMETER_NODE: {
+            pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_optional_keyword_parameter_node_t *)node)->name));
+            pm_serialize_location(parser, &((pm_optional_keyword_parameter_node_t *)node)->name_loc, buffer);
+            pm_serialize_node(parser, (pm_node_t *)((pm_optional_keyword_parameter_node_t *)node)->value, buffer);
+            break;
+        }
         case PM_OPTIONAL_PARAMETER_NODE: {
             pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_optional_parameter_node_t *)node)->name));
             pm_serialize_location(parser, &((pm_optional_parameter_node_t *)node)->name_loc, buffer);
@@ -1482,6 +1478,11 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
             pm_buffer_append_varint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
             break;
         }
+        case PM_REQUIRED_KEYWORD_PARAMETER_NODE: {
+            pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_required_keyword_parameter_node_t *)node)->name));
+            pm_serialize_location(parser, &((pm_required_keyword_parameter_node_t *)node)->name_loc, buffer);
+            break;
+        }
         case PM_REQUIRED_PARAMETER_NODE: {
             pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_required_parameter_node_t *)node)->name));
             break;
@@ -1785,6 +1786,9 @@ pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *bu
     pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(comment->end - comment->start));
 }
 
+/**
+ * Serialize the given list of comments to the given buffer.
+ */
 void
 pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
     pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_list_size(list)));
@@ -1838,6 +1842,9 @@ pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *
     }
 }
 
+/**
+ * Serialize the name of the encoding to the buffer.
+ */
 void
 pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer) {
     size_t encoding_length = strlen(encoding->name);
@@ -1845,10 +1852,14 @@ pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer) {
     pm_buffer_append_string(buffer, encoding->name, encoding_length);
 }
 
-#line 200 "serialize.c.erb"
+#line 206 "serialize.c.erb"
+/**
+ * Serialize the encoding, metadata, nodes, and constant pool.
+ */
 void
 pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
     pm_serialize_encoding(&parser->encoding, buffer);
+    pm_buffer_append_varint(buffer, parser->start_line);
     pm_serialize_comment_list(parser, &parser->comment_list, buffer);
     pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer);
     pm_serialize_diagnostic_list(parser, &parser->error_list, buffer);
@@ -1921,10 +1932,16 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
     pm_buffer_append_varint(buffer, parser->lex_state);
 }
 
+/**
+ * Lex the given source and serialize to the given buffer.
+ */
 PRISM_EXPORTED_FUNCTION void
-pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer) {
+pm_serialize_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, filepath);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
         .data = (void *) buffer,
@@ -1934,10 +1951,11 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
     parser.lex_callback = &lex_callback;
     pm_node_t *node = pm_parse(&parser);
 
-    // Append 0 to mark end of tokens
+    // Append 0 to mark end of tokens.
     pm_buffer_append_byte(buffer, 0);
 
     pm_serialize_encoding(&parser.encoding, buffer);
+    pm_buffer_append_varint(buffer, parser.start_line);
     pm_serialize_comment_list(&parser, &parser.comment_list, buffer);
     pm_serialize_magic_comment_list(&parser, &parser.magic_comment_list, buffer);
     pm_serialize_diagnostic_list(&parser, &parser.error_list, buffer);
@@ -1945,15 +1963,20 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }
 
-// Parse and serialize both the AST and the tokens represented by the given
-// source to the given buffer.
+/**
+ * Parse and serialize both the AST and the tokens represented by the given
+ * source to the given buffer.
+ */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
         .data = (void *) buffer,
@@ -1968,4 +1991,5 @@ pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer,
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }

data/src/token_type.c

@@ -9,7 +9,9 @@
 
 #include "prism/ast.h"
 
-// Returns a string representation of the given token type.
+/**
+ * Returns a string representation of the given token type.
+ */
 PRISM_EXPORTED_FUNCTION const char *
 pm_token_type_to_str(pm_token_type_t token_type)
 {