prism 1.9.0 → 2.0.0.rc1

data/src/options.c

@@ -1,18 +1,78 @@
-#include "prism/options.h"
+#include "prism/internal/options.h"
+
+#include "prism/compiler/inline.h"
+
+#include "prism/internal/allocator.h"
+#include "prism/internal/char.h"
+#include "prism/internal/stringy.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Allocate a new options struct. If the options struct cannot be allocated,
+ * this function aborts the process.
+ */
+pm_options_t *
+pm_options_new(void) {
+    pm_options_t *options = xcalloc(1, sizeof(pm_options_t));
+    if (options == NULL) abort();
+    return options;
+}
+
+/**
+ * Free the internal memory associated with the options.
+ */
+void
+pm_options_cleanup(pm_options_t *options) {
+    pm_string_cleanup(&options->filepath);
+    pm_string_cleanup(&options->encoding);
+
+    for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
+        pm_options_scope_t *scope = &options->scopes[scope_index];
+
+        for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
+            pm_string_cleanup(&scope->locals[local_index]);
+        }
+
+        xfree_sized(scope->locals, scope->locals_count * sizeof(pm_string_t));
+    }
+
+    xfree_sized(options->scopes, options->scopes_count * sizeof(pm_options_scope_t));
+}
+
+/**
+ * Free both the held memory of the given options struct and the struct itself.
+ *
+ * @param options The options struct to free.
+ */
+void
+pm_options_free(pm_options_t *options) {
+    pm_options_cleanup(options);
+    xfree_sized(options, sizeof(pm_options_t));
+}
 
 /**
  * Set the shebang callback option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_shebang_callback_set(pm_options_t *options, pm_options_shebang_callback_t shebang_callback, void *shebang_callback_data) {
     options->shebang_callback = shebang_callback;
     options->shebang_callback_data = shebang_callback_data;
 }
 
+/**
+ * Get the filepath option on the given options struct.
+ */
+const pm_string_t *
+pm_options_filepath(const pm_options_t *options) {
+    return &options->filepath;
+}
+
 /**
  * Set the filepath option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_filepath_set(pm_options_t *options, const char *filepath) {
     pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
 }
@@ -20,7 +80,7 @@ pm_options_filepath_set(pm_options_t *options, const char *filepath) {
 /**
  * Set the encoding option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_encoding_set(pm_options_t *options, const char *encoding) {
     pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
 }
@@ -28,7 +88,7 @@ pm_options_encoding_set(pm_options_t *options, const char *encoding) {
 /**
  * Set the encoding_locked option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked) {
     options->encoding_locked = encoding_locked;
 }
@@ -36,7 +96,7 @@ pm_options_encoding_locked_set(pm_options_t *options, bool encoding_locked) {
 /**
  * Set the line option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_line_set(pm_options_t *options, int32_t line) {
     options->line = line;
 }
@@ -44,7 +104,7 @@ pm_options_line_set(pm_options_t *options, int32_t line) {
 /**
  * Set the frozen string literal option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal) {
     options->frozen_string_literal = frozen_string_literal ? PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED : PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED;
 }
@@ -52,7 +112,7 @@ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_l
 /**
  * Sets the command line option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_command_line_set(pm_options_t *options, uint8_t command_line) {
     options->command_line = command_line;
 }
@@ -60,7 +120,7 @@ pm_options_command_line_set(pm_options_t *options, uint8_t command_line) {
 /**
  * Checks if the given slice represents a number.
  */
-static inline bool
+static PRISM_INLINE bool
 is_number(const char *string, size_t length) {
     return pm_strspn_decimal_digit((const uint8_t *) string, (ptrdiff_t) length) == length;
 }
@@ -70,7 +130,7 @@ is_number(const char *string, size_t length) {
  * string. If the string contains an invalid option, this returns false.
  * Otherwise, it returns true.
  */
-PRISM_EXPORTED_FUNCTION bool
+bool
 pm_options_version_set(pm_options_t *options, const char *version, size_t length) {
     if (version == NULL) {
         options->version = PM_OPTIONS_VERSION_LATEST;
@@ -123,8 +183,8 @@ pm_options_version_set(pm_options_t *options, const char *version, size_t length
         }
     }
 
-    if (length >= 6) {
-        if (strncmp(version, "latest", 7) == 0) { // 7 to compare the \0 as well
+    if (length == 6) {
+        if (strncmp(version, "latest", 6) == 0) {
             options->version = PM_OPTIONS_VERSION_LATEST;
             return true;
         }
@@ -133,10 +193,28 @@ pm_options_version_set(pm_options_t *options, const char *version, size_t length
     return false;
 }
 
+/**
+ * Set the version option on the given options struct to the lowest version of
+ * Ruby that prism supports.
+ */
+void
+pm_options_version_set_lowest(pm_options_t *options) {
+    options->version = PM_OPTIONS_VERSION_CRUBY_3_3;
+}
+
+/**
+ * Set the version option on the given options struct to the highest version of
+ * Ruby that prism supports.
+ */
+void
+pm_options_version_set_highest(pm_options_t *options) {
+    options->version = PM_OPTIONS_VERSION_LATEST;
+}
+
 /**
  * Set the main script option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_main_script_set(pm_options_t *options, bool main_script) {
     options->main_script = main_script;
 }
@@ -144,19 +222,57 @@ pm_options_main_script_set(pm_options_t *options, bool main_script) {
 /**
  * Set the partial script option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_partial_script_set(pm_options_t *options, bool partial_script) {
     options->partial_script = partial_script;
 }
 
+/**
+ * Get the freeze option on the given options struct.
+ */
+bool
+pm_options_freeze(const pm_options_t *options) {
+    return options->freeze;
+}
+
 /**
  * Set the freeze option on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION void
+void
 pm_options_freeze_set(pm_options_t *options, bool freeze) {
     options->freeze = freeze;
 }
 
+/**
+ * Get the attach_comments option on the given options struct.
+ */
+bool
+pm_options_attach_comments(const pm_options_t *options) {
+    return options->attach_comments;
+}
+
+/**
+ * Set the attach_comments option on the given options struct.
+ */
+void
+pm_options_attach_comments_set(pm_options_t *options, bool attach_comments) {
+    options->attach_comments = attach_comments;
+}
+
+/**
+ * Get the raise_error option on the given options struct.
+ */
+uint8_t pm_options_raise_error(const pm_options_t *options) {
+    return options->raise_error;
+}
+
+/**
+ * Set the raise_error option on the given options struct.
+ */
+void pm_options_raise_error_set(pm_options_t *options, uint8_t raise_error) {
+    options->raise_error = raise_error;
+}
+
 // For some reason, GCC analyzer thinks we're leaking allocated scopes and
 // locals here, even though we definitely aren't. This is a false positive.
 // Ideally we wouldn't need to suppress this.
@@ -168,7 +284,7 @@ pm_options_freeze_set(pm_options_t *options, bool freeze) {
 /**
  * Allocate and zero out the scopes array on the given options struct.
  */
-PRISM_EXPORTED_FUNCTION bool
+bool
 pm_options_scopes_init(pm_options_t *options, size_t scopes_count) {
     options->scopes_count = scopes_count;
     options->scopes = xcalloc(scopes_count, sizeof(pm_options_scope_t));
@@ -176,10 +292,20 @@ pm_options_scopes_init(pm_options_t *options, size_t scopes_count) {
 }
 
 /**
- * Return a pointer to the scope at the given index within the given options.
+ * Return a constant pointer to the scope at the given index within the given
+ * options.
  */
-PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
-pm_options_scope_get(const pm_options_t *options, size_t index) {
+const pm_options_scope_t *
+pm_options_scope(const pm_options_t *options, size_t index) {
+    return &options->scopes[index];
+}
+
+/**
+ * Return a mutable pointer to the scope at the given index within the given
+ * options.
+ */
+pm_options_scope_t *
+pm_options_scope_mut(pm_options_t *options, size_t index) {
     return &options->scopes[index];
 }
 
@@ -187,49 +313,38 @@ pm_options_scope_get(const pm_options_t *options, size_t index) {
  * Create a new options scope struct. This will hold a set of locals that are in
  * scope surrounding the code that is being parsed.
  */
-PRISM_EXPORTED_FUNCTION bool
+void
 pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count) {
     scope->locals_count = locals_count;
     scope->locals = xcalloc(locals_count, sizeof(pm_string_t));
     scope->forwarding = PM_OPTIONS_SCOPE_FORWARDING_NONE;
-    return scope->locals != NULL;
+    if (scope->locals == NULL) abort();
 }
 
 /**
- * Return a pointer to the local at the given index within the given scope.
+ * Return a constant pointer to the local at the given index within the given
+ * scope.
  */
-PRISM_EXPORTED_FUNCTION const pm_string_t *
-pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
+const pm_string_t *
+pm_options_scope_local(const pm_options_scope_t *scope, size_t index) {
     return &scope->locals[index];
 }
 
 /**
- * Set the forwarding option on the given scope struct.
+ * Return a mutable pointer to the local at the given index within the given
+ * scope.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding) {
-    scope->forwarding = forwarding;
+pm_string_t *
+pm_options_scope_local_mut(pm_options_scope_t *scope, size_t index) {
+    return &scope->locals[index];
 }
 
 /**
- * Free the internal memory associated with the options.
+ * Set the forwarding option on the given scope struct.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_free(pm_options_t *options) {
-    pm_string_free(&options->filepath);
-    pm_string_free(&options->encoding);
-
-    for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
-        pm_options_scope_t *scope = &options->scopes[scope_index];
-
-        for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
-            pm_string_free(&scope->locals[local_index]);
-        }
-
-        xfree(scope->locals);
-    }
-
-    xfree(options->scopes);
+void
+pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding) {
+    scope->forwarding = forwarding;
 }
 
 /**
@@ -302,6 +417,7 @@ pm_options_read(pm_options_t *options, const char *data) {
     options->main_script = ((uint8_t) *data++) > 0;
     options->partial_script = ((uint8_t) *data++) > 0;
     options->freeze = ((uint8_t) *data++) > 0;
+    options->attach_comments = ((uint8_t) *data++) > 0;
 
     uint32_t scopes_count = pm_options_read_u32(data);
     data += 4;
@@ -314,10 +430,7 @@ pm_options_read(pm_options_t *options, const char *data) {
             data += 4;
 
             pm_options_scope_t *scope = &options->scopes[scope_index];
-            if (!pm_options_scope_init(scope, locals_count)) {
-                pm_options_free(options);
-                return;
-            }
+            pm_options_scope_init(scope, locals_count);
 
             uint8_t forwarding = (uint8_t) *data++;
             pm_options_scope_forwarding_set(&options->scopes[scope_index], forwarding);

data/src/parser.c

@@ -0,0 +1,349 @@
+#include "prism/internal/parser.h"
+
+#include "prism/internal/allocator.h"
+#include "prism/internal/comments.h"
+#include "prism/internal/diagnostic.h"
+#include "prism/internal/encoding.h"
+#include "prism/internal/magic_comments.h"
+
+#include <stdlib.h>
+
+/**
+ * Register a callback that will be called whenever prism changes the encoding
+ * it is using to parse based on the magic comment.
+ */
+void
+pm_parser_encoding_changed_callback_set(pm_parser_t *parser, pm_encoding_changed_callback_t callback) {
+    parser->encoding_changed_callback = callback;
+}
+
+/**
+ * Register a callback that will be called whenever a token is lexed.
+ */
+void
+pm_parser_lex_callback_set(pm_parser_t *parser, pm_lex_callback_t callback, void *data) {
+    parser->lex_callback.callback = callback;
+    parser->lex_callback.data = data;
+}
+
+/**
+ * Returns the opaque data that is passed to the lex callback when it is called.
+ */
+void *
+pm_parser_lex_callback_data(const pm_parser_t *parser) {
+    return parser->lex_callback.data;
+}
+
+/**
+ * Returns the raw pointer to the start of the source that is being parsed.
+ */
+const uint8_t *
+pm_parser_start(const pm_parser_t *parser) {
+    return parser->start;
+}
+
+/**
+ * Returns the raw pointer to the end of the source that is being parsed.
+ */
+const uint8_t *
+pm_parser_end(const pm_parser_t *parser) {
+    return parser->end;
+}
+
+/**
+ * Returns the line that the parser was considered to have started on.
+ *
+ * @param parser the parser whose start line we want to get
+ * @return the line that the parser was considered to have started on
+ */
+int32_t
+pm_parser_start_line(const pm_parser_t *parser) {
+    return parser->start_line;
+}
+
+/**
+ * Returns the name of the encoding that is being used to parse the source.
+ */
+const char *
+pm_parser_encoding_name(const pm_parser_t *parser) {
+    return parser->encoding->name;
+}
+
+/**
+ * Returns the width of the character at the given pointer in the encoding that
+ * is being used to parse the source.
+ */
+size_t
+pm_parser_encoding_char_width(const pm_parser_t *parser, const uint8_t *start, ptrdiff_t remaining) {
+    return parser->encoding->char_width(start, remaining);
+}
+
+/**
+ * Returns whether or not the parser is using the US-ASCII encoding.
+ */
+bool
+pm_parser_encoding_us_ascii(const pm_parser_t *parser) {
+    return parser->encoding == PM_ENCODING_US_ASCII_ENTRY;
+}
+
+/**
+ * Returns the filepath that is being used to parse the source.
+ */
+const pm_string_t *
+pm_parser_filepath(const pm_parser_t *parser) {
+    return &parser->filepath;
+}
+
+/**
+ * Find a constant in the parser's constant pool. Returns the id of the
+ * constant, or 0 if the constant is not found.
+ */
+pm_constant_id_t
+pm_parser_constant_find(const pm_parser_t *parser, const uint8_t *start, size_t length) {
+    return pm_constant_pool_find(&parser->constant_pool, start, length);
+}
+
+/**
+ * Returns the frozen string literal value of the parser.
+ */
+int8_t
+pm_parser_frozen_string_literal(const pm_parser_t *parser) {
+    return parser->frozen_string_literal;
+}
+
+/**
+ * Returns the line offsets that are associated with the given parser.
+ *
+ * @param parser the parser whose line offsets we want to get
+ * @return the line offsets that are associated with the given parser
+ */
+const pm_line_offset_list_t *
+pm_parser_line_offsets(const pm_parser_t *parser) {
+    return &parser->line_offsets;
+}
+
+/**
+ * Returns the location of the __DATA__ section that is associated with the
+ * given parser, if it exists.
+ */
+const pm_location_t *
+pm_parser_data_loc(const pm_parser_t *parser) {
+    return &parser->data_loc;
+}
+
+/**
+ * Returns whether the given parser is continuable, meaning that it could become
+ * valid if more input were appended, as opposed to being definitively invalid.
+ */
+bool
+pm_parser_continuable(const pm_parser_t *parser) {
+    return parser->continuable;
+}
+
+/**
+ * Returns the lex state of the parser. Note that this is an internal detail,
+ * and we are purposefully not returning an instance of the internal enum that
+ * we use to track this. This is only exposed because we need it for some very
+ * niche use cases. Most consumers should avoid this function.
+ */
+int
+pm_parser_lex_state(const pm_parser_t *parser) {
+    return (int) parser->lex_state;
+}
+
+/**
+ * Returns the location associated with the given comment.
+ */
+pm_location_t
+pm_comment_location(const pm_comment_t *comment) {
+    return comment->location;
+}
+
+/**
+ * Returns the type associated with the given comment.
+ */
+pm_comment_type_t
+pm_comment_type(const pm_comment_t *comment) {
+    return comment->type;
+}
+
+/**
+ * Returns the number of comments associated with the given parser.
+ */
+size_t
+pm_parser_comments_size(const pm_parser_t *parser) {
+    return parser->comment_list.size;
+}
+
+/**
+ * Iterates over the comments associated with the given parser and calls the
+ * given callback for each comment.
+ */
+void
+pm_parser_comments_each(const pm_parser_t *parser, pm_comment_callback_t callback, void *data) {
+    const pm_list_node_t *current = parser->comment_list.head;
+    while (current != NULL) {
+        const pm_comment_t *comment = (const pm_comment_t *) current;
+        callback(comment, data);
+        current = current->next;
+    }
+}
+
+/**
+ * Returns the number of comments owned by the given node.
+ */
+uint32_t
+pm_node_comments_size(const pm_parser_t *parser, const pm_node_t *node) {
+    return pm_parser_comment_range(parser, node->node_id).length;
+}
+
+/**
+ * Returns the comment at the given index among the comments owned by the given
+ * node.
+ */
+const pm_comment_t *
+pm_node_comment(const pm_parser_t *parser, const pm_node_t *node, uint32_t index) {
+    pm_comment_range_t range = pm_parser_comment_range(parser, node->node_id);
+    if (index >= range.length) return NULL;
+
+    return parser->comment_attachments.comments[parser->comment_attachments.attachments[range.index + index]];
+}
+
+/**
+ * Returns the index, among the comments associated with the given parser, of
+ * the comment at the given index among the comment attachments.
+ */
+uint32_t
+pm_parser_comment_attachment(const pm_parser_t *parser, uint32_t index) {
+    if (parser->comment_attachments.attachments == NULL || index >= parser->comment_list.size) return UINT32_MAX;
+    return parser->comment_attachments.attachments[index];
+}
+
+/**
+ * Returns the number of node ids that have a comment range.
+ */
+uint32_t
+pm_parser_comment_ranges_size(const pm_parser_t *parser) {
+    return parser->comment_attachments.ranges_size;
+}
+
+/**
+ * Returns the range of comment attachments owned by the node with the given id.
+ */
+pm_comment_range_t
+pm_parser_comment_range(const pm_parser_t *parser, uint32_t node_id) {
+    if (node_id >= parser->comment_attachments.ranges_size) return (pm_comment_range_t) { .index = 0, .length = 0 };
+    return parser->comment_attachments.ranges[node_id];
+}
+
+/**
+ * Returns the location associated with the given magic comment key.
+ */
+pm_location_t
+pm_magic_comment_key(const pm_magic_comment_t *magic_comment) {
+    return magic_comment->key;
+}
+
+/**
+ * Returns the location associated with the given magic comment value.
+ */
+pm_location_t
+pm_magic_comment_value(const pm_magic_comment_t *magic_comment) {
+    return magic_comment->value;
+}
+
+/**
+ * Returns the number of magic comments associated with the given parser.
+ */
+size_t
+pm_parser_magic_comments_size(const pm_parser_t *parser) {
+    return parser->magic_comment_list.size;
+}
+
+/**
+ * Iterates over the magic comments associated with the given parser and calls
+ * the given callback for each magic comment.
+ */
+void
+pm_parser_magic_comments_each(const pm_parser_t *parser, pm_magic_comment_callback_t callback, void *data) {
+    const pm_list_node_t *current = parser->magic_comment_list.head;
+    while (current != NULL) {
+        const pm_magic_comment_t *magic_comment = (const pm_magic_comment_t *) current;
+        callback(magic_comment, data);
+        current = current->next;
+    }
+}
+
+/**
+ * Returns the number of errors associated with the given parser.
+ */
+size_t
+pm_parser_errors_size(const pm_parser_t *parser) {
+    return parser->error_list.size;
+}
+
+/**
+ * Returns the number of warnings associated with the given parser.
+ */
+size_t
+pm_parser_warnings_size(const pm_parser_t *parser) {
+    return parser->warning_list.size;
+}
+
+static inline void
+pm_parser_diagnostics_each(const pm_list_t *list, pm_diagnostic_callback_t callback, void *data) {
+    const pm_list_node_t *current = list->head;
+    while (current != NULL) {
+        const pm_diagnostic_t *diagnostic = (const pm_diagnostic_t *) current;
+        callback(diagnostic, data);
+        current = current->next;
+    }
+}
+
+/**
+ * Iterates over the errors associated with the given parser and calls the
+ * given callback for each error.
+ */
+void
+pm_parser_errors_each(const pm_parser_t *parser, pm_diagnostic_callback_t callback, void *data) {
+    pm_parser_diagnostics_each(&parser->error_list, callback, data);
+}
+
+/**
+ * Iterates over the warnings associated with the given parser and calls the
+ * given callback for each warning.
+ */
+void
+pm_parser_warnings_each(const pm_parser_t *parser, pm_diagnostic_callback_t callback, void *data) {
+    pm_parser_diagnostics_each(&parser->warning_list, callback, data);
+}
+
+/**
+ * Returns the number of constants in the constant pool associated with the
+ * given parser.
+ */
+size_t
+pm_parser_constants_size(const pm_parser_t *parser) {
+    return parser->constant_pool.size;
+}
+
+/**
+ * Iterates over the constants in the constant pool associated with the given
+ * parser and calls the given callback for each constant.
+ */
+void
+pm_parser_constants_each(const pm_parser_t *parser, pm_constant_callback_t callback, void *data) {
+    for (uint32_t index = 0; index < parser->constant_pool.size; index++) {
+        const pm_constant_t *constant = &parser->constant_pool.constants[index];
+        callback(constant, data);
+    }
+}
+
+/**
+ * Returns a pointer to the constant at the given id in the constant pool
+ * associated with the given parser.
+ */
+const pm_constant_t *
+pm_parser_constant(const pm_parser_t *parser, pm_constant_id_t constant_id) {
+    return pm_constant_pool_id_to_constant(&parser->constant_pool, constant_id);
+}