prism 2.0.0.rc1 → 2.0.0

data/include/prism/parser.h

@@ -16,6 +16,10 @@
 #include "prism/magic_comments.h"
 #include "prism/options.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * The parser used to parse Ruby source.
  */
@@ -150,6 +154,24 @@ PRISM_EXPORTED_FUNCTION const pm_string_t * pm_parser_filepath(const pm_parser_t
  */
 PRISM_EXPORTED_FUNCTION pm_constant_id_t pm_parser_constant_find(const pm_parser_t *parser, const uint8_t *start, size_t length) PRISM_NONNULL(1, 2);
 
+/**
+ * Add a constant to the parser's constant pool, for naming a node that was not
+ * parsed from the source. If the constant is already in the pool, its existing
+ * id is returned. Otherwise the parser copies the bytes, so they only need to
+ * remain valid for the duration of the call.
+ *
+ * Adding a constant makes the pool larger, so a cache with one entry per
+ * constant built beforehand from pm_parser_constants_size and
+ * pm_parser_constant_at does not cover it. Build such caches after adding any
+ * constants.
+ *
+ * @param parser the parser whose constant pool we want to add to
+ * @param start a pointer to the start of the string to add
+ * @param length the length of the string to add
+ * @returns the id of the constant
+ */
+PRISM_EXPORTED_FUNCTION pm_constant_id_t pm_parser_constant_insert(pm_parser_t *parser, const uint8_t *start, size_t length) PRISM_NONNULL(1, 2);
+
 /**
  * Returns the frozen string literal value of the parser, as determined by the
  * frozen_string_literal magic comment or the option set on the parser.
@@ -177,6 +199,27 @@ PRISM_EXPORTED_FUNCTION const pm_line_offset_list_t * pm_parser_line_offsets(con
  */
 PRISM_EXPORTED_FUNCTION const pm_location_t * pm_parser_data_loc(const pm_parser_t *parser) PRISM_NONNULL(1);
 
+/**
+ * Returns the checksum of the source code that was parsed. The data section
+ * after an __END__ marker is not code, so the hash covers the source only up to
+ * and including the newline that ends the __END__ line. The __END__ marker is
+ * only found by parsing, so this must be called after pm_parse.
+ *
+ * The hash is computed from the source on each call rather than during
+ * parsing, so that parses that never ask for it do not pay for it. This
+ * requires the source to still be alive, and callers that need the value more
+ * than once should keep it.
+ *
+ * The hash is currently the 64-bit variant of XXH3 with a seed of 0. It is not
+ * cryptographic, and the algorithm may change between versions of prism, so
+ * hashes should only be compared against other hashes computed by the same
+ * version of prism.
+ *
+ * @param parser the parser whose checksum we want to get
+ * @returns the checksum of the source code that was parsed
+ */
+PRISM_EXPORTED_FUNCTION uint64_t pm_parser_checksum(const pm_parser_t *parser) PRISM_NONNULL(1);
+
 /**
  * Returns whether the given parser is continuable, meaning that it could become
  * valid if more input were appended, as opposed to being definitively invalid.
@@ -206,22 +249,19 @@ PRISM_EXPORTED_FUNCTION int pm_parser_lex_state(const pm_parser_t *parser) PRISM
 PRISM_EXPORTED_FUNCTION size_t pm_parser_comments_size(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
- * A callback function that can be used to process comments found while parsing.
- */
-typedef void (*pm_comment_callback_t)(const pm_comment_t *comment, void *data);
-
-/**
- * Iterates over the comments associated with the given parser and calls the
- * given callback for each comment.
+ * Returns the first comment associated with the given parser. Iterate over the
+ * rest with pm_comment_next, as in:
+ *
+ * ```c
+ * for (const pm_comment_t *comment = pm_parser_comments_first(parser); comment != NULL; comment = pm_comment_next(comment)) {
+ *     pm_location_t location = pm_comment_location(comment);
+ * }
+ * ```
  *
  * @param parser the parser whose comments we want to iterate over
- * @param callback the callback function to call for each comment. This function
- *     will be passed a pointer to the comment and the data parameter passed to
- *     this function.
- * @param data the data to pass to the callback function for each comment. This
- *     can be NULL if no data needs to be passed to the callback function.
+ * @returns the first comment, or NULL if there are no comments
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_comments_each(const pm_parser_t *parser, pm_comment_callback_t callback, void *data) PRISM_NONNULL(1);
+PRISM_EXPORTED_FUNCTION const pm_comment_t * pm_parser_comments_first(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
  * Returns the number of comments owned by the given node. This is 0 if the
@@ -288,22 +328,13 @@ PRISM_EXPORTED_FUNCTION pm_comment_range_t pm_parser_comment_range(const pm_pars
 PRISM_EXPORTED_FUNCTION size_t pm_parser_magic_comments_size(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
- * A callback function that can be used to process magic comments found while parsing.
- */
-typedef void (*pm_magic_comment_callback_t)(const pm_magic_comment_t *magic_comment, void *data);
-
-/**
- * Iterates over the magic comments associated with the given parser and calls the
- * given callback for each magic comment.
+ * Returns the first magic comment associated with the given parser. Iterate
+ * over the rest with pm_magic_comment_next.
  *
  * @param parser the parser whose magic comments we want to iterate over
- * @param callback the callback function to call for each magic comment. This
- *     function will be passed a pointer to the magic comment and the data
- *     parameter passed to this function.
- * @param data the data to pass to the callback function for each magic comment.
- *     This can be NULL if no data needs to be passed to the callback function.
+ * @returns the first magic comment, or NULL if there are no magic comments
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_magic_comments_each(const pm_parser_t *parser, pm_magic_comment_callback_t callback, void *data) PRISM_NONNULL(1);
+PRISM_EXPORTED_FUNCTION const pm_magic_comment_t * pm_parser_magic_comments_first(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
  * Returns the number of errors associated with the given parser.
@@ -322,36 +353,22 @@ PRISM_EXPORTED_FUNCTION size_t pm_parser_errors_size(const pm_parser_t *parser)
 PRISM_EXPORTED_FUNCTION size_t pm_parser_warnings_size(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
- * A callback function that can be used to process diagnostics found while
- * parsing.
- */
-typedef void (*pm_diagnostic_callback_t)(const pm_diagnostic_t *diagnostic, void *data);
-
-/**
- * Iterates over the errors associated with the given parser and calls the
- * given callback for each error.
+ * Returns the first error associated with the given parser. Iterate over the
+ * rest with pm_diagnostic_next.
  *
  * @param parser the parser whose errors we want to iterate over
- * @param callback the callback function to call for each error. This function
- *     will be passed a pointer to the error and the data parameter passed to
- *     this function.
- * @param data the data to pass to the callback function for each error. This
- *     can be NULL if no data needs to be passed to the callback function.
+ * @returns the first error, or NULL if there are no errors
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_errors_each(const pm_parser_t *parser, pm_diagnostic_callback_t callback, void *data) PRISM_NONNULL(1);
+PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * pm_parser_errors_first(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
- * Iterates over the warnings associated with the given parser and calls the
- * given callback for each warning.
+ * Returns the first warning associated with the given parser. Iterate over the
+ * rest with pm_diagnostic_next.
  *
  * @param parser the parser whose warnings we want to iterate over
- * @param callback the callback function to call for each warning. This function
- *     will be passed a pointer to the warning and the data parameter passed to
- *     this function.
- * @param data the data to pass to the callback function for each warning. This
- *     can be NULL if no data needs to be passed to the callback function.
+ * @returns the first warning, or NULL if there are no warnings
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_warnings_each(const pm_parser_t *parser, pm_diagnostic_callback_t callback, void *data) PRISM_NONNULL(1);
+PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * pm_parser_warnings_first(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
  * Returns the number of constants in the constant pool associated with the
@@ -365,30 +382,38 @@ PRISM_EXPORTED_FUNCTION void pm_parser_warnings_each(const pm_parser_t *parser,
 PRISM_EXPORTED_FUNCTION size_t pm_parser_constants_size(const pm_parser_t *parser) PRISM_NONNULL(1);
 
 /**
- * A callback function that can be used to process constants found while
- * parsing.
- */
-typedef void (*pm_constant_callback_t)(const pm_constant_t *constant, void *data);
-
-/**
- * Iterates over the constants in the constant pool associated with the given
- * parser and calls the given callback for each constant.
+ * Returns the constant at the given index in the constant pool associated with
+ * the given parser. Together with pm_constant_id_index, this lets a caller
+ * build a cache with one entry per constant and then find the entry for any
+ * constant id, as in:
+ *
+ * ```c
+ * size_t size = pm_parser_constants_size(parser);
+ * for (size_t index = 0; index < size; index++) {
+ *     cache[index] = convert(pm_parser_constant_at(parser, index));
+ * }
+ *
+ * value = cache[pm_constant_id_index(constant_id)];
+ * ```
  *
- * @param parser the parser whose constants we want to iterate over
- * @param callback the callback function to call for each constant. This function
- *     will be passed a pointer to the constant and the data parameter passed to
- *     this function.
- * @param data the data to pass to the callback function for each constant. This
- *     can be NULL if no data needs to be passed to the callback function.
+ * For any constant id, pm_parser_constant_at(parser,
+ * pm_constant_id_index(constant_id)) is pm_parser_constant(parser,
+ * constant_id).
+ *
+ * @param parser the parser whose constant pool we want to look up from
+ * @param index the index of the constant, which must be less than
+ *     pm_parser_constants_size(parser)
+ * @returns a pointer to the constant at the given index
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_constants_each(const pm_parser_t *parser, pm_constant_callback_t callback, void *data) PRISM_NONNULL(1);
+PRISM_EXPORTED_FUNCTION const pm_constant_t * pm_parser_constant_at(const pm_parser_t *parser, size_t index) PRISM_NONNULL(1);
 
 /**
  * Returns a pointer to the constant at the given id in the constant pool
  * associated with the given parser.
  *
  * @param parser the parser whose constant pool we want to look up from
- * @param constant_id the id of the constant to look up (1-based)
+ * @param constant_id the id of the constant to look up. Optional constant
+ *     fields on nodes hold 0 when they are absent, which is not a valid id.
  * @returns a pointer to the constant at the given id
  */
 PRISM_EXPORTED_FUNCTION const pm_constant_t * pm_parser_constant(const pm_parser_t *parser, pm_constant_id_t constant_id) PRISM_NONNULL(1);
@@ -401,4 +426,8 @@ PRISM_EXPORTED_FUNCTION const pm_constant_t * pm_parser_constant(const pm_parser
  */
 PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser) PRISM_NONNULL(1);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

data/include/prism/parser.hpp

@@ -0,0 +1,274 @@
+/**
+ * @file parser.hpp
+ *
+ * Ownership of parsers from C++, and lookups through a parser into the source,
+ * the constant pool, comments, and diagnostics.
+ */
+#ifndef PRISM_PARSER_HPP
+#define PRISM_PARSER_HPP
+
+#include "prism/compiler/cplusplus.hpp"
+#include "prism/compiler/lifetimebound.hpp"
+
+#include "prism/arena.h"
+#include "prism/constant_pool.h"
+#include "prism/line_offset_list.h"
+#include "prism/parser.h"
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <string_view>
+
+namespace prism {
+
+/** Frees a parser with pm_parser_free. */
+struct parser_deleter {
+    /**
+     * Free the given parser.
+     *
+     * @param parser The parser to free.
+     */
+    void operator()(pm_parser_t *parser) const noexcept { pm_parser_free(parser); }
+};
+
+/** An owned parser. */
+using unique_parser = std::unique_ptr<pm_parser_t, parser_deleter>;
+
+/**
+ * Allocate a new parser over the given source. The source, the arena, and the
+ * options (which may be null) are not copied, so all three must outlive the
+ * parser. Allocation failure aborts, so this never throws.
+ *
+ * @param arena The arena to use for allocations.
+ * @param source The source code to parse.
+ * @param options The parse options, or null for defaults.
+ * @return A unique_parser owning the newly allocated parser.
+ */
+[[nodiscard]] inline unique_parser
+make_parser(pm_arena_t *arena PRISM_LIFETIMEBOUND, std::string_view source PRISM_LIFETIMEBOUND, const pm_options_t *options PRISM_LIFETIMEBOUND = nullptr) noexcept {
+    return unique_parser(pm_parser_new(arena, reinterpret_cast<const std::uint8_t *>(source.data()), source.size(), options));
+}
+
+/**
+ * The name of the constant with the given id, which must not be zero. Fields
+ * holding an optional constant use zero for none, so check those first.
+ *
+ * @param parser The parser containing the constant.
+ * @param constant_id The id of the constant.
+ * @return The value of the constant.
+ */
+[[nodiscard]] inline std::string_view
+constant(const pm_parser_t *parser PRISM_LIFETIMEBOUND, pm_constant_id_t constant_id) noexcept {
+    assert(constant_id != 0);
+    const pm_constant_t *entry = pm_parser_constant(parser, constant_id);
+    return std::string_view(reinterpret_cast<const char *>(pm_constant_start(entry)), pm_constant_length(entry));
+}
+
+/**
+ * The bytes of source that the given location covers.
+ *
+ * @param parser The parser containing the source.
+ * @param location The location in the source.
+ * @return The bytes of source covered by the location.
+ */
+[[nodiscard]] inline std::string_view
+slice(const pm_parser_t *parser PRISM_LIFETIMEBOUND, pm_location_t location) noexcept {
+    return std::string_view(reinterpret_cast<const char *>(pm_parser_start(parser) + location.start), location.length);
+}
+
+/**
+ * The line and column of the given byte offset into the source. Lines start at
+ * the parser's start line, and columns are byte offsets from the line start.
+ *
+ * @param parser The parser containing the source.
+ * @param offset The byte offset into the source.
+ * @return The line and column of the byte offset.
+ */
+[[nodiscard]] inline pm_line_column_t
+line_column(const pm_parser_t *parser, std::uint32_t offset) noexcept {
+    return pm_line_offset_list_line_column(pm_parser_line_offsets(parser), offset, pm_parser_start_line(parser));
+}
+
+namespace detail {
+
+/**
+ * The diagnostics, comments, or magic comments found while parsing, usable in a
+ * range-based for loop and with the standard algorithms. Each element is a
+ * pointer to the C struct, which the C accessors read, and Next is the C
+ * function that returns the item after a given one. It refers to the parser
+ * rather than copying from it, so the parser must outlive it. It is used
+ * through diagnostic_range, comment_range, and magic_comment_range.
+ */
+template <typename T, const T *(*Next)(const T *)> class next_range {
+public:
+    /** Steps through the items one after another. */
+    class iterator {
+    public:
+        /** The iterator category, for the standard algorithms. */
+        using iterator_category = std::forward_iterator_tag;
+
+        /** The type of each element. */
+        using value_type = const T *;
+
+        /** The type of the distance between two iterators. */
+        using difference_type = std::ptrdiff_t;
+
+        /** The type of a pointer to an element. */
+        using pointer = const value_type *;
+
+        /** The type of a reference to an element. */
+        using reference = const value_type &;
+
+        /** Create the iterator that marks the end of the items. */
+        iterator() noexcept = default;
+
+        /**
+         * Create an iterator at the given item.
+         *
+         * @param current The current item.
+         */
+        explicit iterator(const T *current) noexcept : current_(current) {}
+
+        /**
+         * The item at this position, which must not be the end.
+         *
+         * @return The current item.
+         */
+        reference operator*() const noexcept { return current_; }
+
+        /**
+         * Advance to the next item.
+         *
+         * @return The iterator after advancing.
+         */
+        iterator &
+        operator++() noexcept {
+            assert(current_ != nullptr);
+            current_ = Next(current_);
+            return *this;
+        }
+
+        /**
+         * Advance to the next item, returning the iterator from before.
+         *
+         * @return The iterator from before advancing.
+         */
+        iterator
+        operator++(int) noexcept {
+            iterator previous = *this;
+            ++*this;
+            return previous;
+        }
+
+        /** Whether the two iterators are at the same item. */
+        friend bool operator==(iterator left, iterator right) noexcept { return left.current_ == right.current_; }
+
+        /** Whether the two iterators are at different items. */
+        friend bool operator!=(iterator left, iterator right) noexcept { return left.current_ != right.current_; }
+
+    private:
+        const T *current_ = nullptr;
+    };
+
+    /**
+     * Create a range of the given number of items starting at first.
+     *
+     * @param first The first item.
+     * @param size The number of items.
+     */
+    next_range(const T *first, std::size_t size) noexcept : first_(first), size_(size) {}
+
+    /**
+     * An iterator at the first item.
+     *
+     * @return An iterator at the first item.
+     */
+    iterator begin() const noexcept { return iterator(first_); }
+
+    /**
+     * An iterator past the last item.
+     *
+     * @return An iterator past the last item.
+     */
+    iterator end() const noexcept { return iterator(); }
+
+    /**
+     * The number of items.
+     *
+     * @return The number of items.
+     */
+    std::size_t size() const noexcept { return size_; }
+
+    /**
+     * Whether there are no items.
+     *
+     * @return Whether there are no items.
+     */
+    [[nodiscard]] bool empty() const noexcept { return size_ == 0; }
+
+private:
+    const T *first_;
+    std::size_t size_;
+};
+
+}
+
+/** The errors or warnings found while parsing. */
+using diagnostic_range = detail::next_range<pm_diagnostic_t, pm_diagnostic_next>;
+
+/** The comments found while parsing. */
+using comment_range = detail::next_range<pm_comment_t, pm_comment_next>;
+
+/** The magic comments found while parsing. */
+using magic_comment_range = detail::next_range<pm_magic_comment_t, pm_magic_comment_next>;
+
+/**
+ * The errors found while parsing.
+ *
+ * @param parser The parser containing the source.
+ * @return The range of errors found while parsing.
+ */
+[[nodiscard]] inline diagnostic_range
+errors(const pm_parser_t *parser PRISM_LIFETIMEBOUND) noexcept {
+    return diagnostic_range(pm_parser_errors_first(parser), pm_parser_errors_size(parser));
+}
+
+/**
+ * The warnings found while parsing.
+ *
+ * @param parser The parser containing the source.
+ * @return The range of warnings found while parsing.
+ */
+[[nodiscard]] inline diagnostic_range
+warnings(const pm_parser_t *parser PRISM_LIFETIMEBOUND) noexcept {
+    return diagnostic_range(pm_parser_warnings_first(parser), pm_parser_warnings_size(parser));
+}
+
+/**
+ * The comments found while parsing, in source order.
+ *
+ * @param parser The parser containing the source.
+ * @return The range of comments found while parsing.
+ */
+[[nodiscard]] inline comment_range
+comments(const pm_parser_t *parser PRISM_LIFETIMEBOUND) noexcept {
+    return comment_range(pm_parser_comments_first(parser), pm_parser_comments_size(parser));
+}
+
+/**
+ * The magic comments found while parsing.
+ *
+ * @param parser The parser containing the source.
+ * @return The range of magic comments found while parsing.
+ */
+[[nodiscard]] inline magic_comment_range
+magic_comments(const pm_parser_t *parser PRISM_LIFETIMEBOUND) noexcept {
+    return magic_comment_range(pm_parser_magic_comments_first(parser), pm_parser_magic_comments_size(parser));
+}
+
+}
+
+#endif

data/include/prism/prettyprint.h

@@ -17,6 +17,10 @@
 #include "prism/buffer.h"
 #include "prism/parser.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * Pretty-prints the AST represented by the given node to the given buffer.
  *
@@ -26,6 +30,10 @@
  */
 PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node) PRISM_NONNULL(1, 2, 3);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif
 
 #endif

data/include/prism/serialize.h

@@ -22,6 +22,10 @@
 #include "prism/source.h"
 #include "prism/stream.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * Serialize the AST represented by the given node to the given buffer.
  *
@@ -110,6 +114,10 @@ PRISM_EXPORTED_FUNCTION bool pm_serialize_parse_success_p(const uint8_t *source,
  */
 PRISM_EXPORTED_FUNCTION int8_t pm_serialize_parse_errors_format(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data, pm_errors_format_type_t format_type) PRISM_NONNULL(1, 2);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif
 
 #endif

data/include/prism/source.h

@@ -15,6 +15,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * An opaque type representing source code being parsed.
  */
@@ -152,4 +156,8 @@ PRISM_EXPORTED_FUNCTION size_t pm_source_length(const pm_source_t *source) PRISM
  */
 PRISM_EXPORTED_FUNCTION const uint8_t * pm_source_source(const pm_source_t *source) PRISM_NONNULL(1);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

data/include/prism/stream.h

@@ -14,6 +14,10 @@
 #include "prism/parser.h"
 #include "prism/source.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * Parse a stream of Ruby source and return the tree.
  *
@@ -25,4 +29,8 @@
  */
 PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t **parser, pm_arena_t *arena, pm_source_t *source, const pm_options_t *options) PRISM_NONNULL(1, 2, 3);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

data/include/prism/string_query.h

@@ -13,6 +13,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * Represents the results of a slice query.
  */
@@ -60,4 +64,19 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_constant(const uint8_t
  */
 PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
 
+/**
+ * Check that the slice is a keyword.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_keyword(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif