prism 2.0.0.rc1 → 2.0.0

data/ext/prism/extension.c

@@ -192,7 +192,7 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
             // Add the local to the scope.
             pm_string_t *scope_local = pm_options_scope_local_mut(options_scope, local_index);
             const char *name = rb_id2name(SYM2ID(local));
-            pm_string_constant_init(scope_local, name, strlen(name));
+            pm_string_init(scope_local, (const uint8_t *) name, strlen(name));
         }
 
         // Now set the forwarding options.
@@ -633,28 +633,19 @@ parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) {
     return rb_class_new_instance_freeze(1, argv, type, freeze);
 }
 
-typedef struct {
-    VALUE comments;
-    VALUE source;
-    bool freeze;
-    long index;
-} parser_comments_each_data_t;
-
-static void
-parser_comments_each(const pm_comment_t *comment, void *data) {
-    parser_comments_each_data_t *each_data = (parser_comments_each_data_t *) data;
-    VALUE value = parser_comment(each_data->source, each_data->freeze, comment);
-    rb_ary_store(each_data->comments, each_data->index++, value);
-}
-
 /**
  * Extract the comments out of the parser into the given array, which was
- * created before the source so that the source could hold it.
+ * created before the source so that the source could hold it. The caller may
+ * have already resized it to the number of comments, so each comment is stored
+ * at its index rather than pushed.
  */
 static void
 parser_comments(const pm_parser_t *parser, VALUE comments, VALUE source, bool freeze) {
-    parser_comments_each_data_t each_data = { .comments = comments, .source = source, .freeze = freeze };
-    pm_parser_comments_each(parser, parser_comments_each, &each_data);
+    long index = 0;
+    for (const pm_comment_t *comment = pm_parser_comments_first(parser); comment != NULL; comment = pm_comment_next(comment)) {
+        rb_ary_store(comments, index++, parser_comment(source, freeze, comment));
+    }
+
     if (freeze) rb_obj_freeze(comments);
 }
 
@@ -673,19 +664,6 @@ parser_magic_comment(VALUE source, bool freeze, const pm_magic_comment_t *magic_
     return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze);
 }
 
-typedef struct {
-    VALUE magic_comments;
-    VALUE source;
-    bool freeze;
-} parser_magic_comments_each_data_t;
-
-static void
-parser_magic_comments_each(const pm_magic_comment_t *magic_comment, void *data) {
-    parser_magic_comments_each_data_t *each_data = (parser_magic_comments_each_data_t *) data;
-    VALUE value = parser_magic_comment(each_data->source, each_data->freeze, magic_comment);
-    rb_ary_push(each_data->magic_comments, value);
-}
-
 /**
  * Extract the magic comments out of the parser into an array.
  */
@@ -693,8 +671,9 @@ static VALUE
 parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
     VALUE magic_comments = rb_ary_new_capa(pm_parser_magic_comments_size(parser));
 
-    parser_magic_comments_each_data_t each_data = { magic_comments, source, freeze };
-    pm_parser_magic_comments_each(parser, parser_magic_comments_each, &each_data);
+    for (const pm_magic_comment_t *magic_comment = pm_parser_magic_comments_first(parser); magic_comment != NULL; magic_comment = pm_magic_comment_next(magic_comment)) {
+        rb_ary_push(magic_comments, parser_magic_comment(source, freeze, magic_comment));
+    }
 
     if (freeze) rb_obj_freeze(magic_comments);
     return magic_comments;
@@ -715,20 +694,14 @@ parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) {
     }
 }
 
-typedef struct {
-    VALUE errors;
-    rb_encoding *encoding;
-    VALUE source;
-    bool freeze;
-} parser_errors_each_data_t;
-
-static void
-parser_errors_each(const pm_diagnostic_t *diagnostic, void *data) {
-    parser_errors_each_data_t *each_data = (parser_errors_each_data_t *) data;
-
+/**
+ * Build a new ParseError instance from the given error.
+ */
+static VALUE
+parser_error(const pm_diagnostic_t *diagnostic, rb_encoding *encoding, VALUE source, bool freeze) {
     VALUE type = ID2SYM(rb_intern(pm_diagnostic_type(diagnostic)));
-    VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), each_data->encoding));
-    VALUE location = PARSER_LOCATION(each_data->source, each_data->freeze, pm_diagnostic_location(diagnostic));
+    VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), encoding));
+    VALUE location = PARSER_LOCATION(source, freeze, pm_diagnostic_location(diagnostic));
 
     pm_error_level_t error_level = pm_diagnostic_error_level(diagnostic);
     VALUE level = Qnil;
@@ -748,8 +721,7 @@ parser_errors_each(const pm_diagnostic_t *diagnostic, void *data) {
     }
 
     VALUE argv[] = { type, message, location, level };
-    VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseError, each_data->freeze);
-    rb_ary_push(each_data->errors, value);
+    return rb_class_new_instance_freeze(4, argv, rb_cPrismParseError, freeze);
 }
 
 /**
@@ -759,27 +731,22 @@ static VALUE
 parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
     VALUE errors = rb_ary_new_capa(pm_parser_errors_size(parser));
 
-    parser_errors_each_data_t each_data = { errors, encoding, source, freeze };
-    pm_parser_errors_each(parser, parser_errors_each, &each_data);
+    for (const pm_diagnostic_t *error = pm_parser_errors_first(parser); error != NULL; error = pm_diagnostic_next(error)) {
+        rb_ary_push(errors, parser_error(error, encoding, source, freeze));
+    }
 
     if (freeze) rb_obj_freeze(errors);
     return errors;
 }
 
-typedef struct {
-    VALUE warnings;
-    rb_encoding *encoding;
-    VALUE source;
-    bool freeze;
-} parser_warnings_each_data_t;
-
-static void
-parser_warnings_each(const pm_diagnostic_t *diagnostic, void *data) {
-    parser_warnings_each_data_t *each_data = (parser_warnings_each_data_t *) data;
-
+/**
+ * Build a new ParseWarning instance from the given warning.
+ */
+static VALUE
+parser_warning(const pm_diagnostic_t *diagnostic, rb_encoding *encoding, VALUE source, bool freeze) {
     VALUE type = ID2SYM(rb_intern(pm_diagnostic_type(diagnostic)));
-    VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), each_data->encoding));
-    VALUE location = PARSER_LOCATION(each_data->source, each_data->freeze, pm_diagnostic_location(diagnostic));
+    VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(pm_diagnostic_message(diagnostic), encoding));
+    VALUE location = PARSER_LOCATION(source, freeze, pm_diagnostic_location(diagnostic));
 
     pm_warning_level_t warning_level = pm_diagnostic_warning_level(diagnostic);
     VALUE level = Qnil;
@@ -796,8 +763,7 @@ parser_warnings_each(const pm_diagnostic_t *diagnostic, void *data) {
     }
 
     VALUE argv[] = { type, message, location, level };
-    VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseWarning, each_data->freeze);
-    rb_ary_push(each_data->warnings, value);
+    return rb_class_new_instance_freeze(4, argv, rb_cPrismParseWarning, freeze);
 }
 
 /**
@@ -807,8 +773,9 @@ static VALUE
 parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
     VALUE warnings = rb_ary_new_capa(pm_parser_warnings_size(parser));
 
-    parser_warnings_each_data_t each_data = { warnings, encoding, source, freeze };
-    pm_parser_warnings_each(parser, parser_warnings_each, &each_data);
+    for (const pm_diagnostic_t *warning = pm_parser_warnings_first(parser); warning != NULL; warning = pm_diagnostic_next(warning)) {
+        rb_ary_push(warnings, parser_warning(warning, encoding, source, freeze));
+    }
 
     if (freeze) rb_obj_freeze(warnings);
     return warnings;
@@ -983,8 +950,9 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o
 
     result_t result = check_raise_error_option(parser, options, path_encoding);
     if (result.type == RESULT_OK) {
-        /* Update the Source object with the correct encoding and line offsets,
-         * which are only available after pm_parse() completes. */
+        /* Update the Source object with the correct encoding, line offsets,
+         * and source hash, which are only available after pm_parse()
+         * completes. */
         rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
         rb_enc_associate(source_string, encoding);
 
@@ -993,6 +961,8 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o
             rb_ary_store(offsets, (long) index, ULONG2NUM(line_offsets->offsets[index]));
         }
 
+        rb_funcall(source, rb_intern("replace_checksum"), 1, ULL2NUM(pm_parser_checksum(parser)));
+
         if (pm_options_freeze(options)) {
             rb_obj_freeze(source_string);
             rb_obj_freeze(offsets);
@@ -1678,6 +1648,20 @@ string_query_method_name_p(VALUE self, VALUE string) {
     return string_query(pm_string_query_method_name(source, RSTRING_LEN(string), rb_enc_get(string)->name));
 }
 
+/**
+ * :markup: markdown
+ * call-seq:
+ *   keyword?(string) -> bool
+ *
+ * Returns true if the string is a valid keyword, such as `if`, `defined?`, or
+ * `__FILE__`.
+ */
+static VALUE
+string_query_keyword_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_keyword(source, RSTRING_LEN(string), rb_enc_get(string)->name));
+}
+
 /******************************************************************************/
 /* Initialization of the extension                                            */
 /******************************************************************************/
@@ -1778,6 +1762,7 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismStringQuery, "local?", string_query_local_p, 1);
     rb_define_singleton_method(rb_cPrismStringQuery, "constant?", string_query_constant_p, 1);
     rb_define_singleton_method(rb_cPrismStringQuery, "method_name?", string_query_method_name_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "keyword?", string_query_keyword_p, 1);
 
     Init_prism_api_node();
 }

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "2.0.0.rc1"
+#define EXPECTED_PRISM_VERSION "2.0.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>

data/include/prism/arena.h

@@ -12,6 +12,10 @@
 
 #include <stddef.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * An opaque pointer to an arena that is used for allocations.
  */
@@ -34,4 +38,8 @@ PRISM_NODISCARD PRISM_EXPORTED_FUNCTION pm_arena_t * pm_arena_new(void);
  */
 PRISM_EXPORTED_FUNCTION void pm_arena_free(pm_arena_t *arena) PRISM_NONNULL(1);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

data/include/prism/arena.hpp

@@ -0,0 +1,42 @@
+/**
+ * @file arena.hpp
+ *
+ * Ownership of arenas from C++.
+ */
+#ifndef PRISM_ARENA_HPP
+#define PRISM_ARENA_HPP
+
+#include "prism/compiler/cplusplus.hpp"
+
+#include "prism/arena.h"
+
+#include <memory>
+
+namespace prism {
+
+/** Frees an arena with pm_arena_free. */
+struct arena_deleter {
+    /**
+     * Frees the given arena.
+     *
+     * @param arena The arena to free.
+     */
+    void operator()(pm_arena_t *arena) const noexcept { pm_arena_free(arena); }
+};
+
+/** An owned arena. */
+using unique_arena = std::unique_ptr<pm_arena_t, arena_deleter>;
+
+/**
+ * Allocate a new arena. Allocation failure aborts, so this never throws.
+ *
+ * @return A unique_arena managing the newly allocated arena.
+ */
+[[nodiscard]] inline unique_arena
+make_arena() noexcept {
+    return unique_arena(pm_arena_new());
+}
+
+}
+
+#endif

data/include/prism/ast.h

@@ -27,6 +27,10 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * This enum represents every type of token in the Ruby source.
  */
@@ -1103,7 +1107,9 @@ typedef struct pm_node {
 
     /**
      * The unique identifier for this node, which is deterministic based on the
-     * source. It is used to identify unique nodes across parses.
+     * source. It is used to identify unique nodes across parses. Nodes created
+     * outside of parsing have no position in the source and can use 0, which
+     * no parsed node uses.
      */
     uint32_t node_id;
 
@@ -10368,4 +10374,8 @@ PRISM_EXPORTED_FUNCTION pm_yield_node_t * pm_yield_node_new(pm_arena_t *arena, u
  */
 #define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS 0
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif