prism 2.0.0.rc1-x86_64-linux-gnu → 2.0.0.rc2-x86_64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc11883eb4ab950f920cf9fafa80b2edde172ccffd5c3ddd485f3f785206b370
-  data.tar.gz: 7e460ef605ed206d9ba12247ece6240d272f24e5a080e2df6a46fb9adfa4e1cb
+  metadata.gz: 98349b6e074f7483057352eec27a0d76d3d012cd04ba771c12319beb07786040
+  data.tar.gz: d0cc79a7745253e2ae1987e7a5df08a63e63a350f1a44a05fc4ce8a0563a234a
 SHA512:
-  metadata.gz: 57d4a9a445c1abce8ddd30f7b8e53d0e5bb46742c08ab44a2309b672e1afb3ad9144de4f583abe6caa653370168d993aaf29ac5615003984c874dfef943ce4dd
-  data.tar.gz: c12fce41b7fab93c320288a6011044744b7a460f2cef6eb61d221589a1afe849658ae5272cb187067a3741d137ac26cfa6003292e2a98f2bce85ade802712e9b
+  metadata.gz: f3cab7ac92216d5b7546e6f4fcb4fcfeff2262ee6540b513d84cc0d28d99c74c7697d3803477d62b81c233ed9aaeb942c131f19b18f58b08b908ad0662afeebb
+  data.tar.gz: cbfcf5a6da8e0be1937906da87d5a38dafe0a67f36f94752b1a37a908904f4a71c82a78ab717ce44d21014db30db8d5dd3d89e9b0dc699957f0f52fde844682a

data/CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 - Precompiled platform-specific variants on rubygems.org.
 - Many documentation improvements, including a new playground.
 - Prism now ships accurate and complete RBS and RBI definitions.
+- The license is now owned by the Ruby developers.
 
 ### Added
 
@@ -29,6 +30,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 - Allow pattern matching with prism nodes to match on location slices, like `node in Prism::StringNode[content: "foo"]`.
 - `Prism.parse_file` and similar now accept anything responding to `to_path` (like `Pathname`) directly.
 - `attach_comments` was added as a valid option. Each comment is attached to the node that ends latest on the comment's line before it, otherwise to the innermost node that contains it (or to that node's statements, if only whitespace and comments come between the comment and the statements), otherwise to the top-level statements. The attached comments are available through `Node#comments` and through `pm_node_comments_size` and `pm_node_comment` in the C API, as well as `pm_parser_comment_attachment`, `pm_parser_comment_ranges_size`, and `pm_parser_comment_range`, which expose the attachments grouped by node. When the option is set, the comments on the parse result are sorted by start offset.
+- `StringQuery#keyword?` was added to check if the string query matches a keyword.
+- `Source#checksum` was added to retrieve the checksum of the source content.
 
 ### Changed
 
@@ -71,6 +74,7 @@ Note that for this major version, almost every change listed below is breaking.
 - Most allocations are now performed through an arena.
 - `newline_list` was renamed to `list_offset_list`.
 - `Diagnostic::message` now returns `&[u8]` instead of `&str`.
+- `StringQuery` is now exposed.
 
 #### JavaScript
 

data/CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at opensource@shopify.com. All
+reported by contacting the project team at ruby-core@ruby-lang.org. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

data/LICENSE.md

@@ -1,4 +1,4 @@
-Copyright 2022-present, Shopify Inc.
+Copyright 2022-present, Ruby developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

data/docs/serialization.md

@@ -106,6 +106,7 @@ The header is structured like the following table:
 | `1` | 1 indicates only semantics fields were serialized, 0 indicates all fields were serialized (including location fields) |
 | string | the encoding name |
 | varsint | the start line |
+| `8` | the checksum of the source up to the end of the `__END__` line in native endian order, see `pm_parser_checksum` |
 | varuint | number of newline offsets |
 | varuint* | newline offsets |
 | varuint | number of comments |

data/ext/prism/api_node.c

@@ -253,7 +253,7 @@ pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, VALUE comments,
         offsets = rb_str_new((const char *) line_offsets->offsets, line_offsets->size * sizeof(uint32_t));
     }
 
-    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 6, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, comment_attachments, comment_ranges);
+    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 7, source_string, LONG2NUM(pm_parser_start_line(parser)), offsets, comments, comment_attachments, comment_ranges, ULL2NUM(pm_parser_checksum(parser)));
     if (freeze) rb_obj_freeze(source);
 
     return source;

data/ext/prism/extension.c

@@ -983,8 +983,9 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o
 
     result_t result = check_raise_error_option(parser, options, path_encoding);
     if (result.type == RESULT_OK) {
-        /* Update the Source object with the correct encoding and line offsets,
-         * which are only available after pm_parse() completes. */
+        /* Update the Source object with the correct encoding, line offsets,
+         * and source hash, which are only available after pm_parse()
+         * completes. */
         rb_encoding *encoding = rb_enc_find(pm_parser_encoding_name(parser));
         rb_enc_associate(source_string, encoding);
 
@@ -993,6 +994,8 @@ parse_lex_input(const uint8_t *input, size_t input_length, const pm_options_t *o
             rb_ary_store(offsets, (long) index, ULONG2NUM(line_offsets->offsets[index]));
         }
 
+        rb_funcall(source, rb_intern("replace_checksum"), 1, ULL2NUM(pm_parser_checksum(parser)));
+
         if (pm_options_freeze(options)) {
             rb_obj_freeze(source_string);
             rb_obj_freeze(offsets);
@@ -1678,6 +1681,20 @@ string_query_method_name_p(VALUE self, VALUE string) {
     return string_query(pm_string_query_method_name(source, RSTRING_LEN(string), rb_enc_get(string)->name));
 }
 
+/**
+ * :markup: markdown
+ * call-seq:
+ *   keyword?(string) -> bool
+ *
+ * Returns true if the string is a valid keyword, such as `if`, `defined?`, or
+ * `__FILE__`.
+ */
+static VALUE
+string_query_keyword_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_keyword(source, RSTRING_LEN(string), rb_enc_get(string)->name));
+}
+
 /******************************************************************************/
 /* Initialization of the extension                                            */
 /******************************************************************************/
@@ -1778,6 +1795,7 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismStringQuery, "local?", string_query_local_p, 1);
     rb_define_singleton_method(rb_cPrismStringQuery, "constant?", string_query_constant_p, 1);
     rb_define_singleton_method(rb_cPrismStringQuery, "method_name?", string_query_method_name_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "keyword?", string_query_keyword_p, 1);
 
     Init_prism_api_node();
 }

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "2.0.0.rc1"
+#define EXPECTED_PRISM_VERSION "2.0.0.rc2"
 
 #include <ruby.h>
 #include <ruby/encoding.h>

data/include/prism/compiler/accel.h

@@ -4,6 +4,8 @@
 #ifndef PRISM_COMPILER_ACCEL_H
 #define PRISM_COMPILER_ACCEL_H
 
+#include "prism/compiler/endianness.h"
+
 /**
  * Platform detection for SIMD/fast-path implementations. At most one of these
  * macros is defined, selecting the best available vectorization strategy.
@@ -17,7 +19,7 @@
 #   define PRISM_HAS_NEON
 #elif (defined(__x86_64__) && defined(__SSE2__)) || (defined(_MSC_VER) && defined(_M_X64))
 #   define PRISM_HAS_SSE2
-#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#elif defined(PRISM_LITTLE_ENDIAN_COMPTIME)
 #   define PRISM_HAS_SWAR
 #endif
 

data/include/prism/compiler/bswap.h

@@ -0,0 +1,46 @@
+/**
+ * @file compiler/bswap.h
+ */
+#ifndef PRISM_COMPILER_BSWAP_H
+#define PRISM_COMPILER_BSWAP_H
+
+/**
+ * Byte swap utilities for 32-bit and 64-bit integers.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#define pm_bswap32(x_) __builtin_bswap32((x_))
+#define pm_bswap64(x_) __builtin_bswap64((x_))
+#elif defined(_MSC_VER)
+#include <stdlib.h>
+
+#define pm_bswap32(x_) _byteswap_ulong((x_))
+#define pm_bswap64(x_) _byteswap_uint64((x_))
+#else
+#include <stdint.h>
+
+static uint32_t
+pm_bswap32(uint32_t x) {
+    return (
+      ((x >> 24) & 0x000000FFUL) |
+      ((x << 24) & 0xFF000000UL) |
+      ((x >> 8)  & 0x0000FF00UL) |
+      ((x << 8)  & 0x00FF0000UL)
+    );
+}
+
+static uint64_t
+pm_bswap64(uint64_t x) {
+    return (
+      ((x >> 56) & 0x00000000000000FFULL) |
+      ((x << 56) & 0xFF00000000000000ULL) |
+      ((x >> 40) & 0x000000000000FF00ULL) |
+      ((x << 40) & 0x00FF000000000000ULL) |
+      ((x >> 24) & 0x0000000000FF0000ULL) |
+      ((x << 24) & 0x0000FF0000000000ULL) |
+      ((x >> 8)  & 0x00000000FF000000ULL) |
+      ((x << 8)  & 0x000000FF00000000ULL)
+    );
+}
+#endif
+
+#endif

data/include/prism/compiler/endianness.h

@@ -0,0 +1,34 @@
+/**
+ * @file compiler/endianness.h
+ */
+#ifndef PRISM_COMPILER_ENDIANNESS_H
+#define PRISM_COMPILER_ENDIANNESS_H
+
+/**
+ * Detect endianness at compile-time through the use of predefined macros.
+ */
+#if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define PRISM_LITTLE_ENDIAN_COMPTIME
+#endif
+
+/**
+ * Determine endianness at runtime, either through predefined macros or through
+ * a runtime check that is usually folded.
+ */
+#if defined(PRISM_LITTLE_ENDIAN_COMPTIME)
+#define PRISM_LITTLE_ENDIAN_RUNTIME 1
+#elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define PRISM_LITTLE_ENDIAN_RUNTIME 0
+#else
+#include <stdint.h>
+
+static inline int
+pm_little_endian(void) {
+    const union { uint32_t u; uint8_t c[4]; } one = { 1 };
+    return one.c[0];
+}
+
+#define PRISM_LITTLE_ENDIAN_RUNTIME pm_little_endian()
+#endif
+
+#endif

data/include/prism/internal/buffer.h

@@ -52,6 +52,9 @@ void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
 /* Append a double to the buffer. */
 void pm_buffer_append_double(pm_buffer_t *buffer, double value);
 
+/* Append a 64-bit unsigned integer to the buffer in native endian order. */
+void pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value);
+
 /* Append a unicode codepoint to the buffer. */
 bool pm_buffer_append_unicode_codepoint(pm_buffer_t *buffer, uint32_t value);
 

data/include/prism/internal/parser.h

@@ -715,9 +715,23 @@ struct pm_parser_t {
     /* The current local scope. */
     pm_scope_t *current_scope;
 
+    /*
+     * Scopes that have been popped, kept for reuse by the next push. Scopes
+     * are allocated from the metadata arena, so they are never freed
+     * individually.
+     */
+    pm_scope_t *free_scopes;
+
     /* The current parsing context. */
     pm_context_node_t *current_context;
 
+    /*
+     * Context nodes that have been popped, kept for reuse by the next push.
+     * Context nodes are allocated from the metadata arena, so they are never
+     * freed individually.
+     */
+    pm_context_node_t *free_contexts;
+
     /*
      * The encoding functions for the current file is attached to the parser as
      * it's parsing so that it can change with a magic comment.
@@ -981,4 +995,10 @@ void pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *sourc
  */
 void pm_parser_cleanup(pm_parser_t *parser);
 
+/*
+ * Returns true if the bytes between start and end spell a keyword, using the
+ * same lookup that the lexer does.
+ */
+bool pm_keyword_p(const uint8_t *start, const uint8_t *end);
+
 #endif

data/include/prism/internal/xxh3.h

@@ -0,0 +1,16 @@
+#ifndef PRISM_INTERNAL_XXH3_H
+#define PRISM_INTERNAL_XXH3_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * Computes the XXH3 digest of the given input.
+ *
+ * @param input The input data to hash.
+ * @param length The length of the input data.
+ * @return The 64-bit XXH3 hash of the input data.
+ */
+uint64_t pm_xxh3(const uint8_t *input, size_t length);
+
+#endif

data/include/prism/parser.h

@@ -177,6 +177,27 @@ PRISM_EXPORTED_FUNCTION const pm_line_offset_list_t * pm_parser_line_offsets(con
  */
 PRISM_EXPORTED_FUNCTION const pm_location_t * pm_parser_data_loc(const pm_parser_t *parser) PRISM_NONNULL(1);
 
+/**
+ * Returns the checksum of the source code that was parsed. The data section
+ * after an __END__ marker is not code, so the hash covers the source only up to
+ * and including the newline that ends the __END__ line. The __END__ marker is
+ * only found by parsing, so this must be called after pm_parse.
+ *
+ * The hash is computed from the source on each call rather than during
+ * parsing, so that parses that never ask for it do not pay for it. This
+ * requires the source to still be alive, and callers that need the value more
+ * than once should keep it.
+ *
+ * The hash is currently the 64-bit variant of XXH3 with a seed of 0. It is not
+ * cryptographic, and the algorithm may change between versions of prism, so
+ * hashes should only be compared against other hashes computed by the same
+ * version of prism.
+ *
+ * @param parser the parser whose checksum we want to get
+ * @returns the checksum of the source code that was parsed
+ */
+PRISM_EXPORTED_FUNCTION uint64_t pm_parser_checksum(const pm_parser_t *parser) PRISM_NONNULL(1);
+
 /**
  * Returns whether the given parser is continuable, meaning that it could become
  * valid if more input were appended, as opposed to being definitively invalid.

data/include/prism/string_query.h

@@ -60,4 +60,15 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_constant(const uint8_t
  */
 PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
 
+/**
+ * Check that the slice is a keyword.
+ *
+ * @param source The source to check.
+ * @param length The length of the source.
+ * @param encoding_name The name of the encoding of the source.
+ * @returns PM_STRING_QUERY_TRUE if the query is true, PM_STRING_QUERY_FALSE if
+ *   the query is false, and PM_STRING_QUERY_ERROR if the encoding was invalid.
+ */
+PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_keyword(const uint8_t *source, size_t length, const char *encoding_name) PRISM_NONNULL(1, 3);
+
 #endif

data/include/prism/version.h

@@ -26,7 +26,7 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "2.0.0.rc1"
+#define PRISM_VERSION "2.0.0.rc2"
 
 /**
  * The prism version and the serialization format.

data/lib/prism/ffi.rb

@@ -124,6 +124,7 @@ module Prism # :nodoc:
       "pm_string_query_local",
       "pm_string_query_constant",
       "pm_string_query_method_name",
+      "pm_string_query_keyword",
       []
     )
 
@@ -722,6 +723,11 @@ module Prism # :nodoc:
         query(LibRubyParser.pm_string_query_method_name(string, string.bytesize, string.encoding.name))
       end
 
+      # Mirrors the C extension's StringQuery::keyword? method.
+      def keyword?(string)
+        query(LibRubyParser.pm_string_query_keyword(string, string.bytesize, string.encoding.name))
+      end
+
       private
 
       # Parse the enum result and return an appropriate boolean.