rbs 4.0.0.dev.4 → 4.1.0.pre.2

data/include/rbs/defines.h

@@ -32,6 +32,24 @@
 #define RBS_ATTRIBUTE_FORMAT(string_index, argument_index)
 #endif
 
+/**
+ * Support RBS_LIKELY and RBS_UNLIKELY to help the compiler optimize its
+ * branch predication.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+/** The compiler should predicate that this branch will be taken. */
+#define RBS_LIKELY(x) __builtin_expect(!!(x), 1)
+
+/** The compiler should predicate that this branch will not be taken. */
+#define RBS_UNLIKELY(x) __builtin_expect(!!(x), 0)
+#else
+/** Void because this platform does not support branch prediction hints. */
+#define RBS_LIKELY(x) (x)
+
+/** Void because this platform does not support branch prediction hints. */
+#define RBS_UNLIKELY(x) (x)
+#endif
+
 /**
  * We use -Wimplicit-fallthrough to guard potentially unintended fall-through between cases of a switch.
  * Use RBS_FALLTHROUGH to explicitly annotate cases where the fallthrough is intentional.
@@ -56,4 +74,26 @@
 #define NODISCARD __attribute__((warn_unused_result))
 #endif
 
+/**
+ * Mark a function or variable as potentially unused to suppress compiler warnings.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#define RBS_ATTRIBUTE_UNUSED __attribute__((unused))
+#else
+#define RBS_ATTRIBUTE_UNUSED
+#endif
+
+/**
+ * Nullability annotations for pointer types.
+ * Clang supports _Nullable and _Nonnull to indicate whether a pointer may be NULL.
+ * On other compilers, these expand to nothing.
+ */
+#ifdef __clang__
+#define RBS_NULLABLE _Nullable
+#define RBS_NONNULL _Nonnull
+#else
+#define RBS_NULLABLE
+#define RBS_NONNULL
+#endif
+
 #endif

data/include/rbs/lexer.h

@@ -30,6 +30,7 @@ enum RBSTokenType {
     pBANG,     /* ! */
     pQUESTION, /* ? */
     pLT,       /* < */
+    pGT,       /* > */
     pEQ,       /* = */
 
     kALIAS,        /* alias */
@@ -39,6 +40,7 @@ enum RBSTokenType {
     kBOOL,         /* bool */
     kBOT,          /* bot */
     kCLASS,        /* class */
+    kCLASSALIAS,   /* class-alias */
     kDEF,          /* def */
     kEND,          /* end */
     kEXTEND,       /* extend */
@@ -48,6 +50,7 @@ enum RBSTokenType {
     kINSTANCE,     /* instance */
     kINTERFACE,    /* interface */
     kMODULE,       /* module */
+    kMODULEALIAS,  /* module-alias */
     kNIL,          /* nil */
     kOUT,          /* out */
     kPREPEND,      /* prepend */
@@ -67,6 +70,7 @@ enum RBSTokenType {
     kATRBS,        /* @rbs */
     kSKIP,         /* skip */
     kRETURN,       /* return */
+    kMODULESELF,   /* module-self */
 
     tLIDENT,    /* Identifiers starting with lower case */
     tUIDENT,    /* Identifiers starting with upper case */
@@ -123,20 +127,26 @@ typedef struct {
  * The lexer state is the curren token.
  *
  * ```
- * ... "a string token"
- *    ^                      start position
- *          ^                current position
- *     ~~~~~~                Token => "a str
+ #.   0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6
+ * ... " a   s t r i n g   t o k e n "
+ *    ^                                   start position (0)
+ *                ^                       current position (6)
+ *                 ^                      current character ('i', bytes = 1)
+ *     ~~~~~~~~~~~                        Token => "a str
  * ```
  * */
 typedef struct {
     rbs_string_t string;
-    int start_pos;            /* The character position that defines the start of the input */
-    int end_pos;              /* The character position that defines the end of the input */
-    rbs_position_t current;   /* The current position */
-    rbs_position_t start;     /* The start position of the current token */
+    int start_pos;          /* The byte position that defines the start of the input */
+    int end_pos;            /* The byte position that defines the end of the input */
+    rbs_position_t current; /* The current position: just before the current_character */
+    rbs_position_t start;   /* The start position of the current token */
+
+    unsigned int current_code_point; /* Current character code point */
+    size_t current_character_bytes;  /* Current character byte length (0 or 1~4) */
+
     bool first_token_of_line; /* This flag is used for tLINECOMMENT */
-    unsigned int last_char;   /* Last peeked character */
+
     const rbs_encoding_t *encoding;
 } rbs_lexer_t;
 
@@ -156,15 +166,23 @@ int rbs_token_bytes(rbs_token_t tok);
 const char *rbs_token_type_str(enum RBSTokenType type);
 
 /**
- * Read next character.
+ * Returns the next character.
  * */
 unsigned int rbs_peek(rbs_lexer_t *lexer);
 
 /**
- * Skip one character.
+ * Advances the current position by one character.
  * */
 void rbs_skip(rbs_lexer_t *lexer);
 
+/**
+ * Read next character and store the codepoint and byte length to the given pointers.
+ * 
+ * This doesn't update the lexer state.
+ * Returns `true` if succeeded, or `false` if reached to EOF.
+ * */
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *bytes);
+
 /**
  * Skip n characters.
  * */
@@ -184,4 +202,6 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer);
 
 void rbs_print_token(rbs_token_t tok);
 
+void rbs_print_lexer(rbs_lexer_t *lexer);
+
 #endif

data/include/rbs/location.h

@@ -1,59 +1,40 @@
 #ifndef RBS__RBS_LOCATION_H
 #define RBS__RBS_LOCATION_H
 
-#include "lexer.h"
-
-#include "rbs/util/rbs_constant_pool.h"
 #include "rbs/util/rbs_allocator.h"
 
-typedef struct {
-    int start;
-    int end;
-} rbs_loc_range;
+#define RBS_LOCATION_NULL_RANGE ((rbs_location_range) { -1, -1, -1, -1 })
+#define RBS_LOCATION_NULL_RANGE_P(rg) ((rg).start_char == -1)
+
+/**
+ * Converts a lexer range (rbs_range_t) to an AST location range (rbs_location_range) by extracting character and byte positions.
+ */
+#define RBS_RANGE_LEX2AST(rg) ((rbs_location_range) { .start_char = (rg).start.char_pos, .start_byte = (rg).start.byte_pos, .end_char = (rg).end.char_pos, .end_byte = (rg).end.byte_pos })
 
 typedef struct {
-    rbs_constant_id_t name;
-    rbs_loc_range rg;
-} rbs_loc_entry;
+    int start_char;
+    int start_byte;
 
-typedef unsigned int rbs_loc_entry_bitmap;
+    int end_char;
+    int end_byte;
+} rbs_location_range;
 
-// The flexible array always allocates, but it's okay.
-// This struct is not allocated when the `rbs_loc` doesn't have children.
-typedef struct {
-    unsigned short len;
-    unsigned short cap;
-    rbs_loc_entry_bitmap required_p;
-    rbs_loc_entry entries[1];
-} rbs_loc_children;
-
-typedef struct rbs_location {
-    rbs_range_t rg;
-    rbs_loc_children *children;
-} rbs_location_t;
-
-typedef struct rbs_location_list_node {
-    rbs_location_t *loc;
-    struct rbs_location_list_node *next;
-} rbs_location_list_node_t;
-
-typedef struct rbs_location_list {
+typedef struct rbs_location_range_list_node {
+    rbs_location_range range;
+    struct rbs_location_range_list_node *next;
+} rbs_location_range_list_node_t;
+
+typedef struct rbs_location_range_list {
     rbs_allocator_t *allocator;
-    rbs_location_list_node_t *head;
-    rbs_location_list_node_t *tail;
+    struct rbs_location_range_list_node *head;
+    struct rbs_location_range_list_node *tail;
     size_t length;
-} rbs_location_list_t;
-
-void rbs_loc_alloc_children(rbs_allocator_t *, rbs_location_t *loc, size_t capacity);
-void rbs_loc_add_required_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r);
-void rbs_loc_add_optional_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r);
+} rbs_location_range_list_t;
 
 /**
- * Allocate new rbs_location_t object through the given allocator.
- * */
-rbs_location_t *rbs_location_new(rbs_allocator_t *, rbs_range_t rg);
-
-rbs_location_list_t *rbs_location_list_new(rbs_allocator_t *allocator);
-void rbs_location_list_append(rbs_location_list_t *list, rbs_location_t *loc);
+ * Allocate new rbs_location_range_list_t object through the given allocator.
+ */
+rbs_location_range_list_t *rbs_location_range_list_new(rbs_allocator_t *allocator);
+void rbs_location_range_list_append(rbs_location_range_list_t *list, rbs_location_range range);
 
 #endif

data/include/rbs/parser.h

@@ -27,9 +27,9 @@ typedef struct rbs_comment_t {
     rbs_position_t start;
     rbs_position_t end;
 
-    size_t line_size;
-    size_t line_count;
-    rbs_token_t *tokens;
+    size_t line_tokens_capacity;
+    size_t line_tokens_count;
+    rbs_token_t *line_tokens;
 
     struct rbs_comment_t *next_comment;
 } rbs_comment_t;
@@ -44,7 +44,7 @@ typedef struct rbs_error_t {
  * An RBS parser is a LL(3) parser.
  * */
 typedef struct {
-    rbs_lexer_t *rbs_lexer_t;
+    rbs_lexer_t *lexer;
 
     rbs_token_t current_token;
     rbs_token_t next_token;  /* The first lookahead token */
@@ -126,8 +126,8 @@ rbs_ast_comment_t *rbs_parser_get_comment(rbs_parser_t *parser, int subject_line
 
 void rbs_parser_set_error(rbs_parser_t *parser, rbs_token_t tok, bool syntax_error, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(4, 5);
 
-bool rbs_parse_type(rbs_parser_t *parser, rbs_node_t **type);
-bool rbs_parse_method_type(rbs_parser_t *parser, rbs_method_type_t **method_type);
+bool rbs_parse_type(rbs_parser_t *parser, rbs_node_t **type, bool void_allowed, bool self_allowed, bool classish_allowed);
+bool rbs_parse_method_type(rbs_parser_t *parser, rbs_method_type_t **method_type, bool require_eof, bool classish_allowed);
 bool rbs_parse_signature(rbs_parser_t *parser, rbs_signature_t **signature);
 
 bool rbs_parse_type_params(rbs_parser_t *parser, bool module_type_params, rbs_node_list_t **params);

data/include/rbs/serialize.h

@@ -0,0 +1,39 @@
+/*----------------------------------------------------------------------------*/
+/* This file is generated by the templates/template.rb script and should not  */
+/* be modified manually.                                                      */
+/* To change the template see                                                 */
+/* templates/include/rbs/serialize.h.erb                                      */
+/*----------------------------------------------------------------------------*/
+
+#ifndef RBS__SERIALIZE_H
+#define RBS__SERIALIZE_H
+
+#include "rbs/ast.h"
+#include "rbs/string.h"
+#include "rbs/util/rbs_allocator.h"
+#include "rbs/util/rbs_constant_pool.h"
+
+/**
+ * Serialize a parsed AST node into a compact, portable binary buffer.
+ *
+ * The format is consumed by RBS::WASM::Deserializer on the Ruby side, which
+ * rebuilds the same `RBS::AST` objects that the C extension would have built
+ * directly. This is what lets RBS run on Ruby implementations that cannot load
+ * the C extension (notably JRuby): the parser runs inside WebAssembly, produces
+ * this buffer, and the host reconstructs the tree in pure Ruby.
+ *
+ * The buffer is allocated from `allocator`, so its lifetime is tied to that
+ * allocator. `constant_pool` must be the pool the node was parsed with; it is
+ * used to resolve interned symbol/identifier ids back into their bytes.
+ *
+ * See `docs/wasm_serialization.md` for the wire format.
+ */
+rbs_string_t rbs_serialize_node(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_t *node);
+
+/**
+ * Like rbs_serialize_node, but for a bare node list (e.g. the result of
+ * rbs_parse_type_params). Decoded by RBS::WASM::Deserializer.deserialize_node_list.
+ */
+rbs_string_t rbs_serialize_node_list(rbs_allocator_t *allocator, rbs_constant_pool_t *constant_pool, rbs_node_list_t *list);
+
+#endif

data/include/rbs/string.h

@@ -44,6 +44,4 @@ size_t rbs_string_len(const rbs_string_t self);
  */
 bool rbs_string_equal(const rbs_string_t lhs, const rbs_string_t rhs);
 
-unsigned int rbs_utf8_string_to_codepoint(const rbs_string_t string);
-
 #endif

data/include/rbs/util/rbs_allocator.h

@@ -3,19 +3,40 @@
 
 #include <stddef.h>
 
-#ifndef alignof
-#if defined(__GNUC__) || defined(__clang__)
-#define alignof(type) __alignof__(type)
+/* Include stdalign.h for C11 and later for alignof support */
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#include <stdalign.h>
+#endif
+
+/*
+ * Define a portable alignment macro that works across all supported environments
+ */
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+/* C11 or later - use _Alignof directly (always available in C11+) */
+#define rbs_alignof(type) _Alignof(type)
+#elif defined(__cplusplus) && __cplusplus >= 201103L
+/* C++11 or later has alignof keyword */
+#define rbs_alignof(type) alignof(type)
+#elif defined(__GNUC__) || defined(__clang__)
+/* GCC and Clang provide __alignof__ */
+#define rbs_alignof(type) __alignof__(type)
 #elif defined(_MSC_VER)
-#define alignof(type) __alignof(type)
+/* MSVC provides __alignof */
+#define rbs_alignof(type) __alignof(type)
 #else
-// Fallback using offset trick
-#define alignof(type) offsetof(struct { char c; type member; }, member)
-#endif
+/* Fallback using offset trick for other compilers */
+#define rbs_alignof(type) offsetof( \
+    struct { char c; type member; },                     \
+    member                          \
+)
 #endif
 
-struct rbs_allocator;
-typedef struct rbs_allocator rbs_allocator_t;
+typedef struct rbs_allocator {
+    // The head of a linked list of pages, starting with the most recently allocated page.
+    struct rbs_allocator_page *page;
+
+    size_t default_page_payload_size;
+} rbs_allocator_t;
 
 rbs_allocator_t *rbs_allocator_init(void);
 void rbs_allocator_free(rbs_allocator_t *);
@@ -26,13 +47,13 @@ void *rbs_allocator_calloc_impl(rbs_allocator_t *, size_t count, size_t size, si
 void *rbs_allocator_realloc_impl(rbs_allocator_t *, void *ptr, size_t old_size, size_t new_size, size_t alignment);
 
 // Use this when allocating memory for a single instance of a type.
-#define rbs_allocator_alloc(allocator, type) ((type *) rbs_allocator_malloc_impl((allocator), sizeof(type), alignof(type)))
+#define rbs_allocator_alloc(allocator, type) ((type *) rbs_allocator_malloc_impl((allocator), sizeof(type), rbs_alignof(type)))
 // Use this when allocating memory that will be immediately written to in full.
 // Such as allocating strings
-#define rbs_allocator_alloc_many(allocator, count, type) ((type *) rbs_allocator_malloc_many_impl((allocator), (count), sizeof(type), alignof(type)))
+#define rbs_allocator_alloc_many(allocator, count, type) ((type *) rbs_allocator_malloc_many_impl((allocator), (count), sizeof(type), rbs_alignof(type)))
 // Use this when allocating memory that will NOT be immediately written to in full.
 // Such as allocating buffers
-#define rbs_allocator_calloc(allocator, count, type) ((type *) rbs_allocator_calloc_impl((allocator), (count), sizeof(type), alignof(type)))
-#define rbs_allocator_realloc(allocator, ptr, old_size, new_size, type) ((type *) rbs_allocator_realloc_impl((allocator), (ptr), (old_size), (new_size), alignof(type)))
+#define rbs_allocator_calloc(allocator, count, type) ((type *) rbs_allocator_calloc_impl((allocator), (count), sizeof(type), rbs_alignof(type)))
+#define rbs_allocator_realloc(allocator, ptr, old_size, new_size, type) ((type *) rbs_allocator_realloc_impl((allocator), (ptr), (old_size), (new_size), rbs_alignof(type)))
 
 #endif

data/include/rbs/util/rbs_assert.h

@@ -4,6 +4,17 @@
 #include "rbs/defines.h"
 #include <stdbool.h>
 
-void rbs_assert(bool condition, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(2, 3);
+/**
+ * RBS_ASSERT macro that calls rbs_assert in debug builds and is removed in release builds.
+ * In debug mode, it forwards all arguments to the rbs_assert function.
+ * In release mode, it expands to nothing.
+ */
+#ifdef NDEBUG
+#define RBS_ASSERT(condition, ...) ((void) 0)
+#else
+#define RBS_ASSERT(condition, ...) rbs_assert_impl(condition, __VA_ARGS__)
+#endif
+
+void rbs_assert_impl(bool condition, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(2, 3);
 
 #endif

data/include/rbs/util/rbs_constant_pool.h

@@ -81,9 +81,6 @@ typedef struct {
     uint32_t capacity;
 } rbs_constant_pool_t;
 
-// A global constant pool for storing permenant keywords, such as the names of location children in `parser.c`.
-extern rbs_constant_pool_t *RBS_GLOBAL_CONSTANT_POOL;
-
 /**
  * Initialize a new constant pool with a given capacity.
  *

data/include/rbs/util/rbs_encoding.h

@@ -6,6 +6,8 @@
 #ifndef RBS_RBS_ENCODING_H
 #define RBS_RBS_ENCODING_H
 
+#include "rbs/defines.h"
+
 #include <assert.h>
 #include <stdbool.h>
 #include <stddef.h>

data/include/rbs/util/rbs_unescape.h

@@ -4,6 +4,7 @@
 #include <stddef.h>
 #include "rbs/util/rbs_allocator.h"
 #include "rbs/string.h"
+#include "rbs/util/rbs_encoding.h"
 
 /**
  * Receives `rbs_parser_t` and `range`, which represents a string token or symbol token, and returns a string VALUE.
@@ -18,6 +19,6 @@
  *
  * @returns A new owned string that will be freed when the allocator is freed.
  * */
-rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input);
+rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input, const rbs_encoding_t *encoding);
 
 #endif // RBS_RBS_UNESCAPE_H

data/include/rbs.h

@@ -1,6 +1,14 @@
 #ifndef RBS_H
 #define RBS_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "rbs/parser.h"
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

data/lib/rbs/annotate/rdoc_annotator.rb

@@ -13,25 +13,22 @@ module RBS
         @include_filename = true
       end
 
-      def annotate_file(path, preserve:)
-        content = path.read()
+      def annotate_file(path, preserve: true)
+        buffer, _, decls = Parser.parse_signature(path.read())
 
-        _, _, decls = Parser.parse_signature(content)
+        rewriter = Rewriter.new(buffer)
+        annotate_decls(decls, rewriter)
 
-        annotate_decls(decls)
-
-        path.open("w") do |io|
-          Writer.new(out: io).preserve!(preserve: preserve).write(decls)
-        end
+        path.write(rewriter.string)
       end
 
-      def annotate_decls(decls, outer: [])
+      def annotate_decls(decls, rewriter, outer: [])
         decls.each do |decl|
           case decl
           when AST::Declarations::Class, AST::Declarations::Module
-            annotate_class(decl, outer: outer)
+            annotate_class(decl, rewriter, outer: outer)
           when AST::Declarations::Constant
-            annotate_constant(decl, outer: outer)
+            annotate_constant(decl, rewriter, outer: outer)
           end
         end
       end
@@ -244,7 +241,7 @@ module RBS
         end
       end
 
-      def annotate_class(decl, outer:)
+      def annotate_class(decl, rewriter, outer:)
         annots = annotations(decl)
 
         full_name = resolve_name(decl.name, outer: outer)
@@ -252,7 +249,7 @@ module RBS
           text = resolve_doc_source(annots.copy_annotation, tester: annots) { doc_for_class(full_name, tester: annots) }
         end
 
-        replace_comment(decl, text)
+        replace_comment(decl, text, rewriter)
 
         unless annots.skip_all?
           outer_ = outer + [decl.name.to_namespace]
@@ -260,28 +257,28 @@ module RBS
           decl.each_member do |member|
             case member
             when AST::Members::MethodDefinition
-              annotate_method(full_name, member)
+              annotate_method(full_name, member, rewriter)
             when AST::Members::Alias
-              annotate_alias(full_name, member)
+              annotate_alias(full_name, member, rewriter)
             when AST::Members::AttrReader, AST::Members::AttrAccessor, AST::Members::AttrWriter
-              annotate_attribute(full_name, member)
+              annotate_attribute(full_name, member, rewriter)
             end
           end
 
-          annotate_decls(decl.each_decl.to_a, outer: outer_)
+          annotate_decls(decl.each_decl.to_a, rewriter, outer: outer_)
         end
       end
 
-      def annotate_constant(const, outer:)
+      def annotate_constant(const, rewriter, outer:)
         annots = Annotations.new([])
 
         full_name = resolve_name(const.name, outer: outer)
         text = doc_for_constant(full_name, tester: annots)
 
-        replace_comment(const, text)
+        replace_comment(const, text, rewriter)
       end
 
-      def annotate_alias(typename, als)
+      def annotate_alias(typename, als, rewriter)
         annots = annotations(als)
 
         unless annots.skip?
@@ -295,7 +292,7 @@ module RBS
           end
         end
 
-        replace_comment(als, text)
+        replace_comment(als, text, rewriter)
       end
 
       def join_docs(docs, separator: "----")
@@ -311,7 +308,7 @@ module RBS
         end
       end
 
-      def annotate_method(typename, method)
+      def annotate_method(typename, method, rewriter)
         annots = annotations(method)
 
         unless annots.skip?
@@ -337,10 +334,10 @@ module RBS
           }
         end
 
-        replace_comment(method, text)
+        replace_comment(method, text, rewriter)
       end
 
-      def annotate_attribute(typename, attr)
+      def annotate_attribute(typename, attr, rewriter)
         annots = annotations(attr)
 
         unless annots.skip?
@@ -368,18 +365,17 @@ module RBS
           end
         end
 
-        replace_comment(attr, text)
+        replace_comment(attr, text, rewriter)
       end
 
-      def replace_comment(commented, string)
+      def replace_comment(commented, string, rewriter)
         if string
           if string.empty?
-            commented.instance_variable_set(:@comment, nil)
+            rewriter.delete_comment(commented.comment) if commented.comment
+          elsif commented.comment
+            rewriter.replace_comment(commented.comment, content: string)
           else
-            commented.instance_variable_set(
-              :@comment,
-              AST::Comment.new(location: nil, string: string)
-            )
+            rewriter.add_comment(commented.location || raise, *commented.annotations.filter_map(&:location), content: string)
           end
         end
       end

data/lib/rbs/ast/annotation.rb

@@ -21,7 +21,7 @@ module RBS
         self.class.hash ^ string.hash
       end
 
-      def to_json(state = _ = nil)
+      def to_json(state = nil)
         { string: string, location: location }.to_json(state)
       end
     end

data/lib/rbs/ast/comment.rb

@@ -21,7 +21,7 @@ module RBS
         self.class.hash ^ string.hash
       end
 
-      def to_json(state = _ = nil)
+      def to_json(state = nil)
         { string: string, location: location }.to_json(state)
       end
     end