rbs 4.0.0.dev.4 → 4.0.0.dev.5

data/include/rbs/lexer.h

@@ -30,6 +30,7 @@ enum RBSTokenType {
     pBANG,     /* ! */
     pQUESTION, /* ? */
     pLT,       /* < */
+    pGT,       /* > */
     pEQ,       /* = */
 
     kALIAS,        /* alias */
@@ -39,6 +40,7 @@ enum RBSTokenType {
     kBOOL,         /* bool */
     kBOT,          /* bot */
     kCLASS,        /* class */
+    kCLASSALIAS,   /* class-alias */
     kDEF,          /* def */
     kEND,          /* end */
     kEXTEND,       /* extend */
@@ -48,6 +50,7 @@ enum RBSTokenType {
     kINSTANCE,     /* instance */
     kINTERFACE,    /* interface */
     kMODULE,       /* module */
+    kMODULEALIAS,  /* module-alias */
     kNIL,          /* nil */
     kOUT,          /* out */
     kPREPEND,      /* prepend */
@@ -123,20 +126,26 @@ typedef struct {
  * The lexer state is the curren token.
  *
  * ```
- * ... "a string token"
- *    ^                      start position
- *          ^                current position
- *     ~~~~~~                Token => "a str
+ #.   0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6
+ * ... " a   s t r i n g   t o k e n "
+ *    ^                                   start position (0)
+ *                ^                       current position (6)
+ *                 ^                      current character ('i', bytes = 1)
+ *     ~~~~~~~~~~~                        Token => "a str
  * ```
  * */
 typedef struct {
     rbs_string_t string;
-    int start_pos;            /* The character position that defines the start of the input */
-    int end_pos;              /* The character position that defines the end of the input */
-    rbs_position_t current;   /* The current position */
-    rbs_position_t start;     /* The start position of the current token */
+    int start_pos;          /* The character position that defines the start of the input */
+    int end_pos;            /* The character position that defines the end of the input */
+    rbs_position_t current; /* The current position: just before the current_character */
+    rbs_position_t start;   /* The start position of the current token */
+
+    unsigned int current_code_point; /* Current character code point */
+    size_t current_character_bytes;  /* Current character byte length (0 or 1~4) */
+
     bool first_token_of_line; /* This flag is used for tLINECOMMENT */
-    unsigned int last_char;   /* Last peeked character */
+
     const rbs_encoding_t *encoding;
 } rbs_lexer_t;
 
@@ -156,15 +165,23 @@ int rbs_token_bytes(rbs_token_t tok);
 const char *rbs_token_type_str(enum RBSTokenType type);
 
 /**
- * Read next character.
+ * Returns the next character.
  * */
 unsigned int rbs_peek(rbs_lexer_t *lexer);
 
 /**
- * Skip one character.
+ * Advances the current position by one character.
  * */
 void rbs_skip(rbs_lexer_t *lexer);
 
+/**
+ * Read next character and store the codepoint and byte length to the given pointers.
+ * 
+ * This doesn't update the lexer state.
+ * Returns `true` if succeeded, or `false` if reached to EOF.
+ * */
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *bytes);
+
 /**
  * Skip n characters.
  * */
@@ -184,4 +201,6 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer);
 
 void rbs_print_token(rbs_token_t tok);
 
+void rbs_print_lexer(rbs_lexer_t *lexer);
+
 #endif

data/include/rbs/parser.h

@@ -27,9 +27,9 @@ typedef struct rbs_comment_t {
     rbs_position_t start;
     rbs_position_t end;
 
-    size_t line_size;
-    size_t line_count;
-    rbs_token_t *tokens;
+    size_t line_tokens_capacity;
+    size_t line_tokens_count;
+    rbs_token_t *line_tokens;
 
     struct rbs_comment_t *next_comment;
 } rbs_comment_t;
@@ -44,7 +44,7 @@ typedef struct rbs_error_t {
  * An RBS parser is a LL(3) parser.
  * */
 typedef struct {
-    rbs_lexer_t *rbs_lexer_t;
+    rbs_lexer_t *lexer;
 
     rbs_token_t current_token;
     rbs_token_t next_token;  /* The first lookahead token */
@@ -126,8 +126,8 @@ rbs_ast_comment_t *rbs_parser_get_comment(rbs_parser_t *parser, int subject_line
 
 void rbs_parser_set_error(rbs_parser_t *parser, rbs_token_t tok, bool syntax_error, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(4, 5);
 
-bool rbs_parse_type(rbs_parser_t *parser, rbs_node_t **type);
-bool rbs_parse_method_type(rbs_parser_t *parser, rbs_method_type_t **method_type);
+bool rbs_parse_type(rbs_parser_t *parser, rbs_node_t **type, bool void_allowed, bool self_allowed);
+bool rbs_parse_method_type(rbs_parser_t *parser, rbs_method_type_t **method_type, bool require_eof);
 bool rbs_parse_signature(rbs_parser_t *parser, rbs_signature_t **signature);
 
 bool rbs_parse_type_params(rbs_parser_t *parser, bool module_type_params, rbs_node_list_t **params);

data/include/rbs/string.h

@@ -44,6 +44,4 @@ size_t rbs_string_len(const rbs_string_t self);
  */
 bool rbs_string_equal(const rbs_string_t lhs, const rbs_string_t rhs);
 
-unsigned int rbs_utf8_string_to_codepoint(const rbs_string_t string);
-
 #endif

data/include/rbs/util/rbs_allocator.h

@@ -3,19 +3,40 @@
 
 #include <stddef.h>
 
-#ifndef alignof
-#if defined(__GNUC__) || defined(__clang__)
-#define alignof(type) __alignof__(type)
+/* Include stdalign.h for C11 and later for alignof support */
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#include <stdalign.h>
+#endif
+
+/*
+ * Define a portable alignment macro that works across all supported environments
+ */
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+/* C11 or later - use _Alignof directly (always available in C11+) */
+#define rbs_alignof(type) _Alignof(type)
+#elif defined(__cplusplus) && __cplusplus >= 201103L
+/* C++11 or later has alignof keyword */
+#define rbs_alignof(type) alignof(type)
+#elif defined(__GNUC__) || defined(__clang__)
+/* GCC and Clang provide __alignof__ */
+#define rbs_alignof(type) __alignof__(type)
 #elif defined(_MSC_VER)
-#define alignof(type) __alignof(type)
+/* MSVC provides __alignof */
+#define rbs_alignof(type) __alignof(type)
 #else
-// Fallback using offset trick
-#define alignof(type) offsetof(struct { char c; type member; }, member)
-#endif
+/* Fallback using offset trick for other compilers */
+#define rbs_alignof(type) offsetof( \
+    struct { char c; type member; },                     \
+    member                          \
+)
 #endif
 
-struct rbs_allocator;
-typedef struct rbs_allocator rbs_allocator_t;
+typedef struct rbs_allocator {
+    // The head of a linked list of pages, starting with the most recently allocated page.
+    struct rbs_allocator_page *page;
+
+    size_t default_page_payload_size;
+} rbs_allocator_t;
 
 rbs_allocator_t *rbs_allocator_init(void);
 void rbs_allocator_free(rbs_allocator_t *);
@@ -26,13 +47,13 @@ void *rbs_allocator_calloc_impl(rbs_allocator_t *, size_t count, size_t size, si
 void *rbs_allocator_realloc_impl(rbs_allocator_t *, void *ptr, size_t old_size, size_t new_size, size_t alignment);
 
 // Use this when allocating memory for a single instance of a type.
-#define rbs_allocator_alloc(allocator, type) ((type *) rbs_allocator_malloc_impl((allocator), sizeof(type), alignof(type)))
+#define rbs_allocator_alloc(allocator, type) ((type *) rbs_allocator_malloc_impl((allocator), sizeof(type), rbs_alignof(type)))
 // Use this when allocating memory that will be immediately written to in full.
 // Such as allocating strings
-#define rbs_allocator_alloc_many(allocator, count, type) ((type *) rbs_allocator_malloc_many_impl((allocator), (count), sizeof(type), alignof(type)))
+#define rbs_allocator_alloc_many(allocator, count, type) ((type *) rbs_allocator_malloc_many_impl((allocator), (count), sizeof(type), rbs_alignof(type)))
 // Use this when allocating memory that will NOT be immediately written to in full.
 // Such as allocating buffers
-#define rbs_allocator_calloc(allocator, count, type) ((type *) rbs_allocator_calloc_impl((allocator), (count), sizeof(type), alignof(type)))
-#define rbs_allocator_realloc(allocator, ptr, old_size, new_size, type) ((type *) rbs_allocator_realloc_impl((allocator), (ptr), (old_size), (new_size), alignof(type)))
+#define rbs_allocator_calloc(allocator, count, type) ((type *) rbs_allocator_calloc_impl((allocator), (count), sizeof(type), rbs_alignof(type)))
+#define rbs_allocator_realloc(allocator, ptr, old_size, new_size, type) ((type *) rbs_allocator_realloc_impl((allocator), (ptr), (old_size), (new_size), rbs_alignof(type)))
 
 #endif

data/include/rbs/util/rbs_assert.h

@@ -4,6 +4,17 @@
 #include "rbs/defines.h"
 #include <stdbool.h>
 
-void rbs_assert(bool condition, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(2, 3);
+/**
+ * RBS_ASSERT macro that calls rbs_assert in debug builds and is removed in release builds.
+ * In debug mode, it forwards all arguments to the rbs_assert function.
+ * In release mode, it expands to nothing.
+ */
+#ifdef NDEBUG
+#define RBS_ASSERT(condition, ...) ((void) 0)
+#else
+#define RBS_ASSERT(condition, ...) rbs_assert_impl(condition, __VA_ARGS__)
+#endif
+
+void rbs_assert_impl(bool condition, const char *fmt, ...) RBS_ATTRIBUTE_FORMAT(2, 3);
 
 #endif

data/include/rbs/util/rbs_encoding.h

@@ -6,6 +6,8 @@
 #ifndef RBS_RBS_ENCODING_H
 #define RBS_RBS_ENCODING_H
 
+#include "rbs/defines.h"
+
 #include <assert.h>
 #include <stdbool.h>
 #include <stddef.h>

data/include/rbs/util/rbs_unescape.h

@@ -4,6 +4,7 @@
 #include <stddef.h>
 #include "rbs/util/rbs_allocator.h"
 #include "rbs/string.h"
+#include "rbs/util/rbs_encoding.h"
 
 /**
  * Receives `rbs_parser_t` and `range`, which represents a string token or symbol token, and returns a string VALUE.
@@ -18,6 +19,6 @@
  *
  * @returns A new owned string that will be freed when the allocator is freed.
  * */
-rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input);
+rbs_string_t rbs_unquote_string(rbs_allocator_t *, const rbs_string_t input, const rbs_encoding_t *encoding);
 
 #endif // RBS_RBS_UNESCAPE_H

data/lib/rbs/ast/annotation.rb

@@ -21,7 +21,7 @@ module RBS
         self.class.hash ^ string.hash
       end
 
-      def to_json(state = _ = nil)
+      def to_json(state = nil)
         { string: string, location: location }.to_json(state)
       end
     end

data/lib/rbs/ast/comment.rb

@@ -21,7 +21,7 @@ module RBS
         self.class.hash ^ string.hash
       end
 
-      def to_json(state = _ = nil)
+      def to_json(state = nil)
         { string: string, location: location }.to_json(state)
       end
     end

data/lib/rbs/ast/declarations.rb

@@ -74,7 +74,7 @@ module RBS
             self.class.hash ^ name.hash ^ args.hash
           end
 
-          def to_json(state = _ = nil)
+          def to_json(state = nil)
             {
               name: name,
               args: args,
@@ -130,7 +130,7 @@ module RBS
           self.class.hash ^ name.hash ^ type_params.hash ^ super_class.hash ^ members.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :class,
             name: name,
@@ -166,7 +166,7 @@ module RBS
             self.class.hash ^ name.hash ^ args.hash ^ location.hash
           end
 
-          def to_json(state = _ = nil)
+          def to_json(state = nil)
             {
               name: name,
               args: args,
@@ -231,7 +231,7 @@ module RBS
           self.class.hash ^ name.hash ^ type_params.hash ^ self_types.hash ^ members.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :module,
             name: name,
@@ -288,7 +288,7 @@ module RBS
           self.class.hash ^ type_params.hash ^ members.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :interface,
             name: name,
@@ -331,7 +331,7 @@ module RBS
           self.class.hash ^ name.hash ^ type_params.hash ^ type.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :alias,
             name: name,
@@ -371,7 +371,7 @@ module RBS
           self.class.hash ^ name.hash ^ type.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :constant,
             name: name,
@@ -409,7 +409,7 @@ module RBS
           self.class.hash ^ name.hash ^ type.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :global,
             name: name,
@@ -445,7 +445,7 @@ module RBS
       end
 
       class ClassAlias < AliasDecl
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :class_alias,
             new_name: new_name,
@@ -457,7 +457,7 @@ module RBS
       end
 
       class ModuleAlias < AliasDecl
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             declaration: :module_alias,
             new_name: new_name,

data/lib/rbs/ast/members.rb

@@ -35,7 +35,7 @@ module RBS
             update(method_type: self.method_type.sub(subst))
           end
 
-          def to_json(state = _ = nil)
+          def to_json(state = nil)
             {
               annotations: annotations,
               method_type: method_type
@@ -103,7 +103,7 @@ module RBS
           )
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :method_definition,
             name: name,
@@ -145,7 +145,7 @@ module RBS
       class InstanceVariable < Base
         include Var
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :instance_variable,
             name: name,
@@ -159,7 +159,7 @@ module RBS
       class ClassInstanceVariable < Base
         include Var
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :class_instance_variable,
             name: name,
@@ -173,7 +173,7 @@ module RBS
       class ClassVariable < Base
         include Var
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :class_variable,
             name: name,
@@ -215,7 +215,7 @@ module RBS
       class Include < Base
         include Mixin
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :include,
             name: name,
@@ -230,7 +230,7 @@ module RBS
       class Extend < Base
         include Mixin
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :extend,
             name: name,
@@ -245,7 +245,7 @@ module RBS
       class Prepend < Base
         include Mixin
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :prepend,
             name: name,
@@ -311,7 +311,7 @@ module RBS
       class AttrReader < Base
         include Attribute
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :attr_reader,
             name: name,
@@ -329,7 +329,7 @@ module RBS
       class AttrAccessor < Base
         include Attribute
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :attr_accessor,
             name: name,
@@ -347,7 +347,7 @@ module RBS
       class AttrWriter < Base
         include Attribute
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :attr_writer,
             name: name,
@@ -383,7 +383,7 @@ module RBS
       class Public < Base
         include LocationOnly
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           { member: :public, location: location }.to_json(state)
         end
       end
@@ -391,7 +391,7 @@ module RBS
       class Private < Base
         include LocationOnly
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           { member: :private, location: location }.to_json(state)
         end
       end
@@ -426,7 +426,7 @@ module RBS
           new_name.hash ^ old_name.hash ^ kind.hash
         end
 
-        def to_json(state = _ = nil)
+        def to_json(state = nil)
           {
             member: :alias,
             new_name: new_name,

data/lib/rbs/ast/ruby/annotations.rb

@@ -31,6 +31,52 @@ module RBS
               type: type.map_type_name { yield _1 }
             ) #: self
           end
+
+          def type_fingerprint
+            [
+              "annots/node_type_assertion",
+              type.to_s
+            ]
+          end
+        end
+
+        class AliasAnnotation < Base
+          attr_reader :keyword_location, :type_name_location, :type_name
+
+          def initialize(location:, prefix_location:, keyword_location:, type_name:, type_name_location:)
+            super(location, prefix_location)
+            @keyword_location = keyword_location
+            @type_name = type_name
+            @type_name_location = type_name_location
+          end
+
+          def map_type_name
+            self.class.new(
+              location:,
+              prefix_location:,
+              keyword_location:,
+              type_name: type_name ? yield(type_name) : nil,
+              type_name_location:
+            ) #: self
+          end
+        end
+
+        class ClassAliasAnnotation < AliasAnnotation
+          def type_fingerprint
+            [
+              "annots/class-alias",
+              type_name&.to_s
+            ]
+          end
+        end
+
+        class ModuleAliasAnnotation < AliasAnnotation
+          def type_fingerprint
+            [
+              "annots/module-alias",
+              type_name&.to_s
+            ]
+          end
         end
 
         class ColonMethodTypeAnnotation < Base
@@ -50,6 +96,14 @@ module RBS
               method_type: method_type.map_type {|type| type.map_type_name { yield _1 }}
             ) #: self
           end
+
+          def type_fingerprint
+            [
+              "annots/colon_method_type",
+              annotations.map(&:to_s),
+              method_type.to_s
+            ]
+          end
         end
 
         class MethodTypesAnnotation < Base
@@ -73,6 +127,13 @@ module RBS
 
             self.class.new(location:, prefix_location:, overloads: ovs, vertical_bar_locations:) #: self
           end
+
+          def type_fingerprint
+            [
+              "annots/method_types",
+              overloads.map { |o| [o.annotations.map(&:to_s), o.method_type.to_s] }
+            ]
+          end
         end
 
         class SkipAnnotation < Base
@@ -83,6 +144,10 @@ module RBS
             @skip_location = skip_location
             @comment_location = comment_location
           end
+
+          def type_fingerprint
+            "annots/skip"
+          end
         end
 
         class ReturnTypeAnnotation < Base
@@ -112,6 +177,78 @@ module RBS
               comment_location: comment_location
             ) #: self
           end
+
+          def type_fingerprint
+            [
+              "annots/return_type",
+              return_type.to_s,
+              comment_location&.source
+            ]
+          end
+        end
+
+        class TypeApplicationAnnotation < Base
+          attr_reader :type_args, :close_bracket_location, :comma_locations
+
+          def initialize(location:, prefix_location:, type_args:, close_bracket_location:, comma_locations:)
+            super(location, prefix_location)
+            @type_args = type_args
+            @close_bracket_location = close_bracket_location
+            @comma_locations = comma_locations
+          end
+
+          def map_type_name(&block)
+            mapped_type_args = type_args.map { |type| type.map_type_name { yield _1 } }
+
+            self.class.new(
+              location:,
+              prefix_location:,
+              type_args: mapped_type_args,
+              close_bracket_location:,
+              comma_locations:
+            ) #: self
+          end
+
+          def type_fingerprint
+            [
+              "annots/type_application",
+              type_args.map(&:to_s)
+            ]
+          end
+        end
+
+        class InstanceVariableAnnotation < Base
+          attr_reader :ivar_name, :ivar_name_location, :colon_location, :type, :comment_location
+
+          def initialize(location:, prefix_location:, ivar_name:, ivar_name_location:, colon_location:, type:, comment_location:)
+            super(location, prefix_location)
+            @ivar_name = ivar_name
+            @ivar_name_location = ivar_name_location
+            @colon_location = colon_location
+            @type = type
+            @comment_location = comment_location
+          end
+
+          def map_type_name(&block)
+            self.class.new(
+              location:,
+              prefix_location:,
+              ivar_name:,
+              ivar_name_location:,
+              colon_location:,
+              type: type.map_type_name { yield _1 },
+              comment_location:
+            ) #: self
+          end
+
+          def type_fingerprint
+            [
+              "annots/instance_variable",
+              ivar_name.to_s,
+              type.to_s,
+              comment_location&.source
+            ]
+          end
         end
       end
     end

data/lib/rbs/ast/ruby/comment_block.rb

@@ -177,6 +177,13 @@ module RBS
           Location.new(comment_buffer, start_offset, end_offset)
         end
 
+        def location()
+          first_comment = comments[0] or raise
+          last_comment = comments[-1] or raise
+
+          comment_buffer.rbs_location(first_comment.location.join last_comment.location)
+        end
+
         def parse_annotation_lines(start_line, end_line, variables)
           start_pos = comment_buffer.ranges[start_line].begin
           end_pos = comment_buffer.ranges[end_line].end
@@ -215,6 +222,23 @@ module RBS
 
           false
         end
+
+        def as_comment
+          lines = [] #: Array[String]
+
+          each_paragraph([]) do |paragraph|
+            case paragraph
+            when Location
+              lines << paragraph.local_source
+            end
+          end
+
+          string = lines.join("\n")
+
+          unless string.strip.empty?
+            AST::Comment.new(string: string, location: location)
+          end
+        end
       end
     end
   end