rbs 4.0.0.dev.4 → 4.0.0.dev.5

data/src/lexer.re

@@ -44,6 +44,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
       ":"   { return rbs_next_token(lexer, pCOLON); }
       "::"  { return rbs_next_token(lexer, pCOLON2); }
       "<"   { return rbs_next_token(lexer, pLT); }
+      ">"   { return rbs_next_token(lexer, pGT); }
       "[]"  { return rbs_next_token(lexer, pAREF_OPR); }
       operator  { return rbs_next_token(lexer, tOPERATOR); }
       "--" [^\x00]* { return rbs_next_token(lexer, tINLINECOMMENT); }
@@ -71,6 +72,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
       "bool"          { return rbs_next_token(lexer, kBOOL); }
       "bot"           { return rbs_next_token(lexer, kBOT); }
       "class"         { return rbs_next_token(lexer, kCLASS); }
+      "class-alias"   { return rbs_next_token(lexer, kCLASSALIAS); }
       "def"           { return rbs_next_token(lexer, kDEF); }
       "end"           { return rbs_next_token(lexer, kEND); }
       "extend"        { return rbs_next_token(lexer, kEXTEND); }
@@ -80,6 +82,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
       "instance"      { return rbs_next_token(lexer, kINSTANCE); }
       "interface"     { return rbs_next_token(lexer, kINTERFACE); }
       "module"        { return rbs_next_token(lexer, kMODULE); }
+      "module-alias"  { return rbs_next_token(lexer, kMODULEALIAS); }
       "nil"           { return rbs_next_token(lexer, kNIL); }
       "out"           { return rbs_next_token(lexer, kOUT); }
       "prepend"       { return rbs_next_token(lexer, kPREPEND); }

data/src/lexstate.c

@@ -1,4 +1,6 @@
+#include "rbs/defines.h"
 #include "rbs/lexer.h"
+#include "rbs/util/rbs_assert.h"
 
 static const char *RBS_TOKENTYPE_NAMES[] = {
     "NullType",
@@ -26,6 +28,7 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
     "pBANG",     /* ! */
     "pQUESTION", /* ? */
     "pLT",       /* < */
+    "pGT",       /* > */
     "pEQ",       /* = */
 
     "kALIAS",        /* alias */
@@ -35,6 +38,7 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
     "kBOOL",         /* bool */
     "kBOT",          /* bot */
     "kCLASS",        /* class */
+    "kCLASSALIAS",   /* class-alias */
     "kDEF",          /* def */
     "kEND",          /* end */
     "kEXTEND",       /* extend */
@@ -44,6 +48,7 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
     "kINSTANCE",     /* instance */
     "kINTERFACE",    /* interface */
     "kMODULE",       /* module */
+    "kMODULEALIAS",  /* module-alias */
     "kNIL",          /* nil */
     "kOUT",          /* out */
     "kPREPEND",      /* prepend */
@@ -109,17 +114,60 @@ int rbs_token_bytes(rbs_token_t tok) {
 }
 
 unsigned int rbs_peek(rbs_lexer_t *lexer) {
-    if (lexer->current.char_pos == lexer->end_pos) {
-        lexer->last_char = '\0';
-        return 0;
+    return lexer->current_code_point;
+}
+
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len) {
+    if (RBS_UNLIKELY(lexer->current.char_pos == lexer->end_pos)) {
+        return false;
+    }
+
+    const char *start = lexer->string.start + lexer->current.byte_pos;
+
+    // Fast path for ASCII (single-byte) characters
+    if ((unsigned int) *start < 128) {
+        *codepoint = (unsigned int) *start;
+        *byte_len = 1;
+        return true;
+    }
+
+    *byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
+
+    if (*byte_len == 1) {
+        *codepoint = (unsigned int) *start;
     } else {
-        rbs_string_t str = rbs_string_new(
-            lexer->string.start + lexer->current.byte_pos,
-            lexer->string.end
-        );
-        unsigned int c = rbs_utf8_string_to_codepoint(str);
-        lexer->last_char = c;
-        return c;
+        *codepoint = 12523; // Dummy data for "ル" from "ルビー" (Ruby) in Unicode
+    }
+
+    return true;
+}
+
+void rbs_skip(rbs_lexer_t *lexer) {
+    RBS_ASSERT(lexer->current_character_bytes > 0, "rbs_skip called with current_character_bytes == 0");
+
+    if (RBS_UNLIKELY(lexer->current_code_point == '\0')) {
+        return;
+    }
+
+    unsigned int codepoint;
+    size_t byte_len;
+
+    lexer->current.byte_pos += lexer->current_character_bytes;
+    lexer->current.char_pos += 1;
+    if (lexer->current_code_point == '\n') {
+        lexer->current.line += 1;
+        lexer->current.column = 0;
+        lexer->first_token_of_line = true;
+    } else {
+        lexer->current.column += 1;
+    }
+
+    if (rbs_next_char(lexer, &codepoint, &byte_len)) {
+        lexer->current_code_point = codepoint;
+        lexer->current_character_bytes = byte_len;
+    } else {
+        lexer->current_character_bytes = 1;
+        lexer->current_code_point = '\0';
     }
 }
 
@@ -153,35 +201,8 @@ rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
     }
 }
 
-void rbs_skip(rbs_lexer_t *lexer) {
-    if (!lexer->last_char) {
-        rbs_peek(lexer);
-    }
-
-    size_t byte_len;
-
-    if (lexer->last_char == '\0') {
-        byte_len = 1;
-    } else {
-        const char *start = lexer->string.start + lexer->current.byte_pos;
-        byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
-    }
-
-    lexer->current.char_pos += 1;
-    lexer->current.byte_pos += byte_len;
-
-    if (lexer->last_char == '\n') {
-        lexer->current.line += 1;
-        lexer->current.column = 0;
-        lexer->first_token_of_line = true;
-    } else {
-        lexer->current.column += 1;
-    }
-}
-
 void rbs_skipn(rbs_lexer_t *lexer, size_t size) {
     for (size_t i = 0; i < size; i++) {
-        rbs_peek(lexer);
         rbs_skip(lexer);
     }
 }

data/src/location.c

@@ -6,9 +6,9 @@
 #define RBS_LOC_CHILDREN_SIZE(cap) (sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * ((cap) - 1))
 
 void rbs_loc_alloc_children(rbs_allocator_t *allocator, rbs_location_t *loc, size_t capacity) {
-    rbs_assert(capacity <= sizeof(rbs_loc_entry_bitmap) * 8, "Capacity %zu is too large. Max is %zu", capacity, sizeof(rbs_loc_entry_bitmap) * 8);
+    RBS_ASSERT(capacity <= sizeof(rbs_loc_entry_bitmap) * 8, "Capacity %zu is too large. Max is %zu", capacity, sizeof(rbs_loc_entry_bitmap) * 8);
 
-    loc->children = rbs_allocator_malloc_impl(allocator, RBS_LOC_CHILDREN_SIZE(capacity), alignof(rbs_loc_children));
+    loc->children = (rbs_loc_children *) rbs_allocator_malloc_impl(allocator, RBS_LOC_CHILDREN_SIZE(capacity), rbs_alignof(rbs_loc_children));
 
     loc->children->len = 0;
     loc->children->required_p = 0;
@@ -16,8 +16,8 @@ void rbs_loc_alloc_children(rbs_allocator_t *allocator, rbs_location_t *loc, siz
 }
 
 void rbs_loc_add_optional_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r) {
-    rbs_assert(loc->children != NULL, "All children should have been pre-allocated with rbs_loc_alloc_children()");
-    rbs_assert((loc->children->len + 1 <= loc->children->cap), "Not enough space was pre-allocated for the children. Children: %hu, Capacity: %hu", loc->children->len, loc->children->cap);
+    RBS_ASSERT(loc->children != NULL, "All children should have been pre-allocated with rbs_loc_alloc_children()");
+    RBS_ASSERT((loc->children->len + 1 <= loc->children->cap), "Not enough space was pre-allocated for the children. Children: %hu, Capacity: %hu", loc->children->len, loc->children->cap);
 
     unsigned short i = loc->children->len++;
     loc->children->entries[i].name = name;