rbs 3.10.0 → 4.0.0.dev.1

data/src/lexer.re

@@ -46,6 +46,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
       "<"   { return rbs_next_token(lexer, pLT); }
       "[]"  { return rbs_next_token(lexer, pAREF_OPR); }
       operator  { return rbs_next_token(lexer, tOPERATOR); }
+      "--" [^\x00]* { return rbs_next_token(lexer, tINLINECOMMENT); }
 
       number = [0-9] [0-9_]*;
       ("-"|"+")? number    { return rbs_next_token(lexer, tINTEGER); }
@@ -95,6 +96,9 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
       "use"           { return rbs_next_token(lexer, kUSE); }
       "as"            { return rbs_next_token(lexer, kAS); }
       "__todo__"      { return rbs_next_token(lexer, k__TODO__); }
+      "@rbs"          { return rbs_next_token(lexer, kATRBS); }
+      "skip"          { return rbs_next_token(lexer, kSKIP); }
+      "return"        { return rbs_next_token(lexer, kRETURN); }
 
       unicode_char = "\\u" [0-9a-fA-F]{4};
       oct_char = "\\x" [0-9a-f]{1,2};

data/src/lexstate.c

@@ -1,205 +1,191 @@
-#include "rbs/defines.h"
 #include "rbs/lexer.h"
-#include "rbs/util/rbs_assert.h"
 
 static const char *RBS_TOKENTYPE_NAMES[] = {
-    "NullType",
-    "pEOF",
-    "ErrorToken",
-
-    "pLPAREN",   /* ( */
-    "pRPAREN",   /* ) */
-    "pCOLON",    /* : */
-    "pCOLON2",   /* :: */
-    "pLBRACKET", /* [ */
-    "pRBRACKET", /* ] */
-    "pLBRACE",   /* { */
-    "pRBRACE",   /* } */
-    "pHAT",      /* ^ */
-    "pARROW",    /* -> */
-    "pFATARROW", /* => */
-    "pCOMMA",    /* , */
-    "pBAR",      /* | */
-    "pAMP",      /* & */
-    "pSTAR",     /* * */
-    "pSTAR2",    /* ** */
-    "pDOT",      /* . */
-    "pDOT3",     /* ... */
-    "pBANG",     /* ! */
-    "pQUESTION", /* ? */
-    "pLT",       /* < */
-    "pEQ",       /* = */
-
-    "kALIAS",        /* alias */
-    "kATTRACCESSOR", /* attr_accessor */
-    "kATTRREADER",   /* attr_reader */
-    "kATTRWRITER",   /* attr_writer */
-    "kBOOL",         /* bool */
-    "kBOT",          /* bot */
-    "kCLASS",        /* class */
-    "kDEF",          /* def */
-    "kEND",          /* end */
-    "kEXTEND",       /* extend */
-    "kFALSE",        /* kFALSE */
-    "kIN",           /* in */
-    "kINCLUDE",      /* include */
-    "kINSTANCE",     /* instance */
-    "kINTERFACE",    /* interface */
-    "kMODULE",       /* module */
-    "kNIL",          /* nil */
-    "kOUT",          /* out */
-    "kPREPEND",      /* prepend */
-    "kPRIVATE",      /* private */
-    "kPUBLIC",       /* public */
-    "kSELF",         /* self */
-    "kSINGLETON",    /* singleton */
-    "kTOP",          /* top */
-    "kTRUE",         /* true */
-    "kTYPE",         /* type */
-    "kUNCHECKED",    /* unchecked */
-    "kUNTYPED",      /* untyped */
-    "kVOID",         /* void */
-    "kUSE",          /* use */
-    "kAS",           /* as */
-    "k__TODO__",     /* __todo__ */
-
-    "tLIDENT",  /* Identifiers starting with lower case */
-    "tUIDENT",  /* Identifiers starting with upper case */
-    "tULIDENT", /* Identifiers starting with `_` */
-    "tULLIDENT",
-    "tGIDENT",  /* Identifiers starting with `$` */
-    "tAIDENT",  /* Identifiers starting with `@` */
-    "tA2IDENT", /* Identifiers starting with `@@` */
-    "tBANGIDENT",
-    "tEQIDENT",
-    "tQIDENT",   /* Quoted identifier */
-    "pAREF_OPR", /* [] */
-    "tOPERATOR", /* Operator identifier */
-
-    "tCOMMENT",
-    "tLINECOMMENT",
-
-    "tTRIVIA",
-
-    "tDQSTRING", /* Double quoted string */
-    "tSQSTRING", /* Single quoted string */
-    "tINTEGER",  /* Integer */
-    "tSYMBOL",   /* Symbol */
-    "tDQSYMBOL",
-    "tSQSYMBOL",
-    "tANNOTATION", /* Annotation */
+  "NullType",
+  "pEOF",
+  "ErrorToken",
+
+  "pLPAREN",          /* ( */
+  "pRPAREN",          /* ) */
+  "pCOLON",           /* : */
+  "pCOLON2",          /* :: */
+  "pLBRACKET",        /* [ */
+  "pRBRACKET",        /* ] */
+  "pLBRACE",          /* { */
+  "pRBRACE",          /* } */
+  "pHAT",             /* ^ */
+  "pARROW",           /* -> */
+  "pFATARROW",        /* => */
+  "pCOMMA",           /* , */
+  "pBAR",             /* | */
+  "pAMP",             /* & */
+  "pSTAR",            /* * */
+  "pSTAR2",           /* ** */
+  "pDOT",             /* . */
+  "pDOT3",            /* ... */
+  "pBANG",            /* ! */
+  "pQUESTION",        /* ? */
+  "pLT",              /* < */
+  "pEQ",              /* = */
+
+  "kALIAS",           /* alias */
+  "kATTRACCESSOR",    /* attr_accessor */
+  "kATTRREADER",      /* attr_reader */
+  "kATTRWRITER",      /* attr_writer */
+  "kBOOL",            /* bool */
+  "kBOT",             /* bot */
+  "kCLASS",           /* class */
+  "kDEF",             /* def */
+  "kEND",             /* end */
+  "kEXTEND",          /* extend */
+  "kFALSE",           /* kFALSE */
+  "kIN",              /* in */
+  "kINCLUDE",         /* include */
+  "kINSTANCE",        /* instance */
+  "kINTERFACE",       /* interface */
+  "kMODULE",          /* module */
+  "kNIL",             /* nil */
+  "kOUT",             /* out */
+  "kPREPEND",         /* prepend */
+  "kPRIVATE",         /* private */
+  "kPUBLIC",          /* public */
+  "kSELF",            /* self */
+  "kSINGLETON",       /* singleton */
+  "kTOP",             /* top */
+  "kTRUE",            /* true */
+  "kTYPE",            /* type */
+  "kUNCHECKED",       /* unchecked */
+  "kUNTYPED",         /* untyped */
+  "kVOID",            /* void */
+  "kUSE",             /* use */
+  "kAS",              /* as */
+  "k__TODO__",        /* __todo__ */
+  "kATRBS",           /* @rbs */
+  "kSKIP",            /* skip */
+  "kRETURN",          /* return */
+
+  "tLIDENT",          /* Identifiers starting with lower case */
+  "tUIDENT",          /* Identifiers starting with upper case */
+  "tULIDENT",         /* Identifiers starting with `_` */
+  "tULLIDENT",
+  "tGIDENT",          /* Identifiers starting with `$` */
+  "tAIDENT",          /* Identifiers starting with `@` */
+  "tA2IDENT",         /* Identifiers starting with `@@` */
+  "tBANGIDENT",
+  "tEQIDENT",
+  "tQIDENT",          /* Quoted identifier */
+  "pAREF_OPR",        /* [] */
+  "tOPERATOR",        /* Operator identifier */
+
+  "tCOMMENT",
+  "tLINECOMMENT",
+  "tINLINECOMMENT",
+
+  "tTRIVIA",
+
+  "tDQSTRING",        /* Double quoted string */
+  "tSQSTRING",        /* Single quoted string */
+  "tINTEGER",         /* Integer */
+  "tSYMBOL",          /* Symbol */
+  "tDQSYMBOL",
+  "tSQSYMBOL",
+  "tANNOTATION",      /* Annotation */
 };
 
-const rbs_position_t NullPosition = { -1, -1, -1, -1 };
-const rbs_range_t NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
-const rbs_token_t NullToken = { .type = NullType, .range = { { 0 }, { 0 } } };
+rbs_token_t NullToken = { .type = NullType, .range = {} };
+rbs_position_t NullPosition = { -1, -1, -1, -1 };
+rbs_range_t NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
 
 const char *rbs_token_type_str(enum RBSTokenType type) {
-    return RBS_TOKENTYPE_NAMES[type];
+  return RBS_TOKENTYPE_NAMES[type];
 }
 
 int rbs_token_chars(rbs_token_t tok) {
-    return tok.range.end.char_pos - tok.range.start.char_pos;
+  return tok.range.end.char_pos - tok.range.start.char_pos;
 }
 
 int rbs_token_bytes(rbs_token_t tok) {
-    return RBS_RANGE_BYTES(tok.range);
+  return RBS_RANGE_BYTES(tok.range);
 }
 
 unsigned int rbs_peek(rbs_lexer_t *lexer) {
-    return lexer->current_code_point;
+  if (lexer->current.char_pos == lexer->end_pos) {
+    lexer->last_char = '\0';
+    return 0;
+  } else {
+    rbs_string_t str = rbs_string_new(
+      lexer->string.start + lexer->current.byte_pos,
+      lexer->string.end
+    );
+    unsigned int c = rbs_utf8_string_to_codepoint(str);
+    lexer->last_char = c;
+    return c;
+  }
 }
 
-bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len) {
-    if (RBS_UNLIKELY(lexer->current.char_pos == lexer->end_pos)) {
-        return false;
-    }
-
-    const char *start = lexer->string.start + lexer->current.byte_pos;
-
-    // Fast path for ASCII (single-byte) characters
-    if ((unsigned int) *start < 128) {
-        *codepoint = (unsigned int) *start;
-        *byte_len = 1;
-        return true;
-    }
-
-    *byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
-
-    if (*byte_len == 1) {
-        *codepoint = (unsigned int) *start;
-    } else {
-        *codepoint = 12523; // Dummy data for "ル" from "ルビー" (Ruby) in Unicode
-    }
+rbs_token_t rbs_next_token(rbs_lexer_t *lexer, enum RBSTokenType type) {
+  rbs_token_t t;
 
-    return true;
-}
+  t.type = type;
+  t.range.start = lexer->start;
+  t.range.end = lexer->current;
+  lexer->start = lexer->current;
+  if (type != tTRIVIA) {
+    lexer->first_token_of_line = false;
+  }
 
-void rbs_skip(rbs_lexer_t *lexer) {
-    RBS_ASSERT(lexer->current_character_bytes > 0, "rbs_skip called with current_character_bytes == 0");
-
-    if (RBS_UNLIKELY(lexer->current_code_point == '\0')) {
-        return;
-    }
-
-    unsigned int codepoint;
-    size_t byte_len;
-
-    lexer->current.byte_pos += lexer->current_character_bytes;
-    lexer->current.char_pos += 1;
-    if (lexer->current_code_point == '\n') {
-        lexer->current.line += 1;
-        lexer->current.column = 0;
-        lexer->first_token_of_line = true;
-    } else {
-        lexer->current.column += 1;
-    }
-
-    if (rbs_next_char(lexer, &codepoint, &byte_len)) {
-        lexer->current_code_point = codepoint;
-        lexer->current_character_bytes = byte_len;
-    } else {
-        lexer->current_character_bytes = 1;
-        lexer->current_code_point = '\0';
-    }
+  return t;
 }
 
-rbs_token_t rbs_next_token(rbs_lexer_t *lexer, enum RBSTokenType type) {
+rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
+  if ((size_t) lexer->current.byte_pos == rbs_string_len(lexer->string) + 1) {
+    // End of String
     rbs_token_t t;
-
-    t.type = type;
+    t.type = pEOF;
     t.range.start = lexer->start;
-    t.range.end = lexer->current;
+    t.range.end = lexer->start;
     lexer->start = lexer->current;
-    if (type != tTRIVIA) {
-        lexer->first_token_of_line = false;
-    }
 
     return t;
+  } else {
+    // NULL byte in the middle of the string
+    return rbs_next_token(lexer, pEOF);
+  }
 }
 
-rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
-    if ((size_t) lexer->current.byte_pos == rbs_string_len(lexer->string) + 1) {
-        // End of String
-        rbs_token_t t;
-        t.type = pEOF;
-        t.range.start = lexer->start;
-        t.range.end = lexer->start;
-        lexer->start = lexer->current;
-
-        return t;
-    } else {
-        // NULL byte in the middle of the string
-        return rbs_next_token(lexer, pEOF);
-    }
+void rbs_skip(rbs_lexer_t *lexer) {
+  if (!lexer->last_char) {
+    rbs_peek(lexer);
+  }
+
+  size_t byte_len;
+
+  if (lexer->last_char == '\0') {
+    byte_len = 1;
+  } else {
+    const char *start = lexer->string.start + lexer->current.byte_pos;
+    byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
+  }
+
+  lexer->current.char_pos += 1;
+  lexer->current.byte_pos += byte_len;
+
+  if (lexer->last_char == '\n') {
+    lexer->current.line += 1;
+    lexer->current.column = 0;
+    lexer->first_token_of_line = true;
+  } else {
+    lexer->current.column += 1;
+  }
 }
 
 void rbs_skipn(rbs_lexer_t *lexer, size_t size) {
-    for (size_t i = 0; i < size; i++) {
-        rbs_skip(lexer);
-    }
+  for (size_t i = 0; i < size; i ++) {
+    rbs_peek(lexer);
+    rbs_skip(lexer);
+  }
 }
 
 char *rbs_peek_token(rbs_lexer_t *lexer, rbs_token_t tok) {
-    return (char *) lexer->string.start + tok.range.start.byte_pos;
+  return (char *) lexer->string.start + tok.range.start.byte_pos;
 }

data/src/location.c

@@ -6,66 +6,66 @@
 #define RBS_LOC_CHILDREN_SIZE(cap) (sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * ((cap) - 1))
 
 void rbs_loc_alloc_children(rbs_allocator_t *allocator, rbs_location_t *loc, size_t capacity) {
-    RBS_ASSERT(capacity <= sizeof(rbs_loc_entry_bitmap) * 8, "Capacity %zu is too large. Max is %zu", capacity, sizeof(rbs_loc_entry_bitmap) * 8);
+  rbs_assert(capacity <= sizeof(rbs_loc_entry_bitmap) * 8, "Capacity %zu is too large. Max is %zu", capacity, sizeof(rbs_loc_entry_bitmap) * 8);
 
-    loc->children = (rbs_loc_children *) rbs_allocator_malloc_impl(allocator, RBS_LOC_CHILDREN_SIZE(capacity), rbs_alignof(rbs_loc_children));
+  loc->children = rbs_allocator_malloc_impl(allocator, RBS_LOC_CHILDREN_SIZE(capacity), alignof(rbs_loc_children));
 
-    loc->children->len = 0;
-    loc->children->required_p = 0;
-    loc->children->cap = capacity;
+  loc->children->len = 0;
+  loc->children->required_p = 0;
+  loc->children->cap = capacity;
 }
 
 void rbs_loc_add_optional_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r) {
-    RBS_ASSERT(loc->children != NULL, "All children should have been pre-allocated with rbs_loc_alloc_children()");
-    RBS_ASSERT((loc->children->len + 1 <= loc->children->cap), "Not enough space was pre-allocated for the children. Children: %hu, Capacity: %hu", loc->children->len, loc->children->cap);
+  rbs_assert(loc->children != NULL, "All children should have been pre-allocated with rbs_loc_alloc_children()");
+  rbs_assert((loc->children->len + 1 <= loc->children->cap), "Not enough space was pre-allocated for the children. Children: %hu, Capacity: %hu", loc->children->len, loc->children->cap);
 
-    unsigned short i = loc->children->len++;
-    loc->children->entries[i].name = name;
-    loc->children->entries[i].rg = (rbs_loc_range) { r.start.char_pos, r.end.char_pos };
+  unsigned short i = loc->children->len++;
+  loc->children->entries[i].name = name;
+  loc->children->entries[i].rg = (rbs_loc_range) { r.start.char_pos, r.end.char_pos };
 }
 
 void rbs_loc_add_required_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r) {
-    rbs_loc_add_optional_child(loc, name, r);
-    unsigned short last_index = loc->children->len - 1;
-    loc->children->required_p |= 1 << last_index;
+  rbs_loc_add_optional_child(loc, name, r);
+  unsigned short last_index = loc->children->len - 1;
+  loc->children->required_p |= 1 << last_index;
 }
 
 rbs_location_t *rbs_location_new(rbs_allocator_t *allocator, rbs_range_t rg) {
-    rbs_location_t *location = rbs_allocator_alloc(allocator, rbs_location_t);
-    *location = (rbs_location_t) {
-        .rg = rg,
-        .children = NULL,
-    };
+  rbs_location_t *location = rbs_allocator_alloc(allocator, rbs_location_t);
+  *location = (rbs_location_t) {
+    .rg = rg,
+    .children = NULL,
+  };
 
-    return location;
+  return location;
 }
 
 rbs_location_list_t *rbs_location_list_new(rbs_allocator_t *allocator) {
-    rbs_location_list_t *list = rbs_allocator_alloc(allocator, rbs_location_list_t);
-    *list = (rbs_location_list_t) {
-        .allocator = allocator,
-        .head = NULL,
-        .tail = NULL,
-        .length = 0,
-    };
+  rbs_location_list_t *list = rbs_allocator_alloc(allocator, rbs_location_list_t);
+  *list = (rbs_location_list_t) {
+    .allocator = allocator,
+    .head = NULL,
+    .tail = NULL,
+    .length = 0,
+  };
 
-    return list;
+  return list;
 }
 
 void rbs_location_list_append(rbs_location_list_t *list, rbs_location_t *loc) {
-    rbs_location_list_node_t *node = rbs_allocator_alloc(list->allocator, rbs_location_list_node_t);
-    *node = (rbs_location_list_node_t) {
-        .loc = loc,
-        .next = NULL,
-    };
+  rbs_location_list_node_t *node = rbs_allocator_alloc(list->allocator, rbs_location_list_node_t);
+  *node = (rbs_location_list_node_t) {
+    .loc = loc,
+    .next = NULL,
+  };
 
-    if (list->head == NULL) {
-        list->head = node;
-        list->tail = node;
-    } else {
-        list->tail->next = node;
-        list->tail = node;
-    }
+  if (list->head == NULL) {
+    list->head = node;
+    list->tail = node;
+  } else {
+    list->tail->next = node;
+    list->tail = node;
+  }
 
-    list->length++;
+  list->length++;
 }