rbs 3.9.4 → 4.2.0

data/src/lexer.re

@@ -0,0 +1,187 @@
+#include "rbs/lexer.h"
+
+rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
+  rbs_lexer_t backup;
+
+  backup = *lexer;
+
+  /*!re2c
+      re2c:flags:u = 1;
+      re2c:api:style = free-form;
+      re2c:flags:input = custom;
+      re2c:define:YYCTYPE = "unsigned int";
+      re2c:define:YYPEEK = "rbs_peek(lexer)";
+      re2c:define:YYSKIP = "rbs_skip(lexer);";
+      re2c:define:YYBACKUP = "backup = *lexer;";
+      re2c:define:YYRESTORE = "*lexer = backup;";
+      re2c:yyfill:enable  = 0;
+
+      // Ruby's rule for what an identifier is made of is `ISALNUM(c) ||
+      // (c) == '_' || !ISASCII(c)`, so define the class by what it leaves
+      // out: every ASCII character that is not a word character, and the
+      // sentinel `rbs_next_char` reports for a byte that is invalid under
+      // the active encoding. Everything else can appear in an identifier,
+      // which is how a non-ASCII character gets in.
+      ident_char = . \ ([\x00-\x7F] \ [a-zA-Z0-9_]) \ [\uFFFD];
+
+      // An ASCII digit cannot lead an identifier, but a non-ASCII one can --
+      // Ruby accepts a name made of full-width digits. Since `rbs_next_char`
+      // never reports a non-ASCII character as `[0-9]`, subtracting the ASCII
+      // digits from `ident_char` is the whole of that rule.
+      ident_start = ident_char \ [0-9];
+
+      operator = "/" | "~" | "[]=" | "!" | "!=" | "!~" | "-" | "-@" | "+" | "+@"
+               | "==" | "===" | "=~" | "<<" | "<=" | "<=>" | ">" | ">=" | ">>" | "%";
+
+      "("   { return rbs_next_token(lexer, pLPAREN); }
+      ")"   { return rbs_next_token(lexer, pRPAREN); }
+      "["   { return rbs_next_token(lexer, pLBRACKET); }
+      "]"   { return rbs_next_token(lexer, pRBRACKET); }
+      "{"   { return rbs_next_token(lexer, pLBRACE); }
+      "}"   { return rbs_next_token(lexer, pRBRACE); }
+      ","   { return rbs_next_token(lexer, pCOMMA); }
+      "|"   { return rbs_next_token(lexer, pBAR); }
+      "^"   { return rbs_next_token(lexer, pHAT); }
+      "&"   { return rbs_next_token(lexer, pAMP); }
+      "?"   { return rbs_next_token(lexer, pQUESTION); }
+      "*"   { return rbs_next_token(lexer, pSTAR); }
+      "**"  { return rbs_next_token(lexer, pSTAR2); }
+      "."   { return rbs_next_token(lexer, pDOT); }
+      "..." { return rbs_next_token(lexer, pDOT3); }
+      "`"   {  return rbs_next_token(lexer, tOPERATOR); }
+      "`"   [^ :\x00] [^`\x00]* "`" { return rbs_next_token(lexer, tQIDENT); }
+      "->"  { return rbs_next_token(lexer, pARROW); }
+      "=>"  { return rbs_next_token(lexer, pFATARROW); }
+      "="   { return rbs_next_token(lexer, pEQ); }
+      ":"   { return rbs_next_token(lexer, pCOLON); }
+      "::"  { return rbs_next_token(lexer, pCOLON2); }
+      "<"   { return rbs_next_token(lexer, pLT); }
+      ">"   { return rbs_next_token(lexer, pGT); }
+      "[]"  { return rbs_next_token(lexer, pAREF_OPR); }
+      operator  { return rbs_next_token(lexer, tOPERATOR); }
+      "--" [^\x00]* { return rbs_next_token(lexer, tINLINECOMMENT); }
+
+      number = [0-9] [0-9_]*;
+      ("-"|"+")? number    { return rbs_next_token(lexer, tINTEGER); }
+
+      "%a{" [^}\x00]* "}"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a(" [^)\x00]* ")"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a[" [^\]\x00]* "]" { return rbs_next_token(lexer, tANNOTATION); }
+      "%a|" [^|\x00]* "|"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a<" [^>\x00]* ">"  { return rbs_next_token(lexer, tANNOTATION); }
+
+      "#" (. \ [\x00\uFFFD])*    {
+        // Keep a bare CR in the comment, but leave the CR in CRLF for a trivia token.
+        if (rbs_peek(lexer) == '\n' && lexer->string.start[lexer->current.byte_pos - 1] == '\r') {
+          lexer->current.byte_pos -= 1;
+          lexer->current.char_pos -= 1;
+          lexer->current.column -= 1;
+          lexer->current_code_point = '\r';
+          lexer->current_character_bytes = 1;
+        }
+
+        return rbs_next_token(
+          lexer,
+          lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT
+        );
+      }
+
+      "alias"         { return rbs_next_token(lexer, kALIAS); }
+      "attr_accessor" { return rbs_next_token(lexer, kATTRACCESSOR); }
+      "attr_reader"   { return rbs_next_token(lexer, kATTRREADER); }
+      "attr_writer"   { return rbs_next_token(lexer, kATTRWRITER); }
+      "bool"          { return rbs_next_token(lexer, kBOOL); }
+      "bot"           { return rbs_next_token(lexer, kBOT); }
+      "class"         { return rbs_next_token(lexer, kCLASS); }
+      "class-alias"   { return rbs_next_token(lexer, kCLASSALIAS); }
+      "def"           { return rbs_next_token(lexer, kDEF); }
+      "end"           { return rbs_next_token(lexer, kEND); }
+      "extend"        { return rbs_next_token(lexer, kEXTEND); }
+      "false"         { return rbs_next_token(lexer, kFALSE); }
+      "in"            { return rbs_next_token(lexer, kIN); }
+      "include"       { return rbs_next_token(lexer, kINCLUDE); }
+      "instance"      { return rbs_next_token(lexer, kINSTANCE); }
+      "interface"     { return rbs_next_token(lexer, kINTERFACE); }
+      "module"        { return rbs_next_token(lexer, kMODULE); }
+      "module-alias"  { return rbs_next_token(lexer, kMODULEALIAS); }
+      "module-self"   { return rbs_next_token(lexer, kMODULESELF); }
+      "nil"           { return rbs_next_token(lexer, kNIL); }
+      "out"           { return rbs_next_token(lexer, kOUT); }
+      "prepend"       { return rbs_next_token(lexer, kPREPEND); }
+      "private"       { return rbs_next_token(lexer, kPRIVATE); }
+      "public"        { return rbs_next_token(lexer, kPUBLIC); }
+      "self"          { return rbs_next_token(lexer, kSELF); }
+      "singleton"     { return rbs_next_token(lexer, kSINGLETON); }
+      "top"           { return rbs_next_token(lexer, kTOP); }
+      "true"          { return rbs_next_token(lexer, kTRUE); }
+      "type"          { return rbs_next_token(lexer, kTYPE); }
+      "unchecked"     { return rbs_next_token(lexer, kUNCHECKED); }
+      "untyped"       { return rbs_next_token(lexer, kUNTYPED); }
+      "void"          { return rbs_next_token(lexer, kVOID); }
+      "use"           { return rbs_next_token(lexer, kUSE); }
+      "as"            { return rbs_next_token(lexer, kAS); }
+      "__todo__"      { return rbs_next_token(lexer, k__TODO__); }
+      "@rbs"          { return rbs_next_token(lexer, kATRBS); }
+      "skip"          { return rbs_next_token(lexer, kSKIP); }
+      "return"        { return rbs_next_token(lexer, kRETURN); }
+
+      unicode_char = "\\u" [0-9a-fA-F]{4};
+      oct_char = "\\x" [0-9a-f]{1,2};
+      hex_char = "\\" [0-7]{1,3};
+
+      dqstring = ["] (unicode_char | oct_char | hex_char | "\\" [^xu] | [^\\"\x00])* ["];
+      sqstring = ['] ("\\"['\\] | [^'\x00])* ['];
+
+      dqstring     { return rbs_next_token(lexer, tDQSTRING); }
+      sqstring     { return rbs_next_token(lexer, tSQSTRING); }
+      ":" dqstring { return rbs_next_token(lexer, tDQSYMBOL); }
+      ":" sqstring { return rbs_next_token(lexer, tSQSYMBOL); }
+
+      symbol_opr = ":|" | ":&" | ":/" | ":%" | ":~" | ":`" | ":^"
+                 | ":==" | ":=~" | ":===" | ":!" | ":!=" | ":!~"
+                 | ":<" | ":<=" | ":<<" | ":<=>" | ":>" | ":>=" | ":>>"
+                 | ":-" | ":-@" | ":+" | ":+@" | ":*" | ":**" | ":[]" | ":[]=";
+
+      global_ident = [0-9]+
+                   | "-" [a-zA-Z0-9_]
+                   | [~*$?!@\\/;,.=:<>"&'`+]
+                   | [^ \t\r\n:;=.,!"$%&()-+~|\\'[\]{}*/<>^\x00]+;
+
+      ":" "@"{0,2} ident_start ident_char* [!?=]?  { return rbs_next_token(lexer, tSYMBOL); }
+      ":$" global_ident                            { return rbs_next_token(lexer, tSYMBOL); }
+      symbol_opr                                   { return rbs_next_token(lexer, tSYMBOL); }
+
+      [a-z] ident_char*          { return rbs_next_token(lexer, tLIDENT); }
+      [A-Z] ident_char*          { return rbs_next_token(lexer, tUIDENT); }
+      "_" [a-z0-9_] ident_char*  { return rbs_next_token(lexer, tULLIDENT); }
+      "_" [A-Z] ident_char*      { return rbs_next_token(lexer, tULIDENT); }
+      "_"                        { return rbs_next_token(lexer, tULLIDENT); }
+
+      // Every ASCII spelling is covered by one of the rules above, which match
+      // exactly as far, and re2c settles a tie of equal length in favour of the
+      // rule written first. So this one is left with the names that open with a
+      // character outside ASCII.
+      //
+      // RBS reads the case of that character to tell a class name from an alias
+      // name, and outside ASCII there is no reading of it that both the lexer
+      // and `TypeName#kind` can agree on. Those names get a token of their own
+      // and the parser takes it only where the question does not come up.
+      ident_start ident_char* { return rbs_next_token(lexer, tNONASCIIIDENT); }
+
+      // None of these four needs to know the case of the first character, so
+      // widening the leading position to `ident_start` is all they need.
+      ident_start ident_char* "!"  { return rbs_next_token(lexer, tBANGIDENT); }
+      ident_start ident_char* "="  { return rbs_next_token(lexer, tEQIDENT); }
+
+      "@" ident_start ident_char*  { return rbs_next_token(lexer, tAIDENT); }
+      "@@" ident_start ident_char* { return rbs_next_token(lexer, tA2IDENT); }
+
+      "$" global_ident      { return rbs_next_token(lexer, tGIDENT); }
+
+      skip = ([ \t]+|[\r\n]);
+
+      skip     { return rbs_next_token(lexer, tTRIVIA); }
+      "\x00"   { return rbs_next_eof_token(lexer); }
+      *        { return rbs_next_token(lexer, ErrorToken); }
+  */
+}

data/src/lexstate.c

@@ -0,0 +1,222 @@
+#include "rbs/defines.h"
+#include "rbs/lexer.h"
+#include "rbs/util/rbs_assert.h"
+
+static const char *RBS_TOKENTYPE_NAMES[] = {
+    "NullType",
+    "pEOF",
+    "ErrorToken",
+
+    "pLPAREN",   /* ( */
+    "pRPAREN",   /* ) */
+    "pCOLON",    /* : */
+    "pCOLON2",   /* :: */
+    "pLBRACKET", /* [ */
+    "pRBRACKET", /* ] */
+    "pLBRACE",   /* { */
+    "pRBRACE",   /* } */
+    "pHAT",      /* ^ */
+    "pARROW",    /* -> */
+    "pFATARROW", /* => */
+    "pCOMMA",    /* , */
+    "pBAR",      /* | */
+    "pAMP",      /* & */
+    "pSTAR",     /* * */
+    "pSTAR2",    /* ** */
+    "pDOT",      /* . */
+    "pDOT3",     /* ... */
+    "pBANG",     /* ! */
+    "pQUESTION", /* ? */
+    "pLT",       /* < */
+    "pGT",       /* > */
+    "pEQ",       /* = */
+
+    "kALIAS",        /* alias */
+    "kATTRACCESSOR", /* attr_accessor */
+    "kATTRREADER",   /* attr_reader */
+    "kATTRWRITER",   /* attr_writer */
+    "kBOOL",         /* bool */
+    "kBOT",          /* bot */
+    "kCLASS",        /* class */
+    "kCLASSALIAS",   /* class-alias */
+    "kDEF",          /* def */
+    "kEND",          /* end */
+    "kEXTEND",       /* extend */
+    "kFALSE",        /* kFALSE */
+    "kIN",           /* in */
+    "kINCLUDE",      /* include */
+    "kINSTANCE",     /* instance */
+    "kINTERFACE",    /* interface */
+    "kMODULE",       /* module */
+    "kMODULEALIAS",  /* module-alias */
+    "kNIL",          /* nil */
+    "kOUT",          /* out */
+    "kPREPEND",      /* prepend */
+    "kPRIVATE",      /* private */
+    "kPUBLIC",       /* public */
+    "kSELF",         /* self */
+    "kSINGLETON",    /* singleton */
+    "kTOP",          /* top */
+    "kTRUE",         /* true */
+    "kTYPE",         /* type */
+    "kUNCHECKED",    /* unchecked */
+    "kUNTYPED",      /* untyped */
+    "kVOID",         /* void */
+    "kUSE",          /* use */
+    "kAS",           /* as */
+    "k__TODO__",     /* __todo__ */
+    "kATRBS",        /* @rbs */
+    "kSKIP",         /* skip */
+    "kRETURN",       /* return */
+    "kMODULESELF",   /* module-self */
+
+    "tLIDENT",  /* Identifiers starting with lower case */
+    "tUIDENT",  /* Identifiers starting with upper case */
+    "tULIDENT", /* Identifiers starting with `_` */
+    "tULLIDENT",
+    "tNONASCIIIDENT",
+    "tGIDENT",  /* Identifiers starting with `$` */
+    "tAIDENT",  /* Identifiers starting with `@` */
+    "tA2IDENT", /* Identifiers starting with `@@` */
+    "tBANGIDENT",
+    "tEQIDENT",
+    "tQIDENT",   /* Quoted identifier */
+    "pAREF_OPR", /* [] */
+    "tOPERATOR", /* Operator identifier */
+
+    "tCOMMENT",
+    "tLINECOMMENT",
+    "tINLINECOMMENT",
+
+    "tTRIVIA",
+
+    "tDQSTRING", /* Double quoted string */
+    "tSQSTRING", /* Single quoted string */
+    "tINTEGER",  /* Integer */
+    "tSYMBOL",   /* Symbol */
+    "tDQSYMBOL",
+    "tSQSYMBOL",
+    "tANNOTATION", /* Annotation */
+};
+
+const rbs_position_t NullPosition = { -1, -1, -1, -1 };
+const rbs_range_t NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
+const rbs_token_t NullToken = { .type = NullType, .range = { { 0 }, { 0 } } };
+
+const char *rbs_token_type_str(enum RBSTokenType type) {
+    return RBS_TOKENTYPE_NAMES[type];
+}
+
+int rbs_token_chars(rbs_token_t tok) {
+    return tok.range.end.char_pos - tok.range.start.char_pos;
+}
+
+int rbs_token_bytes(rbs_token_t tok) {
+    return RBS_RANGE_BYTES(tok.range);
+}
+
+unsigned int rbs_peek(rbs_lexer_t *lexer) {
+    return lexer->current_code_point;
+}
+
+bool rbs_next_char(rbs_lexer_t *lexer, unsigned int *codepoint, size_t *byte_len) {
+    if (RBS_UNLIKELY(lexer->current.byte_pos >= lexer->end_pos)) {
+        return false;
+    }
+
+    const char *start = lexer->string.start + lexer->current.byte_pos;
+
+    // Fast path for ASCII (single-byte) characters
+    if ((unsigned int) *start < 128) {
+        *codepoint = (unsigned int) *start;
+        *byte_len = 1;
+        return true;
+    }
+
+    *byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
+
+    if (*byte_len == 0) {
+        // Invalid byte under the active encoding. Map it to a sentinel code
+        // point (U+FFFD) and advance one byte so the lexer always makes
+        // progress. Token rules that scan until a delimiter exclude this
+        // sentinel, so an invalid byte surfaces as an ErrorToken instead of
+        // being silently swallowed.
+        *byte_len = 1;
+        *codepoint = 0xFFFD;
+    } else if (*byte_len == 1) {
+        *codepoint = (unsigned int) *start;
+    } else {
+        *codepoint = 12523; // Dummy data for "ル" from "ルビー" (Ruby) in Unicode
+    }
+
+    return true;
+}
+
+void rbs_skip(rbs_lexer_t *lexer) {
+    RBS_ASSERT(lexer->current_character_bytes > 0, "rbs_skip called with current_character_bytes == 0");
+
+    if (RBS_UNLIKELY(lexer->current_code_point == '\0')) {
+        return;
+    }
+
+    unsigned int codepoint;
+    size_t byte_len;
+
+    lexer->current.byte_pos += lexer->current_character_bytes;
+    lexer->current.char_pos += 1;
+    if (lexer->current_code_point == '\n') {
+        lexer->current.line += 1;
+        lexer->current.column = 0;
+        lexer->first_token_of_line = true;
+    } else {
+        lexer->current.column += 1;
+    }
+
+    if (rbs_next_char(lexer, &codepoint, &byte_len)) {
+        lexer->current_code_point = codepoint;
+        lexer->current_character_bytes = byte_len;
+    } else {
+        lexer->current_character_bytes = 1;
+        lexer->current_code_point = '\0';
+    }
+}
+
+rbs_token_t rbs_next_token(rbs_lexer_t *lexer, enum RBSTokenType type) {
+    rbs_token_t t;
+
+    t.type = type;
+    t.range.start = lexer->start;
+    t.range.end = lexer->current;
+    lexer->start = lexer->current;
+    if (type != tTRIVIA) {
+        lexer->first_token_of_line = false;
+    }
+
+    return t;
+}
+
+rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
+    if ((size_t) lexer->current.byte_pos == rbs_string_len(lexer->string) + 1) {
+        // End of String
+        rbs_token_t t;
+        t.type = pEOF;
+        t.range.start = lexer->start;
+        t.range.end = lexer->start;
+        lexer->start = lexer->current;
+
+        return t;
+    } else {
+        // NULL byte in the middle of the string
+        return rbs_next_token(lexer, pEOF);
+    }
+}
+
+void rbs_skipn(rbs_lexer_t *lexer, size_t size) {
+    for (size_t i = 0; i < size; i++) {
+        rbs_skip(lexer);
+    }
+}
+
+char *rbs_peek_token(rbs_lexer_t *lexer, rbs_token_t tok) {
+    return (char *) lexer->string.start + tok.range.start.byte_pos;
+}

data/src/location.c

@@ -0,0 +1,31 @@
+#include "rbs/location.h"
+
+rbs_location_range_list_t *rbs_location_range_list_new(rbs_allocator_t *allocator) {
+    rbs_location_range_list_t *list = rbs_allocator_alloc(allocator, rbs_location_range_list_t);
+    *list = (rbs_location_range_list_t) {
+        .allocator = allocator,
+        .head = NULL,
+        .tail = NULL,
+        .length = 0,
+    };
+
+    return list;
+}
+
+void rbs_location_range_list_append(rbs_location_range_list_t *list, rbs_location_range range) {
+    rbs_location_range_list_node_t *node = rbs_allocator_alloc(list->allocator, rbs_location_range_list_node_t);
+    *node = (rbs_location_range_list_node_t) {
+        .range = range,
+        .next = NULL,
+    };
+
+    if (list->head == NULL) {
+        list->head = node;
+        list->tail = node;
+    } else {
+        list->tail->next = node;
+        list->tail = node;
+    }
+
+    list->length++;
+}