rbs 3.9.2 → 4.0.0.dev.1

data/src/lexer.re

@@ -0,0 +1,151 @@
+#include "rbs/lexer.h"
+
+rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) {
+  rbs_lexer_t backup;
+
+  backup = *lexer;
+
+  /*!re2c
+      re2c:flags:u = 1;
+      re2c:api:style = free-form;
+      re2c:flags:input = custom;
+      re2c:define:YYCTYPE = "unsigned int";
+      re2c:define:YYPEEK = "rbs_peek(lexer)";
+      re2c:define:YYSKIP = "rbs_skip(lexer);";
+      re2c:define:YYBACKUP = "backup = *lexer;";
+      re2c:define:YYRESTORE = "*lexer = backup;";
+      re2c:yyfill:enable  = 0;
+
+      word = [a-zA-Z0-9_];
+
+      operator = "/" | "~" | "[]=" | "!" | "!=" | "!~" | "-" | "-@" | "+" | "+@"
+               | "==" | "===" | "=~" | "<<" | "<=" | "<=>" | ">" | ">=" | ">>" | "%";
+
+      "("   { return rbs_next_token(lexer, pLPAREN); }
+      ")"   { return rbs_next_token(lexer, pRPAREN); }
+      "["   { return rbs_next_token(lexer, pLBRACKET); }
+      "]"   { return rbs_next_token(lexer, pRBRACKET); }
+      "{"   { return rbs_next_token(lexer, pLBRACE); }
+      "}"   { return rbs_next_token(lexer, pRBRACE); }
+      ","   { return rbs_next_token(lexer, pCOMMA); }
+      "|"   { return rbs_next_token(lexer, pBAR); }
+      "^"   { return rbs_next_token(lexer, pHAT); }
+      "&"   { return rbs_next_token(lexer, pAMP); }
+      "?"   { return rbs_next_token(lexer, pQUESTION); }
+      "*"   { return rbs_next_token(lexer, pSTAR); }
+      "**"  { return rbs_next_token(lexer, pSTAR2); }
+      "."   { return rbs_next_token(lexer, pDOT); }
+      "..." { return rbs_next_token(lexer, pDOT3); }
+      "`"   {  return rbs_next_token(lexer, tOPERATOR); }
+      "`"   [^ :\x00] [^`\x00]* "`" { return rbs_next_token(lexer, tQIDENT); }
+      "->"  { return rbs_next_token(lexer, pARROW); }
+      "=>"  { return rbs_next_token(lexer, pFATARROW); }
+      "="   { return rbs_next_token(lexer, pEQ); }
+      ":"   { return rbs_next_token(lexer, pCOLON); }
+      "::"  { return rbs_next_token(lexer, pCOLON2); }
+      "<"   { return rbs_next_token(lexer, pLT); }
+      "[]"  { return rbs_next_token(lexer, pAREF_OPR); }
+      operator  { return rbs_next_token(lexer, tOPERATOR); }
+      "--" [^\x00]* { return rbs_next_token(lexer, tINLINECOMMENT); }
+
+      number = [0-9] [0-9_]*;
+      ("-"|"+")? number    { return rbs_next_token(lexer, tINTEGER); }
+
+      "%a{" [^}\x00]* "}"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a(" [^)\x00]* ")"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a[" [^\]\x00]* "]" { return rbs_next_token(lexer, tANNOTATION); }
+      "%a|" [^|\x00]* "|"  { return rbs_next_token(lexer, tANNOTATION); }
+      "%a<" [^>\x00]* ">"  { return rbs_next_token(lexer, tANNOTATION); }
+
+      "#" (. \ [\x00])*    {
+        return rbs_next_token(
+          lexer,
+          lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT
+        );
+      }
+
+      "alias"         { return rbs_next_token(lexer, kALIAS); }
+      "attr_accessor" { return rbs_next_token(lexer, kATTRACCESSOR); }
+      "attr_reader"   { return rbs_next_token(lexer, kATTRREADER); }
+      "attr_writer"   { return rbs_next_token(lexer, kATTRWRITER); }
+      "bool"          { return rbs_next_token(lexer, kBOOL); }
+      "bot"           { return rbs_next_token(lexer, kBOT); }
+      "class"         { return rbs_next_token(lexer, kCLASS); }
+      "def"           { return rbs_next_token(lexer, kDEF); }
+      "end"           { return rbs_next_token(lexer, kEND); }
+      "extend"        { return rbs_next_token(lexer, kEXTEND); }
+      "false"         { return rbs_next_token(lexer, kFALSE); }
+      "in"            { return rbs_next_token(lexer, kIN); }
+      "include"       { return rbs_next_token(lexer, kINCLUDE); }
+      "instance"      { return rbs_next_token(lexer, kINSTANCE); }
+      "interface"     { return rbs_next_token(lexer, kINTERFACE); }
+      "module"        { return rbs_next_token(lexer, kMODULE); }
+      "nil"           { return rbs_next_token(lexer, kNIL); }
+      "out"           { return rbs_next_token(lexer, kOUT); }
+      "prepend"       { return rbs_next_token(lexer, kPREPEND); }
+      "private"       { return rbs_next_token(lexer, kPRIVATE); }
+      "public"        { return rbs_next_token(lexer, kPUBLIC); }
+      "self"          { return rbs_next_token(lexer, kSELF); }
+      "singleton"     { return rbs_next_token(lexer, kSINGLETON); }
+      "top"           { return rbs_next_token(lexer, kTOP); }
+      "true"          { return rbs_next_token(lexer, kTRUE); }
+      "type"          { return rbs_next_token(lexer, kTYPE); }
+      "unchecked"     { return rbs_next_token(lexer, kUNCHECKED); }
+      "untyped"       { return rbs_next_token(lexer, kUNTYPED); }
+      "void"          { return rbs_next_token(lexer, kVOID); }
+      "use"           { return rbs_next_token(lexer, kUSE); }
+      "as"            { return rbs_next_token(lexer, kAS); }
+      "__todo__"      { return rbs_next_token(lexer, k__TODO__); }
+      "@rbs"          { return rbs_next_token(lexer, kATRBS); }
+      "skip"          { return rbs_next_token(lexer, kSKIP); }
+      "return"        { return rbs_next_token(lexer, kRETURN); }
+
+      unicode_char = "\\u" [0-9a-fA-F]{4};
+      oct_char = "\\x" [0-9a-f]{1,2};
+      hex_char = "\\" [0-7]{1,3};
+
+      dqstring = ["] (unicode_char | oct_char | hex_char | "\\" [^xu] | [^\\"\x00])* ["];
+      sqstring = ['] ("\\"['\\] | [^'\x00])* ['];
+
+      dqstring     { return rbs_next_token(lexer, tDQSTRING); }
+      sqstring     { return rbs_next_token(lexer, tSQSTRING); }
+      ":" dqstring { return rbs_next_token(lexer, tDQSYMBOL); }
+      ":" sqstring { return rbs_next_token(lexer, tSQSYMBOL); }
+
+      identifier = [a-zA-Z_] word* [!?=]?;
+      symbol_opr = ":|" | ":&" | ":/" | ":%" | ":~" | ":`" | ":^"
+                 | ":==" | ":=~" | ":===" | ":!" | ":!=" | ":!~"
+                 | ":<" | ":<=" | ":<<" | ":<=>" | ":>" | ":>=" | ":>>"
+                 | ":-" | ":-@" | ":+" | ":+@" | ":*" | ":**" | ":[]" | ":[]=";
+
+      global_ident = [0-9]+
+                   | "-" [a-zA-Z0-9_]
+                   | [~*$?!@\\/;,.=:<>"&'`+]
+                   | [^ \t\r\n:;=.,!"$%&()-+~|\\'[\]{}*/<>^\x00]+;
+
+      ":" identifier     { return rbs_next_token(lexer, tSYMBOL); }
+      ":@" identifier    { return rbs_next_token(lexer, tSYMBOL); }
+      ":@@" identifier   { return rbs_next_token(lexer, tSYMBOL); }
+      ":$" global_ident  { return rbs_next_token(lexer, tSYMBOL); }
+      symbol_opr         { return rbs_next_token(lexer, tSYMBOL); }
+
+      [a-z] word*           { return rbs_next_token(lexer, tLIDENT); }
+      [A-Z] word*           { return rbs_next_token(lexer, tUIDENT); }
+      "_" [a-z0-9_] word*   { return rbs_next_token(lexer, tULLIDENT); }
+      "_" [A-Z] word*       { return rbs_next_token(lexer, tULIDENT); }
+      "_"                   { return rbs_next_token(lexer, tULLIDENT); }
+      [a-zA-Z_] word* "!"   { return rbs_next_token(lexer, tBANGIDENT); }
+      [a-zA-Z_] word* "="   { return rbs_next_token(lexer, tEQIDENT); }
+
+      "@" [a-zA-Z_] word*   { return rbs_next_token(lexer, tAIDENT); }
+      "@@" [a-zA-Z_] word*  { return rbs_next_token(lexer, tA2IDENT); }
+
+      "$" global_ident      { return rbs_next_token(lexer, tGIDENT); }
+
+      skip = ([ \t]+|[\r\n]);
+
+      skip     { return rbs_next_token(lexer, tTRIVIA); }
+      "\x00"   { return rbs_next_eof_token(lexer); }
+      *        { return rbs_next_token(lexer, ErrorToken); }
+  */
+}

data/{ext/rbs_extension → src}/lexstate.c

@@ -1,4 +1,4 @@
-#include "rbs_extension.h"
+#include "rbs/lexer.h"
 
 static const char *RBS_TOKENTYPE_NAMES[] = {
   "NullType",
@@ -60,6 +60,9 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
   "kUSE",             /* use */
   "kAS",              /* as */
   "k__TODO__",        /* __todo__ */
+  "kATRBS",           /* @rbs */
+  "kSKIP",            /* skip */
+  "kRETURN",          /* return */
 
   "tLIDENT",          /* Identifiers starting with lower case */
   "tUIDENT",          /* Identifiers starting with upper case */
@@ -76,6 +79,7 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
 
   "tCOMMENT",
   "tLINECOMMENT",
+  "tINLINECOMMENT",
 
   "tTRIVIA",
 
@@ -88,88 +92,100 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
   "tANNOTATION",      /* Annotation */
 };
 
-token NullToken = { NullType };
-position NullPosition = { -1, -1, -1, -1 };
-range NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
+rbs_token_t NullToken = { .type = NullType, .range = {} };
+rbs_position_t NullPosition = { -1, -1, -1, -1 };
+rbs_range_t NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
 
-const char *token_type_str(enum TokenType type) {
+const char *rbs_token_type_str(enum RBSTokenType type) {
   return RBS_TOKENTYPE_NAMES[type];
 }
 
-int token_chars(token tok) {
+int rbs_token_chars(rbs_token_t tok) {
   return tok.range.end.char_pos - tok.range.start.char_pos;
 }
 
-int token_bytes(token tok) {
-  return RANGE_BYTES(tok.range);
+int rbs_token_bytes(rbs_token_t tok) {
+  return RBS_RANGE_BYTES(tok.range);
 }
 
-unsigned int peek(lexstate *state) {
-  if (state->current.char_pos == state->end_pos) {
-    state->last_char = '\0';
+unsigned int rbs_peek(rbs_lexer_t *lexer) {
+  if (lexer->current.char_pos == lexer->end_pos) {
+    lexer->last_char = '\0';
     return 0;
   } else {
-    unsigned int c = rb_enc_mbc_to_codepoint(RSTRING_PTR(state->string) + state->current.byte_pos, RSTRING_END(state->string), rb_enc_get(state->string));
-    state->last_char = c;
+    rbs_string_t str = rbs_string_new(
+      lexer->string.start + lexer->current.byte_pos,
+      lexer->string.end
+    );
+    unsigned int c = rbs_utf8_string_to_codepoint(str);
+    lexer->last_char = c;
     return c;
   }
 }
 
-token next_token(lexstate *state, enum TokenType type) {
-  token t;
+rbs_token_t rbs_next_token(rbs_lexer_t *lexer, enum RBSTokenType type) {
+  rbs_token_t t;
 
   t.type = type;
-  t.range.start = state->start;
-  t.range.end = state->current;
-  state->start = state->current;
+  t.range.start = lexer->start;
+  t.range.end = lexer->current;
+  lexer->start = lexer->current;
   if (type != tTRIVIA) {
-    state->first_token_of_line = false;
+    lexer->first_token_of_line = false;
   }
 
   return t;
 }
 
-token next_eof_token(lexstate *state) {
-  if (state->current.byte_pos == RSTRING_LEN(state->string)+1) {
+rbs_token_t rbs_next_eof_token(rbs_lexer_t *lexer) {
+  if ((size_t) lexer->current.byte_pos == rbs_string_len(lexer->string) + 1) {
     // End of String
-    token t;
+    rbs_token_t t;
     t.type = pEOF;
-    t.range.start = state->start;
-    t.range.end = state->start;
-    state->start = state->current;
+    t.range.start = lexer->start;
+    t.range.end = lexer->start;
+    lexer->start = lexer->current;
 
     return t;
   } else {
     // NULL byte in the middle of the string
-    return next_token(state, pEOF);
+    return rbs_next_token(lexer, pEOF);
   }
 }
 
-void rbs_skip(lexstate *state) {
-  if (!state->last_char) {
-    peek(state);
+void rbs_skip(rbs_lexer_t *lexer) {
+  if (!lexer->last_char) {
+    rbs_peek(lexer);
   }
-  int byte_len = rb_enc_codelen(state->last_char, rb_enc_get(state->string));
 
-  state->current.char_pos += 1;
-  state->current.byte_pos += byte_len;
+  size_t byte_len;
 
-  if (state->last_char == '\n') {
-    state->current.line += 1;
-    state->current.column = 0;
-    state->first_token_of_line = true;
+  if (lexer->last_char == '\0') {
+    byte_len = 1;
   } else {
-    state->current.column += 1;
+    const char *start = lexer->string.start + lexer->current.byte_pos;
+    byte_len = lexer->encoding->char_width((const uint8_t *) start, (ptrdiff_t) (lexer->string.end - start));
+  }
+
+  lexer->current.char_pos += 1;
+  lexer->current.byte_pos += byte_len;
+
+  if (lexer->last_char == '\n') {
+    lexer->current.line += 1;
+    lexer->current.column = 0;
+    lexer->first_token_of_line = true;
+  } else {
+    lexer->current.column += 1;
   }
 }
 
-void skipn(lexstate *state, size_t size) {
+void rbs_skipn(rbs_lexer_t *lexer, size_t size) {
   for (size_t i = 0; i < size; i ++) {
-    peek(state);
-    rbs_skip(state);
+    rbs_peek(lexer);
+    rbs_skip(lexer);
   }
 }
 
-char *peek_token(lexstate *state, token tok) {
-  return RSTRING_PTR(state->string) + tok.range.start.byte_pos;
+char *rbs_peek_token(rbs_lexer_t *lexer, rbs_token_t tok) {
+  return (char *) lexer->string.start + tok.range.start.byte_pos;
 }

data/src/location.c

@@ -0,0 +1,71 @@
+#include "rbs/location.h"
+#include "rbs/util/rbs_assert.h"
+
+#include <stdio.h>
+
+#define RBS_LOC_CHILDREN_SIZE(cap) (sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * ((cap) - 1))
+
+void rbs_loc_alloc_children(rbs_allocator_t *allocator, rbs_location_t *loc, size_t capacity) {
+  rbs_assert(capacity <= sizeof(rbs_loc_entry_bitmap) * 8, "Capacity %zu is too large. Max is %zu", capacity, sizeof(rbs_loc_entry_bitmap) * 8);
+
+  loc->children = rbs_allocator_malloc_impl(allocator, RBS_LOC_CHILDREN_SIZE(capacity), alignof(rbs_loc_children));
+
+  loc->children->len = 0;
+  loc->children->required_p = 0;
+  loc->children->cap = capacity;
+}
+
+void rbs_loc_add_optional_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r) {
+  rbs_assert(loc->children != NULL, "All children should have been pre-allocated with rbs_loc_alloc_children()");
+  rbs_assert((loc->children->len + 1 <= loc->children->cap), "Not enough space was pre-allocated for the children. Children: %hu, Capacity: %hu", loc->children->len, loc->children->cap);
+
+  unsigned short i = loc->children->len++;
+  loc->children->entries[i].name = name;
+  loc->children->entries[i].rg = (rbs_loc_range) { r.start.char_pos, r.end.char_pos };
+}
+
+void rbs_loc_add_required_child(rbs_location_t *loc, rbs_constant_id_t name, rbs_range_t r) {
+  rbs_loc_add_optional_child(loc, name, r);
+  unsigned short last_index = loc->children->len - 1;
+  loc->children->required_p |= 1 << last_index;
+}
+
+rbs_location_t *rbs_location_new(rbs_allocator_t *allocator, rbs_range_t rg) {
+  rbs_location_t *location = rbs_allocator_alloc(allocator, rbs_location_t);
+  *location = (rbs_location_t) {
+    .rg = rg,
+    .children = NULL,
+  };
+
+  return location;
+}
+
+rbs_location_list_t *rbs_location_list_new(rbs_allocator_t *allocator) {
+  rbs_location_list_t *list = rbs_allocator_alloc(allocator, rbs_location_list_t);
+  *list = (rbs_location_list_t) {
+    .allocator = allocator,
+    .head = NULL,
+    .tail = NULL,
+    .length = 0,
+  };
+
+  return list;
+}
+
+void rbs_location_list_append(rbs_location_list_t *list, rbs_location_t *loc) {
+  rbs_location_list_node_t *node = rbs_allocator_alloc(list->allocator, rbs_location_list_node_t);
+  *node = (rbs_location_list_node_t) {
+    .loc = loc,
+    .next = NULL,
+  };
+
+  if (list->head == NULL) {
+    list->head = node;
+    list->tail = node;
+  } else {
+    list->tail->next = node;
+    list->tail = node;
+  }
+
+  list->length++;
+}