rbs 2.8.4 → 3.8.1

data/ext/rbs_extension/lexer.h

@@ -58,6 +58,9 @@ enum TokenType {
   kUNCHECKED,       /* unchecked */
   kUNTYPED,         /* untyped */
   kVOID,            /* void */
+  kUSE,             /* use */
+  kAS,              /* as */
+  k__TODO__,        /* __todo__ */
 
   tLIDENT,          /* Identifiers starting with lower case */
   tUIDENT,          /* Identifiers starting with upper case */
@@ -75,6 +78,8 @@ enum TokenType {
   tCOMMENT,         /* Comment */
   tLINECOMMENT,     /* Comment of all line */
 
+  tTRIVIA,          /* Trivia tokens -- space and new line */
+
   tDQSTRING,        /* Double quoted string */
   tSQSTRING,        /* Single quoted string */
   tINTEGER,         /* Integer */
@@ -150,7 +155,7 @@ unsigned int peek(lexstate *state);
 /**
  * Skip one character.
  * */
-void skip(lexstate *state);
+void rbs_skip(lexstate *state);
 
 /**
  * Skip n characters.
@@ -162,6 +167,11 @@ void skipn(lexstate *state, size_t size);
  * */
 token next_token(lexstate *state, enum TokenType type);
 
+/**
+ * Return new token with EOF type.
+ * */
+token next_eof_token(lexstate *state);
+
 token rbsparser_next_token(lexstate *state);
 
 void print_token(token tok);

data/ext/rbs_extension/lexer.re

@@ -3,7 +3,6 @@
 token rbsparser_next_token(lexstate *state) {
   lexstate backup;
 
-start:
   backup = *state;
 
   /*!re2c
@@ -12,7 +11,7 @@ start:
       re2c:flags:input = custom;
       re2c:define:YYCTYPE = "unsigned int";
       re2c:define:YYPEEK = "peek(state)";
-      re2c:define:YYSKIP = "skip(state);";
+      re2c:define:YYSKIP = "rbs_skip(state);";
       re2c:define:YYBACKUP = "backup = *state;";
       re2c:define:YYRESTORE = "*state = backup;";
       re2c:yyfill:enable  = 0;
@@ -93,8 +92,15 @@ start:
       "unchecked"     { return next_token(state, kUNCHECKED); }
       "untyped"       { return next_token(state, kUNTYPED); }
       "void"          { return next_token(state, kVOID); }
+      "use"           { return next_token(state, kUSE); }
+      "as"            { return next_token(state, kAS); }
+      "__todo__"      { return next_token(state, k__TODO__); }
 
-      dqstring = ["] ("\\"[abefnrstv"\\] | [^"\\\x00])* ["];
+      unicode_char = "\\u" [0-9a-fA-F]{4};
+      oct_char = "\\x" [0-9a-f]{1,2};
+      hex_char = "\\" [0-7]{1,3};
+
+      dqstring = ["] (unicode_char | oct_char | hex_char | "\\" [^xu] | [^\\"\x00])* ["];
       sqstring = ['] ("\\"['\\] | [^'\x00])* ['];
 
       dqstring     { return next_token(state, tDQSTRING); }
@@ -132,10 +138,10 @@ start:
 
       "$" global_ident      { return next_token(state, tGIDENT); }
 
-      skip = [ \t\n\r]+;
+      skip = ([ \t]+|[\r\n]);
 
-      skip     { state->start = state->current; goto start; }
-      "\x00"   { return next_token(state, pEOF); }
+      skip     { return next_token(state, tTRIVIA); }
+      "\x00"   { return next_eof_token(state); }
       *        { return next_token(state, ErrorToken); }
   */
 }

data/ext/rbs_extension/lexstate.c

@@ -57,6 +57,9 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
   "kUNCHECKED",       /* unchecked */
   "kUNTYPED",         /* untyped */
   "kVOID",            /* void */
+  "kUSE",             /* use */
+  "kAS",              /* as */
+  "k__TODO__",        /* __todo__ */
 
   "tLIDENT",          /* Identifiers starting with lower case */
   "tUIDENT",          /* Identifiers starting with upper case */
@@ -74,6 +77,8 @@ static const char *RBS_TOKENTYPE_NAMES[] = {
   "tCOMMENT",
   "tLINECOMMENT",
 
+  "tTRIVIA",
+
   "tDQSTRING",        /* Double quoted string */
   "tSQSTRING",        /* Single quoted string */
   "tINTEGER",         /* Integer */
@@ -117,12 +122,30 @@ token next_token(lexstate *state, enum TokenType type) {
   t.range.start = state->start;
   t.range.end = state->current;
   state->start = state->current;
-  state->first_token_of_line = false;
+  if (type != tTRIVIA) {
+    state->first_token_of_line = false;
+  }
 
   return t;
 }
 
-void skip(lexstate *state) {
+token next_eof_token(lexstate *state) {
+  if (state->current.byte_pos == RSTRING_LEN(state->string)+1) {
+    // End of String
+    token t;
+    t.type = pEOF;
+    t.range.start = state->start;
+    t.range.end = state->start;
+    state->start = state->current;
+
+    return t;
+  } else {
+    // NULL byte in the middle of the string
+    return next_token(state, pEOF);
+  }
+}
+
+void rbs_skip(lexstate *state) {
   if (!state->last_char) {
     peek(state);
   }
@@ -143,7 +166,7 @@ void skip(lexstate *state) {
 void skipn(lexstate *state, size_t size) {
   for (size_t i = 0; i < size; i ++) {
     peek(state);
-    skip(state);
+    rbs_skip(state);
   }
 }
 

data/ext/rbs_extension/location.c

@@ -1,83 +1,88 @@
 #include "rbs_extension.h"
 
+#define RBS_LOC_REQUIRED_P(loc, i) ((loc)->children->required_p & (1 << (i)))
+#define RBS_LOC_OPTIONAL_P(loc, i) (!RBS_LOC_REQUIRED_P((loc), (i)))
+#define RBS_LOC_CHILDREN_SIZE(cap) (sizeof(rbs_loc_children) + sizeof(rbs_loc_entry) * ((cap) - 1))
+#define NULL_LOC_RANGE_P(rg) ((rg).start == -1)
+
+rbs_loc_range RBS_LOC_NULL_RANGE = { -1, -1 };
 VALUE RBS_Location;
 
-rbs_loc_list *rbs_loc_list_add(rbs_loc_list *list, const ID name, const range r) {
-  rbs_loc_list *new = malloc(sizeof(rbs_loc_list));
-  new->next = list;
-  new->name = name;
-  new->rg = r;
-  return new;
+position rbs_loc_position(int char_pos) {
+  position pos = { 0, char_pos, -1, -1 };
+  return pos;
 }
 
-rbs_loc_list *rbs_loc_list_dup(rbs_loc_list *list) {
-  if (list) {
-    return rbs_loc_list_add(rbs_loc_list_dup(list->next), list->name, list->rg);
-  } else {
-    return NULL;
-  }
+position rbs_loc_position3(int char_pos, int line, int column) {
+  position pos = { 0, char_pos, line, column };
+  return pos;
 }
 
-void rbs_loc_list_free(rbs_loc_list *list) {
-  while (list) {
-    rbs_loc_list *next = list->next;
-    free(list);
-    list = next;
-  }
+rbs_loc_range rbs_new_loc_range(range rg) {
+  rbs_loc_range r = { rg.start.char_pos, rg.end.char_pos };
+  return r;
 }
 
-bool rbs_loc_list_find(const rbs_loc_list *list, ID name, range *rg) {
-  while (list) {
-    if (list->name == name) {
-      *rg = list->rg;
-      return true;
-    }
-
-    list = list->next;
+static void check_children_max(unsigned short n) {
+  size_t max = sizeof(rbs_loc_entry_bitmap) * 8;
+  if (n > max) {
+    rb_raise(rb_eRuntimeError, "Too many children added to location: %d", n);
   }
-
-  return false;
 }
 
-size_t rbs_loc_list_size(const rbs_loc_list *list) {
-  size_t size = 0;
+void rbs_loc_alloc_children(rbs_loc *loc, unsigned short cap) {
+  check_children_max(cap);
 
-  while (list) {
-    size += 1;
-    list = list->next;
-  }
+  size_t s = RBS_LOC_CHILDREN_SIZE(cap);
+  loc->children = malloc(s);
 
-  return size;
+  *loc->children = (rbs_loc_children) {
+    .len = 0,
+    .required_p = 0,
+    .cap = cap,
+    .entries = {{ 0 }},
+  };
 }
 
-position rbs_loc_position(int char_pos) {
-  position pos = { 0, char_pos, -1, -1 };
-  return pos;
-}
-
-position rbs_loc_position3(int char_pos, int line, int column) {
-  position pos = { 0, char_pos, line, column };
-  return pos;
+static void check_children_cap(rbs_loc *loc) {
+  if (loc->children == NULL) {
+    rbs_loc_alloc_children(loc, 1);
+  } else {
+    if (loc->children->len == loc->children->cap) {
+      check_children_max(loc->children->cap + 1);
+      size_t s = RBS_LOC_CHILDREN_SIZE(++loc->children->cap);
+      loc->children = realloc(loc->children, s);
+    }
+  }
 }
 
 void rbs_loc_add_required_child(rbs_loc *loc, ID name, range r) {
-  loc->requireds = rbs_loc_list_add(loc->requireds, name, r);
+  rbs_loc_add_optional_child(loc, name, r);
+
+  unsigned short last_index = loc->children->len - 1;
+  loc->children->required_p |= 1 << last_index;
 }
 
 void rbs_loc_add_optional_child(rbs_loc *loc, ID name, range r) {
-  loc->optionals = rbs_loc_list_add(loc->optionals, name, r);
+  check_children_cap(loc);
+
+  unsigned short i = loc->children->len++;
+  loc->children->entries[i] = (rbs_loc_entry) {
+    .name = name,
+    .rg = rbs_new_loc_range(r),
+  };
 }
 
-void rbs_loc_init(rbs_loc *loc, VALUE buffer, range rg) {
-  loc->buffer = buffer;
-  loc->rg = rg;
-  loc->optionals = NULL;
-  loc->requireds = NULL;
+void rbs_loc_init(rbs_loc *loc, VALUE buffer, rbs_loc_range rg) {
+  *loc = (rbs_loc) {
+    .buffer = buffer,
+    .rg = rg,
+    .children = NULL,
+  };
 }
 
 void rbs_loc_free(rbs_loc *loc) {
-  rbs_loc_list_free(loc->optionals);
-  rbs_loc_list_free(loc->requireds);
+  free(loc->children);
   ruby_xfree(loc);
 }
 
@@ -89,7 +94,11 @@ static void rbs_loc_mark(void *ptr)
 
 static size_t rbs_loc_memsize(const void *ptr) {
   const rbs_loc *loc = ptr;
-  return sizeof(*loc) + (rbs_loc_list_size(loc->optionals) + rbs_loc_list_size(loc->requireds)) * sizeof(rbs_loc_list);
+  if (loc->children == NULL) {
+    return sizeof(rbs_loc);
+  } else {
+    return sizeof(rbs_loc) + RBS_LOC_CHILDREN_SIZE(loc->children->cap);
+  }
 }
 
 static rb_data_type_t location_type = {
@@ -102,7 +111,7 @@ static VALUE location_s_allocate(VALUE klass) {
   rbs_loc *loc;
   VALUE obj = TypedData_Make_Struct(klass, rbs_loc, &location_type, loc);
 
-  rbs_loc_init(loc, Qnil, NULL_RANGE);
+  rbs_loc_init(loc, Qnil, RBS_LOC_NULL_RANGE);
 
   return obj;
 }
@@ -114,12 +123,14 @@ rbs_loc *rbs_check_location(VALUE obj) {
 static VALUE location_initialize(VALUE self, VALUE buffer, VALUE start_pos, VALUE end_pos) {
   rbs_loc *loc = rbs_check_location(self);
 
-  position start = rbs_loc_position(FIX2INT(start_pos));
-  position end = rbs_loc_position(FIX2INT(end_pos));
+  int start = FIX2INT(start_pos);
+  int end = FIX2INT(end_pos);
 
-  loc->buffer = buffer;
-  loc->rg.start = start;
-  loc->rg.end = end;
+  *loc = (rbs_loc) {
+    .buffer = buffer,
+    .rg = (rbs_loc_range) { start, end },
+    .children = NULL,
+  };
 
   return Qnil;
 }
@@ -128,10 +139,16 @@ static VALUE location_initialize_copy(VALUE self, VALUE other) {
   rbs_loc *self_loc = rbs_check_location(self);
   rbs_loc *other_loc = rbs_check_location(other);
 
-  self_loc->buffer = other_loc->buffer;
-  self_loc->rg = other_loc->rg;
-  self_loc->requireds = rbs_loc_list_dup(other_loc->requireds);
-  self_loc->optionals = rbs_loc_list_dup(other_loc->optionals);
+  *self_loc = (rbs_loc) {
+    .buffer = other_loc->buffer,
+    .rg = other_loc->rg,
+    .children = NULL,
+  };
+
+  if (other_loc->children != NULL) {
+    rbs_loc_alloc_children(self_loc, other_loc->children->cap);
+    memcpy(self_loc->children, other_loc->children, RBS_LOC_CHILDREN_SIZE(other_loc->children->cap));
+  }
 
   return Qnil;
 }
@@ -143,38 +160,12 @@ static VALUE location_buffer(VALUE self) {
 
 static VALUE location_start_pos(VALUE self) {
   rbs_loc *loc = rbs_check_location(self);
-  return INT2FIX(loc->rg.start.char_pos);
+  return INT2FIX(loc->rg.start);
 }
 
 static VALUE location_end_pos(VALUE self) {
   rbs_loc *loc = rbs_check_location(self);
-  return INT2FIX(loc->rg.end.char_pos);
-}
-
-static VALUE location_start_loc(VALUE self) {
-  rbs_loc *loc = rbs_check_location(self);
-
-  if (loc->rg.start.line >= 0) {
-    VALUE pair = rb_ary_new_capa(2);
-    rb_ary_push(pair, INT2FIX(loc->rg.start.line));
-    rb_ary_push(pair, INT2FIX(loc->rg.start.column));
-    return pair;
-  } else {
-    return Qnil;
-  }
-}
-
-static VALUE location_end_loc(VALUE self) {
-  rbs_loc *loc = rbs_check_location(self);
-
-  if (loc->rg.end.line >= 0) {
-    VALUE pair = rb_ary_new_capa(2);
-    rb_ary_push(pair, INT2FIX(loc->rg.end.line));
-    rb_ary_push(pair, INT2FIX(loc->rg.end.column));
-    return pair;
-  } else {
-    return Qnil;
-  }
+  return INT2FIX(loc->rg.end);
 }
 
 static VALUE location_add_required_child(VALUE self, VALUE name, VALUE start, VALUE end) {
@@ -213,6 +204,15 @@ VALUE rbs_new_location(VALUE buffer, range rg) {
   rbs_loc *loc;
   VALUE obj = TypedData_Make_Struct(RBS_Location, rbs_loc, &location_type, loc);
 
+  rbs_loc_init(loc, buffer, rbs_new_loc_range(rg));
+
+  return obj;
+}
+
+static VALUE rbs_new_location_from_loc_range(VALUE buffer, rbs_loc_range rg) {
+  rbs_loc *loc;
+  VALUE obj = TypedData_Make_Struct(RBS_Location, rbs_loc, &location_type, loc);
+
   rbs_loc_init(loc, buffer, rg);
 
   return obj;
@@ -221,18 +221,19 @@ VALUE rbs_new_location(VALUE buffer, range rg) {
 static VALUE location_aref(VALUE self, VALUE name) {
   rbs_loc *loc = rbs_check_location(self);
 
-  range result;
   ID id = SYM2ID(name);
 
-  if (rbs_loc_list_find(loc->requireds, id, &result)) {
-    return rbs_new_location(loc->buffer, result);
-  }
-
-  if (rbs_loc_list_find(loc->optionals, id, &result)) {
-    if (null_range_p(result)) {
-      return Qnil;
-    } else {
-      return rbs_new_location(loc->buffer, result);
+  if (loc->children != NULL) {
+    for (unsigned short i = 0; i < loc->children->len; i++) {
+      if (loc->children->entries[i].name == id) {
+        rbs_loc_range result = loc->children->entries[i].rg;
+
+        if (RBS_LOC_OPTIONAL_P(loc, i) && NULL_LOC_RANGE_P(result)) {
+          return Qnil;
+        } else {
+          return rbs_new_location_from_loc_range(loc->buffer, result);
+        }
+      }
     }
   }
 
@@ -244,11 +245,16 @@ static VALUE location_optional_keys(VALUE self) {
   VALUE keys = rb_ary_new();
 
   rbs_loc *loc = rbs_check_location(self);
-  rbs_loc_list *list = loc->optionals;
+  rbs_loc_children *children = loc->children;
+  if (children == NULL) {
+    return keys;
+  }
+
+  for (unsigned short i = 0; i < children->len; i++) {
+    if (RBS_LOC_OPTIONAL_P(loc, i)) {
+      rb_ary_push(keys, ID2SYM(children->entries[i].name));
 
-  while (list) {
-    rb_ary_push(keys, ID2SYM(list->name));
-    list = list->next;
+    }
   }
 
   return keys;
@@ -258,11 +264,15 @@ static VALUE location_required_keys(VALUE self) {
   VALUE keys = rb_ary_new();
 
   rbs_loc *loc = rbs_check_location(self);
-  rbs_loc_list *list = loc->requireds;
+  rbs_loc_children *children = loc->children;
+  if (children == NULL) {
+    return keys;
+  }
 
-  while (list) {
-    rb_ary_push(keys, ID2SYM(list->name));
-    list = list->next;
+  for (unsigned short i = 0; i < children->len; i++) {
+    if (RBS_LOC_REQUIRED_P(loc, i)) {
+      rb_ary_push(keys, ID2SYM(children->entries[i].name));
+    }
   }
 
   return keys;
@@ -284,8 +294,6 @@ void rbs__init_location(void) {
   rb_define_method(RBS_Location, "buffer", location_buffer, 0);
   rb_define_method(RBS_Location, "start_pos", location_start_pos, 0);
   rb_define_method(RBS_Location, "end_pos", location_end_pos, 0);
-  rb_define_private_method(RBS_Location, "_start_loc", location_start_loc, 0);
-  rb_define_private_method(RBS_Location, "_end_loc", location_end_loc, 0);
   rb_define_method(RBS_Location, "_add_required_child", location_add_required_child, 3);
   rb_define_method(RBS_Location, "_add_optional_child", location_add_optional_child, 3);
   rb_define_method(RBS_Location, "_add_optional_no_child", location_add_optional_no_child, 1);

data/ext/rbs_extension/location.h

@@ -9,17 +9,31 @@
  * */
 extern VALUE RBS_Location;
 
-typedef struct rbs_loc_list {
+typedef struct {
+  int start;
+  int end;
+} rbs_loc_range;
+
+typedef struct {
   ID name;
-  range rg;
-  struct rbs_loc_list *next;
-} rbs_loc_list;
+  rbs_loc_range rg;
+} rbs_loc_entry;
+
+typedef unsigned int rbs_loc_entry_bitmap;
+
+// The flexible array always allocates, but it's okay.
+// This struct is not allocated when the `rbs_loc` doesn't have children.
+typedef struct {
+  unsigned short len;
+  unsigned short cap;
+  rbs_loc_entry_bitmap required_p;
+  rbs_loc_entry entries[1];
+} rbs_loc_children;
 
 typedef struct {
   VALUE buffer;
-  range rg;
-  rbs_loc_list *requireds;
-  rbs_loc_list *optionals;
+  rbs_loc_range rg;
+  rbs_loc_children *children; // NULL when no children is allocated
 } rbs_loc;
 
 /**
@@ -32,13 +46,24 @@ VALUE rbs_new_location(VALUE buffer, range rg);
  * */
 rbs_loc *rbs_check_location(VALUE location);
 
+/**
+ * Allocate memory for child locations.
+ *
+ * Do not call twice for the same location.
+ * */
+void rbs_loc_alloc_children(rbs_loc *loc, unsigned short cap);
+
 /**
  * Add a required child range with given name.
+ *
+ * Allocate memory for children with rbs_loc_alloc_children before calling this function.
  * */
 void rbs_loc_add_required_child(rbs_loc *loc, ID name, range r);
 
 /**
  * Add an optional child range with given name.
+ *
+ * Allocate memory for children with rbs_loc_alloc_children before calling this function.
  * */
 void rbs_loc_add_optional_child(rbs_loc *loc, ID name, range r);
 

data/ext/rbs_extension/main.c

@@ -3,6 +3,9 @@
 void
 Init_rbs_extension(void)
 {
+#ifdef HAVE_RB_EXT_RACTOR_SAFE
+  rb_ext_ractor_safe(true);
+#endif  
   rbs__init_constants();
   rbs__init_location();
   rbs__init_parser();