rbs 1.7.0.beta.2 → 1.7.0.beta.3

data/ext/rbs_extension/lexer.h

@@ -29,35 +29,35 @@ enum TokenType {
   pLT,              /* < */
   pEQ,              /* = */
 
+  kALIAS,           /* alias */
+  kATTRACCESSOR,    /* attr_accessor */
+  kATTRREADER,      /* attr_reader */
+  kATTRWRITER,      /* attr_writer */
   kBOOL,            /* bool */
   kBOT,             /* bot */
   kCLASS,           /* class */
+  kDEF,             /* def */
+  kEND,             /* end */
+  kEXTEND,          /* extend */
   kFALSE,           /* false */
+  kIN,              /* in */
+  kINCLUDE,         /* include */
   kINSTANCE,        /* instance */
   kINTERFACE,       /* interface */
+  kMODULE,          /* module */
   kNIL,             /* nil */
+  kOUT,             /* out */
+  kPREPEND,         /* prepend */
+  kPRIVATE,         /* private */
+  kPUBLIC,          /* public */
   kSELF,            /* self */
   kSINGLETON,       /* singleton */
   kTOP,             /* top */
   kTRUE,            /* true */
-  kVOID,            /* void */
   kTYPE,            /* type */
   kUNCHECKED,       /* unchecked */
-  kIN,              /* in */
-  kOUT,             /* out */
-  kEND,             /* end */
-  kDEF,             /* def */
-  kINCLUDE,         /* include */
-  kEXTEND,          /* extend */
-  kPREPEND,         /* prepend */
-  kALIAS,           /* alias */
-  kMODULE,          /* module */
-  kATTRREADER,      /* attr_reader */
-  kATTRWRITER,      /* attr_writer */
-  kATTRACCESSOR,    /* attr_accessor */
-  kPUBLIC,          /* public */
-  kPRIVATE,         /* private */
   kUNTYPED,         /* untyped */
+  kVOID,            /* void */
 
   tLIDENT,          /* Identifiers starting with lower case */
   tUIDENT,          /* Identifiers starting with upper case */
@@ -121,14 +121,13 @@ typedef struct {
   position current;               /* The current position */
   position start;                 /* The start position of the current token */
   bool first_token_of_line;       /* This flag is used for tLINECOMMENT */
+  unsigned int last_char;         /* Last peeked character */
 } lexstate;
 
 extern token NullToken;
 extern position NullPosition;
 extern range NULL_RANGE;
 
-token rbsparser_next_token(lexstate *state);
-
 char *peek_token(lexstate *state, token tok);
 int token_chars(token tok);
 int token_bytes(token tok);
@@ -140,6 +139,23 @@ int token_bytes(token tok);
 
 const char *token_type_str(enum TokenType type);
 
+/**
+ * Read next character.
+ * */
+unsigned int peek(lexstate *state);
+
+/**
+ * Skip one character.
+ * */
+void skip(lexstate *state);
+
+/**
+ * Return new token with given type.
+ * */
+token next_token(lexstate *state, enum TokenType type);
+
+token rbsparser_next_token(lexstate *state);
+
 void print_token(token tok);
 
 #endif

data/ext/rbs_extension/lexer.re

@@ -0,0 +1,140 @@
+#include "rbs_extension.h"
+
+token rbsparser_next_token(lexstate *state) {
+  lexstate backup;
+
+start:
+  backup = *state;
+
+  /*!re2c
+      re2c:flags:u = 1;
+      re2c:api:style = free-form;
+      re2c:flags:input = custom;
+      re2c:define:YYCTYPE = "unsigned int";
+      re2c:define:YYPEEK = "peek(state)";
+      re2c:define:YYSKIP = "skip(state);";
+      re2c:define:YYBACKUP = "backup = *state;";
+      re2c:define:YYRESTORE = "*state = backup;";
+      re2c:yyfill:enable  = 0;
+
+      word = [a-zA-Z0-9_];
+
+      operator = "/" | "~" | "[]" | "[]=" | "!" | "!=" | "!~" | "-" | "-@" | "+" | "+@"
+               | "==" | "===" | "=~" | "<<" | "<=" | "<=>" | ">" | ">=" | ">>" | "%";
+
+      "("   { return next_token(state, pLPAREN); }
+      ")"   { return next_token(state, pRPAREN); }
+      "["   { return next_token(state, pLBRACKET); }
+      "]"   { return next_token(state, pRBRACKET); }
+      "{"   { return next_token(state, pLBRACE); }
+      "}"   { return next_token(state, pRBRACE); }
+      ","   { return next_token(state, pCOMMA); }
+      "|"   { return next_token(state, pBAR); }
+      "^"   { return next_token(state, pHAT); }
+      "&"   { return next_token(state, pAMP); }
+      "?"   { return next_token(state, pQUESTION); }
+      "*"   { return next_token(state, pSTAR); }
+      "**"  { return next_token(state, pSTAR2); }
+      "."   { return next_token(state, pDOT); }
+      "..." { return next_token(state, pDOT3); }
+      "`"   {  return next_token(state, tOPERATOR); }
+      "`"   [^ :\x00] [^`\x00]* "`" { return next_token(state, tQIDENT); }
+      "->"  { return next_token(state, pARROW); }
+      "=>"  { return next_token(state, pFATARROW); }
+      "="   { return next_token(state, pEQ); }
+      ":"   { return next_token(state, pCOLON); }
+      "::"  { return next_token(state, pCOLON2); }
+      "<"   { return next_token(state, pLT); }
+      operator  { return next_token(state, tOPERATOR); }
+
+      number = [0-9] [0-9_]*;
+      ("-"|"+")? number    { return next_token(state, tINTEGER); }
+
+      "%a{" [^}\x00]* "}"  { return next_token(state, tANNOTATION); }
+      "%a(" [^)\x00]* ")"  { return next_token(state, tANNOTATION); }
+      "%a[" [^\]\x00]* "]" { return next_token(state, tANNOTATION); }
+      "%a|" [^|\x00]* "|"  { return next_token(state, tANNOTATION); }
+      "%a<" [^>\x00]* ">"  { return next_token(state, tANNOTATION); }
+
+      "#" (. \ [\x00])*    {
+        return next_token(
+          state,
+          state->first_token_of_line ? tLINECOMMENT : tCOMMENT
+        );
+      }
+
+      "alias"         { return next_token(state, kALIAS); }
+      "attr_accessor" { return next_token(state, kATTRACCESSOR); }
+      "attr_reader"   { return next_token(state, kATTRREADER); }
+      "attr_writer"   { return next_token(state, kATTRWRITER); }
+      "bool"          { return next_token(state, kBOOL); }
+      "bot"           { return next_token(state, kBOT); }
+      "class"         { return next_token(state, kCLASS); }
+      "def"           { return next_token(state, kDEF); }
+      "end"           { return next_token(state, kEND); }
+      "extend"        { return next_token(state, kEXTEND); }
+      "false"         { return next_token(state, kFALSE); }
+      "in"            { return next_token(state, kIN); }
+      "include"       { return next_token(state, kINCLUDE); }
+      "instance"      { return next_token(state, kINSTANCE); }
+      "interface"     { return next_token(state, kINTERFACE); }
+      "module"        { return next_token(state, kMODULE); }
+      "nil"           { return next_token(state, kNIL); }
+      "out"           { return next_token(state, kOUT); }
+      "prepend"       { return next_token(state, kPREPEND); }
+      "private"       { return next_token(state, kPRIVATE); }
+      "public"        { return next_token(state, kPUBLIC); }
+      "self"          { return next_token(state, kSELF); }
+      "singleton"     { return next_token(state, kSINGLETON); }
+      "top"           { return next_token(state, kTOP); }
+      "true"          { return next_token(state, kTRUE); }
+      "type"          { return next_token(state, kTYPE); }
+      "unchecked"     { return next_token(state, kUNCHECKED); }
+      "untyped"       { return next_token(state, kUNTYPED); }
+      "void"          { return next_token(state, kVOID); }
+
+      dqstring = ["] ("\\"["] | [^"\x00])* ["];
+      sqstring = ['] ("\\"['] | [^'\x00])* ['];
+
+      dqstring     { return next_token(state, tDQSTRING); }
+      sqstring     { return next_token(state, tSQSTRING); }
+      ":" dqstring { return next_token(state, tDQSYMBOL); }
+      ":" sqstring { return next_token(state, tSQSYMBOL); }
+
+      identifier = [a-zA-Z_] word* [!?=]?;
+      symbol_opr = ":|" | ":&" | ":/" | ":%" | ":~" | ":`" | ":^"
+                 | ":==" | ":=~" | ":===" | ":!" | ":!=" | ":!~"
+                 | ":<" | ":<=" | ":<<" | ":<=>" | ":>" | ":>=" | ":>>"
+                 | ":-" | ":-@" | ":+" | ":+@" | ":*" | ":**" | ":[]" | ":[]=";
+
+      global_ident = [0-9]+
+                   | "-" [a-zA-Z0-9_]
+                   | [~*$?!@\\/;,.=:<>"&'`+]
+                   | [^ \t\r\n:;=.,!"$%&()-+~|\\'[\]{}*/<>^\x00]+;
+
+      ":" identifier     { return next_token(state, tSYMBOL); }
+      ":@" identifier    { return next_token(state, tSYMBOL); }
+      ":@@" identifier   { return next_token(state, tSYMBOL); }
+      ":$" global_ident  { return next_token(state, tSYMBOL); }
+      symbol_opr         { return next_token(state, tSYMBOL); }
+
+      [a-z] word*           { return next_token(state, tLIDENT); }
+      [A-Z] word*           { return next_token(state, tUIDENT); }
+      "_" [a-z0-9_] word*   { return next_token(state, tULLIDENT); }
+      "_" [A-Z] word*       { return next_token(state, tULIDENT); }
+      "_"                   { return next_token(state, tULLIDENT); }
+      [a-zA-Z_] word* "!"   { return next_token(state, tBANGIDENT); }
+      [a-zA-Z_] word* "="   { return next_token(state, tEQIDENT); }
+
+      "@" [a-zA-Z_] word*   { return next_token(state, tAIDENT); }
+      "@@" [a-zA-Z_] word*  { return next_token(state, tA2IDENT); }
+
+      "$" global_ident      { return next_token(state, tGIDENT); }
+
+      skip = [ \t\n]+;
+
+      skip     { state->start = state->current; goto start; }
+      "\x00"   { return next_token(state, pEOF); }
+      *        { return next_token(state, ErrorToken); }
+  */
+}

data/ext/rbs_extension/lexstate.c

@@ -0,0 +1,139 @@
+#include "rbs_extension.h"
+
+static const char *RBS_TOKENTYPE_NAMES[] = {
+  "NullType",
+  "pEOF",
+  "ErrorToken",
+
+  "pLPAREN",          /* ( */
+  "pRPAREN",          /* ) */
+  "pCOLON",           /* : */
+  "pCOLON2",          /* :: */
+  "pLBRACKET",        /* [ */
+  "pRBRACKET",        /* ] */
+  "pLBRACE",          /* { */
+  "pRBRACE",          /* } */
+  "pHAT",             /* ^ */
+  "pARROW",           /* -> */
+  "pFATARROW",        /* => */
+  "pCOMMA",           /* , */
+  "pBAR",             /* | */
+  "pAMP",             /* & */
+  "pSTAR",            /* * */
+  "pSTAR2",           /* ** */
+  "pDOT",             /* . */
+  "pDOT3",            /* ... */
+  "pBANG",            /* ! */
+  "pQUESTION",        /* ? */
+  "pLT",              /* < */
+  "pEQ",              /* = */
+
+  "kALIAS",           /* alias */
+  "kATTRACCESSOR",    /* attr_accessor */
+  "kATTRREADER",      /* attr_reader */
+  "kATTRWRITER",      /* attr_writer */
+  "kBOOL",            /* bool */
+  "kBOT",             /* bot */
+  "kCLASS",           /* class */
+  "kDEF",             /* def */
+  "kEND",             /* end */
+  "kEXTEND",          /* extend */
+  "kFALSE",           /* kFALSE */
+  "kIN",              /* in */
+  "kINCLUDE",         /* include */
+  "kINSTANCE",        /* instance */
+  "kINTERFACE",       /* interface */
+  "kMODULE",          /* module */
+  "kNIL",             /* nil */
+  "kOUT",             /* out */
+  "kPREPEND",         /* prepend */
+  "kPRIVATE",         /* private */
+  "kPUBLIC",          /* public */
+  "kSELF",            /* self */
+  "kSINGLETON",       /* singleton */
+  "kTOP",             /* top */
+  "kTRUE",            /* true */
+  "kTYPE",            /* type */
+  "kUNCHECKED",       /* unchecked */
+  "kUNTYPED",         /* untyped */
+  "kVOID",            /* void */
+
+  "tLIDENT",          /* Identifiers starting with lower case */
+  "tUIDENT",          /* Identifiers starting with upper case */
+  "tULIDENT",         /* Identifiers starting with `_` */
+  "tULLIDENT",
+  "tGIDENT",          /* Identifiers starting with `$` */
+  "tAIDENT",          /* Identifiers starting with `@` */
+  "tA2IDENT",         /* Identifiers starting with `@@` */
+  "tBANGIDENT",
+  "tEQIDENT",
+  "tQIDENT",          /* Quoted identifier */
+  "tOPERATOR",        /* Operator identifier */
+
+  "tCOMMENT",
+  "tLINECOMMENT",
+
+  "tDQSTRING",        /* Double quoted string */
+  "tSQSTRING",        /* Single quoted string */
+  "tINTEGER",         /* Integer */
+  "tSYMBOL",          /* Symbol */
+  "tDQSYMBOL",
+  "tSQSYMBOL",
+  "tANNOTATION",      /* Annotation */
+};
+
+token NullToken = { NullType };
+position NullPosition = { -1, -1, -1, -1 };
+range NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
+
+const char *token_type_str(enum TokenType type) {
+  return RBS_TOKENTYPE_NAMES[type];
+}
+
+int token_chars(token tok) {
+  return tok.range.end.char_pos - tok.range.start.char_pos;
+}
+
+int token_bytes(token tok) {
+  return RANGE_BYTES(tok.range);
+}
+
+unsigned int peek(lexstate *state) {
+  unsigned int c = rb_enc_mbc_to_codepoint(RSTRING_PTR(state->string) + state->current.byte_pos, RSTRING_END(state->string), rb_enc_get(state->string));
+  state->last_char = c;
+  return c;
+}
+
+token next_token(lexstate *state, enum TokenType type) {
+  token t;
+
+  t.type = type;
+  t.range.start = state->start;
+  t.range.end = state->current;
+  state->start = state->current;
+  state->first_token_of_line = false;
+
+  return t;
+}
+
+void skip(lexstate *state) {
+  if (!state->last_char) {
+    peek(state);
+  }
+  int byte_len = rb_enc_codelen(state->last_char, rb_enc_get(state->string));
+
+  state->current.char_pos += 1;
+  state->current.byte_pos += byte_len;
+
+  if (state->last_char == '\n') {
+    state->current.line += 1;
+    state->current.column = 0;
+    state->first_token_of_line = true;
+  } else {
+    state->current.column += 1;
+  }
+}
+
+char *peek_token(lexstate *state, token tok) {
+  return RSTRING_PTR(state->string) + tok.range.start.byte_pos;
+}

data/ext/rbs_extension/parser.c

@@ -2387,36 +2387,4 @@ void rbs__init_parser() {
   rb_define_singleton_method(RBS_Parser, "_parse_type", rbsparser_parse_type, 4);
   rb_define_singleton_method(RBS_Parser, "_parse_method_type", rbsparser_parse_method_type, 4);
   rb_define_singleton_method(RBS_Parser, "_parse_signature", rbsparser_parse_signature, 3);
-
-  RBS_Parser_KEYWORDS = rb_hash_new();
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("bool"), INT2FIX(kBOOL));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("bot"), INT2FIX(kBOT));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("class"), INT2FIX(kCLASS));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("instance"), INT2FIX(kINSTANCE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("interface"), INT2FIX(kINTERFACE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("nil"), INT2FIX(kNIL));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("self"), INT2FIX(kSELF));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("singleton"), INT2FIX(kSINGLETON));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("top"), INT2FIX(kTOP));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("void"), INT2FIX(kVOID));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("type"), INT2FIX(kTYPE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("unchecked"), INT2FIX(kUNCHECKED));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("in"), INT2FIX(kIN));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("out"), INT2FIX(kOUT));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("end"), INT2FIX(kEND));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("def"), INT2FIX(kDEF));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("include"), INT2FIX(kINCLUDE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("extend"), INT2FIX(kEXTEND));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("prepend"), INT2FIX(kPREPEND));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("alias"), INT2FIX(kALIAS));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("module"), INT2FIX(kMODULE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("attr_reader"), INT2FIX(kATTRREADER));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("attr_writer"), INT2FIX(kATTRWRITER));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("attr_accessor"), INT2FIX(kATTRACCESSOR));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("public"), INT2FIX(kPUBLIC));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("private"), INT2FIX(kPRIVATE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("untyped"), INT2FIX(kUNTYPED));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("true"), INT2FIX(kTRUE));
-  rb_hash_aset(RBS_Parser_KEYWORDS, rb_str_new_literal("false"), INT2FIX(kFALSE));
-  rb_define_const(RBS_Parser, "KEYWORDS", RBS_Parser_KEYWORDS);
 }

data/ext/rbs_extension/parser.h

@@ -9,11 +9,6 @@
  * */
 extern VALUE RBS_Parser;
 
-/**
- * RBS::Parser::KEYWORDS constant, which stores a hash from keyword string to token type fixnum
- * */
-extern VALUE RBS_Parser_KEYWORDS;
-
 VALUE parse_type(parserstate *state);
 VALUE parse_method_type(parserstate *state);
 VALUE parse_signature(parserstate *state);