rbs 1.6.1 → 1.7.0.beta.3

data/ext/rbs_extension/lexer.h

@@ -0,0 +1,161 @@
+#ifndef RBS__LEXER_H
+#define RBS__LEXER_H
+
+enum TokenType {
+  NullType,         /* (Nothing) */
+  pEOF,             /* EOF */
+  ErrorToken,       /* Error */
+
+  pLPAREN,          /* ( */
+  pRPAREN,          /* ) */
+  pCOLON,           /* : */
+  pCOLON2,          /* :: */
+  pLBRACKET,        /* [ */
+  pRBRACKET,        /* ] */
+  pLBRACE,          /* { */
+  pRBRACE,          /* } */
+  pHAT,             /* ^ */
+  pARROW,           /* -> */
+  pFATARROW,        /* => */
+  pCOMMA,           /* , */
+  pBAR,             /* | */
+  pAMP,             /* & */
+  pSTAR,            /* * */
+  pSTAR2,           /* ** */
+  pDOT,             /* . */
+  pDOT3,            /* ... */
+  pBANG,            /* ! */
+  pQUESTION,        /* ? */
+  pLT,              /* < */
+  pEQ,              /* = */
+
+  kALIAS,           /* alias */
+  kATTRACCESSOR,    /* attr_accessor */
+  kATTRREADER,      /* attr_reader */
+  kATTRWRITER,      /* attr_writer */
+  kBOOL,            /* bool */
+  kBOT,             /* bot */
+  kCLASS,           /* class */
+  kDEF,             /* def */
+  kEND,             /* end */
+  kEXTEND,          /* extend */
+  kFALSE,           /* false */
+  kIN,              /* in */
+  kINCLUDE,         /* include */
+  kINSTANCE,        /* instance */
+  kINTERFACE,       /* interface */
+  kMODULE,          /* module */
+  kNIL,             /* nil */
+  kOUT,             /* out */
+  kPREPEND,         /* prepend */
+  kPRIVATE,         /* private */
+  kPUBLIC,          /* public */
+  kSELF,            /* self */
+  kSINGLETON,       /* singleton */
+  kTOP,             /* top */
+  kTRUE,            /* true */
+  kTYPE,            /* type */
+  kUNCHECKED,       /* unchecked */
+  kUNTYPED,         /* untyped */
+  kVOID,            /* void */
+
+  tLIDENT,          /* Identifiers starting with lower case */
+  tUIDENT,          /* Identifiers starting with upper case */
+  tULIDENT,         /* Identifiers starting with `_` followed by upper case */
+  tULLIDENT,        /* Identifiers starting with `_` followed by lower case */
+  tGIDENT,          /* Identifiers starting with `$` */
+  tAIDENT,          /* Identifiers starting with `@` */
+  tA2IDENT,         /* Identifiers starting with `@@` */
+  tBANGIDENT,       /* Identifiers ending with `!` */
+  tEQIDENT,         /* Identifiers ending with `=` */
+  tQIDENT,          /* Quoted identifier */
+  tOPERATOR,         /* Operator identifier */
+
+  tCOMMENT,         /* Comment */
+  tLINECOMMENT,     /* Comment of all line */
+
+  tDQSTRING,        /* Double quoted string */
+  tSQSTRING,        /* Single quoted string */
+  tINTEGER,         /* Integer */
+  tSYMBOL,          /* Symbol */
+  tDQSYMBOL,        /* Double quoted symbol */
+  tSQSYMBOL,        /* Single quoted symbol */
+  tANNOTATION,      /* Annotation */
+};
+
+/**
+ * The `byte_pos` (or `char_pos`) is the primary data.
+ * The rest are cache.
+ *
+ * They can be computed from `byte_pos` (or `char_pos`), but it needs full scan from the beginning of the string (depending on the encoding).
+ * */
+typedef struct {
+  int byte_pos;
+  int char_pos;
+  int line;
+  int column;
+} position;
+
+typedef struct {
+  position start;
+  position end;
+} range;
+
+typedef struct {
+  enum TokenType type;
+  range range;
+} token;
+
+/**
+ * The lexer state is the curren token.
+ *
+ * ```
+ * ... "a string token"
+ *    ^                      start position
+ *          ^                current position
+ *     ~~~~~~                Token => "a str
+ * ```
+ * */
+typedef struct {
+  VALUE string;
+  position current;               /* The current position */
+  position start;                 /* The start position of the current token */
+  bool first_token_of_line;       /* This flag is used for tLINECOMMENT */
+  unsigned int last_char;         /* Last peeked character */
+} lexstate;
+
+extern token NullToken;
+extern position NullPosition;
+extern range NULL_RANGE;
+
+char *peek_token(lexstate *state, token tok);
+int token_chars(token tok);
+int token_bytes(token tok);
+
+#define null_position_p(pos) (pos.byte_pos == -1)
+#define null_range_p(range) (range.start.byte_pos == -1)
+#define nonnull_pos_or(pos1, pos2) (null_position_p(pos1) ? pos2 : pos1)
+#define RANGE_BYTES(range) (range.end.byte_pos - range.start.byte_pos)
+
+const char *token_type_str(enum TokenType type);
+
+/**
+ * Read next character.
+ * */
+unsigned int peek(lexstate *state);
+
+/**
+ * Skip one character.
+ * */
+void skip(lexstate *state);
+
+/**
+ * Return new token with given type.
+ * */
+token next_token(lexstate *state, enum TokenType type);
+
+token rbsparser_next_token(lexstate *state);
+
+void print_token(token tok);
+
+#endif

data/ext/rbs_extension/lexer.re

@@ -0,0 +1,140 @@
+#include "rbs_extension.h"
+
+token rbsparser_next_token(lexstate *state) {
+  lexstate backup;
+
+start:
+  backup = *state;
+
+  /*!re2c
+      re2c:flags:u = 1;
+      re2c:api:style = free-form;
+      re2c:flags:input = custom;
+      re2c:define:YYCTYPE = "unsigned int";
+      re2c:define:YYPEEK = "peek(state)";
+      re2c:define:YYSKIP = "skip(state);";
+      re2c:define:YYBACKUP = "backup = *state;";
+      re2c:define:YYRESTORE = "*state = backup;";
+      re2c:yyfill:enable  = 0;
+
+      word = [a-zA-Z0-9_];
+
+      operator = "/" | "~" | "[]" | "[]=" | "!" | "!=" | "!~" | "-" | "-@" | "+" | "+@"
+               | "==" | "===" | "=~" | "<<" | "<=" | "<=>" | ">" | ">=" | ">>" | "%";
+
+      "("   { return next_token(state, pLPAREN); }
+      ")"   { return next_token(state, pRPAREN); }
+      "["   { return next_token(state, pLBRACKET); }
+      "]"   { return next_token(state, pRBRACKET); }
+      "{"   { return next_token(state, pLBRACE); }
+      "}"   { return next_token(state, pRBRACE); }
+      ","   { return next_token(state, pCOMMA); }
+      "|"   { return next_token(state, pBAR); }
+      "^"   { return next_token(state, pHAT); }
+      "&"   { return next_token(state, pAMP); }
+      "?"   { return next_token(state, pQUESTION); }
+      "*"   { return next_token(state, pSTAR); }
+      "**"  { return next_token(state, pSTAR2); }
+      "."   { return next_token(state, pDOT); }
+      "..." { return next_token(state, pDOT3); }
+      "`"   {  return next_token(state, tOPERATOR); }
+      "`"   [^ :\x00] [^`\x00]* "`" { return next_token(state, tQIDENT); }
+      "->"  { return next_token(state, pARROW); }
+      "=>"  { return next_token(state, pFATARROW); }
+      "="   { return next_token(state, pEQ); }
+      ":"   { return next_token(state, pCOLON); }
+      "::"  { return next_token(state, pCOLON2); }
+      "<"   { return next_token(state, pLT); }
+      operator  { return next_token(state, tOPERATOR); }
+
+      number = [0-9] [0-9_]*;
+      ("-"|"+")? number    { return next_token(state, tINTEGER); }
+
+      "%a{" [^}\x00]* "}"  { return next_token(state, tANNOTATION); }
+      "%a(" [^)\x00]* ")"  { return next_token(state, tANNOTATION); }
+      "%a[" [^\]\x00]* "]" { return next_token(state, tANNOTATION); }
+      "%a|" [^|\x00]* "|"  { return next_token(state, tANNOTATION); }
+      "%a<" [^>\x00]* ">"  { return next_token(state, tANNOTATION); }
+
+      "#" (. \ [\x00])*    {
+        return next_token(
+          state,
+          state->first_token_of_line ? tLINECOMMENT : tCOMMENT
+        );
+      }
+
+      "alias"         { return next_token(state, kALIAS); }
+      "attr_accessor" { return next_token(state, kATTRACCESSOR); }
+      "attr_reader"   { return next_token(state, kATTRREADER); }
+      "attr_writer"   { return next_token(state, kATTRWRITER); }
+      "bool"          { return next_token(state, kBOOL); }
+      "bot"           { return next_token(state, kBOT); }
+      "class"         { return next_token(state, kCLASS); }
+      "def"           { return next_token(state, kDEF); }
+      "end"           { return next_token(state, kEND); }
+      "extend"        { return next_token(state, kEXTEND); }
+      "false"         { return next_token(state, kFALSE); }
+      "in"            { return next_token(state, kIN); }
+      "include"       { return next_token(state, kINCLUDE); }
+      "instance"      { return next_token(state, kINSTANCE); }
+      "interface"     { return next_token(state, kINTERFACE); }
+      "module"        { return next_token(state, kMODULE); }
+      "nil"           { return next_token(state, kNIL); }
+      "out"           { return next_token(state, kOUT); }
+      "prepend"       { return next_token(state, kPREPEND); }
+      "private"       { return next_token(state, kPRIVATE); }
+      "public"        { return next_token(state, kPUBLIC); }
+      "self"          { return next_token(state, kSELF); }
+      "singleton"     { return next_token(state, kSINGLETON); }
+      "top"           { return next_token(state, kTOP); }
+      "true"          { return next_token(state, kTRUE); }
+      "type"          { return next_token(state, kTYPE); }
+      "unchecked"     { return next_token(state, kUNCHECKED); }
+      "untyped"       { return next_token(state, kUNTYPED); }
+      "void"          { return next_token(state, kVOID); }
+
+      dqstring = ["] ("\\"["] | [^"\x00])* ["];
+      sqstring = ['] ("\\"['] | [^'\x00])* ['];
+
+      dqstring     { return next_token(state, tDQSTRING); }
+      sqstring     { return next_token(state, tSQSTRING); }
+      ":" dqstring { return next_token(state, tDQSYMBOL); }
+      ":" sqstring { return next_token(state, tSQSYMBOL); }
+
+      identifier = [a-zA-Z_] word* [!?=]?;
+      symbol_opr = ":|" | ":&" | ":/" | ":%" | ":~" | ":`" | ":^"
+                 | ":==" | ":=~" | ":===" | ":!" | ":!=" | ":!~"
+                 | ":<" | ":<=" | ":<<" | ":<=>" | ":>" | ":>=" | ":>>"
+                 | ":-" | ":-@" | ":+" | ":+@" | ":*" | ":**" | ":[]" | ":[]=";
+
+      global_ident = [0-9]+
+                   | "-" [a-zA-Z0-9_]
+                   | [~*$?!@\\/;,.=:<>"&'`+]
+                   | [^ \t\r\n:;=.,!"$%&()-+~|\\'[\]{}*/<>^\x00]+;
+
+      ":" identifier     { return next_token(state, tSYMBOL); }
+      ":@" identifier    { return next_token(state, tSYMBOL); }
+      ":@@" identifier   { return next_token(state, tSYMBOL); }
+      ":$" global_ident  { return next_token(state, tSYMBOL); }
+      symbol_opr         { return next_token(state, tSYMBOL); }
+
+      [a-z] word*           { return next_token(state, tLIDENT); }
+      [A-Z] word*           { return next_token(state, tUIDENT); }
+      "_" [a-z0-9_] word*   { return next_token(state, tULLIDENT); }
+      "_" [A-Z] word*       { return next_token(state, tULIDENT); }
+      "_"                   { return next_token(state, tULLIDENT); }
+      [a-zA-Z_] word* "!"   { return next_token(state, tBANGIDENT); }
+      [a-zA-Z_] word* "="   { return next_token(state, tEQIDENT); }
+
+      "@" [a-zA-Z_] word*   { return next_token(state, tAIDENT); }
+      "@@" [a-zA-Z_] word*  { return next_token(state, tA2IDENT); }
+
+      "$" global_ident      { return next_token(state, tGIDENT); }
+
+      skip = [ \t\n]+;
+
+      skip     { state->start = state->current; goto start; }
+      "\x00"   { return next_token(state, pEOF); }
+      *        { return next_token(state, ErrorToken); }
+  */
+}

data/ext/rbs_extension/lexstate.c

@@ -0,0 +1,139 @@
+#include "rbs_extension.h"
+
+static const char *RBS_TOKENTYPE_NAMES[] = {
+  "NullType",
+  "pEOF",
+  "ErrorToken",
+
+  "pLPAREN",          /* ( */
+  "pRPAREN",          /* ) */
+  "pCOLON",           /* : */
+  "pCOLON2",          /* :: */
+  "pLBRACKET",        /* [ */
+  "pRBRACKET",        /* ] */
+  "pLBRACE",          /* { */
+  "pRBRACE",          /* } */
+  "pHAT",             /* ^ */
+  "pARROW",           /* -> */
+  "pFATARROW",        /* => */
+  "pCOMMA",           /* , */
+  "pBAR",             /* | */
+  "pAMP",             /* & */
+  "pSTAR",            /* * */
+  "pSTAR2",           /* ** */
+  "pDOT",             /* . */
+  "pDOT3",            /* ... */
+  "pBANG",            /* ! */
+  "pQUESTION",        /* ? */
+  "pLT",              /* < */
+  "pEQ",              /* = */
+
+  "kALIAS",           /* alias */
+  "kATTRACCESSOR",    /* attr_accessor */
+  "kATTRREADER",      /* attr_reader */
+  "kATTRWRITER",      /* attr_writer */
+  "kBOOL",            /* bool */
+  "kBOT",             /* bot */
+  "kCLASS",           /* class */
+  "kDEF",             /* def */
+  "kEND",             /* end */
+  "kEXTEND",          /* extend */
+  "kFALSE",           /* kFALSE */
+  "kIN",              /* in */
+  "kINCLUDE",         /* include */
+  "kINSTANCE",        /* instance */
+  "kINTERFACE",       /* interface */
+  "kMODULE",          /* module */
+  "kNIL",             /* nil */
+  "kOUT",             /* out */
+  "kPREPEND",         /* prepend */
+  "kPRIVATE",         /* private */
+  "kPUBLIC",          /* public */
+  "kSELF",            /* self */
+  "kSINGLETON",       /* singleton */
+  "kTOP",             /* top */
+  "kTRUE",            /* true */
+  "kTYPE",            /* type */
+  "kUNCHECKED",       /* unchecked */
+  "kUNTYPED",         /* untyped */
+  "kVOID",            /* void */
+
+  "tLIDENT",          /* Identifiers starting with lower case */
+  "tUIDENT",          /* Identifiers starting with upper case */
+  "tULIDENT",         /* Identifiers starting with `_` */
+  "tULLIDENT",
+  "tGIDENT",          /* Identifiers starting with `$` */
+  "tAIDENT",          /* Identifiers starting with `@` */
+  "tA2IDENT",         /* Identifiers starting with `@@` */
+  "tBANGIDENT",
+  "tEQIDENT",
+  "tQIDENT",          /* Quoted identifier */
+  "tOPERATOR",        /* Operator identifier */
+
+  "tCOMMENT",
+  "tLINECOMMENT",
+
+  "tDQSTRING",        /* Double quoted string */
+  "tSQSTRING",        /* Single quoted string */
+  "tINTEGER",         /* Integer */
+  "tSYMBOL",          /* Symbol */
+  "tDQSYMBOL",
+  "tSQSYMBOL",
+  "tANNOTATION",      /* Annotation */
+};
+
+token NullToken = { NullType };
+position NullPosition = { -1, -1, -1, -1 };
+range NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
+
+const char *token_type_str(enum TokenType type) {
+  return RBS_TOKENTYPE_NAMES[type];
+}
+
+int token_chars(token tok) {
+  return tok.range.end.char_pos - tok.range.start.char_pos;
+}
+
+int token_bytes(token tok) {
+  return RANGE_BYTES(tok.range);
+}
+
+unsigned int peek(lexstate *state) {
+  unsigned int c = rb_enc_mbc_to_codepoint(RSTRING_PTR(state->string) + state->current.byte_pos, RSTRING_END(state->string), rb_enc_get(state->string));
+  state->last_char = c;
+  return c;
+}
+
+token next_token(lexstate *state, enum TokenType type) {
+  token t;
+
+  t.type = type;
+  t.range.start = state->start;
+  t.range.end = state->current;
+  state->start = state->current;
+  state->first_token_of_line = false;
+
+  return t;
+}
+
+void skip(lexstate *state) {
+  if (!state->last_char) {
+    peek(state);
+  }
+  int byte_len = rb_enc_codelen(state->last_char, rb_enc_get(state->string));
+
+  state->current.char_pos += 1;
+  state->current.byte_pos += byte_len;
+
+  if (state->last_char == '\n') {
+    state->current.line += 1;
+    state->current.column = 0;
+    state->first_token_of_line = true;
+  } else {
+    state->current.column += 1;
+  }
+}
+
+char *peek_token(lexstate *state, token tok) {
+  return RSTRING_PTR(state->string) + tok.range.start.byte_pos;
+}