rbs 3.9.4 → 4.1.1

data/ext/rbs_extension/lexer.h

@@ -1,179 +0,0 @@
-#ifndef RBS__LEXER_H
-#define RBS__LEXER_H
-
-enum TokenType {
-  NullType,         /* (Nothing) */
-  pEOF,             /* EOF */
-  ErrorToken,       /* Error */
-
-  pLPAREN,          /* ( */
-  pRPAREN,          /* ) */
-  pCOLON,           /* : */
-  pCOLON2,          /* :: */
-  pLBRACKET,        /* [ */
-  pRBRACKET,        /* ] */
-  pLBRACE,          /* { */
-  pRBRACE,          /* } */
-  pHAT,             /* ^ */
-  pARROW,           /* -> */
-  pFATARROW,        /* => */
-  pCOMMA,           /* , */
-  pBAR,             /* | */
-  pAMP,             /* & */
-  pSTAR,            /* * */
-  pSTAR2,           /* ** */
-  pDOT,             /* . */
-  pDOT3,            /* ... */
-  pBANG,            /* ! */
-  pQUESTION,        /* ? */
-  pLT,              /* < */
-  pEQ,              /* = */
-
-  kALIAS,           /* alias */
-  kATTRACCESSOR,    /* attr_accessor */
-  kATTRREADER,      /* attr_reader */
-  kATTRWRITER,      /* attr_writer */
-  kBOOL,            /* bool */
-  kBOT,             /* bot */
-  kCLASS,           /* class */
-  kDEF,             /* def */
-  kEND,             /* end */
-  kEXTEND,          /* extend */
-  kFALSE,           /* false */
-  kIN,              /* in */
-  kINCLUDE,         /* include */
-  kINSTANCE,        /* instance */
-  kINTERFACE,       /* interface */
-  kMODULE,          /* module */
-  kNIL,             /* nil */
-  kOUT,             /* out */
-  kPREPEND,         /* prepend */
-  kPRIVATE,         /* private */
-  kPUBLIC,          /* public */
-  kSELF,            /* self */
-  kSINGLETON,       /* singleton */
-  kTOP,             /* top */
-  kTRUE,            /* true */
-  kTYPE,            /* type */
-  kUNCHECKED,       /* unchecked */
-  kUNTYPED,         /* untyped */
-  kVOID,            /* void */
-  kUSE,             /* use */
-  kAS,              /* as */
-  k__TODO__,        /* __todo__ */
-
-  tLIDENT,          /* Identifiers starting with lower case */
-  tUIDENT,          /* Identifiers starting with upper case */
-  tULIDENT,         /* Identifiers starting with `_` followed by upper case */
-  tULLIDENT,        /* Identifiers starting with `_` followed by lower case */
-  tGIDENT,          /* Identifiers starting with `$` */
-  tAIDENT,          /* Identifiers starting with `@` */
-  tA2IDENT,         /* Identifiers starting with `@@` */
-  tBANGIDENT,       /* Identifiers ending with `!` */
-  tEQIDENT,         /* Identifiers ending with `=` */
-  tQIDENT,          /* Quoted identifier */
-  pAREF_OPR,        /* [] */
-  tOPERATOR,        /* Operator identifier */
-
-  tCOMMENT,         /* Comment */
-  tLINECOMMENT,     /* Comment of all line */
-
-  tTRIVIA,          /* Trivia tokens -- space and new line */
-
-  tDQSTRING,        /* Double quoted string */
-  tSQSTRING,        /* Single quoted string */
-  tINTEGER,         /* Integer */
-  tSYMBOL,          /* Symbol */
-  tDQSYMBOL,        /* Double quoted symbol */
-  tSQSYMBOL,        /* Single quoted symbol */
-  tANNOTATION,      /* Annotation */
-};
-
-/**
- * The `byte_pos` (or `char_pos`) is the primary data.
- * The rest are cache.
- *
- * They can be computed from `byte_pos` (or `char_pos`), but it needs full scan from the beginning of the string (depending on the encoding).
- * */
-typedef struct {
-  int byte_pos;
-  int char_pos;
-  int line;
-  int column;
-} position;
-
-typedef struct {
-  position start;
-  position end;
-} range;
-
-typedef struct {
-  enum TokenType type;
-  range range;
-} token;
-
-/**
- * The lexer state is the curren token.
- *
- * ```
- * ... "a string token"
- *    ^                      start position
- *          ^                current position
- *     ~~~~~~                Token => "a str
- * ```
- * */
-typedef struct {
-  VALUE string;
-  int start_pos;                  /* The character position that defines the start of the input */
-  int end_pos;                    /* The character position that defines the end of the input */
-  position current;               /* The current position */
-  position start;                 /* The start position of the current token */
-  bool first_token_of_line;       /* This flag is used for tLINECOMMENT */
-  unsigned int last_char;         /* Last peeked character */
-} lexstate;
-
-extern token NullToken;
-extern position NullPosition;
-extern range NULL_RANGE;
-
-char *peek_token(lexstate *state, token tok);
-int token_chars(token tok);
-int token_bytes(token tok);
-
-#define null_position_p(pos) (pos.byte_pos == -1)
-#define null_range_p(range) (range.start.byte_pos == -1)
-#define nonnull_pos_or(pos1, pos2) (null_position_p(pos1) ? pos2 : pos1)
-#define RANGE_BYTES(range) (range.end.byte_pos - range.start.byte_pos)
-
-const char *token_type_str(enum TokenType type);
-
-/**
- * Read next character.
- * */
-unsigned int peek(lexstate *state);
-
-/**
- * Skip one character.
- * */
-void rbs_skip(lexstate *state);
-
-/**
- * Skip n characters.
- * */
-void skipn(lexstate *state, size_t size);
-
-/**
- * Return new token with given type.
- * */
-token next_token(lexstate *state, enum TokenType type);
-
-/**
- * Return new token with EOF type.
- * */
-token next_eof_token(lexstate *state);
-
-token rbsparser_next_token(lexstate *state);
-
-void print_token(token tok);
-
-#endif

data/ext/rbs_extension/lexer.re

@@ -1,147 +0,0 @@
-#include "rbs_extension.h"
-
-token rbsparser_next_token(lexstate *state) {
-  lexstate backup;
-
-  backup = *state;
-
-  /*!re2c
-      re2c:flags:u = 1;
-      re2c:api:style = free-form;
-      re2c:flags:input = custom;
-      re2c:define:YYCTYPE = "unsigned int";
-      re2c:define:YYPEEK = "peek(state)";
-      re2c:define:YYSKIP = "rbs_skip(state);";
-      re2c:define:YYBACKUP = "backup = *state;";
-      re2c:define:YYRESTORE = "*state = backup;";
-      re2c:yyfill:enable  = 0;
-
-      word = [a-zA-Z0-9_];
-
-      operator = "/" | "~" | "[]=" | "!" | "!=" | "!~" | "-" | "-@" | "+" | "+@"
-               | "==" | "===" | "=~" | "<<" | "<=" | "<=>" | ">" | ">=" | ">>" | "%";
-
-      "("   { return next_token(state, pLPAREN); }
-      ")"   { return next_token(state, pRPAREN); }
-      "["   { return next_token(state, pLBRACKET); }
-      "]"   { return next_token(state, pRBRACKET); }
-      "{"   { return next_token(state, pLBRACE); }
-      "}"   { return next_token(state, pRBRACE); }
-      ","   { return next_token(state, pCOMMA); }
-      "|"   { return next_token(state, pBAR); }
-      "^"   { return next_token(state, pHAT); }
-      "&"   { return next_token(state, pAMP); }
-      "?"   { return next_token(state, pQUESTION); }
-      "*"   { return next_token(state, pSTAR); }
-      "**"  { return next_token(state, pSTAR2); }
-      "."   { return next_token(state, pDOT); }
-      "..." { return next_token(state, pDOT3); }
-      "`"   {  return next_token(state, tOPERATOR); }
-      "`"   [^ :\x00] [^`\x00]* "`" { return next_token(state, tQIDENT); }
-      "->"  { return next_token(state, pARROW); }
-      "=>"  { return next_token(state, pFATARROW); }
-      "="   { return next_token(state, pEQ); }
-      ":"   { return next_token(state, pCOLON); }
-      "::"  { return next_token(state, pCOLON2); }
-      "<"   { return next_token(state, pLT); }
-      "[]"  { return next_token(state, pAREF_OPR); }
-      operator  { return next_token(state, tOPERATOR); }
-
-      number = [0-9] [0-9_]*;
-      ("-"|"+")? number    { return next_token(state, tINTEGER); }
-
-      "%a{" [^}\x00]* "}"  { return next_token(state, tANNOTATION); }
-      "%a(" [^)\x00]* ")"  { return next_token(state, tANNOTATION); }
-      "%a[" [^\]\x00]* "]" { return next_token(state, tANNOTATION); }
-      "%a|" [^|\x00]* "|"  { return next_token(state, tANNOTATION); }
-      "%a<" [^>\x00]* ">"  { return next_token(state, tANNOTATION); }
-
-      "#" (. \ [\x00])*    {
-        return next_token(
-          state,
-          state->first_token_of_line ? tLINECOMMENT : tCOMMENT
-        );
-      }
-
-      "alias"         { return next_token(state, kALIAS); }
-      "attr_accessor" { return next_token(state, kATTRACCESSOR); }
-      "attr_reader"   { return next_token(state, kATTRREADER); }
-      "attr_writer"   { return next_token(state, kATTRWRITER); }
-      "bool"          { return next_token(state, kBOOL); }
-      "bot"           { return next_token(state, kBOT); }
-      "class"         { return next_token(state, kCLASS); }
-      "def"           { return next_token(state, kDEF); }
-      "end"           { return next_token(state, kEND); }
-      "extend"        { return next_token(state, kEXTEND); }
-      "false"         { return next_token(state, kFALSE); }
-      "in"            { return next_token(state, kIN); }
-      "include"       { return next_token(state, kINCLUDE); }
-      "instance"      { return next_token(state, kINSTANCE); }
-      "interface"     { return next_token(state, kINTERFACE); }
-      "module"        { return next_token(state, kMODULE); }
-      "nil"           { return next_token(state, kNIL); }
-      "out"           { return next_token(state, kOUT); }
-      "prepend"       { return next_token(state, kPREPEND); }
-      "private"       { return next_token(state, kPRIVATE); }
-      "public"        { return next_token(state, kPUBLIC); }
-      "self"          { return next_token(state, kSELF); }
-      "singleton"     { return next_token(state, kSINGLETON); }
-      "top"           { return next_token(state, kTOP); }
-      "true"          { return next_token(state, kTRUE); }
-      "type"          { return next_token(state, kTYPE); }
-      "unchecked"     { return next_token(state, kUNCHECKED); }
-      "untyped"       { return next_token(state, kUNTYPED); }
-      "void"          { return next_token(state, kVOID); }
-      "use"           { return next_token(state, kUSE); }
-      "as"            { return next_token(state, kAS); }
-      "__todo__"      { return next_token(state, k__TODO__); }
-
-      unicode_char = "\\u" [0-9a-fA-F]{4};
-      oct_char = "\\x" [0-9a-f]{1,2};
-      hex_char = "\\" [0-7]{1,3};
-
-      dqstring = ["] (unicode_char | oct_char | hex_char | "\\" [^xu] | [^\\"\x00])* ["];
-      sqstring = ['] ("\\"['\\] | [^'\x00])* ['];
-
-      dqstring     { return next_token(state, tDQSTRING); }
-      sqstring     { return next_token(state, tSQSTRING); }
-      ":" dqstring { return next_token(state, tDQSYMBOL); }
-      ":" sqstring { return next_token(state, tSQSYMBOL); }
-
-      identifier = [a-zA-Z_] word* [!?=]?;
-      symbol_opr = ":|" | ":&" | ":/" | ":%" | ":~" | ":`" | ":^"
-                 | ":==" | ":=~" | ":===" | ":!" | ":!=" | ":!~"
-                 | ":<" | ":<=" | ":<<" | ":<=>" | ":>" | ":>=" | ":>>"
-                 | ":-" | ":-@" | ":+" | ":+@" | ":*" | ":**" | ":[]" | ":[]=";
-
-      global_ident = [0-9]+
-                   | "-" [a-zA-Z0-9_]
-                   | [~*$?!@\\/;,.=:<>"&'`+]
-                   | [^ \t\r\n:;=.,!"$%&()-+~|\\'[\]{}*/<>^\x00]+;
-
-      ":" identifier     { return next_token(state, tSYMBOL); }
-      ":@" identifier    { return next_token(state, tSYMBOL); }
-      ":@@" identifier   { return next_token(state, tSYMBOL); }
-      ":$" global_ident  { return next_token(state, tSYMBOL); }
-      symbol_opr         { return next_token(state, tSYMBOL); }
-
-      [a-z] word*           { return next_token(state, tLIDENT); }
-      [A-Z] word*           { return next_token(state, tUIDENT); }
-      "_" [a-z0-9_] word*   { return next_token(state, tULLIDENT); }
-      "_" [A-Z] word*       { return next_token(state, tULIDENT); }
-      "_"                   { return next_token(state, tULLIDENT); }
-      [a-zA-Z_] word* "!"   { return next_token(state, tBANGIDENT); }
-      [a-zA-Z_] word* "="   { return next_token(state, tEQIDENT); }
-
-      "@" [a-zA-Z_] word*   { return next_token(state, tAIDENT); }
-      "@@" [a-zA-Z_] word*  { return next_token(state, tA2IDENT); }
-
-      "$" global_ident      { return next_token(state, tGIDENT); }
-
-      skip = ([ \t]+|[\r\n]);
-
-      skip     { return next_token(state, tTRIVIA); }
-      "\x00"   { return next_eof_token(state); }
-      *        { return next_token(state, ErrorToken); }
-  */
-}

data/ext/rbs_extension/lexstate.c

@@ -1,175 +0,0 @@
-#include "rbs_extension.h"
-
-static const char *RBS_TOKENTYPE_NAMES[] = {
-  "NullType",
-  "pEOF",
-  "ErrorToken",
-
-  "pLPAREN",          /* ( */
-  "pRPAREN",          /* ) */
-  "pCOLON",           /* : */
-  "pCOLON2",          /* :: */
-  "pLBRACKET",        /* [ */
-  "pRBRACKET",        /* ] */
-  "pLBRACE",          /* { */
-  "pRBRACE",          /* } */
-  "pHAT",             /* ^ */
-  "pARROW",           /* -> */
-  "pFATARROW",        /* => */
-  "pCOMMA",           /* , */
-  "pBAR",             /* | */
-  "pAMP",             /* & */
-  "pSTAR",            /* * */
-  "pSTAR2",           /* ** */
-  "pDOT",             /* . */
-  "pDOT3",            /* ... */
-  "pBANG",            /* ! */
-  "pQUESTION",        /* ? */
-  "pLT",              /* < */
-  "pEQ",              /* = */
-
-  "kALIAS",           /* alias */
-  "kATTRACCESSOR",    /* attr_accessor */
-  "kATTRREADER",      /* attr_reader */
-  "kATTRWRITER",      /* attr_writer */
-  "kBOOL",            /* bool */
-  "kBOT",             /* bot */
-  "kCLASS",           /* class */
-  "kDEF",             /* def */
-  "kEND",             /* end */
-  "kEXTEND",          /* extend */
-  "kFALSE",           /* kFALSE */
-  "kIN",              /* in */
-  "kINCLUDE",         /* include */
-  "kINSTANCE",        /* instance */
-  "kINTERFACE",       /* interface */
-  "kMODULE",          /* module */
-  "kNIL",             /* nil */
-  "kOUT",             /* out */
-  "kPREPEND",         /* prepend */
-  "kPRIVATE",         /* private */
-  "kPUBLIC",          /* public */
-  "kSELF",            /* self */
-  "kSINGLETON",       /* singleton */
-  "kTOP",             /* top */
-  "kTRUE",            /* true */
-  "kTYPE",            /* type */
-  "kUNCHECKED",       /* unchecked */
-  "kUNTYPED",         /* untyped */
-  "kVOID",            /* void */
-  "kUSE",             /* use */
-  "kAS",              /* as */
-  "k__TODO__",        /* __todo__ */
-
-  "tLIDENT",          /* Identifiers starting with lower case */
-  "tUIDENT",          /* Identifiers starting with upper case */
-  "tULIDENT",         /* Identifiers starting with `_` */
-  "tULLIDENT",
-  "tGIDENT",          /* Identifiers starting with `$` */
-  "tAIDENT",          /* Identifiers starting with `@` */
-  "tA2IDENT",         /* Identifiers starting with `@@` */
-  "tBANGIDENT",
-  "tEQIDENT",
-  "tQIDENT",          /* Quoted identifier */
-  "pAREF_OPR",        /* [] */
-  "tOPERATOR",        /* Operator identifier */
-
-  "tCOMMENT",
-  "tLINECOMMENT",
-
-  "tTRIVIA",
-
-  "tDQSTRING",        /* Double quoted string */
-  "tSQSTRING",        /* Single quoted string */
-  "tINTEGER",         /* Integer */
-  "tSYMBOL",          /* Symbol */
-  "tDQSYMBOL",
-  "tSQSYMBOL",
-  "tANNOTATION",      /* Annotation */
-};
-
-token NullToken = { NullType };
-position NullPosition = { -1, -1, -1, -1 };
-range NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
-
-const char *token_type_str(enum TokenType type) {
-  return RBS_TOKENTYPE_NAMES[type];
-}
-
-int token_chars(token tok) {
-  return tok.range.end.char_pos - tok.range.start.char_pos;
-}
-
-int token_bytes(token tok) {
-  return RANGE_BYTES(tok.range);
-}
-
-unsigned int peek(lexstate *state) {
-  if (state->current.char_pos == state->end_pos) {
-    state->last_char = '\0';
-    return 0;
-  } else {
-    unsigned int c = rb_enc_mbc_to_codepoint(RSTRING_PTR(state->string) + state->current.byte_pos, RSTRING_END(state->string), rb_enc_get(state->string));
-    state->last_char = c;
-    return c;
-  }
-}
-
-token next_token(lexstate *state, enum TokenType type) {
-  token t;
-
-  t.type = type;
-  t.range.start = state->start;
-  t.range.end = state->current;
-  state->start = state->current;
-  if (type != tTRIVIA) {
-    state->first_token_of_line = false;
-  }
-
-  return t;
-}
-
-token next_eof_token(lexstate *state) {
-  if (state->current.byte_pos == RSTRING_LEN(state->string)+1) {
-    // End of String
-    token t;
-    t.type = pEOF;
-    t.range.start = state->start;
-    t.range.end = state->start;
-    state->start = state->current;
-
-    return t;
-  } else {
-    // NULL byte in the middle of the string
-    return next_token(state, pEOF);
-  }
-}
-
-void rbs_skip(lexstate *state) {
-  if (!state->last_char) {
-    peek(state);
-  }
-  int byte_len = rb_enc_codelen(state->last_char, rb_enc_get(state->string));
-
-  state->current.char_pos += 1;
-  state->current.byte_pos += byte_len;
-
-  if (state->last_char == '\n') {
-    state->current.line += 1;
-    state->current.column = 0;
-    state->first_token_of_line = true;
-  } else {
-    state->current.column += 1;
-  }
-}
-
-void skipn(lexstate *state, size_t size) {
-  for (size_t i = 0; i < size; i ++) {
-    peek(state);
-    rbs_skip(state);
-  }
-}
-
-char *peek_token(lexstate *state, token tok) {
-  return RSTRING_PTR(state->string) + tok.range.start.byte_pos;
-}