rbs 1.6.2 → 1.7.0.beta.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de88dd41e7cee057e4668ffd3ed94e91d5d09746cdbfcd26602cc04b9ef2abd1
-  data.tar.gz: '06092d8a703d157bc214f81155bc24009491db68073fbcedc5fde9b09aca85e7'
+  metadata.gz: 6f2e7e51d08feb7b1a562ac43c679da8fc6f0a7cfc2ecef65e11c1e8c3d0d2b9
+  data.tar.gz: d41ed33c0b5de01cd1746faa3572e56f651ecfc808c046cdb4166a0ec357138e
 SHA512:
-  metadata.gz: 2a53e2b25733fd63c832748ae629275becb31badd9b120cf2b8c72c66cea7e084d0aefefeaa09c00f2364ab1335ee030b0b54e8a14f3f7dbe08e1523cfe1d7e2
-  data.tar.gz: 66b02700d2a81f5d8367451356af01a0cac8b0152194adb5f844abf92ffdfc16bcec975f2e21302477b32ee344242d3e0407105c3cf4e7a78387fa454003a418
+  metadata.gz: bc198dc7d40caaa802604cb1f633098955540d838c4e1adcca635da85880703378d981a4f37c528e7f4b3720fc6f152d7fc6c7bcb11ce3e0abba76fd14a6d47c
+  data.tar.gz: 68c6643f9adf4761bec2a2ffb211e223cb732a37c341652c4625191b5ab6d6791c91196f3c3bc4fd5624d6d853558c33e28f8462f483f923badad8f38498a675

data/.github/workflows/ruby.yml

@@ -19,10 +19,6 @@ jobs:
         - test
         - stdlib_test
         - rubocop validate test_doc build test_generate_stdlib
-        - confirm_parser
-        exclude:
-        - container_tag: master-nightly-focal
-          job: confirm_parser
     container:
       image: rubylang/ruby:${{ matrix.container_tag }}
     steps:

data/.gitignore

@@ -11,3 +11,4 @@
 /vendor/sigs
 /Gemfile.lock
 
+lib/**/*.bundle

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## master
 
+## 1.7.0 (beta.1)
+
+## Library changes
+
+* Replace RBS::Parser ([#788](https://github.com/ruby/rbs/pull/788))
+
 ## 1.6.2 (2021-09-09)
 
 ## Signature updates

data/Gemfile

@@ -5,6 +5,7 @@ gemspec
 
 # Development dependencies
 gem "rake"
+gem "rake-compiler"
 gem "test-unit"
 gem "rspec"
 gem "racc"

data/Rakefile

@@ -1,15 +1,17 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 require "rbconfig"
+require 'rake/extensiontask'
 
 $LOAD_PATH << File.join(__dir__, "test")
 
 ruby = ENV["RUBY"] || RbConfig.ruby
-racc = ENV.fetch("RACC", "racc")
 rbs = File.join(__dir__, "exe/rbs")
 bin = File.join(__dir__, "bin")
 
-Rake::TestTask.new(:test) do |t|
+Rake::ExtensionTask.new("rbs/extension")
+
+Rake::TestTask.new(:test => :compile) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList["test/**/*_test.rb"].reject do |path|
@@ -19,7 +21,7 @@ end
 
 multitask :default => [:test, :stdlib_test, :rubocop, :validate, :test_doc]
 
-task :test_doc => :parser do
+task :test_doc do
   files = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").select do |file| Pathname(file).extname == ".md" end
   end
@@ -27,7 +29,7 @@ task :test_doc => :parser do
   sh "#{ruby} #{__dir__}/bin/run_in_md.rb #{files.join(" ")}"
 end
 
-task :validate => :parser do
+task :validate => :compile do
   sh "#{ruby} #{rbs} validate --silent"
 
   FileList["stdlib/*"].each do |path|
@@ -72,7 +74,7 @@ task :validate => :parser do
 end
 
 FileList["test/stdlib/**/*_test.rb"].each do |test|
-  task test => :parser do
+  task test => :compile do
     sh "#{ruby} -Ilib #{bin}/test_runner.rb #{test}"
   end
   task stdlib_test: test
@@ -82,21 +84,6 @@ task :rubocop do
   sh "rubocop --parallel"
 end
 
-rule ".rb" => ".y" do |t|
-  sh "#{racc} -v -o #{t.name} #{t.source}"
-end
-
-task :parser => "lib/rbs/parser.rb"
-task :test => :parser
-task :stdlib_test => :parser
-task :build => :parser
-
-task :confirm_parser do
-  puts "Testing if parser.rb is updated with respect to parser.y"
-  sh "#{racc} -v -o lib/rbs/parser.rb lib/rbs/parser.y"
-  sh "git diff --exit-code lib/rbs/parser.rb"
-end
-
 namespace :generate do
   desc "Generate a test file for a stdlib class signatures"
   task :stdlib_test, [:class] do |_task, args|
@@ -225,5 +212,3 @@ task :test_generate_stdlib do
   sh "RBS_GENERATE_TEST_PATH=/tmp/Array_test.rb rake 'generate:stdlib_test[Array]'"
   sh "ruby -c /tmp/Array_test.rb"
 end
-
-CLEAN.include("lib/rbs/parser.rb")

data/core/kernel.rbs

@@ -482,8 +482,8 @@ module Kernel : BasicObject
                    | [:child, int] # redirect to the redirected file descriptor
                    | :close # close the file descriptor in child process
 
-  def self?.spawn: (String command, *String args, ?unsetenv_others: boolish, ?pgroup?: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> Integer
-                 | (Hash[string, string?] env, String command, *String args, ?unsetenv_others: boolish, ?pgroup?: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> Integer
+  def self?.spawn: (String command, *String args, ?unsetenv_others: boolish, ?pgroup: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> Integer
+                 | (Hash[string, string?] env, String command, *String args, ?unsetenv_others: boolish, ?pgroup: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> Integer
 
   # Executes *command…* in a subshell. *command…* is one of following forms.
   #
@@ -510,8 +510,8 @@ module Kernel : BasicObject
   #     *
   #
   # See `Kernel.exec` for the standard shell.
-  def self?.system: (String command, *String args, ?unsetenv_others: boolish, ?pgroup?: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> (NilClass | FalseClass | TrueClass)
-                  | (Hash[string, string?] env, String command, *String args, ?unsetenv_others: boolish, ?pgroup?: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> (NilClass | FalseClass | TrueClass)
+  def self?.system: (String command, *String args, ?unsetenv_others: boolish, ?pgroup: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> (NilClass | FalseClass | TrueClass)
+                  | (Hash[string, string?] env, String command, *String args, ?unsetenv_others: boolish, ?pgroup: (true | Integer), ?umask: Integer, ?in: redirect_fd, ?out: redirect_fd, ?err: redirect_fd, ?close_others: boolish, ?chdir: String) -> (NilClass | FalseClass | TrueClass)
 end
 
 Kernel::RUBYGEMS_ACTIVATION_MONITOR: untyped

data/core/trace_point.rbs

@@ -286,5 +286,5 @@ class TracePoint < Object
   # Same as TracePoint#binding:
   #     trace.binding.eval('self')
   #
-  def `self`: () -> Binding
+  def self: () -> Binding
 end

data/ext/rbs/extension/constants.c

@@ -0,0 +1,140 @@
+#include "rbs_extension.h"
+
+VALUE RBS_Parser;
+VALUE RBS_Parser_KEYWORDS;
+
+VALUE RBS;
+VALUE RBS_AST;
+VALUE RBS_AST_Comment;
+VALUE RBS_AST_Annotation;
+
+VALUE RBS_AST_Declarations;
+
+VALUE RBS_AST_Declarations_ModuleTypeParams;
+VALUE RBS_AST_Declarations_ModuleTypeParams_TypeParam;
+
+VALUE RBS_AST_Declarations_Alias;
+VALUE RBS_AST_Declarations_Constant;
+VALUE RBS_AST_Declarations_Global;
+VALUE RBS_AST_Declarations_Interface;
+VALUE RBS_AST_Declarations_Module;
+VALUE RBS_AST_Declarations_Module_Self;
+VALUE RBS_AST_Declarations_Class;
+VALUE RBS_AST_Declarations_Class_Super;
+
+VALUE RBS_AST_Members;
+VALUE RBS_AST_Members_Alias;
+VALUE RBS_AST_Members_AttrAccessor;
+VALUE RBS_AST_Members_AttrReader;
+VALUE RBS_AST_Members_AttrWriter;
+VALUE RBS_AST_Members_ClassInstanceVariable;
+VALUE RBS_AST_Members_ClassVariable;
+VALUE RBS_AST_Members_Extend;
+VALUE RBS_AST_Members_Include;
+VALUE RBS_AST_Members_InstanceVariable;
+VALUE RBS_AST_Members_MethodDefinition;
+VALUE RBS_AST_Members_Prepend;
+VALUE RBS_AST_Members_Private;
+VALUE RBS_AST_Members_Public;
+
+VALUE RBS_Namespace;
+VALUE RBS_TypeName;
+
+VALUE RBS_Types_Alias;
+VALUE RBS_Types_Bases_Any;
+VALUE RBS_Types_Bases_Bool;
+VALUE RBS_Types_Bases_Bottom;
+VALUE RBS_Types_Bases_Class;
+VALUE RBS_Types_Bases_Instance;
+VALUE RBS_Types_Bases_Nil;
+VALUE RBS_Types_Bases_Self;
+VALUE RBS_Types_Bases_Top;
+VALUE RBS_Types_Bases_Void;
+VALUE RBS_Types_Bases;
+VALUE RBS_Types_Block;
+VALUE RBS_Types_ClassInstance;
+VALUE RBS_Types_ClassSingleton;
+VALUE RBS_Types_Function_Param;
+VALUE RBS_Types_Function;
+VALUE RBS_Types_Interface;
+VALUE RBS_Types_Intersection;
+VALUE RBS_Types_Literal;
+VALUE RBS_Types_Optional;
+VALUE RBS_Types_Proc;
+VALUE RBS_Types_Record;
+VALUE RBS_Types_Tuple;
+VALUE RBS_Types_Union;
+VALUE RBS_Types_Variable;
+VALUE RBS_Types;
+VALUE RBS_MethodType;
+
+VALUE RBS_ParsingError;
+
+void rbs__init_constants() {
+  ID id_RBS = rb_intern_const("RBS");
+
+  RBS = rb_const_get(rb_cObject, id_RBS);
+  RBS_ParsingError = rb_const_get(RBS, rb_intern("ParsingError"));
+  RBS_AST = rb_const_get(RBS, rb_intern("AST"));
+  RBS_AST_Comment = rb_const_get(RBS_AST, rb_intern("Comment"));
+  RBS_AST_Annotation = rb_const_get(RBS_AST, rb_intern("Annotation"));
+
+  RBS_AST_Declarations = rb_const_get(RBS_AST, rb_intern("Declarations"));
+
+  RBS_AST_Declarations_ModuleTypeParams = rb_const_get(RBS_AST_Declarations, rb_intern("ModuleTypeParams"));
+  RBS_AST_Declarations_ModuleTypeParams_TypeParam = rb_const_get(RBS_AST_Declarations_ModuleTypeParams, rb_intern("TypeParam"));
+
+  RBS_AST_Declarations_Alias = rb_const_get(RBS_AST_Declarations, rb_intern("Alias"));
+  RBS_AST_Declarations_Constant = rb_const_get(RBS_AST_Declarations, rb_intern("Constant"));
+  RBS_AST_Declarations_Global = rb_const_get(RBS_AST_Declarations, rb_intern("Global"));
+  RBS_AST_Declarations_Interface = rb_const_get(RBS_AST_Declarations, rb_intern("Interface"));
+  RBS_AST_Declarations_Module = rb_const_get(RBS_AST_Declarations, rb_intern("Module"));
+  RBS_AST_Declarations_Module_Self = rb_const_get(RBS_AST_Declarations_Module, rb_intern("Self"));
+  RBS_AST_Declarations_Class = rb_const_get(RBS_AST_Declarations, rb_intern("Class"));
+  RBS_AST_Declarations_Class_Super = rb_const_get(RBS_AST_Declarations_Class, rb_intern("Super"));
+
+  RBS_AST_Members = rb_const_get(RBS_AST, rb_intern("Members"));
+  RBS_AST_Members_Alias = rb_const_get(RBS_AST_Members, rb_intern("Alias"));
+  RBS_AST_Members_AttrAccessor = rb_const_get(RBS_AST_Members, rb_intern("AttrAccessor"));
+  RBS_AST_Members_AttrReader = rb_const_get(RBS_AST_Members, rb_intern("AttrReader"));
+  RBS_AST_Members_AttrWriter = rb_const_get(RBS_AST_Members, rb_intern("AttrWriter"));
+  RBS_AST_Members_ClassInstanceVariable = rb_const_get(RBS_AST_Members, rb_intern("ClassInstanceVariable"));
+  RBS_AST_Members_ClassVariable = rb_const_get(RBS_AST_Members, rb_intern("ClassVariable"));
+  RBS_AST_Members_Extend = rb_const_get(RBS_AST_Members, rb_intern("Extend"));
+  RBS_AST_Members_Include = rb_const_get(RBS_AST_Members, rb_intern("Include"));
+  RBS_AST_Members_InstanceVariable = rb_const_get(RBS_AST_Members, rb_intern("InstanceVariable"));
+  RBS_AST_Members_MethodDefinition = rb_const_get(RBS_AST_Members, rb_intern("MethodDefinition"));
+  RBS_AST_Members_Prepend = rb_const_get(RBS_AST_Members, rb_intern("Prepend"));
+  RBS_AST_Members_Private = rb_const_get(RBS_AST_Members, rb_intern("Private"));
+  RBS_AST_Members_Public = rb_const_get(RBS_AST_Members, rb_intern("Public"));
+
+  RBS_Namespace = rb_const_get(RBS, rb_intern("Namespace"));
+  RBS_TypeName = rb_const_get(RBS, rb_intern("TypeName"));
+  RBS_Types = rb_const_get(RBS, rb_intern("Types"));
+  RBS_Types_Alias = rb_const_get(RBS_Types, rb_intern("Alias"));
+  RBS_Types_Bases = rb_const_get(RBS_Types, rb_intern("Bases"));
+  RBS_Types_Bases_Any = rb_const_get(RBS_Types_Bases, rb_intern("Any"));
+  RBS_Types_Bases_Bool = rb_const_get(RBS_Types_Bases, rb_intern("Bool"));
+  RBS_Types_Bases_Bottom = rb_const_get(RBS_Types_Bases, rb_intern("Bottom"));
+  RBS_Types_Bases_Class = rb_const_get(RBS_Types_Bases, rb_intern("Class"));
+  RBS_Types_Bases_Instance = rb_const_get(RBS_Types_Bases, rb_intern("Instance"));
+  RBS_Types_Bases_Nil = rb_const_get(RBS_Types_Bases, rb_intern("Nil"));
+  RBS_Types_Bases_Self = rb_const_get(RBS_Types_Bases, rb_intern("Self"));
+  RBS_Types_Bases_Top = rb_const_get(RBS_Types_Bases, rb_intern("Top"));
+  RBS_Types_Bases_Void = rb_const_get(RBS_Types_Bases, rb_intern("Void"));
+  RBS_Types_Block = rb_const_get(RBS_Types, rb_intern("Block"));
+  RBS_Types_ClassInstance = rb_const_get(RBS_Types, rb_intern("ClassInstance"));
+  RBS_Types_ClassSingleton = rb_const_get(RBS_Types, rb_intern("ClassSingleton"));
+  RBS_Types_Function = rb_const_get(RBS_Types, rb_intern("Function"));
+  RBS_Types_Function_Param = rb_const_get(RBS_Types_Function, rb_intern("Param"));
+  RBS_Types_Interface = rb_const_get(RBS_Types, rb_intern("Interface"));
+  RBS_Types_Intersection = rb_const_get(RBS_Types, rb_intern("Intersection"));
+  RBS_Types_Literal = rb_const_get(RBS_Types, rb_intern("Literal"));
+  RBS_Types_Optional = rb_const_get(RBS_Types, rb_intern("Optional"));
+  RBS_Types_Proc = rb_const_get(RBS_Types, rb_intern("Proc"));
+  RBS_Types_Record = rb_const_get(RBS_Types, rb_intern("Record"));
+  RBS_Types_Tuple = rb_const_get(RBS_Types, rb_intern("Tuple"));
+  RBS_Types_Union = rb_const_get(RBS_Types, rb_intern("Union"));
+  RBS_Types_Variable = rb_const_get(RBS_Types, rb_intern("Variable"));
+  RBS_MethodType = rb_const_get(RBS, rb_intern("MethodType"));
+}

data/ext/rbs/extension/constants.h

@@ -0,0 +1,72 @@
+#ifndef RBS__CONSTANTS_H
+#define RBS__CONSTANTS_H
+
+extern VALUE RBS;
+
+extern VALUE RBS_AST;
+extern VALUE RBS_AST_Annotation;
+extern VALUE RBS_AST_Comment;
+
+extern VALUE RBS_AST_Declarations;
+extern VALUE RBS_AST_Declarations_Alias;
+extern VALUE RBS_AST_Declarations_Class_Super;
+extern VALUE RBS_AST_Declarations_Class;
+extern VALUE RBS_AST_Declarations_Constant;
+extern VALUE RBS_AST_Declarations_Global;
+extern VALUE RBS_AST_Declarations_Interface;
+extern VALUE RBS_AST_Declarations_Module_Self;
+extern VALUE RBS_AST_Declarations_Module;
+extern VALUE RBS_AST_Declarations_ModuleTypeParams_TypeParam;
+extern VALUE RBS_AST_Declarations_ModuleTypeParams;
+
+extern VALUE RBS_AST_Members;
+extern VALUE RBS_AST_Members_Alias;
+extern VALUE RBS_AST_Members_AttrAccessor;
+extern VALUE RBS_AST_Members_AttrReader;
+extern VALUE RBS_AST_Members_AttrWriter;
+extern VALUE RBS_AST_Members_ClassInstanceVariable;
+extern VALUE RBS_AST_Members_ClassVariable;
+extern VALUE RBS_AST_Members_Extend;
+extern VALUE RBS_AST_Members_Include;
+extern VALUE RBS_AST_Members_InstanceVariable;
+extern VALUE RBS_AST_Members_MethodDefinition;
+extern VALUE RBS_AST_Members_Prepend;
+extern VALUE RBS_AST_Members_Private;
+extern VALUE RBS_AST_Members_Public;
+
+extern VALUE RBS_MethodType;
+extern VALUE RBS_Namespace;
+
+extern VALUE RBS_ParsingError;
+extern VALUE RBS_TypeName;
+
+extern VALUE RBS_Types;
+extern VALUE RBS_Types_Alias;
+extern VALUE RBS_Types_Bases;
+extern VALUE RBS_Types_Bases_Any;
+extern VALUE RBS_Types_Bases_Bool;
+extern VALUE RBS_Types_Bases_Bottom;
+extern VALUE RBS_Types_Bases_Class;
+extern VALUE RBS_Types_Bases_Instance;
+extern VALUE RBS_Types_Bases_Nil;
+extern VALUE RBS_Types_Bases_Self;
+extern VALUE RBS_Types_Bases_Top;
+extern VALUE RBS_Types_Bases_Void;
+extern VALUE RBS_Types_Block;
+extern VALUE RBS_Types_ClassInstance;
+extern VALUE RBS_Types_ClassSingleton;
+extern VALUE RBS_Types_Function_Param;
+extern VALUE RBS_Types_Function;
+extern VALUE RBS_Types_Interface;
+extern VALUE RBS_Types_Intersection;
+extern VALUE RBS_Types_Literal;
+extern VALUE RBS_Types_Optional;
+extern VALUE RBS_Types_Proc;
+extern VALUE RBS_Types_Record;
+extern VALUE RBS_Types_Tuple;
+extern VALUE RBS_Types_Union;
+extern VALUE RBS_Types_Variable;
+
+void rbs__init_constants();
+
+#endif

data/ext/rbs/extension/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+$INCFLAGS << " -I$(top_srcdir)" if $extmk
+create_makefile 'extension'

data/ext/rbs/extension/lexer.c

@@ -0,0 +1,1070 @@
+#include "rbs_extension.h"
+
+#define ONE_CHAR_PATTERN(c, t) case c: tok = next_token(state, t); break
+
+/**
+ * Returns one character at current.
+ *
+ * ... A B C ...
+ *    ^           current => A
+ * */
+#define peek(state) rb_enc_mbc_to_codepoint(RSTRING_PTR(state->string) + state->current.byte_pos, RSTRING_END(state->string), rb_enc_get(state->string))
+
+static const char *RBS_TOKENTYPE_NAMES[] = {
+  "NullType",
+  "pEOF",
+  "ErrorToken",
+
+  "pLPAREN",          /* ( */
+  "pRPAREN",          /* ) */
+  "pCOLON",           /* : */
+  "pCOLON2",          /* :: */
+  "pLBRACKET",        /* [ */
+  "pRBRACKET",        /* ] */
+  "pLBRACE",          /* { */
+  "pRBRACE",          /* } */
+  "pHAT",             /* ^ */
+  "pARROW",           /* -> */
+  "pFATARROW",        /* => */
+  "pCOMMA",           /* , */
+  "pBAR",             /* | */
+  "pAMP",             /* & */
+  "pSTAR",            /* * */
+  "pSTAR2",           /* ** */
+  "pDOT",             /* . */
+  "pDOT3",            /* ... */
+  "pBANG",            /* ! */
+  "pQUESTION",        /* ? */
+  "pLT",              /* < */
+  "pEQ",              /* = */
+
+  "kBOOL",            /* bool */
+  "kBOT",             /* bot */
+  "kCLASS",           /* class */
+  "kFALSE",           /* kFALSE */
+  "kINSTANCE",        /* instance */
+  "kINTERFACE",       /* interface */
+  "kNIL",             /* nil */
+  "kSELF",            /* self */
+  "kSINGLETON",       /* singleton */
+  "kTOP",             /* top */
+  "kTRUE",            /* true */
+  "kVOID",            /* void */
+  "kTYPE",            /* type */
+  "kUNCHECKED",       /* unchecked */
+  "kIN",              /* in */
+  "kOUT",             /* out */
+  "kEND",             /* end */
+  "kDEF",             /* def */
+  "kINCLUDE",         /* include */
+  "kEXTEND",          /* extend */
+  "kPREPEND",         /* prepend */
+  "kALIAS",           /* alias */
+  "kMODULE",          /* module */
+  "kATTRREADER",      /* attr_reader */
+  "kATTRWRITER",      /* attr_writer */
+  "kATTRACCESSOR",    /* attr_accessor */
+  "kPUBLIC",          /* public */
+  "kPRIVATE",         /* private */
+  "kUNTYPED",         /* untyped */
+
+  "tLIDENT",          /* Identifiers starting with lower case */
+  "tUIDENT",          /* Identifiers starting with upper case */
+  "tULIDENT",         /* Identifiers starting with `_` */
+  "tULLIDENT",
+  "tGIDENT",          /* Identifiers starting with `$` */
+  "tAIDENT",          /* Identifiers starting with `@` */
+  "tA2IDENT",         /* Identifiers starting with `@@` */
+  "tBANGIDENT",
+  "tEQIDENT",
+  "tQIDENT",          /* Quoted identifier */
+  "tOPERATOR",        /* Operator identifier */
+
+  "tCOMMENT",
+  "tLINECOMMENT",
+
+  "tDQSTRING",        /* Double quoted string */
+  "tSQSTRING",        /* Single quoted string */
+  "tINTEGER",         /* Integer */
+  "tSYMBOL",          /* Symbol */
+  "tDQSYMBOL",
+  "tSQSYMBOL",
+  "tANNOTATION",      /* Annotation */
+};
+
+token NullToken = { NullType };
+position NullPosition = { -1, -1, -1, -1 };
+range NULL_RANGE = { { -1, -1, -1, -1 }, { -1, -1, -1, -1 } };
+
+const char *token_type_str(enum TokenType type) {
+  return RBS_TOKENTYPE_NAMES[type];
+}
+
+unsigned int peekn(lexstate *state, unsigned int chars[], size_t length) {
+  int byteoffset = 0;
+
+  rb_encoding *encoding = rb_enc_get(state->string);
+  char *start = RSTRING_PTR(state->string) + state->current.byte_pos;
+  char *end = RSTRING_END(state->string);
+
+  for (size_t i = 0; i < length; i++)
+  {
+    chars[i] = rb_enc_mbc_to_codepoint(start + byteoffset, end, encoding);
+    byteoffset += rb_enc_codelen(chars[i], rb_enc_get(state->string));
+  }
+
+  return byteoffset;
+}
+
+int token_chars(token tok) {
+  return tok.range.end.char_pos - tok.range.start.char_pos;
+}
+
+int token_bytes(token tok) {
+  return RANGE_BYTES(tok.range);
+}
+
+/**
+ * ... token ...
+ *    ^             start
+ *          ^       current
+ *
+ * */
+token next_token(lexstate *state, enum TokenType type) {
+  token t;
+
+  t.type = type;
+  t.range.start = state->start;
+  t.range.end = state->current;
+  state->start = state->current;
+  state->first_token_of_line = false;
+
+  return t;
+}
+
+void advance_skip(lexstate *state, unsigned int c, bool skip) {
+  int len = rb_enc_codelen(c, rb_enc_get(state->string));
+
+  state->current.char_pos += 1;
+  state->current.byte_pos += len;
+
+  if (c == '\n') {
+    state->current.line += 1;
+    state->current.column = 0;
+    state->first_token_of_line = true;
+  } else {
+    state->current.column += 1;
+  }
+
+  if (skip) {
+    state->start = state->current;
+  }
+}
+
+void advance_char(lexstate *state, unsigned int c) {
+  advance_skip(state, c, false);
+}
+
+void skip_char(lexstate *state, unsigned int c) {
+  advance_skip(state, c, true);
+}
+
+void skip(lexstate *state) {
+  unsigned char c = peek(state);
+  skip_char(state, c);
+}
+
+void advance(lexstate *state) {
+  unsigned char c = peek(state);
+  advance_char(state, c);
+}
+
+/*
+  1. Peek one character from state
+  2. If read characetr equals to given `c`, skip the character and return true.
+  3. Return false otherwise.
+*/
+static bool advance_next_character_if(lexstate *state, unsigned int c) {
+  if (peek(state) == c) {
+    advance_char(state, c);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+/*
+   ... 0 1 ...
+        ^           current
+          ^         current (return)
+*/
+static token lex_number(lexstate *state) {
+  unsigned int c;
+
+  while (true) {
+    c = peek(state);
+
+    if (rb_isdigit(c) || c == '_') {
+      advance_char(state, c);
+    } else {
+      break;
+    }
+  }
+
+  return next_token(state, tINTEGER);
+}
+
+/*
+  lex_hyphen ::=  -       (tOPERATOR)
+               |  - @     (tOPERATOR)
+               |  - >     (pARROW)
+               |  - 1 ... (tINTEGER)
+*/
+static token lex_hyphen(lexstate* state) {
+  if (advance_next_character_if(state, '>')) {
+    return next_token(state, pARROW);
+  } else if (advance_next_character_if(state, '@')) {
+    return next_token(state, tOPERATOR);
+  } else {
+    unsigned int c = peek(state);
+
+    if (rb_isdigit(c)) {
+      advance_char(state, c);
+      return lex_number(state);
+    } else {
+      return next_token(state, tOPERATOR);
+    }
+  }
+}
+
+/*
+  lex_plus ::= +
+             | + @
+             | + \d
+*/
+static token lex_plus(lexstate *state) {
+  if (advance_next_character_if(state, '@')) {
+    return next_token(state, tOPERATOR);
+  } else if (rb_isdigit(peek(state))) {
+    return lex_number(state);
+  } else {
+    return next_token(state, tOPERATOR);
+  }
+}
+
+/*
+  lex_dot ::= .         pDOT
+            | . . .     pDOT3
+*/
+static token lex_dot(lexstate *state) {
+  unsigned int cs[2];
+
+  peekn(state, cs, 2);
+
+  if (cs[0] == '.' && cs[1] == '.') {
+    advance_char(state, '.');
+    advance_char(state, '.');
+    return next_token(state, pDOT3);
+  } else {
+    return next_token(state, pDOT);
+  }
+}
+
+/*
+  lex_eq ::= =
+           | ==
+           | ===
+           | =~
+           | =>
+*/
+static token lex_eq(lexstate *state) {
+  unsigned int cs[2];
+  peekn(state, cs, 2);
+
+  if (cs[0] == '=' && cs[1] == '=') {
+    // ===
+    advance_char(state, cs[0]);
+    advance_char(state, cs[1]);
+    return next_token(state, tOPERATOR);
+  } else if (cs[0] == '=') {
+    // ==
+    advance_char(state, cs[0]);
+    return next_token(state, tOPERATOR);
+  } else if (cs[0] == '~') {
+    // =~
+    advance_char(state, cs[0]);
+    return next_token(state, tOPERATOR);
+  } else if (cs[0] == '>') {
+    // =>
+    advance_char(state, cs[0]);
+    return next_token(state, pFATARROW);
+  } else {
+    return next_token(state, pEQ);
+  }
+}
+
+/*
+  underscore ::= _A        tULIDENT
+               | _a        tULLIDENT
+               | _         tULLIDENT
+*/
+static token lex_underscore(lexstate *state) {
+  unsigned int c;
+
+  c = peek(state);
+
+  if ('A' <= c && c <= 'Z') {
+    advance_char(state, c);
+
+    while (true) {
+      c = peek(state);
+
+      if (rb_isalnum(c) || c == '_') {
+        // ok
+        advance_char(state, c);
+      } else {
+        break;
+      }
+    }
+
+    return next_token(state, tULIDENT);
+  } else if (rb_isalnum(c) || c == '_') {
+    advance_char(state, c);
+
+    while (true) {
+      c = peek(state);
+
+      if (rb_isalnum(c) || c == '_') {
+        // ok
+        advance_char(state, c);
+      } else {
+        break;
+      }
+    }
+
+    if (c == '!') {
+      advance_char(state, c);
+      return next_token(state, tBANGIDENT);
+    } else if (c == '=') {
+      advance_char(state, c);
+      return next_token(state, tEQIDENT);
+    } else {
+      return next_token(state, tULLIDENT);
+    }
+  } else {
+    return next_token(state, tULLIDENT);
+  }
+}
+
+static bool is_opr(unsigned int c) {
+  switch (c) {
+  case ':':
+  case ';':
+  case '=':
+  case '.':
+  case ',':
+  case '!':
+  case '"':
+  case '$':
+  case '%':
+  case '&':
+  case '(':
+  case ')':
+  case '-':
+  case '+':
+  case '~':
+  case '|':
+  case '\\':
+  case '\'':
+  case '[':
+  case ']':
+  case '{':
+  case '}':
+  case '*':
+  case '/':
+  case '<':
+  case '>':
+  case '^':
+    return true;
+  default:
+    return false;
+  }
+}
+
+static token lex_global(lexstate *state) {
+  unsigned int c;
+
+  c = peek(state);
+
+  if (rb_isspace(c) || c == 0) {
+    return next_token(state, ErrorToken);
+  }
+
+  if (rb_isdigit(c)) {
+    // `$` [`0`-`9`]+
+    advance_char(state, c);
+
+    while (true) {
+      c = peek(state);
+      if (rb_isdigit(c)) {
+        advance_char(state, c);
+      } else {
+        return next_token(state, tGIDENT);
+      }
+    }
+  }
+
+  if (c == '-') {
+    // `$` `-` [a-zA-Z0-9_]
+    advance_char(state, c);
+    c = peek(state);
+
+    if (rb_isalnum(c) || c == '_') {
+      advance_char(state, c);
+      return next_token(state, tGIDENT);
+    } else {
+      return next_token(state, ErrorToken);
+    }
+  }
+
+  switch (c) {
+  case '~':
+  case '*':
+  case '$':
+  case '?':
+  case '!':
+  case '@':
+  case '\\':
+  case '/':
+  case ';':
+  case ',':
+  case '.':
+  case '=':
+  case ':':
+  case '<':
+  case '>':
+  case '"':
+  case '&':
+  case '\'':
+  case '`':
+  case '+':
+    advance_char(state, c);
+    return next_token(state, tGIDENT);
+
+  default:
+    if (is_opr(c) || c == 0) {
+      return next_token(state, ErrorToken);
+    }
+
+    while (true) {
+      advance_char(state, c);
+      c = peek(state);
+
+      if (rb_isspace(c) || is_opr(c) || c == 0) {
+        break;
+      }
+    }
+
+    return next_token(state, tGIDENT);
+  }
+}
+
+void pp(VALUE object) {
+  VALUE inspect = rb_funcall(object, rb_intern("inspect"), 0);
+  printf("pp >> %s\n", RSTRING_PTR(inspect));
+}
+
+static token lex_ident(lexstate *state, enum TokenType default_type) {
+  unsigned int c;
+  token tok;
+
+  while (true) {
+    c = peek(state);
+    if (rb_isalnum(c) || c == '_') {
+      advance_char(state, c);
+    } else if (c == '!') {
+      advance_char(state, c);
+      tok = next_token(state, tBANGIDENT);
+      break;
+    } else if (c == '=') {
+      advance_char(state, c);
+      tok = next_token(state, tEQIDENT);
+      break;
+    } else {
+      tok = next_token(state, default_type);
+      break;
+    }
+  }
+
+  if (tok.type == tLIDENT) {
+    VALUE string = rb_enc_str_new(
+      RSTRING_PTR(state->string) + tok.range.start.byte_pos,
+      RANGE_BYTES(tok.range),
+      rb_enc_get(state->string)
+    );
+
+    VALUE type = rb_hash_aref(RBS_Parser_KEYWORDS, string);
+    if (FIXNUM_P(type)) {
+      tok.type = FIX2INT(type);
+    }
+  }
+
+  return tok;
+}
+
+static token lex_comment(lexstate *state, enum TokenType type) {
+  unsigned int c;
+
+  c = peek(state);
+  if (c == ' ') {
+    advance_char(state, c);
+  }
+
+  while (true) {
+    c = peek(state);
+
+    if (c == '\n' || c == '\0') {
+      break;
+    } else {
+      advance_char(state, c);
+    }
+  }
+
+  token tok = next_token(state, type);
+
+  skip_char(state, c);
+
+  return tok;
+}
+
+/*
+   ... " ... " ...
+      ^               start
+        ^             current
+              ^       current (after)
+*/
+static token lex_dqstring(lexstate *state) {
+  unsigned int c;
+
+  while (true) {
+    c = peek(state);
+    advance_char(state, c);
+
+    if (c == '\\') {
+      if (peek(state) == '"') {
+        advance_char(state, c);
+        c = peek(state);
+      }
+    } else if (c == '"') {
+      break;
+    }
+  }
+
+  return next_token(state, tDQSTRING);
+}
+
+/*
+   ... @ foo ...
+      ^             start
+        ^           current
+            ^       current (return)
+
+   ... @ @ foo ...
+      ^               start
+        ^             current
+              ^       current (return)
+*/
+static token lex_ivar(lexstate *state) {
+  unsigned int c;
+
+  enum TokenType type = tAIDENT;
+
+  c = peek(state);
+
+  if (c == '@') {
+    type = tA2IDENT;
+    advance_char(state, c);
+    c = peek(state);
+  }
+
+  if (rb_isalpha(c) || c == '_') {
+    advance_char(state, c);
+    c = peek(state);
+  } else {
+    return next_token(state, ErrorToken);
+  }
+
+  while (rb_isalnum(c) || c == '_') {
+    advance_char(state, c);
+    c = peek(state);
+  }
+
+  return next_token(state, type);
+}
+
+/*
+   ... ' ... ' ...
+      ^               start
+        ^             current
+              ^       current (after)
+*/
+static token lex_sqstring(lexstate *state) {
+  unsigned int c;
+
+  c = peek(state);
+
+  while (true) {
+    c = peek(state);
+    advance_char(state, c);
+
+    if (c == '\\') {
+      if (peek(state) == '\'') {
+        advance_char(state, c);
+        c = peek(state);
+      }
+    } else if (c == '\'') {
+      break;
+    }
+  }
+
+  return next_token(state, tSQSTRING);
+}
+
+#define EQPOINTS2(c0, c1, s) (c0 == s[0] && c1 == s[1])
+#define EQPOINTS3(c0, c1, c2, s) (c0 == s[0] && c1 == s[1] && c2 == s[2])
+
+/*
+   ... : @ ...
+      ^          start
+        ^        current
+          ^      current (return)
+*/
+static token lex_colon_symbol(lexstate *state) {
+  unsigned int c[3];
+  peekn(state, c, 3);
+
+  switch (c[0]) {
+    case '|':
+    case '&':
+    case '/':
+    case '%':
+    case '~':
+    case '`':
+    case '^':
+      advance_char(state, c[0]);
+      return next_token(state, tSYMBOL);
+    case '=':
+      if (EQPOINTS2(c[0], c[1], "=~")) {
+        // :=~
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+        return next_token(state, tSYMBOL);
+      } else if (EQPOINTS3(c[0], c[1], c[2], "===")) {
+        // :===
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+        advance_char(state, c[2]);
+        return next_token(state, tSYMBOL);
+      } else if (EQPOINTS2(c[0], c[1], "==")) {
+        // :==
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+        return next_token(state, tSYMBOL);
+      }
+      break;
+    case '<':
+      if (EQPOINTS3(c[0], c[1], c[2], "<=>")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+        advance_char(state, c[2]);
+      } else if (EQPOINTS2(c[0], c[1], "<=") || EQPOINTS2(c[0], c[1], "<<")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        advance_char(state, c[0]);
+      }
+      return next_token(state, tSYMBOL);
+    case '>':
+      if (EQPOINTS2(c[0], c[1], ">=") || EQPOINTS2(c[0], c[1], ">>")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        advance_char(state, c[0]);
+      }
+      return next_token(state, tSYMBOL);
+    case '-':
+    case '+':
+      if (EQPOINTS2(c[0], c[1], "+@") || EQPOINTS2(c[0], c[1], "-@")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        advance_char(state, c[0]);
+      }
+      return next_token(state, tSYMBOL);
+    case '*':
+      if (EQPOINTS2(c[0], c[1], "**")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        advance_char(state, c[0]);
+      }
+      return next_token(state, tSYMBOL);
+    case '[':
+      if (EQPOINTS3(c[0], c[1], c[2], "[]=")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+        advance_char(state, c[2]);
+      } else if (EQPOINTS2(c[0], c[1], "[]")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        break;
+      }
+      return next_token(state, tSYMBOL);
+    case '!':
+      if (EQPOINTS2(c[0], c[1], "!=") || EQPOINTS2(c[0], c[1], "!~")) {
+        advance_char(state, c[0]);
+        advance_char(state, c[1]);
+      } else {
+        advance_char(state, c[0]);
+      }
+      return next_token(state, tSYMBOL);
+    case '@': {
+      advance_char(state, '@');
+      token tok = lex_ivar(state);
+      if (tok.type != ErrorToken) {
+        tok.type = tSYMBOL;
+      }
+      return tok;
+    }
+    case '$': {
+      advance_char(state, '$');
+      token tok = lex_global(state);
+      if (tok.type != ErrorToken) {
+        tok.type = tSYMBOL;
+      }
+      return tok;
+    }
+    case '\'': {
+      position start = state->start;
+      advance_char(state, '\'');
+      token tok = lex_sqstring(state);
+      tok.type = tSQSYMBOL;
+      tok.range.start = start;
+      return tok;
+    }
+    case '"': {
+      position start = state->start;
+      advance_char(state, '"');
+      token tok = lex_dqstring(state);
+      tok.type = tDQSYMBOL;
+      tok.range.start = start;
+      return tok;
+    }
+    default:
+      if (rb_isalpha(c[0]) || c[0] == '_') {
+        position start = state->start;
+        token tok = lex_ident(state, NullType);
+        tok.range.start = start;
+
+        if (peek(state) == '?') {
+          if (tok.type != tBANGIDENT && tok.type != tEQIDENT) {
+            skip_char(state, '?');
+            tok.range.end = state->current;
+          }
+        }
+
+        tok.type = tSYMBOL;
+        return tok;
+      }
+  }
+
+  return next_token(state, pCOLON);
+}
+
+/*
+   ... : : ...
+      ^          start
+        ^        current
+          ^      current (return)
+
+   ... :   ...
+      ^          start
+        ^        current (lex_colon_symbol)
+*/
+static token lex_colon(lexstate *state) {
+  unsigned int c = peek(state);
+
+  if (c == ':') {
+    advance_char(state, c);
+    return next_token(state, pCOLON2);
+  } else {
+    return lex_colon_symbol(state);
+  }
+}
+
+/*
+  lex_lt ::= <       (pLT)
+           | < <     (tOPERATOR)
+           | < =     (tOPERATOR)
+           | < = >   (tOPERATOR)
+*/
+static token lex_lt(lexstate *state) {
+  if (advance_next_character_if(state, '<')) {
+    return next_token(state, tOPERATOR);
+  } else if (advance_next_character_if(state, '=')) {
+    advance_next_character_if(state, '>');
+    return next_token(state, tOPERATOR);
+  } else {
+    return next_token(state, pLT);
+  }
+}
+
+/*
+  lex_gt ::= >
+           | > =
+           | > >
+*/
+static token lex_gt(lexstate *state) {
+  advance_next_character_if(state, '=') || advance_next_character_if(state, '>');
+  return next_token(state, tOPERATOR);
+}
+
+/*
+    ... `%` `a` `{` ... `}` ...
+       ^                         start
+           ^                     current
+                           ^     current (exit)
+                    ---          token
+*/
+static token lex_percent(lexstate *state) {
+  unsigned int cs[2];
+  unsigned int end_char;
+
+  peekn(state, cs, 2);
+
+  if (cs[0] != 'a') {
+    return next_token(state, tOPERATOR);
+  }
+
+  switch (cs[1])
+  {
+  case '{':
+    end_char = '}';
+    break;
+  case '(':
+    end_char = ')';
+    break;
+  case '[':
+    end_char = ']';
+    break;
+  case '|':
+    end_char = '|';
+    break;
+  case '<':
+    end_char = '>';
+    break;
+  default:
+    return next_token(state, tOPERATOR);
+  }
+
+  advance_char(state, cs[0]);
+  advance_char(state, cs[1]);
+
+  unsigned int c;
+
+  while ((c = peek(state))) {
+    if (c == end_char) {
+      advance_char(state, c);
+      return next_token(state, tANNOTATION);
+    }
+    advance_char(state, c);
+  }
+
+  return next_token(state, ErrorToken);
+}
+
+/*
+  bracket ::= [       (pLBRACKET)
+             * ^
+            | [ ]     (tOPERATOR)
+             * ^ $
+            | [ ] =   (tOPERATOR)
+             * ^   $
+*/
+static token lex_bracket(lexstate *state) {
+  if (advance_next_character_if(state, ']')) {
+    advance_next_character_if(state, '=');
+    return next_token(state, tOPERATOR);
+  } else {
+    return next_token(state, pLBRACKET);
+  }
+}
+
+/*
+  bracket ::= *
+            | * *
+*/
+static token lex_star(lexstate *state) {
+  if (advance_next_character_if(state, '*')) {
+    return next_token(state, pSTAR2);
+  } else {
+    return next_token(state, pSTAR);
+  }
+}
+
+/*
+  bang ::= !
+         | ! =
+         | ! ~
+*/
+static token lex_bang(lexstate *state) {
+  advance_next_character_if(state, '=') || advance_next_character_if(state, '~');
+  return next_token(state, tOPERATOR);
+}
+
+/*
+  backquote ::= `            (tOPERATOR)
+              | `[^ :][^`]`  (tQIDENT)
+*/
+static token lex_backquote(lexstate *state) {
+  unsigned int c = peek(state);
+
+  if (c == ' ' || c == ':') {
+    return next_token(state, tOPERATOR);
+  } else {
+    while (true) {
+      if (c == '`') {
+        break;
+      }
+
+      c = peek(state);
+      advance_char(state, c);
+    }
+
+    return next_token(state, tQIDENT);
+  }
+}
+
+token rbsparser_next_token(lexstate *state) {
+  token tok = NullToken;
+
+  unsigned int c;
+  bool skipping = true;
+
+  while (skipping) {
+    c = peek(state);
+
+    switch (c) {
+    case ' ':
+    case '\t':
+    case '\n':
+      // nop
+      skip_char(state, c);
+      break;
+    case '\0':
+      return next_token(state, pEOF);
+    default:
+      advance_char(state, c);
+      skipping = false;
+      break;
+    }
+  }
+
+  /* ... c d ..                */
+  /*      ^     state->current */
+  /*    ^       start          */
+  switch (c) {
+    case '\0': tok = next_token(state, pEOF);
+    ONE_CHAR_PATTERN('(', pLPAREN);
+    ONE_CHAR_PATTERN(')', pRPAREN);
+    ONE_CHAR_PATTERN(']', pRBRACKET);
+    ONE_CHAR_PATTERN('{', pLBRACE);
+    ONE_CHAR_PATTERN('}', pRBRACE);
+    ONE_CHAR_PATTERN(',', pCOMMA);
+    ONE_CHAR_PATTERN('|', pBAR);
+    ONE_CHAR_PATTERN('^', pHAT);
+    ONE_CHAR_PATTERN('&', pAMP);
+    ONE_CHAR_PATTERN('?', pQUESTION);
+    ONE_CHAR_PATTERN('/', tOPERATOR);
+    ONE_CHAR_PATTERN('~', tOPERATOR);
+    case '[':
+      tok = lex_bracket(state);
+      break;
+    case '-':
+      tok = lex_hyphen(state);
+      break;
+    case '+':
+      tok = lex_plus(state);
+      break;
+    case '*':
+      tok = lex_star(state);
+      break;
+    case '<':
+      tok = lex_lt(state);
+      break;
+    case '=':
+      tok = lex_eq(state);
+      break;
+    case '>':
+      tok = lex_gt(state);
+      break;
+    case '!':
+      tok = lex_bang(state);
+      break;
+    case '#':
+      if (state->first_token_of_line) {
+        tok = lex_comment(state, tLINECOMMENT);
+      } else {
+        tok = lex_comment(state, tCOMMENT);
+      }
+      break;
+    case ':':
+      tok = lex_colon(state);
+      break;
+    case '.':
+      tok = lex_dot(state);
+      break;
+    case '_':
+      tok = lex_underscore(state);
+      break;
+    case '$':
+      tok = lex_global(state);
+      break;
+    case '@':
+      tok = lex_ivar(state);
+      break;
+    case '"':
+      tok = lex_dqstring(state);
+      break;
+    case '\'':
+      tok = lex_sqstring(state);
+      break;
+    case '%':
+      tok = lex_percent(state);
+      break;
+    case '`':
+      tok = lex_backquote(state);
+      break;
+    default:
+      if (rb_isalpha(c) && rb_isupper(c)) {
+        tok = lex_ident(state, tUIDENT);
+      }
+      if (rb_isalpha(c) && rb_islower(c)) {
+        tok = lex_ident(state, tLIDENT);
+      }
+      if (rb_isdigit(c)) {
+        tok = lex_number(state);
+      }
+  }
+
+  if (tok.type == NullType) {
+    tok = next_token(state, ErrorToken);
+  }
+
+  return tok;
+}
+
+char *peek_token(lexstate *state, token tok) {
+  return RSTRING_PTR(state->string) + tok.range.start.byte_pos;
+}