c_lexer 2.5.1.2.0 → 2.5.3.0.0

data/ext/lexer/lexer.rl

@@ -5,16 +5,16 @@
 #include <stdint.h>
 #include <stdio.h>
 
-#include "stack.h"
-#include "stack_state.h"
+#include "stack_state/stack.h"
+#include "stack_state/stack_state.h"
 #include "lexer.h"
 
-#define INIT_LEXER_STATE(l, s) lexer_state *s; Data_Get_Struct(l, lexer_state, s);
+#define GET_LEXER(self) Data_Get_Struct(self, Lexer, lexer)
 #define STATIC_ENV_DECLARED(name) \
-  state->static_env != Qnil && RTEST(rb_funcall(state->static_env, rb_intern("declared?"), 1, name))
+  lexer->static_env != Qnil && RTEST(rb_funcall(lexer->static_env, rb_intern("declared?"), 1, name))
 
-#include "cmdarg.h"
-#include "cond.h"
+#include "stack_state/cmdarg.h"
+#include "stack_state/cond.h"
 
 #include "literal/methods.h"
 #include "emit_tables.h"
@@ -24,50 +24,50 @@
 
 static VALUE lexer_alloc(VALUE klass)
 {
-  lexer_state *state = xmalloc(sizeof(lexer_state));
-
-  state->cs = state->p = state->pe = 0;
-  state->paren_nest = 0;
-
-  state->cs_stack      = xmalloc(4 * sizeof(int));
-  state->cs_stack_top  = 0;
-  state->cs_stack_size = 4;
-
-  state->source_buffer = Qnil;
-  state->source        = Qnil;
-  state->source_pts    = Qnil;
-  state->token_queue   = Qnil;
-  state->static_env    = Qnil;
-  state->lambda_stack  = Qnil;
-  state->diagnostics   = Qnil;
-  state->tokens        = Qnil;
-  state->comments      = Qnil;
-  state->encoding      = Qnil;
-  state->escape        = Qnil;
-
-  ss_stack_init(&state->cond_stack);
-  ss_stack_init(&state->cmdarg_stack);
-  lit_stack_init(&state->literal_stack);
-
-  return Data_Wrap_Struct(klass, lexer_mark, lexer_dealloc, state);
+  Lexer *lexer = xmalloc(sizeof(Lexer));
+
+  lexer->cs = lexer->p = lexer->pe = 0;
+  lexer->paren_nest = 0;
+
+  lexer->cs_stack      = xmalloc(4 * sizeof(int));
+  lexer->cs_stack_top  = 0;
+  lexer->cs_stack_size = 4;
+
+  lexer->source_buffer = Qnil;
+  lexer->source        = Qnil;
+  lexer->source_pts    = Qnil;
+  lexer->token_queue   = Qnil;
+  lexer->static_env    = Qnil;
+  lexer->lambda_stack  = Qnil;
+  lexer->diagnostics   = Qnil;
+  lexer->tokens        = Qnil;
+  lexer->comments      = Qnil;
+  lexer->encoding      = Qnil;
+  lexer->escape        = Qnil;
+
+  ss_stack_init(&lexer->cond_stack);
+  ss_stack_init(&lexer->cmdarg_stack);
+  lit_stack_init(&lexer->literal_stack);
+
+  return Data_Wrap_Struct(klass, lexer_mark, lexer_dealloc, lexer);
 }
 
 static void lexer_mark(void *ptr)
 {
-  lexer_state *state = ptr;
-  rb_gc_mark(state->source_buffer);
-  rb_gc_mark(state->source);
-  rb_gc_mark(state->source_pts);
-  rb_gc_mark(state->token_queue);
-  rb_gc_mark(state->static_env);
-  rb_gc_mark(state->lambda_stack);
-  rb_gc_mark(state->diagnostics);
-  rb_gc_mark(state->tokens);
-  rb_gc_mark(state->comments);
-  rb_gc_mark(state->encoding);
-  rb_gc_mark(state->escape);
-
-  for (literal *lit = state->literal_stack.bottom; lit < state->literal_stack.top; lit++) {
+  Lexer *lexer = ptr;
+  rb_gc_mark(lexer->source_buffer);
+  rb_gc_mark(lexer->source);
+  rb_gc_mark(lexer->source_pts);
+  rb_gc_mark(lexer->token_queue);
+  rb_gc_mark(lexer->static_env);
+  rb_gc_mark(lexer->lambda_stack);
+  rb_gc_mark(lexer->diagnostics);
+  rb_gc_mark(lexer->tokens);
+  rb_gc_mark(lexer->comments);
+  rb_gc_mark(lexer->encoding);
+  rb_gc_mark(lexer->escape);
+
+  for (literal *lit = lexer->literal_stack.bottom; lit < lexer->literal_stack.top; lit++) {
     rb_gc_mark(lit->buffer);
     rb_gc_mark(lit->start_tok);
     rb_gc_mark(lit->start_delim);
@@ -78,25 +78,25 @@ static void lexer_mark(void *ptr)
 
 static void lexer_dealloc(void *ptr)
 {
-  lexer_state *state = ptr;
-  ss_stack_dealloc(&state->cond_stack);
-  ss_stack_dealloc(&state->cmdarg_stack);
-  lit_stack_dealloc(&state->literal_stack);
+  Lexer *lexer = ptr;
+  ss_stack_dealloc(&lexer->cond_stack);
+  ss_stack_dealloc(&lexer->cmdarg_stack);
+  lit_stack_dealloc(&lexer->literal_stack);
   xfree(ptr);
 }
 
 static VALUE lexer_initialize(VALUE self, VALUE version)
 {
-  INIT_LEXER_STATE(self, state);
+  Lexer* lexer = GET_LEXER(self);
 
-  state->version = NUM2INT(version);
+  lexer->version = NUM2INT(version);
 
   return lexer_reset(0, NULL, self);
 }
 
 static VALUE lexer_reset(int argc, VALUE *argv, VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
+  Lexer* lexer = GET_LEXER(self);
 
   VALUE reset_state;
   rb_scan_args(argc, argv, "01", &reset_state);
@@ -104,21 +104,21 @@ static VALUE lexer_reset(int argc, VALUE *argv, VALUE self)
     reset_state = Qtrue;
 
   if (RTEST(reset_state)) {
-    state->cs = lex_en_line_begin;
+    lexer->cs = lex_en_line_begin;
 
-    state->cond   = 0;
-    state->cmdarg = 0;
-    ss_stack_clear(&state->cond_stack);
-    ss_stack_clear(&state->cmdarg_stack);
+    lexer->cond   = 0;
+    lexer->cmdarg = 0;
+    ss_stack_clear(&lexer->cond_stack);
+    ss_stack_clear(&lexer->cmdarg_stack);
   }
 
-  state->force_utf32 = 0;
+  lexer->force_utf32 = 0;
 
-  state->source       = Qnil;
-  state->source_pts   = Qnil;
-  state->encoding     = Qnil;
+  lexer->source       = Qnil;
+  lexer->source_pts   = Qnil;
+  lexer->encoding     = Qnil;
 
-  state->p            = 0;
+  lexer->p            = 0;
   // @ts is a local variable
   // @te is a local variable
   // @act is a local variable
@@ -127,65 +127,65 @@ static VALUE lexer_reset(int argc, VALUE *argv, VALUE self)
   // @top is handled on prepush
 
   // Lexer state
-  state->token_queue  = rb_ary_new();
-  lit_stack_clear(&state->literal_stack);
+  lexer->token_queue  = rb_ary_new();
+  lit_stack_clear(&lexer->literal_stack);
 
-  state->eq_begin_s   = 0;
+  lexer->eq_begin_s   = 0;
   // @sharp_s is a local variable
 
-  state->newline_s    = 0;
+  lexer->newline_s    = 0;
 
   // @num_base is a local variable
   // @num_digits_s is a local variable
   // @num_suffix_s is a local variable
   // @num_xfrm is a local variable
 
-  state->escape_s     = 0;
-  state->escape       = Qnil;
+  lexer->escape_s     = 0;
+  lexer->escape       = Qnil;
 
-  state->herebody_s   = 0;
+  lexer->herebody_s   = 0;
 
-  state->paren_nest   = 0;
-  state->lambda_stack = rb_ary_new();
+  lexer->paren_nest   = 0;
+  lexer->lambda_stack = rb_ary_new();
 
-  state->dedent_level = -1;
+  lexer->dedent_level = -1;
 
   // @command_state is a local variable
 
-  state->in_kwarg     = 0;
+  lexer->in_kwarg     = 0;
 
-  state->cs_before_block_comment = lex_en_line_begin;
+  lexer->cs_before_block_comment = lex_en_line_begin;
 
   return self;
 }
 
 static VALUE lexer_set_source_buffer(VALUE self, VALUE buffer)
 {
-  INIT_LEXER_STATE(self, state);
+  Lexer* lexer = GET_LEXER(self);
 
-  state->source_buffer = buffer;
+  lexer->source_buffer = buffer;
 
   if (RTEST(buffer)) {
-    state->source = rb_funcall(buffer, rb_intern("source"), 0);
-    state->encoding = rb_obj_encoding(state->source);
+    lexer->source = rb_funcall(buffer, rb_intern("source"), 0);
+    lexer->encoding = rb_obj_encoding(lexer->source);
 
-    if (state->encoding == utf8_encoding) {
-      state->source_pts = rb_funcall(state->source, rb_intern("unpack"), 1, rb_str_new2("U*"));
+    if (lexer->encoding == utf8_encoding) {
+      lexer->source_pts = rb_funcall(lexer->source, rb_intern("unpack"), 1, rb_str_new2("U*"));
     } else {
-      state->source_pts = rb_funcall(state->source, rb_intern("unpack"), 1, rb_str_new2("C*"));
+      lexer->source_pts = rb_funcall(lexer->source, rb_intern("unpack"), 1, rb_str_new2("C*"));
     }
 
-    state->pe = RARRAY_LEN(state->source_pts) + 2; /* pretend there is a null at the end */
+    lexer->pe = RARRAY_LEN(lexer->source_pts) + 2; /* pretend there is a null at the end */
 
-    VALUE source_pt = rb_ary_entry(state->source_pts, 0);
+    VALUE source_pt = rb_ary_entry(lexer->source_pts, 0);
     if (source_pt != Qnil && NUM2INT(source_pt) == 0xfeff) {
-      state->p = 1;
+      lexer->p = 1;
     }
   } else {
-    state->source = Qnil;
-    state->source_pts = Qnil;
-    state->encoding = Qnil;
-    state->pe = 0;
+    lexer->source = Qnil;
+    lexer->source_pts = Qnil;
+    lexer->encoding = Qnil;
+    lexer->pe = 0;
   }
 
   return self;
@@ -193,9 +193,9 @@ static VALUE lexer_set_source_buffer(VALUE self, VALUE buffer)
 
 static VALUE lexer_get_state(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
+  Lexer* lexer = GET_LEXER(self);
 
-  switch (state->cs) {
+  switch (lexer->cs) {
   case lex_en_line_begin:    return ID2SYM(rb_intern("line_begin"));
   case lex_en_expr_dot:      return ID2SYM(rb_intern("expr_dot"));
   case lex_en_expr_fname:    return ID2SYM(rb_intern("expr_fname"));
@@ -220,42 +220,42 @@ static VALUE lexer_get_state(VALUE self)
 
 static VALUE lexer_set_state(VALUE self, VALUE state_sym)
 {
-  INIT_LEXER_STATE(self, state);
+  Lexer* lexer = GET_LEXER(self);
   const char *state_name = rb_id2name(SYM2ID(state_sym));
 
   if (strcmp(state_name, "line_begin") == 0)
-    state->cs = lex_en_line_begin;
+    lexer->cs = lex_en_line_begin;
   else if (strcmp(state_name, "expr_dot") == 0)
-    state->cs = lex_en_expr_dot;
+    lexer->cs = lex_en_expr_dot;
   else if (strcmp(state_name, "expr_fname") == 0)
-    state->cs = lex_en_expr_fname;
+    lexer->cs = lex_en_expr_fname;
   else if (strcmp(state_name, "expr_value") == 0)
-    state->cs = lex_en_expr_value;
+    lexer->cs = lex_en_expr_value;
   else if (strcmp(state_name, "expr_beg") == 0)
-    state->cs = lex_en_expr_beg;
+    lexer->cs = lex_en_expr_beg;
   else if (strcmp(state_name, "expr_mid") == 0)
-    state->cs = lex_en_expr_mid;
+    lexer->cs = lex_en_expr_mid;
   else if (strcmp(state_name, "expr_arg") == 0)
-    state->cs = lex_en_expr_arg;
+    lexer->cs = lex_en_expr_arg;
   else if (strcmp(state_name, "expr_cmdarg") == 0)
-    state->cs = lex_en_expr_cmdarg;
+    lexer->cs = lex_en_expr_cmdarg;
   else if (strcmp(state_name, "expr_end") == 0)
-    state->cs = lex_en_expr_end;
+    lexer->cs = lex_en_expr_end;
   else if (strcmp(state_name, "expr_endarg") == 0)
-    state->cs = lex_en_expr_endarg;
+    lexer->cs = lex_en_expr_endarg;
   else if (strcmp(state_name, "expr_endfn") == 0)
-    state->cs = lex_en_expr_endfn;
+    lexer->cs = lex_en_expr_endfn;
   else if (strcmp(state_name, "expr_labelarg") == 0)
-    state->cs = lex_en_expr_labelarg;
+    lexer->cs = lex_en_expr_labelarg;
 
   else if (strcmp(state_name, "interp_string") == 0)
-    state->cs = lex_en_interp_string;
+    lexer->cs = lex_en_interp_string;
   else if (strcmp(state_name, "interp_words") == 0)
-    state->cs = lex_en_interp_words;
+    lexer->cs = lex_en_interp_words;
   else if (strcmp(state_name, "plain_string") == 0)
-    state->cs = lex_en_plain_string;
+    lexer->cs = lex_en_plain_string;
   else if (strcmp(state_name, "plain_words") == 0)
-    state->cs = lex_en_plain_words;
+    lexer->cs = lex_en_plain_words;
   else
     rb_raise(rb_eArgError, "Invalid state: %s", state_name);
 
@@ -264,52 +264,52 @@ static VALUE lexer_set_state(VALUE self, VALUE state_sym)
 
 static VALUE lexer_push_cmdarg(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  ss_stack_push(&state->cmdarg_stack, state->cmdarg);
-  state->cmdarg = 0;
+  Lexer* lexer = GET_LEXER(self);
+  ss_stack_push(&lexer->cmdarg_stack, lexer->cmdarg);
+  lexer->cmdarg = 0;
   return Qnil;
 }
 
 static VALUE lexer_pop_cmdarg(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  state->cmdarg = ss_stack_pop(&state->cmdarg_stack);
+  Lexer* lexer = GET_LEXER(self);
+  lexer->cmdarg = ss_stack_pop(&lexer->cmdarg_stack);
   return Qnil;
 }
 
 static VALUE lexer_push_cond(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  ss_stack_push(&state->cond_stack, state->cond);
-  state->cond = 0;
+  Lexer* lexer = GET_LEXER(self);
+  ss_stack_push(&lexer->cond_stack, lexer->cond);
+  lexer->cond = 0;
   return Qnil;
 }
 
 static VALUE lexer_pop_cond(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  state->cond = ss_stack_pop(&state->cond_stack);
+  Lexer* lexer = GET_LEXER(self);
+  lexer->cond = ss_stack_pop(&lexer->cond_stack);
   return Qnil;
 }
 
 static VALUE lexer_get_in_kwarg(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  return state->in_kwarg ? Qtrue : Qfalse;
+  Lexer* lexer = GET_LEXER(self);
+  return lexer->in_kwarg ? Qtrue : Qfalse;
 }
 
 static VALUE lexer_set_in_kwarg(VALUE self, VALUE val)
 {
-  INIT_LEXER_STATE(self, state);
-  state->in_kwarg = RTEST(val) ? 1 : 0;
+  Lexer* lexer = GET_LEXER(self);
+  lexer->in_kwarg = RTEST(val) ? 1 : 0;
   return val;
 }
 
 static VALUE lexer_get_dedent_level(VALUE self)
 {
-  INIT_LEXER_STATE(self, state);
-  int result = state->dedent_level;
-  state->dedent_level = -1;
+  Lexer* lexer = GET_LEXER(self);
+  int result = lexer->dedent_level;
+  lexer->dedent_level = -1;
   if (result == -1)
     return Qnil;
   else
@@ -327,22 +327,22 @@ static VALUE lexer_advance(VALUE self)
   int num_base = 0;
   long p, pe, eof, ts = 0, te = 0, tm = 0, sharp_s = 0, heredoc_e = 0;
   long num_digits_s = 0, num_suffix_s = 0;
-  void (*num_xfrm)(lexer_state*, VALUE, long, long); /* numeric suffix-induced transformation */
-  lexer_state *state;
+  void (*num_xfrm)(Lexer*, VALUE, long, long); /* numeric suffix-induced transformation */
+  Lexer *lexer;
   int *stack;
   VALUE ident_tok = Qnil;
   long ident_ts = 0, ident_te = 0;
   long numeric_s = 0;
-  Data_Get_Struct(self, lexer_state, state);
+  Data_Get_Struct(self, Lexer, lexer);
 
-  if (RARRAY_LEN(state->token_queue) > 0)
-    return rb_ary_shift(state->token_queue);
+  if (RARRAY_LEN(lexer->token_queue) > 0)
+    return rb_ary_shift(lexer->token_queue);
 
-  cs = state->cs;
-  p = state->p;
-  pe = eof = state->pe;
-  stack = state->cs_stack;
-  top = state->cs_stack_top;
+  cs = lexer->cs;
+  p = lexer->p;
+  pe = eof = lexer->pe;
+  stack = lexer->cs_stack;
+  top = lexer->cs_stack_top;
 
   command_state = (cs == lex_en_expr_value || cs == lex_en_line_begin);
 
@@ -350,18 +350,18 @@ static VALUE lexer_advance(VALUE self)
     write exec;
   }%%
 
-  state->p = p;
-  state->cs = cs;
-  state->cs_stack_top = top;
+  lexer->p = p;
+  lexer->cs = cs;
+  lexer->cs_stack_top = top;
 
-  if (RARRAY_LEN(state->token_queue) > 0) {
-    return rb_ary_shift(state->token_queue);
+  if (RARRAY_LEN(lexer->token_queue) > 0) {
+    return rb_ary_shift(lexer->token_queue);
   } else if (cs == lex_error) {
-    VALUE info  = rb_ary_new3(2, rb_str_new2("$error"), range(state, p - 1, p));
+    VALUE info  = rb_ary_new3(2, rb_str_new2("$error"), range(lexer, p - 1, p));
     VALUE token = rb_ary_new3(2, Qfalse, info);
     return token;
   } else {
-    VALUE info  = rb_ary_new3(2, rb_str_new2("$eof"), range(state, eof - 2, eof - 2));
+    VALUE info  = rb_ary_new3(2, rb_str_new2("$eof"), range(lexer, eof - 2, eof - 2));
     VALUE token = rb_ary_new3(2, Qfalse, info);
     return token;
   }
@@ -372,62 +372,62 @@ static inline void force_encoding(VALUE str, VALUE enc)
   rb_enc_associate(str, rb_to_encoding(enc));
 }
 
-static void emit_token(lexer_state *state, VALUE type, VALUE value, long start, long end)
+static void emit_token(Lexer *lexer, VALUE type, VALUE value, long start, long end)
 {
-  VALUE info  = rb_ary_new3(2, value, range(state, start, end));
+  VALUE info  = rb_ary_new3(2, value, range(lexer, start, end));
   VALUE token = rb_ary_new3(2, type, info);
 
-  rb_ary_push(state->token_queue, token);
+  rb_ary_push(lexer->token_queue, token);
 
-  if (state->tokens != Qnil)
-    rb_ary_push(state->tokens, token);
+  if (lexer->tokens != Qnil)
+    rb_ary_push(lexer->tokens, token);
 }
 
-static void emit_comment(lexer_state *state, long start, long end)
+static void emit_comment(Lexer *lexer, long start, long end)
 {
   VALUE rng = Qnil;
 
-  if (state->tokens != Qnil) {
-    rng = range(state, start, end);
+  if (lexer->tokens != Qnil) {
+    rng = range(lexer, start, end);
 
-    VALUE info  = rb_ary_new3(2, tok(state, start, end), rng);
+    VALUE info  = rb_ary_new3(2, tok(lexer, start, end), rng);
     VALUE token = rb_ary_new3(2, tCOMMENT, info);
-    rb_ary_push(state->tokens, token);
+    rb_ary_push(lexer->tokens, token);
   }
 
-  if (state->comments != Qnil) {
+  if (lexer->comments != Qnil) {
     if (rng == Qnil)
-      rng = range(state, start, end);
+      rng = range(lexer, start, end);
     VALUE comment = rb_class_new_instance(1, &rng, comment_klass);
-    rb_ary_push(state->comments, comment);
+    rb_ary_push(lexer->comments, comment);
   }
 }
 
-static void emit_do(lexer_state *state, int do_block, long ts, long te)
+static void emit_do(Lexer *lexer, int do_block, long ts, long te)
 {
-  if (stack_state_active(&state->cond))
+  if (stack_state_active(&lexer->cond))
     emit(kDO_COND);
-  else if (stack_state_active(&state->cmdarg) || do_block)
+  else if (stack_state_active(&lexer->cmdarg) || do_block)
     emit(kDO_BLOCK);
   else
     emit(kDO);
 }
 
-static VALUE tok(lexer_state *state, long start, long end)
+static VALUE tok(Lexer *lexer, long start, long end)
 {
-  return rb_str_substr(state->source, start, end - start);
+  return rb_str_substr(lexer->source, start, end - start);
 }
 
-static VALUE range(lexer_state *state, long start, long end)
+static VALUE range(Lexer *lexer, long start, long end)
 {
   VALUE args[3];
-  args[0] = state->source_buffer;
+  args[0] = lexer->source_buffer;
   args[1] = INT2NUM(start);
   args[2] = INT2NUM(end);
   return rb_class_new_instance(3, args, range_klass);
 }
 
-static void diagnostic(lexer_state *state, VALUE type, VALUE reason,
+static void diagnostic(Lexer *lexer, VALUE type, VALUE reason,
                        VALUE arguments, VALUE loc, VALUE hilights)
 {
   VALUE args[5];
@@ -437,15 +437,15 @@ static void diagnostic(lexer_state *state, VALUE type, VALUE reason,
   args[3] = loc;
   args[4] = hilights;
   VALUE diagnostic = rb_class_new_instance(5, args, diagnostic_klass);
-  rb_funcall(state->diagnostics, rb_intern("process"), 1, diagnostic);
+  rb_funcall(lexer->diagnostics, rb_intern("process"), 1, diagnostic);
 }
 
-static int get_codepoint(lexer_state *state, long p)
+static int get_codepoint(Lexer *lexer, long p)
 {
-  if (p >= RARRAY_LEN(state->source_pts))
+  if (p >= RARRAY_LEN(lexer->source_pts))
     return 0;
   else
-    return NUM2INT(rb_ary_entry(state->source_pts, p));
+    return NUM2INT(rb_ary_entry(lexer->source_pts, p));
 }
 
 static int arg_or_cmdarg(int command_state)
@@ -558,53 +558,53 @@ static int find_8_or_9(VALUE str)
   return -1;
 }
 
-static void emit_int(lexer_state *state, VALUE val, long start, long end)
+static void emit_int(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tINTEGER, val, start, end);
+  emit_token(lexer, tINTEGER, val, start, end);
 }
 
-static void emit_rational(lexer_state *state, VALUE val, long start, long end)
+static void emit_rational(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tRATIONAL, rb_Rational1(val), start, end);
+  emit_token(lexer, tRATIONAL, rb_Rational1(val), start, end);
 }
 
-static void emit_complex(lexer_state *state, VALUE val, long start, long end)
+static void emit_complex(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tIMAGINARY, rb_Complex(Qzero, val), start, end);
+  emit_token(lexer, tIMAGINARY, rb_Complex(Qzero, val), start, end);
 }
 
-static void emit_complex_rational(lexer_state *state, VALUE val, long start, long end)
+static void emit_complex_rational(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tIMAGINARY, rb_Complex(Qzero, rb_Rational1(val)), start, end);
+  emit_token(lexer, tIMAGINARY, rb_Complex(Qzero, rb_Rational1(val)), start, end);
 }
 
-static void emit_float(lexer_state *state, VALUE val, long start, long end)
+static void emit_float(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tFLOAT, rb_Float(val), start, end);
+  emit_token(lexer, tFLOAT, rb_Float(val), start, end);
 }
 
-static void emit_complex_float(lexer_state *state, VALUE val, long start, long end)
+static void emit_complex_float(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tIMAGINARY, rb_Complex(Qzero, rb_Float(val)), start, end);
+  emit_token(lexer, tIMAGINARY, rb_Complex(Qzero, rb_Float(val)), start, end);
 }
 
-static void emit_int_followed_by_if(lexer_state *state, VALUE val, long start, long end)
+static void emit_int_followed_by_if(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tINTEGER, val, start, end);
+  emit_token(lexer, tINTEGER, val, start, end);
 }
 
-static void emit_int_followed_by_rescue(lexer_state *state, VALUE val, long start, long end)
+static void emit_int_followed_by_rescue(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tINTEGER, val, start, end);
+  emit_token(lexer, tINTEGER, val, start, end);
 }
 
-static void emit_float_followed_by_if(lexer_state *state, VALUE val, long start, long end)
+static void emit_float_followed_by_if(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tFLOAT, rb_Float(val), start, end);
+  emit_token(lexer, tFLOAT, rb_Float(val), start, end);
 }
-static void emit_float_followed_by_rescue(lexer_state *state, VALUE val, long start, long end)
+static void emit_float_followed_by_rescue(Lexer *lexer, VALUE val, long start, long end)
 {
-  emit_token(state, tFLOAT, rb_Float(val), start, end);
+  emit_token(lexer, tFLOAT, rb_Float(val), start, end);
 }
 
 static int next_state_for_literal(literal *lit) {
@@ -635,23 +635,23 @@ static int next_state_for_literal(literal *lit) {
   }
 }
 
-static int push_literal(lexer_state *state, VALUE str_type, VALUE delimiter,
+static int push_literal(Lexer *lexer, VALUE str_type, VALUE delimiter,
                         long str_s, long heredoc_e, int indent, int dedent_body,
                         int label_allowed)
 {
   literal lit;
-  literal_init(&lit, state, str_type, delimiter, str_s, heredoc_e, indent,
+  literal_init(&lit, lexer, str_type, delimiter, str_s, heredoc_e, indent,
                dedent_body, label_allowed);
-  lit_stack_push(&state->literal_stack, lit);
+  lit_stack_push(&lexer->literal_stack, lit);
 
   return next_state_for_literal(&lit);
 }
 
-static int pop_literal(lexer_state *state)
+static int pop_literal(Lexer *lexer)
 {
-  literal old_literal = lit_stack_pop(&state->literal_stack);
+  literal old_literal = lit_stack_pop(&lexer->literal_stack);
 
-  state->dedent_level = old_literal.dedent_level;
+  lexer->dedent_level = old_literal.dedent_level;
 
   if (old_literal.start_tok == tREGEXP_BEG) {
     return lex_en_regexp_modifiers;
@@ -986,20 +986,20 @@ void Init_lexer()
 
 %%{
   alphtype int;
-  getkey (get_codepoint(state, p));
+  getkey (get_codepoint(lexer, p));
 
   prepush {
     /* grow the state stack as needed */
-    if (state->cs_stack_top == state->cs_stack_size) {
-      int *new_stack = xmalloc(state->cs_stack_size * 2 * sizeof(int));
-      memcpy(new_stack, state->cs_stack, state->cs_stack_size * sizeof(int));
-      xfree(state->cs_stack);
-      stack = state->cs_stack = new_stack;
-      state->cs_stack_size = state->cs_stack_size * 2;
+    if (lexer->cs_stack_top == lexer->cs_stack_size) {
+      int *new_stack = xmalloc(lexer->cs_stack_size * 2 * sizeof(int));
+      memcpy(new_stack, lexer->cs_stack, lexer->cs_stack_size * sizeof(int));
+      xfree(lexer->cs_stack);
+      stack = lexer->cs_stack = new_stack;
+      lexer->cs_stack_size = lexer->cs_stack_size * 2;
     }
   }
 
-  action do_nl { state->newline_s = p; }
+  action do_nl { lexer->newline_s = p; }
 
   c_nl       = '\n' $ do_nl;
   c_space    = [ \t\r\f\v];
@@ -1094,30 +1094,30 @@ void Init_lexer()
   escaped_nl = "\\" c_nl;
 
   action unicode_points {
-    state->escape = rb_str_new2("");
+    lexer->escape = rb_str_new2("");
 
-    VALUE codepoints = tok(state, state->escape_s + 2, p - 1);
-    long codepoint_s = state->escape_s + 2;
+    VALUE codepoints = tok(lexer, lexer->escape_s + 2, p - 1);
+    long codepoint_s = lexer->escape_s + 2;
 
     VALUE regexp;
 
-    if (state->version < 24) {
+    if (lexer->version < 24) {
       if (str_start_with_p(codepoints, " ") || str_start_with_p(codepoints, "\t")) {
-        diagnostic(state, severity_error, invalid_unicode_escape, Qnil,
-                     range(state, state->escape_s + 2, state->escape_s + 3), empty_array);
+        diagnostic(lexer, severity_error, invalid_unicode_escape, Qnil,
+                     range(lexer, lexer->escape_s + 2, lexer->escape_s + 3), empty_array);
       }
 
       regexp = rb_reg_regcomp(rb_str_new2("[ \\t]{2}"));
       VALUE space_p = rb_funcall(codepoints, rb_intern("index"), 1, regexp);
 
       if (RTEST(space_p)) {
-        diagnostic(state, severity_error, invalid_unicode_escape, Qnil,
-                     range(state, codepoint_s + NUM2INT(space_p) + 1, codepoint_s + NUM2INT(space_p) + 1), empty_array);
+        diagnostic(lexer, severity_error, invalid_unicode_escape, Qnil,
+                     range(lexer, codepoint_s + NUM2INT(space_p) + 1, codepoint_s + NUM2INT(space_p) + 1), empty_array);
       }
 
       if (str_end_with_p(codepoints, " ") || str_end_with_p(codepoints, "\t")) {
-        diagnostic(state, severity_error, invalid_unicode_escape, Qnil,
-                     range(state, p - 1, p), empty_array);
+        diagnostic(lexer, severity_error, invalid_unicode_escape, Qnil,
+                     range(lexer, p - 1, p), empty_array);
       }
     }
 
@@ -1136,93 +1136,93 @@ void Init_lexer()
       } else {
         VALUE codepoint = rb_str_to_inum(codepoint_str, 16, 0);
         if (NUM2INT(codepoint) >= 0x110000) {
-          diagnostic(state, severity_error, unicode_point_too_large, Qnil,
-                     range(state, codepoint_s, codepoint_s + RSTRING_LEN(codepoint_str)), empty_array);
+          diagnostic(lexer, severity_error, unicode_point_too_large, Qnil,
+                     range(lexer, codepoint_s, codepoint_s + RSTRING_LEN(codepoint_str)), empty_array);
           break;
         }
 
         codepoint = rb_funcall(codepoint, rb_intern("chr"), 1, utf8_encoding);
-        state->escape = rb_str_plus(state->escape, codepoint);
+        lexer->escape = rb_str_plus(lexer->escape, codepoint);
         codepoint_s += RSTRING_LEN(codepoint_str);
       }
     }
   }
 
   action unescape_char {
-    char c = NUM2INT(rb_ary_entry(state->source_pts, p - 1));
-    state->escape = unescape_char(c);
+    char c = NUM2INT(rb_ary_entry(lexer->source_pts, p - 1));
+    lexer->escape = unescape_char(c);
 
-    if (state->escape == Qnil) {
-      VALUE codepoint = rb_funcall(state->source_buffer, rb_intern("slice"), 1, INT2NUM(p - 1));
-      state->escape = codepoint;
-      force_encoding(codepoint, state->encoding);
+    if (lexer->escape == Qnil) {
+      VALUE codepoint = rb_funcall(lexer->source_buffer, rb_intern("slice"), 1, INT2NUM(p - 1));
+      lexer->escape = codepoint;
+      force_encoding(codepoint, lexer->encoding);
     }
   }
 
   action invalid_complex_escape {
-    diagnostic(state, fatal, invalid_escape, Qnil, range(state, ts, te),
+    diagnostic(lexer, fatal, invalid_escape, Qnil, range(lexer, ts, te),
                empty_array);
   }
 
   action slash_c_char {
-    char c = *RSTRING_PTR(state->escape) & 0x9f;
-    state->escape = rb_str_new(&c, 1);
-    force_encoding(state->escape, state->encoding);
+    char c = *RSTRING_PTR(lexer->escape) & 0x9f;
+    lexer->escape = rb_str_new(&c, 1);
+    force_encoding(lexer->escape, lexer->encoding);
   }
 
   action slash_m_char {
-    char c = *RSTRING_PTR(state->escape) | 0x80;
-    state->escape = rb_str_new(&c, 1);
-    force_encoding(state->escape, state->encoding);
+    char c = *RSTRING_PTR(lexer->escape) | 0x80;
+    lexer->escape = rb_str_new(&c, 1);
+    force_encoding(lexer->escape, lexer->encoding);
   }
 
   maybe_escaped_char = (
         '\\' c_any      %unescape_char
-    | ( c_any - [\\] )  % { state->escape = rb_str_substr(state->source, p - 1, 1); }
+    | ( c_any - [\\] )  % { lexer->escape = rb_str_substr(lexer->source, p - 1, 1); }
   );
 
   maybe_escaped_ctrl_char = (
         '\\' c_any      %unescape_char %slash_c_char
-    |   '?'             % { state->escape = rb_str_new2("\x7f"); }
-    | ( c_any - [\\?] ) % { state->escape = rb_str_substr(state->source, p - 1, 1); } %slash_c_char
+    |   '?'             % { lexer->escape = rb_str_new2("\x7f"); }
+    | ( c_any - [\\?] ) % { lexer->escape = rb_str_substr(lexer->source, p - 1, 1); } %slash_c_char
   );
 
   escape = (
       [0-7]{1,3} % {
-        VALUE token = tok(state, state->escape_s, p);
+        VALUE token = tok(lexer, lexer->escape_s, p);
         char c = NUM2INT(rb_str_to_inum(token, 8, 0));
         c = c % 0x100;
-        state->escape = rb_str_new(&c, 1);
-        force_encoding(state->escape, state->encoding);
+        lexer->escape = rb_str_new(&c, 1);
+        force_encoding(lexer->escape, lexer->encoding);
       }
 
     | 'x' xdigit{1,2} % {
-        VALUE token = tok(state, state->escape_s + 1, p);
+        VALUE token = tok(lexer, lexer->escape_s + 1, p);
         char c = NUM2INT(rb_str_to_inum(token, 16, 0));
-        state->escape = rb_str_new(&c, 1);
-        force_encoding(state->escape, state->encoding);
+        lexer->escape = rb_str_new(&c, 1);
+        force_encoding(lexer->escape, lexer->encoding);
       }
 
     | 'x' ( c_any - xdigit )
       % {
-        diagnostic(state, fatal, invalid_hex_escape, Qnil,
-                   range(state, state->escape_s - 1, p + 2), empty_array);
+        diagnostic(lexer, fatal, invalid_hex_escape, Qnil,
+                   range(lexer, lexer->escape_s - 1, p + 2), empty_array);
       }
 
     | 'u' xdigit{4} % {
-        VALUE token = tok(state, state->escape_s + 1, p);
+        VALUE token = tok(lexer, lexer->escape_s + 1, p);
         int i = NUM2INT(rb_str_to_inum(token, 16, 0));
-        state->escape = rb_enc_uint_chr(i, rb_to_encoding(utf8_encoding));
+        lexer->escape = rb_enc_uint_chr(i, rb_to_encoding(utf8_encoding));
       }
 
     | 'u' xdigit{0,3} % {
-        diagnostic(state, fatal, invalid_unicode_escape, Qnil,
-                   range(state, state->escape_s - 1, p), empty_array);
+        diagnostic(lexer, fatal, invalid_unicode_escape, Qnil,
+                   range(lexer, lexer->escape_s - 1, p), empty_array);
       }
 
     | 'u{' ( c_any - xdigit - [ \t}] )* '}' % {
-        diagnostic(state, fatal, invalid_unicode_escape, Qnil,
-                   range(state, state->escape_s - 1, p), empty_array);
+        diagnostic(lexer, fatal, invalid_unicode_escape, Qnil,
+                   range(lexer, lexer->escape_s - 1, p), empty_array);
       }
 
     | 'u{' [ \t]* ( xdigit{1,6} [ \t]+ )*
@@ -1235,8 +1235,8 @@ void Init_lexer()
           | ( c_any - [ \t}] )* c_eof
           | xdigit{7,}
         ) % {
-          diagnostic(state, fatal, invalid_unicode_escape, Qnil,
-                   range(state, p - 1, p), empty_array);
+          diagnostic(lexer, fatal, invalid_unicode_escape, Qnil,
+                   range(lexer, p - 1, p), empty_array);
         }
       )
 
@@ -1259,42 +1259,42 @@ void Init_lexer()
     | ( c_any - [0-7xuCMc] ) %unescape_char
 
     | c_eof % {
-        diagnostic(state, fatal, escape_eof, Qnil, range(state, p - 1, p),
+        diagnostic(lexer, fatal, escape_eof, Qnil, range(lexer, p - 1, p),
                    empty_array);
       }
   );
 
   e_bs = '\\' % {
-    state->escape_s = p;
-    state->escape = Qnil;
+    lexer->escape_s = p;
+    lexer->escape = Qnil;
   };
 
   e_heredoc_nl = c_nl % {
-    if (state->herebody_s) {
-      p = state->herebody_s;
-      state->herebody_s = 0;
+    if (lexer->herebody_s) {
+      p = lexer->herebody_s;
+      lexer->herebody_s = 0;
     }
   };
 
   action extend_string {
-    VALUE string = tok(state, ts, te);
+    VALUE string = tok(lexer, ts, te);
     VALUE lookahead = Qnil;
 
-    if (state->version >= 22 && !stack_state_active(&state->cond)) {
-      lookahead = tok(state, te, te + 2);
+    if (lexer->version >= 22 && !stack_state_active(&lexer->cond)) {
+      lookahead = tok(lexer, te, te + 2);
     }
 
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
 
     if (!current_literal->heredoc_e &&
          literal_nest_and_try_closing(current_literal, string, ts, te, lookahead)) {
-      VALUE token = array_last(state->token_queue);
+      VALUE token = array_last(lexer->token_queue);
       if (rb_ary_entry(token, 0) == tLABEL_END) {
         p += 1;
-        pop_literal(state);
+        pop_literal(lexer);
         fnext expr_labelarg;
       } else {
-        fnext *pop_literal(state);
+        fnext *pop_literal(lexer);
       }
 
       fbreak;
@@ -1304,93 +1304,93 @@ void Init_lexer()
   }
 
   action extend_string_escaped {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
-    VALUE escaped_char = rb_str_substr(state->source, state->escape_s, 1);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
+    VALUE escaped_char = rb_str_substr(lexer->source, lexer->escape_s, 1);
 
     if (literal_munge_escape_p(current_literal, escaped_char)) {
       if (literal_regexp_p(current_literal) && is_regexp_metachar(escaped_char)) {
-        literal_extend_string(current_literal, tok(state, ts, te), ts, te);
+        literal_extend_string(current_literal, tok(lexer, ts, te), ts, te);
       } else {
         literal_extend_string(current_literal, escaped_char, ts, te);
       }
     } else {
       if (literal_regexp_p(current_literal)) {
-        VALUE token = tok(state, ts, te);
+        VALUE token = tok(lexer, ts, te);
         rb_funcall(token, rb_intern("gsub!"), 2, escaped_newline, blank_string);
         literal_extend_string(current_literal, token, ts, te);
       } else if (literal_heredoc_p(current_literal) && newline_char_p(escaped_char)) {
         if (literal_squiggly_heredoc_p(current_literal)) {
-          literal_extend_string(current_literal, tok(state, ts, te), ts, te);
+          literal_extend_string(current_literal, tok(lexer, ts, te), ts, te);
         } else {
-          VALUE token = tok(state, ts, te);
+          VALUE token = tok(lexer, ts, te);
           rb_funcall(token, rb_intern("gsub!"), 2, escaped_newline, blank_string);
           literal_extend_string(current_literal, token, ts, te);
         }
-      } else if (state->escape == Qnil) {
-        literal_extend_string(current_literal, tok(state, ts, te), ts, te);
+      } else if (lexer->escape == Qnil) {
+        literal_extend_string(current_literal, tok(lexer, ts, te), ts, te);
       } else {
-        literal_extend_string(current_literal, state->escape, ts, te);
+        literal_extend_string(current_literal, lexer->escape, ts, te);
       }
     }
   }
 
   action extend_string_eol {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     long str_s = current_literal->str_s;
 
     if (te == pe) {
-      diagnostic(state, fatal, string_eof, Qnil,
-                 range(state, str_s, str_s + 1), empty_array);
+      diagnostic(lexer, fatal, string_eof, Qnil,
+                 range(lexer, str_s, str_s + 1), empty_array);
     }
 
     if (literal_heredoc_p(current_literal)) {
-      VALUE line = tok(state, state->herebody_s, ts);
+      VALUE line = tok(lexer, lexer->herebody_s, ts);
       rb_funcall(line, rb_intern("gsub!"), 2, crs_to_eol, blank_string);
 
-      if (state->version >= 18 && state->version <= 20) {
+      if (lexer->version >= 18 && lexer->version <= 20) {
         rb_funcall(line, rb_intern("gsub!"), 2, cr_then_anything_to_eol, blank_string);
       }
 
-      if (literal_nest_and_try_closing(current_literal, line, state->herebody_s, ts, Qnil)) {
-        state->herebody_s = te;
+      if (literal_nest_and_try_closing(current_literal, line, lexer->herebody_s, ts, Qnil)) {
+        lexer->herebody_s = te;
         p = current_literal->heredoc_e - 1;
-        fnext *pop_literal(state); fbreak;
+        fnext *pop_literal(lexer); fbreak;
       } else {
         literal_infer_indent_level(current_literal, line);
-        state->herebody_s = te;
+        lexer->herebody_s = te;
       }
     } else {
-      if (literal_nest_and_try_closing(current_literal, tok(state, ts, te), ts, te, Qnil)) {
-        fnext *pop_literal(state); fbreak;
+      if (literal_nest_and_try_closing(current_literal, tok(lexer, ts, te), ts, te, Qnil)) {
+        fnext *pop_literal(lexer); fbreak;
       }
 
-      if (state->herebody_s) {
-        p = state->herebody_s - 1;
-        state->herebody_s = 0;
+      if (lexer->herebody_s) {
+        p = lexer->herebody_s - 1;
+        lexer->herebody_s = 0;
       }
     }
 
-    if (literal_words_p(current_literal) && !eof_codepoint(get_codepoint(state, p))) {
+    if (literal_words_p(current_literal) && !eof_codepoint(get_codepoint(lexer, p))) {
       literal_extend_space(current_literal, ts, te);
     } else {
-      literal_extend_string(current_literal, tok(state, ts, te), ts, te);
+      literal_extend_string(current_literal, tok(lexer, ts, te), ts, te);
       literal_flush_string(current_literal);
     }
   }
 
   action extend_string_space {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     literal_extend_space(current_literal, ts, te);
   }
 
   interp_var = '#' ( global_var | class_var_v | instance_var_v );
 
   action extend_interp_var {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     literal_flush_string(current_literal);
     literal_extend_content(current_literal);
 
-    emit_token(state, tSTRING_DVAR, Qnil, ts, ts + 1);
+    emit_token(lexer, tSTRING_DVAR, Qnil, ts, ts + 1);
 
     p = ts;
     fcall expr_variable;
@@ -1399,34 +1399,34 @@ void Init_lexer()
   interp_code = '#{';
 
   e_lbrace = '{' % {
-    stack_state_push(&state->cond, 0);
-    stack_state_push(&state->cmdarg, 0);
+    stack_state_push(&lexer->cond, 0);
+    stack_state_push(&lexer->cmdarg, 0);
 
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     if (current_literal != NULL) {
       literal_start_interp_brace(current_literal);
     }
   };
 
   e_rbrace = '}' % {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     if (current_literal != NULL) {
       if (literal_end_interp_brace_and_try_closing(current_literal)) {
-        if (state->version == 18 || state->version == 19) {
-          emit_token(state, tRCURLY, rb_str_new2("}"), p - 1, p);
-          if (state->version < 24) {
-            stack_state_lexpop(&state->cond);
-            stack_state_lexpop(&state->cmdarg);
+        if (lexer->version == 18 || lexer->version == 19) {
+          emit_token(lexer, tRCURLY, rb_str_new2("}"), p - 1, p);
+          if (lexer->version < 24) {
+            stack_state_lexpop(&lexer->cond);
+            stack_state_lexpop(&lexer->cmdarg);
           } else {
-            stack_state_pop(&state->cond);
-            stack_state_pop(&state->cmdarg);
+            stack_state_pop(&lexer->cond);
+            stack_state_pop(&lexer->cmdarg);
           }
         } else {
-          emit_token(state, tSTRING_DEND, rb_str_new2("}"), p - 1, p);
+          emit_token(lexer, tSTRING_DEND, rb_str_new2("}"), p - 1, p);
         }
 
         if (current_literal->herebody_s) {
-          state->herebody_s = current_literal->herebody_s;
+          lexer->herebody_s = current_literal->herebody_s;
         }
 
         fhold;
@@ -1437,15 +1437,15 @@ void Init_lexer()
   };
 
   action extend_interp_code {
-    literal *current_literal = lit_stack_top(&state->literal_stack);
+    literal *current_literal = lit_stack_top(&lexer->literal_stack);
     literal_flush_string(current_literal);
     literal_extend_content(current_literal);
 
-    emit_token(state, tSTRING_DBEG, rb_str_new2("#{"), ts, te);
+    emit_token(lexer, tSTRING_DBEG, rb_str_new2("#{"), ts, te);
 
     if (current_literal->heredoc_e) {
-      current_literal->herebody_s = state->herebody_s;
-      state->herebody_s = 0;
+      current_literal->herebody_s = lexer->herebody_s;
+      lexer->herebody_s = 0;
     }
 
     literal_start_interp_brace(current_literal);
@@ -1513,13 +1513,13 @@ void Init_lexer()
   regexp_modifiers := |*
       [A-Za-z]+
       => {
-        VALUE unknown_options = find_unknown_options(tok(state, ts, te));
+        VALUE unknown_options = find_unknown_options(tok(lexer, ts, te));
 
         if (unknown_options != Qnil) {
           VALUE hash = rb_hash_new();
           rb_hash_aset(hash, ID2SYM(rb_intern("options")), unknown_options);
-          diagnostic(state, severity_error, regexp_options, hash,
-                     range(state, ts, te), empty_array);
+          diagnostic(lexer, severity_error, regexp_options, hash,
+                     range(lexer, ts, te), empty_array);
         }
 
         emit(tREGEXP_OPT);
@@ -1529,7 +1529,7 @@ void Init_lexer()
 
       any
       => {
-        emit_token(state, tREGEXP_OPT, tok(state, ts, te - 1), ts, te - 1);
+        emit_token(lexer, tREGEXP_OPT, tok(lexer, ts, te - 1), ts, te - 1);
         fhold;
         fgoto expr_end;
       };
@@ -1542,7 +1542,7 @@ void Init_lexer()
 
   w_comment =
       '#'     %{ sharp_s = p - 1; }
-      c_line* %{ emit_comment(state, sharp_s, p == pe ? p - 2 : p); }
+      c_line* %{ emit_comment(lexer, sharp_s, p == pe ? p - 2 : p); }
     ;
 
   w_space_comment =
@@ -1583,22 +1583,22 @@ void Init_lexer()
   ;
 
   e_lbrack = '[' % {
-    stack_state_push(&state->cond, 0);
-    stack_state_push(&state->cmdarg, 0);
+    stack_state_push(&lexer->cond, 0);
+    stack_state_push(&lexer->cmdarg, 0);
   };
 
   e_lparen = '(' % {
-    stack_state_push(&state->cond, 0);
-    stack_state_push(&state->cmdarg, 0);
-    state->paren_nest += 1;
+    stack_state_push(&lexer->cond, 0);
+    stack_state_push(&lexer->cmdarg, 0);
+    lexer->paren_nest += 1;
   };
 
   e_rparen = ')' % {
-    state->paren_nest -= 1;
+    lexer->paren_nest -= 1;
   };
 
   action local_ident {
-    VALUE str = tok(state, ts, te);
+    VALUE str = tok(lexer, ts, te);
     emit(tIDENTIFIER);
 
     if (STATIC_ENV_DECLARED(str)) {
@@ -1610,11 +1610,11 @@ void Init_lexer()
 
   expr_variable := |*
       global_var => {
-        VALUE str = tok(state, ts, te);
+        VALUE str = tok(lexer, ts, te);
 
         if (is_nthref(str)) {
-          VALUE integer = rb_str_to_inum(tok(state, ts + 1, te), 10, 0);
-          emit_token(state, tNTH_REF, integer, ts, te);
+          VALUE integer = rb_str_to_inum(tok(lexer, ts + 1, te), 10, 0);
+          emit_token(lexer, tNTH_REF, integer, ts, te);
         } else if (is_backref(str)) {
           emit(tBACK_REF);
         } else {
@@ -1625,12 +1625,12 @@ void Init_lexer()
       };
 
       class_var_v => {
-        VALUE str = tok(state, ts, te);
+        VALUE str = tok(lexer, ts, te);
 
         if (bad_cvar_name(str)) {
           VALUE hash = rb_hash_new();
           rb_hash_aset(hash, ID2SYM(rb_intern("name")), str);
-          diagnostic(state, severity_error, cvar_name, hash, range(state, ts, te), empty_array);
+          diagnostic(lexer, severity_error, cvar_name, hash, range(lexer, ts, te), empty_array);
         }
 
         emit(tCVAR);
@@ -1638,12 +1638,12 @@ void Init_lexer()
       };
 
       instance_var_v => {
-        VALUE str = tok(state, ts, te);
+        VALUE str = tok(lexer, ts, te);
 
         if (bad_ivar_name(str)) {
           VALUE hash = rb_hash_new();
           rb_hash_aset(hash, ID2SYM(rb_intern("name")), str);
-          diagnostic(state, severity_error, ivar_name, hash, range(state, ts, te), empty_array);
+          diagnostic(lexer, severity_error, ivar_name, hash, range(lexer, ts, te), empty_array);
         }
 
         emit(tIVAR);
@@ -1653,7 +1653,7 @@ void Init_lexer()
 
   expr_fname := |*
       keyword
-      => { emit_table_KEYWORDS_BEGIN(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS_BEGIN(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_endfn; fbreak; };
 
       constant        => { emit(tCONSTANT); fnext expr_endfn; fbreak; };
@@ -1665,7 +1665,7 @@ void Init_lexer()
       operator_fname      |
       operator_arithmetic |
       operator_rest
-      => { emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_endfn; fbreak; };
 
       '::' => { fhold; fhold; fgoto expr_end; };
@@ -1674,13 +1674,13 @@ void Init_lexer()
 
       '%s' c_any
       => {
-        if (state->version == 23) {
-          VALUE type = rb_str_substr(state->source, ts, te - ts - 1);
-          VALUE delimiter = rb_str_substr(state->source, te - 1, 1);
+        if (lexer->version == 23) {
+          VALUE type = rb_str_substr(lexer->source, ts, te - ts - 1);
+          VALUE delimiter = rb_str_substr(lexer->source, te - 1, 1);
           if (delimiter == Qnil)
             delimiter = blank_string;
 
-          fgoto *push_literal(state, type, delimiter, ts, 0, 0, 0, 0);
+          fgoto *push_literal(lexer, type, delimiter, ts, 0, 0, 0, 0);
         } else {
           p = ts - 1;
           fgoto expr_end;
@@ -1696,7 +1696,7 @@ void Init_lexer()
 
   expr_endfn := |*
       label ( any - ':' ) => {
-        emit_token(state, tLABEL, tok(state, ts, te - 2), ts, te - 1);
+        emit_token(lexer, tLABEL, tok(lexer, ts, te - 2), ts, te - 1);
         fhold; fnext expr_labelarg; fbreak;
       };
 
@@ -1713,13 +1713,13 @@ void Init_lexer()
       call_or_var => { emit(tIDENTIFIER); fnext *arg_or_cmdarg(command_state); fbreak; };
 
       bareword ambiguous_fid_suffix
-      => { emit_token(state, tFID, tok(state, ts, tm), ts, tm);
+      => { emit_token(lexer, tFID, tok(lexer, ts, tm), ts, tm);
            fnext *arg_or_cmdarg(command_state); p = tm - 1; fbreak; };
 
       operator_fname      |
       operator_arithmetic |
       operator_rest
-      => { emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_arg; fbreak; };
 
       w_any;
@@ -1732,11 +1732,11 @@ void Init_lexer()
 
   expr_arg := |*
       w_space+ e_lparen => {
-        if (state->version == 18) {
-          emit_token(state, tLPAREN2, rb_str_new2("("), te - 1, te);
+        if (lexer->version == 18) {
+          emit_token(lexer, tLPAREN2, rb_str_new2("("), te - 1, te);
           fnext expr_value; fbreak;
         } else {
-          emit_token(state, tLPAREN_ARG, rb_str_new2("("), te - 1, te);
+          emit_token(lexer, tLPAREN_ARG, rb_str_new2("("), te - 1, te);
           fnext expr_beg; fbreak;
         }
       };
@@ -1744,17 +1744,17 @@ void Init_lexer()
       e_lparen => { emit(tLPAREN2); fnext expr_beg; fbreak; };
 
       w_space+ e_lbrack => {
-        emit_token(state, tLBRACK, rb_str_new2("["), te - 1, te);
+        emit_token(lexer, tLBRACK, rb_str_new2("["), te - 1, te);
         fnext expr_beg; fbreak;
       };
 
       w_space* e_lbrace => {
-        VALUE val = array_last(state->lambda_stack);
-        if (val != Qnil && NUM2INT(val) == state->paren_nest) {
-          rb_ary_pop(state->lambda_stack);
-          emit_token(state, tLAMBEG, rb_str_new2("{"), te - 1, te);
+        VALUE val = array_last(lexer->lambda_stack);
+        if (val != Qnil && NUM2INT(val) == lexer->paren_nest) {
+          rb_ary_pop(lexer->lambda_stack);
+          emit_token(lexer, tLAMBEG, rb_str_new2("{"), te - 1, te);
         } else {
-          emit_token(state, tLCURLY, rb_str_new2("{"), te - 1, te);
+          emit_token(lexer, tLCURLY, rb_str_new2("{"), te - 1, te);
         }
         fnext expr_value; fbreak;
       };
@@ -1765,9 +1765,9 @@ void Init_lexer()
 
       w_space+ %{ tm = p; }
       ( [%/] ( c_any - c_space_nl - '=' ) | '<<' ) => {
-        if (NUM2INT(rb_ary_entry(state->source_pts, tm)) == '/') {
-          diagnostic(state, warning, ambiguous_literal, Qnil,
-                     range(state, tm, tm + 1), empty_array);
+        if (NUM2INT(rb_ary_entry(lexer->source_pts, tm)) == '/') {
+          diagnostic(lexer, warning, ambiguous_literal, Qnil,
+                     range(lexer, tm, tm + 1), empty_array);
         }
 
         p = tm - 1;
@@ -1776,9 +1776,9 @@ void Init_lexer()
 
       w_space+ %{ tm = p; } ( '+' | '-' | '*' | '&' | '**' ) => {
         VALUE hash = rb_hash_new();
-        VALUE str  = tok(state, tm, te);
+        VALUE str  = tok(lexer, tm, te);
         rb_hash_aset(hash, prefix, str);
-        diagnostic(state, warning, ambiguous_prefix, hash, range(state, tm, te),
+        diagnostic(lexer, warning, ambiguous_prefix, hash, range(lexer, tm, te),
                    empty_array);
 
         p = tm - 1;
@@ -1817,8 +1817,8 @@ void Init_lexer()
   expr_cmdarg := |*
       w_space+ e_lparen
       => {
-        emit_token(state, tLPAREN_ARG, rb_str_new2("("), te - 1, te);
-        if (state->version == 18) {
+        emit_token(lexer, tLPAREN_ARG, rb_str_new2("("), te - 1, te);
+        if (lexer->version == 18) {
           fnext expr_value; fbreak;
         } else {
           fnext expr_beg; fbreak;
@@ -1827,10 +1827,10 @@ void Init_lexer()
 
       w_space* 'do'
       => {
-        if (stack_state_active(&state->cond)) {
-          emit_token(state, kDO_COND, rb_str_new2("do"), te - 2, te);
+        if (stack_state_active(&lexer->cond)) {
+          emit_token(lexer, kDO_COND, rb_str_new2("do"), te - 2, te);
         } else {
-          emit_token(state, kDO, rb_str_new2("do"), te - 2, te);
+          emit_token(lexer, kDO, rb_str_new2("do"), te - 2, te);
         }
         fnext expr_value; fbreak;
       };
@@ -1846,17 +1846,17 @@ void Init_lexer()
 
   expr_endarg := |*
       e_lbrace => {
-        VALUE val = array_last(state->lambda_stack);
-        if (val != Qnil && NUM2INT(val) == state->paren_nest) {
-          rb_ary_pop(state->lambda_stack);
-          emit_token(state, tLAMBEG, rb_str_new2("{"), te - 1, te);
+        VALUE val = array_last(lexer->lambda_stack);
+        if (val != Qnil && NUM2INT(val) == lexer->paren_nest) {
+          rb_ary_pop(lexer->lambda_stack);
+          emit_token(lexer, tLAMBEG, rb_str_new2("{"), te - 1, te);
         } else {
-          emit_token(state, tLBRACE_ARG, rb_str_new2("{"), te - 1, te);
+          emit_token(lexer, tLBRACE_ARG, rb_str_new2("{"), te - 1, te);
         }
         fnext expr_value; fbreak;
       };
 
-      'do' => { emit_do(state, 1, ts, te); fnext expr_value; fbreak; };
+      'do' => { emit_do(lexer, 1, ts, te); fnext expr_value; fbreak; };
 
       w_space_comment;
 
@@ -1868,7 +1868,7 @@ void Init_lexer()
 
   expr_mid := |*
       keyword_modifier
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_beg; fbreak; };
 
       bareword => { p = ts - 1; fgoto expr_beg; };
@@ -1884,38 +1884,38 @@ void Init_lexer()
 
   expr_beg := |*
       [+\-] w_any* [0-9] => {
-        emit_token(state, tUNARY_NUM, tok(state, ts, ts + 1), ts, ts + 1);
+        emit_token(lexer, tUNARY_NUM, tok(lexer, ts, ts + 1), ts, ts + 1);
         fhold; fnext expr_end; fbreak;
       };
 
       '*' => { emit(tSTAR); fbreak; };
 
       '/' c_any => {
-        VALUE delimiter = rb_str_substr(state->source, ts, 1);
-        fhold; fgoto *push_literal(state, delimiter, delimiter, ts, 0, 0, 0, 0);
+        VALUE delimiter = rb_str_substr(lexer->source, ts, 1);
+        fhold; fgoto *push_literal(lexer, delimiter, delimiter, ts, 0, 0, 0, 0);
       };
 
       '%' ( any - [A-Za-z] ) => {
-        VALUE type = rb_str_substr(state->source, ts, 1);
-        VALUE delimiter = rb_str_substr(state->source, te - 1, 1);
+        VALUE type = rb_str_substr(lexer->source, ts, 1);
+        VALUE delimiter = rb_str_substr(lexer->source, te - 1, 1);
         if (delimiter == Qnil)
           delimiter = blank_string;
 
-        fgoto *push_literal(state, type, delimiter, ts, 0, 0, 0, 0);
+        fgoto *push_literal(lexer, type, delimiter, ts, 0, 0, 0, 0);
       };
 
       '%' [A-Za-z]+ c_any => {
-        VALUE type = rb_str_substr(state->source, ts, te - ts - 1);
-        VALUE delimiter = rb_str_substr(state->source, te - 1, 1);
+        VALUE type = rb_str_substr(lexer->source, ts, te - ts - 1);
+        VALUE delimiter = rb_str_substr(lexer->source, te - 1, 1);
         if (delimiter == Qnil)
           delimiter = blank_string;
 
-        fgoto *push_literal(state, type, delimiter, ts, 0, 0, 0, 0);
+        fgoto *push_literal(lexer, type, delimiter, ts, 0, 0, 0, 0);
       };
 
       '%' c_eof => {
-        diagnostic(state, fatal, string_eof, Qnil,
-                   range(state, ts, ts + 1), empty_array);
+        diagnostic(lexer, fatal, string_eof, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
       };
 
       '<<' [~\-]?
@@ -1923,9 +1923,9 @@ void Init_lexer()
         | "'" ( any - "'" )* "'"
         | "`" ( any - "`" )* "`"
         | bareword ) % { heredoc_e = p; }
-        c_line* c_nl % { if (!state->herebody_s) state->herebody_s = p; } => {
+        c_line* c_nl % { if (!lexer->herebody_s) lexer->herebody_s = p; } => {
 
-        VALUE heredoc = tok(state, ts, heredoc_e);
+        VALUE heredoc = tok(lexer, ts, heredoc_e);
         VALUE type;
         char *cp = RSTRING_PTR(heredoc);
         int indent = 0, dedent_body = 0;
@@ -1959,100 +1959,100 @@ void Init_lexer()
           type = rb_str_new2("<<\"");
         }
 
-        VALUE delimiter = tok(state, rng_s, rng_e);
+        VALUE delimiter = tok(lexer, rng_s, rng_e);
 
-        if (state->version >= 24) {
+        if (lexer->version >= 24) {
           if (NUM2INT(rb_funcall(delimiter, rb_intern("count"), 1, newline)) > 0) {
             if (str_end_with_p(delimiter, "\n")) {
-              diagnostic(state, warning, heredoc_id_ends_with_nl, Qnil,
-                   range(state, ts, ts + 1), empty_array);
+              diagnostic(lexer, warning, heredoc_id_ends_with_nl, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
 
               delimiter = rb_funcall(delimiter, rb_intern("rstrip"), 0);
             } else {
-              diagnostic(state, fatal, heredoc_id_has_newline, Qnil,
-                   range(state, ts, ts + 1), empty_array);
+              diagnostic(lexer, fatal, heredoc_id_has_newline, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
             }
           }
         }
 
-        if (dedent_body && state->version >= 18 && state->version <= 22) {
-          emit_token(state, tLSHFT, rb_str_new2("<<"), ts, ts + 2);
+        if (dedent_body && lexer->version >= 18 && lexer->version <= 22) {
+          emit_token(lexer, tLSHFT, rb_str_new2("<<"), ts, ts + 2);
           p = ts + 1;
           fnext expr_beg; fbreak;
         } else {
-          fnext *push_literal(state, type, delimiter, ts, heredoc_e, indent,
+          fnext *push_literal(lexer, type, delimiter, ts, heredoc_e, indent,
                               dedent_body, 0);
-          p = state->herebody_s - 1;
+          p = lexer->herebody_s - 1;
         }
       };
 
       ':' ('&&' | '||') => {
         fhold; fhold;
-        emit_token(state, tSYMBEG, tok(state, ts, ts + 1), ts, ts + 1);
+        emit_token(lexer, tSYMBEG, tok(lexer, ts, ts + 1), ts, ts + 1);
         fgoto expr_fname;
       };
 
       ':' ['"] => { /* ' */
-        VALUE type = tok(state, ts, te);
-        VALUE delimiter = tok(state, te - 1, te);
-        fgoto *push_literal(state, type, delimiter, ts, 0, 0, 0, 0);
+        VALUE type = tok(lexer, ts, te);
+        VALUE delimiter = tok(lexer, te - 1, te);
+        fgoto *push_literal(lexer, type, delimiter, ts, 0, 0, 0, 0);
       };
 
       ':' [!~] '@'
       => {
-        emit_token(state, tSYMBOL, tok(state, ts + 1, ts + 2), ts, te);
+        emit_token(lexer, tSYMBOL, tok(lexer, ts + 1, ts + 2), ts, te);
         fnext expr_end; fbreak;
       };
 
       ':' bareword ambiguous_symbol_suffix => {
-        emit_token(state, tSYMBOL, tok(state, ts + 1, tm), ts, tm);
+        emit_token(lexer, tSYMBOL, tok(lexer, ts + 1, tm), ts, tm);
         p = tm - 1;
         fnext expr_end; fbreak;
       };
 
       ':' ( bareword | global_var | class_var | instance_var |
             operator_fname | operator_arithmetic | operator_rest ) => {
-        emit_token(state, tSYMBOL, tok(state, ts + 1, te), ts, te);
+        emit_token(lexer, tSYMBOL, tok(lexer, ts + 1, te), ts, te);
         fnext expr_end; fbreak;
       };
 
       '?' ( e_bs ( escape - ( '\u{' (xdigit+ [ \t]+)+ xdigit+ '}' ))
-          | (c_any - c_space_nl - e_bs) % { state->escape = Qnil; }
+          | (c_any - c_space_nl - e_bs) % { lexer->escape = Qnil; }
           ) => {
-        VALUE value = state->escape;
+        VALUE value = lexer->escape;
         if (value == Qnil)
-          value = tok(state, ts + 1, te);
+          value = tok(lexer, ts + 1, te);
 
-        if (state->version == 18)
-          emit_token(state, tINTEGER, rb_funcall(value, rb_intern("getbyte"), 1, INT2NUM(0)), ts, te);
+        if (lexer->version == 18)
+          emit_token(lexer, tINTEGER, rb_funcall(value, rb_intern("getbyte"), 1, INT2NUM(0)), ts, te);
         else
-          emit_token(state, tCHARACTER, value, ts, te);
+          emit_token(lexer, tCHARACTER, value, ts, te);
 
         fnext expr_end; fbreak;
       };
 
       '?' c_space_nl => {
-        VALUE escape = escape_char(rb_str_subseq(state->source, ts + 1, 1));
+        VALUE escape = escape_char(rb_str_subseq(lexer->source, ts + 1, 1));
         VALUE hash = rb_hash_new();
         rb_hash_aset(hash, ID2SYM(rb_intern("escape")), escape);
-        diagnostic(state, warning, invalid_escape_use, hash,
-                   range(state, ts, te), empty_array);
+        diagnostic(lexer, warning, invalid_escape_use, hash,
+                   range(lexer, ts, te), empty_array);
 
         p = ts - 1;
         fgoto expr_end;
       };
 
       '?' c_eof => {
-        diagnostic(state, fatal, incomplete_escape, Qnil,
-                   range(state, ts, ts + 1), empty_array);
+        diagnostic(lexer, fatal, incomplete_escape, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
       };
 
       '?' [A-Za-z_] bareword => { p = ts - 1; fgoto expr_end; };
 
       e_lbrace => {
-        VALUE val = array_last(state->lambda_stack);
-        if (val != Qnil && NUM2INT(val) == state->paren_nest) {
-          rb_ary_pop(state->lambda_stack);
+        VALUE val = array_last(lexer->lambda_stack);
+        if (val != Qnil && NUM2INT(val) == lexer->paren_nest) {
+          rb_ary_pop(lexer->lambda_stack);
           emit(tLAMBEG);
         } else {
           emit(tLBRACE);
@@ -2061,37 +2061,37 @@ void Init_lexer()
       };
 
       e_lbrack => {
-        emit_token(state, tLBRACK, tok(state, ts, te), ts, te);
+        emit_token(lexer, tLBRACK, tok(lexer, ts, te), ts, te);
         fbreak;
       };
 
       e_lparen => {
-        emit_token(state, tLPAREN, tok(state, ts, te), ts, te);
+        emit_token(lexer, tLPAREN, tok(lexer, ts, te), ts, te);
         fbreak;
       };
 
       punctuation_begin
-      => { emit_table_PUNCTUATION_BEGIN(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION_BEGIN(lexer, tok(lexer, ts, te), ts, te);
            fbreak; };
 
       'rescue' %{ tm = p; } '=>'? => {
-        emit_token(state, kRESCUE, tok(state, ts, tm), ts, tm);
+        emit_token(lexer, kRESCUE, tok(lexer, ts, tm), ts, tm);
         p = tm - 1;
         fnext expr_mid; fbreak;
       };
 
       keyword_modifier
-      => { emit_table_KEYWORDS_BEGIN(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS_BEGIN(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_value; fbreak; };
 
       label ( any - ':' )
       => {
         fhold;
 
-        if (state->version == 18) {
-          VALUE ident = tok(state, ts, te - 2);
+        if (lexer->version == 18) {
+          VALUE ident = tok(lexer, ts, te - 2);
 
-          emit_token(state, is_capitalized(ident) ? tCONSTANT : tIDENTIFIER,
+          emit_token(lexer, is_capitalized(ident) ? tCONSTANT : tIDENTIFIER,
                ident, ts, te - 2);
           fhold;
 
@@ -2101,7 +2101,7 @@ void Init_lexer()
             fnext *arg_or_cmdarg(command_state);
           }
         } else {
-          emit_token(state, tLABEL, tok(state, ts, te - 2), ts, te - 1);
+          emit_token(lexer, tLABEL, tok(lexer, ts, te - 2), ts, te - 1);
           fnext expr_labelarg;
         }
 
@@ -2113,13 +2113,13 @@ void Init_lexer()
       call_or_var => local_ident;
 
       (call_or_var - keyword)
-        % { ident_tok = tok(state, ts, te); ident_ts = ts; ident_te = te; }
+        % { ident_tok = tok(lexer, ts, te); ident_ts = ts; ident_te = te; }
       w_space+ '('
       => {
-        emit_token(state, tIDENTIFIER, ident_tok, ident_ts, ident_te);
+        emit_token(lexer, tIDENTIFIER, ident_tok, ident_ts, ident_te);
         p = ident_te - 1;
 
-        if (STATIC_ENV_DECLARED(ident_tok) && state->version < 25) {
+        if (STATIC_ENV_DECLARED(ident_tok) && lexer->version < 25) {
           fnext expr_endfn;
         } else {
           fnext expr_cmdarg;
@@ -2132,7 +2132,7 @@ void Init_lexer()
 
       e_heredoc_nl '=begin' ( c_space | c_nl_zlen ) => {
         p = ts - 1;
-        state->cs_before_block_comment = state->cs;
+        lexer->cs_before_block_comment = lexer->cs;
         fgoto line_begin;
       };
 
@@ -2149,7 +2149,7 @@ void Init_lexer()
       w_space_comment;
 
       w_newline => {
-        if (state->in_kwarg) {
+        if (lexer->in_kwarg) {
           fhold; fgoto expr_end;
         } else {
           fgoto line_begin;
@@ -2165,8 +2165,8 @@ void Init_lexer()
       label (any - ':') => { p = ts - 1; fgoto expr_end; };
 
       ['"] => { /* ' */
-        VALUE type = tok(state, ts, te);
-        fgoto *push_literal(state, type, type, ts, 0, 0, 0, 0);
+        VALUE type = tok(lexer, ts, te);
+        fgoto *push_literal(lexer, type, type, ts, 0, 0, 0, 0);
       };
 
       w_space_comment;
@@ -2180,15 +2180,15 @@ void Init_lexer()
 
   expr_end := |*
       '->' => {
-        emit_token(state, tLAMBDA, tok(state, ts, ts + 2), ts, ts + 2);
-        rb_ary_push(state->lambda_stack, INT2NUM(state->paren_nest));
+        emit_token(lexer, tLAMBDA, tok(lexer, ts, ts + 2), ts, ts + 2);
+        rb_ary_push(lexer->lambda_stack, INT2NUM(lexer->paren_nest));
         fnext expr_endfn; fbreak;
       };
 
       e_lbrace => {
-        VALUE val = array_last(state->lambda_stack);
-        if (val != Qnil && NUM2INT(val) == state->paren_nest) {
-          rb_ary_pop(state->lambda_stack);
+        VALUE val = array_last(lexer->lambda_stack);
+        if (val != Qnil && NUM2INT(val) == lexer->paren_nest) {
+          rb_ary_pop(lexer->lambda_stack);
           emit(tLAMBEG);
         } else {
           emit(tLCURLY);
@@ -2197,43 +2197,43 @@ void Init_lexer()
       };
 
       'do' => {
-        VALUE val = array_last(state->lambda_stack);
-        if (val != Qnil && NUM2INT(val) == state->paren_nest) {
-          rb_ary_pop(state->lambda_stack);
+        VALUE val = array_last(lexer->lambda_stack);
+        if (val != Qnil && NUM2INT(val) == lexer->paren_nest) {
+          rb_ary_pop(lexer->lambda_stack);
           emit(kDO_LAMBDA);
         } else {
-          emit_do(state, 0, ts, te);
+          emit_do(lexer, 0, ts, te);
         }
         fnext expr_value; fbreak;
       };
 
       keyword_with_fname
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_fname; fbreak; };
 
       'class' w_any* '<<'
-      => { emit_token(state, kCLASS, rb_str_new2("class"), ts, ts + 5);
-           emit_token(state, tLSHFT, rb_str_new2("<<"),    te - 2, te);
+      => { emit_token(lexer, kCLASS, rb_str_new2("class"), ts, ts + 5);
+           emit_token(lexer, tLSHFT, rb_str_new2("<<"),    te - 2, te);
            fnext expr_value; fbreak; };
 
       keyword_modifier
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_beg; fbreak; };
 
       keyword_with_value
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_value; fbreak; };
 
       keyword_with_mid
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_mid; fbreak; };
 
       keyword_with_arg
       => {
-        VALUE keyword = tok(state, ts, te);
-        emit_table_KEYWORDS(state, keyword, ts, te);
+        VALUE keyword = tok(lexer, ts, te);
+        emit_table_KEYWORDS(lexer, keyword, ts, te);
 
-        if (state->version == 18 && strcmp(RSTRING_PTR(keyword), "not") == 0) {
+        if (lexer->version == 18 && strcmp(RSTRING_PTR(keyword), "not") == 0) {
           fnext expr_beg; fbreak;
         } else {
           fnext expr_arg; fbreak;
@@ -2241,8 +2241,8 @@ void Init_lexer()
       };
 
       '__ENCODING__' => {
-        if (state->version == 18) {
-          VALUE str = tok(state, ts, te);
+        if (lexer->version == 18) {
+          VALUE str = tok(lexer, ts, te);
           emit(tIDENTIFIER);
 
           if (STATIC_ENV_DECLARED(str)) {
@@ -2257,7 +2257,7 @@ void Init_lexer()
       };
 
       keyword_with_end
-      => { emit_table_KEYWORDS(state, tok(state, ts, te), ts, te);
+      => { emit_table_KEYWORDS(lexer, tok(lexer, ts, te), ts, te);
            fbreak; };
 
       ( '0' [Xx] %{ num_base = 16; num_digits_s = p; } int_hex
@@ -2269,30 +2269,30 @@ void Init_lexer()
       ) %{ num_suffix_s = p; } int_suffix
       => {
         int invalid_idx;
-        VALUE digits = tok(state, num_digits_s, num_suffix_s);
+        VALUE digits = tok(lexer, num_digits_s, num_suffix_s);
 
-        if (NUM2INT(rb_ary_entry(state->source_pts, num_suffix_s - 1)) == '_') {
+        if (NUM2INT(rb_ary_entry(lexer->source_pts, num_suffix_s - 1)) == '_') {
           VALUE hash = rb_hash_new();
           rb_hash_aset(hash, character, rb_str_new2("_"));
-          diagnostic(state, severity_error, trailing_in_number, hash,
-                     range(state, te - 1, te), empty_array);
-        } else if (RSTRING_LEN(digits) == 0 && num_base == 8 && state->version == 18) {
+          diagnostic(lexer, severity_error, trailing_in_number, hash,
+                     range(lexer, te - 1, te), empty_array);
+        } else if (RSTRING_LEN(digits) == 0 && num_base == 8 && lexer->version == 18) {
           digits = rb_str_new2("0");
         } else if (RSTRING_LEN(digits) == 0) {
-          diagnostic(state, severity_error, empty_numeric, Qnil,
-                     range(state, ts, te), empty_array);
+          diagnostic(lexer, severity_error, empty_numeric, Qnil,
+                     range(lexer, ts, te), empty_array);
         } else if (num_base == 8 && (invalid_idx = find_8_or_9(digits)) != -1) {
           long invalid_s = num_digits_s + invalid_idx;
-          diagnostic(state, severity_error, invalid_octal, Qnil,
-                     range(state, invalid_s, invalid_s + 1), empty_array);
+          diagnostic(lexer, severity_error, invalid_octal, Qnil,
+                     range(lexer, invalid_s, invalid_s + 1), empty_array);
         }
 
         VALUE integer = rb_str_to_inum(digits, num_base, 0);
-        if (state->version >= 18 && state->version <= 20) {
-          emit_token(state, tINTEGER, integer, numeric_s, num_suffix_s);
+        if (lexer->version >= 18 && lexer->version <= 20) {
+          emit_token(lexer, tINTEGER, integer, numeric_s, num_suffix_s);
           p = num_suffix_s - 1;
         } else {
-          num_xfrm(state, integer, numeric_s, te);
+          num_xfrm(lexer, integer, numeric_s, te);
         }
 
         fbreak;
@@ -2300,34 +2300,34 @@ void Init_lexer()
 
       flo_frac flo_pow?
       => {
-        diagnostic(state, severity_error, no_dot_digit_literal, Qnil,
-                   range(state, ts, te), empty_array);
+        diagnostic(lexer, severity_error, no_dot_digit_literal, Qnil,
+                   range(lexer, ts, te), empty_array);
       };
 
       flo_int [eE]
       => {
-        if (state->version >= 18 && state->version <= 20) {
+        if (lexer->version >= 18 && lexer->version <= 20) {
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, character, tok(state, te - 1, te));
-          diagnostic(state, severity_error, trailing_in_number, hash,
-                     range(state, te - 1, te), empty_array);
+          rb_hash_aset(hash, character, tok(lexer, te - 1, te));
+          diagnostic(lexer, severity_error, trailing_in_number, hash,
+                     range(lexer, te - 1, te), empty_array);
         } else {
-          VALUE integer = rb_str_to_inum(tok(state, ts, te - 1), 10, 0);
-          emit_token(state, tINTEGER, integer, ts, te - 1);
+          VALUE integer = rb_str_to_inum(tok(lexer, ts, te - 1), 10, 0);
+          emit_token(lexer, tINTEGER, integer, ts, te - 1);
           fhold; fbreak;
         }
       };
 
       flo_int flo_frac [eE]
       => {
-        if (state->version >= 18 && state->version <= 20) {
+        if (lexer->version >= 18 && lexer->version <= 20) {
           VALUE hash = rb_hash_new();
-          rb_hash_aset(hash, character, tok(state, te - 1, te));
-          diagnostic(state, severity_error, trailing_in_number, hash,
-                     range(state, te - 1, te), empty_array);
+          rb_hash_aset(hash, character, tok(lexer, te - 1, te));
+          diagnostic(lexer, severity_error, trailing_in_number, hash,
+                     range(lexer, te - 1, te), empty_array);
         } else {
-          VALUE fval = rb_funcall(tok(state, ts, te - 1), rb_intern("to_f"), 0);
-          emit_token(state, tFLOAT, fval, ts, te - 1);
+          VALUE fval = rb_funcall(tok(lexer, ts, te - 1), rb_intern("to_f"), 0);
+          emit_token(lexer, tFLOAT, fval, ts, te - 1);
           fhold; fbreak;
         }
       };
@@ -2337,28 +2337,28 @@ void Init_lexer()
       | flo_frac          %{ num_suffix_s = p; } flo_suffix
       )
       => {
-        VALUE digits = tok(state, ts, num_suffix_s);
+        VALUE digits = tok(lexer, ts, num_suffix_s);
 
-        if (state->version >= 18 && state->version <= 20) {
+        if (lexer->version >= 18 && lexer->version <= 20) {
           VALUE fval = rb_Float(digits);
-          emit_token(state, tFLOAT, fval, ts, num_suffix_s);
+          emit_token(lexer, tFLOAT, fval, ts, num_suffix_s);
           p = num_suffix_s - 1;
         } else {
-          num_xfrm(state, digits, ts, te);
+          num_xfrm(lexer, digits, ts, te);
         }
         fbreak;
       };
 
       '`' | ['"] => { /* ' */
-        VALUE type = tok(state, ts, te);
-        VALUE delimiter = tok(state, te - 1, te);
-        fgoto *push_literal(state, type, delimiter, ts, 0, 0, 0, 1);
+        VALUE type = tok(lexer, ts, te);
+        VALUE delimiter = tok(lexer, te - 1, te);
+        fgoto *push_literal(lexer, type, delimiter, ts, 0, 0, 0, 1);
       };
 
       constant => { emit(tCONSTANT); fnext *arg_or_cmdarg(command_state); fbreak; };
 
       constant ambiguous_const_suffix => {
-        emit_token(state, tCONSTANT, tok(state, ts, tm), ts, tm);
+        emit_token(lexer, tCONSTANT, tok(lexer, ts, tm), ts, tm);
         p = tm - 1;
         fbreak;
       };
@@ -2367,7 +2367,7 @@ void Init_lexer()
       => { p = ts - 1; fcall expr_variable; };
 
       '.' | '&.' | '::'
-      => { emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_dot; fbreak; };
 
       call_or_var => local_ident;
@@ -2376,39 +2376,39 @@ void Init_lexer()
         if (tm == te) {
           emit(tFID);
         } else {
-          emit_token(state, tIDENTIFIER, tok(state, ts, tm), ts, tm);
+          emit_token(lexer, tIDENTIFIER, tok(lexer, ts, tm), ts, tm);
           p = tm - 1;
         }
         fnext expr_arg; fbreak;
       };
 
-      '*' => {
-        emit(tSTAR2);
+      '*' | '=>' => {
+        emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
         fgoto expr_value;
       };
 
       ( operator_arithmetic | operator_rest ) - ( '|' | '~' | '!' | '*' )
       => {
-        emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+        emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
         fnext expr_value; fbreak;
       };
 
       ( e_lparen | '|' | '~' | '!' )
-      => { emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_beg; fbreak; };
 
       e_rbrace => {
         emit(tRCURLY);
 
-        if (state->version < 24) {
-          stack_state_lexpop(&state->cond);
-          stack_state_lexpop(&state->cmdarg);
+        if (lexer->version < 24) {
+          stack_state_lexpop(&lexer->cond);
+          stack_state_lexpop(&lexer->cmdarg);
         } else {
-          stack_state_pop(&state->cond);
-          stack_state_pop(&state->cmdarg);
+          stack_state_pop(&lexer->cond);
+          stack_state_pop(&lexer->cmdarg);
         }
 
-        if (state->version >= 25) {
+        if (lexer->version >= 25) {
           fnext expr_end;
         } else {
           fnext expr_endarg;
@@ -2420,12 +2420,12 @@ void Init_lexer()
       e_rparen => {
         emit(tRPAREN);
 
-        if (state->version < 24) {
-          stack_state_lexpop(&state->cond);
-          stack_state_lexpop(&state->cmdarg);
+        if (lexer->version < 24) {
+          stack_state_lexpop(&lexer->cond);
+          stack_state_lexpop(&lexer->cmdarg);
         } else {
-          stack_state_pop(&state->cond);
-          stack_state_pop(&state->cmdarg);
+          stack_state_pop(&lexer->cond);
+          stack_state_pop(&lexer->cmdarg);
         }
 
         fbreak;
@@ -2434,15 +2434,15 @@ void Init_lexer()
       ']' => {
         emit(tRBRACK);
 
-        if (state->version < 24) {
-          stack_state_lexpop(&state->cond);
-          stack_state_lexpop(&state->cmdarg);
+        if (lexer->version < 24) {
+          stack_state_lexpop(&lexer->cond);
+          stack_state_lexpop(&lexer->cmdarg);
         } else {
-          stack_state_pop(&state->cond);
-          stack_state_pop(&state->cmdarg);
+          stack_state_pop(&lexer->cond);
+          stack_state_pop(&lexer->cmdarg);
         }
 
-        if (state->version >= 25) {
+        if (lexer->version >= 25) {
           fnext expr_end;
         } else {
           fnext expr_endarg;
@@ -2452,7 +2452,7 @@ void Init_lexer()
       };
 
       operator_arithmetic '='
-      => { emit_token(state, tOP_ASGN, tok(state, ts, te - 1), ts, te);
+      => { emit_token(lexer, tOP_ASGN, tok(lexer, ts, te - 1), ts, te);
            fnext expr_beg; fbreak; };
 
       '?' => { emit(tEH); fnext expr_value; fbreak; };
@@ -2460,7 +2460,7 @@ void Init_lexer()
       e_lbrack => { emit(tLBRACK2); fnext expr_beg; fbreak; };
 
       punctuation_end
-      => { emit_table_PUNCTUATION(state, tok(state, ts, te), ts, te);
+      => { emit_table_PUNCTUATION(lexer, tok(lexer, ts, te), ts, te);
            fnext expr_beg; fbreak; };
 
       w_space_comment;
@@ -2470,17 +2470,17 @@ void Init_lexer()
       ';' => { emit(tSEMI); fnext expr_value; fbreak; };
 
       '\\' c_line {
-        diagnostic(state, severity_error, bare_backslash, Qnil,
-                   range(state, ts, ts + 1), empty_array);
+        diagnostic(lexer, severity_error, bare_backslash, Qnil,
+                   range(lexer, ts, ts + 1), empty_array);
         fhold;
       };
 
       c_any
       => {
         VALUE hash = rb_hash_new();
-        VALUE str  = rb_str_inspect(tok(state, ts, te));
+        VALUE str  = rb_str_inspect(tok(lexer, ts, te));
         rb_hash_aset(hash, character, rb_str_substr(str, 1, NUM2INT(rb_str_length(str)) - 2));
-        diagnostic(state, fatal, unexpected, hash, range(state, ts, te), empty_array);
+        diagnostic(lexer, fatal, unexpected, hash, range(lexer, ts, te), empty_array);
       };
 
       c_eof => do_eof;
@@ -2490,22 +2490,22 @@ void Init_lexer()
       c_space* %{ tm = p; } ('.' | '&.') => { p = tm - 1; fgoto expr_end; };
 
       any => {
-        emit_token(state, tNL, Qnil, state->newline_s, state->newline_s + 1);
+        emit_token(lexer, tNL, Qnil, lexer->newline_s, lexer->newline_s + 1);
         fhold; fnext line_begin; fbreak;
       };
   *|;
 
   line_comment := |*
       '=end' c_line* c_nl_zlen => {
-        emit_comment(state, state->eq_begin_s, te);
-        fgoto *state->cs_before_block_comment;
+        emit_comment(lexer, lexer->eq_begin_s, te);
+        fgoto *lexer->cs_before_block_comment;
       };
 
       c_line* c_nl;
 
       c_line* zlen => {
-        diagnostic(state, fatal, embedded_document, Qnil,
-                   range(state, state->eq_begin_s, state->eq_begin_s + 6),
+        diagnostic(lexer, fatal, embedded_document, Qnil,
+                   range(lexer, lexer->eq_begin_s, lexer->eq_begin_s + 6),
                    empty_array);
       };
   *|;
@@ -2514,7 +2514,7 @@ void Init_lexer()
       w_any;
 
       '=begin' ( c_space | c_nl_zlen ) => {
-        state->eq_begin_s = ts;
+        lexer->eq_begin_s = ts;
         fgoto line_comment;
       };