gloss 0.0.1

data/ext/gloss/src/gloss.cr

@@ -0,0 +1,31 @@
+require "./lib/cr_ruby"
+require "./cr_ast"
+require "./rb_ast"
+require "./parser"
+
+def parse_buffer(self : CrRuby::VALUE, buffer : CrRuby::VALUE)
+  plain_buffer = CrRuby.rb_str_to_str(buffer)
+  c_buffer = CrRuby.rb_string_value_cstr(pointerof(plain_buffer))
+  crystal_buffer = String.new(c_buffer)
+
+  output = parse_buffer(crystal_buffer)
+
+  CrRuby.rb_str_new_cstr(output)
+end
+
+def parse_buffer(buffer : String)
+  begin
+    tree = Gloss::Parser.parse buffer
+    tree.to_rb.to_json
+  rescue e : Crystal::SyntaxException
+    pp e.backtrace
+    e.to_s
+  end
+end
+
+fun init = Init_gloss
+  GC.init
+  LibCrystalMain.__crystal_main(0, Pointer(Pointer(UInt8)).null)
+  gloss = CrRuby.rb_define_module("Gloss");
+  CrRuby.rb_define_singleton_method(gloss, "parse_buffer", ->parse_buffer(CrRuby::VALUE, CrRuby::VALUE), 1);
+end

data/ext/gloss/src/lexer.cr

@@ -0,0 +1,1186 @@
+require "compiler/crystal/syntax/*"
+
+module Crystal
+  class Lexer
+    def next_token
+      reset_token
+
+      # Skip comments
+      while current_char == '#'
+        start = current_pos
+
+        # Check #<loc:...> pragma comment
+        if next_char_no_column_increment == '<' &&
+           next_char_no_column_increment == 'l' &&
+           next_char_no_column_increment == 'o' &&
+           next_char_no_column_increment == 'c' &&
+           next_char_no_column_increment == ':'
+          next_char_no_column_increment
+          consume_loc_pragma
+          start = current_pos
+        else
+          if @doc_enabled
+            consume_doc
+          elsif @comments_enabled
+            return consume_comment(start)
+          else
+            skip_comment
+          end
+        end
+      end
+
+      start = current_pos
+
+      # Fix location by `macro_expansion_pragmas`.
+      if me_pragmas = macro_expansion_pragmas
+        # It might happen that the current "start" already passed some
+        # location pragmas, so we must consume all of those. For example
+        # if one does `@{{...}}` inside a macro, "start" will be one less
+        # number than the pragma, and after consuming "@..." if we don't
+        # consume the pragmas generated by `{{...}}` we'll be in an
+        # incorrect location.
+        while me_pragmas.first_key?.try &.<=(start)
+          _, pragmas = me_pragmas.shift
+          pragmas.each &.run_pragma self
+        end
+      end
+
+      reset_regex_flags = true
+
+      case current_char
+      when '\0'
+        @token.type = :EOF
+      when ' ', '\t'
+        consume_whitespace
+        reset_regex_flags = false
+      when '\\'
+        if next_char == '\n'
+          incr_line_number
+          @token.passed_backslash_newline = true
+          consume_whitespace
+          reset_regex_flags = false
+        else
+          unknown_token
+        end
+      when '\n'
+        @token.type = :NEWLINE
+        next_char
+        incr_line_number
+        reset_regex_flags = false
+        consume_newlines
+      when '\r'
+        if next_char == '\n'
+          next_char
+          @token.type = :NEWLINE
+          incr_line_number
+          consume_newlines
+        else
+          raise "expected '\\n' after '\\r'"
+        end
+      when '='
+        case next_char
+        when '='
+          case next_char
+          when '='
+            next_char :"==="
+          else
+            @token.type = :"=="
+          end
+        when '>'
+          next_char :"=>"
+        when '~'
+          next_char :"=~"
+        else
+          @token.type = :"="
+        end
+      when '!'
+        case next_char
+        when '='
+          next_char :"!="
+        when '~'
+          next_char :"!~"
+        else
+          @token.type = :"!"
+        end
+      when '<'
+        case next_char
+        when '='
+          case next_char
+          when '>'
+            next_char :"<=>"
+          else
+            @token.type = :"<="
+          end
+        when '<'
+          case next_char
+          when '='
+            next_char :"<<="
+          when '-'
+            has_single_quote = false
+            found_closing_single_quote = false
+
+            char = next_char
+            start_here = current_pos
+
+            if char == '\''
+              has_single_quote = true
+              char = next_char
+              start_here = current_pos
+            end
+
+            unless ident_part?(char)
+              raise "heredoc identifier starts with invalid character"
+            end
+
+            end_here = 0
+
+            while true
+              char = next_char
+              case
+              when char == '\r'
+                if peek_next_char == '\n'
+                  end_here = current_pos
+                  next_char
+                  break
+                else
+                  raise "expecting '\\n' after '\\r'"
+                end
+              when char == '\n'
+                end_here = current_pos
+                break
+              when ident_part?(char)
+                # ok
+              when char == '\0'
+                raise "Unexpected EOF on heredoc identifier"
+              else
+                if char == '\'' && has_single_quote
+                  found_closing_single_quote = true
+                  end_here = current_pos
+                  next_char
+                  break
+                elsif has_single_quote
+                  # wait until another quote
+                else
+                  end_here = current_pos
+                  break
+                end
+              end
+            end
+
+            if has_single_quote && !found_closing_single_quote
+              raise "expecting closing single quote"
+            end
+
+            here = string_range(start_here, end_here)
+
+            delimited_pair :heredoc, here, here, start, allow_escapes: !has_single_quote, advance: false
+          else
+            @token.type = :"<<"
+          end
+        else
+          @token.type = :"<"
+        end
+      when '>'
+        case next_char
+        when '='
+          next_char :">="
+        when '>'
+          case next_char
+          when '='
+            next_char :">>="
+          else
+            @token.type = :">>"
+          end
+        else
+          @token.type = :">"
+        end
+      when '+'
+        @token.start = start
+        case next_char
+        when '='
+          next_char :"+="
+        when '0'
+          scan_zero_number(start)
+        when '1', '2', '3', '4', '5', '6', '7', '8', '9'
+          scan_number(start)
+        when '+'
+          raise "postfix increment is not supported, use `exp += 1`"
+        else
+          @token.type = :"+"
+        end
+      when '-'
+        @token.start = start
+        case next_char
+        when '='
+          next_char :"-="
+        when '>'
+          next_char :"->"
+        when '0'
+          scan_zero_number start, negative: true
+        when '1', '2', '3', '4', '5', '6', '7', '8', '9'
+          scan_number start, negative: true
+        when '-'
+          raise "postfix decrement is not supported, use `exp -= 1`"
+        else
+          @token.type = :"-"
+        end
+      when '*'
+        case next_char
+        when '='
+          next_char :"*="
+        when '*'
+          case next_char
+          when '='
+            next_char :"**="
+          else
+            @token.type = :"**"
+          end
+        else
+          @token.type = :"*"
+        end
+      when '/'
+        line = @line_number
+        column = @column_number
+        char = next_char
+        if (@wants_def_or_macro_name || !@slash_is_regex) && char == '/'
+          case next_char
+          when '='
+            next_char :"//="
+          else
+            @token.type = :"//"
+          end
+        elsif !@slash_is_regex && char == '='
+          next_char :"/="
+        elsif @wants_def_or_macro_name
+          @token.type = :"/"
+        elsif @slash_is_regex
+          @token.type = :DELIMITER_START
+          @token.delimiter_state = Token::DelimiterState.new(:regex, '/', '/')
+          @token.raw = "/"
+        elsif char.ascii_whitespace? || char == '\0'
+          @token.type = :"/"
+        elsif @wants_regex
+          @token.type = :DELIMITER_START
+          @token.delimiter_state = Token::DelimiterState.new(:regex, '/', '/')
+          @token.raw = "/"
+        else
+          @token.type = :"/"
+        end
+      when '%'
+        if @wants_def_or_macro_name
+          next_char :"%"
+        else
+          case next_char
+          when '='
+            next_char :"%="
+          when '(', '[', '{', '<', '|'
+            delimited_pair :string, current_char, closing_char, start
+          when 'i'
+            case peek_next_char
+            when '(', '{', '[', '<', '|'
+              start_char = next_char
+              next_char :SYMBOL_ARRAY_START
+              @token.raw = "%i#{start_char}" if @wants_raw
+              @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char))
+            else
+              @token.type = :"%"
+            end
+          when 'q'
+            case peek_next_char
+            when '(', '{', '[', '<', '|'
+              next_char
+              delimited_pair :string, current_char, closing_char, start, allow_escapes: false
+            else
+              @token.type = :"%"
+            end
+          when 'Q'
+            case peek_next_char
+            when '(', '{', '[', '<', '|'
+              next_char
+              delimited_pair :string, current_char, closing_char, start
+            else
+              @token.type = :"%"
+            end
+          when 'r'
+            case next_char
+            when '(', '[', '{', '<', '|'
+              delimited_pair :regex, current_char, closing_char, start
+            else
+              raise "unknown %r char"
+            end
+          when 'x'
+            case next_char
+            when '(', '[', '{', '<', '|'
+              delimited_pair :command, current_char, closing_char, start
+            else
+              raise "unknown %x char"
+            end
+          when 'w'
+            case peek_next_char
+            when '(', '{', '[', '<', '|'
+              start_char = next_char
+              next_char :STRING_ARRAY_START
+              @token.raw = "%w#{start_char}" if @wants_raw
+              @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char))
+            else
+              @token.type = :"%"
+            end
+          when '}'
+            next_char :"%}"
+          else
+            @token.type = :"%"
+          end
+        end
+      when '(' then next_char :"("
+      when ')' then next_char :")"
+      when '{'
+        char = next_char
+        case char
+        when '%'
+          next_char :"{%"
+        when '{'
+          next_char :"{{"
+        else
+          @token.type = :"{"
+        end
+      when '}' then next_char :"}"
+      when '['
+        case next_char
+        when ']'
+          case next_char
+          when '='
+            next_char :"[]="
+          when '?'
+            next_char :"[]?"
+          else
+            @token.type = :"[]"
+          end
+        else
+          @token.type = :"["
+        end
+      when ']' then next_char :"]"
+      when ',' then next_char :","
+      when '?' then next_char :"?"
+      when ';'
+        reset_regex_flags = false
+        next_char :";"
+      when ':'
+        char = next_char
+
+        if @wants_symbol
+          case char
+          when ':'
+            next_char :"::"
+          when '+'
+            next_char_and_symbol "+"
+          when '-'
+            next_char_and_symbol "-"
+          when '*'
+            if next_char == '*'
+              next_char_and_symbol "**"
+            else
+              symbol "*"
+            end
+          when '/'
+            case next_char
+            when '/'
+              next_char_and_symbol "//"
+            else
+              symbol "/"
+            end
+          when '='
+            case next_char
+            when '='
+              if next_char == '='
+                next_char_and_symbol "==="
+              else
+                symbol "=="
+              end
+            when '~'
+              next_char_and_symbol "=~"
+            else
+              unknown_token
+            end
+          when '!'
+            case next_char
+            when '='
+              next_char_and_symbol "!="
+            when '~'
+              next_char_and_symbol "!~"
+            else
+              symbol "!"
+            end
+          when '<'
+            case next_char
+            when '='
+              if next_char == '>'
+                next_char_and_symbol "<=>"
+              else
+                symbol "<="
+              end
+            when '<'
+              next_char_and_symbol "<<"
+            else
+              symbol "<"
+            end
+          when '>'
+            case next_char
+            when '='
+              next_char_and_symbol ">="
+            when '>'
+              next_char_and_symbol ">>"
+            else
+              symbol ">"
+            end
+          when '&'
+            case next_char
+            when '+'
+              next_char_and_symbol "&+"
+            when '-'
+              next_char_and_symbol "&-"
+            when '*'
+              case next_char
+              when '*'
+                next_char_and_symbol "&**"
+              else
+                symbol "&*"
+              end
+            else
+              symbol "&"
+            end
+          when '|'
+            next_char_and_symbol "|"
+          when '^'
+            next_char_and_symbol "^"
+          when '~'
+            next_char_and_symbol "~"
+          when '%'
+            next_char_and_symbol "%"
+          when '['
+            if next_char == ']'
+              case next_char
+              when '='
+                next_char_and_symbol "[]="
+              when '?'
+                next_char_and_symbol "[]?"
+              else
+                symbol "[]"
+              end
+            else
+              unknown_token
+            end
+          when '"'
+            line = @line_number
+            column = @column_number
+            start = current_pos + 1
+            io = IO::Memory.new
+            while true
+              char = next_char
+              case char
+              when '\\'
+                case char = next_char
+                when 'a'
+                  io << '\a'
+                when 'b'
+                  io << '\b'
+                when 'n'
+                  io << '\n'
+                when 'r'
+                  io << '\r'
+                when 't'
+                  io << '\t'
+                when 'v'
+                  io << '\v'
+                when 'f'
+                  io << '\f'
+                when 'e'
+                  io << '\e'
+                when 'x'
+                  io.write_byte consume_string_hex_escape
+                when 'u'
+                  io << consume_string_unicode_escape
+                when '0', '1', '2', '3', '4', '5', '6', '7'
+                  io.write_byte consume_octal_escape(char)
+                when '\n'
+                  incr_line_number nil
+                  io << '\n'
+                when '\0'
+                  raise "unterminated quoted symbol", line, column
+                else
+                  io << char
+                end
+              when '"'
+                break
+              when '\0'
+                raise "unterminated quoted symbol", line, column
+              else
+                io << char
+              end
+            end
+
+            @token.type = :SYMBOL
+            @token.value = io.to_s
+            next_char
+            set_token_raw_from_start(start - 2)
+          else
+            if ident_start?(char)
+              start = current_pos
+              while ident_part?(next_char)
+                # Nothing to do
+              end
+              if current_char == '?' || ((current_char == '!' || current_char == '=') && peek_next_char != '=')
+                next_char
+              end
+              @token.type = :SYMBOL
+              @token.value = string_range_from_pool(start)
+              set_token_raw_from_start(start - 1)
+            else
+              @token.type = :":"
+            end
+          end
+        else
+          case char
+          when ':'
+            next_char :"::"
+          else
+            @token.type = :":"
+          end
+        end
+      when '~'
+        next_char :"~"
+      when '.'
+        case next_char
+        when '.'
+          case next_char
+          when '.'
+            next_char :"..."
+          else
+            @token.type = :".."
+          end
+        else
+          @token.type = :"."
+        end
+      when '&'
+        case next_char
+        when '&'
+          case next_char
+          when '='
+            next_char :"&&="
+          else
+            @token.type = :"&&"
+          end
+        when '='
+          next_char :"&="
+        when '+'
+          case next_char
+          when '='
+            next_char :"&+="
+          else
+            @token.type = :"&+"
+          end
+        when '-'
+          # Check if '>' comes after '&-', making it '&->'.
+          # We want to parse that like '&(->...)',
+          # so we only return '&' for now.
+          if peek_next_char == '>'
+            @token.type = :"&"
+          else
+            case next_char
+            when '='
+              next_char :"&-="
+            else
+              @token.type = :"&-"
+            end
+          end
+        when '*'
+          case next_char
+          when '*'
+            next_char :"&**"
+          when '='
+            next_char :"&*="
+          else
+            @token.type = :"&*"
+          end
+        else
+          @token.type = :"&"
+        end
+      when '|'
+        case next_char
+        when '|'
+          case next_char
+          when '='
+            next_char :"||="
+          else
+            @token.type = :"||"
+          end
+        when '='
+          next_char :"|="
+        else
+          @token.type = :"|"
+        end
+      when '^'
+        case next_char
+        when '='
+          next_char :"^="
+        else
+          @token.type = :"^"
+        end
+      when '"', '`', '\''
+        delimiter = current_char
+        if delimiter == '`' && @wants_def_or_macro_name
+          next_char :"`"
+        else
+          next_char
+          @token.type = :DELIMITER_START
+          @token.delimiter_state = Token::DelimiterState.new(delimiter == '`' ? :command : :string, delimiter, delimiter)
+          set_token_raw_from_start(start)
+        end
+      when '0'
+        scan_zero_number(start)
+      when '1', '2', '3', '4', '5', '6', '7', '8', '9'
+        scan_number current_pos
+      when '@'
+        start = current_pos
+        case next_char
+        when '['
+          next_char :"@["
+        else
+          class_var = false
+          if current_char == '@'
+            class_var = true
+            next_char
+          end
+          if ident_start?(current_char)
+            while ident_part?(next_char)
+              # Nothing to do
+            end
+            @token.type = class_var ? :CLASS_VAR : :INSTANCE_VAR
+            @token.value = string_range_from_pool(start)
+          else
+            unknown_token
+          end
+        end
+      when '$'
+        start = current_pos
+        next_char
+        case current_char
+        when '~'
+          next_char
+          @token.type = :"$~"
+        when '?'
+          next_char
+          @token.type = :"$?"
+        when .ascii_number?
+          start = current_pos
+          char = next_char
+          if char == '0'
+            char = next_char
+          else
+            while char.ascii_number?
+              char = next_char
+            end
+            char = next_char if char == '?'
+          end
+          @token.type = :GLOBAL_MATCH_DATA_INDEX
+          @token.value = string_range_from_pool(start)
+        else
+          if ident_start?(current_char)
+            while ident_part?(next_char)
+              # Nothing to do
+            end
+            @token.type = :GLOBAL
+            @token.value = string_range_from_pool(start)
+          else
+            unknown_token
+          end
+        end
+      when 'a'
+        case next_char
+        when 'b'
+          if next_char == 's' && next_char == 't' && next_char == 'r' && next_char == 'a' && next_char == 'c' && next_char == 't'
+            return check_ident_or_keyword(:abstract, start)
+          end
+        when 'l'
+          if next_char == 'i' && next_char == 'a' && next_char == 's'
+            return check_ident_or_keyword(:alias, start)
+          end
+        when 's'
+          peek = peek_next_char
+          case peek
+          when 'm'
+            next_char
+            return check_ident_or_keyword(:asm, start)
+          when '?'
+            next_char
+            next_char
+            @token.type = :IDENT
+            @token.value = :as?
+            return @token
+          else
+            return check_ident_or_keyword(:as, start)
+          end
+        when 'n'
+          case next_char
+          when 'd'
+            @token.type = :and
+          when 'n'
+            if next_char == 'o' && next_char == 't' && next_char == 'a' && next_char == 't' && next_char == 'i' && next_char == 'o' && next_char == 'n'
+              return check_ident_or_keyword(:annotation, start)
+            end
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'b'
+        case next_char
+        when 'e'
+          if next_char == 'g' && next_char == 'i' && next_char == 'n'
+            return check_ident_or_keyword(:begin, start)
+          end
+        when 'r'
+          if next_char == 'e' && next_char == 'a' && next_char == 'k'
+            return check_ident_or_keyword(:break, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'c'
+        case next_char
+        when 'a'
+          if next_char == 's' && next_char == 'e'
+            return check_ident_or_keyword(:case, start)
+          end
+        when 'l'
+          if next_char == 'a' && next_char == 's' && next_char == 's'
+            return check_ident_or_keyword(:class, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'd'
+        case next_char
+        when 'e'
+          if next_char == 'f'
+            return check_ident_or_keyword(:def, start)
+          end
+        when 'o' then return check_ident_or_keyword(:do, start)
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'e'
+        case next_char
+        when 'l'
+          case next_char
+          when 's'
+            case next_char
+            when 'e' then return check_ident_or_keyword(:else, start)
+            when 'i'
+              if next_char == 'f'
+                return check_ident_or_keyword(:elsif, start)
+              end
+            else
+              # scan_ident
+            end
+          else
+            # scan_ident
+          end
+        when 'n'
+          case next_char
+          when 'd'
+            return check_ident_or_keyword(:end, start)
+          when 's'
+            if next_char == 'u' && next_char == 'r' && next_char == 'e'
+              return check_ident_or_keyword(:ensure, start)
+            end
+          when 'u'
+            if next_char == 'm'
+              return check_ident_or_keyword(:enum, start)
+            end
+          else
+            # scan_ident
+          end
+        when 'x'
+          if next_char == 't' && next_char == 'e' && next_char == 'n' && next_char == 'd'
+            return check_ident_or_keyword(:extend, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'f'
+        case next_char
+        when 'a'
+          if next_char == 'l' && next_char == 's' && next_char == 'e'
+            return check_ident_or_keyword(:false, start)
+          end
+        when 'o'
+          if next_char == 'r'
+            return check_ident_or_keyword(:for, start)
+          end
+        when 'u'
+          if next_char == 'n'
+            return check_ident_or_keyword(:fun, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'i'
+        case next_char
+        when 'f'
+          return check_ident_or_keyword(:if, start)
+        when 'n'
+          if ident_part_or_end?(peek_next_char)
+            case next_char
+            when 'c'
+              if next_char == 'l' && next_char == 'u' && next_char == 'd' && next_char == 'e'
+                return check_ident_or_keyword(:include, start)
+              end
+            when 's'
+              if next_char == 't' && next_char == 'a' && next_char == 'n' && next_char == 'c' && next_char == 'e' && next_char == '_' && next_char == 's' && next_char == 'i' && next_char == 'z' && next_char == 'e' && next_char == 'o' && next_char == 'f'
+                return check_ident_or_keyword(:instance_sizeof, start)
+              end
+            else
+              # scan_ident
+            end
+          else
+            next_char
+            @token.type = :IDENT
+            @token.value = :in
+            return @token
+          end
+        when 's'
+          if next_char == '_' && next_char == 'a' && next_char == '?'
+            return check_ident_or_keyword(:is_a?, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'l'
+        case next_char
+        when 'i'
+          if next_char == 'b'
+            return check_ident_or_keyword(:lib, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'm'
+        case next_char
+        when 'a'
+          if next_char == 'c' && next_char == 'r' && next_char == 'o'
+            return check_ident_or_keyword(:macro, start)
+          end
+        when 'o'
+          case next_char
+          when 'd'
+            if next_char == 'u' && next_char == 'l' && next_char == 'e'
+              return check_ident_or_keyword(:module, start)
+            end
+          else
+            # scan_ident
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'n'
+        case next_char
+        when 'e'
+          if next_char == 'x' && next_char == 't'
+            return check_ident_or_keyword(:next, start)
+          end
+        when 'i'
+          case next_char
+          when 'l'
+            if peek_next_char == '?'
+              next_char
+              return check_ident_or_keyword(:nil?, start)
+            else
+              return check_ident_or_keyword(:nil, start)
+            end
+          else
+            # scan_ident
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'o'
+        case next_char
+        when 'f'
+          if peek_next_char == 'f'
+            next_char
+            if next_char == 's' && next_char == 'e' && next_char == 't' && next_char == 'o' && next_char == 'f'
+              return check_ident_or_keyword(:offsetof, start)
+            end
+          else
+            return check_ident_or_keyword(:of, start)
+          end
+        when 'u'
+          if next_char == 't'
+            return check_ident_or_keyword(:out, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'p'
+        case next_char
+        when 'o'
+          if next_char == 'i' && next_char == 'n' && next_char == 't' && next_char == 'e' && next_char == 'r' && next_char == 'o' && next_char == 'f'
+            return check_ident_or_keyword(:pointerof, start)
+          end
+        when 'r'
+          case next_char
+          when 'i'
+            if next_char == 'v' && next_char == 'a' && next_char == 't' && next_char == 'e'
+              return check_ident_or_keyword(:private, start)
+            end
+          when 'o'
+            if next_char == 't' && next_char == 'e' && next_char == 'c' && next_char == 't' && next_char == 'e' && next_char == 'd'
+              return check_ident_or_keyword(:protected, start)
+            end
+          else
+            # scan_ident
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'r'
+        case next_char
+        when 'e'
+          case next_char
+          when 's'
+            case next_char
+            when 'c'
+              if next_char == 'u' && next_char == 'e'
+                return check_ident_or_keyword(:rescue, start)
+              end
+            when 'p'
+              if next_char == 'o' && next_char == 'n' && next_char == 'd' && next_char == 's' && next_char == '_' && next_char == 't' && next_char == 'o' && next_char == '?'
+                return check_ident_or_keyword(:responds_to?, start)
+              end
+            else
+              # scan_ident
+            end
+          when 't'
+            if next_char == 'u' && next_char == 'r' && next_char == 'n'
+              return check_ident_or_keyword(:return, start)
+            end
+          when 'q'
+            if next_char == 'u' && next_char == 'i' && next_char == 'r' && next_char == 'e'
+              return check_ident_or_keyword(:require, start)
+            end
+          else
+            # scan_ident
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 's'
+        case next_char
+        when 'e'
+          if next_char == 'l'
+            case next_char
+            when 'e'
+              if next_char == 'c' && next_char == 't'
+                return check_ident_or_keyword(:select, start)
+              end
+            when 'f'
+              return check_ident_or_keyword(:self, start)
+            else
+              # scan_ident
+            end
+          end
+        when 'i'
+          if next_char == 'z' && next_char == 'e' && next_char == 'o' && next_char == 'f'
+            return check_ident_or_keyword(:sizeof, start)
+          end
+        when 't'
+          if next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't'
+            return check_ident_or_keyword(:struct, start)
+          end
+        when 'u'
+          if next_char == 'p' && next_char == 'e' && next_char == 'r'
+            return check_ident_or_keyword(:super, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 't'
+        case next_char
+        when 'h'
+          if next_char == 'e' && next_char == 'n'
+            return check_ident_or_keyword(:then, start)
+          end
+        when 'r'
+          if next_char == 'u' && next_char == 'e'
+            return check_ident_or_keyword(:true, start)
+          end
+        when 'y'
+          if next_char == 'p' && next_char == 'e'
+            if peek_next_char == 'o'
+              next_char
+              if next_char == 'f'
+                return check_ident_or_keyword(:typeof, start)
+              end
+            else
+              return check_ident_or_keyword(:type, start)
+            end
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'u'
+        if next_char == 'n'
+          case next_char
+          when 'i'
+            case next_char
+            when 'o'
+              if next_char == 'n'
+                return check_ident_or_keyword(:union, start)
+              end
+            when 'n'
+              if next_char == 'i' && next_char == 't' && next_char == 'i' && next_char == 'a' && next_char == 'l' && next_char == 'i' && next_char == 'z' && next_char == 'e' && next_char == 'd'
+                return check_ident_or_keyword(:uninitialized, start)
+              end
+            else
+              # scan_ident
+            end
+          when 'l'
+            if next_char == 'e' && next_char == 's' && next_char == 's'
+              return check_ident_or_keyword(:unless, start)
+            end
+          when 't'
+            if next_char == 'i' && next_char == 'l'
+              return check_ident_or_keyword(:until, start)
+            end
+          else
+            # scan_ident
+          end
+        end
+        scan_ident(start)
+      when 'v'
+        if next_char == 'e' && next_char == 'r' && next_char == 'b' && next_char == 'a' && next_char == 't' && next_char == 'i' && next_char == 'm'
+          return check_ident_or_keyword(:verbatim, start)
+        end
+        scan_ident(start)
+      when 'w'
+        case next_char
+        when 'h'
+          case next_char
+          when 'e'
+            if next_char == 'n'
+              return check_ident_or_keyword(:when, start)
+            end
+          when 'i'
+            if next_char == 'l' && next_char == 'e'
+              return check_ident_or_keyword(:while, start)
+            end
+          else
+            # scan_ident
+          end
+        when 'i'
+          if next_char == 't' && next_char == 'h'
+            return check_ident_or_keyword(:with, start)
+          end
+        else
+          # scan_ident
+        end
+        scan_ident(start)
+      when 'y'
+        if next_char == 'i' && next_char == 'e' && next_char == 'l' && next_char == 'd'
+          return check_ident_or_keyword(:yield, start)
+        end
+        scan_ident(start)
+      when '_'
+        case next_char
+        when '_'
+          case next_char
+          when 'D'
+            if next_char == 'I' && next_char == 'R' && next_char == '_' && next_char == '_'
+              if ident_part_or_end?(peek_next_char)
+                scan_ident(start)
+              else
+                next_char
+                @token.type = :__DIR__
+                return @token
+              end
+            end
+          when 'E'
+            if next_char == 'N' && next_char == 'D' && next_char == '_' && next_char == 'L' && next_char == 'I' && next_char == 'N' && next_char == 'E' && next_char == '_' && next_char == '_'
+              if ident_part_or_end?(peek_next_char)
+                scan_ident(start)
+              else
+                next_char
+                @token.type = :__END_LINE__
+                return @token
+              end
+            end
+          when 'F'
+            if next_char == 'I' && next_char == 'L' && next_char == 'E' && next_char == '_' && next_char == '_'
+              if ident_part_or_end?(peek_next_char)
+                scan_ident(start)
+              else
+                next_char
+                @token.type = :__FILE__
+                return @token
+              end
+            end
+          when 'L'
+            if next_char == 'I' && next_char == 'N' && next_char == 'E' && next_char == '_' && next_char == '_'
+              if ident_part_or_end?(peek_next_char)
+                scan_ident(start)
+              else
+                next_char
+                @token.type = :__LINE__
+                return @token
+              end
+            end
+          else
+            # scan_ident
+          end
+        else
+          unless ident_part?(current_char)
+            @token.type = :UNDERSCORE
+            return @token
+          end
+        end
+
+        scan_ident(start)
+      else
+        if current_char.ascii_uppercase?
+          start = current_pos
+          while ident_part?(next_char)
+            # Nothing to do
+          end
+          @token.type = :CONST
+          @token.value = string_range_from_pool(start)
+        elsif current_char.ascii_lowercase? || current_char == '_' || current_char.ord > 0x9F
+          next_char
+          scan_ident(start)
+        else
+          unknown_token
+        end
+      end
+
+      if reset_regex_flags
+        @wants_regex = true
+        @slash_is_regex = false
+      end
+
+      @token
+    end
+  end
+end