yard 0.9.16 → 0.9.17

data/lib/yard/parser/ruby/legacy/ruby_lex.rb

@@ -1,1354 +1,1354 @@
-require "e2mmap"
-require "irb/slex"
-
-module YARD
-  module Parser::Ruby::Legacy
-    # Legacy lexical tokenizer module.
-    module RubyToken
-      EXPR_BEG   = :EXPR_BEG
-      EXPR_MID   = :EXPR_MID
-      EXPR_END   = :EXPR_END
-      EXPR_ARG   = :EXPR_ARG
-      EXPR_FNAME = :EXPR_FNAME
-      EXPR_DOT   = :EXPR_DOT
-      EXPR_CLASS = :EXPR_CLASS
-
-      # Represents a token in the Ruby lexer
-      class Token
-        # @return [Integer] the line number in the file/stream the token is
-        #   located.
-        attr_reader :line_no
-
-        # @return [Integer] the character number in the file/stream the token
-        #   is located.
-        attr_reader :char_no
-
-        # @return [String] the token text value
-        attr_reader :text
-
-        # @return [Symbol] the lexical state at the token
-        attr_accessor :lex_state
-
-        # @private
-        NO_TEXT = "??".freeze
-
-        # Creates a new Token object
-        # @param [Integer] line_no the line number to initialize the token to
-        # @param [Integer] char_no the char number to initialize the token to
-        def initialize(line_no, char_no)
-          @line_no = line_no
-          @char_no = char_no
-          @text    = NO_TEXT
-        end
-
-        # Chainable way to sets the text attribute
-        #
-        # @param [String] text the new text
-        # @return [Token] this token object
-        def set_text(text)
-          @text = text
-          self
-        end
-      end
-
-      # Represents a block
-      class TkBlockContents < Token
-        def text; '...' end
-      end
-
-      # Represents an end statement
-      class TkStatementEnd < Token
-        def text; '' end
-      end
-
-      class TkNode < Token
-        attr :node
-      end
-
-      # Represents whitespace
-      class TkWhitespace < Token
-      end
-
-      # Represents a Ruby identifier
-      class TkId < Token
-        def initialize(line_no, char_no, name)
-          super(line_no, char_no)
-          @name = name
-        end
-        attr :name
-      end
-
-      # Represents a Ruby keyword
-      class TkKW < TkId
-      end
-
-      # Represents a Ruby value
-      class TkVal < Token
-        def initialize(line_no, char_no, value = nil)
-          super(line_no, char_no)
-          set_text(value)
-        end
-      end
-
-      class TkOp < Token
-        def name
-          self.class.op_name
-        end
-      end
-
-      class TkOPASGN < TkOp
-        def initialize(line_no, char_no, op)
-          super(line_no, char_no)
-          op = TkReading2Token[op] unless op.is_a?(Symbol)
-          @op = op
-        end
-        attr :op
-      end
-
-      class TkUnknownChar < Token
-        def initialize(line_no, char_no, _id)
-          super(line_no, char_no)
-          @name = char_no > 255 ? '?' : char_no.chr
-        end
-        attr :name
-      end
-
-      class TkError < Token
-      end
-
-      # @private
-      def set_token_position(line, char)
-        @prev_line_no = line
-        @prev_char_no = char
-      end
-
-      # @private
-      def Token(token, value = nil) # rubocop:disable Style/MethodName
-        tk = nil
-        case token
-        when String, Symbol
-          source = token.is_a?(String) ? TkReading2Token : TkSymbol2Token
-          if (tk = source[token]).nil?
-            IRB.fail TkReading2TokenNoKey, token
-          end
-          tk = Token(tk[0], value)
-        else
-          if token
-            tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
-                   token.new(@prev_line_no, @prev_char_no)
-                 else
-                   token.new(@prev_line_no, @prev_char_no, value)
-                 end
-          end
-        end
-        tk
-      end
-
-      # @private
-      TokenDefinitions = [
-        [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
-        [:TkMODULE,     TkKW,  "module", EXPR_BEG],
-        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
-        [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
-        [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
-        [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
-        [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
-        [:TkEND,        TkKW,  "end",    EXPR_END],
-        [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
-        [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
-        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
-        [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
-        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
-        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
-        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
-        [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
-        [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
-        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
-        [:TkBREAK,      TkKW,  "break",  EXPR_END],
-        [:TkNEXT,       TkKW,  "next",   EXPR_END],
-        [:TkREDO,       TkKW,  "redo",   EXPR_END],
-        [:TkRETRY,      TkKW,  "retry",  EXPR_END],
-        [:TkIN,         TkKW,  "in",     EXPR_BEG],
-        [:TkDO,         TkKW,  "do",     EXPR_BEG],
-        [:TkRETURN,     TkKW,  "return", EXPR_MID],
-        [:TkYIELD,      TkKW,  "yield",  EXPR_END],
-        [:TkSUPER,      TkKW,  "super",  EXPR_END],
-        [:TkSELF,       TkKW,  "self",   EXPR_END],
-        [:TkNIL,        TkKW,  "nil",    EXPR_END],
-        [:TkTRUE,       TkKW,  "true",   EXPR_END],
-        [:TkFALSE,      TkKW,  "false",  EXPR_END],
-        [:TkAND,        TkKW,  "and",    EXPR_BEG],
-        [:TkOR,         TkKW,  "or",     EXPR_BEG],
-        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
-        [:TkIF_MOD,     TkKW],
-        [:TkUNLESS_MOD, TkKW],
-        [:TkWHILE_MOD,  TkKW],
-        [:TkUNTIL_MOD,  TkKW],
-        [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
-        [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
-        [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
-        [:TklEND,       TkKW,  "END",      EXPR_END],
-        [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
-        [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
-        [:TkIDENTIFIER, TkId],
-        [:TkFID,        TkId],
-        [:TkGVAR,       TkId],
-        [:TkIVAR,       TkId],
-        [:TkCONSTANT,   TkId],
-        [:TkINTEGER,    TkVal],
-        [:TkFLOAT,      TkVal],
-        [:TkSYMBOL,     TkVal],
-        [:TkLABEL,      TkVal],
-        [:TkSTRING,     TkVal],
-        [:TkXSTRING,    TkVal],
-        [:TkREGEXP,     TkVal],
-        [:TkCOMMENT,    TkVal],
-        [:TkDSTRING,    TkNode],
-        [:TkDXSTRING,   TkNode],
-        [:TkDREGEXP,    TkNode],
-        [:TkNTH_REF,    TkId],
-        [:TkBACK_REF,   TkId],
-        [:TkUPLUS,      TkOp,   "+@"],
-        [:TkUMINUS,     TkOp,   "-@"],
-        [:TkPOW,        TkOp,   "**"],
-        [:TkCMP,        TkOp,   "<=>"],
-        [:TkEQ,         TkOp,   "=="],
-        [:TkEQQ,        TkOp,   "==="],
-        [:TkNEQ,        TkOp,   "!="],
-        [:TkGEQ,        TkOp,   ">="],
-        [:TkLEQ,        TkOp,   "<="],
-        [:TkANDOP,      TkOp,   "&&"],
-        [:TkOROP,       TkOp,   "||"],
-        [:TkMATCH,      TkOp,   "=~"],
-        [:TkNMATCH,     TkOp,   "!~"],
-        [:TkDOT2,       TkOp,   ".."],
-        [:TkDOT3,       TkOp,   "..."],
-        [:TkAREF,       TkOp,   "[]"],
-        [:TkASET,       TkOp,   "[]="],
-        [:TkLSHFT,      TkOp,   "<<"],
-        [:TkRSHFT,      TkOp,   ">>"],
-        [:TkCOLON2,     TkOp],
-        [:TkCOLON3,     TkOp],
-        [:OPASGN,       TkOp], # +=, -=  etc. #
-        [:TkASSOC,      TkOp,   "=>"],
-        [:TkQUESTION,   TkOp,   "?"], #?
-        [:TkCOLON,      TkOp,   ":"], #:
-        [:TkSTAR],            # *arg
-        [:TkAMPER],           # &arg #
-        [:TkSYMBEG,     TkId],
-        [:TkGT,         TkOp,   ">"],
-        [:TkLT,         TkOp,   "<"],
-        [:TkPLUS,       TkOp,   "+"],
-        [:TkMINUS,      TkOp,   "-"],
-        [:TkMULT,       TkOp,   "*"],
-        [:TkDIV,        TkOp,   "/"],
-        [:TkMOD,        TkOp,   "%"],
-        [:TkBITOR,      TkOp,   "|"],
-        [:TkBITXOR,     TkOp,   "^"],
-        [:TkBITAND,     TkOp,   "&"],
-        [:TkBITNOT,     TkOp,   "~"],
-        [:TkNOTOP,      TkOp,   "!"],
-        [:TkBACKQUOTE,  TkOp,   "`"],
-        [:TkASSIGN,     Token,  "="],
-        [:TkDOT,        Token,  "."],
-        [:TkLPAREN,     Token,  "("],  # (exp)
-        [:TkLBRACK,     Token,  "["],  # [arry]
-        [:TkLBRACE,     Token,  "{"],  # {hash}
-        [:TkRPAREN,     Token,  ")"],
-        [:TkRBRACK,     Token,  "]"],
-        [:TkRBRACE,     Token,  "}"],
-        [:TkCOMMA,      Token,  ","],
-        [:TkSEMICOLON,  Token,  ";"],
-        [:TkSPACE,          TkWhitespace],
-        [:TkNL,             TkWhitespace],
-        [:TkEND_OF_SCRIPT,  TkWhitespace],
-        [:TkBACKSLASH,  TkUnknownChar,  "\\"],
-        [:TkAT,         TkUnknownChar,  "@"],
-        [:TkDOLLAR,     TkUnknownChar,  "\$"]
-      ]
-
-      # { reading => token_class }
-      # { reading => [token_class, *opt] }
-      TkReading2Token = {}
-      TkSymbol2Token = {}
-
-      # @private
-      def self.def_token(token_n, super_token = Token, reading = nil, *opts)
-        token_n = token_n.id2name unless token_n.is_a?(String)
-        if RubyToken.const_defined?(token_n)
-          # IRB.fail AlreadyDefinedToken, token_n
-        end
-
-        token_c = Class.new super_token
-        RubyToken.const_set token_n, token_c
-        # token_c.inspect
-
-        if reading
-          if TkReading2Token[reading]
-            IRB.fail TkReading2TokenDuplicateError, token_n, reading
-          end
-          if opts.empty?
-            TkReading2Token[reading] = [token_c]
-          else
-            TkReading2Token[reading] = [token_c].concat(opts)
-          end
-        end
-        TkSymbol2Token[token_n.intern] = token_c
-
-        if token_c <= TkOp
-          token_c.class_eval %{
-            def self.op_name; "#{reading}"; end
-          }
-        end
-      end
-
-      for defs in TokenDefinitions
-        def_token(*defs)
-      end
-
-      NEWLINE_TOKEN = TkNL.new(0, 0)
-      NEWLINE_TOKEN.set_text("\n")
-    end
-
-    # Lexical analyzer for Ruby source
-    # @private
-    class RubyLex
-      # Read an input stream character by character. We allow for unlimited
-      # ungetting of characters just read.
-      #
-      # We simplify the implementation greatly by reading the entire input
-      # into a buffer initially, and then simply traversing it using
-      # pointers.
-      #
-      # We also have to allow for the <i>here document diversion</i>. This
-      # little gem comes about when the lexer encounters a here
-      # document. At this point we effectively need to split the input
-      # stream into two parts: one to read the body of the here document,
-      # the other to read the rest of the input line where the here
-      # document was initially encountered. For example, we might have
-      #
-      #   do_something(<<-A, <<-B)
-      #     stuff
-      #     for
-      #   A
-      #     stuff
-      #     for
-      #   B
-      #
-      # When the lexer encounters the <<A, it reads until the end of the
-      # line, and keeps it around for later. It then reads the body of the
-      # here document.  Once complete, it needs to read the rest of the
-      # original line, but then skip the here document body.
-      #
-      # @private
-      class BufferedReader
-        attr_reader :line_num
-
-        def initialize(content)
-          if /\t/ =~ content
-            tab_width = 2
-            content = content.split(/\n/).map do |line|
-              1 while line.gsub!(/\t+/) { ' ' * (tab_width * $&.length - $`.length % tab_width) } && $~ #`
-              line
-            end .join("\n")
-          end
-          @content = String.new(content)
-          @content << "\n" unless @content[-1, 1] == "\n"
-          @size      = @content.size
-          @offset    = 0
-          @hwm       = 0
-          @line_num  = 1
-          @read_back_offset = 0
-          @last_newline = 0
-          @newline_pending = false
-        end
-
-        def column
-          @offset - @last_newline
-        end
-
-        def getc
-          return nil if @offset >= @size
-          ch = @content[@offset, 1]
-
-          @offset += 1
-          @hwm = @offset if @hwm < @offset
-
-          if @newline_pending
-            @line_num += 1
-            @last_newline = @offset - 1
-            @newline_pending = false
-          end
-
-          if ch == "\n"
-            @newline_pending = true
-          end
-          ch
-        end
-
-        def getc_already_read
-          getc
-        end
-
-        def ungetc(_ch)
-          raise "unget past beginning of file" if @offset <= 0
-          @offset -= 1
-          if @content[@offset] == ?\n
-            @newline_pending = false
-          end
-        end
-
-        def get_read
-          res = @content[@read_back_offset...@offset]
-          @read_back_offset = @offset
-          res
-        end
-
-        def peek(at)
-          pos = @offset + at
-          if pos >= @size
-            nil
-          else
-            @content[pos, 1]
-          end
-        end
-
-        def peek_equal(str)
-          @content[@offset, str.length] == str
-        end
-
-        def divert_read_from(reserve)
-          @content[@offset, 0] = reserve
-          @size = @content.size
-        end
-      end
-
-      # end of nested class BufferedReader
-
-      extend Exception2MessageMapper
-      def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
-      def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
-      def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-      def_exception(:TkReading2TokenDuplicateError,
-        "key duplicate(token_n='%s', key='%s')")
-      def_exception(:SyntaxError, "%s")
-
-      include RubyToken
-      include IRB
-
-      attr_reader :continue
-      attr_reader :lex_state
-
-      def self.debug?
-        false
-      end
-
-      def initialize(content)
-        lex_init
-
-        @reader = BufferedReader.new(content)
-
-        @exp_line_no = @line_no = 1
-        @base_char_no = 0
-        @indent = 0
-
-        @ltype = nil
-        @quoted = nil
-        @lex_state = EXPR_BEG
-        @space_seen = false
-
-        @continue = false
-        @line = ""
-
-        @skip_space = false
-        @read_auto_clean_up = false
-        @exception_on_syntax_error = true
-
-        @colonblock_seen = false
-      end
-
-      attr_accessor :skip_space
-      attr_accessor :read_auto_clean_up
-      attr_accessor :exception_on_syntax_error
-
-      attr :indent
-
-      # io functions
-      def line_no
-        @reader.line_num
-      end
-
-      def char_no
-        @reader.column
-      end
-
-      def get_read
-        @reader.get_read
-      end
-
-      def getc
-        @reader.getc
-      end
-
-      def getc_of_rests
-        @reader.getc_already_read
-      end
-
-      def gets
-        (c = getc) || return
-        l = ""
-        begin
-          l.concat c unless c == "\r"
-          break if c == "\n"
-        end while c = getc # rubocop:disable Lint/Loop
-        l
-      end
-
-      def ungetc(c = nil)
-        @reader.ungetc(c)
-      end
-
-      def peek_equal?(str)
-        @reader.peek_equal(str)
-      end
-
-      def peek(i = 0)
-        @reader.peek(i)
-      end
-
-      def lex
-        catch(:eof) do
-          until ((tk = token).is_a?(TkNL) || tk.is_a?(TkEND_OF_SCRIPT)) &&
-                !@continue ||
-                tk.nil?
-          end
-          line = get_read
-
-          if line == "" && tk.is_a?(TkEND_OF_SCRIPT) || tk.nil?
-            nil
-          else
-            line
-          end
-        end
-      end
-
-      def token
-        set_token_position(line_no, char_no)
-        catch(:eof) do
-          begin
-            begin
-              tk = @OP.match(self)
-              @space_seen = tk.is_a?(TkSPACE)
-            rescue SyntaxError
-              abort if @exception_on_syntax_error
-              tk = TkError.new(line_no, char_no)
-            end
-          end while @skip_space && tk.is_a?(TkSPACE)
-          if @read_auto_clean_up
-            get_read
-          end
-          # throw :eof unless tk
-          p tk if $DEBUG
-          tk.lex_state = lex_state if tk
-          tk
-        end
-      end
-
-      ENINDENT_CLAUSE = [
-        "case", "class", "def", "do", "for", "if",
-        "module", "unless", "until", "while", "begin"
-      ] #, "when"
-      ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
-      DEINDENT_CLAUSE = ["end"] #, "when"
-
-      PERCENT_LTYPE = {
-        "q" => "\'",
-        "Q" => "\"",
-        "x" => "\`",
-        "r" => "/",
-        "w" => "]",
-        "W" => "]"
-      }
-
-      PERCENT_PAREN = {
-        "{" => "}",
-        "[" => "]",
-        "<" => ">",
-        "(" => ")"
-      }
-
-      Ltype2Token = {
-        "\'" => TkSTRING,
-        "\"" => TkSTRING,
-        "\`" => TkXSTRING,
-        "/" => TkREGEXP,
-        "]" => TkDSTRING
-      }
-      Ltype2Token.default = TkSTRING
-
-      DLtype2Token = {
-        "\"" => TkDSTRING,
-        "\`" => TkDXSTRING,
-        "/" => TkDREGEXP
-      }
-
-      def lex_init()
-        @OP = SLex.new
-        @OP.def_rules("\0", "\004", "\032") do |chars, _io|
-          Token(TkEND_OF_SCRIPT).set_text(chars)
-        end
-
-        @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, _io|
-          @space_seen = true
-          while (ch = getc) =~ /[ \t\f\r\13]/
-            chars << ch
-          end
-          ungetc
-          Token(TkSPACE).set_text(chars)
-        end
-
-        @OP.def_rule("#") do |_op, _io|
-          identify_comment
-        end
-
-        @OP.def_rule("=begin", proc { @prev_char_no == 0 && peek(0) =~ /\s/ }) do |op, _io|
-          str = String.new(op)
-          @ltype = "="
-
-          begin
-            line = String.new
-            begin
-              ch = getc
-              line << ch
-            end until ch == "\n"
-            str << line
-          end until line =~ /^=end/
-
-          ungetc
-
-          @ltype = nil
-
-          if str =~ /\A=begin\s+rdoc/i
-            str.sub!(/\A=begin.*\n/, '')
-            str.sub!(/^=end.*/m, '')
-            Token(TkCOMMENT).set_text(str)
-          else
-            Token(TkCOMMENT).set_text(str)
-          end
-        end
-
-        @OP.def_rule("\n") do
-          print "\\n\n" if RubyLex.debug?
-          @colonblock_seen = false
-          case @lex_state
-          when EXPR_BEG, EXPR_FNAME, EXPR_DOT
-            @continue = true
-          else
-            @continue = false
-            @lex_state = EXPR_BEG
-          end
-          Token(TkNL).set_text("\n")
-        end
-
-        @OP.def_rules("*", "**",
-          "!", "!=", "!~",
-          "=", "==", "===",
-          "=~", "<=>",
-          "<", "<=",
-          ">", ">=", ">>") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rules("<<") do |op, _io|
-          tk = nil
-          if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
-             (@lex_state != EXPR_ARG || @space_seen)
-            c = peek(0)
-            tk = identify_here_document if /[-\w\"\'\`]/ =~ c
-          end
-          if !tk
-            @lex_state = EXPR_BEG
-            tk = Token(op).set_text(op)
-          end
-          tk
-        end
-
-        @OP.def_rules("'", '"') do |op, _io|
-          identify_string(op)
-        end
-
-        @OP.def_rules("`") do |op, _io|
-          if @lex_state == EXPR_FNAME
-            Token(op).set_text(op)
-          else
-            identify_string(op)
-          end
-        end
-
-        @OP.def_rules('?') do |op, _io|
-          if @lex_state == EXPR_END
-            @lex_state = EXPR_BEG
-            Token(TkQUESTION).set_text(op)
-          else
-            ch = getc
-            if @lex_state == EXPR_ARG && ch !~ /\s/
-              ungetc
-              @lex_state = EXPR_BEG
-              Token(TkQUESTION).set_text(op)
-            else
-              str = String.new(op)
-              str << ch
-              if ch == '\\' #'
-                str << read_escape
-              end
-              @lex_state = EXPR_END
-              Token(TkINTEGER).set_text(str)
-            end
-          end
-        end
-
-        @OP.def_rules("&", "&&", "|", "||") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rules("+=", "-=", "*=", "**=",
-          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, _io|
-          @lex_state = EXPR_BEG
-          op =~ /^(.*)=$/
-          Token(TkOPASGN, $1).set_text(op)
-        end
-
-        @OP.def_rule("+@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
-          Token(TkUPLUS).set_text(op)
-        end
-
-        @OP.def_rule("-@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
-          Token(TkUMINUS).set_text(op)
-        end
-
-        @OP.def_rules("+", "-") do |op, _io|
-          catch(:RET) do
-            if @lex_state == EXPR_ARG
-              if @space_seen && peek(0) =~ /[0-9]/
-                throw :RET, identify_number(op)
-              else
-                @lex_state = EXPR_BEG
-              end
-            elsif @lex_state != EXPR_END && peek(0) =~ /[0-9]/
-              throw :RET, identify_number(op)
-            else
-              @lex_state = EXPR_BEG
-            end
-            Token(op).set_text(op)
-          end
-        end
-
-        @OP.def_rule(".") do
-          @lex_state = EXPR_BEG
-          if peek(0) =~ /[0-9]/
-            ungetc
-            identify_number("")
-          else
-            # for obj.if
-            @lex_state = EXPR_DOT
-            Token(TkDOT).set_text(".")
-          end
-        end
-
-        @OP.def_rules("..", "...") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        lex_int2
-      end
-
-      def lex_int2
-        @OP.def_rules("]", "}", ")") do |op, _io|
-          @lex_state = EXPR_END
-          @indent -= 1
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rule(":") do
-          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
-            @lex_state = EXPR_BEG
-            tk = Token(TkCOLON)
-          else
-            @lex_state = EXPR_FNAME
-            tk = Token(TkSYMBEG)
-          end
-          tk.set_text(":")
-        end
-
-        @OP.def_rule("::") do
-          # p @lex_state.id2name, @space_seen
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_ARG && @space_seen
-            @lex_state = EXPR_BEG
-            tk = Token(TkCOLON3)
-          else
-            @lex_state = EXPR_DOT
-            tk = Token(TkCOLON2)
-          end
-          tk.set_text("::")
-        end
-
-        @OP.def_rule("/") do |op, _io|
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            identify_string(op)
-          elsif peek(0) == '='
-            getc
-            @lex_state = EXPR_BEG
-            Token(TkOPASGN, :/).set_text("/=") #")
-          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
-            identify_string(op)
-          else
-            @lex_state = EXPR_BEG
-            Token("/").set_text(op)
-          end
-        end
-
-        @OP.def_rules("^") do
-          @lex_state = EXPR_BEG
-          Token("^").set_text("^")
-        end
-
-        # @OP.def_rules("^=") do
-        #   @lex_state = EXPR_BEG
-        #   Token(TkOPASGN, :^)
-        # end
-
-        @OP.def_rules(",", ";") do |op, _io|
-          @colonblock_seen = false
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rule("~") do
-          @lex_state = EXPR_BEG
-          Token("~").set_text("~")
-        end
-
-        @OP.def_rule("~@", proc { @lex_state = EXPR_FNAME }) do
-          @lex_state = EXPR_BEG
-          Token("~").set_text("~@")
-        end
-
-        @OP.def_rule("(") do
-          @indent += 1
-            # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            #  @lex_state = EXPR_BEG
-            #  tk = Token(TkfLPAREN)
-            # else
-            @lex_state = EXPR_BEG
-            tk = Token(TkLPAREN)
-          # end
-          tk.set_text("(")
-        end
-
-        @OP.def_rule("[]", proc { @lex_state == EXPR_FNAME }) do
-          Token("[]").set_text("[]")
-        end
-
-        @OP.def_rule("[]=", proc { @lex_state == EXPR_FNAME }) do
-          Token("[]=").set_text("[]=")
-        end
-
-        @OP.def_rule("[") do
-          @indent += 1
-          # if @lex_state == EXPR_FNAME
-          #   t = Token(TkfLBRACK)
-          # else
-          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-          #     t = Token(TkLBRACK)
-          #   elsif @lex_state == EXPR_ARG && @space_seen
-          #   else
-          #     t = Token(TkfLBRACK)
-          #   end
-          # end
-          t = Token(TkLBRACK)
-          @lex_state = EXPR_BEG
-          t.set_text("[")
-        end
-
-        @OP.def_rule("{") do
-          @indent += 1
-          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
-          #   t = Token(TkLBRACE)
-          # else
-          #   t = Token(TkfLBRACE)
-          # end
-          t = Token(TkLBRACE)
-          @lex_state = EXPR_BEG
-          t.set_text("{")
-        end
-
-        @OP.def_rule('\\') do #'
-          if getc == "\n"
-            @space_seen = true
-            @continue = true
-            Token(TkSPACE).set_text("\\\n")
-          else
-            ungetc
-            Token("\\").set_text("\\") #"
-          end
-        end
-
-        @OP.def_rule('%') do |_op, _io|
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            identify_quotation('%')
-          elsif peek(0) == '='
-            getc
-            Token(TkOPASGN, "%").set_text("%=")
-          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
-            identify_quotation('%')
-          else
-            @lex_state = EXPR_BEG
-            Token("%").set_text("%")
-          end
-        end
-
-        @OP.def_rule('$') do #'
-          identify_gvar
-        end
-
-        @OP.def_rule('@') do
-          if peek(0) =~ /[@\w]/
-            ungetc
-            identify_identifier
-          else
-            Token("@").set_text("@")
-          end
-        end
-
-        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
-        #   |op, io|
-        #   @indent += 1
-        #   @lex_state = EXPR_FNAME
-        # # @lex_state = EXPR_END
-        # # until @rests[0] == "\n" or @rests[0] == ";"
-        # #   rests.shift
-        # # end
-        # end
-
-        @OP.def_rule("__END__", proc { @prev_char_no == 0 && peek(0) =~ /[\r\n]/ }) do
-          throw :eof
-        end
-
-        @OP.def_rule("") do |op, io|
-          printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
-          if peek(0) =~ /[0-9]/
-            t = identify_number("")
-          elsif peek(0) =~ /[\w]/
-            t = identify_identifier
-          end
-          printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
-          t
-        end
-
-        p @OP if RubyLex.debug?
-      end
-
-      def identify_gvar
-        @lex_state = EXPR_END
-        str = String.new("$")
-
-        tk = case ch = getc
-             when %r{[~_*$?!@/\\;,=:<>".]}
-               str << ch
-               Token(TkGVAR, str)
-
-             when "-"
-               str << "-" << getc
-               Token(TkGVAR, str)
-
-             when "&", "`", "'", "+"
-               str << ch
-               Token(TkBACK_REF, str)
-
-             when /[1-9]/
-               str << ch
-               while (ch = getc) =~ /[0-9]/
-                 str << ch
-               end
-               ungetc
-               Token(TkNTH_REF)
-             when /\w/
-               ungetc
-               ungetc
-               return identify_identifier
-             else
-               ungetc
-               Token("$")
-             end
-        tk.set_text(str)
-      end
-
-      def identify_identifier
-        token = ""
-        token.concat getc if peek(0) =~ /[$@]/
-        token.concat getc if peek(0) == "@"
-
-        while (ch = getc) =~ /\w|_/
-          print ":", ch, ":" if RubyLex.debug?
-          token.concat ch
-        end
-        ungetc
-
-        if ch == "!" || ch == "?"
-          token.concat getc
-        end
-        # fix token
-
-        # $stderr.puts "identifier - #{token}, state = #@lex_state"
-
-        case token
-        when /^\$/
-          return Token(TkGVAR, token).set_text(token)
-        when /^\@/
-          @lex_state = EXPR_END
-          return Token(TkIVAR, token).set_text(token)
-        end
-
-        if @lex_state != EXPR_DOT
-          print token, "\n" if RubyLex.debug?
-
-          token_c, *trans = TkReading2Token[token]
-          if token_c
-            # reserved word?
-
-            if @lex_state != EXPR_BEG &&
-               @lex_state != EXPR_FNAME &&
-               trans[1]
-              # modifiers
-              token_c = TkSymbol2Token[trans[1]]
-              @lex_state = trans[0]
-            else
-              if @lex_state != EXPR_FNAME
-                if ENINDENT_CLAUSE.include?(token)
-                  @indent += 1
-
-                  if ACCEPTS_COLON.include?(token)
-                    @colonblock_seen = true
-                  else
-                    @colonblock_seen = false
-                  end
-                elsif DEINDENT_CLAUSE.include?(token)
-                  @indent -= 1
-                  @colonblock_seen = false
-                end
-                @lex_state = trans[0]
-              else
-                @lex_state = EXPR_END
-              end
-            end
-            return Token(token_c, token).set_text(token)
-          end
-        end
-
-        if @lex_state == EXPR_FNAME
-          @lex_state = EXPR_END
-          if peek(0) == '='
-            token.concat getc
-          end
-        elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
-          @lex_state = EXPR_ARG
-        else
-          @lex_state = EXPR_END
-        end
-
-        if token[0, 1] =~ /[A-Z]/
-          return Token(TkCONSTANT, token).set_text(token)
-        elsif token[token.size - 1, 1] =~ /[!?]/
-          return Token(TkFID, token).set_text(token)
-        else
-          return Token(TkIDENTIFIER, token).set_text(token)
-        end
-      end
-
-      def identify_here_document
-        ch = getc
-        if ch == "-"
-          ch = getc
-          indent = true
-        end
-        if /['"`]/ =~ ch # '
-          lt = ch
-          quoted = ""
-          while (c = getc) && c != lt
-            quoted.concat c
-          end
-        else
-          lt = '"'
-          quoted = ch.dup
-          while (c = getc) && c =~ /\w/
-            quoted.concat c
-          end
-          ungetc
-        end
-
-        ltback, @ltype = @ltype, lt
-        reserve = String.new
-
-        while ch = getc
-          reserve << ch
-          if ch == "\\" #"
-            ch = getc
-            reserve << ch
-          elsif ch == "\n"
-            break
-          end
-        end
-
-        str = String.new
-        while (l = gets)
-          l.chomp!
-          l.strip! if indent
-          break if l == quoted
-          str << l.chomp << "\n"
-        end
-
-        @reader.divert_read_from(reserve)
-
-        @ltype = ltback
-        @lex_state = EXPR_END
-        Token(Ltype2Token[lt], str).set_text(str.dump)
-      end
-
-      def identify_quotation(initial_char)
-        ch = getc
-        if lt = PERCENT_LTYPE[ch]
-          initial_char += ch
-          ch = getc
-        elsif ch =~ /\W/
-          lt = "\""
-        else
-          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
-        end
-        # if ch !~ /\W/
-        #   ungetc
-        #   next
-        # end
-        # @ltype = lt
-        @quoted = ch unless @quoted = PERCENT_PAREN[ch]
-        identify_string(lt, @quoted, ch, initial_char) if lt
-      end
-
-      def identify_number(start)
-        str = start.dup
-
-        if start == "+" || start == "-" || start == ""
-          start = getc
-          str << start
-        end
-
-        @lex_state = EXPR_END
-
-        if start == "0"
-          if peek(0) == "x"
-            ch = getc
-            str << ch
-            match = /[0-9a-f_]/
-          else
-            match = /[0-7_]/
-          end
-          while ch = getc
-            if ch !~ match
-              ungetc
-              break
-            else
-              str << ch
-            end
-          end
-          return Token(TkINTEGER).set_text(str)
-        end
-
-        type = TkINTEGER
-        allow_point = true
-        allow_e = true
-        while ch = getc
-          case ch
-          when /[0-9_]/
-            str << ch
-
-          when allow_point && "."
-            type = TkFLOAT
-            if peek(0) !~ /[0-9]/
-              ungetc
-              break
-            end
-            str << ch
-            allow_point = false
-
-          when allow_e && "e", allow_e && "E"
-            str << ch
-            type = TkFLOAT
-            if peek(0) =~ /[+-]/
-              str << getc
-            end
-            allow_e = false
-            allow_point = false
-          else
-            ungetc
-            break
-          end
-        end
-        Token(type).set_text(str)
-      end
-
-      def identify_string(ltype, quoted = ltype, opener = nil, initial_char = nil)
-        @ltype = ltype
-        @quoted = quoted
-        subtype = nil
-
-        str = String.new
-        str << initial_char if initial_char
-        str << (opener || quoted)
-
-        nest = 0
-        begin
-          while ch = getc
-            str << ch
-            if @quoted == ch
-              if nest == 0
-                break
-              else
-                nest -= 1
-              end
-            elsif opener == ch
-              nest += 1
-            elsif @ltype != "'" && @ltype != "]" && ch == "#"
-              ch = getc
-              if ch == "{"
-                subtype = true
-                str << ch << skip_inner_expression
-              else
-                ungetc(ch)
-              end
-            elsif ch == '\\' #'
-              str << read_escape
-            end
-          end
-          if @ltype == "/"
-            if peek(0) =~ /i|o|n|e|s/
-              str << getc
-            end
-          end
-          if subtype
-            Token(DLtype2Token[ltype], str)
-          else
-            Token(Ltype2Token[ltype], str)
-          end.set_text(str)
-        ensure
-          @ltype = nil
-          @quoted = nil
-          @lex_state = EXPR_END
-        end
-      end
-
-      def skip_inner_expression
-        res = String.new
-        nest = 0
-        while (ch = getc)
-          res << ch
-          if ch == '}'
-            break if nest == 0
-            nest -= 1
-          elsif ch == '{'
-            nest += 1
-          end
-        end
-        res
-      end
-
-      def identify_comment
-        @ltype = "#"
-        comment = String.new("#")
-        while ch = getc
-          if ch == "\\"
-            ch = getc
-            if ch == "\n"
-              ch = " "
-            else
-              comment << "\\"
-            end
-          else
-            if ch == "\n"
-              @ltype = nil
-              ungetc
-              break
-            end
-          end
-          comment << ch
-        end
-        Token(TkCOMMENT).set_text(comment)
-      end
-
-      def read_escape
-        res = String.new
-        case ch = getc
-        when /[0-7]/
-          ungetc ch
-          3.times do
-          case ch = getc
-          when /[0-7]/
-          when nil
-            break
-          else
-            ungetc
-            break
-          end
-          res << ch
-          end
-
-        when "x"
-          res << ch
-          2.times do
-          case ch = getc
-          when /[0-9a-fA-F]/
-          when nil
-            break
-          else
-            ungetc
-            break
-          end
-            res << ch
-          end
-
-        when "M"
-          res << ch
-          if (ch = getc) != '-'
-            ungetc
-          else
-            res << ch
-            if (ch = getc) == "\\" #"
-              res << ch
-              res << read_escape
-            else
-              res << ch
-            end
-          end
-
-        when "C", "c" #, "^"
-          res << ch
-          if ch == "C" && (ch = getc) != "-"
-            ungetc
-          else
-            res << ch
-            if (ch = getc) == "\\" #"
-              res << ch
-              res << read_escape
-            else
-              res << ch
-            end
-          end
-        else
-          res << ch
-        end
-        res
-      end
-    end
-  end
-end
+require "e2mmap"
+require "irb/slex"
+
+module YARD
+  module Parser::Ruby::Legacy
+    # Legacy lexical tokenizer module.
+    module RubyToken
+      EXPR_BEG   = :EXPR_BEG
+      EXPR_MID   = :EXPR_MID
+      EXPR_END   = :EXPR_END
+      EXPR_ARG   = :EXPR_ARG
+      EXPR_FNAME = :EXPR_FNAME
+      EXPR_DOT   = :EXPR_DOT
+      EXPR_CLASS = :EXPR_CLASS
+
+      # Represents a token in the Ruby lexer
+      class Token
+        # @return [Integer] the line number in the file/stream the token is
+        #   located.
+        attr_reader :line_no
+
+        # @return [Integer] the character number in the file/stream the token
+        #   is located.
+        attr_reader :char_no
+
+        # @return [String] the token text value
+        attr_reader :text
+
+        # @return [Symbol] the lexical state at the token
+        attr_accessor :lex_state
+
+        # @private
+        NO_TEXT = "??".freeze
+
+        # Creates a new Token object
+        # @param [Integer] line_no the line number to initialize the token to
+        # @param [Integer] char_no the char number to initialize the token to
+        def initialize(line_no, char_no)
+          @line_no = line_no
+          @char_no = char_no
+          @text    = NO_TEXT
+        end
+
+        # Chainable way to sets the text attribute
+        #
+        # @param [String] text the new text
+        # @return [Token] this token object
+        def set_text(text)
+          @text = text
+          self
+        end
+      end
+
+      # Represents a block
+      class TkBlockContents < Token
+        def text; '...' end
+      end
+
+      # Represents an end statement
+      class TkStatementEnd < Token
+        def text; '' end
+      end
+
+      class TkNode < Token
+        attr :node
+      end
+
+      # Represents whitespace
+      class TkWhitespace < Token
+      end
+
+      # Represents a Ruby identifier
+      class TkId < Token
+        def initialize(line_no, char_no, name)
+          super(line_no, char_no)
+          @name = name
+        end
+        attr :name
+      end
+
+      # Represents a Ruby keyword
+      class TkKW < TkId
+      end
+
+      # Represents a Ruby value
+      class TkVal < Token
+        def initialize(line_no, char_no, value = nil)
+          super(line_no, char_no)
+          set_text(value)
+        end
+      end
+
+      class TkOp < Token
+        def name
+          self.class.op_name
+        end
+      end
+
+      class TkOPASGN < TkOp
+        def initialize(line_no, char_no, op)
+          super(line_no, char_no)
+          op = TkReading2Token[op] unless op.is_a?(Symbol)
+          @op = op
+        end
+        attr :op
+      end
+
+      class TkUnknownChar < Token
+        def initialize(line_no, char_no, _id)
+          super(line_no, char_no)
+          @name = char_no > 255 ? '?' : char_no.chr
+        end
+        attr :name
+      end
+
+      class TkError < Token
+      end
+
+      # @private
+      def set_token_position(line, char)
+        @prev_line_no = line
+        @prev_char_no = char
+      end
+
+      # @private
+      def Token(token, value = nil) # rubocop:disable Style/MethodName
+        tk = nil
+        case token
+        when String, Symbol
+          source = token.is_a?(String) ? TkReading2Token : TkSymbol2Token
+          if (tk = source[token]).nil?
+            IRB.fail TkReading2TokenNoKey, token
+          end
+          tk = Token(tk[0], value)
+        else
+          if token
+            tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
+                   token.new(@prev_line_no, @prev_char_no)
+                 else
+                   token.new(@prev_line_no, @prev_char_no, value)
+                 end
+          end
+        end
+        tk
+      end
+
+      # @private
+      TokenDefinitions = [
+        [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
+        [:TkMODULE,     TkKW,  "module", EXPR_BEG],
+        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
+        [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
+        [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
+        [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
+        [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
+        [:TkEND,        TkKW,  "end",    EXPR_END],
+        [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
+        [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
+        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
+        [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
+        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
+        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
+        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
+        [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
+        [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
+        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
+        [:TkBREAK,      TkKW,  "break",  EXPR_END],
+        [:TkNEXT,       TkKW,  "next",   EXPR_END],
+        [:TkREDO,       TkKW,  "redo",   EXPR_END],
+        [:TkRETRY,      TkKW,  "retry",  EXPR_END],
+        [:TkIN,         TkKW,  "in",     EXPR_BEG],
+        [:TkDO,         TkKW,  "do",     EXPR_BEG],
+        [:TkRETURN,     TkKW,  "return", EXPR_MID],
+        [:TkYIELD,      TkKW,  "yield",  EXPR_END],
+        [:TkSUPER,      TkKW,  "super",  EXPR_END],
+        [:TkSELF,       TkKW,  "self",   EXPR_END],
+        [:TkNIL,        TkKW,  "nil",    EXPR_END],
+        [:TkTRUE,       TkKW,  "true",   EXPR_END],
+        [:TkFALSE,      TkKW,  "false",  EXPR_END],
+        [:TkAND,        TkKW,  "and",    EXPR_BEG],
+        [:TkOR,         TkKW,  "or",     EXPR_BEG],
+        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
+        [:TkIF_MOD,     TkKW],
+        [:TkUNLESS_MOD, TkKW],
+        [:TkWHILE_MOD,  TkKW],
+        [:TkUNTIL_MOD,  TkKW],
+        [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
+        [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
+        [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
+        [:TklEND,       TkKW,  "END",      EXPR_END],
+        [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
+        [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
+        [:TkIDENTIFIER, TkId],
+        [:TkFID,        TkId],
+        [:TkGVAR,       TkId],
+        [:TkIVAR,       TkId],
+        [:TkCONSTANT,   TkId],
+        [:TkINTEGER,    TkVal],
+        [:TkFLOAT,      TkVal],
+        [:TkSYMBOL,     TkVal],
+        [:TkLABEL,      TkVal],
+        [:TkSTRING,     TkVal],
+        [:TkXSTRING,    TkVal],
+        [:TkREGEXP,     TkVal],
+        [:TkCOMMENT,    TkVal],
+        [:TkDSTRING,    TkNode],
+        [:TkDXSTRING,   TkNode],
+        [:TkDREGEXP,    TkNode],
+        [:TkNTH_REF,    TkId],
+        [:TkBACK_REF,   TkId],
+        [:TkUPLUS,      TkOp,   "+@"],
+        [:TkUMINUS,     TkOp,   "-@"],
+        [:TkPOW,        TkOp,   "**"],
+        [:TkCMP,        TkOp,   "<=>"],
+        [:TkEQ,         TkOp,   "=="],
+        [:TkEQQ,        TkOp,   "==="],
+        [:TkNEQ,        TkOp,   "!="],
+        [:TkGEQ,        TkOp,   ">="],
+        [:TkLEQ,        TkOp,   "<="],
+        [:TkANDOP,      TkOp,   "&&"],
+        [:TkOROP,       TkOp,   "||"],
+        [:TkMATCH,      TkOp,   "=~"],
+        [:TkNMATCH,     TkOp,   "!~"],
+        [:TkDOT2,       TkOp,   ".."],
+        [:TkDOT3,       TkOp,   "..."],
+        [:TkAREF,       TkOp,   "[]"],
+        [:TkASET,       TkOp,   "[]="],
+        [:TkLSHFT,      TkOp,   "<<"],
+        [:TkRSHFT,      TkOp,   ">>"],
+        [:TkCOLON2,     TkOp],
+        [:TkCOLON3,     TkOp],
+        [:OPASGN,       TkOp], # +=, -=  etc. #
+        [:TkASSOC,      TkOp,   "=>"],
+        [:TkQUESTION,   TkOp,   "?"], #?
+        [:TkCOLON,      TkOp,   ":"], #:
+        [:TkSTAR],            # *arg
+        [:TkAMPER],           # &arg #
+        [:TkSYMBEG,     TkId],
+        [:TkGT,         TkOp,   ">"],
+        [:TkLT,         TkOp,   "<"],
+        [:TkPLUS,       TkOp,   "+"],
+        [:TkMINUS,      TkOp,   "-"],
+        [:TkMULT,       TkOp,   "*"],
+        [:TkDIV,        TkOp,   "/"],
+        [:TkMOD,        TkOp,   "%"],
+        [:TkBITOR,      TkOp,   "|"],
+        [:TkBITXOR,     TkOp,   "^"],
+        [:TkBITAND,     TkOp,   "&"],
+        [:TkBITNOT,     TkOp,   "~"],
+        [:TkNOTOP,      TkOp,   "!"],
+        [:TkBACKQUOTE,  TkOp,   "`"],
+        [:TkASSIGN,     Token,  "="],
+        [:TkDOT,        Token,  "."],
+        [:TkLPAREN,     Token,  "("],  # (exp)
+        [:TkLBRACK,     Token,  "["],  # [arry]
+        [:TkLBRACE,     Token,  "{"],  # {hash}
+        [:TkRPAREN,     Token,  ")"],
+        [:TkRBRACK,     Token,  "]"],
+        [:TkRBRACE,     Token,  "}"],
+        [:TkCOMMA,      Token,  ","],
+        [:TkSEMICOLON,  Token,  ";"],
+        [:TkSPACE,          TkWhitespace],
+        [:TkNL,             TkWhitespace],
+        [:TkEND_OF_SCRIPT,  TkWhitespace],
+        [:TkBACKSLASH,  TkUnknownChar,  "\\"],
+        [:TkAT,         TkUnknownChar,  "@"],
+        [:TkDOLLAR,     TkUnknownChar,  "\$"]
+      ]
+
+      # { reading => token_class }
+      # { reading => [token_class, *opt] }
+      TkReading2Token = {}
+      TkSymbol2Token = {}
+
+      # @private
+      def self.def_token(token_n, super_token = Token, reading = nil, *opts)
+        token_n = token_n.id2name unless token_n.is_a?(String)
+        if RubyToken.const_defined?(token_n)
+          # IRB.fail AlreadyDefinedToken, token_n
+        end
+
+        token_c = Class.new super_token
+        RubyToken.const_set token_n, token_c
+        # token_c.inspect
+
+        if reading
+          if TkReading2Token[reading]
+            IRB.fail TkReading2TokenDuplicateError, token_n, reading
+          end
+          if opts.empty?
+            TkReading2Token[reading] = [token_c]
+          else
+            TkReading2Token[reading] = [token_c].concat(opts)
+          end
+        end
+        TkSymbol2Token[token_n.intern] = token_c
+
+        if token_c <= TkOp
+          token_c.class_eval %{
+            def self.op_name; "#{reading}"; end
+          }
+        end
+      end
+
+      for defs in TokenDefinitions
+        def_token(*defs)
+      end
+
+      NEWLINE_TOKEN = TkNL.new(0, 0)
+      NEWLINE_TOKEN.set_text("\n")
+    end
+
+    # Lexical analyzer for Ruby source
+    # @private
+    class RubyLex
+      # Read an input stream character by character. We allow for unlimited
+      # ungetting of characters just read.
+      #
+      # We simplify the implementation greatly by reading the entire input
+      # into a buffer initially, and then simply traversing it using
+      # pointers.
+      #
+      # We also have to allow for the <i>here document diversion</i>. This
+      # little gem comes about when the lexer encounters a here
+      # document. At this point we effectively need to split the input
+      # stream into two parts: one to read the body of the here document,
+      # the other to read the rest of the input line where the here
+      # document was initially encountered. For example, we might have
+      #
+      #   do_something(<<-A, <<-B)
+      #     stuff
+      #     for
+      #   A
+      #     stuff
+      #     for
+      #   B
+      #
+      # When the lexer encounters the <<A, it reads until the end of the
+      # line, and keeps it around for later. It then reads the body of the
+      # here document.  Once complete, it needs to read the rest of the
+      # original line, but then skip the here document body.
+      #
+      # @private
+      class BufferedReader
+        attr_reader :line_num
+
+        def initialize(content)
+          if /\t/ =~ content
+            tab_width = 2
+            content = content.split(/\n/).map do |line|
+              1 while line.gsub!(/\t+/) { ' ' * (tab_width * $&.length - $`.length % tab_width) } && $~ #`
+              line
+            end .join("\n")
+          end
+          @content = String.new(content)
+          @content << "\n" unless @content[-1, 1] == "\n"
+          @size      = @content.size
+          @offset    = 0
+          @hwm       = 0
+          @line_num  = 1
+          @read_back_offset = 0
+          @last_newline = 0
+          @newline_pending = false
+        end
+
+        def column
+          @offset - @last_newline
+        end
+
+        def getc
+          return nil if @offset >= @size
+          ch = @content[@offset, 1]
+
+          @offset += 1
+          @hwm = @offset if @hwm < @offset
+
+          if @newline_pending
+            @line_num += 1
+            @last_newline = @offset - 1
+            @newline_pending = false
+          end
+
+          if ch == "\n"
+            @newline_pending = true
+          end
+          ch
+        end
+
+        def getc_already_read
+          getc
+        end
+
+        def ungetc(_ch)
+          raise "unget past beginning of file" if @offset <= 0
+          @offset -= 1
+          if @content[@offset] == ?\n
+            @newline_pending = false
+          end
+        end
+
+        def get_read
+          res = @content[@read_back_offset...@offset]
+          @read_back_offset = @offset
+          res
+        end
+
+        def peek(at)
+          pos = @offset + at
+          if pos >= @size
+            nil
+          else
+            @content[pos, 1]
+          end
+        end
+
+        def peek_equal(str)
+          @content[@offset, str.length] == str
+        end
+
+        def divert_read_from(reserve)
+          @content[@offset, 0] = reserve
+          @size = @content.size
+        end
+      end
+
+      # end of nested class BufferedReader
+
+      extend Exception2MessageMapper
+      def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
+      def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
+      def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
+      def_exception(:TkReading2TokenDuplicateError,
+        "key duplicate(token_n='%s', key='%s')")
+      def_exception(:SyntaxError, "%s")
+
+      include RubyToken
+      include IRB
+
+      attr_reader :continue
+      attr_reader :lex_state
+
+      def self.debug?
+        false
+      end
+
+      def initialize(content)
+        lex_init
+
+        @reader = BufferedReader.new(content)
+
+        @exp_line_no = @line_no = 1
+        @base_char_no = 0
+        @indent = 0
+
+        @ltype = nil
+        @quoted = nil
+        @lex_state = EXPR_BEG
+        @space_seen = false
+
+        @continue = false
+        @line = ""
+
+        @skip_space = false
+        @read_auto_clean_up = false
+        @exception_on_syntax_error = true
+
+        @colonblock_seen = false
+      end
+
+      attr_accessor :skip_space
+      attr_accessor :read_auto_clean_up
+      attr_accessor :exception_on_syntax_error
+
+      attr :indent
+
+      # io functions
+      def line_no
+        @reader.line_num
+      end
+
+      def char_no
+        @reader.column
+      end
+
+      def get_read
+        @reader.get_read
+      end
+
+      def getc
+        @reader.getc
+      end
+
+      def getc_of_rests
+        @reader.getc_already_read
+      end
+
+      def gets
+        (c = getc) || return
+        l = ""
+        begin
+          l.concat c unless c == "\r"
+          break if c == "\n"
+        end while c = getc # rubocop:disable Lint/Loop
+        l
+      end
+
+      def ungetc(c = nil)
+        @reader.ungetc(c)
+      end
+
+      def peek_equal?(str)
+        @reader.peek_equal(str)
+      end
+
+      def peek(i = 0)
+        @reader.peek(i)
+      end
+
+      def lex
+        catch(:eof) do
+          until ((tk = token).is_a?(TkNL) || tk.is_a?(TkEND_OF_SCRIPT)) &&
+                !@continue ||
+                tk.nil?
+          end
+          line = get_read
+
+          if line == "" && tk.is_a?(TkEND_OF_SCRIPT) || tk.nil?
+            nil
+          else
+            line
+          end
+        end
+      end
+
+      def token
+        set_token_position(line_no, char_no)
+        catch(:eof) do
+          begin
+            begin
+              tk = @OP.match(self)
+              @space_seen = tk.is_a?(TkSPACE)
+            rescue SyntaxError
+              abort if @exception_on_syntax_error
+              tk = TkError.new(line_no, char_no)
+            end
+          end while @skip_space && tk.is_a?(TkSPACE)
+          if @read_auto_clean_up
+            get_read
+          end
+          # throw :eof unless tk
+          p tk if $DEBUG
+          tk.lex_state = lex_state if tk
+          tk
+        end
+      end
+
+      ENINDENT_CLAUSE = [
+        "case", "class", "def", "do", "for", "if",
+        "module", "unless", "until", "while", "begin"
+      ] #, "when"
+      ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
+      DEINDENT_CLAUSE = ["end"] #, "when"
+
+      PERCENT_LTYPE = {
+        "q" => "\'",
+        "Q" => "\"",
+        "x" => "\`",
+        "r" => "/",
+        "w" => "]",
+        "W" => "]"
+      }
+
+      PERCENT_PAREN = {
+        "{" => "}",
+        "[" => "]",
+        "<" => ">",
+        "(" => ")"
+      }
+
+      Ltype2Token = {
+        "\'" => TkSTRING,
+        "\"" => TkSTRING,
+        "\`" => TkXSTRING,
+        "/" => TkREGEXP,
+        "]" => TkDSTRING
+      }
+      Ltype2Token.default = TkSTRING
+
+      DLtype2Token = {
+        "\"" => TkDSTRING,
+        "\`" => TkDXSTRING,
+        "/" => TkDREGEXP
+      }
+
+      def lex_init()
+        @OP = SLex.new
+        @OP.def_rules("\0", "\004", "\032") do |chars, _io|
+          Token(TkEND_OF_SCRIPT).set_text(chars)
+        end
+
+        @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, _io|
+          @space_seen = true
+          while (ch = getc) =~ /[ \t\f\r\13]/
+            chars << ch
+          end
+          ungetc
+          Token(TkSPACE).set_text(chars)
+        end
+
+        @OP.def_rule("#") do |_op, _io|
+          identify_comment
+        end
+
+        @OP.def_rule("=begin", proc { @prev_char_no == 0 && peek(0) =~ /\s/ }) do |op, _io|
+          str = String.new(op)
+          @ltype = "="
+
+          begin
+            line = String.new
+            begin
+              ch = getc
+              line << ch
+            end until ch == "\n"
+            str << line
+          end until line =~ /^=end/
+
+          ungetc
+
+          @ltype = nil
+
+          if str =~ /\A=begin\s+rdoc/i
+            str.sub!(/\A=begin.*\n/, '')
+            str.sub!(/^=end.*/m, '')
+            Token(TkCOMMENT).set_text(str)
+          else
+            Token(TkCOMMENT).set_text(str)
+          end
+        end
+
+        @OP.def_rule("\n") do
+          print "\\n\n" if RubyLex.debug?
+          @colonblock_seen = false
+          case @lex_state
+          when EXPR_BEG, EXPR_FNAME, EXPR_DOT
+            @continue = true
+          else
+            @continue = false
+            @lex_state = EXPR_BEG
+          end
+          Token(TkNL).set_text("\n")
+        end
+
+        @OP.def_rules("*", "**",
+          "!", "!=", "!~",
+          "=", "==", "===",
+          "=~", "<=>",
+          "<", "<=",
+          ">", ">=", ">>") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rules("<<") do |op, _io|
+          tk = nil
+          if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
+             (@lex_state != EXPR_ARG || @space_seen)
+            c = peek(0)
+            tk = identify_here_document if /[-\w\"\'\`]/ =~ c
+          end
+          if !tk
+            @lex_state = EXPR_BEG
+            tk = Token(op).set_text(op)
+          end
+          tk
+        end
+
+        @OP.def_rules("'", '"') do |op, _io|
+          identify_string(op)
+        end
+
+        @OP.def_rules("`") do |op, _io|
+          if @lex_state == EXPR_FNAME
+            Token(op).set_text(op)
+          else
+            identify_string(op)
+          end
+        end
+
+        @OP.def_rules('?') do |op, _io|
+          if @lex_state == EXPR_END
+            @lex_state = EXPR_BEG
+            Token(TkQUESTION).set_text(op)
+          else
+            ch = getc
+            if @lex_state == EXPR_ARG && ch !~ /\s/
+              ungetc
+              @lex_state = EXPR_BEG
+              Token(TkQUESTION).set_text(op)
+            else
+              str = String.new(op)
+              str << ch
+              if ch == '\\' #'
+                str << read_escape
+              end
+              @lex_state = EXPR_END
+              Token(TkINTEGER).set_text(str)
+            end
+          end
+        end
+
+        @OP.def_rules("&", "&&", "|", "||") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rules("+=", "-=", "*=", "**=",
+          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, _io|
+          @lex_state = EXPR_BEG
+          op =~ /^(.*)=$/
+          Token(TkOPASGN, $1).set_text(op)
+        end
+
+        @OP.def_rule("+@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
+          Token(TkUPLUS).set_text(op)
+        end
+
+        @OP.def_rule("-@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
+          Token(TkUMINUS).set_text(op)
+        end
+
+        @OP.def_rules("+", "-") do |op, _io|
+          catch(:RET) do
+            if @lex_state == EXPR_ARG
+              if @space_seen && peek(0) =~ /[0-9]/
+                throw :RET, identify_number(op)
+              else
+                @lex_state = EXPR_BEG
+              end
+            elsif @lex_state != EXPR_END && peek(0) =~ /[0-9]/
+              throw :RET, identify_number(op)
+            else
+              @lex_state = EXPR_BEG
+            end
+            Token(op).set_text(op)
+          end
+        end
+
+        @OP.def_rule(".") do
+          @lex_state = EXPR_BEG
+          if peek(0) =~ /[0-9]/
+            ungetc
+            identify_number("")
+          else
+            # for obj.if
+            @lex_state = EXPR_DOT
+            Token(TkDOT).set_text(".")
+          end
+        end
+
+        @OP.def_rules("..", "...") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        lex_int2
+      end
+
+      def lex_int2
+        @OP.def_rules("]", "}", ")") do |op, _io|
+          @lex_state = EXPR_END
+          @indent -= 1
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rule(":") do
+          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON)
+          else
+            @lex_state = EXPR_FNAME
+            tk = Token(TkSYMBEG)
+          end
+          tk.set_text(":")
+        end
+
+        @OP.def_rule("::") do
+          # p @lex_state.id2name, @space_seen
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_ARG && @space_seen
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON3)
+          else
+            @lex_state = EXPR_DOT
+            tk = Token(TkCOLON2)
+          end
+          tk.set_text("::")
+        end
+
+        @OP.def_rule("/") do |op, _io|
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            identify_string(op)
+          elsif peek(0) == '='
+            getc
+            @lex_state = EXPR_BEG
+            Token(TkOPASGN, :/).set_text("/=") #")
+          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
+            identify_string(op)
+          else
+            @lex_state = EXPR_BEG
+            Token("/").set_text(op)
+          end
+        end
+
+        @OP.def_rules("^") do
+          @lex_state = EXPR_BEG
+          Token("^").set_text("^")
+        end
+
+        # @OP.def_rules("^=") do
+        #   @lex_state = EXPR_BEG
+        #   Token(TkOPASGN, :^)
+        # end
+
+        @OP.def_rules(",", ";") do |op, _io|
+          @colonblock_seen = false
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rule("~") do
+          @lex_state = EXPR_BEG
+          Token("~").set_text("~")
+        end
+
+        @OP.def_rule("~@", proc { @lex_state = EXPR_FNAME }) do
+          @lex_state = EXPR_BEG
+          Token("~").set_text("~@")
+        end
+
+        @OP.def_rule("(") do
+          @indent += 1
+            # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            #  @lex_state = EXPR_BEG
+            #  tk = Token(TkfLPAREN)
+            # else
+            @lex_state = EXPR_BEG
+            tk = Token(TkLPAREN)
+          # end
+          tk.set_text("(")
+        end
+
+        @OP.def_rule("[]", proc { @lex_state == EXPR_FNAME }) do
+          Token("[]").set_text("[]")
+        end
+
+        @OP.def_rule("[]=", proc { @lex_state == EXPR_FNAME }) do
+          Token("[]=").set_text("[]=")
+        end
+
+        @OP.def_rule("[") do
+          @indent += 1
+          # if @lex_state == EXPR_FNAME
+          #   t = Token(TkfLBRACK)
+          # else
+          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+          #     t = Token(TkLBRACK)
+          #   elsif @lex_state == EXPR_ARG && @space_seen
+          #   else
+          #     t = Token(TkfLBRACK)
+          #   end
+          # end
+          t = Token(TkLBRACK)
+          @lex_state = EXPR_BEG
+          t.set_text("[")
+        end
+
+        @OP.def_rule("{") do
+          @indent += 1
+          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
+          #   t = Token(TkLBRACE)
+          # else
+          #   t = Token(TkfLBRACE)
+          # end
+          t = Token(TkLBRACE)
+          @lex_state = EXPR_BEG
+          t.set_text("{")
+        end
+
+        @OP.def_rule('\\') do #'
+          if getc == "\n"
+            @space_seen = true
+            @continue = true
+            Token(TkSPACE).set_text("\\\n")
+          else
+            ungetc
+            Token("\\").set_text("\\") #"
+          end
+        end
+
+        @OP.def_rule('%') do |_op, _io|
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            identify_quotation('%')
+          elsif peek(0) == '='
+            getc
+            Token(TkOPASGN, "%").set_text("%=")
+          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
+            identify_quotation('%')
+          else
+            @lex_state = EXPR_BEG
+            Token("%").set_text("%")
+          end
+        end
+
+        @OP.def_rule('$') do #'
+          identify_gvar
+        end
+
+        @OP.def_rule('@') do
+          if peek(0) =~ /[@\w]/
+            ungetc
+            identify_identifier
+          else
+            Token("@").set_text("@")
+          end
+        end
+
+        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
+        #   |op, io|
+        #   @indent += 1
+        #   @lex_state = EXPR_FNAME
+        # # @lex_state = EXPR_END
+        # # until @rests[0] == "\n" or @rests[0] == ";"
+        # #   rests.shift
+        # # end
+        # end
+
+        @OP.def_rule("__END__", proc { @prev_char_no == 0 && peek(0) =~ /[\r\n]/ }) do
+          throw :eof
+        end
+
+        @OP.def_rule("") do |op, io|
+          printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
+          if peek(0) =~ /[0-9]/
+            t = identify_number("")
+          elsif peek(0) =~ /[\w]/
+            t = identify_identifier
+          end
+          printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
+          t
+        end
+
+        p @OP if RubyLex.debug?
+      end
+
+      def identify_gvar
+        @lex_state = EXPR_END
+        str = String.new("$")
+
+        tk = case ch = getc
+             when %r{[~_*$?!@/\\;,=:<>".]}
+               str << ch
+               Token(TkGVAR, str)
+
+             when "-"
+               str << "-" << getc
+               Token(TkGVAR, str)
+
+             when "&", "`", "'", "+"
+               str << ch
+               Token(TkBACK_REF, str)
+
+             when /[1-9]/
+               str << ch
+               while (ch = getc) =~ /[0-9]/
+                 str << ch
+               end
+               ungetc
+               Token(TkNTH_REF)
+             when /\w/
+               ungetc
+               ungetc
+               return identify_identifier
+             else
+               ungetc
+               Token("$")
+             end
+        tk.set_text(str)
+      end
+
+      def identify_identifier
+        token = ""
+        token.concat getc if peek(0) =~ /[$@]/
+        token.concat getc if peek(0) == "@"
+
+        while (ch = getc) =~ /\w|_/
+          print ":", ch, ":" if RubyLex.debug?
+          token.concat ch
+        end
+        ungetc
+
+        if ch == "!" || ch == "?"
+          token.concat getc
+        end
+        # fix token
+
+        # $stderr.puts "identifier - #{token}, state = #@lex_state"
+
+        case token
+        when /^\$/
+          return Token(TkGVAR, token).set_text(token)
+        when /^\@/
+          @lex_state = EXPR_END
+          return Token(TkIVAR, token).set_text(token)
+        end
+
+        if @lex_state != EXPR_DOT
+          print token, "\n" if RubyLex.debug?
+
+          token_c, *trans = TkReading2Token[token]
+          if token_c
+            # reserved word?
+
+            if @lex_state != EXPR_BEG &&
+               @lex_state != EXPR_FNAME &&
+               trans[1]
+              # modifiers
+              token_c = TkSymbol2Token[trans[1]]
+              @lex_state = trans[0]
+            else
+              if @lex_state != EXPR_FNAME
+                if ENINDENT_CLAUSE.include?(token)
+                  @indent += 1
+
+                  if ACCEPTS_COLON.include?(token)
+                    @colonblock_seen = true
+                  else
+                    @colonblock_seen = false
+                  end
+                elsif DEINDENT_CLAUSE.include?(token)
+                  @indent -= 1
+                  @colonblock_seen = false
+                end
+                @lex_state = trans[0]
+              else
+                @lex_state = EXPR_END
+              end
+            end
+            return Token(token_c, token).set_text(token)
+          end
+        end
+
+        if @lex_state == EXPR_FNAME
+          @lex_state = EXPR_END
+          if peek(0) == '='
+            token.concat getc
+          end
+        elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_END
+        end
+
+        if token[0, 1] =~ /[A-Z]/
+          return Token(TkCONSTANT, token).set_text(token)
+        elsif token[token.size - 1, 1] =~ /[!?]/
+          return Token(TkFID, token).set_text(token)
+        else
+          return Token(TkIDENTIFIER, token).set_text(token)
+        end
+      end
+
+      def identify_here_document
+        ch = getc
+        if ch == "-"
+          ch = getc
+          indent = true
+        end
+        if /['"`]/ =~ ch # '
+          lt = ch
+          quoted = ""
+          while (c = getc) && c != lt
+            quoted.concat c
+          end
+        else
+          lt = '"'
+          quoted = ch.dup
+          while (c = getc) && c =~ /\w/
+            quoted.concat c
+          end
+          ungetc
+        end
+
+        ltback, @ltype = @ltype, lt
+        reserve = String.new
+
+        while ch = getc
+          reserve << ch
+          if ch == "\\" #"
+            ch = getc
+            reserve << ch
+          elsif ch == "\n"
+            break
+          end
+        end
+
+        str = String.new
+        while (l = gets)
+          l.chomp!
+          l.strip! if indent
+          break if l == quoted
+          str << l.chomp << "\n"
+        end
+
+        @reader.divert_read_from(reserve)
+
+        @ltype = ltback
+        @lex_state = EXPR_END
+        Token(Ltype2Token[lt], str).set_text(str.dump)
+      end
+
+      def identify_quotation(initial_char)
+        ch = getc
+        if lt = PERCENT_LTYPE[ch]
+          initial_char += ch
+          ch = getc
+        elsif ch =~ /\W/
+          lt = "\""
+        else
+          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
+        end
+        # if ch !~ /\W/
+        #   ungetc
+        #   next
+        # end
+        # @ltype = lt
+        @quoted = ch unless @quoted = PERCENT_PAREN[ch]
+        identify_string(lt, @quoted, ch, initial_char) if lt
+      end
+
+      def identify_number(start)
+        str = start.dup
+
+        if start == "+" || start == "-" || start == ""
+          start = getc
+          str << start
+        end
+
+        @lex_state = EXPR_END
+
+        if start == "0"
+          if peek(0) == "x"
+            ch = getc
+            str << ch
+            match = /[0-9a-f_]/
+          else
+            match = /[0-7_]/
+          end
+          while ch = getc
+            if ch !~ match
+              ungetc
+              break
+            else
+              str << ch
+            end
+          end
+          return Token(TkINTEGER).set_text(str)
+        end
+
+        type = TkINTEGER
+        allow_point = true
+        allow_e = true
+        while ch = getc
+          case ch
+          when /[0-9_]/
+            str << ch
+
+          when allow_point && "."
+            type = TkFLOAT
+            if peek(0) !~ /[0-9]/
+              ungetc
+              break
+            end
+            str << ch
+            allow_point = false
+
+          when allow_e && "e", allow_e && "E"
+            str << ch
+            type = TkFLOAT
+            if peek(0) =~ /[+-]/
+              str << getc
+            end
+            allow_e = false
+            allow_point = false
+          else
+            ungetc
+            break
+          end
+        end
+        Token(type).set_text(str)
+      end
+
+      def identify_string(ltype, quoted = ltype, opener = nil, initial_char = nil)
+        @ltype = ltype
+        @quoted = quoted
+        subtype = nil
+
+        str = String.new
+        str << initial_char if initial_char
+        str << (opener || quoted)
+
+        nest = 0
+        begin
+          while ch = getc
+            str << ch
+            if @quoted == ch
+              if nest == 0
+                break
+              else
+                nest -= 1
+              end
+            elsif opener == ch
+              nest += 1
+            elsif @ltype != "'" && @ltype != "]" && ch == "#"
+              ch = getc
+              if ch == "{"
+                subtype = true
+                str << ch << skip_inner_expression
+              else
+                ungetc(ch)
+              end
+            elsif ch == '\\' #'
+              str << read_escape
+            end
+          end
+          if @ltype == "/"
+            if peek(0) =~ /i|o|n|e|s/
+              str << getc
+            end
+          end
+          if subtype
+            Token(DLtype2Token[ltype], str)
+          else
+            Token(Ltype2Token[ltype], str)
+          end.set_text(str)
+        ensure
+          @ltype = nil
+          @quoted = nil
+          @lex_state = EXPR_END
+        end
+      end
+
+      def skip_inner_expression
+        res = String.new
+        nest = 0
+        while (ch = getc)
+          res << ch
+          if ch == '}'
+            break if nest == 0
+            nest -= 1
+          elsif ch == '{'
+            nest += 1
+          end
+        end
+        res
+      end
+
+      def identify_comment
+        @ltype = "#"
+        comment = String.new("#")
+        while ch = getc
+          if ch == "\\"
+            ch = getc
+            if ch == "\n"
+              ch = " "
+            else
+              comment << "\\"
+            end
+          else
+            if ch == "\n"
+              @ltype = nil
+              ungetc
+              break
+            end
+          end
+          comment << ch
+        end
+        Token(TkCOMMENT).set_text(comment)
+      end
+
+      def read_escape
+        res = String.new
+        case ch = getc
+        when /[0-7]/
+          ungetc ch
+          3.times do
+          case ch = getc
+          when /[0-7]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
+          res << ch
+          end
+
+        when "x"
+          res << ch
+          2.times do
+          case ch = getc
+          when /[0-9a-fA-F]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
+            res << ch
+          end
+
+        when "M"
+          res << ch
+          if (ch = getc) != '-'
+            ungetc
+          else
+            res << ch
+            if (ch = getc) == "\\" #"
+              res << ch
+              res << read_escape
+            else
+              res << ch
+            end
+          end
+
+        when "C", "c" #, "^"
+          res << ch
+          if ch == "C" && (ch = getc) != "-"
+            ungetc
+          else
+            res << ch
+            if (ch = getc) == "\\" #"
+              res << ch
+              res << read_escape
+            else
+              res << ch
+            end
+          end
+        else
+          res << ch
+        end
+        res
+      end
+    end
+  end
+end