etude_for_ruby 0.1.4 → 0.2.3

data/vendor/bundle/ruby/2.4.0/gems/yard-0.9.8/lib/yard/parser/ruby/legacy/ruby_lex.rb

@@ -1,1354 +0,0 @@
-require "e2mmap"
-require "irb/slex"
-
-module YARD
-  module Parser::Ruby::Legacy
-    # Legacy lexical tokenizer module.
-    module RubyToken
-      EXPR_BEG   = :EXPR_BEG
-      EXPR_MID   = :EXPR_MID
-      EXPR_END   = :EXPR_END
-      EXPR_ARG   = :EXPR_ARG
-      EXPR_FNAME = :EXPR_FNAME
-      EXPR_DOT   = :EXPR_DOT
-      EXPR_CLASS = :EXPR_CLASS
-
-      # Represents a token in the Ruby lexer
-      class Token
-        # @return [Integer] the line number in the file/stream the token is
-        #   located.
-        attr_reader :line_no
-
-        # @return [Integer] the character number in the file/stream the token
-        #   is located.
-        attr_reader :char_no
-
-        # @return [String] the token text value
-        attr_reader :text
-
-        # @return [Symbol] the lexical state at the token
-        attr_accessor :lex_state
-
-        # @private
-        NO_TEXT = "??".freeze
-
-        # Creates a new Token object
-        # @param [Integer] line_no the line number to initialize the token to
-        # @param [Integer] char_no the char number to initialize the token to
-        def initialize(line_no, char_no)
-          @line_no = line_no
-          @char_no = char_no
-          @text    = NO_TEXT
-        end
-
-        # Chainable way to sets the text attribute
-        #
-        # @param [String] text the new text
-        # @return [Token] this token object
-        def set_text(text)
-          @text = text
-          self
-        end
-      end
-
-      # Represents a block
-      class TkBlockContents < Token
-        def text; '...' end
-      end
-
-      # Represents an end statement
-      class TkStatementEnd < Token
-        def text; '' end
-      end
-
-      class TkNode < Token
-        attr :node
-      end
-
-      # Represents whitespace
-      class TkWhitespace < Token
-      end
-
-      # Represents a Ruby identifier
-      class TkId < Token
-        def initialize(line_no, char_no, name)
-          super(line_no, char_no)
-          @name = name
-        end
-        attr :name
-      end
-
-      # Represents a Ruby keyword
-      class TkKW < TkId
-      end
-
-      # Represents a Ruby value
-      class TkVal < Token
-        def initialize(line_no, char_no, value = nil)
-          super(line_no, char_no)
-          set_text(value)
-        end
-      end
-
-      class TkOp < Token
-        def name
-          self.class.op_name
-        end
-      end
-
-      class TkOPASGN < TkOp
-        def initialize(line_no, char_no, op)
-          super(line_no, char_no)
-          op = TkReading2Token[op] unless op.is_a?(Symbol)
-          @op = op
-        end
-        attr :op
-      end
-
-      class TkUnknownChar < Token
-        def initialize(line_no, char_no, _id)
-          super(line_no, char_no)
-          @name = char_no > 255 ? '?' : char_no.chr
-        end
-        attr :name
-      end
-
-      class TkError < Token
-      end
-
-      # @private
-      def set_token_position(line, char)
-        @prev_line_no = line
-        @prev_char_no = char
-      end
-
-      # @private
-      def Token(token, value = nil) # rubocop:disable Style/MethodName
-        tk = nil
-        case token
-        when String, Symbol
-          source = token.is_a?(String) ? TkReading2Token : TkSymbol2Token
-          if (tk = source[token]).nil?
-            IRB.fail TkReading2TokenNoKey, token
-          end
-          tk = Token(tk[0], value)
-        else
-          if token
-            tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
-                   token.new(@prev_line_no, @prev_char_no)
-                 else
-                   token.new(@prev_line_no, @prev_char_no, value)
-                 end
-          end
-        end
-        tk
-      end
-
-      # @private
-      TokenDefinitions = [
-        [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
-        [:TkMODULE,     TkKW,  "module", EXPR_BEG],
-        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
-        [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
-        [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
-        [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
-        [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
-        [:TkEND,        TkKW,  "end",    EXPR_END],
-        [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
-        [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
-        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
-        [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
-        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
-        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
-        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
-        [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
-        [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
-        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
-        [:TkBREAK,      TkKW,  "break",  EXPR_END],
-        [:TkNEXT,       TkKW,  "next",   EXPR_END],
-        [:TkREDO,       TkKW,  "redo",   EXPR_END],
-        [:TkRETRY,      TkKW,  "retry",  EXPR_END],
-        [:TkIN,         TkKW,  "in",     EXPR_BEG],
-        [:TkDO,         TkKW,  "do",     EXPR_BEG],
-        [:TkRETURN,     TkKW,  "return", EXPR_MID],
-        [:TkYIELD,      TkKW,  "yield",  EXPR_END],
-        [:TkSUPER,      TkKW,  "super",  EXPR_END],
-        [:TkSELF,       TkKW,  "self",   EXPR_END],
-        [:TkNIL,        TkKW,  "nil",    EXPR_END],
-        [:TkTRUE,       TkKW,  "true",   EXPR_END],
-        [:TkFALSE,      TkKW,  "false",  EXPR_END],
-        [:TkAND,        TkKW,  "and",    EXPR_BEG],
-        [:TkOR,         TkKW,  "or",     EXPR_BEG],
-        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
-        [:TkIF_MOD,     TkKW],
-        [:TkUNLESS_MOD, TkKW],
-        [:TkWHILE_MOD,  TkKW],
-        [:TkUNTIL_MOD,  TkKW],
-        [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
-        [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
-        [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
-        [:TklEND,       TkKW,  "END",      EXPR_END],
-        [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
-        [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
-        [:TkIDENTIFIER, TkId],
-        [:TkFID,        TkId],
-        [:TkGVAR,       TkId],
-        [:TkIVAR,       TkId],
-        [:TkCONSTANT,   TkId],
-        [:TkINTEGER,    TkVal],
-        [:TkFLOAT,      TkVal],
-        [:TkSYMBOL,     TkVal],
-        [:TkLABEL,      TkVal],
-        [:TkSTRING,     TkVal],
-        [:TkXSTRING,    TkVal],
-        [:TkREGEXP,     TkVal],
-        [:TkCOMMENT,    TkVal],
-        [:TkDSTRING,    TkNode],
-        [:TkDXSTRING,   TkNode],
-        [:TkDREGEXP,    TkNode],
-        [:TkNTH_REF,    TkId],
-        [:TkBACK_REF,   TkId],
-        [:TkUPLUS,      TkOp,   "+@"],
-        [:TkUMINUS,     TkOp,   "-@"],
-        [:TkPOW,        TkOp,   "**"],
-        [:TkCMP,        TkOp,   "<=>"],
-        [:TkEQ,         TkOp,   "=="],
-        [:TkEQQ,        TkOp,   "==="],
-        [:TkNEQ,        TkOp,   "!="],
-        [:TkGEQ,        TkOp,   ">="],
-        [:TkLEQ,        TkOp,   "<="],
-        [:TkANDOP,      TkOp,   "&&"],
-        [:TkOROP,       TkOp,   "||"],
-        [:TkMATCH,      TkOp,   "=~"],
-        [:TkNMATCH,     TkOp,   "!~"],
-        [:TkDOT2,       TkOp,   ".."],
-        [:TkDOT3,       TkOp,   "..."],
-        [:TkAREF,       TkOp,   "[]"],
-        [:TkASET,       TkOp,   "[]="],
-        [:TkLSHFT,      TkOp,   "<<"],
-        [:TkRSHFT,      TkOp,   ">>"],
-        [:TkCOLON2,     TkOp],
-        [:TkCOLON3,     TkOp],
-        [:OPASGN,       TkOp], # +=, -=  etc. #
-        [:TkASSOC,      TkOp,   "=>"],
-        [:TkQUESTION,   TkOp,   "?"], #?
-        [:TkCOLON,      TkOp,   ":"], #:
-        [:TkSTAR],            # *arg
-        [:TkAMPER],           # &arg #
-        [:TkSYMBEG,     TkId],
-        [:TkGT,         TkOp,   ">"],
-        [:TkLT,         TkOp,   "<"],
-        [:TkPLUS,       TkOp,   "+"],
-        [:TkMINUS,      TkOp,   "-"],
-        [:TkMULT,       TkOp,   "*"],
-        [:TkDIV,        TkOp,   "/"],
-        [:TkMOD,        TkOp,   "%"],
-        [:TkBITOR,      TkOp,   "|"],
-        [:TkBITXOR,     TkOp,   "^"],
-        [:TkBITAND,     TkOp,   "&"],
-        [:TkBITNOT,     TkOp,   "~"],
-        [:TkNOTOP,      TkOp,   "!"],
-        [:TkBACKQUOTE,  TkOp,   "`"],
-        [:TkASSIGN,     Token,  "="],
-        [:TkDOT,        Token,  "."],
-        [:TkLPAREN,     Token,  "("],  # (exp)
-        [:TkLBRACK,     Token,  "["],  # [arry]
-        [:TkLBRACE,     Token,  "{"],  # {hash}
-        [:TkRPAREN,     Token,  ")"],
-        [:TkRBRACK,     Token,  "]"],
-        [:TkRBRACE,     Token,  "}"],
-        [:TkCOMMA,      Token,  ","],
-        [:TkSEMICOLON,  Token,  ";"],
-        [:TkSPACE,          TkWhitespace],
-        [:TkNL,             TkWhitespace],
-        [:TkEND_OF_SCRIPT,  TkWhitespace],
-        [:TkBACKSLASH,  TkUnknownChar,  "\\"],
-        [:TkAT,         TkUnknownChar,  "@"],
-        [:TkDOLLAR,     TkUnknownChar,  "\$"]
-      ]
-
-      # { reading => token_class }
-      # { reading => [token_class, *opt] }
-      TkReading2Token = {}
-      TkSymbol2Token = {}
-
-      # @private
-      def self.def_token(token_n, super_token = Token, reading = nil, *opts)
-        token_n = token_n.id2name unless token_n.is_a?(String)
-        if RubyToken.const_defined?(token_n)
-          # IRB.fail AlreadyDefinedToken, token_n
-        end
-
-        token_c = Class.new super_token
-        RubyToken.const_set token_n, token_c
-        # token_c.inspect
-
-        if reading
-          if TkReading2Token[reading]
-            IRB.fail TkReading2TokenDuplicateError, token_n, reading
-          end
-          if opts.empty?
-            TkReading2Token[reading] = [token_c]
-          else
-            TkReading2Token[reading] = [token_c].concat(opts)
-          end
-        end
-        TkSymbol2Token[token_n.intern] = token_c
-
-        if token_c <= TkOp
-          token_c.class_eval %{
-            def self.op_name; "#{reading}"; end
-          }
-        end
-      end
-
-      for defs in TokenDefinitions
-        def_token(*defs)
-      end
-
-      NEWLINE_TOKEN = TkNL.new(0, 0)
-      NEWLINE_TOKEN.set_text("\n")
-    end
-
-    # Lexical analyzer for Ruby source
-    # @private
-    class RubyLex
-      # Read an input stream character by character. We allow for unlimited
-      # ungetting of characters just read.
-      #
-      # We simplify the implementation greatly by reading the entire input
-      # into a buffer initially, and then simply traversing it using
-      # pointers.
-      #
-      # We also have to allow for the <i>here document diversion</i>. This
-      # little gem comes about when the lexer encounters a here
-      # document. At this point we effectively need to split the input
-      # stream into two parts: one to read the body of the here document,
-      # the other to read the rest of the input line where the here
-      # document was initially encountered. For example, we might have
-      #
-      #   do_something(<<-A, <<-B)
-      #     stuff
-      #     for
-      #   A
-      #     stuff
-      #     for
-      #   B
-      #
-      # When the lexer encounters the <<A, it reads until the end of the
-      # line, and keeps it around for later. It then reads the body of the
-      # here document.  Once complete, it needs to read the rest of the
-      # original line, but then skip the here document body.
-      #
-      # @private
-      class BufferedReader
-        attr_reader :line_num
-
-        def initialize(content)
-          if /\t/ =~ content
-            tab_width = 2
-            content = content.split(/\n/).map do |line|
-              1 while line.gsub!(/\t+/) { ' ' * (tab_width * $&.length - $`.length % tab_width) } && $~ #`
-              line
-            end .join("\n")
-          end
-          @content = String.new(content)
-          @content << "\n" unless @content[-1, 1] == "\n"
-          @size      = @content.size
-          @offset    = 0
-          @hwm       = 0
-          @line_num  = 1
-          @read_back_offset = 0
-          @last_newline = 0
-          @newline_pending = false
-        end
-
-        def column
-          @offset - @last_newline
-        end
-
-        def getc
-          return nil if @offset >= @size
-          ch = @content[@offset, 1]
-
-          @offset += 1
-          @hwm = @offset if @hwm < @offset
-
-          if @newline_pending
-            @line_num += 1
-            @last_newline = @offset - 1
-            @newline_pending = false
-          end
-
-          if ch == "\n"
-            @newline_pending = true
-          end
-          ch
-        end
-
-        def getc_already_read
-          getc
-        end
-
-        def ungetc(_ch)
-          raise "unget past beginning of file" if @offset <= 0
-          @offset -= 1
-          if @content[@offset] == ?\n
-            @newline_pending = false
-          end
-        end
-
-        def get_read
-          res = @content[@read_back_offset...@offset]
-          @read_back_offset = @offset
-          res
-        end
-
-        def peek(at)
-          pos = @offset + at
-          if pos >= @size
-            nil
-          else
-            @content[pos, 1]
-          end
-        end
-
-        def peek_equal(str)
-          @content[@offset, str.length] == str
-        end
-
-        def divert_read_from(reserve)
-          @content[@offset, 0] = reserve
-          @size = @content.size
-        end
-      end
-
-      # end of nested class BufferedReader
-
-      extend Exception2MessageMapper
-      def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
-      def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
-      def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-      def_exception(:TkReading2TokenDuplicateError,
-        "key duplicate(token_n='%s', key='%s')")
-      def_exception(:SyntaxError, "%s")
-
-      include RubyToken
-      include IRB
-
-      attr_reader :continue
-      attr_reader :lex_state
-
-      def self.debug?
-        false
-      end
-
-      def initialize(content)
-        lex_init
-
-        @reader = BufferedReader.new(content)
-
-        @exp_line_no = @line_no = 1
-        @base_char_no = 0
-        @indent = 0
-
-        @ltype = nil
-        @quoted = nil
-        @lex_state = EXPR_BEG
-        @space_seen = false
-
-        @continue = false
-        @line = ""
-
-        @skip_space = false
-        @read_auto_clean_up = false
-        @exception_on_syntax_error = true
-
-        @colonblock_seen = false
-      end
-
-      attr_accessor :skip_space
-      attr_accessor :read_auto_clean_up
-      attr_accessor :exception_on_syntax_error
-
-      attr :indent
-
-      # io functions
-      def line_no
-        @reader.line_num
-      end
-
-      def char_no
-        @reader.column
-      end
-
-      def get_read
-        @reader.get_read
-      end
-
-      def getc
-        @reader.getc
-      end
-
-      def getc_of_rests
-        @reader.getc_already_read
-      end
-
-      def gets
-        (c = getc) || return
-        l = ""
-        begin
-          l.concat c unless c == "\r"
-          break if c == "\n"
-        end while c = getc # rubocop:disable Lint/Loop
-        l
-      end
-
-      def ungetc(c = nil)
-        @reader.ungetc(c)
-      end
-
-      def peek_equal?(str)
-        @reader.peek_equal(str)
-      end
-
-      def peek(i = 0)
-        @reader.peek(i)
-      end
-
-      def lex
-        catch(:eof) do
-          until ((tk = token).is_a?(TkNL) || tk.is_a?(TkEND_OF_SCRIPT)) &&
-                !@continue ||
-                tk.nil?
-          end
-          line = get_read
-
-          if line == "" && tk.is_a?(TkEND_OF_SCRIPT) || tk.nil?
-            nil
-          else
-            line
-          end
-        end
-      end
-
-      def token
-        set_token_position(line_no, char_no)
-        catch(:eof) do
-          begin
-            begin
-              tk = @OP.match(self)
-              @space_seen = tk.is_a?(TkSPACE)
-            rescue SyntaxError
-              abort if @exception_on_syntax_error
-              tk = TkError.new(line_no, char_no)
-            end
-          end while @skip_space && tk.is_a?(TkSPACE)
-          if @read_auto_clean_up
-            get_read
-          end
-          # throw :eof unless tk
-          p tk if $DEBUG
-          tk.lex_state = lex_state if tk
-          tk
-        end
-      end
-
-      ENINDENT_CLAUSE = [
-        "case", "class", "def", "do", "for", "if",
-        "module", "unless", "until", "while", "begin"
-      ] #, "when"
-      ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
-      DEINDENT_CLAUSE = ["end"] #, "when"
-
-      PERCENT_LTYPE = {
-        "q" => "\'",
-        "Q" => "\"",
-        "x" => "\`",
-        "r" => "/",
-        "w" => "]",
-        "W" => "]"
-      }
-
-      PERCENT_PAREN = {
-        "{" => "}",
-        "[" => "]",
-        "<" => ">",
-        "(" => ")"
-      }
-
-      Ltype2Token = {
-        "\'" => TkSTRING,
-        "\"" => TkSTRING,
-        "\`" => TkXSTRING,
-        "/" => TkREGEXP,
-        "]" => TkDSTRING
-      }
-      Ltype2Token.default = TkSTRING
-
-      DLtype2Token = {
-        "\"" => TkDSTRING,
-        "\`" => TkDXSTRING,
-        "/" => TkDREGEXP
-      }
-
-      def lex_init()
-        @OP = SLex.new
-        @OP.def_rules("\0", "\004", "\032") do |chars, _io|
-          Token(TkEND_OF_SCRIPT).set_text(chars)
-        end
-
-        @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, _io|
-          @space_seen = TRUE
-          while (ch = getc) =~ /[ \t\f\r\13]/
-            chars << ch
-          end
-          ungetc
-          Token(TkSPACE).set_text(chars)
-        end
-
-        @OP.def_rule("#") do |_op, _io|
-          identify_comment
-        end
-
-        @OP.def_rule("=begin", proc { @prev_char_no == 0 && peek(0) =~ /\s/ }) do |op, _io|
-          str = String.new(op)
-          @ltype = "="
-
-          begin
-            line = String.new
-            begin
-              ch = getc
-              line << ch
-            end until ch == "\n"
-            str << line
-          end until line =~ /^=end/
-
-          ungetc
-
-          @ltype = nil
-
-          if str =~ /\A=begin\s+rdoc/i
-            str.sub!(/\A=begin.*\n/, '')
-            str.sub!(/^=end.*/m, '')
-            Token(TkCOMMENT).set_text(str)
-          else
-            Token(TkCOMMENT).set_text(str)
-          end
-        end
-
-        @OP.def_rule("\n") do
-          print "\\n\n" if RubyLex.debug?
-          @colonblock_seen = false
-          case @lex_state
-          when EXPR_BEG, EXPR_FNAME, EXPR_DOT
-            @continue = TRUE
-          else
-            @continue = FALSE
-            @lex_state = EXPR_BEG
-          end
-          Token(TkNL).set_text("\n")
-        end
-
-        @OP.def_rules("*", "**",
-          "!", "!=", "!~",
-          "=", "==", "===",
-          "=~", "<=>",
-          "<", "<=",
-          ">", ">=", ">>") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rules("<<") do |op, _io|
-          tk = nil
-          if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
-             (@lex_state != EXPR_ARG || @space_seen)
-            c = peek(0)
-            tk = identify_here_document if /[-\w\"\'\`]/ =~ c
-          end
-          if !tk
-            @lex_state = EXPR_BEG
-            tk = Token(op).set_text(op)
-          end
-          tk
-        end
-
-        @OP.def_rules("'", '"') do |op, _io|
-          identify_string(op)
-        end
-
-        @OP.def_rules("`") do |op, _io|
-          if @lex_state == EXPR_FNAME
-            Token(op).set_text(op)
-          else
-            identify_string(op)
-          end
-        end
-
-        @OP.def_rules('?') do |op, _io|
-          if @lex_state == EXPR_END
-            @lex_state = EXPR_BEG
-            Token(TkQUESTION).set_text(op)
-          else
-            ch = getc
-            if @lex_state == EXPR_ARG && ch !~ /\s/
-              ungetc
-              @lex_state = EXPR_BEG
-              Token(TkQUESTION).set_text(op)
-            else
-              str = String.new(op)
-              str << ch
-              if ch == '\\' #'
-                str << read_escape
-              end
-              @lex_state = EXPR_END
-              Token(TkINTEGER).set_text(str)
-            end
-          end
-        end
-
-        @OP.def_rules("&", "&&", "|", "||") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rules("+=", "-=", "*=", "**=",
-          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, _io|
-          @lex_state = EXPR_BEG
-          op =~ /^(.*)=$/
-          Token(TkOPASGN, $1).set_text(op)
-        end
-
-        @OP.def_rule("+@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
-          Token(TkUPLUS).set_text(op)
-        end
-
-        @OP.def_rule("-@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
-          Token(TkUMINUS).set_text(op)
-        end
-
-        @OP.def_rules("+", "-") do |op, _io|
-          catch(:RET) do
-            if @lex_state == EXPR_ARG
-              if @space_seen && peek(0) =~ /[0-9]/
-                throw :RET, identify_number(op)
-              else
-                @lex_state = EXPR_BEG
-              end
-            elsif @lex_state != EXPR_END && peek(0) =~ /[0-9]/
-              throw :RET, identify_number(op)
-            else
-              @lex_state = EXPR_BEG
-            end
-            Token(op).set_text(op)
-          end
-        end
-
-        @OP.def_rule(".") do
-          @lex_state = EXPR_BEG
-          if peek(0) =~ /[0-9]/
-            ungetc
-            identify_number("")
-          else
-            # for obj.if
-            @lex_state = EXPR_DOT
-            Token(TkDOT).set_text(".")
-          end
-        end
-
-        @OP.def_rules("..", "...") do |op, _io|
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        lex_int2
-      end
-
-      def lex_int2
-        @OP.def_rules("]", "}", ")") do |op, _io|
-          @lex_state = EXPR_END
-          @indent -= 1
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rule(":") do
-          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
-            @lex_state = EXPR_BEG
-            tk = Token(TkCOLON)
-          else
-            @lex_state = EXPR_FNAME
-            tk = Token(TkSYMBEG)
-          end
-          tk.set_text(":")
-        end
-
-        @OP.def_rule("::") do
-          # p @lex_state.id2name, @space_seen
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_ARG && @space_seen
-            @lex_state = EXPR_BEG
-            tk = Token(TkCOLON3)
-          else
-            @lex_state = EXPR_DOT
-            tk = Token(TkCOLON2)
-          end
-          tk.set_text("::")
-        end
-
-        @OP.def_rule("/") do |op, _io|
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            identify_string(op)
-          elsif peek(0) == '='
-            getc
-            @lex_state = EXPR_BEG
-            Token(TkOPASGN, :/).set_text("/=") #")
-          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
-            identify_string(op)
-          else
-            @lex_state = EXPR_BEG
-            Token("/").set_text(op)
-          end
-        end
-
-        @OP.def_rules("^") do
-          @lex_state = EXPR_BEG
-          Token("^").set_text("^")
-        end
-
-        # @OP.def_rules("^=") do
-        #   @lex_state = EXPR_BEG
-        #   Token(TkOPASGN, :^)
-        # end
-
-        @OP.def_rules(",", ";") do |op, _io|
-          @colonblock_seen = false
-          @lex_state = EXPR_BEG
-          Token(op).set_text(op)
-        end
-
-        @OP.def_rule("~") do
-          @lex_state = EXPR_BEG
-          Token("~").set_text("~")
-        end
-
-        @OP.def_rule("~@", proc { @lex_state = EXPR_FNAME }) do
-          @lex_state = EXPR_BEG
-          Token("~").set_text("~@")
-        end
-
-        @OP.def_rule("(") do
-          @indent += 1
-            # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            #  @lex_state = EXPR_BEG
-            #  tk = Token(TkfLPAREN)
-            # else
-            @lex_state = EXPR_BEG
-            tk = Token(TkLPAREN)
-          # end
-          tk.set_text("(")
-        end
-
-        @OP.def_rule("[]", proc { @lex_state == EXPR_FNAME }) do
-          Token("[]").set_text("[]")
-        end
-
-        @OP.def_rule("[]=", proc { @lex_state == EXPR_FNAME }) do
-          Token("[]=").set_text("[]=")
-        end
-
-        @OP.def_rule("[") do
-          @indent += 1
-          # if @lex_state == EXPR_FNAME
-          #   t = Token(TkfLBRACK)
-          # else
-          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-          #     t = Token(TkLBRACK)
-          #   elsif @lex_state == EXPR_ARG && @space_seen
-          #   else
-          #     t = Token(TkfLBRACK)
-          #   end
-          # end
-          t = Token(TkLBRACK)
-          @lex_state = EXPR_BEG
-          t.set_text("[")
-        end
-
-        @OP.def_rule("{") do
-          @indent += 1
-          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
-          #   t = Token(TkLBRACE)
-          # else
-          #   t = Token(TkfLBRACE)
-          # end
-          t = Token(TkLBRACE)
-          @lex_state = EXPR_BEG
-          t.set_text("{")
-        end
-
-        @OP.def_rule('\\') do #'
-          if getc == "\n"
-            @space_seen = true
-            @continue = true
-            Token(TkSPACE).set_text("\\\n")
-          else
-            ungetc
-            Token("\\").set_text("\\") #"
-          end
-        end
-
-        @OP.def_rule('%') do |_op, _io|
-          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-            identify_quotation('%')
-          elsif peek(0) == '='
-            getc
-            Token(TkOPASGN, "%").set_text("%=")
-          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
-            identify_quotation('%')
-          else
-            @lex_state = EXPR_BEG
-            Token("%").set_text("%")
-          end
-        end
-
-        @OP.def_rule('$') do #'
-          identify_gvar
-        end
-
-        @OP.def_rule('@') do
-          if peek(0) =~ /[@\w]/
-            ungetc
-            identify_identifier
-          else
-            Token("@").set_text("@")
-          end
-        end
-
-        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
-        #   |op, io|
-        #   @indent += 1
-        #   @lex_state = EXPR_FNAME
-        # # @lex_state = EXPR_END
-        # # until @rests[0] == "\n" or @rests[0] == ";"
-        # #   rests.shift
-        # # end
-        # end
-
-        @OP.def_rule("__END__", proc { @prev_char_no == 0 && peek(0) =~ /[\r\n]/ }) do
-          throw :eof
-        end
-
-        @OP.def_rule("") do |op, io|
-          printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
-          if peek(0) =~ /[0-9]/
-            t = identify_number("")
-          elsif peek(0) =~ /[\w]/
-            t = identify_identifier
-          end
-          printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
-          t
-        end
-
-        p @OP if RubyLex.debug?
-      end
-
-      def identify_gvar
-        @lex_state = EXPR_END
-        str = String.new("$")
-
-        tk = case ch = getc
-             when %r{[~_*$?!@/\\;,=:<>".]}
-               str << ch
-               Token(TkGVAR, str)
-
-             when "-"
-               str << "-" << getc
-               Token(TkGVAR, str)
-
-             when "&", "`", "'", "+"
-               str << ch
-               Token(TkBACK_REF, str)
-
-             when /[1-9]/
-               str << ch
-               while (ch = getc) =~ /[0-9]/
-                 str << ch
-               end
-               ungetc
-               Token(TkNTH_REF)
-             when /\w/
-               ungetc
-               ungetc
-               return identify_identifier
-             else
-               ungetc
-               Token("$")
-             end
-        tk.set_text(str)
-      end
-
-      def identify_identifier
-        token = ""
-        token.concat getc if peek(0) =~ /[$@]/
-        token.concat getc if peek(0) == "@"
-
-        while (ch = getc) =~ /\w|_/
-          print ":", ch, ":" if RubyLex.debug?
-          token.concat ch
-        end
-        ungetc
-
-        if ch == "!" || ch == "?"
-          token.concat getc
-        end
-        # fix token
-
-        # $stderr.puts "identifier - #{token}, state = #@lex_state"
-
-        case token
-        when /^\$/
-          return Token(TkGVAR, token).set_text(token)
-        when /^\@/
-          @lex_state = EXPR_END
-          return Token(TkIVAR, token).set_text(token)
-        end
-
-        if @lex_state != EXPR_DOT
-          print token, "\n" if RubyLex.debug?
-
-          token_c, *trans = TkReading2Token[token]
-          if token_c
-            # reserved word?
-
-            if @lex_state != EXPR_BEG &&
-               @lex_state != EXPR_FNAME &&
-               trans[1]
-              # modifiers
-              token_c = TkSymbol2Token[trans[1]]
-              @lex_state = trans[0]
-            else
-              if @lex_state != EXPR_FNAME
-                if ENINDENT_CLAUSE.include?(token)
-                  @indent += 1
-
-                  if ACCEPTS_COLON.include?(token)
-                    @colonblock_seen = true
-                  else
-                    @colonblock_seen = false
-                  end
-                elsif DEINDENT_CLAUSE.include?(token)
-                  @indent -= 1
-                  @colonblock_seen = false
-                end
-                @lex_state = trans[0]
-              else
-                @lex_state = EXPR_END
-              end
-            end
-            return Token(token_c, token).set_text(token)
-          end
-        end
-
-        if @lex_state == EXPR_FNAME
-          @lex_state = EXPR_END
-          if peek(0) == '='
-            token.concat getc
-          end
-        elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
-          @lex_state = EXPR_ARG
-        else
-          @lex_state = EXPR_END
-        end
-
-        if token[0, 1] =~ /[A-Z]/
-          return Token(TkCONSTANT, token).set_text(token)
-        elsif token[token.size - 1, 1] =~ /[!?]/
-          return Token(TkFID, token).set_text(token)
-        else
-          return Token(TkIDENTIFIER, token).set_text(token)
-        end
-      end
-
-      def identify_here_document
-        ch = getc
-        if ch == "-"
-          ch = getc
-          indent = true
-        end
-        if /['"`]/ =~ ch # '
-          lt = ch
-          quoted = ""
-          while (c = getc) && c != lt
-            quoted.concat c
-          end
-        else
-          lt = '"'
-          quoted = ch.dup
-          while (c = getc) && c =~ /\w/
-            quoted.concat c
-          end
-          ungetc
-        end
-
-        ltback, @ltype = @ltype, lt
-        reserve = String.new
-
-        while ch = getc
-          reserve << ch
-          if ch == "\\" #"
-            ch = getc
-            reserve << ch
-          elsif ch == "\n"
-            break
-          end
-        end
-
-        str = String.new
-        while (l = gets)
-          l.chomp!
-          l.strip! if indent
-          break if l == quoted
-          str << l.chomp << "\n"
-        end
-
-        @reader.divert_read_from(reserve)
-
-        @ltype = ltback
-        @lex_state = EXPR_END
-        Token(Ltype2Token[lt], str).set_text(str.dump)
-      end
-
-      def identify_quotation(initial_char)
-        ch = getc
-        if lt = PERCENT_LTYPE[ch]
-          initial_char += ch
-          ch = getc
-        elsif ch =~ /\W/
-          lt = "\""
-        else
-          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
-        end
-        # if ch !~ /\W/
-        #   ungetc
-        #   next
-        # end
-        # @ltype = lt
-        @quoted = ch unless @quoted = PERCENT_PAREN[ch]
-        identify_string(lt, @quoted, ch, initial_char) if lt
-      end
-
-      def identify_number(start)
-        str = start.dup
-
-        if start == "+" || start == "-" || start == ""
-          start = getc
-          str << start
-        end
-
-        @lex_state = EXPR_END
-
-        if start == "0"
-          if peek(0) == "x"
-            ch = getc
-            str << ch
-            match = /[0-9a-f_]/
-          else
-            match = /[0-7_]/
-          end
-          while ch = getc
-            if ch !~ match
-              ungetc
-              break
-            else
-              str << ch
-            end
-          end
-          return Token(TkINTEGER).set_text(str)
-        end
-
-        type = TkINTEGER
-        allow_point = TRUE
-        allow_e = TRUE
-        while ch = getc
-          case ch
-          when /[0-9_]/
-            str << ch
-
-          when allow_point && "."
-            type = TkFLOAT
-            if peek(0) !~ /[0-9]/
-              ungetc
-              break
-            end
-            str << ch
-            allow_point = false
-
-          when allow_e && "e", allow_e && "E"
-            str << ch
-            type = TkFLOAT
-            if peek(0) =~ /[+-]/
-              str << getc
-            end
-            allow_e = false
-            allow_point = false
-          else
-            ungetc
-            break
-          end
-        end
-        Token(type).set_text(str)
-      end
-
-      def identify_string(ltype, quoted = ltype, opener = nil, initial_char = nil)
-        @ltype = ltype
-        @quoted = quoted
-        subtype = nil
-
-        str = String.new
-        str << initial_char if initial_char
-        str << (opener || quoted)
-
-        nest = 0
-        begin
-          while ch = getc
-            str << ch
-            if @quoted == ch
-              if nest == 0
-                break
-              else
-                nest -= 1
-              end
-            elsif opener == ch
-              nest += 1
-            elsif @ltype != "'" && @ltype != "]" && ch == "#"
-              ch = getc
-              if ch == "{"
-                subtype = true
-                str << ch << skip_inner_expression
-              else
-                ungetc(ch)
-              end
-            elsif ch == '\\' #'
-              str << read_escape
-            end
-          end
-          if @ltype == "/"
-            if peek(0) =~ /i|o|n|e|s/
-              str << getc
-            end
-          end
-          if subtype
-            Token(DLtype2Token[ltype], str)
-          else
-            Token(Ltype2Token[ltype], str)
-          end.set_text(str)
-        ensure
-          @ltype = nil
-          @quoted = nil
-          @lex_state = EXPR_END
-        end
-      end
-
-      def skip_inner_expression
-        res = String.new
-        nest = 0
-        while (ch = getc)
-          res << ch
-          if ch == '}'
-            break if nest == 0
-            nest -= 1
-          elsif ch == '{'
-            nest += 1
-          end
-        end
-        res
-      end
-
-      def identify_comment
-        @ltype = "#"
-        comment = String.new("#")
-        while ch = getc
-          if ch == "\\"
-            ch = getc
-            if ch == "\n"
-              ch = " "
-            else
-              comment << "\\"
-            end
-          else
-            if ch == "\n"
-              @ltype = nil
-              ungetc
-              break
-            end
-          end
-          comment << ch
-        end
-        Token(TkCOMMENT).set_text(comment)
-      end
-
-      def read_escape
-        res = String.new
-        case ch = getc
-        when /[0-7]/
-          ungetc ch
-          3.times do
-          case ch = getc
-          when /[0-7]/
-          when nil
-            break
-          else
-            ungetc
-            break
-          end
-          res << ch
-          end
-
-        when "x"
-          res << ch
-          2.times do
-          case ch = getc
-          when /[0-9a-fA-F]/
-          when nil
-            break
-          else
-            ungetc
-            break
-          end
-            res << ch
-          end
-
-        when "M"
-          res << ch
-          if (ch = getc) != '-'
-            ungetc
-          else
-            res << ch
-            if (ch = getc) == "\\" #"
-              res << ch
-              res << read_escape
-            else
-              res << ch
-            end
-          end
-
-        when "C", "c" #, "^"
-          res << ch
-          if ch == "C" && (ch = getc) != "-"
-            ungetc
-          else
-            res << ch
-            if (ch = getc) == "\\" #"
-              res << ch
-              res << read_escape
-            else
-              res << ch
-            end
-          end
-        else
-          res << ch
-        end
-        res
-      end
-    end
-  end
-end