yard 0.2.1 → 0.2.2

data/lib/ruby_lex.rb

@@ -1,1321 +0,0 @@
-require "e2mmap"
-require "irb/slex"
-
-module YARD
-  module RubyToken
-    EXPR_BEG   = :EXPR_BEG
-    EXPR_MID   = :EXPR_MID
-    EXPR_END   = :EXPR_END
-    EXPR_ARG   = :EXPR_ARG
-    EXPR_FNAME = :EXPR_FNAME
-    EXPR_DOT   = :EXPR_DOT
-    EXPR_CLASS = :EXPR_CLASS
-  
-    class Token
-      NO_TEXT = "??".freeze
-      attr :text
-
-      def initialize(line_no, char_no)
-        @line_no = line_no
-        @char_no = char_no
-        @text    = NO_TEXT
-      end
-
-      # Because we're used in contexts that expect to return a token,
-      # we set the text string and then return ourselves
-      def set_text(text)
-        @text = text
-        self
-      end
-
-      attr_reader :line_no, :char_no, :text
-      attr_accessor :lex_state
-    end
-
-    class TkNode < Token
-      attr :node
-    end
-
-    class TkId < Token
-      def initialize(line_no, char_no, name)
-        super(line_no, char_no)
-        @name = name
-      end
-      attr :name
-    end
-
-    class TkKW < TkId
-    end
-
-    class TkVal < Token
-      def initialize(line_no, char_no, value = nil)
-        super(line_no, char_no)
-        set_text(value)
-      end
-    end
-
-    class TkOp < Token
-      def name
-        self.class.op_name
-      end
-    end
-
-    class TkOPASGN < TkOp
-      def initialize(line_no, char_no, op)
-        super(line_no, char_no)
-        op = TkReading2Token[op] unless op.kind_of?(Symbol)
-        @op = op
-      end
-      attr :op
-    end
-
-    class TkUnknownChar < Token
-      def initialize(line_no, char_no, id)
-        super(line_no, char_no)
-        @name = char_no.chr
-      end
-      attr :name
-    end
-
-    class TkError < Token
-    end
-
-    def set_token_position(line, char)
-      @prev_line_no = line
-      @prev_char_no = char
-    end
-
-    def Token(token, value = nil)
-      tk = nil
-      case token
-      when String, Symbol
-        source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
-        if (tk = source[token]).nil?
-        	IRB.fail TkReading2TokenNoKey, token
-        end
-        tk = Token(tk[0], value) 
-      else 
-        tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
-               token.new(@prev_line_no, @prev_char_no)
-             else
-               token.new(@prev_line_no, @prev_char_no, value)
-             end
-      end
-      tk
-    end
-
-    TokenDefinitions = [
-      [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
-      [:TkMODULE,     TkKW,  "module", EXPR_BEG],
-      [:TkDEF,	      TkKW,  "def",    EXPR_FNAME],
-      [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
-      [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
-      [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
-      [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
-      [:TkEND,	      TkKW,  "end",    EXPR_END],
-      [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
-      [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
-      [:TkTHEN,	      TkKW,  "then",   EXPR_BEG],
-      [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
-      [:TkELSE,	      TkKW,  "else",   EXPR_BEG],
-      [:TkCASE,	      TkKW,  "case",   EXPR_BEG],
-      [:TkWHEN,	      TkKW,  "when",   EXPR_BEG],
-      [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
-      [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
-      [:TkFOR,	      TkKW,  "for",    EXPR_BEG],
-      [:TkBREAK,      TkKW,  "break",  EXPR_END],
-      [:TkNEXT,	      TkKW,  "next",   EXPR_END],
-      [:TkREDO,	      TkKW,  "redo",   EXPR_END],
-      [:TkRETRY,      TkKW,  "retry",  EXPR_END],
-      [:TkIN,	        TkKW,  "in",     EXPR_BEG],
-      [:TkDO,	        TkKW,  "do",     EXPR_BEG],
-      [:TkRETURN,     TkKW,  "return", EXPR_MID],
-      [:TkYIELD,      TkKW,  "yield",  EXPR_END],
-      [:TkSUPER,      TkKW,  "super",  EXPR_END],
-      [:TkSELF,	      TkKW,  "self",   EXPR_END],
-      [:TkNIL, 	      TkKW,  "nil",    EXPR_END],
-      [:TkTRUE,	      TkKW,  "true",   EXPR_END],
-      [:TkFALSE,      TkKW,  "false",  EXPR_END],
-      [:TkAND,	      TkKW,  "and",    EXPR_BEG],
-      [:TkOR, 	      TkKW,  "or",     EXPR_BEG],
-      [:TkNOT,	      TkKW,  "not",    EXPR_BEG],
-      [:TkIF_MOD,     TkKW],
-      [:TkUNLESS_MOD, TkKW],
-      [:TkWHILE_MOD,  TkKW],
-      [:TkUNTIL_MOD,  TkKW],
-      [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
-      [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
-      [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
-      [:TklEND,	      TkKW,  "END",      EXPR_END],
-      [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
-      [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
-
-      [:TkIDENTIFIER, TkId],
-      [:TkFID,	      TkId],
-      [:TkGVAR,	      TkId],
-      [:TkIVAR,	      TkId],
-      [:TkCONSTANT,   TkId],
-
-      [:TkINTEGER,    TkVal],
-      [:TkFLOAT,      TkVal],
-      [:TkSTRING,     TkVal],
-      [:TkXSTRING,    TkVal],
-      [:TkREGEXP,     TkVal],
-      [:TkCOMMENT,    TkVal],
-
-      [:TkDSTRING,    TkNode],
-      [:TkDXSTRING,   TkNode],
-      [:TkDREGEXP,    TkNode],
-      [:TkNTH_REF,    TkId],
-      [:TkBACK_REF,   TkId],
-
-      [:TkUPLUS,      TkOp,   "+@"],
-      [:TkUMINUS,     TkOp,   "-@"],
-      [:TkPOW,	      TkOp,   "**"],
-      [:TkCMP,	      TkOp,   "<=>"],
-      [:TkEQ,	        TkOp,   "=="],
-      [:TkEQQ,	      TkOp,   "==="],
-      [:TkNEQ,	      TkOp,   "!="],
-      [:TkGEQ,	      TkOp,   ">="],
-      [:TkLEQ,	      TkOp,   "<="],
-      [:TkANDOP,      TkOp,   "&&"],
-      [:TkOROP,	      TkOp,   "||"],
-      [:TkMATCH,      TkOp,   "=~"],
-      [:TkNMATCH,     TkOp,   "!~"],
-      [:TkDOT2,	      TkOp,   ".."],
-      [:TkDOT3,	      TkOp,   "..."],
-      [:TkAREF,	      TkOp,   "[]"],
-      [:TkASET,	      TkOp,   "[]="],
-      [:TkLSHFT,      TkOp,   "<<"],
-      [:TkRSHFT,      TkOp,   ">>"],
-      [:TkCOLON2,     TkOp],
-      [:TkCOLON3,     TkOp],
-      [:OPASGN,	      TkOp],               # +=, -=  etc. #
-      [:TkASSOC,      TkOp,   "=>"],
-      [:TkQUESTION,   TkOp,   "?"],	 #?
-      [:TkCOLON,      TkOp,   ":"],        #:
-    
-      [:TkfLPAREN],         # func( #
-      [:TkfLBRACK],         # func[ #
-      [:TkfLBRACE],         # func{ #
-      [:TkSTAR],            # *arg
-      [:TkAMPER],           # &arg #
-      [:TkSYMBOL,     TkId],          # :SYMBOL
-      [:TkSYMBEG,     TkId], 
-      [:TkGT,	        TkOp,   ">"],
-      [:TkLT,	        TkOp,   "<"],
-      [:TkPLUS,	      TkOp,   "+"],
-      [:TkMINUS,      TkOp,   "-"],
-      [:TkMULT,	      TkOp,   "*"],
-      [:TkDIV,	      TkOp,   "/"],
-      [:TkMOD,	      TkOp,   "%"],
-      [:TkBITOR,      TkOp,   "|"],
-      [:TkBITXOR,     TkOp,   "^"],
-      [:TkBITAND,     TkOp,   "&"],
-      [:TkBITNOT,     TkOp,   "~"],
-      [:TkNOTOP,      TkOp,   "!"],
-
-      [:TkBACKQUOTE,  TkOp,   "`"],
-
-      [:TkASSIGN,     Token,  "="],
-      [:TkDOT,	      Token,  "."],
-      [:TkLPAREN,     Token,  "("],  #(exp)
-      [:TkLBRACK,     Token,  "["],  #[arry]
-      [:TkLBRACE,     Token,  "{"],  #{hash}
-      [:TkRPAREN,     Token,  ")"],
-      [:TkRBRACK,     Token,  "]"],
-      [:TkRBRACE,     Token,  "}"],
-      [:TkCOMMA,      Token,  ","],
-      [:TkSEMICOLON,  Token,  ";"],
-
-      [:TkRD_COMMENT],
-      [:TkSPACE],
-      [:TkNL],
-      [:TkEND_OF_SCRIPT],
-
-      [:TkBACKSLASH,  TkUnknownChar,  "\\"],
-      [:TkAT,	        TkUnknownChar,  "@"],
-      [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
-    ]
-
-    # {reading => token_class}
-    # {reading => [token_class, *opt]}
-    TkReading2Token = {}
-    TkSymbol2Token = {}
-
-    def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
-      token_n = token_n.id2name unless token_n.kind_of?(String)
-      if RubyToken.const_defined?(token_n)
-        #IRB.fail AlreadyDefinedToken, token_n
-      end
-
-      token_c =  Class.new super_token
-      RubyToken.const_set token_n, token_c
-  #    token_c.inspect
- 
-      if reading
-        if TkReading2Token[reading]
-  	      IRB.fail TkReading2TokenDuplicateError, token_n, reading
-        end
-        if opts.empty?
-  	      TkReading2Token[reading] = [token_c]
-        else
-  	      TkReading2Token[reading] = [token_c].concat(opts)
-        end
-      end
-      TkSymbol2Token[token_n.intern] = token_c
-
-      if token_c <= TkOp
-        token_c.class_eval %{
-          def self.op_name; "#{reading}"; end
-        }
-      end
-    end
-
-    for defs in TokenDefinitions
-      def_token(*defs)
-    end
-
-    NEWLINE_TOKEN = TkNL.new(0,0)
-    NEWLINE_TOKEN.set_text("\n")
-
-  end
-
-
-
-  # Lexical analyzer for Ruby source
-
-  class RubyLex
-
-    ######################################################################
-    #
-    # Read an input stream character by character. We allow for unlimited
-    # ungetting of characters just read.
-    #
-    # We simplify the implementation greatly by reading the entire input
-    # into a buffer initially, and then simply traversing it using
-    # pointers.
-    #
-    # We also have to allow for the <i>here document diversion</i>. This
-    # little gem comes about when the lexer encounters a here
-    # document. At this point we effectively need to split the input
-    # stream into two parts: one to read the body of the here document,
-    # the other to read the rest of the input line where the here
-    # document was initially encountered. For example, we might have
-    #
-    #   do_something(<<-A, <<-B)
-    #     stuff
-    #     for
-    #   A
-    #     stuff
-    #     for
-    #   B
-    #
-    # When the lexer encounters the <<A, it reads until the end of the
-    # line, and keeps it around for later. It then reads the body of the
-    # here document.  Once complete, it needs to read the rest of the
-    # original line, but then skip the here document body.
-    #
-  
-    class BufferedReader
-    
-      attr_reader :line_num
-    
-      def initialize(content)
-        if /\t/ =~ content
-          tab_width = 2
-          content = content.split(/\n/).map do |line|
-            1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)}  && $~ #`
-            line
-          end .join("\n")
-        end
-        @content   = content
-        @content << "\n" unless @content[-1,1] == "\n"
-        @size      = @content.size
-        @offset    = 0
-        @hwm       = 0
-        @line_num  = 1
-        @read_back_offset = 0
-        @last_newline = 0
-        @newline_pending = false
-      end
-    
-      def column
-        @offset - @last_newline
-      end
-    
-      def getc
-        return nil if @offset >= @size
-        ch = @content[@offset, 1]
-      
-        @offset += 1
-        @hwm = @offset if @hwm < @offset
-      
-        if @newline_pending
-          @line_num += 1
-          @last_newline = @offset - 1
-          @newline_pending = false
-        end
-      
-        if ch == "\n"
-          @newline_pending = true
-        end
-        ch
-      end
-    
-      def getc_already_read
-        getc
-      end
-    
-      def ungetc(ch)
-        raise "unget past beginning of file" if @offset <= 0
-        @offset -= 1
-        if @content[@offset] == ?\n
-          @newline_pending = false
-        end
-      end
-    
-      def get_read
-        res = @content[@read_back_offset...@offset]
-        @read_back_offset = @offset
-        res
-      end
-    
-      def peek(at)
-        pos = @offset + at
-        if pos >= @size
-          nil
-        else
-          @content[pos, 1]
-        end
-      end
-    
-      def peek_equal(str)
-        @content[@offset, str.length] == str
-      end
-    
-      def divert_read_from(reserve)
-        @content[@offset, 0] = reserve
-        @size      = @content.size
-      end
-    end
-
-    # end of nested class BufferedReader
-
-    extend Exception2MessageMapper
-    def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
-    def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
-    def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-    def_exception(:TkReading2TokenDuplicateError, 
-  		"key duplicate(token_n='%s', key='%s')")
-    def_exception(:SyntaxError, "%s")
-  
-    include RubyToken
-    include IRB
-
-    attr_reader :continue
-    attr_reader :lex_state
-
-    def RubyLex.debug?
-      false
-    end
-
-    def initialize(content)
-      lex_init
-
-      @reader = BufferedReader.new(content)
-
-      @exp_line_no = @line_no = 1
-      @base_char_no = 0
-      @indent = 0
-
-      @ltype = nil
-      @quoted = nil
-      @lex_state = EXPR_BEG
-      @space_seen = false
-    
-      @continue = false
-      @line = ""
-
-      @skip_space = false
-      @read_auto_clean_up = false
-      @exception_on_syntax_error = true
-    end
-
-    attr :skip_space, true
-    attr :read_auto_clean_up, true
-    attr :exception_on_syntax_error, true
-
-    attr :indent
-
-    # io functions
-    def line_no
-      @reader.line_num
-    end
-
-    def char_no
-      @reader.column
-    end
-
-    def get_read
-      @reader.get_read
-    end
-
-    def getc
-      @reader.getc
-    end
-
-    def getc_of_rests
-      @reader.getc_already_read
-    end
-
-    def gets
-      c = getc or return
-      l = ""
-      begin
-        l.concat c unless c == "\r"
-        break if c == "\n"
-      end while c = getc
-      l
-    end
-
-
-    def ungetc(c = nil)
-      @reader.ungetc(c)
-    end
-
-    def peek_equal?(str)
-      @reader.peek_equal(str)
-    end
-
-    def peek(i = 0)
-      @reader.peek(i)
-    end
-
-    def lex
-      until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
-  	     !@continue or
-  	     tk.nil?)
-      end
-      line = get_read
-
-      if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
-        nil
-      else
-        line
-      end
-    end
-
-    def token
-      set_token_position(line_no, char_no)
-      begin
-        begin
-  	      tk = @OP.match(self)
-  	      @space_seen = tk.kind_of?(TkSPACE)
-        rescue SyntaxError
-  	      abort if @exception_on_syntax_error
-  	      tk = TkError.new(line_no, char_no)
-        end
-      end while @skip_space and tk.kind_of?(TkSPACE)
-      if @read_auto_clean_up
-        get_read
-      end
-  #   throw :eof unless tk
-      p tk if $DEBUG
-      tk.lex_state = lex_state if tk
-      tk
-    end
-  
-    ENINDENT_CLAUSE = [
-      "case", "class", "def", "do", "for", "if",
-      "module", "unless", "until", "while", "begin" #, "when"
-    ]
-    DEINDENT_CLAUSE = ["end" #, "when"
-    ]
-
-    PERCENT_LTYPE = {
-      "q" => "\'",
-      "Q" => "\"",
-      "x" => "\`",
-      "r" => "/",
-      "w" => "]",
-      "W" => "]"
-    }
-  
-    PERCENT_PAREN = {
-      "{" => "}",
-      "[" => "]",
-      "<" => ">",
-      "(" => ")"
-    }
-
-    Ltype2Token = {
-      "\'" => TkSTRING,
-      "\"" => TkSTRING,
-      "\`" => TkXSTRING,
-      "/" => TkREGEXP,
-      "]" => TkDSTRING
-    }
-    Ltype2Token.default = TkSTRING
-
-    DLtype2Token = {
-      "\"" => TkDSTRING,
-      "\`" => TkDXSTRING,
-      "/" => TkDREGEXP,
-    }
-
-    def lex_init()
-      @OP = SLex.new
-      @OP.def_rules("\0", "\004", "\032") do |chars, io|
-        Token(TkEND_OF_SCRIPT).set_text(chars)
-      end
-
-      @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
-        @space_seen = TRUE
-        while (ch = getc) =~ /[ \t\f\r\13]/
-          chars << ch
-        end
-        ungetc
-        Token(TkSPACE).set_text(chars)
-      end
-
-      @OP.def_rule("#") do
-        |op, io|
-        identify_comment
-      end
-
-      @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
-        |op, io|
-        str = op
-        @ltype = "="
-
-
-        begin
-          line = ""
-          begin
-            ch = getc
-            line << ch
-          end until ch == "\n"
-          str << line
-        end until line =~ /^=end/
-
-        ungetc
-
-        @ltype = nil
-
-        if str =~ /\A=begin\s+rdoc/i
-          str.sub!(/\A=begin.*\n/, '')
-          str.sub!(/^=end.*/m, '')
-          Token(TkCOMMENT).set_text(str)
-        else
-          Token(TkRD_COMMENT)#.set_text(str)
-        end
-      end
-
-      @OP.def_rule("\n") do
-        print "\\n\n" if RubyLex.debug?
-        case @lex_state
-        when EXPR_BEG, EXPR_FNAME, EXPR_DOT
-  	      @continue = TRUE
-        else
-  	      @continue = FALSE
-  	      @lex_state = EXPR_BEG
-        end
-        Token(TkNL).set_text("\n")
-      end
-
-      @OP.def_rules("*", "**",	
-  		  "!", "!=", "!~",
-  		  "=", "==", "===", 
-  		  "=~", "<=>",	
-  		  "<", "<=",
-  		  ">", ">=", ">>") do |op, io|
-        @lex_state = EXPR_BEG
-        Token(op).set_text(op)
-      end
-
-      @OP.def_rules("<<") do |op, io|
-        tk = nil
-        if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && 
-           (@lex_state != EXPR_ARG || @space_seen)
-        	c = peek(0)
-        	tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
-        end
-        if !tk
-          @lex_state = EXPR_BEG
-          tk = Token(op).set_text(op)
-        end
-        tk
-      end
-
-      @OP.def_rules("'", '"') do |op, io|
-        identify_string(op)
-      end
-
-      @OP.def_rules("`") do |op, io|
-        if @lex_state == EXPR_FNAME
-  	      Token(op).set_text(op)
-        else
-  	      identify_string(op)
-        end
-      end
-
-      @OP.def_rules('?') do |op, io|
-        if @lex_state == EXPR_END
-  	      @lex_state = EXPR_BEG
-  	      Token(TkQUESTION).set_text(op)
-        else
-  	      ch = getc
-  	      if @lex_state == EXPR_ARG && ch !~ /\s/
-  	        ungetc
-  	        @lex_state = EXPR_BEG
-  	        Token(TkQUESTION).set_text(op)
-  	      else
-            str = op
-            str << ch
-        	  if (ch == '\\') #'
-        	    str << read_escape
-        	  end
-        	  @lex_state = EXPR_END
-        	  Token(TkINTEGER).set_text(str)
-  	      end
-        end
-      end
-
-      @OP.def_rules("&", "&&", "|", "||") do |op, io|
-        @lex_state = EXPR_BEG
-        Token(op).set_text(op)
-      end
-    
-      @OP.def_rules("+=", "-=", "*=", "**=", 
-  		  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
-        @lex_state = EXPR_BEG
-        op =~ /^(.*)=$/
-        Token(TkOPASGN, $1).set_text(op)
-      end
-
-      @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
-        Token(TkUPLUS).set_text(op)
-      end
-
-      @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
-        Token(TkUMINUS).set_text(op)
-      end
-
-      @OP.def_rules("+", "-") do |op, io|
-        catch(:RET) do
-        	if @lex_state == EXPR_ARG
-        	  if @space_seen and peek(0) =~ /[0-9]/
-        	    throw :RET, identify_number(op)
-        	  else
-        	    @lex_state = EXPR_BEG
-        	  end
-        	elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
-        	  throw :RET, identify_number(op)
-        	else
-        	  @lex_state = EXPR_BEG
-        	end
-  	      Token(op).set_text(op)
-        end
-      end
-
-      @OP.def_rule(".") do
-        @lex_state = EXPR_BEG
-        if peek(0) =~ /[0-9]/
-        	ungetc
-        	identify_number("")
-        else
-  	      # for obj.if
-  	      @lex_state = EXPR_DOT
-  	      Token(TkDOT).set_text(".")
-        end
-      end
-
-      @OP.def_rules("..", "...") do |op, io|
-        @lex_state = EXPR_BEG
-        Token(op).set_text(op)
-      end
-
-      lex_int2
-    end
-  
-    def lex_int2
-      @OP.def_rules("]", "}", ")") do
-        |op, io|
-        @lex_state = EXPR_END
-        @indent -= 1
-        Token(op).set_text(op)
-      end
-
-      @OP.def_rule(":") do
-        if @lex_state == EXPR_END || peek(0) =~ /\s/
-  	      @lex_state = EXPR_BEG
-  	      tk = Token(TkCOLON)
-        else
-  	      @lex_state = EXPR_FNAME
-  	      tk = Token(TkSYMBEG)
-        end
-        tk.set_text(":")
-      end
-
-      @OP.def_rule("::") do
-  #      p @lex_state.id2name, @space_seen
-        if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
-  	      @lex_state = EXPR_BEG
-  	      tk = Token(TkCOLON3)
-        else
-  	      @lex_state = EXPR_DOT
-  	      tk = Token(TkCOLON2)
-        end
-        tk.set_text("::")
-      end
-
-      @OP.def_rule("/") do |op, io|
-        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-  	      identify_string(op)
-        elsif peek(0) == '='
-  	      getc
-  	      @lex_state = EXPR_BEG
-  	      Token(TkOPASGN, :/).set_text("/=") #")
-        elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
-  	      identify_string(op)
-        else 
-  	      @lex_state = EXPR_BEG
-          Token("/").set_text(op)
-        end
-      end
-
-      @OP.def_rules("^") do
-        @lex_state = EXPR_BEG
-        Token("^").set_text("^")
-      end
-
-      #       @OP.def_rules("^=") do
-      # 	@lex_state = EXPR_BEG
-      # 	Token(TkOPASGN, :^)
-      #       end
-    
-      @OP.def_rules(",", ";") do |op, io|
-        @lex_state = EXPR_BEG
-        Token(op).set_text(op)
-      end
-
-      @OP.def_rule("~") do
-        @lex_state = EXPR_BEG
-        Token("~").set_text("~")
-      end
-
-      @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
-        @lex_state = EXPR_BEG
-        Token("~").set_text("~@")
-      end
-    
-      @OP.def_rule("(") do
-        @indent += 1
-        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-  	      @lex_state = EXPR_BEG
-  	      tk = Token(TkfLPAREN)
-        else
-  	      @lex_state = EXPR_BEG
-  	      tk = Token(TkLPAREN)
-        end
-        tk.set_text("(")
-      end
-
-      @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
-        Token("[]").set_text("[]")
-      end
-
-      @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
-        Token("[]=").set_text("[]=")
-      end
-
-      @OP.def_rule("[") do
-        @indent += 1
-        if @lex_state == EXPR_FNAME
-  	      t = Token(TkfLBRACK)
-        else
-  	      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-  	        t = Token(TkLBRACK)
-  	      elsif @lex_state == EXPR_ARG && @space_seen
-  	        t = Token(TkLBRACK)
-  	      else
-  	        t = Token(TkfLBRACK)
-  	      end
-  	      @lex_state = EXPR_BEG
-        end
-        t.set_text("[")
-      end
-
-      @OP.def_rule("{") do
-        @indent += 1
-        if @lex_state != EXPR_END && @lex_state != EXPR_ARG
-  	      t = Token(TkLBRACE)
-        else
-  	      t = Token(TkfLBRACE)
-        end
-        @lex_state = EXPR_BEG
-        t.set_text("{")
-      end
-
-      @OP.def_rule('\\') do   #'
-        if getc == "\n" 
-  	      @space_seen = true
-  	      @continue = true
-  	      Token(TkSPACE).set_text("\\\n")
-        else 
-  	      ungetc
-  	      Token("\\").set_text("\\")  #"
-        end 
-      end 
-
-      @OP.def_rule('%') do
-        |op, io|
-        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-  	      identify_quotation('%')
-        elsif peek(0) == '='
-  	      getc
-  	      Token(TkOPASGN, "%").set_text("%=")
-        elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
-  	      identify_quotation('%')
-        else
-  	      @lex_state = EXPR_BEG
-  	      Token("%").set_text("%")
-        end
-      end
-
-      @OP.def_rule('$') do  #'
-        identify_gvar
-      end
-
-      @OP.def_rule('@') do
-        if peek(0) =~ /[@\w_]/
-  	      ungetc
-  	      identify_identifier
-        else
-  	      Token("@").set_text("@")
-        end
-      end
-
-      #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
-      # 	|op, io|
-      # 	@indent += 1
-      # 	@lex_state = EXPR_FNAME
-      # #	@lex_state = EXPR_END
-      # #	until @rests[0] == "\n" or @rests[0] == ";"
-      # #	  rests.shift
-      # #	end
-      #       end
-
-      @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
-        throw :eof
-      end
-
-      @OP.def_rule("") do
-        |op, io|
-        printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
-        if peek(0) =~ /[0-9]/
-  	t = identify_number("")
-        elsif peek(0) =~ /[\w_]/
-  	t = identify_identifier
-        end
-        printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
-        t
-      end
-    
-      p @OP if RubyLex.debug?
-    end
-  
-    def identify_gvar
-      @lex_state = EXPR_END
-      str = "$"
-
-      tk = case ch = getc
-           when /[~_*$?!@\/\\;,=:<>".]/   #"
-             str << ch
-             Token(TkGVAR, str)
-           
-           when "-"
-             str << "-" << getc
-             Token(TkGVAR, str)
-           
-           when "&", "`", "'", "+"
-             str << ch
-             Token(TkBACK_REF, str)
-           
-           when /[1-9]/
-             str << ch
-             while (ch = getc) =~ /[0-9]/
-               str << ch
-             end
-             ungetc
-             Token(TkNTH_REF)
-           when /\w/
-             ungetc
-             ungetc
-             return identify_identifier
-           else 
-             ungetc
-             Token("$")     
-           end
-      tk.set_text(str)
-    end
-  
-    def identify_identifier
-      token = ""
-      token.concat getc if peek(0) =~ /[$@]/
-      token.concat getc if peek(0) == "@"
-
-      while (ch = getc) =~ /\w|_/
-        print ":", ch, ":" if RubyLex.debug?
-        token.concat ch
-      end
-      ungetc
-    
-      if ch == "!" or ch == "?"
-        token.concat getc
-      end
-      # fix token
-
-      # $stderr.puts "identifier - #{token}, state = #@lex_state"
-
-      case token
-      when /^\$/
-        return Token(TkGVAR, token).set_text(token)
-      when /^\@/
-        @lex_state = EXPR_END
-        return Token(TkIVAR, token).set_text(token)
-      end
-    
-      if @lex_state != EXPR_DOT
-        print token, "\n" if RubyLex.debug?
-
-        token_c, *trans = TkReading2Token[token]
-        if token_c
-  	# reserved word?
-
-  	if (@lex_state != EXPR_BEG &&
-  	    @lex_state != EXPR_FNAME &&
-  	    trans[1])
-  	  # modifiers
-  	  token_c = TkSymbol2Token[trans[1]]
-  	  @lex_state = trans[0]
-  	else
-  	  if @lex_state != EXPR_FNAME
-  	    if ENINDENT_CLAUSE.include?(token)
-  	      @indent += 1
-  	    elsif DEINDENT_CLAUSE.include?(token)
-  	      @indent -= 1
-  	    end
-  	    @lex_state = trans[0]
-  	  else
-  	    @lex_state = EXPR_END
-  	  end
-  	end
-  	return Token(token_c, token).set_text(token)
-        end
-      end
-
-      if @lex_state == EXPR_FNAME
-        @lex_state = EXPR_END
-        if peek(0) == '='
-  	token.concat getc
-        end
-      elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
-        @lex_state = EXPR_ARG
-      else
-        @lex_state = EXPR_END
-      end
-
-      if token[0, 1] =~ /[A-Z]/
-        return Token(TkCONSTANT, token).set_text(token)
-      elsif token[token.size - 1, 1] =~ /[!?]/
-        return Token(TkFID, token).set_text(token)
-      else
-        return Token(TkIDENTIFIER, token).set_text(token)
-      end
-    end
-
-    def identify_here_document
-      ch = getc
-      if ch == "-"
-        ch = getc
-        indent = true
-      end
-      if /['"`]/ =~ ch            # '
-        lt = ch
-        quoted = ""
-        while (c = getc) && c != lt
-  	quoted.concat c
-        end
-      else
-        lt = '"'
-        quoted = ch.dup
-        while (c = getc) && c =~ /\w/
-  	quoted.concat c
-        end
-        ungetc
-      end
-
-      ltback, @ltype = @ltype, lt
-      reserve = ""
-
-      while ch = getc
-        reserve << ch
-        if ch == "\\"    #"
-          ch = getc
-  	reserve << ch
-        elsif ch == "\n"
-  	break
-        end
-      end
-
-      str = ""
-      while (l = gets)
-        l.chomp!
-        l.strip! if indent
-        break if l == quoted
-        str << l.chomp << "\n"
-      end
-
-      @reader.divert_read_from(reserve)
-
-      @ltype = ltback
-      @lex_state = EXPR_END
-      Token(Ltype2Token[lt], str).set_text(str.dump)
-    end
-  
-    def identify_quotation(initial_char)
-      ch = getc
-      if lt = PERCENT_LTYPE[ch]
-        initial_char += ch
-        ch = getc
-      elsif ch =~ /\W/
-        lt = "\""
-      else
-        RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
-      end
-  #     if ch !~ /\W/
-  #       ungetc
-  #       next
-  #     end
-      #@ltype = lt
-      @quoted = ch unless @quoted = PERCENT_PAREN[ch]
-      identify_string(lt, @quoted, ch, initial_char)
-    end
-
-    def identify_number(start)
-      str = start.dup
-
-      if start == "+" or start == "-" or start == ""
-        start = getc
-        str << start
-      end
-
-      @lex_state = EXPR_END
-
-      if start == "0"
-        if peek(0) == "x"
-          ch = getc
-          str << ch
-          match = /[0-9a-f_]/
-        else
-          match = /[0-7_]/
-        end
-        while ch = getc
-          if ch !~ match
-            ungetc
-            break
-          else
-            str << ch
-          end
-        end
-        return Token(TkINTEGER).set_text(str)
-      end
-
-      type = TkINTEGER
-      allow_point = TRUE
-      allow_e = TRUE
-      while ch = getc
-        case ch
-        when /[0-9_]/
-          str << ch
-
-        when allow_point && "."
-  	type = TkFLOAT
-  	if peek(0) !~ /[0-9]/
-  	  ungetc
-  	  break
-  	end
-          str << ch
-  	allow_point = false
-
-        when allow_e && "e", allow_e && "E"
-          str << ch
-  	type = TkFLOAT
-  	if peek(0) =~ /[+-]/
-  	  str << getc
-  	end
-  	allow_e = false
-  	allow_point = false
-        else
-  	ungetc
-  	break
-        end
-      end
-      Token(type).set_text(str)
-    end
-  
-    def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
-      @ltype = ltype
-      @quoted = quoted
-      subtype = nil
-
-      str = ""
-      str << initial_char if initial_char
-      str << (opener||quoted)
-
-      nest = 0
-      begin
-        while ch = getc 
-  	str << ch
-  	if @quoted == ch 
-            if nest == 0
-              break
-            else
-              nest -= 1
-            end
-          elsif opener == ch
-            nest += 1
-  	elsif @ltype != "'" && @ltype != "]" and ch == "#"
-            ch = getc
-            if ch == "{"
-              subtype = true
-              str << ch << skip_inner_expression
-            else
-              ungetc(ch)
-            end
-  	elsif ch == '\\' #'
-  	  str << read_escape
-  	end
-        end
-        if @ltype == "/"
-  	if peek(0) =~ /i|o|n|e|s/
-  	  str << getc
-  	end
-        end
-        if subtype
-  	Token(DLtype2Token[ltype], str)
-        else
-  	Token(Ltype2Token[ltype], str)
-        end.set_text(str)
-      ensure
-        @ltype = nil
-        @quoted = nil
-        @lex_state = EXPR_END
-      end
-    end
-
-    def skip_inner_expression
-      res = ""
-      nest = 0
-      while (ch = getc)
-        res << ch
-        if ch == '}'
-          break if nest.zero?
-          nest -= 1
-        elsif ch == '{'
-          nest += 1
-        end
-      end
-      res
-    end
-
-    def identify_comment
-      @ltype = "#"
-      comment = "#"
-      while ch = getc
-        if ch == "\\"
-          ch = getc
-          if ch == "\n"
-            ch = " "
-          else
-            comment << "\\" 
-          end
-        else
-          if ch == "\n"
-            @ltype = nil
-            ungetc
-            break
-          end
-        end
-        comment << ch
-      end
-      return Token(TkCOMMENT).set_text(comment)
-    end
-  
-    def read_escape
-      res = ""
-      case ch = getc
-      when /[0-7]/
-        ungetc ch
-        3.times do
-  	case ch = getc
-  	when /[0-7]/
-  	when nil
-  	  break
-  	else
-  	  ungetc
-  	  break
-  	end
-          res << ch
-        end
-      
-      when "x"
-        res << ch
-        2.times do
-  	case ch = getc
-  	when /[0-9a-fA-F]/
-  	when nil
-  	  break
-  	else
-  	  ungetc
-  	  break
-  	end
-          res << ch
-        end
-
-      when "M"
-        res << ch
-        if (ch = getc) != '-'
-  	ungetc
-        else
-          res << ch
-  	if (ch = getc) == "\\" #"
-            res << ch
-  	  res << read_escape
-          else
-            res << ch
-  	end
-        end
-
-      when "C", "c" #, "^"
-        res << ch
-        if ch == "C" and (ch = getc) != "-"
-  	ungetc
-        else
-          res << ch
-          if (ch = getc) == "\\" #"
-            res << ch
-            res << read_escape
-          else
-            res << ch
-          end
-        end
-      else
-        res << ch
-      end
-      res
-    end
-  end
-end