hyper-pro-gem 0.0.1

data/yard-0.9.44/lib/yard/parser/ruby/legacy/ruby_lex.rb

@@ -0,0 +1,1360 @@
+require_relative 'irb/slex'
+
+module YARD
+  module Parser::Ruby::Legacy
+    # Legacy lexical tokenizer module.
+    module RubyToken
+      EXPR_BEG   = :EXPR_BEG
+      EXPR_MID   = :EXPR_MID
+      EXPR_END   = :EXPR_END
+      EXPR_ARG   = :EXPR_ARG
+      EXPR_FNAME = :EXPR_FNAME
+      EXPR_DOT   = :EXPR_DOT
+      EXPR_CLASS = :EXPR_CLASS
+
+      # Represents a token in the Ruby lexer
+      class Token
+        # @return [Integer] the line number in the file/stream the token is
+        #   located.
+        attr_reader :line_no
+
+        # @return [Integer] the character number in the file/stream the token
+        #   is located.
+        attr_reader :char_no
+
+        # @return [String] the token text value
+        attr_reader :text
+
+        # @return [Symbol] the lexical state at the token
+        attr_accessor :lex_state
+
+        # @private
+        NO_TEXT = "??".freeze
+
+        # Creates a new Token object
+        # @param [Integer] line_no the line number to initialize the token to
+        # @param [Integer] char_no the char number to initialize the token to
+        def initialize(line_no, char_no)
+          @line_no = line_no
+          @char_no = char_no
+          @text    = NO_TEXT
+        end
+
+        # Chainable way to sets the text attribute
+        #
+        # @param [String] text the new text
+        # @return [Token] this token object
+        def set_text(text)
+          @text = text
+          self
+        end
+      end
+
+      # Represents a block
+      class TkBlockContents < Token
+        def text; '...' end
+      end
+
+      # Represents an end statement
+      class TkStatementEnd < Token
+        def text; '' end
+      end
+
+      class TkNode < Token
+        attr :node
+      end
+
+      # Represents whitespace
+      class TkWhitespace < Token
+      end
+
+      # Represents a Ruby identifier
+      class TkId < Token
+        def initialize(line_no, char_no, name)
+          super(line_no, char_no)
+          @name = name
+        end
+        attr :name
+      end
+
+      # Represents a Ruby keyword
+      class TkKW < TkId
+      end
+
+      # Represents a Ruby value
+      class TkVal < Token
+        def initialize(line_no, char_no, value = nil)
+          super(line_no, char_no)
+          set_text(value)
+        end
+      end
+
+      class TkOp < Token
+        def name
+          self.class.op_name
+        end
+      end
+
+      class TkOPASGN < TkOp
+        def initialize(line_no, char_no, op)
+          super(line_no, char_no)
+          op = TkReading2Token[op] unless op.is_a?(Symbol)
+          @op = op
+        end
+        attr :op
+      end
+
+      class TkUnknownChar < Token
+        def initialize(line_no, char_no, _id)
+          super(line_no, char_no)
+          @name = char_no > 255 ? '?' : char_no.chr
+        end
+        attr :name
+      end
+
+      class TkError < Token
+      end
+
+      # @private
+      def set_token_position(line, char)
+        @prev_line_no = line
+        @prev_char_no = char
+      end
+
+      # @private
+      def Token(token, value = nil) # rubocop:disable Style/MethodName
+        tk = nil
+        case token
+        when String, Symbol
+          source = token.is_a?(String) ? TkReading2Token : TkSymbol2Token
+          if (tk = source[token]).nil?
+            raise "no key #{token}"
+          end
+          tk = Token(tk[0], value)
+        else
+          if token
+            tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
+                   token.new(@prev_line_no, @prev_char_no)
+                 else
+                   token.new(@prev_line_no, @prev_char_no, value)
+                 end
+          end
+        end
+        tk
+      end
+
+      # @private
+      TokenDefinitions = [
+        [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
+        [:TkMODULE,     TkKW,  "module", EXPR_BEG],
+        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
+        [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
+        [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
+        [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
+        [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
+        [:TkEND,        TkKW,  "end",    EXPR_END],
+        [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
+        [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
+        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
+        [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
+        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
+        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
+        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
+        [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
+        [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
+        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
+        [:TkBREAK,      TkKW,  "break",  EXPR_END],
+        [:TkNEXT,       TkKW,  "next",   EXPR_END],
+        [:TkREDO,       TkKW,  "redo",   EXPR_END],
+        [:TkRETRY,      TkKW,  "retry",  EXPR_END],
+        [:TkIN,         TkKW,  "in",     EXPR_BEG],
+        [:TkDO,         TkKW,  "do",     EXPR_BEG],
+        [:TkRETURN,     TkKW,  "return", EXPR_MID],
+        [:TkYIELD,      TkKW,  "yield",  EXPR_END],
+        [:TkSUPER,      TkKW,  "super",  EXPR_END],
+        [:TkSELF,       TkKW,  "self",   EXPR_END],
+        [:TkNIL,        TkKW,  "nil",    EXPR_END],
+        [:TkTRUE,       TkKW,  "true",   EXPR_END],
+        [:TkFALSE,      TkKW,  "false",  EXPR_END],
+        [:TkAND,        TkKW,  "and",    EXPR_BEG],
+        [:TkOR,         TkKW,  "or",     EXPR_BEG],
+        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
+        [:TkIF_MOD,     TkKW],
+        [:TkUNLESS_MOD, TkKW],
+        [:TkWHILE_MOD,  TkKW],
+        [:TkUNTIL_MOD,  TkKW],
+        [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
+        [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
+        [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
+        [:TklEND,       TkKW,  "END",      EXPR_END],
+        [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
+        [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
+        [:TkIDENTIFIER, TkId],
+        [:TkFID,        TkId],
+        [:TkGVAR,       TkId],
+        [:TkIVAR,       TkId],
+        [:TkCONSTANT,   TkId],
+        [:TkINTEGER,    TkVal],
+        [:TkFLOAT,      TkVal],
+        [:TkSYMBOL,     TkVal],
+        [:TkLABEL,      TkVal],
+        [:TkSTRING,     TkVal],
+        [:TkXSTRING,    TkVal],
+        [:TkREGEXP,     TkVal],
+        [:TkCOMMENT,    TkVal],
+        [:TkDSTRING,    TkNode],
+        [:TkDXSTRING,   TkNode],
+        [:TkDREGEXP,    TkNode],
+        [:TkNTH_REF,    TkId],
+        [:TkBACK_REF,   TkId],
+        [:TkUPLUS,      TkOp,   "+@"],
+        [:TkUMINUS,     TkOp,   "-@"],
+        [:TkPOW,        TkOp,   "**"],
+        [:TkCMP,        TkOp,   "<=>"],
+        [:TkEQ,         TkOp,   "=="],
+        [:TkEQQ,        TkOp,   "==="],
+        [:TkNEQ,        TkOp,   "!="],
+        [:TkGEQ,        TkOp,   ">="],
+        [:TkLEQ,        TkOp,   "<="],
+        [:TkANDOP,      TkOp,   "&&"],
+        [:TkOROP,       TkOp,   "||"],
+        [:TkMATCH,      TkOp,   "=~"],
+        [:TkNMATCH,     TkOp,   "!~"],
+        [:TkDOT2,       TkOp,   ".."],
+        [:TkDOT3,       TkOp,   "..."],
+        [:TkAREF,       TkOp,   "[]"],
+        [:TkASET,       TkOp,   "[]="],
+        [:TkLSHFT,      TkOp,   "<<"],
+        [:TkRSHFT,      TkOp,   ">>"],
+        [:TkCOLON2,     TkOp],
+        [:TkCOLON3,     TkOp],
+        [:OPASGN,       TkOp], # +=, -=  etc. #
+        [:TkASSOC,      TkOp,   "=>"],
+        [:TkQUESTION,   TkOp,   "?"], #?
+        [:TkCOLON,      TkOp,   ":"], #:
+        [:TkSTAR],            # *arg
+        [:TkAMPER],           # &arg #
+        [:TkSYMBEG,     TkId],
+        [:TkGT,         TkOp,   ">"],
+        [:TkLT,         TkOp,   "<"],
+        [:TkPLUS,       TkOp,   "+"],
+        [:TkMINUS,      TkOp,   "-"],
+        [:TkMULT,       TkOp,   "*"],
+        [:TkDIV,        TkOp,   "/"],
+        [:TkMOD,        TkOp,   "%"],
+        [:TkBITOR,      TkOp,   "|"],
+        [:TkBITXOR,     TkOp,   "^"],
+        [:TkBITAND,     TkOp,   "&"],
+        [:TkBITNOT,     TkOp,   "~"],
+        [:TkNOTOP,      TkOp,   "!"],
+        [:TkBACKQUOTE,  TkOp,   "`"],
+        [:TkASSIGN,     Token,  "="],
+        [:TkDOT,        Token,  "."],
+        [:TkLPAREN,     Token,  "("],  # (exp)
+        [:TkLBRACK,     Token,  "["],  # [arry]
+        [:TkLBRACE,     Token,  "{"],  # {hash}
+        [:TkRPAREN,     Token,  ")"],
+        [:TkRBRACK,     Token,  "]"],
+        [:TkRBRACE,     Token,  "}"],
+        [:TkCOMMA,      Token,  ","],
+        [:TkSEMICOLON,  Token,  ";"],
+        [:TkSPACE,          TkWhitespace],
+        [:TkNL,             TkWhitespace],
+        [:TkEND_OF_SCRIPT,  TkWhitespace],
+        [:TkBACKSLASH,  TkUnknownChar,  "\\"],
+        [:TkAT,         TkUnknownChar,  "@"],
+        [:TkDOLLAR,     TkUnknownChar,  "\$"]
+      ]
+
+      # { reading => token_class }
+      # { reading => [token_class, *opt] }
+      TkReading2Token = {}
+      TkSymbol2Token = {}
+
+      # @private
+      def self.def_token(token_n, super_token = Token, reading = nil, *opts)
+        token_n = token_n.id2name unless token_n.is_a?(String)
+        if RubyToken.const_defined?(token_n)
+          # IRB.fail AlreadyDefinedToken, token_n
+        end
+
+        token_c = Class.new super_token
+        RubyToken.const_set token_n, token_c
+        # token_c.inspect
+
+        if reading
+          if TkReading2Token[reading]
+            raise "duplicate #{token_n} #{reading}"
+          end
+          if opts.empty?
+            TkReading2Token[reading] = [token_c]
+          else
+            TkReading2Token[reading] = [token_c].concat(opts)
+          end
+        end
+        TkSymbol2Token[token_n.intern] = token_c
+
+        if token_c <= TkOp
+          token_c.class_eval %{
+            def self.op_name; "#{reading}"; end
+          }
+        end
+      end
+
+      for defs in TokenDefinitions
+        def_token(*defs)
+      end
+
+      NEWLINE_TOKEN = TkNL.new(0, 0)
+      NEWLINE_TOKEN.set_text("\n")
+    end
+
+    # Lexical analyzer for Ruby source
+    # @private
+    class RubyLex
+      # Read an input stream character by character. We allow for unlimited
+      # ungetting of characters just read.
+      #
+      # We simplify the implementation greatly by reading the entire input
+      # into a buffer initially, and then simply traversing it using
+      # pointers.
+      #
+      # We also have to allow for the <i>here document diversion</i>. This
+      # little gem comes about when the lexer encounters a here
+      # document. At this point we effectively need to split the input
+      # stream into two parts: one to read the body of the here document,
+      # the other to read the rest of the input line where the here
+      # document was initially encountered. For example, we might have
+      #
+      #   do_something(<<-A, <<-B)
+      #     stuff
+      #     for
+      #   A
+      #     stuff
+      #     for
+      #   B
+      #
+      # When the lexer encounters the <<A, it reads until the end of the
+      # line, and keeps it around for later. It then reads the body of the
+      # here document.  Once complete, it needs to read the rest of the
+      # original line, but then skip the here document body.
+      #
+      # @private
+      class BufferedReader
+        attr_reader :line_num
+
+        def initialize(content)
+          if /\t/ =~ content
+            tab_width = 2
+            content = content.split(/\n/).map do |line|
+              1 while line.gsub!(/\t+/) { ' ' * (tab_width * $&.length - $`.length % tab_width) } && $~ #`
+              line
+            end .join("\n")
+          end
+          @content = String.new(content)
+          @content << "\n" unless @content[-1, 1] == "\n"
+          @size      = @content.size
+          @offset    = 0
+          @hwm       = 0
+          @line_num  = 1
+          @read_back_offset = 0
+          @last_newline = 0
+          @newline_pending = false
+        end
+
+        def column
+          @offset - @last_newline
+        end
+
+        def getc
+          return nil if @offset >= @size
+          ch = @content[@offset, 1]
+
+          @offset += 1
+          @hwm = @offset if @hwm < @offset
+
+          if @newline_pending
+            @line_num += 1
+            @last_newline = @offset - 1
+            @newline_pending = false
+          end
+
+          if ch == "\n"
+            @newline_pending = true
+          end
+          ch
+        end
+
+        def getc_already_read
+          getc
+        end
+
+        def ungetc(_ch)
+          raise "unget past beginning of file" if @offset <= 0
+          @offset -= 1
+          if @content[@offset] == ?\n
+            @newline_pending = false
+          end
+        end
+
+        def get_read
+          res = @content[@read_back_offset...@offset]
+          @read_back_offset = @offset
+          res
+        end
+
+        def peek(at)
+          pos = @offset + at
+          if pos >= @size
+            nil
+          else
+            @content[pos, 1]
+          end
+        end
+
+        def peek_equal(str)
+          @content[@offset, str.length] == str
+        end
+
+        def divert_read_from(reserve)
+          @content[@offset, 0] = reserve
+          @size = @content.size
+        end
+      end
+
+      # end of nested class BufferedReader
+
+      include RubyToken
+      include IRB
+
+      attr_reader :continue
+      attr_reader :lex_state
+
+      def self.debug?
+        false
+      end
+
+      def initialize(content)
+        lex_init
+
+        @reader = BufferedReader.new(content)
+
+        @exp_line_no = @line_no = 1
+        @base_char_no = 0
+        @indent = 0
+
+        @ltype = nil
+        @quoted = nil
+        @lex_state = EXPR_BEG
+        @space_seen = false
+
+        @continue = false
+        @line = ""
+
+        @skip_space = false
+        @read_auto_clean_up = false
+        @exception_on_syntax_error = true
+
+        @colonblock_seen = false
+      end
+
+      attr_accessor :skip_space
+      attr_accessor :read_auto_clean_up
+      attr_accessor :exception_on_syntax_error
+
+      attr :indent
+
+      # io functions
+      def line_no
+        @reader.line_num
+      end
+
+      def char_no
+        @reader.column
+      end
+
+      def get_read
+        @reader.get_read
+      end
+
+      def getc
+        @reader.getc
+      end
+
+      def getc_of_rests
+        @reader.getc_already_read
+      end
+
+      def gets
+        (c = getc) || return
+        l = ""
+        begin
+          l.concat c unless c == "\r"
+          break if c == "\n"
+        end while c = getc # rubocop:disable Lint/Loop
+        l
+      end
+
+      def ungetc(c = nil)
+        @reader.ungetc(c)
+      end
+
+      def peek_equal?(str)
+        @reader.peek_equal(str)
+      end
+
+      def peek(i = 0)
+        @reader.peek(i)
+      end
+
+      def lex
+        catch(:eof) do
+          until ((tk = token).is_a?(TkNL) || tk.is_a?(TkEND_OF_SCRIPT)) &&
+                !@continue ||
+                tk.nil?
+          end
+          line = get_read
+
+          if line == "" && tk.is_a?(TkEND_OF_SCRIPT) || tk.nil?
+            nil
+          else
+            line
+          end
+        end
+      end
+
+      def token
+        set_token_position(line_no, char_no)
+        catch(:eof) do
+          begin
+            begin
+              tk = @OP.match(self)
+              @space_seen = tk.is_a?(TkSPACE)
+            rescue SyntaxError
+              abort if @exception_on_syntax_error
+              tk = TkError.new(line_no, char_no)
+            end
+          end while @skip_space && tk.is_a?(TkSPACE)
+          if @read_auto_clean_up
+            get_read
+          end
+          # throw :eof unless tk
+          p tk if $DEBUG
+          tk.lex_state = lex_state if tk
+          tk
+        end
+      end
+
+      ENINDENT_CLAUSE = [
+        "case", "class", "def", "do", "for", "if",
+        "module", "unless", "until", "while", "begin"
+      ] #, "when"
+      ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
+      DEINDENT_CLAUSE = ["end"] #, "when"
+
+      PERCENT_LTYPE = {
+        "q" => "\'",
+        "Q" => "\"",
+        "x" => "\`",
+        "r" => "/",
+        "w" => "]",
+        "W" => "]"
+      }
+
+      PERCENT_PAREN = {
+        "{" => "}",
+        "[" => "]",
+        "<" => ">",
+        "(" => ")"
+      }
+
+      Ltype2Token = {
+        "\'" => TkSTRING,
+        "\"" => TkSTRING,
+        "\`" => TkXSTRING,
+        "/" => TkREGEXP,
+        "]" => TkDSTRING
+      }
+      Ltype2Token.default = TkSTRING
+
+      DLtype2Token = {
+        "\"" => TkDSTRING,
+        "\`" => TkDXSTRING,
+        "/" => TkDREGEXP
+      }
+
+      def lex_init()
+        @OP = SLex.new
+        @OP.def_rules("\0", "\004", "\032") do |chars, _io|
+          Token(TkEND_OF_SCRIPT).set_text(chars)
+        end
+
+        @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, _io|
+          @space_seen = true
+          while (ch = getc) =~ /[ \t\f\r\13]/
+            chars << ch
+          end
+          ungetc
+          Token(TkSPACE).set_text(chars)
+        end
+
+        @OP.def_rule("#") do |_op, _io|
+          identify_comment
+        end
+
+        @OP.def_rule("=begin", proc { @prev_char_no == 0 && peek(0) =~ /\s/ }) do |op, _io|
+          str = String.new(op)
+          @ltype = "="
+
+          begin
+            line = String.new
+            begin
+              ch = getc
+              line << ch
+            end until ch == "\n"
+            str << line
+          end until line =~ /^=end/
+
+          ungetc
+
+          @ltype = nil
+
+          if str =~ /\A=begin\s+rdoc/i
+            str.sub!(/\A=begin.*\n/, '')
+            str.sub!(/^=end.*/m, '')
+            Token(TkCOMMENT).set_text(str)
+          else
+            Token(TkCOMMENT).set_text(str)
+          end
+        end
+
+        @OP.def_rule("\n") do
+          print "\\n\n" if RubyLex.debug?
+          @colonblock_seen = false
+          case @lex_state
+          when EXPR_BEG, EXPR_FNAME, EXPR_DOT
+            @continue = true
+          else
+            @continue = false
+            @lex_state = EXPR_BEG
+          end
+          Token(TkNL).set_text("\n")
+        end
+
+        @OP.def_rules("*", "**",
+          "!", "!=", "!~",
+          "=", "==", "===",
+          "=~", "<=>",
+          "<", "<=",
+          ">", ">=", ">>") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rules("<<") do |op, _io|
+          tk = nil
+          if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
+             (@lex_state != EXPR_ARG || @space_seen)
+            c = peek(0)
+            tk = identify_here_document if /[-~\w\"\'\`]/ =~ c
+          end
+          if !tk
+            @lex_state = EXPR_BEG
+            tk = Token(op).set_text(op)
+          end
+          tk
+        end
+
+        @OP.def_rules("'", '"') do |op, _io|
+          identify_string(op)
+        end
+
+        @OP.def_rules("`") do |op, _io|
+          if @lex_state == EXPR_FNAME
+            Token(op).set_text(op)
+          else
+            identify_string(op)
+          end
+        end
+
+        @OP.def_rules('?') do |op, _io|
+          if @lex_state == EXPR_END
+            @lex_state = EXPR_BEG
+            Token(TkQUESTION).set_text(op)
+          else
+            ch = getc
+            if @lex_state == EXPR_ARG && ch !~ /\s/
+              ungetc
+              @lex_state = EXPR_BEG
+              Token(TkQUESTION).set_text(op)
+            else
+              str = String.new(op)
+              str << ch
+              if ch == '\\' #'
+                str << read_escape
+              end
+              @lex_state = EXPR_END
+              Token(TkINTEGER).set_text(str)
+            end
+          end
+        end
+
+        @OP.def_rules("&", "&&", "|", "||") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rules("+=", "-=", "*=", "**=",
+          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, _io|
+          @lex_state = EXPR_BEG
+          op =~ /^(.*)=$/
+          Token(TkOPASGN, $1).set_text(op)
+        end
+
+        @OP.def_rule("+@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
+          Token(TkUPLUS).set_text(op)
+        end
+
+        @OP.def_rule("-@", proc { @lex_state == EXPR_FNAME }) do |op, _io|
+          Token(TkUMINUS).set_text(op)
+        end
+
+        @OP.def_rules("+", "-") do |op, _io|
+          catch(:RET) do
+            if @lex_state == EXPR_ARG
+              if @space_seen && peek(0) =~ /[0-9]/
+                throw :RET, identify_number(op)
+              else
+                @lex_state = EXPR_BEG
+              end
+            elsif @lex_state != EXPR_END && peek(0) =~ /[0-9]/
+              throw :RET, identify_number(op)
+            else
+              @lex_state = EXPR_BEG
+            end
+            Token(op).set_text(op)
+          end
+        end
+
+        @OP.def_rule(".") do
+          @lex_state = EXPR_BEG
+          if peek(0) =~ /[0-9]/
+            ungetc
+            identify_number("")
+          else
+            # for obj.if
+            @lex_state = EXPR_DOT
+            Token(TkDOT).set_text(".")
+          end
+        end
+
+        @OP.def_rules("..", "...") do |op, _io|
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        lex_int2
+      end
+
+      def lex_int2
+        @OP.def_rules("]", "}", ")") do |op, _io|
+          @lex_state = EXPR_END
+          @indent -= 1
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rule(":") do
+          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON)
+          else
+            @lex_state = EXPR_FNAME
+            tk = Token(TkSYMBEG)
+          end
+          tk.set_text(":")
+        end
+
+        @OP.def_rule("::") do
+          # p @lex_state.id2name, @space_seen
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_ARG && @space_seen
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON3)
+          else
+            @lex_state = EXPR_DOT
+            tk = Token(TkCOLON2)
+          end
+          tk.set_text("::")
+        end
+
+        @OP.def_rule("/") do |op, _io|
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            identify_string(op)
+          elsif peek(0) == '='
+            getc
+            @lex_state = EXPR_BEG
+            Token(TkOPASGN, :/).set_text("/=") #")
+          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
+            identify_string(op)
+          else
+            @lex_state = EXPR_BEG
+            Token("/").set_text(op)
+          end
+        end
+
+        @OP.def_rules("^") do
+          @lex_state = EXPR_BEG
+          Token("^").set_text("^")
+        end
+
+        # @OP.def_rules("^=") do
+        #   @lex_state = EXPR_BEG
+        #   Token(TkOPASGN, :^)
+        # end
+
+        @OP.def_rules(",", ";") do |op, _io|
+          @colonblock_seen = false
+          @lex_state = EXPR_BEG
+          Token(op).set_text(op)
+        end
+
+        @OP.def_rule("~") do
+          @lex_state = EXPR_BEG
+          Token("~").set_text("~")
+        end
+
+        @OP.def_rule("~@", proc { @lex_state = EXPR_FNAME }) do
+          @lex_state = EXPR_BEG
+          Token("~").set_text("~@")
+        end
+
+        @OP.def_rule("(") do
+          @indent += 1
+            # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            #  @lex_state = EXPR_BEG
+            #  tk = Token(TkfLPAREN)
+            # else
+            @lex_state = EXPR_BEG
+            tk = Token(TkLPAREN)
+          # end
+          tk.set_text("(")
+        end
+
+        @OP.def_rule("[]", proc { @lex_state == EXPR_FNAME }) do
+          Token("[]").set_text("[]")
+        end
+
+        @OP.def_rule("[]=", proc { @lex_state == EXPR_FNAME }) do
+          Token("[]=").set_text("[]=")
+        end
+
+        @OP.def_rule("[") do
+          @indent += 1
+          # if @lex_state == EXPR_FNAME
+          #   t = Token(TkfLBRACK)
+          # else
+          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+          #     t = Token(TkLBRACK)
+          #   elsif @lex_state == EXPR_ARG && @space_seen
+          #   else
+          #     t = Token(TkfLBRACK)
+          #   end
+          # end
+          t = Token(TkLBRACK)
+          @lex_state = EXPR_BEG
+          t.set_text("[")
+        end
+
+        @OP.def_rule("{") do
+          @indent += 1
+          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
+          #   t = Token(TkLBRACE)
+          # else
+          #   t = Token(TkfLBRACE)
+          # end
+          t = Token(TkLBRACE)
+          @lex_state = EXPR_BEG
+          t.set_text("{")
+        end
+
+        @OP.def_rule('\\') do #'
+          if getc == "\n"
+            @space_seen = true
+            @continue = true
+            Token(TkSPACE).set_text("\\\n")
+          else
+            ungetc
+            Token("\\").set_text("\\") #"
+          end
+        end
+
+        @OP.def_rule('%') do |_op, _io|
+          if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+            identify_quotation('%')
+          elsif peek(0) == '='
+            getc
+            Token(TkOPASGN, "%").set_text("%=")
+          elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/
+            identify_quotation('%')
+          else
+            @lex_state = EXPR_BEG
+            Token("%").set_text("%")
+          end
+        end
+
+        @OP.def_rule('$') do #'
+          identify_gvar
+        end
+
+        @OP.def_rule('@') do
+          if peek(0) =~ /[@\w]/
+            ungetc
+            identify_identifier
+          else
+            Token("@").set_text("@")
+          end
+        end
+
+        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
+        #   |op, io|
+        #   @indent += 1
+        #   @lex_state = EXPR_FNAME
+        # # @lex_state = EXPR_END
+        # # until @rests[0] == "\n" or @rests[0] == ";"
+        # #   rests.shift
+        # # end
+        # end
+
+        @OP.def_rule("__END__", proc { @prev_char_no == 0 && peek(0) =~ /[\r\n]/ }) do
+          throw :eof
+        end
+
+        @OP.def_rule("") do |op, io|
+          printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
+          if peek(0) =~ /[0-9]/
+            t = identify_number("")
+          elsif peek(0) =~ /[\w]/
+            t = identify_identifier
+          end
+          printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
+          t
+        end
+
+        p @OP if RubyLex.debug?
+      end
+
+      def identify_gvar
+        @lex_state = EXPR_END
+        str = String.new("$")
+
+        tk = case ch = getc
+             when %r{[~_*$?!@/\\;,=:<>".]}
+               str << ch
+               Token(TkGVAR, str)
+
+             when "-"
+               str << "-" << getc
+               Token(TkGVAR, str)
+
+             when "&", "`", "'", "+"
+               str << ch
+               Token(TkBACK_REF, str)
+
+             when /[1-9]/
+               str << ch
+               while (ch = getc) =~ /[0-9]/
+                 str << ch
+               end
+               ungetc
+               Token(TkNTH_REF)
+             when /\w/
+               ungetc
+               ungetc
+               return identify_identifier
+             else
+               ungetc
+               Token("$")
+             end
+        tk.set_text(str)
+      end
+
+      def identify_identifier
+        token = String.new
+        token.concat getc if peek(0) =~ /[$@]/
+        token.concat getc if peek(0) == "@"
+
+        while (ch = getc) =~ /\w|_/
+          print ":", ch, ":" if RubyLex.debug?
+          token.concat ch
+        end
+        ungetc
+
+        if ch == "!" || ch == "?"
+          token.concat getc
+        end
+        # fix token
+
+        # $stderr.puts "identifier - #{token}, state = #@lex_state"
+
+        case token
+        when /^\$/
+          return Token(TkGVAR, token).set_text(token)
+        when /^\@/
+          @lex_state = EXPR_END
+          return Token(TkIVAR, token).set_text(token)
+        end
+
+        if @lex_state != EXPR_DOT
+          print token, "\n" if RubyLex.debug?
+
+          token_c, *trans = TkReading2Token[token]
+          if token_c
+            # reserved word?
+
+            if @lex_state != EXPR_BEG &&
+               @lex_state != EXPR_FNAME &&
+               trans[1]
+              # modifiers
+              token_c = TkSymbol2Token[trans[1]]
+              @lex_state = trans[0]
+            else
+              if @lex_state != EXPR_FNAME
+                if ENINDENT_CLAUSE.include?(token)
+                  @indent += 1
+
+                  if ACCEPTS_COLON.include?(token)
+                    @colonblock_seen = true
+                  else
+                    @colonblock_seen = false
+                  end
+                elsif DEINDENT_CLAUSE.include?(token)
+                  @indent -= 1
+                  @colonblock_seen = false
+                end
+                @lex_state = trans[0]
+              else
+                @lex_state = EXPR_END
+              end
+            end
+            return Token(token_c, token).set_text(token)
+          end
+        end
+
+        if @lex_state == EXPR_FNAME
+          @lex_state = EXPR_END
+          if peek(0) == '='
+            token.concat getc
+          end
+        elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_END
+        end
+
+        if token[0, 1] =~ /[A-Z]/
+          return Token(TkCONSTANT, token).set_text(token)
+        elsif token[token.size - 1, 1] =~ /[!?]/
+          return Token(TkFID, token).set_text(token)
+        else
+          return Token(TkIDENTIFIER, token).set_text(token)
+        end
+      end
+
+      def identify_here_document
+        ch = getc
+        if ch == "-"
+          ch = getc
+        elsif ch == "~"
+          ch = getc
+          indent = true
+        end
+        if /['"`]/ =~ ch # '
+          lt = ch
+          quoted = ""
+          while (c = getc) && c != lt
+            quoted.concat c
+          end
+        else
+          lt = '"'
+          quoted = ch.dup
+          while (c = getc) && c =~ /\w/
+            quoted.concat c
+          end
+          ungetc
+        end
+
+        ltback, @ltype = @ltype, lt
+        reserve = String.new
+
+        while ch = getc
+          reserve << ch
+          if ch == "\\" #"
+            ch = getc
+            reserve << ch
+          elsif ch == "\n"
+            break
+          end
+        end
+
+        str = String.new
+        while (l = gets)
+          l.chomp!
+          if l == quoted
+            str = dedent(str) if indent
+            break
+          else
+            str << l.chomp << "\n"
+          end
+        end
+
+        @reader.divert_read_from(reserve)
+
+        @ltype = ltback
+        @lex_state = EXPR_END
+        Token(Ltype2Token[lt], str).set_text(str.dump)
+      end
+
+      def dedent(str)
+        lines = str.split("\n", -1)
+        dedent_amt = lines.map do |line|
+          line =~ /\S/ ? line.match(/^ */).offset(0)[1] : nil
+        end.compact.min || 0
+        return str if dedent_amt.zero?
+
+        lines.map { |line| line =~ /\S/ ? line.gsub(/^ {#{dedent_amt}}/, "") : line }.join("\n")
+      end
+
+      def identify_quotation(initial_char)
+        ch = getc
+        if lt = PERCENT_LTYPE[ch]
+          initial_char += ch
+          ch = getc
+        elsif ch =~ /\W/
+          lt = "\""
+        else
+          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
+        end
+        # if ch !~ /\W/
+        #   ungetc
+        #   next
+        # end
+        # @ltype = lt
+        @quoted = ch unless @quoted = PERCENT_PAREN[ch]
+        identify_string(lt, @quoted, ch, initial_char) if lt
+      end
+
+      def identify_number(start)
+        str = start.dup
+
+        if start == "+" || start == "-" || start == ""
+          start = getc
+          str << start
+        end
+
+        @lex_state = EXPR_END
+
+        if start == "0"
+          if peek(0) == "x"
+            ch = getc
+            str << ch
+            match = /[0-9a-f_]/
+          else
+            match = /[0-7_]/
+          end
+          while ch = getc
+            if ch !~ match
+              ungetc
+              break
+            else
+              str << ch
+            end
+          end
+          return Token(TkINTEGER).set_text(str)
+        end
+
+        type = TkINTEGER
+        allow_point = true
+        allow_e = true
+        while ch = getc
+          case ch
+          when /[0-9_]/
+            str << ch
+
+          when allow_point && "."
+            type = TkFLOAT
+            if peek(0) !~ /[0-9]/
+              ungetc
+              break
+            end
+            str << ch
+            allow_point = false
+
+          when allow_e && "e", allow_e && "E"
+            str << ch
+            type = TkFLOAT
+            if peek(0) =~ /[+-]/
+              str << getc
+            end
+            allow_e = false
+            allow_point = false
+          else
+            ungetc
+            break
+          end
+        end
+        Token(type).set_text(str)
+      end
+
+      def identify_string(ltype, quoted = ltype, opener = nil, initial_char = nil)
+        @ltype = ltype
+        @quoted = quoted
+        subtype = nil
+
+        str = String.new
+        str << initial_char if initial_char
+        str << (opener || quoted)
+
+        nest = 0
+        begin
+          while ch = getc
+            str << ch
+            if @quoted == ch
+              if nest == 0
+                break
+              else
+                nest -= 1
+              end
+            elsif opener == ch
+              nest += 1
+            elsif @ltype != "'" && @ltype != "]" && ch == "#"
+              ch = getc
+              if ch == "{"
+                subtype = true
+                str << ch << skip_inner_expression
+              else
+                ungetc(ch)
+              end
+            elsif ch == '\\' #'
+              str << read_escape
+            end
+          end
+          if @ltype == "/"
+            if peek(0) =~ /i|o|n|e|s/
+              str << getc
+            end
+          end
+          if subtype
+            Token(DLtype2Token[ltype], str)
+          else
+            Token(Ltype2Token[ltype], str)
+          end.set_text(str)
+        ensure
+          @ltype = nil
+          @quoted = nil
+          @lex_state = EXPR_END
+        end
+      end
+
+      def skip_inner_expression
+        res = String.new
+        nest = 0
+        while (ch = getc)
+          res << ch
+          if ch == '}'
+            break if nest == 0
+            nest -= 1
+          elsif ch == '{'
+            nest += 1
+          end
+        end
+        res
+      end
+
+      def identify_comment
+        @ltype = "#"
+        comment = String.new("#")
+        while ch = getc
+          if ch == "\\"
+            ch = getc
+            if ch == "\n"
+              ch = " "
+            else
+              comment << "\\"
+            end
+          else
+            if ch == "\n"
+              @ltype = nil
+              ungetc
+              break
+            end
+          end
+          comment << ch
+        end
+        Token(TkCOMMENT).set_text(comment)
+      end
+
+      def read_escape
+        res = String.new
+        case ch = getc
+        when /[0-7]/
+          ungetc ch
+          3.times do
+          case ch = getc
+          when /[0-7]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
+          res << ch
+          end
+
+        when "x"
+          res << ch
+          2.times do
+          case ch = getc
+          when /[0-9a-fA-F]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
+            res << ch
+          end
+
+        when "M"
+          res << ch
+          if (ch = getc) != '-'
+            ungetc
+          else
+            res << ch
+            if (ch = getc) == "\\" #"
+              res << ch
+              res << read_escape
+            else
+              res << ch
+            end
+          end
+
+        when "C", "c" #, "^"
+          res << ch
+          if ch == "C" && (ch = getc) != "-"
+            ungetc
+          else
+            res << ch
+            if (ch = getc) == "\\" #"
+              res << ch
+              res << read_escape
+            else
+              res << ch
+            end
+          end
+        else
+          res << ch
+        end
+        res
+      end
+    end
+  end
+end