rdoc 2.4.3 → 2.5

data/lib/rdoc/ri/paths.rb

@@ -1,45 +1,63 @@
 require 'rdoc/ri'
 
 ##
-# Encapsulate all the strangeness to do with finding out where to find RDoc
-# files
-#
-# We basically deal with three directories:
-#
-# 1. The 'system' documentation directory, which holds the documentation
-#    distributed with Ruby, and which is managed by the Ruby install process
-# 2. The 'site' directory, which contains site-wide documentation added
-#    locally.
-# 3. The 'user' documentation directory, stored under the user's own home
-#    directory.
-#
-# There's contention about all this, but for now:
-#
-# system:: $datadir/ri/<ver>/system/...
-# site::   $datadir/ri/<ver>/site/...
-# user::   ~/.rdoc
+# The directories where ri data lives.
 
 module RDoc::RI::Paths
 
   #:stopdoc:
   require 'rbconfig'
 
-  DOC_DIR  = "doc/rdoc"
+  version = RbConfig::CONFIG['ruby_version']
+
+  base    = File.join RbConfig::CONFIG['datadir'], "ri", version
+  SYSDIR  = File.join base, "system"
+  SITEDIR = File.join base, "site"
+  homedir = File.expand_path('~') ||
+            ENV['HOME'] || ENV['USERPROFILE'] || ENV['HOMEPATH']
+
+  HOMEDIR = if homedir then
+              File.join homedir, ".rdoc"
+            end
+  #:startdoc:
+
+  @gemdirs = nil
+
+  ##
+  # Iterates over each selected path yielding the directory and type.
+  #
+  # Yielded types:
+  # :system:: Where Ruby's ri data is stored.  Yielded when +system+ is
+  #           true
+  # :site:: Where ri for installed libraries are stored.  Yielded when
+  #         +site+ is true.  Normally no ri data is stored here.
+  # :home:: ~/.ri.  Yielded when +home+ is true.
+  # :gem:: ri data for an installed gem.  Yielded when +gems+ is true.
+  # :extra:: ri data directory from the command line.  Yielded for each
+  #          entry in +extra_dirs+
+
+  def self.each system, site, home, gems, *extra_dirs # :yields: directory, type
+    extra_dirs.each do |dir|
+      yield dir, :extra
+    end
 
-  VERSION = RbConfig::CONFIG['ruby_version']
+    yield SYSDIR,  :system if system
+    yield SITEDIR, :site   if site
+    yield HOMEDIR, :home   if home
 
-  base    = File.join(RbConfig::CONFIG['datadir'], "ri", VERSION)
-  SYSDIR  = File.join(base, "system")
-  SITEDIR = File.join(base, "site")
-  homedir = ENV['HOME'] || ENV['USERPROFILE'] || ENV['HOMEPATH']
+    gemdirs.each do |dir|
+      yield dir, :gem
+    end if gems
 
-  if homedir then
-    HOMEDIR = File.join(homedir, ".rdoc")
-  else
-    HOMEDIR = nil
+    nil
   end
 
-  begin
+  ##
+  # The latest installed gems' ri directories
+
+  def self.gemdirs
+    return @gemdirs if @gemdirs
+
     require 'rubygems' unless defined?(Gem) and defined?(Gem::Enable) and
                               Gem::Enable
 
@@ -63,31 +81,38 @@ module RDoc::RI::Paths
       end
     end
 
-    GEMDIRS = ri_paths.map { |k,v| v.last }.sort
+    @gemdirs = ri_paths.map { |k,v| v.last }.sort
   rescue LoadError
-    GEMDIRS = []
+    @gemdirs = []
   end
 
-  # Returns the selected documentation directories as an Array, or PATH if no
-  # overriding directories were given.
+  ##
+  # Returns existing directories from the selected documentation directories
+  # as an Array.
+  #
+  # See also ::each
+
+  def self.path(system, site, home, gems, *extra_dirs)
+    path = raw_path system, site, home, gems, *extra_dirs
 
-  def self.path(use_system, use_site, use_home, use_gems, *extra_dirs)
-    path = raw_path(use_system, use_site, use_home, use_gems, *extra_dirs)
-    return path.select { |directory| File.directory? directory }
+    path.select { |directory| File.directory? directory }
   end
 
-  # Returns the selected documentation directories including nonexistent
-  # directories.  Used to print out what paths were searched if no ri was
-  # found.
+  ##
+  # Returns selected documentation directories including nonexistent
+  # directories.
+  #
+  # See also ::each
 
-  def self.raw_path(use_system, use_site, use_home, use_gems, *extra_dirs)
+  def self.raw_path(system, site, home, gems, *extra_dirs)
     path = []
-    path << extra_dirs unless extra_dirs.empty?
-    path << SYSDIR if use_system
-    path << SITEDIR if use_site
-    path << HOMEDIR if use_home
-    path << GEMDIRS if use_gems
 
-    return path.flatten.compact
+    each(system, site, home, gems, *extra_dirs) do |dir, type|
+      path << dir
+    end
+
+    path.compact
   end
+
 end
+

data/lib/rdoc/ri/store.rb

@@ -0,0 +1,248 @@
+require 'rdoc/code_objects'
+require 'fileutils'
+
+##
+# A set of ri data.
+#
+# The store manages reading and writing ri data for a project (gem, path,
+# etc.) and maintains a cache of methods, classes and ancestors in the
+# store.
+
+class RDoc::RI::Store
+
+  ##
+  # Path this store reads or writes
+
+  attr_accessor :path
+
+  ##
+  # Type of ri datastore this was loaded from.  See RDoc::RI::Driver,
+  # RDoc::RI::Paths.
+
+  attr_accessor :type
+
+  attr_reader :cache
+
+  ##
+  # Creates a new Store of +type+ that will load or save to +path+
+
+  def initialize path, type = nil
+    @type = type
+    @path = path
+
+    @cache = {
+      :class_methods    => {},
+      :instance_methods => {},
+      :attributes       => {},
+      :modules          => [],
+      :ancestors        => {},
+    }
+  end
+
+  ##
+  # Ancestors cache accessor.  Maps a klass name to an Array of its ancestors
+  # in this store.  If Foo in this store inherits from Object, Kernel won't be
+  # listed (it will be included from ruby's ri store).
+
+  def ancestors
+    @cache[:ancestors]
+  end
+
+  ##
+  # Attributes cache accessor.  Maps a class to an Array of its attributes.
+
+  def attributes
+    @cache[:attributes]
+  end
+
+  ##
+  # Path to the cache file
+
+  def cache_path
+    File.join @path, 'cache.ri'
+  end
+
+  ##
+  # Path to the ri data for +klass_name+
+
+  def class_file klass_name
+    name = klass_name.split('::').last
+    File.join class_path(klass_name), "cdesc-#{name}.ri"
+  end
+
+  ##
+  # Class methods cache accessor.  Maps a class to an Array of its class
+  # methods (not full name).
+
+  def class_methods
+    @cache[:class_methods]
+  end
+
+  ##
+  # Path where data for +klass_name+ will be stored (methods or class data)
+
+  def class_path klass_name
+    File.join @path, *klass_name.split('::')
+  end
+
+  ##
+  # Friendly rendition of #path
+
+  def friendly_path
+    case type
+    when :gem    then
+      sep = Regexp.union(*['/', File::ALT_SEPARATOR].compact)
+      @path =~ /#{sep}doc#{sep}(.*?)#{sep}ri$/
+      "gem #{$1}"
+    when :home   then '~/.ri'
+    when :site   then 'ruby site'
+    when :system then 'ruby core'
+    else @path
+    end
+  end
+
+  def inspect # :nodoc:
+    "#<%s:0x%x %s %p>" % [self.class, object_id, @path, modules.sort]
+  end
+
+  ##
+  # Instance methods cache accessor.  Maps a class to an Array of its
+  # instance methods (not full name).
+
+  def instance_methods
+    @cache[:instance_methods]
+  end
+
+  ##
+  # Loads cache file for this store
+
+  def load_cache
+    open cache_path, 'rb' do |io|
+      @cache = Marshal.load io.read
+    end
+  rescue Errno::ENOENT
+  end
+
+  ##
+  # Loads ri data for +klass_name+
+
+  def load_class klass_name
+    open class_file(klass_name), 'rb' do |io|
+      Marshal.load io.read
+    end
+  end
+
+  ##
+  # Loads ri data for +method_name+ in +klass_name+
+
+  def load_method klass_name, method_name
+    open method_file(klass_name, method_name), 'rb' do |io|
+      Marshal.load io.read
+    end
+  end
+
+  ##
+  # Path to the ri data for +method_name+ in +klass_name+
+
+  def method_file klass_name, method_name
+    method_name = method_name.split('::').last
+    method_name =~ /#(.*)/
+    method_type = $1 ? 'i' : 'c'
+    method_name = $1 if $1
+
+    method_name = if ''.respond_to? :ord then
+                    method_name.gsub(/\W/) { "%%%02x" % $&[0].ord }
+                  else
+                    method_name.gsub(/\W/) { "%%%02x" % $&[0] }
+                  end
+
+    File.join class_path(klass_name), "#{method_name}-#{method_type}.ri"
+  end
+
+  ##
+  # Modules cache accessor.  An Array of all the modules (and classes) in the
+  # store.
+
+  def modules
+    @cache[:modules]
+  end
+
+  ##
+  # Writes the cache file for this store
+
+  def save_cache
+    # HACK mongrel-1.1.5 documents its files twice
+    @cache[:attributes].      each do |_, m| m.uniq!; m.sort! end
+    @cache[:class_methods].   each do |_, m| m.uniq!; m.sort! end
+    @cache[:instance_methods].each do |_, m| m.uniq!; m.sort! end
+
+    open cache_path, 'wb' do |io|
+      Marshal.dump @cache, io
+    end
+  end
+
+  ##
+  # Writes the ri data for +klass+
+
+  def save_class klass
+    FileUtils.mkdir_p class_path(klass.full_name)
+
+    @cache[:modules] << klass.full_name
+
+    path = class_file klass.full_name
+
+    begin
+      disk_klass = nil
+
+      open path, 'rb' do |io|
+        disk_klass = Marshal.load io.read
+      end
+
+      klass.merge disk_klass
+    rescue Errno::ENOENT
+    end
+
+    # BasicObject has no ancestors
+    ancestors = klass.ancestors.compact.map do |ancestor|
+      # HACK for classes we don't know about (class X < RuntimeError)
+      String === ancestor ? ancestor : ancestor.full_name
+    end
+
+    @cache[:ancestors][klass.full_name] ||= []
+    @cache[:ancestors][klass.full_name].push(*ancestors)
+
+    attributes = klass.attributes.map do |attribute|
+      "#{attribute.type} #{attribute.name}"
+    end
+
+    unless attributes.empty? then
+      @cache[:attributes][klass.full_name] ||= []
+      @cache[:attributes][klass.full_name].push(*attributes)
+    end
+
+    open path, 'wb' do |io|
+      Marshal.dump klass, io
+    end
+  end
+
+  ##
+  # Writes the ri data for +method+ on +klass+
+
+  def save_method klass, method
+    FileUtils.mkdir_p class_path(klass.full_name)
+
+    cache = if method.singleton then
+              @cache[:class_methods]
+            else
+              @cache[:instance_methods]
+            end
+    cache[klass.full_name] ||= []
+    cache[klass.full_name] << method.name
+
+    open method_file(klass.full_name, method.full_name), 'wb' do |io|
+      Marshal.dump method, io
+    end
+  end
+
+end
+

data/lib/rdoc/ruby_lex.rb

@@ -0,0 +1,1284 @@
+#--
+#   irb/ruby-lex.rb - ruby lexcal analyzer
+#   	$Release Version: 0.9.5$
+#   	$Revision: 17979 $
+#   	$Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+#++
+
+require "e2mmap"
+require "irb/slex"
+require "rdoc/ruby_token"
+require "stringio"
+
+##
+# Ruby lexer adapted from irb.
+#
+# The internals are not documented because they are scary.
+
+class RDoc::RubyLex
+
+  # :stopdoc:
+
+  extend Exception2MessageMapper
+
+  def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
+  def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
+  def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
+  def_exception(:TkReading2TokenDuplicateError,
+                "key duplicate(token_n='%s', key='%s')")
+  def_exception(:SyntaxError, "%s")
+
+  def_exception(:TerminateLineInput, "Terminate Line Input")
+
+  include RDoc::RubyToken
+  include IRB
+
+  attr_reader :continue
+  attr_reader :lex_state
+  attr_reader :reader
+
+  class << self
+    attr_accessor :debug_level
+  end
+
+  def self.debug?
+    @debug_level > 0
+  end
+
+  self.debug_level = 0
+
+  def initialize(content, options)
+    lex_init
+
+    if /\t/ =~ content then
+      tab_width = options.tab_width
+      content = content.split(/\n/).map do |line|
+        1 while line.gsub!(/\t+/) {
+          ' ' * (tab_width*$&.length - $`.length % tab_width)
+        }  && $~
+        line
+      end.join("\n")
+    end
+
+    content << "\n" unless content[-1, 1] == "\n"
+
+    set_input StringIO.new content
+
+    @base_char_no = 0
+    @char_no = 0
+    @exp_line_no = @line_no = 1
+    @here_readed = []
+    @readed = []
+    @rests = []
+    @seek = 0
+
+    @here_header = false
+    @indent = 0
+    @indent_stack = []
+    @lex_state = EXPR_BEG
+    @space_seen = false
+
+    @continue = false
+    @line = ""
+
+    @skip_space = false
+    @readed_auto_clean_up = false
+    @exception_on_syntax_error = true
+
+    @prompt = nil
+    @prev_seek = nil
+  end
+
+  def inspect # :nodoc:
+    "#<%s:0x%x lex_state %p space_seen %p>" % [
+      self.class, object_id,
+      @lex_state, @space_seen,
+    ]
+  end
+
+  attr_accessor :skip_space
+  attr_accessor :readed_auto_clean_up
+  attr_accessor :exception_on_syntax_error
+
+  attr_reader :seek
+  attr_reader :char_no
+  attr_reader :line_no
+  attr_reader :indent
+
+  # io functions
+  def set_input(io, p = nil, &block)
+    @io = io
+    if p.respond_to?(:call)
+      @input = p
+    elsif block_given?
+      @input = block
+    else
+      @input = Proc.new{@io.gets}
+    end
+  end
+
+  def get_readed
+    if idx = @readed.reverse.index("\n")
+      @base_char_no = idx
+    else
+      @base_char_no += @readed.size
+    end
+
+    readed = @readed.join("")
+    @readed = []
+    readed
+  end
+
+  def getc
+    while @rests.empty?
+      #      return nil unless buf_input
+      @rests.push nil unless buf_input
+    end
+    c = @rests.shift
+    if @here_header
+      @here_readed.push c
+    else
+      @readed.push c
+    end
+    @seek += 1
+    if c == "\n"
+      @line_no += 1 
+      @char_no = 0
+    else
+      @char_no += 1
+    end
+    c
+  end
+
+  def gets
+    l = ""
+    while c = getc
+      l.concat(c)
+      break if c == "\n"
+    end
+    return nil if l == "" and c.nil?
+    l
+  end
+
+  def eof?
+    @io.eof?
+  end
+
+  def getc_of_rests
+    if @rests.empty?
+      nil
+    else
+      getc
+    end
+  end
+
+  def ungetc(c = nil)
+    if @here_readed.empty?
+      c2 = @readed.pop
+    else
+      c2 = @here_readed.pop
+    end
+    c = c2 unless c
+    @rests.unshift c #c = 
+    @seek -= 1
+    if c == "\n"
+      @line_no -= 1 
+      if idx = @readed.reverse.index("\n")
+        @char_no = @readed.size - idx
+      else
+        @char_no = @base_char_no + @readed.size
+      end
+    else
+      @char_no -= 1
+    end
+  end
+
+  def peek_equal?(str)
+    chrs = str.split(//)
+    until @rests.size >= chrs.size
+      return false unless buf_input
+    end
+    @rests[0, chrs.size] == chrs
+  end
+
+  def peek_match?(regexp)
+    while @rests.empty?
+      return false unless buf_input
+    end
+    regexp =~ @rests.join("")
+  end
+
+  def peek(i = 0)
+    while @rests.size <= i
+      return nil unless buf_input
+    end
+    @rests[i]
+  end
+
+  def buf_input
+    prompt
+    line = @input.call
+    return nil unless line
+    @rests.concat line.split(//)
+    true
+  end
+  private :buf_input
+
+  def set_prompt(p = nil, &block)
+    p = block if block_given?
+    if p.respond_to?(:call)
+      @prompt = p
+    else
+      @prompt = Proc.new{print p}
+    end
+  end
+
+  def prompt
+    if @prompt
+      @prompt.call(@ltype, @indent, @continue, @line_no)
+    end
+  end
+
+  def initialize_input
+    @ltype = nil
+    @quoted = nil
+    @indent = 0
+    @indent_stack = []
+    @lex_state = EXPR_BEG
+    @space_seen = false
+    @here_header = false
+
+    @continue = false
+    prompt
+
+    @line = ""
+    @exp_line_no = @line_no
+  end
+
+  def each_top_level_statement
+    initialize_input
+    catch(:TERM_INPUT) do
+      loop do
+        begin
+          @continue = false
+          prompt
+          unless l = lex
+            throw :TERM_INPUT if @line == ''
+          else
+            #p l
+            @line.concat l
+            if @ltype or @continue or @indent > 0
+              next
+            end
+          end
+          if @line != "\n"
+            yield @line, @exp_line_no
+          end
+          break unless l
+          @line = ''
+          @exp_line_no = @line_no
+
+          @indent = 0
+          @indent_stack = []
+          prompt
+        rescue TerminateLineInput
+          initialize_input
+          prompt
+          get_readed
+        end
+      end
+    end
+  end
+
+  def lex
+    until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
+           !@continue or
+      tk.nil?)
+      #p tk
+      #p @lex_state
+      #p self
+    end
+    line = get_readed
+    #      print self.inspect
+    if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
+      nil
+    else
+      line
+    end
+  end
+
+  def token
+    #      require "tracer"
+    #      Tracer.on
+    @prev_seek = @seek
+    @prev_line_no = @line_no
+    @prev_char_no = @char_no
+    begin
+      begin
+        tk = @OP.match(self)
+        @space_seen = tk.kind_of?(TkSPACE)
+      rescue SyntaxError => e
+        raise RDoc::Error, "syntax error: #{e.message}" if
+          @exception_on_syntax_error
+
+        tk = TkError.new(@seek, @line_no, @char_no)
+      end
+    end while @skip_space and tk.kind_of?(TkSPACE)
+
+    if @readed_auto_clean_up
+      get_readed
+    end
+    #      Tracer.off
+    tk
+  end
+
+  ENINDENT_CLAUSE = [
+    "case", "class", "def", "do", "for", "if",
+    "module", "unless", "until", "while", "begin" #, "when"
+  ]
+
+  DEINDENT_CLAUSE = ["end" #, "when"
+  ]
+
+  PERCENT_LTYPE = {
+    "q" => "\'",
+    "Q" => "\"",
+    "x" => "\`",
+    "r" => "/",
+    "w" => "]",
+    "W" => "]",
+    "s" => ":"
+  }
+
+  PERCENT_PAREN = {
+    "{" => "}",
+    "[" => "]",
+    "<" => ">",
+    "(" => ")"
+  }
+
+  Ltype2Token = {
+    "\'" => TkSTRING,
+    "\"" => TkSTRING,
+    "\`" => TkXSTRING,
+    "/" => TkREGEXP,
+    "]" => TkDSTRING,
+    ":" => TkSYMBOL
+  }
+  DLtype2Token = {
+    "\"" => TkDSTRING,
+    "\`" => TkDXSTRING,
+    "/" => TkDREGEXP,
+  }
+
+  def lex_init()
+    @OP = IRB::SLex.new
+    @OP.def_rules("\0", "\004", "\032") do |op, io|
+      Token(TkEND_OF_SCRIPT)
+    end
+
+    @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
+      @space_seen = true
+      str = op
+      while (ch = getc) =~ /[ \t\f\r\13]/ do
+        str << ch
+      end
+      ungetc
+      Token TkSPACE, str
+    end
+
+    @OP.def_rule("#") do |op, io|
+      identify_comment
+    end
+
+    @OP.def_rule("=begin",
+                 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do 
+      |op, io|
+      @ltype = "="
+      res = ''
+      until (ch = getc) == "\n" do res << ch end
+      until peek_equal?("=end") && peek(4) =~ /\s/ do
+        until (ch = getc) == "\n" do res << ch end
+      end
+      res << gets
+      @ltype = nil
+      Token(TkRD_COMMENT, res)
+    end
+
+    @OP.def_rule("\n") do |op, io|
+      print "\\n\n" if RDoc::RubyLex.debug?
+      case @lex_state
+      when EXPR_BEG, EXPR_FNAME, EXPR_DOT
+        @continue = true
+      else
+        @continue = false
+        @lex_state = EXPR_BEG
+        until (@indent_stack.empty? || 
+               [TkLPAREN, TkLBRACK, TkLBRACE, 
+                 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+          @indent_stack.pop
+        end
+      end
+      @here_header = false
+      @here_readed = []
+      Token(TkNL)
+    end
+
+    @OP.def_rules("*", "**",
+                  "=", "==", "===", 
+                  "=~", "<=>",
+                  "<", "<=",
+                  ">", ">=", ">>") do
+      |op, io|
+      case @lex_state
+      when EXPR_FNAME, EXPR_DOT
+        @lex_state = EXPR_ARG
+      else
+        @lex_state = EXPR_BEG
+      end
+      Token(op)
+    end
+
+    @OP.def_rules("!", "!=", "!~") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token(op)
+    end
+
+    @OP.def_rules("<<") do
+      |op, io|
+      tk = nil
+      if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
+        (@lex_state != EXPR_ARG || @space_seen)
+        c = peek(0)
+        if /\S/ =~ c && (/["'`]/ =~ c || /[\w_]/ =~ c || c == "-")
+          tk = identify_here_document
+        end
+      end
+      unless tk
+        tk = Token(op)
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      end
+      tk
+    end
+
+    @OP.def_rules("'", '"') do
+      |op, io|
+      identify_string(op)
+    end
+
+    @OP.def_rules("`") do
+      |op, io|
+      if @lex_state == EXPR_FNAME
+        @lex_state = EXPR_END
+        Token(op)
+      else
+        identify_string(op)
+      end
+    end
+
+    @OP.def_rules('?') do
+      |op, io|
+      if @lex_state == EXPR_END
+        @lex_state = EXPR_BEG
+        Token(TkQUESTION)
+      else
+        ch = getc
+        if @lex_state == EXPR_ARG && ch =~ /\s/
+          ungetc
+          @lex_state = EXPR_BEG;
+          Token(TkQUESTION)
+        else
+          str = ch
+          if ch == '\\'
+            str << read_escape
+          end
+          @lex_state = EXPR_END
+          str << (ch.respond_to?(:ord) ? ch.ord : ch[0])
+          Token(TkINTEGER, str)
+        end
+      end
+    end
+
+    @OP.def_rules("&", "&&", "|", "||") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token(op)
+    end
+
+    @OP.def_rules("+=", "-=", "*=", "**=",
+                  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      op =~ /^(.*)=$/
+      Token(TkOPASGN, $1)
+    end
+
+    @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
+      |op, io|
+      @lex_state = EXPR_ARG
+      Token(op)
+    end
+
+    @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
+      |op, io|
+      @lex_state = EXPR_ARG
+      Token(op)
+    end
+
+    @OP.def_rules("+", "-") do
+      |op, io|
+      catch(:RET) do
+        if @lex_state == EXPR_ARG
+          if @space_seen and peek(0) =~ /[0-9]/
+            throw :RET, identify_number
+          else
+            @lex_state = EXPR_BEG
+          end
+        elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
+          throw :RET, identify_number
+        else
+          @lex_state = EXPR_BEG
+        end
+        Token(op)
+      end
+    end
+
+    @OP.def_rule(".") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      if peek(0) =~ /[0-9]/
+        ungetc
+        identify_number
+      else
+        # for "obj.if" etc.
+        @lex_state = EXPR_DOT
+        Token(TkDOT)
+      end
+    end
+
+    @OP.def_rules("..", "...") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token(op)
+    end
+
+    lex_int2
+  end
+
+  def lex_int2
+    @OP.def_rules("]", "}", ")") do
+      |op, io|
+      @lex_state = EXPR_END
+      @indent -= 1
+      @indent_stack.pop
+      Token(op)
+    end
+
+    @OP.def_rule(":") do
+      |op, io|
+      if @lex_state == EXPR_END || peek(0) =~ /\s/
+        @lex_state = EXPR_BEG
+        Token(TkCOLON)
+      else
+        @lex_state = EXPR_FNAME;
+        Token(TkSYMBEG)
+      end
+    end
+
+    @OP.def_rule("::") do
+      |op, io|
+      #      p @lex_state.id2name, @space_seen
+      if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
+        @lex_state = EXPR_BEG
+        Token(TkCOLON3)
+      else
+        @lex_state = EXPR_DOT
+        Token(TkCOLON2)
+      end
+    end
+
+    @OP.def_rule("/") do
+      |op, io|
+      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+        identify_string(op)
+      elsif peek(0) == '='
+        getc
+        @lex_state = EXPR_BEG
+        Token(TkOPASGN, "/") #/)
+      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
+        identify_string(op)
+      else
+        @lex_state = EXPR_BEG
+        Token("/") #/)
+      end
+    end
+
+    @OP.def_rules("^") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token("^")
+    end
+
+    #       @OP.def_rules("^=") do
+    # 	@lex_state = EXPR_BEG
+    # 	Token(OP_ASGN, :^)
+    #       end
+
+    @OP.def_rules(",") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token(op)
+    end
+
+    @OP.def_rules(";") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      until (@indent_stack.empty? || 
+             [TkLPAREN, TkLBRACK, TkLBRACE, 
+               TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+        @indent_stack.pop
+      end
+      Token(op)
+    end
+
+    @OP.def_rule("~") do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token("~")
+    end
+
+    @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
+      |op, io|
+      @lex_state = EXPR_BEG
+      Token("~")
+    end
+
+    @OP.def_rule("(") do
+      |op, io|
+      @indent += 1
+      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+        @lex_state = EXPR_BEG
+        tk_c = TkfLPAREN
+      else
+        @lex_state = EXPR_BEG
+        tk_c = TkLPAREN
+      end
+      @indent_stack.push tk_c
+      tk = Token(tk_c)
+    end
+
+    @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
+      |op, io|
+      @lex_state = EXPR_ARG
+      Token("[]")
+    end
+
+    @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
+      |op, io|
+      @lex_state = EXPR_ARG
+      Token("[]=")
+    end
+
+    @OP.def_rule("[") do
+      |op, io|
+      @indent += 1
+      if @lex_state == EXPR_FNAME
+        tk_c = TkfLBRACK
+      else
+        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+          tk_c = TkLBRACK
+        elsif @lex_state == EXPR_ARG && @space_seen
+          tk_c = TkLBRACK
+        else
+          tk_c = TkfLBRACK
+        end
+        @lex_state = EXPR_BEG
+      end
+      @indent_stack.push tk_c
+      Token(tk_c)
+    end
+
+    @OP.def_rule("{") do
+      |op, io|
+      @indent += 1
+      if @lex_state != EXPR_END && @lex_state != EXPR_ARG
+        tk_c = TkLBRACE
+      else
+        tk_c = TkfLBRACE
+      end
+      @lex_state = EXPR_BEG
+      @indent_stack.push tk_c
+      Token(tk_c)
+    end
+
+    @OP.def_rule('\\') do
+      |op, io|
+      if getc == "\n"
+        @space_seen = true
+        @continue = true
+        Token(TkSPACE)
+      else
+        ungetc
+        Token("\\")
+      end
+    end
+
+    @OP.def_rule('%') do
+      |op, io|
+      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+        identify_quotation
+      elsif peek(0) == '='
+        getc
+        Token(TkOPASGN, :%)
+      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
+        identify_quotation
+      else
+        @lex_state = EXPR_BEG
+        Token("%") #))
+      end
+    end
+
+    @OP.def_rule('$') do
+      |op, io|
+      identify_gvar
+    end
+
+    @OP.def_rule('@') do
+      |op, io|
+      if peek(0) =~ /[\w@]/
+        ungetc
+        identify_identifier
+      else
+        Token("@")
+      end
+    end
+
+    #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
+    # 	|op, io|
+    # 	@indent += 1
+    # 	@lex_state = EXPR_FNAME
+    # #	@lex_state = EXPR_END
+    # #	until @rests[0] == "\n" or @rests[0] == ";"
+    # #	  rests.shift
+    # #	end
+    #       end
+
+    @OP.def_rule("_") do
+      if peek_match?(/_END__/) and @lex_state == EXPR_BEG then
+        Token(TkEND_OF_SCRIPT)
+      else
+        ungetc
+        identify_identifier
+      end
+    end
+
+    @OP.def_rule("") do
+      |op, io|
+      printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
+      if peek(0) =~ /[0-9]/
+        t = identify_number
+      else
+        t = identify_identifier
+      end
+      printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
+      t
+    end
+
+    p @OP if RDoc::RubyLex.debug?
+  end
+
+  def identify_gvar
+    @lex_state = EXPR_END
+
+    case ch = getc
+    when /[~_*$?!@\/\\;,=:<>".]/   #"
+      Token(TkGVAR, "$" + ch)
+    when "-"
+      Token(TkGVAR, "$-" + getc)
+    when "&", "`", "'", "+"
+      Token(TkBACK_REF, "$"+ch)
+    when /[1-9]/
+      ref = ch
+      while (ch = getc) =~ /[0-9]/ do ref << ch end
+      ungetc
+      Token(TkNTH_REF, "$#{ref}")
+    when /\w/
+      ungetc
+      ungetc
+      identify_identifier
+    else
+      ungetc
+      Token("$")
+    end
+  end
+
+  def identify_identifier
+    token = ""
+    if peek(0) =~ /[$@]/
+      token.concat(c = getc)
+      if c == "@" and peek(0) == "@"
+        token.concat getc
+      end
+    end
+
+    # HACK to avoid a warning the regexp is hidden behind an eval
+    # HACK need a better way to detect oniguruma
+    @identifier_re ||= if defined? Encoding then
+                         eval '/[\p{Alnum}_]/u'
+                       else
+                         eval '/[\w\x80-\xff]/'
+                       end
+
+    while (ch = getc) =~ @identifier_re
+      print " :#{ch}: " if RDoc::RubyLex.debug?
+      token.concat ch
+    end
+
+    ungetc
+
+    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
+      token.concat getc
+    end
+
+    # almost fix token
+
+    case token
+    when /^\$/
+      return Token(TkGVAR, token)
+    when /^\@\@/
+      @lex_state = EXPR_END
+      # p Token(TkCVAR, token)
+      return Token(TkCVAR, token)
+    when /^\@/
+      @lex_state = EXPR_END
+      return Token(TkIVAR, token)
+    end
+
+    if @lex_state != EXPR_DOT
+      print token, "\n" if RDoc::RubyLex.debug?
+
+      token_c, *trans = TkReading2Token[token]
+      if token_c
+        # reserved word?
+
+        if (@lex_state != EXPR_BEG &&
+            @lex_state != EXPR_FNAME &&
+            trans[1])
+          # modifiers
+          token_c = TkSymbol2Token[trans[1]]
+          @lex_state = trans[0]
+        else
+          if @lex_state != EXPR_FNAME
+            if ENINDENT_CLAUSE.include?(token)
+              # check for ``class = val'' etc.
+              valid = true
+              case token
+              when "class"
+                valid = false unless peek_match?(/^\s*(<<|\w|::)/)
+              when "def"
+                valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
+              when "do"
+                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
+              when *ENINDENT_CLAUSE
+                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
+              else
+                # no nothing
+              end
+              if valid
+                if token == "do"
+                  if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
+                    @indent += 1
+                    @indent_stack.push token_c
+                  end
+                else
+                  @indent += 1
+                  @indent_stack.push token_c
+                end
+              end
+
+            elsif DEINDENT_CLAUSE.include?(token)
+              @indent -= 1
+              @indent_stack.pop
+            end
+            @lex_state = trans[0]
+          else
+            @lex_state = EXPR_END
+          end
+        end
+        return Token(token_c, token)
+      end
+    end
+
+    if @lex_state == EXPR_FNAME
+      @lex_state = EXPR_END
+      if peek(0) == '='
+        token.concat getc
+      end
+    elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
+      @lex_state = EXPR_ARG
+    else
+      @lex_state = EXPR_END
+    end
+
+    if token[0, 1] =~ /[A-Z]/
+      return Token(TkCONSTANT, token)
+    elsif token[token.size - 1, 1] =~ /[!?]/
+      return Token(TkFID, token)
+    else
+      return Token(TkIDENTIFIER, token)
+    end
+  end
+
+  def identify_here_document
+    ch = getc
+    #    if lt = PERCENT_LTYPE[ch]
+    if ch == "-"
+      ch = getc
+      indent = true
+    end
+    if /['"`]/ =~ ch
+      lt = ch
+      quoted = ""
+      while (c = getc) && c != lt
+        quoted.concat c
+      end
+    else
+      lt = '"'
+      quoted = ch.dup
+      while (c = getc) && c =~ /\w/
+        quoted.concat c
+      end
+      ungetc
+    end
+
+    ltback, @ltype = @ltype, lt
+    reserve = []
+    while ch = getc
+      reserve.push ch
+      if ch == "\\"
+        reserve.push ch = getc
+      elsif ch == "\n"
+        break
+      end
+    end
+
+    @here_header = false
+    doc = ''
+    while l = gets
+      l = l.sub(/(:?\r)?\n\z/, '')
+      if (indent ? l.strip : l) == quoted
+        break
+      end
+      doc << l
+    end
+
+    @here_header = true
+    @here_readed.concat reserve
+    while ch = reserve.pop
+      ungetc ch
+    end
+
+    @ltype = ltback
+    @lex_state = EXPR_END
+    Token(Ltype2Token[lt], doc)
+  end
+
+  def identify_quotation
+    ch = getc
+    if lt = PERCENT_LTYPE[ch]
+      ch = getc
+    elsif ch =~ /\W/
+      lt = "\""
+    else
+      raise RDoc::Error, "unknown type of %string #{ch.inspect}"
+    end
+    #     if ch !~ /\W/
+    #       ungetc
+    #       next
+    #     end
+    #@ltype = lt
+    @quoted = ch unless @quoted = PERCENT_PAREN[ch]
+    identify_string(lt, @quoted)
+  end
+
+  def identify_number
+    @lex_state = EXPR_END
+
+    num = ''
+
+    if peek(0) == "0" && peek(1) !~ /[.eE]/
+      num << getc
+
+      case peek(0)
+      when /[xX]/
+        ch = getc
+        match = /[0-9a-fA-F_]/
+      when /[bB]/
+        ch = getc
+        match = /[01_]/
+      when /[oO]/
+        ch = getc
+        match = /[0-7_]/
+      when /[dD]/
+        ch = getc
+        match = /[0-9_]/
+      when /[0-7]/
+        match = /[0-7_]/
+      when /[89]/
+        raise RDoc::Error, "Illegal octal digit"
+      else
+        return Token(TkINTEGER, num)
+      end
+
+      num << ch if ch
+
+      len0 = true
+      non_digit = false
+      while ch = getc
+        num << ch
+        if match =~ ch
+          if ch == "_"
+            if non_digit
+              raise RDoc::Error, "trailing `#{ch}' in number"
+            else
+              non_digit = ch
+            end
+          else
+            non_digit = false
+            len0 = false
+          end
+        else
+          ungetc
+          num[-1, 1] = ''
+          if len0
+            raise RDoc::Error, "numeric literal without digits"
+          end
+          if non_digit
+            raise RDoc::Error, "trailing `#{non_digit}' in number"
+          end
+          break
+        end
+      end
+      return Token(TkINTEGER, num)
+    end
+
+    type = TkINTEGER
+    allow_point = true
+    allow_e = true
+    non_digit = false
+    while ch = getc
+      num << ch
+      case ch
+      when /[0-9]/
+        non_digit = false
+      when "_"
+        non_digit = ch
+      when allow_point && "."
+        if non_digit
+          raise RDoc::Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkFLOAT
+        if peek(0) !~ /[0-9]/
+          type = TkINTEGER
+          ungetc
+          num[-1, 1] = ''
+          break
+        end
+        allow_point = false
+      when allow_e && "e", allow_e && "E"
+        if non_digit
+          raise RDoc::Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkFLOAT
+        if peek(0) =~ /[+-]/
+          num << getc
+        end
+        allow_e = false
+        allow_point = false
+        non_digit = ch
+      else
+        if non_digit
+          raise RDoc::Error, "trailing `#{non_digit}' in number"
+        end
+        ungetc
+        num[-1, 1] = ''
+        break
+      end
+    end
+
+    Token(type, num)
+  end
+
+  def identify_string(ltype, quoted = ltype)
+    @ltype = ltype
+    @quoted = quoted
+    str = @ltype.dup
+
+    subtype = nil
+    begin
+      nest = 0
+
+      while ch = getc
+        str << ch
+
+        if @quoted == ch and nest == 0
+          break
+        elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
+          ch = getc
+          subtype = true
+          if ch == "{" then
+            str << ch << skip_inner_expression
+          else
+            ungetc
+          end
+        elsif ch == '\\' and @ltype == "'" #'
+          case ch = getc
+          when "\\", "\n", "'"
+          else
+            ungetc
+          end
+        elsif ch == '\\' #'
+          str << read_escape
+        end
+
+        if PERCENT_PAREN.values.include?(@quoted) 
+          if PERCENT_PAREN[ch] == @quoted
+            nest += 1
+          elsif ch == @quoted
+            nest -= 1
+          end
+        end
+      end
+
+      if @ltype == "/"
+        if peek(0) =~ /i|m|x|o|e|s|u|n/
+          getc
+        end
+      end
+
+      if subtype
+        Token(DLtype2Token[ltype], str)
+      else
+        Token(Ltype2Token[ltype], str)
+      end
+    ensure
+      @ltype = nil
+      @quoted = nil
+      @lex_state = EXPR_END
+    end
+  end
+
+  def skip_inner_expression
+    res = ""
+    nest = 0
+    while (ch = getc)
+      res << ch
+      if ch == '}'
+        break if nest.zero?
+        nest -= 1
+      elsif ch == '{'
+        nest += 1
+      end
+    end
+    res
+  end
+
+  def identify_comment
+    @ltype = "#"
+
+    comment = '#'
+
+    while ch = getc
+      # if ch == "\\" #"
+      #   read_escape
+      # end
+      if ch == "\n"
+        @ltype = nil
+        ungetc
+        break
+      end
+
+      comment << ch
+    end
+
+    return Token(TkCOMMENT, comment)
+  end
+
+  def read_escape
+    escape = ''
+    ch = getc
+    escape << ch
+
+    case ch
+    when "\n", "\r", "\f"
+    when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
+    when /[0-7]/
+      ungetc ch
+      3.times do
+        ch = getc
+        escape << ch
+        case ch
+        when /[0-7]/
+        when nil
+          break
+        else
+          ungetc
+          break
+        end
+      end
+
+    when "x"
+      2.times do
+        ch = getc
+        escape << ch
+        case ch
+        when /[0-9a-fA-F]/
+        when nil
+          break
+        else
+          ungetc
+          break
+        end
+      end
+
+    when "M"
+      ch = getc
+      escape << ch
+      if ch != '-'
+        ungetc
+      else
+        ch = getc
+        escape << ch
+        if ch == "\\" #"
+          escape << read_escape
+        end
+      end
+
+    when "C", "c" #, "^"
+      if ch == "C" and (ch = getc) != "-"
+        escape << ch
+        ungetc
+      elsif (ch = getc) == "\\" #"
+        escape << ch << read_escape
+      end
+    else
+      # other characters 
+    end
+
+    escape
+  end
+
+  # :startdoc:
+
+end
+
+#RDoc::RubyLex.debug_level = 1
+