rdoc 3.1 → 6.3.3

data/lib/rdoc/parser/ruby.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This file contains stuff stolen outright from:
 #
@@ -7,17 +8,6 @@
 #       by Keiju ISHITSUKA (Nippon Rational Inc.)
 #
 
-require 'rdoc/ruby_token'
-require 'rdoc/ruby_lex'
-
-require 'rdoc/code_objects'
-require 'rdoc/token_stream'
-require 'rdoc/markup/pre_process'
-require 'rdoc/parser'
-require 'rdoc/parser/ruby_tools'
-
-$TOKEN_DEBUG ||= nil
-
 ##
 # Extracts code elements from a source file returning a TopLevel object
 # containing the constituent file elements.
@@ -32,6 +22,7 @@ $TOKEN_DEBUG ||= nil
 # * aliases
 # * private, public, protected
 # * private_class_function, public_class_function
+# * private_constant, public_constant
 # * module_function
 # * attr, attr_reader, attr_writer, attr_accessor
 # * extra accessors given on the command line
@@ -103,7 +94,7 @@ $TOKEN_DEBUG ||= nil
 # You can force the name of a method using the :method: directive:
 #
 #   ##
-#   # :method: woo_hoo!
+#   # :method: some_method!
 #
 # By default, meta-methods are instance methods.  To indicate that a method is
 # a singleton method instead use the :singleton-method: directive:
@@ -114,7 +105,10 @@ $TOKEN_DEBUG ||= nil
 # You can also use the :singleton-method: directive with a name:
 #
 #   ##
-#   # :singleton-method: woo_hoo!
+#   # :singleton-method: some_method!
+#
+# You can define arguments for metaprogrammed methods via either the
+# :call-seq:, :arg: or :args: directives.
 #
 # Additionally you can mark a method as an attribute by
 # using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:.  Just like
@@ -144,11 +138,13 @@ $TOKEN_DEBUG ||= nil
 # Note that by default, the :method: directive will be ignored if there is a
 # standard rdocable item following it.
 
+require 'ripper'
+require_relative 'ripper_state_lex'
+
 class RDoc::Parser::Ruby < RDoc::Parser
 
   parse_files_matching(/\.rbw?$/)
 
-  include RDoc::RubyToken
   include RDoc::TokenStream
   include RDoc::Parser::RubyTools
 
@@ -168,40 +164,110 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def initialize(top_level, file_name, content, options, stats)
     super
 
+    if /\t/ =~ content then
+      tab_width = @options.tab_width
+      content = content.split(/\n/).map do |line|
+        1 while line.gsub!(/\t+/) {
+          ' ' * (tab_width*$&.length - $`.length % tab_width)
+        }  && $~
+        line
+      end.join("\n")
+    end
+
     @size = 0
     @token_listeners = nil
-    @scanner = RDoc::RubyLex.new content, @options
-    @scanner.exception_on_syntax_error = false
+    content = RDoc::Encoding.remove_magic_comment content
+    @scanner = RDoc::Parser::RipperStateLex.parse(content)
+    @content = content
+    @scanner_point = 0
     @prev_seek = nil
+    @markup = @options.markup
+    @track_visibility = :nodoc != @options.visibility
+    @encoding = @options.encoding
 
     reset
   end
 
+  def tk_nl?(tk)
+    :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+  end
+
+  ##
+  # Retrieves the read token stream and replaces +pattern+ with +replacement+
+  # using gsub.  If the result is only a ";" returns an empty string.
+
+  def get_tkread_clean pattern, replacement # :nodoc:
+    read = get_tkread.gsub(pattern, replacement).strip
+    return '' if read == ';'
+    read
+  end
+
+  ##
+  # Extracts the visibility information for the visibility token +tk+
+  # and +single+ class type identifier.
+  #
+  # Returns the visibility type (a string), the visibility (a symbol) and
+  # +singleton+ if the methods following should be converted to singleton
+  # methods.
+
+  def get_visibility_information tk, single # :nodoc:
+    vis_type  = tk[:text]
+    singleton = single == SINGLE
+
+    vis =
+      case vis_type
+      when 'private'   then :private
+      when 'protected' then :protected
+      when 'public'    then :public
+      when 'private_class_method' then
+        singleton = true
+        :private
+      when 'public_class_method' then
+        singleton = true
+        :public
+      when 'module_function' then
+        singleton = true
+        :public
+      else
+        raise RDoc::Error, "Invalid visibility: #{tk.name}"
+      end
+
+    return vis_type, vis, singleton
+  end
+
   ##
   # Look for the first comment in a file that isn't a shebang line.
 
   def collect_first_comment
     skip_tkspace
-    comment = ''
+    comment = ''.dup
+    comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
     first_line = true
+    first_comment_tk_kind = nil
+    line_no = nil
 
     tk = get_tk
 
-    while TkCOMMENT === tk
-      if first_line and tk.text =~ /\A#!/ then
+    while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
+      comment_body = retrieve_comment_body(tk)
+      if first_line and comment_body =~ /\A#!/ then
         skip_tkspace
         tk = get_tk
-      elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+      elsif first_line and comment_body =~ /\A#\s*-\*-/ then
         first_line = false
         skip_tkspace
         tk = get_tk
       else
+        break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
+        first_comment_tk_kind = tk[:kind]
+
+        line_no = tk[:line_no] if first_line
         first_line = false
-        comment << tk.text << "\n"
+        comment << comment_body
         tk = get_tk
 
-        if TkNL === tk then
-          skip_tkspace false
+        if :on_nl === tk then
+          skip_tkspace_without_nl
           tk = get_tk
         end
       end
@@ -209,45 +275,61 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     unget_tk tk
 
-    comment
+    new_comment comment, line_no
   end
 
   ##
-  # Aborts with +msg+
+  # Consumes trailing whitespace from the token stream
 
-  def error(msg)
-    msg = make_message msg
+  def consume_trailing_spaces # :nodoc:
+    skip_tkspace_without_nl
+  end
 
-    abort msg
+  ##
+  # Creates a new attribute in +container+ with +name+.
+
+  def create_attr container, single, name, rw, comment # :nodoc:
+    att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+    record_location att
+
+    container.add_attribute att
+    @stats.add_attribute att
+
+    att
   end
 
   ##
-  # Look for a 'call-seq' in the comment, and override the normal parameter
-  # stuff
-  #--
-  # TODO handle undent
+  # Creates a module alias in +container+ at +rhs_name+ (or at the top-level
+  # for "::") with the name from +constant+.
 
-  def extract_call_seq(comment, meth)
-    if comment.sub!(/:?call-seq:(.*?)(^\s*#?\s*$|\z)/m, '') then
-      seq = $1
-      seq.gsub!(/^\s*\#\s*/, '')
-      meth.call_seq = seq
-    end
+  def create_module_alias container, constant, rhs_name # :nodoc:
+    mod = if rhs_name =~ /^::/ then
+            @store.find_class_or_module rhs_name
+          else
+            container.find_module_named rhs_name
+          end
 
-    meth
+    container.add_module_alias mod, rhs_name, constant, @top_level
+  end
+
+  ##
+  # Aborts with +msg+
+
+  def error(msg)
+    msg = make_message msg
+
+    abort msg
   end
 
   ##
-  # Looks for a true or false token.  Returns false if TkFALSE or TkNIL are
-  # found.
+  # Looks for a true or false token.
 
   def get_bool
     skip_tkspace
     tk = get_tk
-    case tk
-    when TkTRUE
+    if :on_kw == tk[:kind] && 'true' == tk[:text]
       true
-    when TkFALSE, TkNIL
+    elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
       false
     else
       unget_tk tk
@@ -260,58 +342,87 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # with :: separated named) and return the ultimate name, the associated
   # container, and the given name (with the ::).
 
-  def get_class_or_module(container)
+  def get_class_or_module container, ignore_constants = false
     skip_tkspace
     name_t = get_tk
-    given_name = ''
+    given_name = ''.dup
 
     # class ::A -> A is in the top level
-    case name_t
-    when TkCOLON2, TkCOLON3 then # bug
+    if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
       name_t = get_tk
       container = @top_level
       given_name << '::'
     end
 
-    skip_tkspace false
-    given_name << name_t.name
+    skip_tkspace_without_nl
+    given_name << name_t[:text]
 
-    while TkCOLON2 === peek_tk do
+    is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
+    new_modules = []
+    while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
       prev_container = container
-      container = container.find_module_named name_t.name
-      unless container then
-        container = prev_container.add_module RDoc::NormalModule, name_t.name
-      end
+      container = container.find_module_named name_t[:text]
+      container ||=
+        if ignore_constants then
+          c = RDoc::NormalModule.new name_t[:text]
+          c.store = @store
+          new_modules << [prev_container, c]
+          c
+        else
+          c = prev_container.add_module RDoc::NormalModule, name_t[:text]
+          c.ignore unless prev_container.document_children
+          @top_level.add_to_classes_or_modules c
+          c
+        end
+
+      record_location container
+
       get_tk
+      skip_tkspace
+      if :on_lparen == peek_tk[:kind] # ProcObjectInConstant::()
+        parse_method_or_yield_parameters
+        break
+      end
       name_t = get_tk
-      given_name << '::' << name_t.name
+      unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
+        raise RDoc::Error, "Invalid class or module definition: #{given_name}"
+      end
+      if prev_container == container and !ignore_constants
+        given_name = name_t[:text]
+      else
+        given_name << '::' + name_t[:text]
+      end
     end
-    skip_tkspace false
-    return [container, name_t, given_name]
+
+    skip_tkspace_without_nl
+
+    return [container, name_t, given_name, new_modules]
   end
 
   ##
   # Return a superclass, which can be either a constant of an expression
 
   def get_class_specification
-    tk = get_tk
-    return "self" if TkSELF === tk
-
-    res = ""
-    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
-      res += tk.name
-      tk = get_tk
+    tk = peek_tk
+    if tk.nil?
+      return ''
+    elsif :on_kw == tk[:kind] && 'self' == tk[:text]
+      return 'self'
+    elsif :on_gvar == tk[:kind]
+      return ''
     end
 
-    unget_tk(tk)
-    skip_tkspace false
+    res = get_constant
+
+    skip_tkspace_without_nl
 
     get_tkread # empty out read buffer
 
     tk = get_tk
+    return res unless tk
 
-    case tk
-    when TkNL, TkCOMMENT, TkSEMICOLON then
+    case tk[:kind]
+    when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
       unget_tk(tk)
       return res
     end
@@ -326,42 +437,164 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_constant
     res = ""
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
 
-    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
-      res += tk.name
+    while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
+      res += tk[:text]
       tk = get_tk
     end
 
-#      if res.empty?
-#        warn("Unexpected token #{tk} in constant")
-#      end
     unget_tk(tk)
     res
   end
 
   ##
-  # Get a constant that may be surrounded by parens
+  # Get an included module that may be surrounded by parens
+
+  def get_included_module_with_optional_parens
+    skip_tkspace_without_nl
+    get_tkread
+    tk = get_tk
+    end_token = get_end_token tk
+    return '' unless end_token
 
-  def get_constant_with_optional_parens
-    skip_tkspace false
     nest = 0
-    while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
-      get_tk
-      skip_tkspace
-      nest += 1
+    continue = false
+    only_constant = true
+
+    while tk != nil do
+      is_element_of_constant = false
+      case tk[:kind]
+      when :on_semicolon then
+        break if nest == 0
+      when :on_lbracket then
+        nest += 1
+      when :on_rbracket then
+        nest -= 1
+      when :on_lbrace then
+        nest += 1
+      when :on_rbrace then
+        nest -= 1
+        if nest <= 0
+          # we might have a.each { |i| yield i }
+          unget_tk(tk) if nest < 0
+          break
+        end
+      when :on_lparen then
+        nest += 1
+      when end_token[:kind] then
+        if end_token[:kind] == :on_rparen
+          nest -= 1
+          break if nest <= 0
+        else
+          break if nest <= 0
+        end
+      when :on_rparen then
+        nest -= 1
+      when :on_comment, :on_embdoc then
+        @read.pop
+        if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+          (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
+          break if !continue and nest <= 0
+        end
+      when :on_comma then
+        continue = true
+      when :on_ident then
+        continue = false if continue
+      when :on_kw then
+        case tk[:text]
+        when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
+          nest += 1
+        when 'if', 'unless', 'while', 'until', 'rescue'
+          # postfix if/unless/while/until/rescue must be EXPR_LABEL
+          nest += 1 unless (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0
+        when 'end'
+          nest -= 1
+          break if nest == 0
+        end
+      when :on_const then
+        is_element_of_constant = true
+      when :on_op then
+        is_element_of_constant = true if '::' == tk[:text]
+      end
+      only_constant = false unless is_element_of_constant
+      tk = get_tk
     end
 
-    name = get_constant
+    if only_constant
+      get_tkread_clean(/\s+/, ' ')
+    else
+      ''
+    end
+  end
 
-    while nest > 0
-      skip_tkspace
-      tk = get_tk
-      nest -= 1 if TkRPAREN === tk
+  ##
+  # Little hack going on here. In the statement:
+  #
+  #   f = 2*(1+yield)
+  #
+  # We see the RPAREN as the next token, so we need to exit early.  This still
+  # won't catch all cases (such as "a = yield + 1"
+
+  def get_end_token tk # :nodoc:
+    case tk[:kind]
+    when :on_lparen
+      token = RDoc::Parser::RipperStateLex::Token.new
+      token[:kind] = :on_rparen
+      token[:text] = ')'
+      token
+    when :on_rparen
+      nil
+    else
+      token = RDoc::Parser::RipperStateLex::Token.new
+      token[:kind] = :on_nl
+      token[:text] = "\n"
+      token
+    end
+  end
+
+  ##
+  # Retrieves the method container for a singleton method.
+
+  def get_method_container container, name_t # :nodoc:
+    prev_container = container
+    container = container.find_module_named(name_t[:text])
+
+    unless container then
+      constant = prev_container.constants.find do |const|
+        const.name == name_t[:text]
+      end
+
+      if constant then
+        parse_method_dummy prev_container
+        return
+      end
+    end
+
+    unless container then
+      # TODO seems broken, should starting at Object in @store
+      obj = name_t[:text].split("::").inject(Object) do |state, item|
+        state.const_get(item)
+      end rescue nil
+
+      type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
+
+      unless [Class, Module].include?(obj.class) then
+        warn("Couldn't find #{name_t[:text]}. Assuming it's a module")
+      end
+
+      if type == RDoc::NormalClass then
+        sclass = obj.superclass ? obj.superclass.name : nil
+        container = prev_container.add_class type, name_t[:text], sclass
+      else
+        container = prev_container.add_module type, name_t[:text]
+      end
+
+      record_location container
     end
 
-    name
+    container
   end
 
   ##
@@ -369,28 +602,36 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_symbol_or_name
     tk = get_tk
-    case tk
-    when TkSYMBOL then
-      text = tk.text.sub(/^:/, '')
+    case tk[:kind]
+    when :on_symbol then
+      text = tk[:text].sub(/^:/, '')
 
-      if TkASSIGN === peek_tk then
+      next_tk = peek_tk
+      if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
         get_tk
         text << '='
       end
 
       text
-    when TkId, TkOp then
-      tk.name
-    when TkAMPER,
-         TkDSTRING,
-         TkSTAR,
-         TkSTRING then
-      tk.text
+    when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
+      tk[:text]
+    when :on_tstring, :on_dstring then
+      tk[:text][1..-2]
     else
       raise RDoc::Error, "Name or symbol expected (got #{tk})"
     end
   end
 
+  ##
+  # Marks containers between +container+ and +ancestor+ as ignored
+
+  def suppress_parents container, ancestor # :nodoc:
+    while container and container != ancestor do
+      container.suppress unless container.documented?
+      container = container.parent
+    end
+  end
+
   ##
   # Look for directives in a normal comment block:
   #
@@ -399,77 +640,69 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   # This routine modifies its +comment+ parameter.
 
-  def look_for_directives_in(context, comment)
-    preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
-
-    preprocess.handle comment, context do |directive, param|
+  def look_for_directives_in container, comment
+    @preprocess.handle comment, container do |directive, param|
       case directive
-      when 'enddoc' then
-        context.done_documenting = true
-        ''
-      when 'main' then
-        @options.main_page = param if @options.respond_to? :main_page
-        ''
       when 'method', 'singleton-method',
            'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
         false # handled elsewhere
       when 'section' then
-        context.set_current_section param, comment
-        comment.replace ''
+        break unless container.kind_of?(RDoc::Context)
+        container.set_current_section param, comment.dup
+        comment.text = ''
         break
-      when 'startdoc' then
-        context.start_doc
-        context.force_documentation = true
-        ''
-      when 'stopdoc' then
-        context.stop_doc
-        ''
-      when 'title' then
-        @options.default_title = param if @options.respond_to? :default_title=
-        ''
       end
     end
 
-    remove_private_comments comment
+    comment.remove_private
   end
 
   ##
   # Adds useful info about the parser to +message+
 
   def make_message message
-    prefix = "#{@file_name}:"
+    prefix = "#{@file_name}:".dup
 
-    prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+    tk = peek_tk
+    prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk
 
     "#{prefix} #{message}"
   end
 
+  ##
+  # Creates a comment with the correct format
+
+  def new_comment comment, line_no = nil
+    c = RDoc::Comment.new comment, @top_level, :ruby
+    c.line = line_no
+    c.format = @markup
+    c
+  end
+
   ##
   # Creates an RDoc::Attr for the name following +tk+, setting the comment to
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
+    line_no = tk[:line_no]
+
     args = parse_symbol_arg 1
     if args.size > 0 then
       name = args[0]
       rw = "R"
-      skip_tkspace false
+      skip_tkspace_without_nl
       tk = get_tk
 
-      if TkCOMMA === tk then
+      if :on_comma == tk[:kind] then
         rw = "RW" if get_bool
       else
         unget_tk tk
       end
 
-      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
-      att.record_location @top_level
+      att = create_attr context, single, name, rw, comment
+      att.line   = line_no
 
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
-
-      context.add_attribute att if att.document_self
-
-      @stats.add_attribute att
     else
       warn "'attr' ignored - looks like a variable"
     end
@@ -480,14 +713,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
+    line_no = tk[:line_no]
+
     args = parse_symbol_arg
     rw = "?"
 
     tmp = RDoc::CodeObject.new
     read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
-    return unless tmp.document_self
+    # TODO In most other places we let the context keep track of document_self
+    # and add found items appropriately but here we do not.  I'm not sure why.
+    return if @track_visibility and not tmp.document_self
 
-    case tk.name
+    case tk[:text]
     when "attr_reader"   then rw = "R"
     when "attr_writer"   then rw = "W"
     when "attr_accessor" then rw = "RW"
@@ -496,11 +733,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     for name in args
-      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
-      att.record_location @top_level
-
-      context.add_attribute att
-      @stats.add_attribute att
+      att = create_attr context, single, name, rw, comment
+      att.line   = line_no
     end
   end
 
@@ -508,19 +742,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
+    line_no = tk[:line_no]
+
     skip_tkspace
 
-    if TkLPAREN === peek_tk then
+    if :on_lparen === peek_tk[:kind] then
       get_tk
       skip_tkspace
     end
 
     new_name = get_symbol_or_name
 
-    @scanner.instance_eval { @lex_state = EXPR_FNAME }
-
     skip_tkspace
-    if TkCOMMA === peek_tk then
+    if :on_comma === peek_tk[:kind] then
       get_tk
       skip_tkspace
     end
@@ -533,10 +767,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                          single == SINGLE)
-    al.record_location @top_level
+    record_location al
+    al.line   = line_no
 
     read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
-    context.add_alias al if al.document_self
+    context.add_alias al
     @stats.add_alias al
 
     al
@@ -546,260 +781,458 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Extracts call parameters from the token stream.
 
   def parse_call_parameters(tk)
-    end_token = case tk
-                when TkLPAREN, TkfLPAREN
-                  TkRPAREN
-                when TkRPAREN
+    end_token = case tk[:kind]
+                when :on_lparen
+                  :on_rparen
+                when :on_rparen
                   return ""
                 else
-                  TkNL
+                  :on_nl
                 end
     nest = 0
 
     loop do
-      case tk
-      when TkSEMICOLON
+      break if tk.nil?
+      case tk[:kind]
+      when :on_semicolon
         break
-      when TkLPAREN, TkfLPAREN
+      when :on_lparen
         nest += 1
       when end_token
-        if end_token == TkRPAREN
+        if end_token == :on_rparen
           nest -= 1
-          break if @scanner.lex_state == EXPR_END and nest <= 0
+          break if RDoc::Parser::RipperStateLex.end?(tk) and nest <= 0
         else
-          break unless @scanner.continue
+          break if RDoc::Parser::RipperStateLex.end?(tk)
         end
-      when TkCOMMENT, TkASSIGN, TkOPASGN
+      when :on_comment, :on_embdoc
         unget_tk(tk)
         break
-      when nil then
-        break
+      when :on_op
+        if tk[:text] =~ /^(.{1,2})?=$/
+          unget_tk(tk)
+          break
+        end
       end
       tk = get_tk
     end
-    res = get_tkread.tr("\n", " ").strip
-    res = "" if res == ";"
-    res
+
+    get_tkread_clean "\n", " "
   end
 
   ##
   # Parses a class in +context+ with +comment+
 
-  def parse_class(container, single, tk, comment)
-    declaration_context = container
-    container, name_t, given_name = get_class_or_module container
+  def parse_class container, single, tk, comment
+    line_no = tk[:line_no]
 
-    case name_t
-    when TkCONSTANT
-      name = name_t.name
-      superclass = '::Object'
+    declaration_context = container
+    container, name_t, given_name, = get_class_or_module container
 
-      if TkLT === peek_tk then
-        get_tk
-        skip_tkspace
-        superclass = get_class_specification
-        superclass = '(unknown)' if superclass.empty?
+    if name_t[:kind] == :on_const
+      cls = parse_class_regular container, declaration_context, single,
+        name_t, given_name, comment
+    elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
+      case name = get_class_specification
+      when 'self', container.name
+        read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+        parse_statements container, SINGLE
+        return # don't update line
+      else
+        cls = parse_class_singleton container, name, comment
       end
+    else
+      warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
+      return
+    end
 
-      cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
-      cls = declaration_context.add_class cls_type, given_name, superclass
+    cls.line   = line_no
 
-      read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
-      cls.record_location @top_level
-      cls.comment = comment if cls.document_self
+    # after end modifiers
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
 
-      @top_level.add_to_classes_or_modules cls
-      @stats.add_class cls
+    cls
+  end
 
-      parse_statements cls
-    when TkLSHFT
-      case name = get_class_specification
-      when "self", container.name
-        parse_statements container, SINGLE
-      else
-        other = RDoc::TopLevel.find_class_named name
+  ##
+  # Parses and creates a regular class
 
-        unless other then
-          other = container.add_module RDoc::NormalModule, name
-          other.record_location @top_level
-          other.comment = comment
-        end
+  def parse_class_regular container, declaration_context, single, # :nodoc:
+                          name_t, given_name, comment
+    superclass = '::Object'
 
-        # notify :nodoc: all if not a constant-named class/module
-        # (and remove any comment)
-        unless name =~ /\A(::)?[A-Z]/
-          other.document_self = nil
-          other.document_children = false
-          other.clear_comment
-        end
+    if given_name =~ /^::/ then
+      declaration_context = @top_level
+      given_name = $'
+    end
+
+    tk = peek_tk
+    if tk[:kind] == :on_op && tk[:text] == '<' then
+      get_tk
+      skip_tkspace
+      superclass = get_class_specification
+      superclass = '(unknown)' if superclass.empty?
+    end
+
+    cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
+    cls = declaration_context.add_class cls_type, given_name, superclass
+    cls.ignore unless container.document_children
+
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+    record_location cls
+
+    cls.add_comment comment, @top_level
+
+    @top_level.add_to_classes_or_modules cls
+    @stats.add_class cls
+
+    suppress_parents container, declaration_context unless cls.document_self
+
+    parse_statements cls
+
+    cls
+  end
 
-        @top_level.add_to_classes_or_modules other
-        @stats.add_class other
+  ##
+  # Parses a singleton class in +container+ with the given +name+ and
+  # +comment+.
 
-        read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
-        parse_statements(other, SINGLE)
+  def parse_class_singleton container, name, comment # :nodoc:
+    other = @store.find_class_named name
+
+    unless other then
+      if name =~ /^::/ then
+        name = $'
+        container = @top_level
       end
 
-    else
-      warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
+      other = container.add_module RDoc::NormalModule, name
+      record_location other
+
+      # class << $gvar
+      other.ignore if name.empty?
+
+      other.add_comment comment, @top_level
+    end
+
+    # notify :nodoc: all if not a constant-named class/module
+    # (and remove any comment)
+    unless name =~ /\A(::)?[A-Z]/ then
+      other.document_self = nil
+      other.document_children = false
+      other.clear_comment
     end
+
+    @top_level.add_to_classes_or_modules other
+    @stats.add_class other
+
+    read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
+    parse_statements(other, SINGLE)
+
+    other
   end
 
   ##
-  # Parses a constant in +context+ with +comment+
+  # Parses a constant in +context+ with +comment+.  If +ignore_constants+ is
+  # true, no found constants will be added to RDoc.
+
+  def parse_constant container, tk, comment, ignore_constants = false
+    line_no = tk[:line_no]
 
-  def parse_constant(container, tk, comment)
-    name = tk.name
-    skip_tkspace false
+    name = tk[:text]
+    skip_tkspace_without_nl
 
     return unless name =~ /^\w+$/
 
-    eq_tk = get_tk
+    new_modules = []
+    if :on_op == peek_tk[:kind] && '::' == peek_tk[:text] then
+      unget_tk tk
 
-    unless TkASSIGN === eq_tk then
-      unget_tk eq_tk
-      return
+      container, name_t, _, new_modules = get_class_or_module container, true
+
+      name = name_t[:text]
     end
 
-    nest = 0
-    get_tkread
+    is_array_or_hash = false
+    if peek_tk && :on_lbracket == peek_tk[:kind]
+      get_tk
+      nest = 1
+      while bracket_tk = get_tk
+        case bracket_tk[:kind]
+        when :on_lbracket
+          nest += 1
+        when :on_rbracket
+          nest -= 1
+          break if nest == 0
+        end
+      end
+      skip_tkspace_without_nl
+      is_array_or_hash = true
+    end
 
-    tk = get_tk
+    unless peek_tk && :on_op == peek_tk[:kind] && '=' == peek_tk[:text] then
+      return false
+    end
+    get_tk
 
-    if TkGT === tk then
-      unget_tk tk
-      unget_tk eq_tk
-      return
+    unless ignore_constants
+      new_modules.each do |prev_c, new_module|
+        prev_c.add_module_by_normal_module new_module
+        new_module.ignore unless prev_c.document_children
+        @top_level.add_to_classes_or_modules new_module
+      end
     end
 
-    rhs_name = ''
+    value = ''
+    con = RDoc::Constant.new name, value, comment
+
+    body = parse_constant_body container, con, is_array_or_hash
+
+    return unless body
+
+    con.value = body
+    record_location con
+    con.line   = line_no
+    read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+
+    return if is_array_or_hash
+
+    @stats.add_constant con
+    container.add_constant con
+
+    true
+  end
+
+  def parse_constant_body container, constant, is_array_or_hash # :nodoc:
+    nest     = 0
+    rhs_name = ''.dup
+
+    get_tkread
+
+    tk = get_tk
 
+    body = nil
     loop do
-      case tk
-      when TkSEMICOLON then
+      break if tk.nil?
+      if :on_semicolon == tk[:kind] then
         break if nest <= 0
-      when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
-           TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+      elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then
+        nest += 1
+      elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
         nest += 1
-      when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+      elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
+        if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
+          nest += 1
+        end
+      elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
+            (:on_kw == tk[:kind] && 'end' == tk[:text]) then
         nest -= 1
-      when TkCOMMENT then
-        if nest <= 0 && @scanner.lex_state == EXPR_END
-          unget_tk tk
+      elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
+        unget_tk tk
+        if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
+          body = get_tkread_clean(/^[ \t]+/, '')
+          read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
           break
+        else
+          read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
         end
-      when TkCONSTANT then
-        rhs_name << tk.name
-
-        if nest <= 0 and TkNL === peek_tk then
-          mod = if rhs_name =~ /^::/ then
-                  RDoc::TopLevel.find_class_or_module rhs_name
-                else
-                  container.find_module_named rhs_name
-                end
+      elsif :on_const == tk[:kind] then
+        rhs_name << tk[:text]
 
-          container.add_module_alias mod, name if mod
-          get_tk # TkNL
+        next_tk = peek_tk
+        if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then
+          create_module_alias container, constant, rhs_name unless is_array_or_hash
           break
         end
-      when TkNL then
-        if nest <= 0 &&
-           (@scanner.lex_state == EXPR_END || !@scanner.continue) then
+      elsif :on_nl == tk[:kind] then
+        if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
           unget_tk tk
           break
         end
-      when TkCOLON2, TkCOLON3 then
+      elsif :on_op == tk[:kind] && '::' == tk[:text]
         rhs_name << '::'
-      when nil then
-        break
       end
       tk = get_tk
     end
 
-    res = get_tkread.gsub(/^[ \t]+/, '').strip
-    res = "" if res == ";"
-
-    con = RDoc::Constant.new name, res, comment
-    con.record_location @top_level
-    read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
-
-    @stats.add_constant con
-    container.add_constant con if con.document_self
+    body ? body : get_tkread_clean(/^[ \t]+/, '')
   end
 
   ##
   # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
   # :method: or :attr: directives in +comment+.
 
-  def parse_comment(container, tk, comment)
-    line_no = tk.line_no
-    column  = tk.char_no
+  def parse_comment container, tk, comment
+    return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
+    column  = tk[:char_no]
+    line_no = comment.line.nil? ? tk[:line_no] : comment.line
+
+    comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    singleton = !!$~
 
-    singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    co =
+      if (comment.text = comment.text.sub(/^# +:?method: *(\S*).*?\n/i, '')) && !!$~ then
+        line_no += $`.count("\n")
+        parse_comment_ghost container, comment.text, $1, column, line_no, comment
+      elsif (comment.text = comment.text.sub(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '')) && !!$~ then
+        parse_comment_attr container, $1, $3, comment
+      end
 
-    # REFACTOR
-    if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
-      name = $1 unless $1.empty?
+    if co then
+      co.singleton = singleton
+      co.line      = line_no
+    end
 
-      meth = RDoc::GhostMethod.new get_tkread, name
-      meth.record_location @top_level
-      meth.singleton = singleton
+    true
+  end
 
-      meth.start_collecting_tokens
-      indent = TkSPACE.new nil, 1, 1
-      indent.set_text " " * column
+  ##
+  # Parse a comment that is describing an attribute in +container+ with the
+  # given +name+ and +comment+.
 
-      position_comment = TkCOMMENT.new nil, line_no, 1
-      position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
-      meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+  def parse_comment_attr container, type, name, comment # :nodoc:
+    return if name.empty?
 
-      meth.params = ''
+    rw = case type
+         when 'attr_reader' then 'R'
+         when 'attr_writer' then 'W'
+         else 'RW'
+         end
 
-      extract_call_seq comment, meth
+    create_attr container, NORMAL, name, rw, comment
+  end
 
-      return unless meth.name
+  def parse_comment_ghost container, text, name, column, line_no, # :nodoc:
+                          comment
+    name = nil if name.empty?
 
-      container.add_method meth if meth.document_self
+    meth = RDoc::GhostMethod.new get_tkread, name
+    record_location meth
 
-      meth.comment = comment
+    meth.start_collecting_tokens
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
+    meth.add_tokens [position_comment, newline, indent]
+
+    meth.params =
+      if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
+        $1
+      else
+        ''
+      end
 
-      @stats.add_method meth
-    elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?:) *(\S*).*?\n/i, '') then
-      rw = case $1
-           when 'attr_reader' then 'R'
-           when 'attr_writer' then 'W'
-           else 'RW'
-           end
+    comment.normalize
+    comment.extract_call_seq meth
 
-      name = $3 unless $3.empty?
+    return unless meth.name
 
-      # TODO authorize 'singleton-attr...'?
-      att = RDoc::Attr.new get_tkread, name, rw, comment
-      att.record_location @top_level
+    container.add_method meth
 
-      container.add_attribute att
+    meth.comment = comment
 
-      @stats.add_attribute att
-    end
+    @stats.add_method meth
 
-    true
+    meth
   end
 
   ##
-  # Parses an +include+ in +context+ with +comment+
+  # Creates an RDoc::Method on +container+ from +comment+ if there is a
+  # Signature section in the comment
 
-  def parse_include(context, comment)
+  def parse_comment_tomdoc container, tk, comment
+    return unless signature = RDoc::TomDoc.signature(comment)
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
+
+    name, = signature.split %r%[ \(]%, 2
+
+    meth = RDoc::GhostMethod.new get_tkread, name
+    record_location meth
+    meth.line      = line_no
+
+    meth.start_collecting_tokens
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
+    meth.add_tokens [position_comment, newline, indent]
+
+    meth.call_seq = signature
+
+    comment.normalize
+
+    return unless meth.name
+
+    container.add_method meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to
+  # +container+ # with +comment+
+
+  def parse_extend_or_include klass, container, comment # :nodoc:
     loop do
       skip_tkspace_comment
 
-      name = get_constant_with_optional_parens
-      context.add_include RDoc::Include.new(name, comment) unless name.empty?
+      name = get_included_module_with_optional_parens
+
+      unless name.empty? then
+        obj = container.add klass, name, comment
+        record_location obj
+      end
+
+      return if peek_tk.nil? || :on_comma != peek_tk[:kind]
 
-      return unless TkCOMMA === peek_tk
       get_tk
     end
   end
 
+  ##
+  # Parses identifiers that can create new methods or change visibility.
+  #
+  # Returns true if the comment was not consumed.
+
+  def parse_identifier container, single, tk, comment # :nodoc:
+    case tk[:text]
+    when 'private', 'protected', 'public', 'private_class_method',
+         'public_class_method', 'module_function' then
+      parse_visibility container, single, tk
+      return true
+    when 'private_constant', 'public_constant'
+      parse_constant_visibility container, single, tk
+      return true
+    when 'attr' then
+      parse_attr container, single, tk, comment
+    when /^attr_(reader|writer|accessor)$/ then
+      parse_attr_accessor container, single, tk, comment
+    when 'alias_method' then
+      parse_alias container, single, tk, comment
+    when 'require', 'include' then
+      # ignore
+    else
+      if comment.text =~ /\A#\#$/ then
+        case comment.text
+        when /^# +:?attr(_reader|_writer|_accessor)?:/ then
+          parse_meta_attr container, single, tk, comment
+        else
+          method = parse_meta_method container, single, tk, comment
+          method.params = container.params if
+            container.params
+          method.block_params = container.block_params if
+            container.block_params
+        end
+      end
+    end
+
+    false
+  end
+
   ##
   # Parses a meta-programmed attribute and creates an RDoc::Attr.
   #
@@ -837,9 +1270,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     tmp = RDoc::CodeObject.new
     read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
-    return unless tmp.document_self
 
-    if comment.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+    regexp = /^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i
+    if regexp =~ comment.text then
+      comment.text = comment.text.sub(regexp, '')
       rw = case $1
            when 'attr_reader' then 'R'
            when 'attr_writer' then 'W'
@@ -849,216 +1283,175 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     if name then
-      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
-      att.record_location @top_level
-
-      context.add_attribute att
-      @stats.add_attribute att
+      att = create_attr context, single, name, rw, comment
     else
       args.each do |attr_name|
-        att = RDoc::Attr.new(get_tkread, attr_name, rw, comment,
-                             single == SINGLE)
-        att.record_location @top_level
-
-        context.add_attribute att
-        @stats.add_attribute att
+        att = create_attr context, single, attr_name, rw, comment
       end
     end
+
+    att
   end
 
   ##
   # Parses a meta-programmed method
 
   def parse_meta_method(container, single, tk, comment)
-    line_no = tk.line_no
-    column  = tk.char_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     start_collecting_tokens
     add_token tk
     add_token_listener self
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
-    singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    singleton = !!$~
 
-    if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
-      name = $1 unless $1.empty?
-    end
+    name = parse_meta_method_name comment, tk
 
-    if name.nil? then
-      name_t = get_tk
-      case name_t
-      when TkSYMBOL then
-        name = name_t.text[1..-1]
-      when TkSTRING then
-        name = name_t.value[1..-2]
-      when TkASSIGN then # ignore
-        remove_token_listener self
-        return
-      else
-        warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
-        name = 'unknown'
-      end
-    end
+    return unless name
 
     meth = RDoc::MetaMethod.new get_tkread, name
-    meth.record_location @top_level
+    record_location meth
+    meth.line   = line_no
     meth.singleton = singleton
 
     remove_token_listener self
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new nil, 1, 1
-    indent.set_text " " * column
-
-    position_comment = TkCOMMENT.new nil, line_no, 1
-    position_comment.value = "# File #{@top_level.absolute_name}, line #{line_no}"
-    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
+    meth.add_tokens [position_comment, newline, indent]
     meth.add_tokens @token_stream
 
+    parse_meta_method_params container, single, meth, tk, comment
+
+    meth.comment = comment
+
+    @stats.add_method meth
+
+    meth
+  end
+
+  ##
+  # Parses the name of a metaprogrammed method.  +comment+ is used to
+  # determine the name while +tk+ is used in an error message if the name
+  # cannot be determined.
+
+  def parse_meta_method_name comment, tk # :nodoc:
+    if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+      return $1 unless $1.empty?
+    end
+
+    name_t = get_tk
+
+    if :on_symbol == name_t[:kind] then
+      name_t[:text][1..-1]
+    elsif :on_tstring == name_t[:kind] then
+      name_t[:text][1..-2]
+    elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore
+      remove_token_listener self
+
+      nil
+    else
+      warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'"
+      'unknown'
+    end
+  end
+
+  ##
+  # Parses the parameters and block for a meta-programmed method.
+
+  def parse_meta_method_params container, single, meth, tk, comment # :nodoc:
     token_listener meth do
       meth.params = ''
 
-      extract_call_seq comment, meth
+      look_for_directives_in meth, comment
+      comment.normalize
+      comment.extract_call_seq meth
 
-      container.add_method meth if meth.document_self
+      container.add_method meth
 
       last_tk = tk
 
       while tk = get_tk do
-        case tk
-        when TkSEMICOLON then
+        if :on_semicolon == tk[:kind] then
           break
-        when TkNL then
-          break unless last_tk and TkCOMMA === last_tk
-        when TkSPACE then
+        elsif :on_nl == tk[:kind] then
+          break unless last_tk and :on_comma == last_tk[:kind]
+        elsif :on_sp == tk[:kind] then
           # expression continues
+        elsif :on_kw == tk[:kind] && 'do' == tk[:text] then
+          parse_statements container, single, meth
+          break
         else
           last_tk = tk
         end
       end
     end
-
-    meth.comment = comment
-
-    @stats.add_method meth
   end
 
   ##
   # Parses a normal method defined by +def+
 
   def parse_method(container, single, tk, comment)
-    added_container = nil
-    meth = nil
+    singleton = nil
+    added_container = false
     name = nil
-    line_no = tk.line_no
-    column  = tk.char_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     start_collecting_tokens
     add_token tk
 
     token_listener self do
-      @scanner.instance_eval do @lex_state = EXPR_FNAME end
+      prev_container = container
+      name, container, singleton = parse_method_name container
+      added_container = container != prev_container
+    end
 
-      skip_tkspace
-      name_t = get_tk
-      back_tk = skip_tkspace
-      meth = nil
-      added_container = false
+    return unless name
 
-      dot = get_tk
-      if TkDOT === dot or TkCOLON2 === dot then
-        @scanner.instance_eval do @lex_state = EXPR_FNAME end
-        skip_tkspace
-        name_t2 = get_tk
-
-        case name_t
-        when TkSELF, TkMOD then
-          name = name_t2.name
-        when TkCONSTANT then
-          name = name_t2.name
-          prev_container = container
-          container = container.find_module_named(name_t.name)
-          unless container then
-            added_container = true
-            obj = name_t.name.split("::").inject(Object) do |state, item|
-              state.const_get(item)
-            end rescue nil
-
-            type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
-
-            unless [Class, Module].include?(obj.class) then
-              warn("Couldn't find #{name_t.name}. Assuming it's a module")
-            end
+    meth = RDoc::AnyMethod.new get_tkread, name
+    look_for_directives_in meth, comment
+    meth.singleton = single == SINGLE ? true : singleton
 
-            if type == RDoc::NormalClass then
-              sclass = obj.superclass ? obj.superclass.name : nil
-              container = prev_container.add_class type, name_t.name, sclass
-            else
-              container = prev_container.add_module type, name_t.name
-            end
+    record_location meth
+    meth.line   = line_no
 
-            container.record_location @top_level
-          end
-        when TkIDENTIFIER, TkIVAR, TkGVAR then
-          dummy = RDoc::Context.new
-          dummy.parent = container
-          skip_method dummy
-          return
-        when TkTRUE, TkFALSE, TkNIL then
-          klass_name = "#{name_t.name.capitalize}Class"
-          container = RDoc::TopLevel.find_class_named klass_name
-          container ||= @top_level.add_class RDoc::NormalClass, klass_name
+    meth.start_collecting_tokens
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
+    token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
+    meth.add_tokens [token, newline, indent]
+    meth.add_tokens @token_stream
 
-          name = name_t2.name
-        else
-          warn "unexpected method name token #{name_t.inspect}"
-          # break
-          skip_method container
-          return
-        end
+    parse_method_params_and_body container, single, meth, added_container
 
-        meth = RDoc::AnyMethod.new(get_tkread, name)
-        meth.singleton = true
-      else
-        unget_tk dot
-        back_tk.reverse_each do |token|
-          unget_tk token
-        end
+    comment.normalize
+    comment.extract_call_seq meth
 
-        name = case name_t
-               when TkSTAR, TkAMPER then
-                 name_t.text
-               else
-                 unless name_t.respond_to? :name then
-                   warn "expected method name token, . or ::, got #{name_t.inspect}"
-                   skip_method container
-                   return
-                 end
-                 name_t.name
-               end
-
-        meth = RDoc::AnyMethod.new get_tkread, name
-        meth.singleton = (single == SINGLE)
-      end
-    end
+    meth.comment = comment
 
-    meth.record_location @top_level
+    # after end modifiers
+    read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
 
-    meth.start_collecting_tokens
-    indent = TkSPACE.new nil, 1, 1
-    indent.set_text " " * column
+    @stats.add_method meth
+  end
 
-    token = TkCOMMENT.new nil, line_no, 1
-    token.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
-    meth.add_tokens [token, NEWLINE_TOKEN, indent]
-    meth.add_tokens @token_stream
+  ##
+  # Parses the parameters and body of +meth+
 
+  def parse_method_params_and_body container, single, meth, added_container
     token_listener meth do
-      @scanner.instance_eval do @continue = false end
       parse_method_parameters meth
 
-      if meth.document_self then
+      if meth.document_self or not @track_visibility then
         container.add_method meth
       elsif added_container then
         container.document_self = false
@@ -1067,7 +1460,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       # Having now read the method parameters and documentation modifiers, we
       # now know whether we have to rename #initialize to ::new
 
-      if name == "initialize" && !meth.singleton then
+      if meth.name == "initialize" && !meth.singleton then
         if meth.dont_rename_initialize then
           meth.visibility = :protected
         else
@@ -1079,12 +1472,108 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       parse_statements container, single, meth
     end
+  end
 
-    extract_call_seq comment, meth
+  ##
+  # Parses a method that needs to be ignored.
 
-    meth.comment = comment
+  def parse_method_dummy container
+    dummy = RDoc::Context.new
+    dummy.parent = container
+    dummy.store  = container.store
+    skip_method dummy
+  end
 
-    @stats.add_method meth
+  ##
+  # Parses the name of a method in +container+.
+  #
+  # Returns the method name, the container it is in (for def Foo.name) and if
+  # it is a singleton or regular method.
+
+  def parse_method_name container # :nodoc:
+    skip_tkspace
+    name_t = get_tk
+    back_tk = skip_tkspace_without_nl
+    singleton = false
+
+    dot = get_tk
+    if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then
+      singleton = true
+
+      name, container = parse_method_name_singleton container, name_t
+    else
+      unget_tk dot
+      back_tk.reverse_each do |token|
+        unget_tk token
+      end
+
+      name = parse_method_name_regular container, name_t
+    end
+
+    return name, container, singleton
+  end
+
+  ##
+  # For the given +container+ and initial name token +name_t+ the method name
+  # is parsed from the token stream for a regular method.
+
+  def parse_method_name_regular container, name_t # :nodoc:
+    if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then
+      name_t[:text]
+    else
+      unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then
+        warn "expected method name token, . or ::, got #{name_t.inspect}"
+        skip_method container
+        return
+      end
+      name_t[:text]
+    end
+  end
+
+  ##
+  # For the given +container+ and initial name token +name_t+ the method name
+  # and the new +container+ (if necessary) are parsed from the token stream
+  # for a singleton method.
+
+  def parse_method_name_singleton container, name_t # :nodoc:
+    skip_tkspace
+    name_t2 = get_tk
+
+    if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then
+      # NOTE: work around '[' being consumed early
+      if :on_lbracket == name_t2[:kind]
+        get_tk
+        name = '[]'
+      else
+        name = name_t2[:text]
+      end
+    elsif :on_const == name_t[:kind] then
+      name = name_t2[:text]
+
+      container = get_method_container container, name_t
+
+      return unless container
+
+      name
+    elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then
+      parse_method_dummy container
+
+      name = nil
+    elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then
+      klass_name = "#{name_t[:text].capitalize}Class"
+      container = @store.find_class_named klass_name
+      container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+      name = name_t2[:text]
+    else
+      warn "unexpected method name token #{name_t.inspect}"
+      # break
+      skip_method container
+
+      name = nil
+    end
+
+    return name, container
   end
 
   ##
@@ -1092,63 +1581,61 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_method_or_yield_parameters(method = nil,
                                        modifiers = RDoc::METHOD_MODIFIERS)
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
+    end_token = get_end_token tk
+    return '' unless end_token
 
-    # Little hack going on here. In the statement
-    #  f = 2*(1+yield)
-    # We see the RPAREN as the next token, so we need
-    # to exit early. This still won't catch all cases
-    # (such as "a = yield + 1"
-    end_token = case tk
-                when TkLPAREN, TkfLPAREN
-                  TkRPAREN
-                when TkRPAREN
-                  return ""
-                else
-                  TkNL
-                end
     nest = 0
+    continue = false
 
-    loop do
-      case tk
-      when TkSEMICOLON then
+    while tk != nil do
+      case tk[:kind]
+      when :on_semicolon then
         break if nest == 0
-      when TkLBRACE, TkfLBRACE then
+      when :on_lbracket then
+        nest += 1
+      when :on_rbracket then
+        nest -= 1
+      when :on_lbrace then
         nest += 1
-      when TkRBRACE then
+      when :on_rbrace then
         nest -= 1
         if nest <= 0
           # we might have a.each { |i| yield i }
           unget_tk(tk) if nest < 0
           break
         end
-      when TkLPAREN, TkfLPAREN then
+      when :on_lparen then
         nest += 1
-      when end_token then
-        if end_token == TkRPAREN
+      when end_token[:kind] then
+        if end_token[:kind] == :on_rparen
           nest -= 1
-          break if @scanner.lex_state == EXPR_END and nest <= 0
+          break if nest <= 0
         else
-          break unless @scanner.continue
+          break
         end
-      when TkRPAREN then
+      when :on_rparen then
         nest -= 1
-      when method && method.block_params.nil? && TkCOMMENT then
-        unget_tk tk
-        read_documentation_modifiers method, modifiers
+      when :on_comment, :on_embdoc then
         @read.pop
-      when TkCOMMENT then
-        @read.pop
-      when nil then
-        break
+        if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+          (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
+          if method && method.block_params.nil? then
+            unget_tk tk
+            read_documentation_modifiers method, modifiers
+          end
+          break if !continue and nest <= 0
+        end
+      when :on_comma then
+        continue = true
+      when :on_ident then
+        continue = false if continue
       end
       tk = get_tk
     end
 
-    res = get_tkread.gsub(/\s+/, ' ').strip
-    res = '' if res == ';'
-    res
+    get_tkread_clean(/\s+/, ' ')
   end
 
   ##
@@ -1159,34 +1646,37 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   # and add this as the block_params for the method
 
-  def parse_method_parameters(method)
+  def parse_method_parameters method
     res = parse_method_or_yield_parameters method
 
     res = "(#{res})" unless res =~ /\A\(/
     method.params = res unless method.params
 
-    if method.block_params.nil? then
-      skip_tkspace false
-      read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
-    end
+    return if  method.block_params
+
+    skip_tkspace_without_nl
+    read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
   end
 
   ##
   # Parses an RDoc::NormalModule in +container+ with +comment+
 
-  def parse_module(container, single, tk, comment)
+  def parse_module container, single, tk, comment
     container, name_t, = get_class_or_module container
 
-    name = name_t.name
+    name = name_t[:text]
 
     mod = container.add_module RDoc::NormalModule, name
-    mod.record_location @top_level
+    mod.ignore unless container.document_children
+    record_location mod
 
     read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
-    mod.comment = comment if mod.document_self
-    parse_statements(mod)
+    mod.add_comment comment, @top_level
+    parse_statements mod
+
+    # after end modifiers
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
 
-    @top_level.add_to_classes_or_modules mod
     @stats.add_module mod
   end
 
@@ -1197,12 +1687,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
     skip_tkspace_comment
     tk = get_tk
 
-    if TkLPAREN === tk then
+    if :on_lparen == tk[:kind] then
       skip_tkspace_comment
       tk = get_tk
     end
 
-    name = tk.text if TkSTRING === tk
+    name = tk[:text][1..-2] if :on_tstring == tk[:kind]
 
     if name then
       @top_level.add_require RDoc::Require.new(name, comment)
@@ -1212,10 +1702,44 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
-  # The core of the ruby parser.
+  # Parses a rescue
+
+  def parse_rescue
+    skip_tkspace_without_nl
+
+    while tk = get_tk
+      case tk[:kind]
+      when :on_nl, :on_semicolon, :on_comment then
+        break
+      when :on_comma then
+        skip_tkspace_without_nl
+
+        get_tk if :on_nl == peek_tk[:kind]
+      end
+
+      skip_tkspace_without_nl
+    end
+  end
+
+  ##
+  # Retrieve comment body without =begin/=end
+
+  def retrieve_comment_body(tk)
+    if :on_embdoc == tk[:kind]
+      tk[:text].gsub(/\A=begin.*\n/, '').gsub(/=end\n?\z/, '')
+    else
+      tk[:text]
+    end
+  end
+
+  ##
+  # The core of the Ruby parser.
 
   def parse_statements(container, single = NORMAL, current_method = nil,
-                       comment = '')
+                       comment = new_comment(''))
+    raise 'no' unless RDoc::Comment === comment
+    comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+
     nest = 1
     save_visibility = container.visibility
 
@@ -1223,35 +1747,63 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     while tk = get_tk do
       keep_comment = false
+      try_parse_comment = false
 
-      non_comment_seen = true unless TkCOMMENT === tk
+      non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
 
-      case tk
-      when TkNL then
-        skip_tkspace
-        tk = get_tk
+      case tk[:kind]
+      when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
+        if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+          skip_tkspace
+          tk = get_tk
+        else
+          past_tokens = @read.size > 1 ? @read[0..-2] : []
+          nl_position = 0
+          past_tokens.reverse.each_with_index do |read_tk, i|
+            if read_tk =~ /^\n$/ then
+              nl_position = (past_tokens.size - 1) - i
+              break
+            elsif read_tk =~ /^#.*\n$/ then
+              nl_position = ((past_tokens.size - 1) - i) + 1
+              break
+            end
+          end
+          comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
+          unless comment_only_line then
+            tk = get_tk
+          end
+        end
 
-        if TkCOMMENT === tk then
+        if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
           if non_comment_seen then
             # Look for RDoc in a comment about to be thrown away
             non_comment_seen = parse_comment container, tk, comment unless
               comment.empty?
 
             comment = ''
+            comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
           end
 
-          while TkCOMMENT === tk do
-            comment << tk.text << "\n"
+          line_no = nil
+          while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
+            comment_body = retrieve_comment_body(tk)
+            line_no = tk[:line_no] if comment.empty?
+            comment += comment_body
+            comment << "\n" unless comment_body =~ /\n\z/
 
-            tk = get_tk        # this is the newline
-            skip_tkspace false # leading spaces
+            if comment_body.size > 1 && comment_body =~ /\n\z/ then
+              skip_tkspace_without_nl # leading spaces
+            end
             tk = get_tk
           end
 
+          comment = new_comment comment, line_no
+
           unless comment.empty? then
             look_for_directives_in container, comment
 
             if container.done_documenting then
+              throw :eof if RDoc::TopLevel === container
               container.ongoing_visibility = save_visibility
             end
           end
@@ -1263,171 +1815,179 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
         unget_tk tk
         keep_comment = true
+        container.current_line_visibility = nil
 
-      when TkCLASS then
-        if container.document_children then
+      when :on_kw then
+        case tk[:text]
+        when 'class' then
           parse_class container, single, tk, comment
-        else
-          nest += 1
-        end
 
-      when TkMODULE then
-        if container.document_children then
+        when 'module' then
           parse_module container, single, tk, comment
-        else
-          nest += 1
-        end
 
-      when TkDEF then
-        if container.document_self then
+        when 'def' then
           parse_method container, single, tk, comment
-        else
+
+        when 'alias' then
+          parse_alias container, single, tk, comment unless current_method
+
+        when 'yield' then
+          if current_method.nil? then
+            warn "Warning: yield outside of method" if container.document_self
+          else
+            parse_yield container, single, tk, current_method
+          end
+
+        when 'until', 'while' then
+          if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
+            nest += 1
+            skip_optional_do_after_expression
+          end
+
+        # Until and While can have a 'do', which shouldn't increase the nesting.
+        # We can't solve the general case, but we can handle most occurrences by
+        # ignoring a do at the end of a line.
+
+        # 'for' is trickier
+        when 'for' then
           nest += 1
-        end
+          skip_for_variable
+          skip_optional_do_after_expression
 
-      when TkCONSTANT then
-        if container.document_self then
-          parse_constant container, tk, comment
-        end
+        when 'case', 'do', 'if', 'unless', 'begin' then
+          if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
+            nest += 1
+          end
 
-      when TkALIAS then
-        if container.document_self and not current_method then
-          parse_alias container, single, tk, comment
-        end
+        when 'super' then
+          current_method.calls_super = true if current_method
 
-      when TkYIELD then
-        if current_method.nil? then
-          warn "Warning: yield outside of method" if container.document_self
-        else
-          parse_yield container, single, tk, current_method
-        end
+        when 'rescue' then
+          parse_rescue
 
-      # Until and While can have a 'do', which shouldn't increase the nesting.
-      # We can't solve the general case, but we can handle most occurrences by
-      # ignoring a do at the end of a line.
+        when 'end' then
+          nest -= 1
+          if nest == 0 then
+            container.ongoing_visibility = save_visibility
 
-      when TkUNTIL, TkWHILE then
-        nest += 1
-        skip_optional_do_after_expression
+            parse_comment container, tk, comment unless comment.empty?
 
-      # 'for' is trickier
-      when TkFOR then
-        nest += 1
-        skip_for_variable
-        skip_optional_do_after_expression
+            return
+          end
+        end
 
-      when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
-        nest += 1
+      when :on_const then
+        unless parse_constant container, tk, comment, current_method then
+          try_parse_comment = true
+        end
 
-      when TkIDENTIFIER then
+      when :on_ident then
         if nest == 1 and current_method.nil? then
-          case tk.name
-          when 'private', 'protected', 'public', 'private_class_method',
-               'public_class_method', 'module_function' then
-            parse_visibility container, single, tk
-            keep_comment = true
-          when 'attr' then
-            parse_attr container, single, tk, comment
-          when /^attr_(reader|writer|accessor)$/ then
-            parse_attr_accessor container, single, tk, comment
-          when 'alias_method' then
-            parse_alias container, single, tk, comment if
-              container.document_self
-          when 'require', 'include' then
-            # ignore
-          else
-            if container.document_self and comment =~ /\A#\#$/ then
-              case comment
-              when /^# +:?attr(_reader|_writer|_accessor)?:/ then
-                parse_meta_attr container, single, tk, comment
-              else
-                parse_meta_method container, single, tk, comment
-              end
-            end
-          end
+          keep_comment = parse_identifier container, single, tk, comment
         end
 
-        case tk.name
+        case tk[:text]
         when "require" then
           parse_require container, comment
         when "include" then
-          parse_include container, comment
+          parse_extend_or_include RDoc::Include, container, comment
+        when "extend" then
+          parse_extend_or_include RDoc::Extend, container, comment
         end
 
-      when TkEND then
-        nest -= 1
-        if nest == 0 then
-          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
-          container.ongoing_visibility = save_visibility
-
-          parse_comment container, tk, comment unless comment.empty?
-
-          return
-        end
       else
+        try_parse_comment = nest == 1
+      end
+
+      if try_parse_comment then
         non_comment_seen = parse_comment container, tk, comment unless
           comment.empty?
 
-        comment = ''
+        keep_comment = false
       end
 
-      comment = '' unless keep_comment
+      unless keep_comment then
+        comment = new_comment ''
+        comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+        container.params = nil
+        container.block_params = nil
+      end
 
-      begin
-        get_tkread
-        skip_tkspace false
-      end while peek_tk == TkNL
+      consume_trailing_spaces
     end
+
+    container.params = nil
+    container.block_params = nil
   end
 
   ##
   # Parse up to +no+ symbol arguments
 
   def parse_symbol_arg(no = nil)
-    args = []
-
     skip_tkspace_comment
 
-    case tk = get_tk
-    when TkLPAREN
-      loop do
-        skip_tkspace_comment
-        if tk1 = parse_symbol_in_arg
-          args.push tk1
-          break if no and args.size >= no
-        end
+    tk = get_tk
+    if tk[:kind] == :on_lparen
+      parse_symbol_arg_paren no
+    else
+      parse_symbol_arg_space no, tk
+    end
+  end
 
-        skip_tkspace_comment
-        case tk2 = get_tk
-        when TkRPAREN
-          break
-        when TkCOMMA
-        else
-          warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
-          break
-        end
+  ##
+  # Parses up to +no+ symbol arguments surrounded by () and places them in
+  # +args+.
+
+  def parse_symbol_arg_paren no # :nodoc:
+    args = []
+
+    loop do
+      skip_tkspace_comment
+      if tk1 = parse_symbol_in_arg
+        args.push tk1
+        break if no and args.size >= no
       end
-    else
-      unget_tk tk
-      if tk = parse_symbol_in_arg
-        args.push tk
-        return args if no and args.size >= no
+
+      skip_tkspace_comment
+      case (tk2 = get_tk)[:kind]
+      when :on_rparen
+        break
+      when :on_comma
+      else
+        warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
+        break
       end
+    end
 
-      loop do
-        skip_tkspace false
+    args
+  end
 
-        tk1 = get_tk
-        unless TkCOMMA === tk1 then
-          unget_tk tk1
-          break
-        end
+  ##
+  # Parses up to +no+ symbol arguments separated by spaces and places them in
+  # +args+.
 
-        skip_tkspace_comment
-        if tk = parse_symbol_in_arg
-          args.push tk
-          break if no and args.size >= no
-        end
+  def parse_symbol_arg_space no, tk # :nodoc:
+    args = []
+
+    unget_tk tk
+    if tk = parse_symbol_in_arg
+      args.push tk
+      return args if no and args.size >= no
+    end
+
+    loop do
+      skip_tkspace_without_nl
+
+      tk1 = get_tk
+      if tk1.nil? || :on_comma != tk1[:kind] then
+        unget_tk tk1
+        break
+      end
+
+      skip_tkspace_comment
+      if tk = parse_symbol_in_arg
+        args.push tk
+        break if no and args.size >= no
       end
     end
 
@@ -1438,12 +1998,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Returns symbol text from the next token
 
   def parse_symbol_in_arg
-    case tk = get_tk
-    when TkSYMBOL
-      tk.text.sub(/^:/, '')
-    when TkSTRING
-      eval @read[-1]
-    when TkDSTRING, TkIDENTIFIER then
+    tk = get_tk
+    if :on_symbol == tk[:kind] then
+      tk[:text].sub(/^:/, '')
+    elsif :on_tstring == tk[:kind] then
+      tk[:text][1..-2]
+    elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
       nil # ignore
     else
       warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
@@ -1452,12 +2012,20 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
-  # Parses statements at the toplevel in +container+
+  # Parses statements in the top-level +container+
 
-  def parse_top_level_statements(container)
+  def parse_top_level_statements container
     comment = collect_first_comment
-    look_for_directives_in(container, comment)
+
+    look_for_directives_in container, comment
+
+    throw :eof if container.done_documenting
+
+    @markup = comment.format
+
+    # HACK move if to RDoc::Context#comment=
     container.comment = comment if container.document_self unless comment.empty?
+
     parse_statements container, NORMAL, nil, comment
   end
 
@@ -1465,65 +2033,39 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Determines the visibility in +container+ from +tk+
 
   def parse_visibility(container, single, tk)
-    singleton = (single == SINGLE)
-
-    vis_type = tk.name
-
-    vis = case vis_type
-          when 'private'   then :private
-          when 'protected' then :protected
-          when 'public'    then :public
-          when 'private_class_method' then
-            singleton = true
-            :private
-          when 'public_class_method' then
-            singleton = true
-            :public
-          when 'module_function' then
-            singleton = true
-            :public
-          else
-            raise RDoc::Error, "Invalid visibility: #{tk.name}"
-          end
+    vis_type, vis, singleton = get_visibility_information tk, single
 
     skip_tkspace_comment false
 
-    case peek_tk
-      # Ryan Davis suggested the extension to ignore modifiers, because he
-      # often writes
-      #
-      #   protected unless $TESTING
-      #
-    when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+    ptk = peek_tk
+    # Ryan Davis suggested the extension to ignore modifiers, because he
+    # often writes
+    #
+    #   protected unless $TESTING
+    #
+    if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
       container.ongoing_visibility = vis
+    elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
+      container.current_line_visibility = vis
     else
-      if vis_type == 'module_function' then
-        args = parse_symbol_arg
-        container.set_visibility_for args, :private, false
-
-        module_functions = []
-
-        container.methods_matching args do |m|
-          s_m = m.dup
-          s_m.record_location @top_level
-          s_m.singleton = true
-          s_m.visibility = :public
-          module_functions << s_m
-        end
+      update_visibility container, vis_type, vis, singleton
+    end
+  end
 
-        module_functions.each do |s_m|
-          case s_m
-          when RDoc::AnyMethod then
-            container.add_method s_m
-          when RDoc::Attr then
-            container.add_attribute s_m
-          end
-        end
-      else
-        args = parse_symbol_arg
-        container.set_visibility_for args, vis, singleton
-      end
+  ##
+  # Parses a Module#private_constant or Module#public_constant call from +tk+.
+
+  def parse_constant_visibility(container, single, tk)
+    args = parse_symbol_arg
+    case tk[:text]
+    when 'private_constant'
+      vis = :private
+    when 'public_constant'
+      vis = :public
+    else
+      raise RDoc::Error, 'Unreachable'
     end
+    container.set_constant_visibility_for args, vis
   end
 
   ##
@@ -1533,7 +2075,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     return if method.block_params
 
     get_tkread
-    @scanner.instance_eval { @continue = false }
     method.block_params = parse_method_or_yield_parameters
   end
 
@@ -1547,49 +2088,72 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   #   class MyClass # :nodoc:
   #
-  # We return the directive name and any parameters as a two element array
+  # We return the directive name and any parameters as a two element array if
+  # the name is in +allowed+.  A directive can be found anywhere up to the end
+  # of the current line.
 
   def read_directive allowed
-    tk = get_tk
+    tokens = []
+
+    while tk = get_tk do
+      tokens << tk
 
-    if TkCOMMENT === tk then
-      return unless tk.text =~ /\s*:?(\w+):\s*(.*)/
+      if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
+        return
+      elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
+        return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/
 
-      directive = $1.downcase
+        directive = $1.downcase
 
-      return [directive, $2] if allowed.include? directive
-    else
-      unget_tk tk
+        return [directive, $2] if allowed.include? directive
+
+        return
+      end
+    end
+  ensure
+    unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
+      tokens.reverse_each do |token|
+        unget_tk token
+      end
     end
   end
 
   ##
-  # Handles the directive for +context+ if the directive is listed in +allow+.
-  # This method is called for directives following a definition.
+  # Handles directives following the definition for +context+ (any
+  # RDoc::CodeObject) if the directives are +allowed+ at this point.
+  #
+  # See also RDoc::Markup::PreProcess#handle_directive
 
-  def read_documentation_modifiers(context, allow)
-    directive, value = read_directive allow
+  def read_documentation_modifiers context, allowed
+    skip_tkspace_without_nl
+    directive, value = read_directive allowed
 
     return unless directive
 
-    case directive
-    when 'notnew', 'not_new', 'not-new' then
-      context.dont_rename_initialize = true
-    else
-      RDoc::Parser.process_directive context, directive, value
+    @preprocess.handle_directive '', directive, value, context do |dir, param|
+      if %w[notnew not_new not-new].include? dir then
+        context.dont_rename_initialize = true
+
+        true
+      end
     end
   end
 
   ##
-  # Removes private comments from +comment+
+  # Records the location of this +container+ in the file for this parser and
+  # adds it to the list of classes and modules in the file.
 
-  def remove_private_comments(comment)
-    comment.gsub!(/^#--\n.*?^#\+\+\n?/m, '')
-    comment.sub!(/^#--\n.*\n?/m, '')
+  def record_location container # :nodoc:
+    case container
+    when RDoc::ClassModule then
+      @top_level.add_to_classes_or_modules container
+    end
+
+    container.record_location @top_level
   end
 
   ##
-  # Scans this ruby file for ruby constructs
+  # Scans this Ruby file for Ruby constructs
 
   def scan
     reset
@@ -1597,30 +2161,35 @@ class RDoc::Parser::Ruby < RDoc::Parser
     catch :eof do
       begin
         parse_top_level_statements @top_level
+
       rescue StandardError => e
-        bytes = ''
-
-        20.times do @scanner.ungetc end
-        count = 0
-        60.times do |i|
-          count = i
-          byte = @scanner.getc
-          break unless byte
-          bytes << byte
+        if @content.include?('<%') and @content.include?('%>') then
+          # Maybe, this is ERB.
+          $stderr.puts "\033[2KRDoc detects ERB file. Skips it for compatibility:"
+          $stderr.puts @file_name
+          return
         end
-        count -= 20
-        count.times do @scanner.ungetc end
+
+        if @scanner_point >= @scanner.size
+          now_line_no = @scanner[@scanner.size - 1][:line_no]
+        else
+          now_line_no = peek_tk[:line_no]
+        end
+        first_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
+        last_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 }
+        last_tk_index = last_tk_index ? last_tk_index - 1 : @scanner.size - 1
+        code = @scanner[first_tk_index..last_tk_index].map{ |t| t[:text] }.join
 
         $stderr.puts <<-EOF
 
-#{self.class} failure around line #{@scanner.line_no} of
+#{self.class} failure around line #{now_line_no} of
 #{@file_name}
 
         EOF
 
-        unless bytes.empty? then
+        unless code.empty? then
+          $stderr.puts code
           $stderr.puts
-          $stderr.puts bytes.inspect
         end
 
         raise e
@@ -1634,58 +2203,52 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # while, until, and for have an optional do
 
   def skip_optional_do_after_expression
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
-    case tk
-    when TkLPAREN, TkfLPAREN then
-      end_token = TkRPAREN
-    else
-      end_token = TkNL
-    end
 
     b_nest = 0
     nest = 0
-    @scanner.instance_eval { @continue = false }
 
     loop do
-      case tk
-      when TkSEMICOLON then
+      break unless tk
+      case tk[:kind]
+      when :on_semicolon, :on_nl, :on_ignored_nl then
         break if b_nest.zero?
-      when TkLPAREN, TkfLPAREN then
+      when :on_lparen then
         nest += 1
-      when TkBEGIN then
-        b_nest += 1
-      when TkEND then
-        b_nest -= 1
-      when TkDO
-        break if nest.zero?
-      when end_token then
-        if end_token == TkRPAREN
-          nest -= 1
-          break if @scanner.lex_state == EXPR_END and nest.zero?
-        else
-          break unless @scanner.continue
+      when :on_rparen then
+        nest -= 1
+      when :on_kw then
+        case tk[:text]
+        when 'begin'
+          b_nest += 1
+        when 'end'
+          b_nest -= 1
+        when 'do'
+          break if nest.zero?
+        end
+      when :on_comment, :on_embdoc then
+        if b_nest.zero? and "\n" == tk[:text][-1] then
+          break
         end
-      when nil then
-        break
       end
       tk = get_tk
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
-    get_tk if TkDO === peek_tk
+    get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
   end
 
   ##
   # skip the var [in] part of a 'for' statement
 
   def skip_for_variable
-    skip_tkspace false
-    tk = get_tk
-    skip_tkspace false
+    skip_tkspace_without_nl
+    get_tk
+    skip_tkspace_without_nl
     tk = get_tk
-    unget_tk(tk) unless TkIN === tk
+    unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
   end
 
   ##
@@ -1702,20 +2265,63 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def skip_tkspace_comment(skip_nl = true)
     loop do
-      skip_tkspace skip_nl
-      return unless TkCOMMENT === peek_tk
+      skip_nl ? skip_tkspace : skip_tkspace_without_nl
+      next_tk = peek_tk
+      return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
       get_tk
     end
   end
 
   ##
-  # Prints +msg+ to +$stderr+ unless we're being quiet
+  # Updates visibility in +container+ from +vis_type+ and +vis+.
 
-  def warn(msg)
-    return if @options.quiet
-    msg = make_message msg
-    $stderr.puts msg
+  def update_visibility container, vis_type, vis, singleton # :nodoc:
+    new_methods = []
+
+    case vis_type
+    when 'module_function' then
+      args = parse_symbol_arg
+      container.set_visibility_for args, :private, false
+
+      container.methods_matching args do |m|
+        s_m = m.dup
+        record_location s_m
+        s_m.singleton = true
+        new_methods << s_m
+      end
+    when 'public_class_method', 'private_class_method' then
+      args = parse_symbol_arg
+
+      container.methods_matching args, true do |m|
+        if m.parent != container then
+          m = m.dup
+          record_location m
+          new_methods << m
+        end
+
+        m.visibility = vis
+      end
+    else
+      args = parse_symbol_arg
+      container.set_visibility_for args, vis, singleton
+    end
+
+    new_methods.each do |method|
+      case method
+      when RDoc::AnyMethod then
+        container.add_method method
+      when RDoc::Attr then
+        container.add_attribute method
+      end
+      method.visibility = vis
+    end
   end
 
-end
+  ##
+  # Prints +message+ to +$stderr+ unless we're being quiet
 
+  def warn message
+    @options.warn make_message message
+  end
+
+end