horseman 0.0.1 → 0.0.2

data/vendor/ruby/1.9.1/gems/rdoc-3.11/lib/rdoc/parser/ruby.rb

@@ -0,0 +1,1811 @@
+##
+# This file contains stuff stolen outright from:
+#
+#   rtags.rb -
+#   ruby-lex.rb - ruby lexcal analyzer
+#   ruby-token.rb - ruby tokens
+#       by Keiju ISHITSUKA (Nippon Rational Inc.)
+#
+
+$TOKEN_DEBUG ||= nil
+
+##
+# Extracts code elements from a source file returning a TopLevel object
+# containing the constituent file elements.
+#
+# This file is based on rtags
+#
+# RubyParser understands how to document:
+# * classes
+# * modules
+# * methods
+# * constants
+# * aliases
+# * private, public, protected
+# * private_class_function, public_class_function
+# * module_function
+# * attr, attr_reader, attr_writer, attr_accessor
+# * extra accessors given on the command line
+# * metaprogrammed methods
+# * require
+# * include
+#
+# == Method Arguments
+#
+#--
+# NOTE: I don't think this works, needs tests, remove the paragraph following
+# this block when known to work
+#
+# The parser extracts the arguments from the method definition.  You can
+# override this with a custom argument definition using the :args: directive:
+#
+#   ##
+#   # This method tries over and over until it is tired
+#
+#   def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try
+#     puts thing_to_try
+#     go_go_go thing_to_try, tries - 1
+#   end
+#
+# If you have a more-complex set of overrides you can use the :call-seq:
+# directive:
+#++
+#
+# The parser extracts the arguments from the method definition.  You can
+# override this with a custom argument definition using the :call-seq:
+# directive:
+#
+#   ##
+#   # This method can be called with a range or an offset and length
+#   #
+#   # :call-seq:
+#   #   my_method(Range)
+#   #   my_method(offset, length)
+#
+#   def my_method(*args)
+#   end
+#
+# The parser extracts +yield+ expressions from method bodies to gather the
+# yielded argument names.  If your method manually calls a block instead of
+# yielding or you want to override the discovered argument names use
+# the :yields: directive:
+#
+#   ##
+#   # My method is awesome
+#
+#   def my_method(&block) # :yields: happy, times
+#     block.call 1, 2
+#   end
+#
+# == Metaprogrammed Methods
+#
+# To pick up a metaprogrammed method, the parser looks for a comment starting
+# with '##' before an identifier:
+#
+#   ##
+#   # This is a meta-programmed method!
+#
+#   add_my_method :meta_method, :arg1, :arg2
+#
+# The parser looks at the token after the identifier to determine the name, in
+# this example, :meta_method.  If a name cannot be found, a warning is printed
+# and 'unknown is used.
+#
+# You can force the name of a method using the :method: directive:
+#
+#   ##
+#   # :method: woo_hoo!
+#
+# By default, meta-methods are instance methods.  To indicate that a method is
+# a singleton method instead use the :singleton-method: directive:
+#
+#   ##
+#   # :singleton-method:
+#
+# You can also use the :singleton-method: directive with a name:
+#
+#   ##
+#   # :singleton-method: woo_hoo!
+#
+# Additionally you can mark a method as an attribute by
+# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:.  Just like
+# for :method:, the name is optional.
+#
+#   ##
+#   # :attr_reader: my_attr_name
+#
+# == Hidden methods and attributes
+#
+# You can provide documentation for methods that don't appear using
+# the :method:, :singleton-method: and :attr: directives:
+#
+#   ##
+#   # :attr_writer: ghost_writer
+#   # There is an attribute here, but you can't see it!
+#
+#   ##
+#   # :method: ghost_method
+#   # There is a method here, but you can't see it!
+#
+#   ##
+#   # this is a comment for a regular method
+#
+#   def regular_method() end
+#
+# Note that by default, the :method: directive will be ignored if there is a
+# standard rdocable item following it.
+
+class RDoc::Parser::Ruby < RDoc::Parser
+
+  parse_files_matching(/\.rbw?$/)
+
+  include RDoc::RubyToken
+  include RDoc::TokenStream
+  include RDoc::Parser::RubyTools
+
+  ##
+  # RDoc::NormalClass type
+
+  NORMAL = "::"
+
+  ##
+  # RDoc::SingleClass type
+
+  SINGLE = "<<"
+
+  ##
+  # Creates a new Ruby parser.
+
+  def initialize(top_level, file_name, content, options, stats)
+    super
+
+    @size = 0
+    @token_listeners = nil
+    @scanner = RDoc::RubyLex.new content, @options
+    @scanner.exception_on_syntax_error = false
+    @prev_seek = nil
+    @markup = @options.markup
+
+    @encoding = nil
+    @encoding = @options.encoding if Object.const_defined? :Encoding
+
+    reset
+  end
+
+  ##
+  # Look for the first comment in a file that isn't a shebang line.
+
+  def collect_first_comment
+    skip_tkspace
+    comment = ''
+    comment.force_encoding @encoding if @encoding
+    first_line = true
+
+    tk = get_tk
+
+    while TkCOMMENT === tk
+      if first_line and tk.text =~ /\A#!/ then
+        skip_tkspace
+        tk = get_tk
+      elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+        first_line = false
+        skip_tkspace
+        tk = get_tk
+      else
+        first_line = false
+        comment << tk.text << "\n"
+        tk = get_tk
+
+        if TkNL === tk then
+          skip_tkspace false
+          tk = get_tk
+        end
+      end
+    end
+
+    unget_tk tk
+
+    new_comment comment
+  end
+
+  ##
+  # Aborts with +msg+
+
+  def error(msg)
+    msg = make_message msg
+
+    abort msg
+  end
+
+  ##
+  # Looks for a true or false token.  Returns false if TkFALSE or TkNIL are
+  # found.
+
+  def get_bool
+    skip_tkspace
+    tk = get_tk
+    case tk
+    when TkTRUE
+      true
+    when TkFALSE, TkNIL
+      false
+    else
+      unget_tk tk
+      true
+    end
+  end
+
+  ##
+  # Look for the name of a class of module (optionally with a leading :: or
+  # with :: separated named) and return the ultimate name, the associated
+  # container, and the given name (with the ::).
+
+  def get_class_or_module(container)
+    skip_tkspace
+    name_t = get_tk
+    given_name = ''
+
+    # class ::A -> A is in the top level
+    case name_t
+    when TkCOLON2, TkCOLON3 then # bug
+      name_t = get_tk
+      container = @top_level
+      given_name << '::'
+    end
+
+    skip_tkspace false
+    given_name << name_t.name
+
+    while TkCOLON2 === peek_tk do
+      prev_container = container
+      container = container.find_module_named name_t.name
+      unless container then
+        container = prev_container.add_module RDoc::NormalModule, name_t.name
+      end
+      get_tk
+      name_t = get_tk
+      given_name << '::' << name_t.name
+    end
+    skip_tkspace false
+    return [container, name_t, given_name]
+  end
+
+  ##
+  # Return a superclass, which can be either a constant of an expression
+
+  def get_class_specification
+    tk = get_tk
+    return "self" if TkSELF === tk
+
+    res = ""
+    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
+      res += tk.name
+      tk = get_tk
+    end
+
+    unget_tk(tk)
+    skip_tkspace false
+
+    get_tkread # empty out read buffer
+
+    tk = get_tk
+
+    case tk
+    when TkNL, TkCOMMENT, TkSEMICOLON then
+      unget_tk(tk)
+      return res
+    end
+
+    res += parse_call_parameters(tk)
+    res
+  end
+
+  ##
+  # Parse a constant, which might be qualified by one or more class or module
+  # names
+
+  def get_constant
+    res = ""
+    skip_tkspace false
+    tk = get_tk
+
+    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
+      res += tk.name
+      tk = get_tk
+    end
+
+#      if res.empty?
+#        warn("Unexpected token #{tk} in constant")
+#      end
+    unget_tk(tk)
+    res
+  end
+
+  ##
+  # Get a constant that may be surrounded by parens
+
+  def get_constant_with_optional_parens
+    skip_tkspace false
+
+    nest = 0
+
+    while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
+      get_tk
+      skip_tkspace
+      nest += 1
+    end
+
+    name = get_constant
+
+    while nest > 0
+      skip_tkspace
+      tk = get_tk
+      nest -= 1 if TkRPAREN === tk
+    end
+
+    name
+  end
+
+  ##
+  # Extracts a name or symbol from the token stream.
+
+  def get_symbol_or_name
+    tk = get_tk
+    case tk
+    when TkSYMBOL then
+      text = tk.text.sub(/^:/, '')
+
+      if TkASSIGN === peek_tk then
+        get_tk
+        text << '='
+      end
+
+      text
+    when TkId, TkOp then
+      tk.name
+    when TkAMPER,
+         TkDSTRING,
+         TkSTAR,
+         TkSTRING then
+      tk.text
+    else
+      raise RDoc::Error, "Name or symbol expected (got #{tk})"
+    end
+  end
+
+  ##
+  # Look for directives in a normal comment block:
+  #
+  #   # :stopdoc:
+  #   # Don't display comment from this point forward
+  #
+  # This routine modifies its +comment+ parameter.
+
+  def look_for_directives_in context, comment
+    @preprocess.handle comment, context do |directive, param|
+      case directive
+      when 'method', 'singleton-method',
+           'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
+        false # handled elsewhere
+      when 'section' then
+        context.set_current_section param, comment.dup
+        comment.text = ''
+        break
+      end
+    end
+
+    remove_private_comments comment
+  end
+
+  ##
+  # Adds useful info about the parser to +message+
+
+  def make_message message
+    prefix = "#{@file_name}:"
+
+    prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+
+    "#{prefix} #{message}"
+  end
+
+  ##
+  # Creates a comment with the correct format
+
+  def new_comment comment
+    c = RDoc::Comment.new comment, @top_level
+    c.format = @markup
+    c
+  end
+
+  ##
+  # Creates an RDoc::Attr for the name following +tk+, setting the comment to
+  # +comment+.
+
+  def parse_attr(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    args = parse_symbol_arg 1
+    if args.size > 0 then
+      name = args[0]
+      rw = "R"
+      skip_tkspace false
+      tk = get_tk
+
+      if TkCOMMA === tk then
+        rw = "RW" if get_bool
+      else
+        unget_tk tk
+      end
+
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+      att.offset = offset
+      att.line   = line_no
+
+      read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
+
+      context.add_attribute att
+
+      @stats.add_attribute att
+    else
+      warn "'attr' ignored - looks like a variable"
+    end
+  end
+
+  ##
+  # Creates an RDoc::Attr for each attribute listed after +tk+, setting the
+  # comment for each to +comment+.
+
+  def parse_attr_accessor(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    args = parse_symbol_arg
+    rw = "?"
+
+    tmp = RDoc::CodeObject.new
+    read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+    # TODO In most other places we let the context keep track of document_self
+    # and add found items appropriately but here we do not.  I'm not sure why.
+    return unless tmp.document_self
+
+    case tk.name
+    when "attr_reader"   then rw = "R"
+    when "attr_writer"   then rw = "W"
+    when "attr_accessor" then rw = "RW"
+    else
+      rw = '?'
+    end
+
+    for name in args
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+      att.offset = offset
+      att.line   = line_no
+
+      context.add_attribute att
+      @stats.add_attribute att
+    end
+  end
+
+  ##
+  # Parses an +alias+ in +context+ with +comment+
+
+  def parse_alias(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    skip_tkspace
+
+    if TkLPAREN === peek_tk then
+      get_tk
+      skip_tkspace
+    end
+
+    new_name = get_symbol_or_name
+
+    @scanner.instance_eval { @lex_state = EXPR_FNAME }
+
+    skip_tkspace
+    if TkCOMMA === peek_tk then
+      get_tk
+      skip_tkspace
+    end
+
+    begin
+      old_name = get_symbol_or_name
+    rescue RDoc::Error
+      return
+    end
+
+    al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
+                         single == SINGLE)
+    al.record_location @top_level
+    al.offset = offset
+    al.line   = line_no
+
+    read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
+    context.add_alias al
+    @stats.add_alias al
+
+    al
+  end
+
+  ##
+  # Extracts call parameters from the token stream.
+
+  def parse_call_parameters(tk)
+    end_token = case tk
+                when TkLPAREN, TkfLPAREN
+                  TkRPAREN
+                when TkRPAREN
+                  return ""
+                else
+                  TkNL
+                end
+    nest = 0
+
+    loop do
+      case tk
+      when TkSEMICOLON
+        break
+      when TkLPAREN, TkfLPAREN
+        nest += 1
+      when end_token
+        if end_token == TkRPAREN
+          nest -= 1
+          break if @scanner.lex_state == EXPR_END and nest <= 0
+        else
+          break unless @scanner.continue
+        end
+      when TkCOMMENT, TkASSIGN, TkOPASGN
+        unget_tk(tk)
+        break
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+    res = get_tkread.tr("\n", " ").strip
+    res = "" if res == ";"
+    res
+  end
+
+  ##
+  # Parses a class in +context+ with +comment+
+
+  def parse_class container, single, tk, comment
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    declaration_context = container
+    container, name_t, given_name = get_class_or_module container
+
+    case name_t
+    when TkCONSTANT
+      name = name_t.name
+      superclass = '::Object'
+
+      if TkLT === peek_tk then
+        get_tk
+        skip_tkspace
+        superclass = get_class_specification
+        superclass = '(unknown)' if superclass.empty?
+      end
+
+      cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
+      cls = declaration_context.add_class cls_type, given_name, superclass
+      cls.ignore unless container.document_children
+
+      read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+      cls.record_location @top_level
+      cls.offset = offset
+      cls.line   = line_no
+
+      cls.add_comment comment, @top_level
+
+      @top_level.add_to_classes_or_modules cls
+      @stats.add_class cls
+
+      parse_statements cls
+    when TkLSHFT
+      case name = get_class_specification
+      when "self", container.name
+        parse_statements container, SINGLE
+      else
+        other = RDoc::TopLevel.find_class_named name
+
+        unless other then
+          other = container.add_module RDoc::NormalModule, name
+          other.record_location @top_level
+          other.offset  = offset
+          other.line    = line_no
+
+          other.add_comment comment, @top_level
+        end
+
+        # notify :nodoc: all if not a constant-named class/module
+        # (and remove any comment)
+        unless name =~ /\A(::)?[A-Z]/ then
+          other.document_self = nil
+          other.document_children = false
+          other.clear_comment
+        end
+
+        @top_level.add_to_classes_or_modules other
+        @stats.add_class other
+
+        read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
+        parse_statements(other, SINGLE)
+      end
+    else
+      warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
+    end
+  end
+
+  ##
+  # Parses a constant in +context+ with +comment+
+
+  def parse_constant container, tk, comment
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    name = tk.name
+    skip_tkspace false
+
+    return unless name =~ /^\w+$/
+
+    eq_tk = get_tk
+
+    unless TkASSIGN === eq_tk then
+      unget_tk eq_tk
+      return false
+    end
+
+    nest = 0
+    get_tkread
+
+    tk = get_tk
+
+    if TkGT === tk then
+      unget_tk tk
+      unget_tk eq_tk
+      return false
+    end
+
+    rhs_name = ''
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if nest <= 0
+      when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
+           TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+        nest += 1
+      when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+        nest -= 1
+      when TkCOMMENT then
+        if nest <= 0 &&
+           (@scanner.lex_state == EXPR_END || !@scanner.continue) then
+          unget_tk tk
+          break
+        end
+      when TkCONSTANT then
+        rhs_name << tk.name
+
+        if nest <= 0 and TkNL === peek_tk then
+          mod = if rhs_name =~ /^::/ then
+                  RDoc::TopLevel.find_class_or_module rhs_name
+                else
+                  container.find_module_named rhs_name
+                end
+
+          container.add_module_alias mod, name, @top_level if mod
+          break
+        end
+      when TkNL then
+        if nest <= 0 &&
+           (@scanner.lex_state == EXPR_END || !@scanner.continue) then
+          unget_tk tk
+          break
+        end
+      when TkCOLON2, TkCOLON3 then
+        rhs_name << '::'
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    res = get_tkread.gsub(/^[ \t]+/, '').strip
+    res = "" if res == ";"
+
+    con = RDoc::Constant.new name, res, comment
+    con.record_location @top_level
+    con.offset = offset
+    con.line   = line_no
+    read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+
+    @stats.add_constant con
+    con = container.add_constant con
+
+    true
+  end
+
+  ##
+  # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
+  # :method: or :attr: directives in +comment+.
+
+  def parse_comment container, tk, comment
+    return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    text = comment.text
+
+    singleton = !!text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+    # REFACTOR
+    if text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+      name = $1 unless $1.empty?
+
+      meth = RDoc::GhostMethod.new get_tkread, name
+      meth.record_location @top_level
+      meth.singleton = singleton
+      meth.offset    = offset
+      meth.line      = line_no
+
+      meth.start_collecting_tokens
+      indent = TkSPACE.new 0, 1, 1
+      indent.set_text " " * column
+
+      position_comment = TkCOMMENT.new 0, line_no, 1
+      position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
+      meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+      meth.params = ''
+
+      comment.normalize
+      comment.extract_call_seq meth
+
+      return unless meth.name
+
+      container.add_method meth
+
+      meth.comment = comment
+
+      @stats.add_method meth
+    elsif text.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+      rw = case $1
+           when 'attr_reader' then 'R'
+           when 'attr_writer' then 'W'
+           else 'RW'
+           end
+
+      name = $3 unless $3.empty?
+
+      # TODO authorize 'singleton-attr...'?
+      att = RDoc::Attr.new get_tkread, name, rw, comment
+      att.record_location @top_level
+      att.offset    = offset
+      att.line      = line_no
+
+      container.add_attribute att
+      @stats.add_attribute att
+    end
+
+    true
+  end
+
+  ##
+  # Creates an RDoc::Method on +container+ from +comment+ if there is a
+  # Signature section in the comment
+
+  def parse_comment_tomdoc container, tk, comment
+    return unless signature = RDoc::TomDoc.signature(comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    name, = signature.split %r%[ \(]%, 2
+
+    meth = RDoc::GhostMethod.new get_tkread, name
+    meth.record_location @top_level
+    meth.offset    = offset
+    meth.line      = line_no
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * offset
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+    meth.call_seq = signature
+
+    comment.normalize
+
+    return unless meth.name
+
+    container.add_method meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Parses an +include+ in +context+ with +comment+
+
+  def parse_include context, comment
+    loop do
+      skip_tkspace_comment
+
+      name = get_constant_with_optional_parens
+
+      unless name.empty? then
+        incl = context.add_include RDoc::Include.new(name, comment)
+        incl.record_location @top_level
+      end
+
+      return unless TkCOMMA === peek_tk
+
+      get_tk
+    end
+  end
+
+  ##
+  # Parses a meta-programmed attribute and creates an RDoc::Attr.
+  #
+  # To create foo and bar attributes on class C with comment "My attributes":
+  #
+  #   class C
+  #
+  #     ##
+  #     # :attr:
+  #     #
+  #     # My attributes
+  #
+  #     my_attr :foo, :bar
+  #
+  #   end
+  #
+  # To create a foo attribute on class C with comment "My attribute":
+  #
+  #   class C
+  #
+  #     ##
+  #     # :attr: foo
+  #     #
+  #     # My attribute
+  #
+  #     my_attr :foo, :bar
+  #
+  #   end
+
+  def parse_meta_attr(context, single, tk, comment)
+    args = parse_symbol_arg
+    rw = "?"
+
+    # If nodoc is given, don't document any of them
+
+    tmp = RDoc::CodeObject.new
+    read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+
+    if comment.text.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+      rw = case $1
+           when 'attr_reader' then 'R'
+           when 'attr_writer' then 'W'
+           else 'RW'
+           end
+      name = $3 unless $3.empty?
+    end
+
+    if name then
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+
+      context.add_attribute att
+      @stats.add_attribute att
+    else
+      args.each do |attr_name|
+        att = RDoc::Attr.new(get_tkread, attr_name, rw, comment,
+                             single == SINGLE)
+        att.record_location @top_level
+
+        context.add_attribute att
+        @stats.add_attribute att
+      end
+    end
+  end
+
+  ##
+  # Parses a meta-programmed method
+
+  def parse_meta_method(container, single, tk, comment)
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    start_collecting_tokens
+    add_token tk
+    add_token_listener self
+
+    skip_tkspace false
+
+    singleton = !!comment.text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+
+    if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+      name = $1 unless $1.empty?
+    end
+
+    if name.nil? then
+      name_t = get_tk
+      case name_t
+      when TkSYMBOL then
+        name = name_t.text[1..-1]
+      when TkSTRING then
+        name = name_t.value[1..-2]
+      when TkASSIGN then # ignore
+        remove_token_listener self
+        return
+      else
+        warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
+        name = 'unknown'
+      end
+    end
+
+    meth = RDoc::MetaMethod.new get_tkread, name
+    meth.record_location @top_level
+    meth.offset = offset
+    meth.line   = line_no
+    meth.singleton = singleton
+
+    remove_token_listener self
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * column
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.value = "# File #{@top_level.absolute_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    meth.add_tokens @token_stream
+
+    token_listener meth do
+      meth.params = ''
+
+      comment.normalize
+      comment.extract_call_seq meth
+
+      container.add_method meth
+
+      last_tk = tk
+
+      while tk = get_tk do
+        case tk
+        when TkSEMICOLON then
+          break
+        when TkNL then
+          break unless last_tk and TkCOMMA === last_tk
+        when TkSPACE then
+          # expression continues
+        when TkDO then
+          unget_tk tk
+          parse_statements container, single, meth
+          break
+        else
+          last_tk = tk
+        end
+      end
+    end
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Parses a normal method defined by +def+
+
+  def parse_method(container, single, tk, comment)
+    added_container = nil
+    meth = nil
+    name = nil
+    column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    start_collecting_tokens
+    add_token tk
+
+    token_listener self do
+      @scanner.instance_eval do @lex_state = EXPR_FNAME end
+
+      skip_tkspace
+      name_t = get_tk
+      back_tk = skip_tkspace
+      meth = nil
+      added_container = false
+
+      dot = get_tk
+      if TkDOT === dot or TkCOLON2 === dot then
+        @scanner.instance_eval do @lex_state = EXPR_FNAME end
+        skip_tkspace
+        name_t2 = get_tk
+
+        case name_t
+        when TkSELF, TkMOD then
+          name = name_t2.name
+        when TkCONSTANT then
+          name = name_t2.name
+          prev_container = container
+          container = container.find_module_named(name_t.name)
+          unless container then
+            added_container = true
+            obj = name_t.name.split("::").inject(Object) do |state, item|
+              state.const_get(item)
+            end rescue nil
+
+            type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
+
+            unless [Class, Module].include?(obj.class) then
+              warn("Couldn't find #{name_t.name}. Assuming it's a module")
+            end
+
+            if type == RDoc::NormalClass then
+              sclass = obj.superclass ? obj.superclass.name : nil
+              container = prev_container.add_class type, name_t.name, sclass
+            else
+              container = prev_container.add_module type, name_t.name
+            end
+
+            container.record_location @top_level
+          end
+        when TkIDENTIFIER, TkIVAR, TkGVAR then
+          dummy = RDoc::Context.new
+          dummy.parent = container
+          skip_method dummy
+          return
+        when TkTRUE, TkFALSE, TkNIL then
+          klass_name = "#{name_t.name.capitalize}Class"
+          container = RDoc::TopLevel.find_class_named klass_name
+          container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+          name = name_t2.name
+        else
+          warn "unexpected method name token #{name_t.inspect}"
+          # break
+          skip_method container
+          return
+        end
+
+        meth = RDoc::AnyMethod.new(get_tkread, name)
+        meth.singleton = true
+      else
+        unget_tk dot
+        back_tk.reverse_each do |token|
+          unget_tk token
+        end
+
+        name = case name_t
+               when TkSTAR, TkAMPER then
+                 name_t.text
+               else
+                 unless name_t.respond_to? :name then
+                   warn "expected method name token, . or ::, got #{name_t.inspect}"
+                   skip_method container
+                   return
+                 end
+                 name_t.name
+               end
+
+        meth = RDoc::AnyMethod.new get_tkread, name
+        meth.singleton = (single == SINGLE)
+      end
+    end
+
+    meth.record_location @top_level
+    meth.offset = offset
+    meth.line   = line_no
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * column
+
+    token = TkCOMMENT.new 0, line_no, 1
+    token.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
+    meth.add_tokens [token, NEWLINE_TOKEN, indent]
+    meth.add_tokens @token_stream
+
+    token_listener meth do
+      @scanner.instance_eval do @continue = false end
+      parse_method_parameters meth
+
+      if meth.document_self then
+        container.add_method meth
+      elsif added_container then
+        container.document_self = false
+      end
+
+      # Having now read the method parameters and documentation modifiers, we
+      # now know whether we have to rename #initialize to ::new
+
+      if name == "initialize" && !meth.singleton then
+        if meth.dont_rename_initialize then
+          meth.visibility = :protected
+        else
+          meth.singleton = true
+          meth.name = "new"
+          meth.visibility = :public
+        end
+      end
+
+      parse_statements container, single, meth
+    end
+
+    comment.normalize
+    comment.extract_call_seq meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
+  # Extracts +yield+ parameters from +method+
+
+  def parse_method_or_yield_parameters(method = nil,
+                                       modifiers = RDoc::METHOD_MODIFIERS)
+    skip_tkspace false
+    tk = get_tk
+
+    # Little hack going on here. In the statement
+    #  f = 2*(1+yield)
+    # We see the RPAREN as the next token, so we need
+    # to exit early. This still won't catch all cases
+    # (such as "a = yield + 1"
+    end_token = case tk
+                when TkLPAREN, TkfLPAREN
+                  TkRPAREN
+                when TkRPAREN
+                  return ""
+                else
+                  TkNL
+                end
+    nest = 0
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if nest == 0
+      when TkLBRACE, TkfLBRACE then
+        nest += 1
+      when TkRBRACE then
+        nest -= 1
+        if nest <= 0
+          # we might have a.each { |i| yield i }
+          unget_tk(tk) if nest < 0
+          break
+        end
+      when TkLPAREN, TkfLPAREN then
+        nest += 1
+      when end_token then
+        if end_token == TkRPAREN
+          nest -= 1
+          break if @scanner.lex_state == EXPR_END and nest <= 0
+        else
+          break unless @scanner.continue
+        end
+      when TkRPAREN then
+        nest -= 1
+      when method && method.block_params.nil? && TkCOMMENT then
+        unget_tk tk
+        read_documentation_modifiers method, modifiers
+        @read.pop
+      when TkCOMMENT then
+        @read.pop
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    res = get_tkread.gsub(/\s+/, ' ').strip
+    res = '' if res == ';'
+    res
+  end
+
+  ##
+  # Capture the method's parameters. Along the way, look for a comment
+  # containing:
+  #
+  #    # yields: ....
+  #
+  # and add this as the block_params for the method
+
+  def parse_method_parameters(method)
+    res = parse_method_or_yield_parameters method
+
+    res = "(#{res})" unless res =~ /\A\(/
+    method.params = res unless method.params
+
+    if method.block_params.nil? then
+      skip_tkspace false
+      read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
+    end
+  end
+
+  ##
+  # Parses an RDoc::NormalModule in +container+ with +comment+
+
+  def parse_module container, single, tk, comment
+    container, name_t, = get_class_or_module container
+
+    name = name_t.name
+
+    mod = container.add_module RDoc::NormalModule, name
+    mod.ignore unless container.document_children
+    mod.record_location @top_level
+
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+    mod.add_comment comment, @top_level
+    parse_statements mod
+
+    @top_level.add_to_classes_or_modules mod
+    @stats.add_module mod
+  end
+
+  ##
+  # Parses an RDoc::Require in +context+ containing +comment+
+
+  def parse_require(context, comment)
+    skip_tkspace_comment
+    tk = get_tk
+
+    if TkLPAREN === tk then
+      skip_tkspace_comment
+      tk = get_tk
+    end
+
+    name = tk.text if TkSTRING === tk
+
+    if name then
+      @top_level.add_require RDoc::Require.new(name, comment)
+    else
+      unget_tk tk
+    end
+  end
+
+  ##
+  # The core of the ruby parser.
+
+  def parse_statements(container, single = NORMAL, current_method = nil,
+                       comment = new_comment(''))
+    raise 'no' unless RDoc::Comment === comment
+    comment.force_encoding @encoding if @encoding
+
+    nest = 1
+    save_visibility = container.visibility
+
+    non_comment_seen = true
+
+    while tk = get_tk do
+      keep_comment = false
+      try_parse_comment = false
+
+      non_comment_seen = true unless TkCOMMENT === tk
+
+      case tk
+      when TkNL then
+        skip_tkspace
+        tk = get_tk
+
+        if TkCOMMENT === tk then
+          if non_comment_seen then
+            # Look for RDoc in a comment about to be thrown away
+            non_comment_seen = parse_comment container, tk, comment unless
+              comment.empty?
+
+            comment = ''
+            comment.force_encoding @encoding if @encoding
+          end
+
+          while TkCOMMENT === tk do
+            comment << tk.text << "\n"
+
+            tk = get_tk
+
+            if TkNL === tk then
+              skip_tkspace false # leading spaces
+              tk = get_tk
+            end
+          end
+
+          comment = new_comment comment
+
+          unless comment.empty? then
+            look_for_directives_in container, comment
+
+            if container.done_documenting then
+              container.ongoing_visibility = save_visibility
+            end
+          end
+
+          keep_comment = true
+        else
+          non_comment_seen = true
+        end
+
+        unget_tk tk # TODO peek instead of get then unget
+        keep_comment = true
+
+      when TkCLASS then
+        parse_class container, single, tk, comment
+
+      when TkMODULE then
+        parse_module container, single, tk, comment
+
+      when TkDEF then
+        parse_method container, single, tk, comment
+
+      when TkCONSTANT then
+        unless parse_constant container, tk, comment then
+          try_parse_comment = true
+        end
+
+      when TkALIAS then
+        parse_alias container, single, tk, comment unless current_method
+
+      when TkYIELD then
+        if current_method.nil? then
+          warn "Warning: yield outside of method" if container.document_self
+        else
+          parse_yield container, single, tk, current_method
+        end
+
+      # Until and While can have a 'do', which shouldn't increase the nesting.
+      # We can't solve the general case, but we can handle most occurrences by
+      # ignoring a do at the end of a line.
+
+      when TkUNTIL, TkWHILE then
+        nest += 1
+        skip_optional_do_after_expression
+
+      # 'for' is trickier
+      when TkFOR then
+        nest += 1
+        skip_for_variable
+        skip_optional_do_after_expression
+
+      when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
+        nest += 1
+
+      when TkIDENTIFIER then
+        if nest == 1 and current_method.nil? then
+          case tk.name
+          when 'private', 'protected', 'public', 'private_class_method',
+               'public_class_method', 'module_function' then
+            parse_visibility container, single, tk
+            keep_comment = true
+          when 'attr' then
+            parse_attr container, single, tk, comment
+          when /^attr_(reader|writer|accessor)$/ then
+            parse_attr_accessor container, single, tk, comment
+          when 'alias_method' then
+            parse_alias container, single, tk, comment
+          when 'require', 'include' then
+            # ignore
+          else
+            if comment.text =~ /\A#\#$/ then
+              case comment.text
+              when /^# +:?attr(_reader|_writer|_accessor)?:/ then
+                parse_meta_attr container, single, tk, comment
+              else
+                parse_meta_method container, single, tk, comment
+              end
+            end
+          end
+        end
+
+        case tk.name
+        when "require" then
+          parse_require container, comment
+        when "include" then
+          parse_include container, comment
+        end
+
+      when TkEND then
+        nest -= 1
+        if nest == 0 then
+          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
+          container.ongoing_visibility = save_visibility
+
+          parse_comment container, tk, comment unless comment.empty?
+
+          return
+        end
+      else
+        try_parse_comment = nest == 1
+      end
+
+      if try_parse_comment then
+        non_comment_seen = parse_comment container, tk, comment unless
+          comment.empty?
+
+        keep_comment = false
+      end
+
+      unless keep_comment then
+        comment = new_comment ''
+        comment.force_encoding @encoding if @encoding
+      end
+
+      begin
+        get_tkread
+        skip_tkspace false
+      end while peek_tk == TkNL
+    end
+  end
+
+  ##
+  # Parse up to +no+ symbol arguments
+
+  def parse_symbol_arg(no = nil)
+    args = []
+
+    skip_tkspace_comment
+
+    case tk = get_tk
+    when TkLPAREN
+      loop do
+        skip_tkspace_comment
+        if tk1 = parse_symbol_in_arg
+          args.push tk1
+          break if no and args.size >= no
+        end
+
+        skip_tkspace_comment
+        case tk2 = get_tk
+        when TkRPAREN
+          break
+        when TkCOMMA
+        else
+          warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
+          break
+        end
+      end
+    else
+      unget_tk tk
+      if tk = parse_symbol_in_arg
+        args.push tk
+        return args if no and args.size >= no
+      end
+
+      loop do
+        skip_tkspace false
+
+        tk1 = get_tk
+        unless TkCOMMA === tk1 then
+          unget_tk tk1
+          break
+        end
+
+        skip_tkspace_comment
+        if tk = parse_symbol_in_arg
+          args.push tk
+          break if no and args.size >= no
+        end
+      end
+    end
+
+    args
+  end
+
+  ##
+  # Returns symbol text from the next token
+
+  def parse_symbol_in_arg
+    case tk = get_tk
+    when TkSYMBOL
+      tk.text.sub(/^:/, '')
+    when TkSTRING
+      eval @read[-1]
+    when TkDSTRING, TkIDENTIFIER then
+      nil # ignore
+    else
+      warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
+      nil
+    end
+  end
+
+  ##
+  # Parses statements in the top-level +container+
+
+  def parse_top_level_statements container
+    comment = collect_first_comment
+
+    look_for_directives_in container, comment
+
+    @markup = comment.format
+
+    # HACK move if to RDoc::Context#comment=
+    container.comment = comment if container.document_self unless comment.empty?
+
+    parse_statements container, NORMAL, nil, comment
+  end
+
+  ##
+  # Determines the visibility in +container+ from +tk+
+
+  def parse_visibility(container, single, tk)
+    singleton = (single == SINGLE)
+
+    vis_type = tk.name
+
+    vis = case vis_type
+          when 'private'   then :private
+          when 'protected' then :protected
+          when 'public'    then :public
+          when 'private_class_method' then
+            singleton = true
+            :private
+          when 'public_class_method' then
+            singleton = true
+            :public
+          when 'module_function' then
+            singleton = true
+            :public
+          else
+            raise RDoc::Error, "Invalid visibility: #{tk.name}"
+          end
+
+    skip_tkspace_comment false
+
+    case peek_tk
+      # Ryan Davis suggested the extension to ignore modifiers, because he
+      # often writes
+      #
+      #   protected unless $TESTING
+      #
+    when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+      container.ongoing_visibility = vis
+    else
+      new_methods = []
+
+      case vis_type
+      when 'module_function' then
+        args = parse_symbol_arg
+        container.set_visibility_for args, :private, false
+
+        container.methods_matching args do |m|
+          s_m = m.dup
+          s_m.record_location @top_level
+          s_m.singleton = true
+          new_methods << s_m
+        end
+      when 'public_class_method', 'private_class_method' then
+        args = parse_symbol_arg
+
+        container.methods_matching args, true do |m|
+          if m.parent != container then
+            m = m.dup
+            m.record_location @top_level
+            new_methods << m
+          end
+
+          m.visibility = vis
+        end
+      else
+        args = parse_symbol_arg
+        container.set_visibility_for args, vis, singleton
+      end
+
+      new_methods.each do |method|
+        case method
+        when RDoc::AnyMethod then
+          container.add_method method
+        when RDoc::Attr then
+          container.add_attribute method
+        end
+        method.visibility = vis
+      end
+    end
+  end
+
+  ##
+  # Determines the block parameter for +context+
+
+  def parse_yield(context, single, tk, method)
+    return if method.block_params
+
+    get_tkread
+    @scanner.instance_eval { @continue = false }
+    method.block_params = parse_method_or_yield_parameters
+  end
+
+  ##
+  # Directives are modifier comments that can appear after class, module, or
+  # method names. For example:
+  #
+  #   def fred # :yields: a, b
+  #
+  # or:
+  #
+  #   class MyClass # :nodoc:
+  #
+  # We return the directive name and any parameters as a two element array
+
+  def read_directive allowed
+    tk = get_tk
+
+    if TkCOMMENT === tk then
+      return unless tk.text =~ /\s*:?(\w+):\s*(.*)/
+
+      directive = $1.downcase
+
+      return [directive, $2] if allowed.include? directive
+    else
+      unget_tk tk
+    end
+  end
+
+  ##
+  # Handles the directive for +context+ if the directive is listed in +allow+.
+  # This method is called for directives following a definition.
+
+  def read_documentation_modifiers context, allow
+    directive, value = read_directive allow
+
+    return unless directive
+
+    @preprocess.handle_directive '', directive, value, context do |dir, param|
+      if %w[notnew not_new not-new].include? dir then
+        context.dont_rename_initialize = true
+
+        true
+      end
+    end
+  end
+
+  ##
+  # Removes private comments from +comment+
+  #--
+  # TODO remove
+
+  def remove_private_comments comment
+    comment.remove_private
+  end
+
+  ##
+  # Scans this ruby file for ruby constructs
+
+  def scan
+    reset
+
+    catch :eof do
+      begin
+        parse_top_level_statements @top_level
+
+      rescue StandardError => e
+        bytes = ''
+
+        20.times do @scanner.ungetc end
+        count = 0
+        60.times do |i|
+          count = i
+          byte = @scanner.getc
+          break unless byte
+          bytes << byte
+        end
+        count -= 20
+        count.times do @scanner.ungetc end
+
+        $stderr.puts <<-EOF
+
+#{self.class} failure around line #{@scanner.line_no} of
+#{@file_name}
+
+        EOF
+
+        unless bytes.empty? then
+          $stderr.puts
+          $stderr.puts bytes.inspect
+        end
+
+        raise e
+      end
+    end
+
+    @top_level
+  end
+
+  ##
+  # while, until, and for have an optional do
+
+  def skip_optional_do_after_expression
+    skip_tkspace false
+    tk = get_tk
+    case tk
+    when TkLPAREN, TkfLPAREN then
+      end_token = TkRPAREN
+    else
+      end_token = TkNL
+    end
+
+    b_nest = 0
+    nest = 0
+    @scanner.instance_eval { @continue = false }
+
+    loop do
+      case tk
+      when TkSEMICOLON then
+        break if b_nest.zero?
+      when TkLPAREN, TkfLPAREN then
+        nest += 1
+      when TkBEGIN then
+        b_nest += 1
+      when TkEND then
+        b_nest -= 1
+      when TkDO
+        break if nest.zero?
+      when end_token then
+        if end_token == TkRPAREN
+          nest -= 1
+          break if @scanner.lex_state == EXPR_END and nest.zero?
+        else
+          break unless @scanner.continue
+        end
+      when nil then
+        break
+      end
+      tk = get_tk
+    end
+
+    skip_tkspace false
+
+    get_tk if TkDO === peek_tk
+  end
+
+  ##
+  # skip the var [in] part of a 'for' statement
+
+  def skip_for_variable
+    skip_tkspace false
+    tk = get_tk
+    skip_tkspace false
+    tk = get_tk
+    unget_tk(tk) unless TkIN === tk
+  end
+
+  ##
+  # Skips the next method in +container+
+
+  def skip_method container
+    meth = RDoc::AnyMethod.new "", "anon"
+    parse_method_parameters meth
+    parse_statements container, false, meth
+  end
+
+  ##
+  # Skip spaces until a comment is found
+
+  def skip_tkspace_comment(skip_nl = true)
+    loop do
+      skip_tkspace skip_nl
+      return unless TkCOMMENT === peek_tk
+      get_tk
+    end
+  end
+
+  ##
+  # Prints +message+ to +$stderr+ unless we're being quiet
+
+  def warn message
+    @options.warn make_message message
+  end
+
+end
+