rdoc 2.5.11 → 3.0

data/lib/rdoc/parser/ruby.rb

@@ -11,8 +11,8 @@ require 'rdoc/ruby_token'
 require 'rdoc/ruby_lex'
 
 require 'rdoc/code_objects'
-require 'rdoc/tokenstream'
-require 'rdoc/markup/preprocess'
+require 'rdoc/token_stream'
+require 'rdoc/markup/pre_process'
 require 'rdoc/parser'
 require 'rdoc/parser/ruby_tools'
 
@@ -162,6 +162,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   SINGLE = "<<"
 
+  ##
+  # Creates a new Ruby parser.
+
   def initialize(top_level, file_name, content, options, stats)
     super
 
@@ -209,10 +212,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
     comment
   end
 
+  ##
+  # Aborts with +msg+
+
   def error(msg)
     msg = make_message msg
-    $stderr.puts msg
-    exit false
+
+    abort msg
   end
 
   ##
@@ -229,6 +235,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth
   end
 
+  ##
+  # Looks for a true or false token.  Returns false if TkFALSE or TkNIL are
+  # found.
+
   def get_bool
     skip_tkspace
     tk = get_tk
@@ -245,20 +255,24 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   ##
   # Look for the name of a class of module (optionally with a leading :: or
-  # with :: separated named) and return the ultimate name and container
+  # with :: separated named) and return the ultimate name, the associated
+  # container, and the given name (with the ::).
 
   def get_class_or_module(container)
     skip_tkspace
     name_t = get_tk
+    given_name = ''
 
     # class ::A -> A is in the top level
     case name_t
     when TkCOLON2, TkCOLON3 then # bug
       name_t = get_tk
       container = @top_level
+      given_name << '::'
     end
 
     skip_tkspace false
+    given_name << name_t.name
 
     while TkCOLON2 === peek_tk do
       prev_container = container
@@ -268,9 +282,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
       get_tk
       name_t = get_tk
+      given_name << '::' << name_t.name
     end
     skip_tkspace false
-    return [container, name_t]
+    return [container, name_t, given_name]
   end
 
   ##
@@ -347,6 +362,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     name
   end
 
+  ##
+  # Extracts a name or symbol from the token stream.
+
   def get_symbol_or_name
     tk = get_tk
     case tk
@@ -361,7 +379,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       text
     when TkId, TkOp then
       tk.name
-    when TkSTRING, TkDSTRING then
+    when TkAMPER,
+         TkDSTRING,
+         TkSTAR,
+         TkSTRING then
       tk.text
     else
       raise RDoc::Error, "Name or symbol expected (got #{tk})"
@@ -374,7 +395,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #   # :stopdoc:
   #   # Don't display comment from this point forward
   #
-  # This routine modifies it's parameter
+  # This routine modifies its +comment+ parameter.
 
   def look_for_directives_in(context, comment)
     preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
@@ -382,9 +403,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     preprocess.handle comment, context do |directive, param|
       case directive
       when 'enddoc' then
-        throw :enddoc
+        context.done_documenting = true
+        ''
       when 'main' then
-        @options.main_page = param
+        @options.main_page = param if @options.respond_to? :main_page
         ''
       when 'method', 'singleton-method',
            'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
@@ -401,7 +423,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         context.stop_doc
         ''
       when 'title' then
-        @options.title = param
+        @options.default_title = param if @options.respond_to? :default_title=
         ''
       end
     end
@@ -426,23 +448,28 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_attr(context, single, tk, comment)
     args = parse_symbol_arg 1
-    if args.size > 0
+    if args.size > 0 then
       name = args[0]
       rw = "R"
       skip_tkspace false
       tk = get_tk
+
       if TkCOMMA === tk then
         rw = "RW" if get_bool
       else
         unget_tk tk
       end
-      att = RDoc::Attr.new get_tkread, name, rw, comment
+
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
-      if att.document_self
-        context.add_attribute(att)
-      end
+
+      context.add_attribute att if att.document_self
+
+      @stats.add_attribute att
     else
-      warn("'attr' ignored - looks like a variable")
+      warn "'attr' ignored - looks like a variable"
     end
   end
 
@@ -452,11 +479,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_attr_accessor(context, single, tk, comment)
     args = parse_symbol_arg
-    read = get_tkread
     rw = "?"
 
-    # TODO If nodoc is given, don't document any of them
-
     tmp = RDoc::CodeObject.new
     read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
     return unless tmp.document_self
@@ -470,17 +494,25 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     for name in args
-      att = RDoc::Attr.new get_tkread, name, rw, comment
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+
       context.add_attribute att
+      @stats.add_attribute att
     end
   end
 
+  ##
+  # Parses an +alias+ in +context+ with +comment+
+
   def parse_alias(context, single, tk, comment)
     skip_tkspace
+
     if TkLPAREN === peek_tk then
       get_tk
       skip_tkspace
     end
+
     new_name = get_symbol_or_name
 
     @scanner.instance_eval { @lex_state = EXPR_FNAME }
@@ -497,16 +529,20 @@ class RDoc::Parser::Ruby < RDoc::Parser
       return
     end
 
-    al = RDoc::Alias.new get_tkread, old_name, new_name, comment
-    al.singleton = SINGLE == single
-    read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
+    al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
+                         single == SINGLE)
+    al.record_location @top_level
 
+    read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
     context.add_alias al if al.document_self
     @stats.add_alias al
 
     al
   end
 
+  ##
+  # Extracts call parameters from the token stream.
+
   def parse_call_parameters(tk)
     end_token = case tk
                 when TkLPAREN, TkfLPAREN
@@ -531,7 +567,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         else
           break unless @scanner.continue
         end
-      when TkCOMMENT
+      when TkCOMMENT, TkASSIGN, TkOPASGN
         unget_tk(tk)
         break
       when nil then
@@ -544,28 +580,33 @@ class RDoc::Parser::Ruby < RDoc::Parser
     res
   end
 
+  ##
+  # Parses a class in +context+ with +comment+
+
   def parse_class(container, single, tk, comment)
-    container, name_t = get_class_or_module container
+    declaration_context = container
+    container, name_t, given_name = get_class_or_module container
 
     case name_t
     when TkCONSTANT
       name = name_t.name
-      superclass = "Object"
+      superclass = '::Object'
 
       if TkLT === peek_tk then
         get_tk
         skip_tkspace
         superclass = get_class_specification
-        superclass = "<unknown>" if superclass.empty?
+        superclass = '(unknown)' if superclass.empty?
       end
 
       cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
-      cls = container.add_class cls_type, name, superclass
+      cls = declaration_context.add_class cls_type, given_name, superclass
 
       read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
       cls.record_location @top_level
-      cls.comment = comment
+      cls.comment = comment if cls.document_self
 
+      @top_level.add_to_classes_or_modules cls
       @stats.add_class cls
 
       parse_statements cls
@@ -582,6 +623,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
           other.comment = comment
         end
 
+        # notify :nodoc: all if not a constant-named class/module
+        # (and remove any comment)
+        unless name =~ /\A(::)?[A-Z]/
+          other.document_self = nil
+          other.document_children = false
+          other.clear_comment
+        end
+
+        @top_level.add_to_classes_or_modules other
         @stats.add_class other
 
         read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
@@ -593,9 +643,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
   end
 
+  ##
+  # Parses a constant in +context+ with +comment+
+
   def parse_constant(container, tk, comment)
     name = tk.name
     skip_tkspace false
+
+    return unless name =~ /^\w+$/
+
     eq_tk = get_tk
 
     unless TkASSIGN === eq_tk then
@@ -619,9 +675,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     loop do
       case tk
       when TkSEMICOLON then
-        break
-      when TkLPAREN, TkfLPAREN, TkLBRACE, TkLBRACK, TkDO, TkIF, TkUNLESS,
-           TkCASE then
+        break if nest <= 0
+      when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
+           TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
         nest += 1
       when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
         nest -= 1
@@ -658,10 +714,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
       tk = get_tk
     end
 
-    res = get_tkread.tr("\n", " ").strip
+    res = get_tkread.gsub(/^[ \t]+/, '').strip
     res = "" if res == ";"
 
     con = RDoc::Constant.new name, res, comment
+    con.record_location @top_level
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
     @stats.add_constant con
@@ -683,6 +740,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       name = $1 unless $1.empty?
 
       meth = RDoc::GhostMethod.new get_tkread, name
+      meth.record_location @top_level
       meth.singleton = singleton
 
       meth.start_collecting_tokens
@@ -713,13 +771,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       name = $3 unless $3.empty?
 
+      # TODO authorize 'singleton-attr...'?
       att = RDoc::Attr.new get_tkread, name, rw, comment
+      att.record_location @top_level
+
       container.add_attribute att
 
-      @stats.add_method att
+      @stats.add_attribute att
     end
   end
 
+  ##
+  # Parses an +include+ in +context+ with +comment+
+
   def parse_include(context, comment)
     loop do
       skip_tkspace_comment
@@ -763,7 +827,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_meta_attr(context, single, tk, comment)
     args = parse_symbol_arg
-    read = get_tkread
     rw = "?"
 
     # If nodoc is given, don't document any of them
@@ -782,12 +845,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     if name then
-      att = RDoc::Attr.new get_tkread, name, rw, comment
+      att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+      att.record_location @top_level
+
       context.add_attribute att
+      @stats.add_attribute att
     else
       args.each do |attr_name|
-        att = RDoc::Attr.new get_tkread, attr_name, rw, comment
+        att = RDoc::Attr.new(get_tkread, attr_name, rw, comment,
+                             single == SINGLE)
+        att.record_location @top_level
+
         context.add_attribute att
+        @stats.add_attribute att
       end
     end
   end
@@ -828,6 +898,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     meth = RDoc::MetaMethod.new get_tkread, name
+    meth.record_location @top_level
     meth.singleton = singleton
 
     remove_token_listener self
@@ -885,7 +956,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     token_listener self do
       @scanner.instance_eval do @lex_state = EXPR_FNAME end
 
-      skip_tkspace false
+      skip_tkspace
       name_t = get_tk
       back_tk = skip_tkspace
       meth = nil
@@ -925,11 +996,17 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
             container.record_location @top_level
           end
-        when TkIDENTIFIER, TkIVAR then
+        when TkIDENTIFIER, TkIVAR, TkGVAR then
           dummy = RDoc::Context.new
           dummy.parent = container
           skip_method dummy
           return
+        when TkTRUE, TkFALSE, TkNIL then
+          klass_name = "#{name_t.name.capitalize}Class"
+          container = RDoc::TopLevel.find_class_named klass_name
+          container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+          name = name_t2.name
         else
           warn "unexpected method name token #{name_t.inspect}"
           # break
@@ -962,6 +1039,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
     end
 
+    meth.record_location @top_level
+
     meth.start_collecting_tokens
     indent = TkSPACE.new nil, 1, 1
     indent.set_text " " * column
@@ -1004,6 +1083,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     @stats.add_method meth
   end
 
+  ##
+  # Extracts +yield+ parameters from +method+
+
   def parse_method_or_yield_parameters(method = nil,
                                        modifiers = RDoc::METHOD_MODIFIERS)
     skip_tkspace false
@@ -1027,14 +1109,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
     loop do
       case tk
       when TkSEMICOLON then
-        break
-      when TkLBRACE then
+        break if nest == 0
+      when TkLBRACE, TkfLBRACE then
         nest += 1
       when TkRBRACE then
-        # we might have a.each {|i| yield i }
-        unget_tk(tk) if nest.zero?
         nest -= 1
-        break if nest <= 0
+        if nest <= 0
+          # we might have a.each { |i| yield i }
+          unget_tk(tk) if nest < 0
+          break
+        end
       when TkLPAREN, TkfLPAREN then
         nest += 1
       when end_token then
@@ -1044,6 +1128,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
         else
           break unless @scanner.continue
         end
+      when TkRPAREN then
+        nest -= 1
       when method && method.block_params.nil? && TkCOMMENT then
         unget_tk tk
         read_documentation_modifiers method, modifiers
@@ -1081,8 +1167,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
   end
 
+  ##
+  # Parses an RDoc::NormalModule in +container+ with +comment+
+
   def parse_module(container, single, tk, comment)
-    container, name_t = get_class_or_module container
+    container, name_t, = get_class_or_module container
 
     name = name_t.name
 
@@ -1090,12 +1179,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
     mod.record_location @top_level
 
     read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+    mod.comment = comment if mod.document_self
     parse_statements(mod)
-    mod.comment = comment
 
+    @top_level.add_to_classes_or_modules mod
     @stats.add_module mod
   end
 
+  ##
+  # Parses an RDoc::Require in +context+ containing +comment+
+
   def parse_require(context, comment)
     skip_tkspace_comment
     tk = get_tk
@@ -1108,7 +1201,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     name = tk.text if TkSTRING === tk
 
     if name then
-      context.add_require RDoc::Require.new(name, comment)
+      @top_level.add_require RDoc::Require.new(name, comment)
     else
       unget_tk tk
     end
@@ -1209,7 +1302,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       # We can't solve the general case, but we can handle most occurrences by
       # ignoring a do at the end of a line.
 
-      when  TkUNTIL, TkWHILE then
+      when TkUNTIL, TkWHILE then
         nest += 1
         skip_optional_do_after_expression
 
@@ -1278,9 +1371,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
   end
 
+  ##
+  # Parse up to +no+ symbol arguments
+
   def parse_symbol_arg(no = nil)
     args = []
+
     skip_tkspace_comment
+
     case tk = get_tk
     when TkLPAREN
       loop do
@@ -1323,28 +1421,40 @@ class RDoc::Parser::Ruby < RDoc::Parser
         end
       end
     end
+
     args
   end
 
+  ##
+  # Returns symbol text from the next token
+
   def parse_symbol_in_arg
     case tk = get_tk
     when TkSYMBOL
       tk.text.sub(/^:/, '')
     when TkSTRING
       eval @read[-1]
+    when TkDSTRING, TkIDENTIFIER then
+      nil # ignore
     else
       warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
       nil
     end
   end
 
+  ##
+  # Parses statements at the toplevel in +container+
+
   def parse_top_level_statements(container)
     comment = collect_first_comment
     look_for_directives_in(container, comment)
-    container.comment = comment unless comment.empty?
+    container.comment = comment if container.document_self unless comment.empty?
     parse_statements container, NORMAL, nil, comment
   end
 
+  ##
+  # Determines the visibility in +container+ from +tk+
+
   def parse_visibility(container, single, tk)
     singleton = (single == SINGLE)
 
@@ -1386,7 +1496,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
         container.methods_matching args do |m|
           s_m = m.dup
-          s_m.singleton = true if RDoc::AnyMethod === s_m
+          s_m.record_location @top_level
+          s_m.singleton = true
           s_m.visibility = :public
           module_functions << s_m
         end
@@ -1406,6 +1517,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
   end
 
+  ##
+  # Determines the block parameter for +context+
+
   def parse_yield(context, single, tk, method)
     return if method.block_params
 
@@ -1426,93 +1540,81 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   # We return the directive name and any parameters as a two element array
 
-  def read_directive(allowed)
+  def read_directive allowed
     tk = get_tk
-    result = nil
 
     if TkCOMMENT === tk then
-      if tk.text =~ /\s*:?(\w+):\s*(.*)/ then
-        directive = $1.downcase
-        if allowed.include? directive then
-          result = [directive, $2]
-        end
-      end
+      return unless tk.text =~ /\s*:?(\w+):\s*(.*)/
+
+      directive = $1.downcase
+
+      return [directive, $2] if allowed.include? directive
     else
       unget_tk tk
     end
-
-    result
   end
 
-  def read_documentation_modifiers(context, allow)
-    dir = read_directive(allow)
-
-    case dir[0]
-    when "notnew", "not_new", "not-new" then
-      context.dont_rename_initialize = true
-
-    when "nodoc" then
-      context.document_self = false
-      if dir[1].downcase == "all"
-        context.document_children = false
-      end
-
-    when "doc" then
-      context.document_self = true
-      context.force_documentation = true
+  ##
+  # Handles the directive for +context+ if the directive is listed in +allow+.
+  # This method is called for directives following a definition.
 
-    when "yield", "yields" then
-      unless context.params.nil?
-        context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
-      end
+  def read_documentation_modifiers(context, allow)
+    directive, value = read_directive allow
 
-      context.block_params = dir[1]
+    return unless directive
 
-    when "arg", "args" then
-      context.params = dir[1]
-    end if dir
+    case directive
+    when 'notnew', 'not_new', 'not-new' then
+      context.dont_rename_initialize = true
+    else
+      RDoc::Parser.process_directive context, directive, value
+    end
   end
 
+  ##
+  # Removes private comments from +comment+
+
   def remove_private_comments(comment)
-    comment.gsub!(/^#--\n.*?^#\+\+/m, '')
-    comment.sub!(/^#--\n.*/m, '')
+    comment.gsub!(/^#--\n.*?^#\+\+\n?/m, '')
+    comment.sub!(/^#--\n.*\n?/m, '')
   end
 
+  ##
+  # Scans this ruby file for ruby constructs
+
   def scan
     reset
 
     catch :eof do
-      catch :enddoc do
-        begin
-          parse_top_level_statements @top_level
-        rescue StandardError => e
-          bytes = ''
-
-          20.times do @scanner.ungetc end
-          count = 0
-          60.times do |i|
-            count = i
-            byte = @scanner.getc
-            break unless byte
-            bytes << byte
-          end
-          count -= 20
-          count.times do @scanner.ungetc end
+      begin
+        parse_top_level_statements @top_level
+      rescue StandardError => e
+        bytes = ''
+
+        20.times do @scanner.ungetc end
+        count = 0
+        60.times do |i|
+          count = i
+          byte = @scanner.getc
+          break unless byte
+          bytes << byte
+        end
+        count -= 20
+        count.times do @scanner.ungetc end
 
-          $stderr.puts <<-EOF
+        $stderr.puts <<-EOF
 
 #{self.class} failure around line #{@scanner.line_no} of
 #{@file_name}
 
-          EOF
+        EOF
 
-          unless bytes.empty? then
-            $stderr.puts
-            $stderr.puts bytes.inspect
-          end
-
-          raise e
+        unless bytes.empty? then
+          $stderr.puts
+          $stderr.puts bytes.inspect
         end
+
+        raise e
       end
     end
 
@@ -1577,6 +1679,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     unget_tk(tk) unless TkIN === tk
   end
 
+  ##
+  # Skips the next method in +container+
+
   def skip_method container
     meth = RDoc::AnyMethod.new "", "anon"
     parse_method_parameters meth
@@ -1594,6 +1699,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
   end
 
+  ##
+  # Prints +msg+ to +$stderr+ unless we're being quiet
+
   def warn(msg)
     return if @options.quiet
     msg = make_message msg