rdoc 2.4.3 → 2.5

data/lib/rdoc/parser/ruby_tools.rb

@@ -0,0 +1,157 @@
+##
+# Collection of methods for writing parsers against RDoc::RubyLex and
+# RDoc::RubyToken
+
+module RDoc::Parser::RubyTools
+
+  include RDoc::RubyToken
+
+  ##
+  # Adds a token listener +obj+, but you should probably use token_listener
+
+  def add_token_listener(obj)
+    @token_listeners ||= []
+    @token_listeners << obj
+  end
+
+  ##
+  # Fetches the next token from the scanner
+
+  def get_tk
+    tk = nil
+
+    if @tokens.empty? then
+      tk = @scanner.token
+      @read.push @scanner.get_readed
+      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+    else
+      @read.push @unget_read.shift
+      tk = @tokens.shift
+      puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
+    end
+
+    tk = nil if TkEND_OF_SCRIPT === tk
+
+    if TkSYMBEG === tk then
+      set_token_position tk.line_no, tk.char_no
+
+      case tk1 = get_tk
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
+        end
+
+        # remove the identifier we just read (we're about to replace it with a
+        # symbol)
+        @token_listeners.each do |obj|
+          obj.pop_token
+        end if @token_listeners
+      else
+        warn("':' not followed by identifier or operator")
+        tk = tk1
+      end
+    end
+
+    # inform any listeners of our shiny new token
+    @token_listeners.each do |obj|
+      obj.add_token(tk)
+    end if @token_listeners
+
+    tk
+  end
+
+  def get_tk_until(*tokens)
+    read = []
+
+    loop do
+      tk = get_tk
+      case tk when *tokens then unget_tk tk; break end
+      read << tk
+    end
+
+    read
+  end
+
+  ##
+  # Retrieves a String representation of the read tokens
+
+  def get_tkread
+    read = @read.join("")
+    @read = []
+    read
+  end
+
+  ##
+  # Peek equivalent for get_tkread
+
+  def peek_read
+    @read.join('')
+  end
+
+  ##
+  # Peek at the next token, but don't remove it from the stream
+
+  def peek_tk
+    unget_tk(tk = get_tk)
+    tk
+  end
+
+  ##
+  # Removes the token listener +obj+
+
+  def remove_token_listener(obj)
+    @token_listeners.delete(obj)
+  end
+
+  ##
+  # Resets the tools
+
+  def reset
+    @read       = []
+    @tokens     = []
+    @unget_read = []
+    @nest = 0
+  end
+
+  ##
+  # Skips whitespace tokens including newlines if +skip_nl+ is true
+
+  def skip_tkspace(skip_nl = true) # HACK dup
+    tokens = []
+
+    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
+      tokens.push tk
+    end
+
+    unget_tk tk
+    tokens
+  end
+
+  ##
+  # Has +obj+ listen to tokens
+
+  def token_listener(obj)
+    add_token_listener obj
+    yield
+  ensure
+    remove_token_listener obj
+  end
+
+  ##
+  # Returns +tk+ to the scanner
+
+  def unget_tk(tk)
+    @tokens.unshift tk
+    @unget_read.unshift @read.pop
+
+    # Remove this token from any listeners
+    @token_listeners.each do |obj|
+      obj.pop_token
+    end if @token_listeners
+  end
+
+end
+
+

data/lib/rdoc/parser/simple.rb

@@ -1,5 +1,3 @@
-require 'rdoc/parser'
-
 ##
 # Parse a non-source file. We basically take the whole thing as one big
 # comment. If the first character in the file is '#', we strip leading pound

data/lib/rdoc/rdoc.rb

@@ -11,12 +11,9 @@ require 'rdoc/parser/perl'
 require 'rdoc/stats'
 require 'rdoc/options'
 
-require 'rdoc/diagram'
-
 require 'find'
 require 'fileutils'
 require 'time'
-require 'thread'
 
 ##
 # Encapsulate the production of rdoc documentation. Basically you can use this
@@ -30,6 +27,11 @@ require 'thread'
 
 class RDoc::RDoc
 
+  ##
+  # File pattern to exclude
+
+  attr_accessor :exclude
+
   ##
   # Generator instance used for creating output
 
@@ -38,7 +40,7 @@ class RDoc::RDoc
   ##
   # RDoc options
 
-  attr_reader :options
+  attr_accessor :options
 
   ##
   # Accessor for statistics.  Available after each call to parse_files
@@ -73,9 +75,13 @@ class RDoc::RDoc
   end
 
   def initialize
-    @generator = nil
-    @options = nil
-    @stats = nil
+    @current      = nil
+    @exclude      = nil
+    @generator    = nil
+    @last_created = nil
+    @old_siginfo  = nil
+    @options      = nil
+    @stats        = nil
   end
 
   ##
@@ -85,6 +91,20 @@ class RDoc::RDoc
     raise RDoc::Error, msg
   end
 
+  ##
+  # Gathers a set of parseable files from the files and directories listed in
+  # +files+.
+
+  def gather_files files
+    files = ["."] if files.empty?
+
+    file_list = normalized_file_list files, true, @exclude
+
+    file_list = file_list.uniq
+
+    file_list = remove_unparseable file_list
+  end
+
   ##
   # Turns RDoc from stdin into HTML
 
@@ -96,6 +116,17 @@ class RDoc::RDoc
     $stdout.write out
   end
 
+  ##
+  # Installs a siginfo handler that prints the current filename.
+
+  def install_siginfo_handler
+    return unless Signal.list.include? 'INFO'
+
+    @old_siginfo = trap 'INFO' do
+      puts @current if @current
+    end
+  end
+
   ##
   # Create an output dir if it doesn't exist. If it does exist, but doesn't
   # contain the flag file <tt>created.rid</tt> then we refuse to use it, as
@@ -122,6 +153,7 @@ class RDoc::RDoc
     else
       FileUtils.mkdir_p(op_dir)
     end
+
     last
   end
 
@@ -144,7 +176,7 @@ class RDoc::RDoc
   # representing candidates for documentation. It may also contain comments
   # (starting with '#')
 
-  def parse_dot_doc_file(in_dir, filename, options)
+  def parse_dot_doc_file in_dir, filename
     # read and strip comments
     patterns = File.read(filename).gsub(/#.*/, '')
 
@@ -152,7 +184,7 @@ class RDoc::RDoc
 
     patterns.split.each do |patt|
       candidates = Dir.glob(File.join(in_dir, patt))
-      result.concat(normalized_file_list(options,  candidates))
+      result.concat normalized_file_list(candidates)
     end
 
     result
@@ -170,7 +202,7 @@ class RDoc::RDoc
   # The effect of this is that if you want a file with a non-standard
   # extension parsed, you must name it explicitly.
 
-  def normalized_file_list(options, relative_files, force_doc = false,
+  def normalized_file_list(relative_files, force_doc = false,
                            exclude_pattern = nil)
     file_list = []
 
@@ -187,18 +219,20 @@ class RDoc::RDoc
         end
       when "directory"
         next if rel_file_name == "CVS" || rel_file_name == ".svn"
-        dot_doc = File.join(rel_file_name, RDoc::DOT_DOC_FILENAME)
-        if File.file?(dot_doc)
-          file_list.concat(parse_dot_doc_file(rel_file_name, dot_doc, options))
+
+        dot_doc = File.join rel_file_name, RDoc::DOT_DOC_FILENAME
+
+        if File.file?(dot_doc) then
+          file_list << parse_dot_doc_file(rel_file_name, dot_doc)
         else
-          file_list.concat(list_files_in_directory(rel_file_name, options))
+          file_list << list_files_in_directory(rel_file_name)
         end
       else
         raise RDoc::Error, "I can't deal with a #{type} #{rel_file_name}"
       end
     end
 
-    file_list
+    file_list.flatten
   end
 
   ##
@@ -207,74 +241,82 @@ class RDoc::RDoc
   # files. However we may well contain subdirectories which must be tested
   # for .document files.
 
-  def list_files_in_directory(dir, options)
+  def list_files_in_directory dir
     files = Dir.glob File.join(dir, "*")
 
-    normalized_file_list options, files, false, options.exclude
+    normalized_file_list files, false, @options.exclude
   end
 
   ##
-  # Parse each file on the command line, recursively entering directories.
+  # Parses +filename+ and returns an RDoc::TopLevel
 
-  def parse_files(options)
-    files = options.files
-    files = ["."] if files.empty?
+  def parse_file filename
+    @stats.add_file filename
+    content = read_file_contents filename
+
+    return unless content
+
+    top_level = RDoc::TopLevel.new filename
+
+    parser = RDoc::Parser.for top_level, filename, content, @options, @stats
+
+    return unless parser
+
+    parser.scan
+  rescue => e
+    $stderr.puts <<-EOF
+Before reporting this, could you check that the file you're documenting
+compiles cleanly--RDoc is not a full Ruby parser, and gets confused easily if
+fed invalid programs.
+
+The internal error was:
+
+\t(#{e.class}) #{e.message}
+
+    EOF
 
-    file_list = normalized_file_list(options, files, true, options.exclude)
+    $stderr.puts e.backtrace.join("\n\t") if $RDOC_DEBUG
+
+    raise e
+    nil
+  end
+
+  ##
+  # Parse each file on the command line, recursively entering directories.
+
+  def parse_files files
+    file_list = gather_files files
 
     return [] if file_list.empty?
 
-    jobs = SizedQueue.new(@options.threads * 3)
-    workers = []
     file_info = []
-    file_info_lock = Mutex.new
-
-    Thread.abort_on_exception = true
-    @stats = RDoc::Stats.new(file_list.size, options.verbosity)
-    @stats.begin_adding @options.threads
-
-    # Create worker threads.
-    @options.threads.times do
-      thread = Thread.new do
-        while (filename = jobs.pop)
-          @stats.add_file(filename)
-          content = read_file_contents(filename)
-          top_level = RDoc::TopLevel.new filename
-
-          parser = RDoc::Parser.for(top_level, filename, content, options,
-                                    @stats)
-          result = parser.scan
-
-          file_info_lock.synchronize do
-            file_info << result
-          end
-        end
-      end
-      workers << thread
-    end
 
-    # Feed filenames to the parser worker threads...
-    file_list.each do |filename|
-      jobs << filename
-    end
-    workers.size.times do
-      jobs << nil
-    end
+    @stats = RDoc::Stats.new file_list.size, @options.verbosity
+    @stats.begin_adding
 
-    # ...and wait until they're done.
-    workers.each do |thread|
-      thread.join
-    end
+    file_info = file_list.map do |filename|
+      @current = filename
+      parse_file filename
+    end.compact
 
     @stats.done_adding
 
     file_info
   end
 
+  ##
+  # Removes file extensions known to be unparseable from +files+
+
+  def remove_unparseable files
+    files.reject do |file|
+      file =~ /\.(?:class|eps|erb|scpt\.txt|ttf|yml)$/i
+    end
+  end
+
   ##
   # Format up one or more files according to the given arguments.
   #
-  # For simplicity, _argv_ is an array of strings, equivalent to the strings
+  # For simplicity, +argv+ is an array of strings, equivalent to the strings
   # that would be passed on the command line. (This isn't a coincidence, as
   # we _do_ pass in ARGV when running interactively). For a list of options,
   # see rdoc/rdoc.rb. By default, output will be stored in a directory
@@ -285,6 +327,8 @@ class RDoc::RDoc
 
   def document(argv)
     RDoc::TopLevel.reset
+    RDoc::Parser::C.reset
+    RDoc::AnyMethod.reset
 
     @options = RDoc::Options.new
     @options.parse argv
@@ -294,11 +338,13 @@ class RDoc::RDoc
       exit
     end
 
+    @exclude = @options.exclude
+
     @last_created = setup_output_dir @options.op_dir, @options.force_update
 
     start_time = Time.now
 
-    file_info = parse_files @options
+    file_info = parse_files @options.files
 
     @options.title = "RDoc Documentation"
 
@@ -315,17 +361,15 @@ class RDoc::RDoc
 
       pwd = Dir.pwd
 
-      Dir.chdir @options.op_dir
+      Dir.chdir @options.op_dir do
+        begin
+          self.class.current = self
 
-      begin
-        self.class.current = self
-
-        RDoc::Diagram.new(file_info, @options).draw if @options.diagram
-        @generator.generate file_info
-        update_output_dir ".", start_time
-      ensure
-        self.class.current = nil
-        Dir.chdir pwd
+          @generator.generate file_info
+          update_output_dir ".", start_time
+        ensure
+          self.class.current = nil
+        end
       end
     end
 
@@ -335,8 +379,6 @@ class RDoc::RDoc
     end
   end
 
-  private
-
   def read_file_contents(filename)
     content = if RUBY_VERSION >= '1.9' then
                 File.open(filename, "r:ascii-8bit") { |f| f.read }
@@ -345,7 +387,7 @@ class RDoc::RDoc
               end
 
     if defined? Encoding then
-      if /coding:\s*(\S+)/ =~ content[/\A(?:.*\n){0,2}/]
+      if /coding:\s*([\w-]+)/i =~ content[/\A(?:.*\n){0,2}/]
         if enc = ::Encoding.find($1)
           content.force_encoding(enc)
         end
@@ -353,20 +395,38 @@ class RDoc::RDoc
     end
 
     content
+  rescue Errno::EISDIR, Errno::ENOENT
+    nil
+  end
+
+  ##
+  # Removes a siginfo handler and replaces the previous
+
+  def remove_siginfo_handler
+    return unless Signal.list.key? 'INFO'
+
+    handler = @old_siginfo || 'DEFAULT'
+
+    trap 'INFO', handler
   end
 
 end
 
-if Gem.respond_to? :find_files then
-  rdoc_extensions = Gem.find_files 'rdoc/discover'
+begin
+  require 'rubygems'
 
-  rdoc_extensions.each do |extension|
-    begin
-      load extension
-    rescue => e
-      warn "error loading #{extension.inspect}: #{e.message} (#{e.class})"
+  if Gem.respond_to? :find_files then
+    rdoc_extensions = Gem.find_files 'rdoc/discover'
+
+    rdoc_extensions.each do |extension|
+      begin
+        load extension
+      rescue => e
+        warn "error loading #{extension.inspect}: #{e.message} (#{e.class})"
+      end
     end
   end
+rescue LoadError
 end
 
 # require built-in generators after discovery in case they've been replaced