rdoc 3.1 → 6.3.3

data/lib/rdoc/parser/ruby_tools.rb

@@ -1,11 +1,9 @@
+# frozen_string_literal: true
 ##
-# Collection of methods for writing parsers against RDoc::RubyLex and
-# RDoc::RubyToken
+# Collection of methods for writing parsers
 
 module RDoc::Parser::RubyTools
 
-  include RDoc::RubyToken
-
   ##
   # Adds a token listener +obj+, but you should probably use token_listener
 
@@ -21,38 +19,24 @@ module RDoc::Parser::RubyTools
     tk = nil
 
     if @tokens.empty? then
-      tk = @scanner.token
-      @read.push @scanner.get_readed
-      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+      if @scanner_point >= @scanner.size
+        return nil
+      else
+        tk = @scanner[@scanner_point]
+        @scanner_point += 1
+        @read.push tk[:text]
+      end
     else
       @read.push @unget_read.shift
       tk = @tokens.shift
-      puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
     end
 
-    tk = nil if TkEND_OF_SCRIPT === tk
-
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read (we're about to replace it with a
-        # symbol)
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
+    if tk == nil || :on___end__ == tk[:kind]
+      tk = nil
     end
 
+    return nil unless tk
+
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)
@@ -70,7 +54,13 @@ module RDoc::Parser::RubyTools
 
     loop do
       tk = get_tk
-      case tk when *tokens then unget_tk tk; break end
+
+      case tk
+      when *tokens then
+        unget_tk tk
+        break
+      end
+
       read << tk
     end
 
@@ -116,19 +106,34 @@ module RDoc::Parser::RubyTools
     @tokens     = []
     @unget_read = []
     @nest = 0
+    @scanner_point = 0
   end
 
   ##
-  # Skips whitespace tokens including newlines if +skip_nl+ is true
+  # Skips whitespace tokens including newlines
 
-  def skip_tkspace(skip_nl = true) # HACK dup
+  def skip_tkspace
     tokens = []
 
-    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
-      tokens.push tk
+    while (tk = get_tk) and (:on_sp == tk[:kind] or :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]) do
+      tokens.push(tk)
     end
 
-    unget_tk tk
+    unget_tk(tk)
+    tokens
+  end
+
+  ##
+  # Skips whitespace tokens excluding newlines
+
+  def skip_tkspace_without_nl
+    tokens = []
+
+    while (tk = get_tk) and :on_sp == tk[:kind] do
+      tokens.push(tk)
+    end
+
+    unget_tk(tk)
     tokens
   end
 
@@ -153,6 +158,8 @@ module RDoc::Parser::RubyTools
     @token_listeners.each do |obj|
       obj.pop_token
     end if @token_listeners
+
+    nil
   end
 
 end

data/lib/rdoc/parser/simple.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
 ##
 # Parse a non-source file. We basically take the whole thing as one big
 # comment.
 
 class RDoc::Parser::Simple < RDoc::Parser
 
+  include RDoc::Parser::Text
+
   parse_files_matching(//)
 
   attr_reader :content # :nodoc:
@@ -16,7 +19,7 @@ class RDoc::Parser::Simple < RDoc::Parser
 
     preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
 
-    preprocess.handle @content, @top_level
+    @content = preprocess.handle @content, @top_level
   end
 
   ##
@@ -24,20 +27,14 @@ class RDoc::Parser::Simple < RDoc::Parser
 
   def scan
     comment = remove_coding_comment @content
-    comment = remove_private_comments comment
+    comment = remove_private_comment comment
+
+    comment = RDoc::Comment.new comment, @top_level
 
     @top_level.comment = comment
-    @top_level.parser = self.class
     @top_level
   end
 
-  ##
-  # Removes comments wrapped in <tt>--/++</tt>
-
-  def remove_private_comments text
-    text.gsub(/^--\n.*?^\+\+/m, '').sub(/^--\n.*/m, '')
-  end
-
   ##
   # Removes the encoding magic comment from +text+
 
@@ -45,5 +42,20 @@ class RDoc::Parser::Simple < RDoc::Parser
     text.sub(/\A# .*coding[=:].*$/, '')
   end
 
-end
+  ##
+  # Removes private comments.
+  #
+  # Unlike RDoc::Comment#remove_private this implementation only looks for two
+  # dashes at the beginning of the line.  Three or more dashes are considered
+  # to be a rule and ignored.
+
+  def remove_private_comment comment
+    # Workaround for gsub encoding for Ruby 1.9.2 and earlier
+    empty = ''
+    empty = RDoc::Encoding.change_encoding empty, comment.encoding
+
+    comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
+    comment.sub(%r%^--\n.*%m, empty)
+  end
 
+end

data/lib/rdoc/parser/text.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+##
+# Indicates this parser is text and doesn't contain code constructs.
+#
+# Include this module in a RDoc::Parser subclass to make it show up as a file,
+# not as part of a class or module.
+#--
+# This is not named File to avoid overriding ::File
+
+module RDoc::Parser::Text
+end
+

data/lib/rdoc/parser.rb

@@ -1,42 +1,34 @@
-require 'rdoc'
-require 'rdoc/code_objects'
-require 'rdoc/markup/pre_process'
-require 'rdoc/stats'
+# -*- coding: us-ascii -*-
+# frozen_string_literal: true
 
 ##
-# A parser is simple a class that implements
+# A parser is simple a class that subclasses RDoc::Parser and implements #scan
+# to fill in an RDoc::TopLevel with parsed data.
 #
-#   #initialize(file_name, body, options)
+# The initialize method takes an RDoc::TopLevel to fill with parsed content,
+# the name of the file to be parsed, the content of the file, an RDoc::Options
+# object and an RDoc::Stats object to inform the user of parsed items.  The
+# scan method is then called to parse the file and must return the
+# RDoc::TopLevel object.  By calling super these items will be set for you.
 #
-# and
+# In order to be used by RDoc the parser needs to register the file extensions
+# it can parse.  Use ::parse_files_matching to register extensions.
 #
-#   #scan
-#
-# The initialize method takes a file name to be used, the body of the file,
-# and an RDoc::Options object. The scan method is then called to return an
-# appropriately parsed TopLevel code object.
-#
-# The ParseFactory is used to redirect to the correct parser given a
-# filename extension. This magic works because individual parsers have to
-# register themselves with us as they are loaded in. The do this using the
-# following incantation
-#
-#   require "rdoc/parser"
+#   require 'rdoc'
 #
 #   class RDoc::Parser::Xyz < RDoc::Parser
-#     parse_files_matching /\.xyz$/ # <<<<
+#     parse_files_matching /\.xyz$/
 #
-#     def initialize(file_name, body, options)
-#       ...
+#     def initialize top_level, file_name, content, options, stats
+#       super
+#
+#       # extra initialization if needed
 #     end
 #
 #     def scan
-#       ...
+#       # parse file and fill in @top_level
 #     end
 #   end
-#
-# Just to make life interesting, if we suspect a plain text file, we also
-# look for a shebang line just in case it's a potential shell script
 
 class RDoc::Parser
 
@@ -45,8 +37,8 @@ class RDoc::Parser
   class << self
 
     ##
-    # A Hash that maps file exetensions regular expressions to parsers that
-    # will consume them.
+    # An Array of arrays that maps file extension (or name) regular
+    # expressions to parser classes that will parse matching filenames.
     #
     # Use parse_files_matching to register a parser's file extensions.
 
@@ -54,6 +46,11 @@ class RDoc::Parser
 
   end
 
+  ##
+  # The name of the file being parsed
+
+  attr_reader :file_name
+
   ##
   # Alias an extension to another extension. After this call, files ending
   # "new_ext" will be parsed using the same parser as "old_ext"
@@ -62,7 +59,7 @@ class RDoc::Parser
     old_ext = old_ext.sub(/^\.(.*)/, '\1')
     new_ext = new_ext.sub(/^\.(.*)/, '\1')
 
-    parser = can_parse "xxx.#{old_ext}"
+    parser = can_parse_by_name "xxx.#{old_ext}"
     return false unless parser
 
     RDoc::Parser.parsers.unshift [/\.#{new_ext}$/, parser]
@@ -79,48 +76,15 @@ class RDoc::Parser
 
     s = File.read(file, 1024) or return false
 
-    have_encoding = s.respond_to? :encoding
-
-    if have_encoding then
-      return false if s.encoding != Encoding::ASCII_8BIT and s.valid_encoding?
-    end
-
     return true if s[0, 2] == Marshal.dump('')[0, 2] or s.index("\x00")
 
-    if have_encoding then
-      s.force_encoding Encoding.default_external
+    mode = 'r:utf-8' # default source encoding has been changed to utf-8
+    s.sub!(/\A#!.*\n/, '')     # assume shebang line isn't longer than 1024.
+    encoding = s[/^\s*\#\s*(?:-\*-\s*)?(?:en)?coding:\s*([^\s;]+?)(?:-\*-|[\s;])/, 1]
+    mode = "rb:#{encoding}" if encoding
+    s = File.open(file, mode) {|f| f.gets(nil, 1024)}
 
-      not s.valid_encoding?
-    else
-      if 0.respond_to? :fdiv then
-        s.count("\x00-\x7F", "^ -~\t\r\n").fdiv(s.size) > 0.3
-      else # HACK 1.8.6
-        (s.count("\x00-\x7F", "^ -~\t\r\n").to_f / s.size) > 0.3
-      end
-    end
-  end
-
-  ##
-  # Processes common directives for CodeObjects for the C and Ruby parsers.
-  #
-  # Applies +directive+'s +value+ to +code_object+, if appropriate
-
-  def self.process_directive code_object, directive, value
-    case directive
-    when 'nodoc' then
-      code_object.document_self = nil # notify nodoc
-      code_object.document_children = value.downcase != 'all'
-    when 'doc' then
-      code_object.document_self = true
-      code_object.force_documentation = true
-    when 'yield', 'yields' then
-      # remove parameter &block
-      code_object.params.sub!(/,?\s*&\w+/, '') if code_object.params
-
-      code_object.block_params = value
-    when 'arg', 'args' then
-      code_object.params = value
-    end
+    not s.valid_encoding?
   end
 
   ##
@@ -133,46 +97,100 @@ class RDoc::Parser
     zip_signature == "PK\x03\x04" or
       zip_signature == "PK\x05\x06" or
       zip_signature == "PK\x07\x08"
+  rescue
+    false
   end
 
   ##
   # Return a parser that can handle a particular extension
 
-  def self.can_parse(file_name)
-    parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }.last
+  def self.can_parse file_name
+    parser = can_parse_by_name file_name
 
     # HACK Selenium hides a jar file using a .txt extension
     return if parser == RDoc::Parser::Simple and zip? file_name
 
+    parser
+  end
+
+  ##
+  # Returns a parser that can handle the extension for +file_name+.  This does
+  # not depend upon the file being readable.
+
+  def self.can_parse_by_name file_name
+    _, parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }
+
     # The default parser must not parse binary files
     ext_name = File.extname file_name
     return parser if ext_name.empty?
-    return if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/
+
+    if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/ then
+      case check_modeline file_name
+      when nil, 'rdoc' then # continue
+      else return nil
+      end
+    end
 
     parser
+  rescue Errno::EACCES
+  end
+
+  ##
+  # Returns the file type from the modeline in +file_name+
+
+  def self.check_modeline file_name
+    line = File.open file_name do |io|
+      io.gets
+    end
+
+    /-\*-\s*(.*?\S)\s*-\*-/ =~ line
+
+    return nil unless type = $1
+
+    if /;/ =~ type then
+      return nil unless /(?:\s|\A)mode:\s*([^\s;]+)/i =~ type
+      type = $1
+    end
+
+    return nil if /coding:/i =~ type
+
+    type.downcase
+  rescue ArgumentError
+  rescue Encoding::InvalidByteSequenceError # invalid byte sequence
+
   end
 
   ##
-  # Find the correct parser for a particular file name. Return a SimpleParser
-  # for ones that we don't know
+  # Finds and instantiates the correct parser for the given +file_name+ and
+  # +content+.
 
-  def self.for(top_level, file_name, body, options, stats)
+  def self.for top_level, file_name, content, options, stats
     return if binary? file_name
 
-    # If no extension, look for shebang
-    if file_name !~ /\.\w+$/ && body =~ %r{\A#!(.+)} then
-      shebang = $1
-      case shebang
-      when %r{env\s+ruby}, %r{/ruby}
-        file_name = "dummy.rb"
+    parser = use_markup content
+
+    unless parser then
+      parse_name = file_name
+
+      # If no extension, look for shebang
+      if file_name !~ /\.\w+$/ && content =~ %r{\A#!(.+)} then
+        shebang = $1
+        case shebang
+        when %r{env\s+ruby}, %r{/ruby}
+          parse_name = 'dummy.rb'
+        end
       end
-    end
 
-    parser = can_parse file_name
+      parser = can_parse parse_name
+    end
 
     return unless parser
 
-    parser.new top_level, file_name, body, options, stats
+    content = remove_modeline content
+
+    parser.new top_level, file_name, content, options, stats
+  rescue SystemCallError
+    nil
   end
 
   ##
@@ -185,20 +203,75 @@ class RDoc::Parser
   end
 
   ##
-  # Creates a new Parser storing +top_level+, +file_name+, +content+, 
-  # +options+ and +stats+ in instance variables.
+  # Removes an emacs-style modeline from the first line of the document
+
+  def self.remove_modeline content
+    content.sub(/\A.*-\*-\s*(.*?\S)\s*-\*-.*\r?\n/, '')
+  end
+
+  ##
+  # If there is a <tt>markup: parser_name</tt> comment at the front of the
+  # file, use it to determine the parser.  For example:
+  #
+  #   # markup: rdoc
+  #   # Class comment can go here
+  #
+  #   class C
+  #   end
   #
-  # Usually invoked by +super+
+  # The comment should appear as the first line of the +content+.
+  #
+  # If the content contains a shebang or editor modeline the comment may
+  # appear on the second or third line.
+  #
+  # Any comment style may be used to hide the markup comment.
+
+  def self.use_markup content
+    markup = content.lines.first(3).grep(/markup:\s+(\w+)/) { $1 }.first
+
+    return unless markup
+
+    # TODO Ruby should be returned only when the filename is correct
+    return RDoc::Parser::Ruby if %w[tomdoc markdown].include? markup
+
+    markup = Regexp.escape markup
+
+    _, selected = RDoc::Parser.parsers.find do |_, parser|
+      /^#{markup}$/i =~ parser.name.sub(/.*:/, '')
+    end
 
-  def initialize(top_level, file_name, content, options, stats)
+    selected
+  end
+
+  ##
+  # Creates a new Parser storing +top_level+, +file_name+, +content+,
+  # +options+ and +stats+ in instance variables.  In +@preprocess+ an
+  # RDoc::Markup::PreProcess object is created which allows processing of
+  # directives.
+
+  def initialize top_level, file_name, content, options, stats
     @top_level = top_level
+    @top_level.parser = self.class
+    @store = @top_level.store
+
     @file_name = file_name
     @content = content
     @options = options
     @stats = stats
+
+    @preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+    @preprocess.options = @options
   end
 
-end
+  autoload :RubyTools, 'rdoc/parser/ruby_tools'
+  autoload :Text,      'rdoc/parser/text'
 
-require 'rdoc/parser/simple'
+end
 
+# simple must come first in order to show up last in the parsers list
+require_relative 'parser/simple'
+require_relative 'parser/c'
+require_relative 'parser/changelog'
+require_relative 'parser/markdown'
+require_relative 'parser/rd'
+require_relative 'parser/ruby'