rdoc 6.17.0 → 8.1.0

data/lib/rdoc/token_stream.rb

@@ -1,130 +1,112 @@
 # frozen_string_literal: true
 
-##
-# A TokenStream is a list of tokens, gathered during the parse of some entity
-# (say a method). Entities populate these streams by being registered with the
-# lexer. Any class can collect tokens by including TokenStream. From the
-# outside, you use such an object by calling the start_collecting_tokens
-# method, followed by calls to add_token and pop_token.
-
-module RDoc::TokenStream
-
+module RDoc
   ##
-  # Converts +token_stream+ to HTML wrapping various tokens with
-  # <tt><span></tt> elements. Some tokens types are wrapped in spans
-  # with the given class names. Other token types are not wrapped in spans.
-
-  def self.to_html(token_stream)
-    starting_title = false
-
-    token_stream.map do |t|
-      next unless t
-
-      style = case t[:kind]
-              when :on_const   then 'ruby-constant'
-              when :on_kw      then 'ruby-keyword'
-              when :on_ivar    then 'ruby-ivar'
-              when :on_cvar    then 'ruby-identifier'
-              when :on_gvar    then 'ruby-identifier'
-              when '=' != t[:text] && :on_op
-                               then 'ruby-operator'
-              when :on_tlambda then 'ruby-operator'
-              when :on_ident   then 'ruby-identifier'
-              when :on_label   then 'ruby-value'
-              when :on_backref, :on_dstring
-                               then 'ruby-node'
-              when :on_comment then 'ruby-comment'
-              when :on_embdoc  then 'ruby-comment'
-              when :on_regexp  then 'ruby-regexp'
-              when :on_tstring then 'ruby-string'
-              when :on_int, :on_float,
-                   :on_rational, :on_imaginary,
-                   :on_heredoc,
-                   :on_symbol, :on_CHAR then 'ruby-value'
-              when :on_heredoc_beg, :on_heredoc_end
-                               then 'ruby-identifier'
-              end
-
-      comment_with_nl = false
-      if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind]
-        comment_with_nl = true if "\n" == t[:text][-1]
-        text = t[:text].rstrip
-      else
-        text = t[:text]
-      end
+  # A TokenStream is a list of tokens, gathered during the parse of some entity
+  # (say a method). Entities populate these streams by being registered with the
+  # lexer. Any class can collect tokens by including TokenStream. From the
+  # outside, you use such an object by calling the start_collecting_tokens
+  # method, followed by calls to add_token and pop_token.
+
+  module TokenStream
+
+    ##
+    # Converts +token_stream+ to HTML wrapping various tokens with
+    # <tt><span></tt> elements. Some tokens types are wrapped in spans
+    # with the given class names. Other token types are not wrapped in spans.
+
+    def self.to_html(token_stream)
+      token_stream.map do |t|
+        next unless t
+
+        style = case t[:kind]
+                when :operator   then 'ruby-operator'
+                when :keyword    then 'ruby-keyword'
+                when :constant   then 'ruby-constant'
+                when :ivar       then 'ruby-ivar'
+                when :comment    then 'ruby-comment'
+                when :value      then 'ruby-value'
+                when :string     then 'ruby-string'
+                when :symbol     then 'ruby-value'
+                when :x_string   then 'ruby-string'
+                when :regexp     then 'ruby-regexp'
+                when :identifier then 'ruby-identifier'
+                end
 
-      if :on_ident == t[:kind] && starting_title
-        starting_title = false
-        style = 'ruby-identifier ruby-title'
-      end
+        text = t[:text]
 
-      if :on_kw == t[:kind] and 'def' == t[:text]
-        starting_title = true
-      end
+        text = CGI.escapeHTML text
 
-      text = CGI.escapeHTML text
+        if style
+          end_with_newline = text.end_with?("\n")
+          text = text.chomp if end_with_newline
+          "<span class=\"#{style}\">#{text}</span>#{"\n" if end_with_newline}"
+        else
+          text
+        end
+      end.join
+    end
 
-      if style then
-        "<span class=\"#{style}\">#{text}</span>#{"\n" if comment_with_nl}"
-      else
-        text
-      end
-    end.join
-  end
+    ##
+    # Adds +tokens+ to the collected tokens
 
-  ##
-  # Adds +tokens+ to the collected tokens
+    def add_tokens(tokens)
+      token_stream.concat(tokens)
+    end
 
-  def add_tokens(tokens)
-    @token_stream.concat(tokens)
-  end
+    ##
+    # Adds one +token+ to the collected tokens
 
-  ##
-  # Adds one +token+ to the collected tokens
+    def add_token(token)
+      token_stream.push(token)
+    end
 
-  def add_token(token)
-    @token_stream.push(token)
-  end
+    ##
+    # Starts collecting tokens
+    #
+    # The optional +loader+ is called once on first access and its result is reused.
 
-  ##
-  # Starts collecting tokens
-  #
+    def collect_tokens(language, loader: nil)
+      @token_stream = []
+      @token_stream_loader = loader
+      @token_stream_language = language
+    end
 
-  def collect_tokens(language)
-    @token_stream = []
-    @token_stream_language = language
-  end
+    alias start_collecting_tokens collect_tokens
 
-  alias start_collecting_tokens collect_tokens
+    ##
+    # Remove the last token from the collected tokens
 
-  ##
-  # Remove the last token from the collected tokens
+    def pop_token
+      token_stream.pop
+    end
 
-  def pop_token
-    @token_stream.pop
-  end
+    ##
+    # Current token stream
 
-  ##
-  # Current token stream
+    def token_stream
+      if @token_stream_loader
+        @token_stream = @token_stream_loader.call
+        @token_stream_loader = nil
+      end
+      @token_stream
+    end
 
-  def token_stream
-    @token_stream
-  end
+    ##
+    # Returns a string representation of the token stream
 
-  ##
-  # Returns a string representation of the token stream
+    def tokens_to_s
+      (token_stream or return '').compact.map { |token| token[:text] }.join ''
+    end
 
-  def tokens_to_s
-    (token_stream or return '').compact.map { |token| token[:text] }.join ''
-  end
+    ##
+    # Returns the source language of the token stream as a string
+    #
+    # Returns 'c' or 'ruby'
 
-  ##
-  # Returns the source language of the token stream as a string
-  #
-  # Returns 'c' or 'ruby'
+    def source_language
+      @token_stream_language == :c ? 'c' : 'ruby'
+    end
 
-  def source_language
-    @token_stream_language == :c ? 'c' : 'ruby'
   end
-
 end

data/lib/rdoc/tom_doc.rb

@@ -1,257 +1,259 @@
 # frozen_string_literal: true
 # :markup: tomdoc
 
-# A parser for TomDoc based on TomDoc 1.0.0-rc1 (02adef9b5a)
-#
-# The TomDoc specification can be found at http://tomdoc.org.
-#
-# To choose TomDoc as your only default format see RDoc::Options@Saved+Options
-# for instructions on setting up a <code>.rdoc_options</code> file to store
-# your project default.
-#
-# There are a few differences between this parser and the specification.  A
-# best-effort was made to follow the specification as closely as possible but
-# some choices to deviate were made.
-#
-# A future version of RDoc will warn when a MUST or MUST NOT is violated and
-# may warn when a SHOULD or SHOULD NOT is violated.  RDoc will always try
-# to emit documentation even if given invalid TomDoc.
-#
-# Here are some implementation choices this parser currently makes:
-#
-# This parser allows rdoc-style inline markup but you should not depended on
-# it.
-#
-# This parser allows a space between the comment and the method body.
-#
-# This parser does not require the default value to be described for an
-# optional argument.
-#
-# This parser does not examine the order of sections.  An Examples section may
-# precede the Arguments section.
-#
-# This class is documented in TomDoc format.  Since this is a subclass of the
-# RDoc markup parser there isn't much to see here, unfortunately.
-
-class RDoc::TomDoc < RDoc::Markup::Parser
-
-  # Internal: Token accessor
-
-  attr_reader :tokens
-
-  # Internal: Adds a post-processor which sets the RDoc section based on the
-  # comment's status.
+module RDoc
+  # A parser for TomDoc based on TomDoc 1.0.0-rc1 (02adef9b5a)
   #
-  # Returns nothing.
-
-  def self.add_post_processor # :nodoc:
-    RDoc::Markup::PreProcess.post_process do |comment, code_object|
-      next unless code_object and
-                  RDoc::Comment === comment and comment.format == 'tomdoc'
-
-      comment.text.gsub!(/\A(\s*# |)(Public|Internal|Deprecated):\s+/) do
-        section = code_object.add_section $2
-        code_object.temporary_section = section
-
-        $1
-      end
-    end
-  end
-
-  add_post_processor
-
-  # Public: Parses TomDoc from text
+  # The TomDoc specification can be found at http://tomdoc.org.
   #
-  # text - A String containing TomDoc-format text.
+  # To choose TomDoc as your only default format see RDoc::Options@Saved+Options
+  # for instructions on setting up a <code>.rdoc_options</code> file to store
+  # your project default.
   #
-  # Examples
+  # There are a few differences between this parser and the specification.  A
+  # best-effort was made to follow the specification as closely as possible but
+  # some choices to deviate were made.
   #
-  #   RDoc::TomDoc.parse <<-TOMDOC
-  #   This method does some things
+  # A future version of RDoc will warn when a MUST or MUST NOT is violated and
+  # may warn when a SHOULD or SHOULD NOT is violated.  RDoc will always try
+  # to emit documentation even if given invalid TomDoc.
   #
-  #   Returns nothing.
-  #   TOMDOC
-  #   # => #<RDoc::Markup::Document:0xXXX @parts=[...], @file=nil>
+  # Here are some implementation choices this parser currently makes:
   #
-  # Returns an RDoc::Markup::Document representing the TomDoc format.
-
-  def self.parse(text)
-    parser = new
-
-    parser.tokenize text
-    doc = RDoc::Markup::Document.new
-    parser.parse doc
-    doc
-  end
-
-  # Internal: Extracts the Signature section's method signature
+  # This parser allows rdoc-style inline markup but you should not depended on
+  # it.
   #
-  # comment - An RDoc::Comment that will be parsed and have the signature
-  #           extracted
+  # This parser allows a space between the comment and the method body.
   #
-  # Returns a String containing the signature and nil if not
-
-  def self.signature(comment)
-    return unless comment.tomdoc?
+  # This parser does not require the default value to be described for an
+  # optional argument.
+  #
+  # This parser does not examine the order of sections.  An Examples section may
+  # precede the Arguments section.
+  #
+  # This class is documented in TomDoc format.  Since this is a subclass of the
+  # RDoc markup parser there isn't much to see here, unfortunately.
 
-    document = comment.parse
+  class TomDoc < Markup::Parser
 
-    signature = nil
-    found_heading = false
-    found_signature = false
+    # Internal: Token accessor
 
-    document.parts.delete_if do |part|
-      next false if found_signature
+    attr_reader :tokens
 
-      found_heading ||=
-        RDoc::Markup::Heading === part && part.text == 'Signature'
+    # Internal: Adds a post-processor which sets the RDoc section based on the
+    # comment's status.
+    #
+    # Returns nothing.
 
-      next false unless found_heading
+    def self.add_post_processor # :nodoc:
+      Markup::PreProcess.post_process do |comment, code_object|
+        next unless code_object and
+                    Comment === comment and comment.format == 'tomdoc'
 
-      next true if RDoc::Markup::BlankLine === part
+        comment.text.gsub!(/\A(\s*# |)(Public|Internal|Deprecated):\s+/) do
+          section = code_object.add_section $2
+          code_object.temporary_section = section
 
-      if RDoc::Markup::Verbatim === part then
-        signature = part
-        found_signature = true
+          $1
+        end
       end
     end
 
-    signature and signature.text
-  end
+    add_post_processor
+
+    # Public: Parses TomDoc from text
+    #
+    # text - A String containing TomDoc-format text.
+    #
+    # Examples
+    #
+    #   RDoc::TomDoc.parse <<-TOMDOC
+    #   This method does some things
+    #
+    #   Returns nothing.
+    #   TOMDOC
+    #   # => #<RDoc::Markup::Document:0xXXX @parts=[...], @file=nil>
+    #
+    # Returns an RDoc::Markup::Document representing the TomDoc format.
+
+    def self.parse(text)
+      parser = new
+
+      parser.tokenize text
+      doc = Markup::Document.new
+      parser.parse doc
+      doc
+    end
 
-  # Public: Creates a new TomDoc parser.  See also RDoc::Markup::parse
+    # Internal: Extracts the Signature section's method signature
+    #
+    # comment - An RDoc::Comment that will be parsed and have the signature
+    #           extracted
+    #
+    # Returns a String containing the signature and nil if not
 
-  def initialize
-    super
+    def self.signature(comment)
+      return unless comment.tomdoc?
 
-    @section      = nil
-    @seen_returns = false
-  end
+      document = comment.parse
 
-  # Internal: Builds a heading from the token stream
-  #
-  # level - The level of heading to create
-  #
-  # Returns an RDoc::Markup::Heading
+      signature = nil
+      found_heading = false
+      found_signature = false
 
-  def build_heading(level)
-    heading = super
+      document.parts.delete_if do |part|
+        next false if found_signature
 
-    @section = heading.text
+        found_heading ||=
+          Markup::Heading === part && part.text == 'Signature'
 
-    heading
-  end
+        next false unless found_heading
 
-  # Internal: Builds a verbatim from the token stream.  A verbatim in the
-  # Examples section will be marked as in Ruby format.
-  #
-  # margin - The indentation from the margin for lines that belong to this
-  #          verbatim section.
-  #
-  # Returns an RDoc::Markup::Verbatim
+        next true if Markup::BlankLine === part
+
+        if Markup::Verbatim === part
+          signature = part
+          found_signature = true
+        end
+      end
 
-  def build_verbatim(margin)
-    verbatim = super
+      signature and signature.text
+    end
 
-    verbatim.format = :ruby if @section == 'Examples'
+    # Public: Creates a new TomDoc parser.  See also RDoc::Markup::parse
 
-    verbatim
-  end
+    def initialize
+      super
 
-  # Internal: Builds a paragraph from the token stream
-  #
-  # margin - Unused
-  #
-  # Returns an RDoc::Markup::Paragraph.
+      @section      = nil
+      @seen_returns = false
+    end
 
-  def build_paragraph(margin)
-    p :paragraph_start => margin if @debug
+    # Internal: Builds a heading from the token stream
+    #
+    # level - The level of heading to create
+    #
+    # Returns an RDoc::Markup::Heading
 
-    paragraph = RDoc::Markup::Paragraph.new
+    def build_heading(level)
+      heading = super
 
-    until @tokens.empty? do
-      type, data, = get
+      @section = heading.text
 
-      case type
-      when :TEXT then
-        @section = 'Returns' if data =~ /\A(Returns|Raises)/
+      heading
+    end
 
-        paragraph << data
-      when :NEWLINE then
-        if :TEXT == peek_token[0] then
-          # Lines beginning with 'Raises' in the Returns section should not be
-          # treated as multiline text
-          if 'Returns' == @section and
-            peek_token[1].start_with?('Raises') then
-            break
+    # Internal: Builds a verbatim from the token stream.  A verbatim in the
+    # Examples section will be marked as in Ruby format.
+    #
+    # margin - The indentation from the margin for lines that belong to this
+    #          verbatim section.
+    #
+    # Returns an RDoc::Markup::Verbatim
+
+    def build_verbatim(margin)
+      verbatim = super
+
+      verbatim.format = :ruby if @section == 'Examples'
+
+      verbatim
+    end
+
+    # Internal: Builds a paragraph from the token stream
+    #
+    # margin - Unused
+    #
+    # Returns an RDoc::Markup::Paragraph.
+
+    def build_paragraph(margin)
+      p :paragraph_start => margin if @debug
+
+      paragraph = Markup::Paragraph.new
+
+      until @tokens.empty? do
+        type, data, = get
+
+        case type
+        when :TEXT
+          @section = 'Returns' if data =~ /\A(Returns|Raises)/
+
+          paragraph << data
+        when :NEWLINE
+          if :TEXT == peek_token[0]
+            # Lines beginning with 'Raises' in the Returns section should not be
+            # treated as multiline text
+            if 'Returns' == @section and
+              peek_token[1].start_with?('Raises')
+              break
+            else
+              paragraph << ' '
+            end
           else
-            paragraph << ' '
+            break
           end
         else
+          unget
           break
         end
-      else
-        unget
-        break
       end
-    end
 
-    p :paragraph_end => margin if @debug
+      p :paragraph_end => margin if @debug
 
-    paragraph
-  end
+      paragraph
+    end
+
+    ##
+    # Detects a section change to "Returns" and adds a heading
 
-  ##
-  # Detects a section change to "Returns" and adds a heading
+    def parse_text(parent, indent) # :nodoc:
+      paragraph = build_paragraph indent
 
-  def parse_text(parent, indent) # :nodoc:
-    paragraph = build_paragraph indent
+      if false == @seen_returns and 'Returns' == @section
+        @seen_returns = true
+        parent << Markup::Heading.new(3, 'Returns')
+        parent << Markup::BlankLine.new
+      end
 
-    if false == @seen_returns and 'Returns' == @section then
-      @seen_returns = true
-      parent << RDoc::Markup::Heading.new(3, 'Returns')
-      parent << RDoc::Markup::BlankLine.new
+      parent << paragraph
     end
 
-    parent << paragraph
-  end
+    # Internal: Turns text into an Array of tokens
+    #
+    # text - A String containing TomDoc-format text.
+    #
+    # Returns self.
+
+    def tokenize(text)
+      text = text.sub(/\A(Public|Internal|Deprecated):\s+/, '')
+
+      setup_scanner text
+
+      until @s.eos? do
+        pos = @s.pos
+
+        # leading spaces will be reflected by the column of the next token
+        # the only thing we loose are trailing spaces at the end of the file
+        next if @s.scan(/ +/)
+
+        @tokens << case
+                   when @s.scan(/\r?\n/)
+                     token = [:NEWLINE, @s.matched, *pos]
+                     @s.newline!
+                     token
+                   when @s.scan(/(Examples|Signature)$/)
+                     @tokens << [:HEADER, 3, *pos]
+
+                     [:TEXT, @s[1], *pos]
+                   when @s.scan(/([:\w][\w\[\]]*)[ ]+- /)
+                     [:NOTE, @s[1], *pos]
+                   else
+                     @s.scan(/.*/)
+                     [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
+                   end
+      end
 
-  # Internal: Turns text into an Array of tokens
-  #
-  # text - A String containing TomDoc-format text.
-  #
-  # Returns self.
-
-  def tokenize(text)
-    text = text.sub(/\A(Public|Internal|Deprecated):\s+/, '')
-
-    setup_scanner text
-
-    until @s.eos? do
-      pos = @s.pos
-
-      # leading spaces will be reflected by the column of the next token
-      # the only thing we loose are trailing spaces at the end of the file
-      next if @s.scan(/ +/)
-
-      @tokens << case
-                 when @s.scan(/\r?\n/) then
-                   token = [:NEWLINE, @s.matched, *pos]
-                   @s.newline!
-                   token
-                 when @s.scan(/(Examples|Signature)$/) then
-                   @tokens << [:HEADER, 3, *pos]
-
-                   [:TEXT, @s[1], *pos]
-                 when @s.scan(/([:\w][\w\[\]]*)[ ]+- /) then
-                   [:NOTE, @s[1], *pos]
-                 else
-                   @s.scan(/.*/)
-                   [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
-                 end
+      self
     end
 
-    self
   end
-
 end