rdoc 7.1.0 → 8.0.0

data/lib/rdoc/markup/document.rb

@@ -26,7 +26,7 @@ class RDoc::Markup::Document
   ##
   # Creates a new Document with +parts+
 
-  def initialize *parts
+  def initialize(*parts)
     @parts = []
     @parts.concat parts
 
@@ -148,7 +148,7 @@ class RDoc::Markup::Document
   ##
   # Appends +parts+ to the document
 
-  def push *parts
+  def push(*parts)
     self.parts.concat parts
   end
 

data/lib/rdoc/markup/formatter.rb

@@ -10,13 +10,9 @@
 # RDoc::Markup::FormatterTestCase.  If you're writing a text-output formatter
 # use RDoc::Markup::TextFormatterTestCase which provides extra test cases.
 
-class RDoc::Markup::Formatter
-
-  ##
-  # Tag for inline markup containing a +bit+ for the bitmask and the +on+ and
-  # +off+ triggers.
+require 'rdoc/markup/inline_parser'
 
-  InlineTag = Struct.new(:bit, :on, :off)
+class RDoc::Markup::Formatter
 
   ##
   # Converts a target url to one that is relative to a given path
@@ -45,21 +41,9 @@ class RDoc::Markup::Formatter
   ##
   # Creates a new Formatter
 
-  def initialize(options, markup = nil)
-    @options = options
-
-    @markup = markup || RDoc::Markup.new
-    @am     = @markup.attribute_manager
-    @am.add_regexp_handling(/<br>/, :HARD_BREAK)
+  def initialize
+    @markup = RDoc::Markup.new
 
-    @attributes = @am.attributes
-
-    @attr_tags = []
-
-    @in_tt = 0
-    @tt_bit = @attributes.bitmap_for :TT
-
-    @hard_break = ''
     @from_path = '.'
   end
 
@@ -84,29 +68,6 @@ class RDoc::Markup::Formatter
     @markup.add_regexp_handling(/rdoc-[a-z]+:[^\s\]]+/, :RDOCLINK)
   end
 
-  ##
-  # Adds a regexp handling for links of the form {<text>}[<url>] and
-  # <word>[<url>]
-
-  def add_regexp_handling_TIDYLINK
-    @markup.add_regexp_handling(/(?:
-                                  \{[^{}]*\} | # multi-word label
-                                  \b[^\s{}]+? # single-word label
-                                 )
-
-                                 \[\S+?\]     # link target
-                                /x, :TIDYLINK)
-  end
-
-  ##
-  # Add a new set of tags for an attribute. We allow separate start and end
-  # tags for flexibility
-
-  def add_tag(name, start, stop)
-    attr = @attributes.bitmap_for name
-    @attr_tags << InlineTag.new(attr, start, stop)
-  end
-
   ##
   # Allows +tag+ to be decorated with additional information.
 
@@ -121,117 +82,169 @@ class RDoc::Markup::Formatter
     @markup.convert content, self
   end
 
-  ##
-  # Converts flow items +flow+
-
-  def convert_flow(flow)
-    res = []
+  # Applies regexp handling to +text+ and returns an array of [text, converted?] pairs.
 
-    flow.each do |item|
-      case item
-      when String then
-        res << convert_string(item)
-      when RDoc::Markup::AttrChanger then
-        off_tags res, item
-        on_tags res, item
-      when RDoc::Markup::RegexpHandling then
-        res << convert_regexp_handling(item)
-      else
-        raise "Unknown flow element: #{item.inspect}"
+  def apply_regexp_handling(text)
+    matched = []
+    @markup.regexp_handlings.each_with_index do |(pattern, name), priority|
+      text.scan(pattern) do
+        m = Regexp.last_match
+        idx = m[1] ? 1 : 0
+        matched << [m.begin(idx), m.end(idx), m[idx], name, priority]
       end
     end
+    # If the start positions are the same, prefer the earlier-registered one
+    # (lower numeric priority from each_with_index).
+    matched.sort_by! {|beg_pos, _, _, _, priority| [beg_pos, priority] }
+
+    pos = 0
+    output = []
+    matched.each do |beg_pos, end_pos, s, name|
+      next if beg_pos < pos
+
+      output << [text[pos...beg_pos], false] if beg_pos != pos
+      handled = public_send(:"handle_regexp_#{name}", s)
+      output << [handled, true]
+      pos = end_pos
+    end
 
-    res.join
+    output << [text[pos..], false] if pos < text.size
+    output
   end
 
-  ##
-  # Converts added regexp handlings. See RDoc::Markup#add_regexp_handling
+  # Called when processing plain text while traversing inline nodes from handle_inline.
+  # +text+ may need proper escaping.
+
+  def handle_PLAIN_TEXT(text)
+  end
 
-  def convert_regexp_handling(target)
-    return target.text if in_tt?
+  # Called when processing regexp-handling-processed text while traversing inline nodes from handle_inline.
+  # +text+ may contain markup tags.
 
-    handled = false
+  def handle_REGEXP_HANDLING_TEXT(text)
+  end
 
-    @attributes.each_name_of target.type do |name|
-      method_name = "handle_regexp_#{name}"
+  # Called when processing text node while traversing inline nodes from handle_inline.
+  # Apply regexp handling and dispatch to the appropriate handler: handle_REGEXP_HANDLING_TEXT or handle_PLAIN_TEXT.
 
-      if respond_to? method_name then
-        target.text = public_send method_name, target
-        handled = true
+  def handle_TEXT(text)
+    apply_regexp_handling(text).each do |part, converted|
+      if converted
+        handle_REGEXP_HANDLING_TEXT(part)
+      else
+        handle_PLAIN_TEXT(part)
       end
     end
+  end
 
-    unless handled then
-      target_name = @attributes.as_string target.type
+  # Called when processing a hard break while traversing inline nodes from handle_inline.
 
-      raise RDoc::Error, "Unhandled regexp handling #{target_name}: #{target}"
-    end
+  def handle_HARD_BREAK
+  end
 
-    target.text
+  # Called when processing bold nodes while traversing inline nodes from handle_inline.
+  # Traverse the children nodes and dispatch to the appropriate handlers.
+
+  def handle_BOLD(nodes)
+    traverse_inline_nodes(nodes)
   end
 
-  ##
-  # Converts a string to be fancier if desired
+  # Called when processing emphasis nodes while traversing inline nodes from handle_inline.
+  # Traverse the children nodes and dispatch to the appropriate handlers.
 
-  def convert_string(string)
-    string
+  def handle_EM(nodes)
+    traverse_inline_nodes(nodes)
   end
 
-  ##
-  # Use ignore in your subclass to ignore the content of a node.
-  #
-  #   ##
-  #   # We don't support raw nodes in ToNoRaw
-  #
-  #   alias accept_raw ignore
+  # Called when processing bold word nodes while traversing inline nodes from handle_inline.
+  # +word+ may need proper escaping.
 
-  def ignore *node
+  def handle_BOLD_WORD(word)
+    handle_PLAIN_TEXT(word)
   end
 
-  ##
-  # Are we currently inside tt tags?
+  # Called when processing emphasis word nodes while traversing inline nodes from handle_inline.
+  # +word+ may need proper escaping.
 
-  def in_tt?
-    @in_tt > 0
+  def handle_EM_WORD(word)
+    handle_PLAIN_TEXT(word)
   end
 
-  def tt_tag?(attr_mask, reverse = false)
-    each_attr_tag(attr_mask, reverse) do |tag|
-      return true if tt? tag
-    end
-    false
+  # Called when processing tt nodes while traversing inline nodes from handle_inline.
+  # +code+ may need proper escaping.
+
+  def handle_TT(code)
+    handle_PLAIN_TEXT(code)
   end
 
-  ##
-  # Turns on tags for +item+ on +res+
+  # Called when processing strike nodes while traversing inline nodes from handle_inline.
+  # Traverse the children nodes and dispatch to the appropriate handlers.
 
-  def on_tags(res, item)
-    each_attr_tag(item.turn_on) do |tag|
-      res << annotate(tag.on)
-      @in_tt += 1 if tt? tag
-    end
+  def handle_STRIKE(nodes)
+    traverse_inline_nodes(nodes)
   end
 
-  ##
-  # Turns off tags for +item+ on +res+
+  # Called when processing tidylink nodes while traversing inline nodes from handle_inline.
+  # +label_part+ is an array of strings or nodes representing the link label.
+  # +url+ is the link URL.
+  # Traverse the label_part nodes and dispatch to the appropriate handlers.
 
-  def off_tags(res, item)
-    each_attr_tag(item.turn_off, true) do |tag|
-      @in_tt -= 1 if tt? tag
-      res << annotate(tag.off)
-    end
+  def handle_TIDYLINK(label_part, url)
+    traverse_inline_nodes(label_part)
   end
 
-  def each_attr_tag(attr_mask, reverse = false)
-    return if attr_mask.zero?
+  # Parses inline +text+, traverse the resulting nodes, and calls the appropriate handler methods.
+
+  def handle_inline(text)
+    nodes = RDoc::Markup::InlineParser.new(text).parse
+    traverse_inline_nodes(nodes)
+  end
 
-    @attr_tags.public_send(reverse ? :reverse_each : :each) do |tag|
-      if attr_mask & tag.bit != 0 then
-        yield tag
+  # Traverses +nodes+ and calls the appropriate handler methods
+  # Nodes formats are described in RDoc::Markup::InlineParser#parse
+
+  def traverse_inline_nodes(nodes)
+    nodes.each do |node|
+      next handle_TEXT(node) if String === node
+      case node[:type]
+      when :TIDYLINK
+        handle_TIDYLINK(node[:children], node[:url])
+      when :HARD_BREAK
+        handle_HARD_BREAK
+      when :BOLD
+        handle_BOLD(node[:children])
+      when :BOLD_WORD
+        handle_BOLD_WORD(node[:children][0] || '')
+      when :EM
+        handle_EM(node[:children])
+      when :EM_WORD
+        handle_EM_WORD(node[:children][0] || '')
+      when :TT
+        handle_TT(node[:children][0] || '')
+      when :STRIKE
+        handle_STRIKE(node[:children])
       end
     end
   end
 
+  ##
+  # Converts a string to be fancier if desired
+
+  def convert_string(string)
+    string
+  end
+
+  ##
+  # Use ignore in your subclass to ignore the content of a node.
+  #
+  #   ##
+  #   # We don't support raw nodes in ToNoRaw
+  #
+  #   alias accept_raw ignore
+
+  def ignore(*node)
+  end
+
   ##
   # Extracts and a scheme, url and an anchor id from +url+ and returns them.
 

data/lib/rdoc/markup/heading.rb

@@ -52,13 +52,10 @@ module RDoc
       #: () -> RDoc::Markup::ToHtml
       def self.to_html
         @to_html ||= begin
-          markup = Markup.new
-          markup.add_regexp_handling CrossReference::CROSSREF_REGEXP, :CROSSREF
+          to_html = Markup::ToHtml.new
 
-          to_html = Markup::ToHtml.new nil
-
-          def to_html.handle_regexp_CROSSREF(target)
-            target.text.sub(/^\\/, '')
+          def to_html.handle_regexp_CROSSREF(text)
+            text.sub(/^\\/, '')
           end
 
           to_html

data/lib/rdoc/markup/indented_paragraph.rb

@@ -13,7 +13,7 @@ class RDoc::Markup::IndentedParagraph < RDoc::Markup::Raw
   # Creates a new IndentedParagraph containing +parts+ indented with +indent+
   # spaces
 
-  def initialize indent, *parts
+  def initialize(indent, *parts)
     @indent = indent
 
     super(*parts)

data/lib/rdoc/markup/inline_parser.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'strscan'
+
+# Parses inline markup in RDoc text.
+# This parser handles em, bold, strike, tt, hard break, and tidylink.
+# Block-level constructs are handled in RDoc::Markup::Parser.
+
+class RDoc::Markup::InlineParser
+
+  # TT, BOLD_WORD, EM_WORD: regexp-handling(example: crossref) is disabled
+  WORD_PAIRS = {
+    '*' => :BOLD_WORD,
+    '**' => :BOLD_WORD,
+    '_' => :EM_WORD,
+    '__' => :EM_WORD,
+    '+' => :TT,
+    '++' => :TT,
+    '`' => :TT,
+    '``' => :TT
+  } # :nodoc:
+
+  # Other types: regexp-handling(example: crossref) is enabled
+  TAGS = {
+    'em' => :EM,
+    'i' => :EM,
+    'b' => :BOLD,
+    's' => :STRIKE,
+    'del' => :STRIKE,
+  } # :nodoc:
+
+  STANDALONE_TAGS = { 'br' => :HARD_BREAK } # :nodoc:
+
+  CODEBLOCK_TAGS = %w[tt code] # :nodoc:
+
+  TOKENS = {
+    **WORD_PAIRS.transform_values { [:word_pair, nil] },
+    **TAGS.keys.to_h {|tag| ["<#{tag}>", [:open_tag, tag]] },
+    **TAGS.keys.to_h {|tag| ["</#{tag}>", [:close_tag, tag]] },
+    **CODEBLOCK_TAGS.to_h {|tag| ["<#{tag}>", [:code_start, tag]] },
+    **STANDALONE_TAGS.keys.to_h {|tag| ["<#{tag}>", [:standalone_tag, tag]] },
+    '{' => [:tidylink_start, nil],
+    '}' => [:tidylink_mid, nil],
+    '\\' => [:escape, nil],
+    '[' => nil # To make `label[url]` scan as separate tokens
+  } # :nodoc:
+
+  multi_char_tokens_regexp = Regexp.union(TOKENS.keys.select {|s| s.size > 1 }).source
+  token_starts_regexp = TOKENS.keys.map {|s| s[0] }.uniq.map {|s| Regexp.escape(s) }.join
+
+  SCANNER_REGEXP =
+    /(?:
+      #{multi_char_tokens_regexp}
+      |[^#{token_starts_regexp}\sa-zA-Z0-9\.]+ # chunk of normal text
+      |\s+|[a-zA-Z0-9\.]+|.
+    )/x # :nodoc:
+
+  # Characters that can be escaped with backslash.
+  ESCAPING_CHARS = '\\*_+`{}[]<>' # :nodoc:
+
+  # Pattern to match code block content until <code></tt></code> or <tt></code></tt>.
+  CODEBLOCK_REGEXPS = CODEBLOCK_TAGS.to_h {|name| [name, /((?:\\.|[^\\])*?)<\/#{name}>/] } # :nodoc:
+
+  # Word contains alphanumeric and <tt>_./:[]-</tt> characters.
+  # Word may start with <tt>#</tt> and may end with any non-space character. (e.g. <tt>#eql?</tt>).
+  # Underscore delimiter have special rules.
+  WORD_REGEXPS = {
+    # Words including _, longest match.
+    # Example: `_::A_` `_-42_` `_A::B::C.foo_bar[baz]_` `_kwarg:_`
+    # Content must not include _ followed by non-alphanumeric character
+    # Example: `_host_:_port_` will be `_host_` + `:` + `_port_`
+    '_' => /#?([a-zA-Z0-9.\/:\[\]-]|_+[a-zA-Z0-9])+[^\s]?_(?=[^a-zA-Z0-9_]|\z)/,
+    # Words allowing _ but not allowing __
+    '__' => /#?[a-zA-Z0-9.\/:\[\]-]*(_[a-zA-Z0-9.\/:\[\]-]+)*[^\s]?__(?=[^a-zA-Z0-9]|\z)/,
+    **%w[* ** + ++ ` ``].to_h do |s|
+      # normal words that can be used within +word+ or *word*
+      [s, /#?[a-zA-Z0-9_.\/:\[\]-]+[^\s]?#{Regexp.escape(s)}(?=[^a-zA-Z0-9]|\z)/]
+    end
+  } # :nodoc:
+
+  def initialize(string)
+    @scanner = StringScanner.new(string)
+    @last_match = nil
+    @scanner_negative_cache = Set.new
+    @stack = []
+    @delimiters = {}
+  end
+
+  # Return the current parsing node on <tt>@stack</tt>.
+
+  def current
+    @stack.last
+  end
+
+  # Parse and return an array of nodes.
+  # Node format:
+  #   {
+  #     type: :EM | :BOLD | :BOLD_WORD | :EM_WORD | :TT | :STRIKE | :HARD_BREAK | :TIDYLINK,
+  #     url: string # only for :TIDYLINK
+  #     children: [string_or_node, ...]
+  #   }
+
+  def parse
+    stack_push(:root, nil)
+    while true
+      type, token, value = scan_token
+      close = nil
+      tidylink_url = nil
+      case type
+      when :node
+        current[:children] << value
+        invalidate_open_tidylinks if value[:type] == :TIDYLINK
+      when :eof
+        close = :root
+      when :tidylink_open
+        stack_push(:tidylink, token)
+      when :tidylink_close
+        close = :tidylink
+        if value
+          tidylink_url = value
+        else
+          # Tidylink closing brace without URL part. Treat opening and closing braces as normal text
+          # `{labelnodes}...` case.
+          current[:children] << token
+        end
+      when :invalidated_tidylink_close
+        # `{...{label}[url]...}` case. Nested tidylink invalidates outer one. The last `}` closes the invalidated tidylink.
+        current[:children] << token
+        close = :invalidated_tidylink
+      when :text
+        current[:children] << token
+      when :open
+        stack_push(value, token)
+      when :close
+        if @delimiters[value]
+          close = value
+        else
+          # closing tag without matching opening tag. Treat as normal text.
+          current[:children] << token
+        end
+      end
+
+      next unless close
+
+      while current[:delimiter] != close
+        children = current[:children]
+        open_token = current[:token]
+        stack_pop
+        current[:children] << open_token if open_token
+        current[:children].concat(children)
+      end
+
+      token = current[:token]
+      children = compact_string(current[:children])
+      stack_pop
+
+      return children if close == :root
+
+      if close == :tidylink || close == :invalidated_tidylink
+        if tidylink_url
+          current[:children] << { type: :TIDYLINK, children: children, url: tidylink_url }
+          invalidate_open_tidylinks
+        else
+          current[:children] << token
+          current[:children].concat(children)
+        end
+      else
+        current[:children] << { type: TAGS[close], children: children }
+      end
+    end
+  end
+
+  private
+
+  # When a valid tidylink node is encountered, invalidate all nested tidylinks.
+
+  def invalidate_open_tidylinks
+    return unless @delimiters[:tidylink]
+
+    @delimiters[:invalidated_tidylink] ||= []
+    @delimiters[:tidylink].each do |idx|
+      @delimiters[:invalidated_tidylink] << idx
+      @stack[idx][:delimiter] = :invalidated_tidylink
+    end
+    @delimiters.delete(:tidylink)
+  end
+
+  # Pop the top node off the stack when node is closed by a closing delimiter or an error.
+
+  def stack_pop
+    delimiter = current[:delimiter]
+    @delimiters[delimiter].pop
+    @delimiters.delete(delimiter) if @delimiters[delimiter].empty?
+    @stack.pop
+  end
+
+  # Push a new node onto the stack when encountering an opening delimiter.
+
+  def stack_push(delimiter, token)
+    node = { delimiter: delimiter, token: token, children: [] }
+    (@delimiters[delimiter] ||= []) << @stack.size
+    @stack << node
+  end
+
+  # Compacts adjacent strings in +nodes+ into a single string.
+
+  def compact_string(nodes)
+    nodes.chunk {|e| String === e }.flat_map do |is_str, elems|
+      is_str ? elems.join : elems
+    end
+  end
+
+  # Scan from StringScanner with +pattern+
+  # If +negative_cache+ is true, caches scan failure result. <tt>scan(pattern, negative_cache: true)</tt> return nil when it is called again after a failure.
+  # Be careful to use +negative_cache+ with a pattern and position that does not match after previous failure.
+
+  def strscan(pattern, negative_cache: false)
+    return if negative_cache && @scanner_negative_cache.include?(pattern)
+
+    string = @scanner.scan(pattern)
+    @last_match = string if string
+    @scanner_negative_cache << pattern if !string && negative_cache
+    string
+  end
+
+  # Scan and return the next token for parsing.
+  # Returns <tt>[token_type, token_string_or_nil, extra_info]</tt>
+
+  def scan_token
+    last_match = @last_match
+    token = strscan(SCANNER_REGEXP)
+    type, name = TOKENS[token]
+
+    case type
+    when :word_pair
+      # If the character before word pair delimiter is alphanumeric, do not treat as word pair.
+      word_pair = strscan(WORD_REGEXPS[token]) unless /[a-zA-Z0-9]\z/.match?(last_match)
+
+      if word_pair.nil?
+        [:text, token, nil]
+      elsif token == '__' && word_pair.match?(/\A[a-zA-Z]+__\z/)
+        # Special exception: __FILE__, __LINE__, __send__ should be treated as normal text.
+        [:text, "#{token}#{word_pair}", nil]
+      else
+        [:node, nil, { type: WORD_PAIRS[token], children: [word_pair.delete_suffix(token)] }]
+      end
+    when :open_tag
+      [:open, token, name]
+    when :close_tag
+      [:close, token, name]
+    when :code_start
+      if (codeblock = strscan(CODEBLOCK_REGEXPS[name], negative_cache: true))
+        # Need to unescape `\\` and `\<`.
+        # RDoc also unescapes backslash + word separators, but this is not really necessary.
+        content = codeblock.delete_suffix("</#{name}>").gsub(/\\(.)/) { '\\<*+_`'.include?($1) ? $1 : $& }
+        [:node, nil, { type: :TT, children: content.empty? ? [] : [content] }]
+      else
+        [:text, token, nil]
+      end
+    when :standalone_tag
+      [:node, nil, { type: STANDALONE_TAGS[name], children: [] }]
+    when :tidylink_start
+      [:tidylink_open, token, nil]
+    when :tidylink_mid
+      if @delimiters[:tidylink]
+        if (url = read_tidylink_url)
+          [:tidylink_close, nil, url]
+        else
+          [:tidylink_close, token, nil]
+        end
+      elsif @delimiters[:invalidated_tidylink]
+        [:invalidated_tidylink_close, token, nil]
+      else
+        [:text, token, nil]
+      end
+    when :escape
+      next_char = strscan(/./)
+      if next_char.nil?
+        # backslash at end of string
+        [:text, '\\', nil]
+      elsif next_char && ESCAPING_CHARS.include?(next_char)
+        # escaped character
+        [:text, next_char, nil]
+      else
+        # If next_char not an escaping character, it is treated as text token with backslash + next_char
+        # For example, backslash of `\Ruby` (suppressed crossref) remains.
+        [:text, "\\#{next_char}", nil]
+      end
+    else
+      if token.nil?
+        [:eof, nil, nil]
+      elsif token.match?(/\A[A-Za-z0-9]*\z/) && (url = read_tidylink_url)
+        # Simplified tidylink: label[url]
+        [:node, nil, { type: :TIDYLINK, children: [token], url: url }]
+      else
+        [:text, token, nil]
+      end
+    end
+  end
+
+  # Read the URL part of a tidylink from the current position.
+  # Returns nil if no valid URL part is found.
+  # URL part is enclosed in square brackets and may contain escaped brackets.
+  # Example: <tt>[http://example.com/?q=\[\]]</tt> represents <tt>http://example.com/?q=[]</tt>.
+  # If we're accepting rdoc-style links in markdown, url may include <tt>*+<_</tt> with backslash escape.
+
+  def read_tidylink_url
+    bracketed_url = strscan(/\[([^\s\[\]\\]|\\[\[\]\\*+<_])+\]/)
+    bracketed_url[1...-1].gsub(/\\(.)/, '\1') if bracketed_url
+  end
+end

data/lib/rdoc/markup/list.rb

@@ -37,7 +37,7 @@ class RDoc::Markup::List
   # Creates a new list of +type+ with +items+.  Valid list types are:
   # +:BULLET+, +:LABEL+, +:LALPHA+, +:NOTE+, +:NUMBER+, +:UALPHA+
 
-  def initialize type = nil, *items
+  def initialize(type = nil, *items)
     @type = type
     @items = []
     @items.concat items
@@ -94,7 +94,7 @@ class RDoc::Markup::List
   ##
   # Appends +items+ to the list
 
-  def push *items
+  def push(*items)
     @items.concat items
   end