rdoc 7.1.0 → 8.0.0

data/lib/rdoc/markup/to_html.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'cgi/escape'
 require 'cgi/util' unless defined?(CGI::EscapeExt)
+require 'prism'
 
 ##
 # Outputs RDoc markup as HTML.
@@ -40,22 +41,104 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
   # :section:
 
+  # Maps an encoding to a Hash of characters properly transcoded for that
+  # encoding.
+  #
+  # See also encode_fallback.
+
+  TO_HTML_CHARACTERS = Hash.new do |h, encoding|
+    h[encoding] = {
+      close_dquote: encode_fallback('”', encoding, '"'),
+      close_squote: encode_fallback('’', encoding, '\''),
+      copyright:    encode_fallback('©', encoding, '(c)'),
+      ellipsis:     encode_fallback('…', encoding, '...'),
+      dot_ellipsis: encode_fallback('.…', encoding, '....'),
+      em_dash:      encode_fallback('—', encoding, '---'),
+      en_dash:      encode_fallback('–', encoding, '--'),
+      open_dquote:  encode_fallback('“', encoding, '"'),
+      open_squote:  encode_fallback('‘', encoding, '\''),
+      trademark:    encode_fallback('®', encoding, '(r)'),
+    }
+  end
+
+  HTML_CHARACTER_ALIASES = {
+    '(c)' => :copyright,
+    '(C)' => :copyright,
+    '(r)' => :trademark,
+    '(R)' => :trademark,
+    '---' => :em_dash,
+    '--'  => :en_dash,
+    '....' => :dot_ellipsis,
+    '...' => :ellipsis,
+    '``' => :open_dquote,
+    "''" => :close_dquote,
+  }
+
+  # Transcodes +character+ to +encoding+ with a +fallback+ character.
+
+  def self.encode_fallback(character, encoding, fallback)
+    character.encode(
+      encoding,
+      fallback: { character => fallback },
+      undef: :replace,
+      replace: fallback
+    )
+  end
+
+  # Converts ascii quote pairs to multibyte quote characters
+  class QuoteConverter
+
+    def initialize
+      @in_dquote = false
+      @in_squote = false
+    end
+
+    def convert(quote, after_word:)
+      case quote
+      when '"'
+        type = @in_dquote ? :close_dquote : :open_dquote
+        @in_dquote = !@in_dquote
+      when "'"
+        if @in_squote
+          type = :close_squote
+          @in_squote = false
+        elsif after_word
+          # Mary's dog, my parents' house: do not start paired quotes
+          type = :close_squote
+        else
+          type = :open_squote
+          @in_squote = true
+        end
+      when '`'
+        # Opening quote of <tt>`quoted sentence'</tt>.
+        # This will conflict with code blocks <tt>`puts('hello')`</tt> in the future.
+        if !@in_squote && !after_word
+          type = :open_squote
+          @in_squote = true
+        end
+      end
+      TO_HTML_CHARACTERS[quote.encoding][type] if type
+    end
+  end
+
   ##
   # Creates a new formatter that will output HTML
 
-  def initialize(options, markup = nil)
-    super
+  def initialize(pipe: false, output_decoration: true)
+    super()
 
+    @pipe = pipe
+    @output_decoration = output_decoration
     @code_object = nil
     @from_path = ''
     @in_list_entry = nil
     @list = nil
     @th = nil
+    @quote_converter = nil
+    @in_tidylink_label = false
     @hard_break = "<br>\n"
 
     init_regexp_handlings
-
-    init_tags
   end
 
   # :section: Regexp Handling
@@ -72,6 +155,15 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     # external links
     @markup.add_regexp_handling(/(?:link:|https?:|mailto:|ftp:|irc:|www\.)#{URL_CHARACTERS_REGEXP_STR}+\w/,
                                 :HYPERLINK)
+
+    # suppress crossref: \#method \::method \ClassName \method_with_underscores
+    @markup.add_regexp_handling(/\\(?:[#:A-Z]|[a-z]+_[a-z0-9])/, :SUPPRESSED_CROSSREF)
+
+    @markup.add_regexp_handling(Regexp.union(HTML_CHARACTER_ALIASES.keys), :HTML_CHARACTERS)
+
+    @markup.add_regexp_handling(/\b['"`]/, :QUOTE_AFTER_WORD)
+    @markup.add_regexp_handling(/\B['"`]/, :QUOTE_NOT_AFTER_WORD)
+
     init_link_notation_regexp_handlings
   end
 
@@ -80,7 +172,6 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
   def init_link_notation_regexp_handlings
     add_regexp_handling_RDOCLINK
-    add_regexp_handling_TIDYLINK
   end
 
   def handle_RDOCLINK(url) # :nodoc:
@@ -88,6 +179,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     when /^rdoc-ref:/
       CGI.escapeHTML($')
     when /^rdoc-label:/
+      return CGI.escapeHTML(url) if in_tidylink_label?
       text = $'
 
       text = case text
@@ -113,11 +205,143 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     end
   end
 
-  ##
-  # +target+ is a <code><br></code>
+  def handle_PLAIN_TEXT(text)
+    emit_inline(convert_string(text))
+  end
+
+  def handle_REGEXP_HANDLING_TEXT(text)
+    emit_inline(text)
+  end
+
+  def handle_BOLD(nodes)
+    emit_inline('<strong>')
+    super
+    emit_inline('</strong>')
+  end
+
+  def handle_EM(nodes)
+    emit_inline('<em>')
+    super
+    emit_inline('</em>')
+  end
+
+  def handle_BOLD_WORD(word)
+    emit_inline('<strong>')
+    super
+    emit_inline('</strong>')
+  end
+
+  def handle_EM_WORD(word)
+    emit_inline('<em>')
+    super
+    emit_inline('</em>')
+  end
 
-  def handle_regexp_HARD_BREAK(target)
-    '<br>'
+  def handle_TT(code)
+    emit_inline('<code>')
+    super
+    emit_inline('</code>')
+  end
+
+  def handle_STRIKE(nodes)
+    emit_inline('<del>')
+    super
+    emit_inline('</del>')
+  end
+
+  def handle_HARD_BREAK
+    emit_inline('<br>')
+  end
+
+  def emit_inline(text)
+    @inline_output << text
+  end
+
+  # Returns true if we are processing inside a tidy link label.
+
+  def in_tidylink_label?
+    @in_tidylink_label
+  end
+
+  # Special handling for tidy link labels.
+  # When a tidy link is <tt>{rdoc-image:path/to/image.jpg:alt text}[http://example.com]</tt>,
+  # label part is normally considered RDOCLINK <tt>rdoc-image:path/to/image.jpg:alt</tt> and a text <tt>" text"</tt>
+  # but RDoc's test code expects the whole label part to be treated as RDOCLINK only in tidy link label.
+  # When a tidy link is <tt>{^1}[url]</tt> or <tt>{*1}[url]</tt>, the label part needs to drop leading * or ^.
+  # TODO: reconsider this workaround.
+
+  def apply_tidylink_label_special_handling(label, url)
+    # ^1 *1 will be converted to just 1 in tidy link label.
+    return label[1..] if label.match?(/\A[*^]\d+\z/)
+
+    # rdoc-image in label specially allows spaces in alt text.
+    return handle_RDOCLINK(label) if label.start_with?('rdoc-image:')
+  end
+
+  def handle_TIDYLINK(label_part, url)
+    # When url is an image, ignore label part (maybe bug?) and just generate img tag.
+    if url.match?(/\Ahttps?:\/\/.+\.(png|gif|jpg|jpeg|bmp)\z/)
+      emit_inline("<img src=\"#{CGI.escapeHTML(url)}\" />")
+      return
+    elsif url.match?(/\Ardoc-image:/)
+      emit_inline(handle_RDOCLINK(url))
+      return
+    end
+
+    if label_part.size == 1 && String === label_part[0]
+      raw_label = label_part[0]
+
+      @in_tidylink_label = true
+      special = apply_tidylink_label_special_handling(raw_label, url)
+      @in_tidylink_label = false
+
+      if special
+        tag = gen_url(CGI.escapeHTML(url), special)
+        unless tag.empty?
+          emit_inline(tag)
+          return
+        end
+      end
+    end
+
+    tag = gen_url(CGI.escapeHTML(url), '')
+    open_tag, close_tag = tag.split(/(?=<\/a>)/, 2)
+    valid_tag = open_tag && close_tag
+    emit_inline(open_tag) if valid_tag
+    @in_tidylink_label = true
+    traverse_inline_nodes(label_part)
+    @in_tidylink_label = false
+    emit_inline(close_tag) if valid_tag
+  end
+
+  def handle_inline(text) # :nodoc:
+    @inline_output = +''
+    @quote_converter = QuoteConverter.new
+    super
+    out = @inline_output
+    @inline_output = nil
+    @quote_converter = nil
+    out
+  end
+
+  # Converts <tt>(c), (r), --, --- , ..., ...., ``, ''</tt> to HTML characters.
+  def handle_regexp_HTML_CHARACTERS(text)
+    name = HTML_CHARACTER_ALIASES[text]
+    TO_HTML_CHARACTERS[text.encoding][name] if name
+  end
+
+  def handle_regexp_QUOTE_NOT_AFTER_WORD(text)
+    @quote_converter.convert(text, after_word: false) || convert_string(text)
+  end
+
+  def handle_regexp_QUOTE_AFTER_WORD(text)
+    @quote_converter.convert(text, after_word: true) || convert_string(text)
+  end
+
+  # Converts suppressed cross-reference +text+ to HTML by removing the leading backslash.
+
+  def handle_regexp_SUPPRESSED_CROSSREF(text)
+    convert_string(text.delete_prefix('\\'))
   end
 
   ##
@@ -132,9 +356,10 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # <tt>link:</tt>::
   #   Reference to a local file relative to the output directory.
 
-  def handle_regexp_HYPERLINK(target)
-    url = CGI.escapeHTML(target.text)
+  def handle_regexp_HYPERLINK(text)
+    return convert_string(text) if in_tidylink_label?
 
+    url = CGI.escapeHTML(text)
     gen_url url, url
   end
 
@@ -147,27 +372,8 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # For the +rdoc-label+ scheme the footnote and label prefixes are stripped
   # when creating a link.  All other contents will be linked verbatim.
 
-  def handle_regexp_RDOCLINK(target)
-    handle_RDOCLINK target.text
-  end
-
-  ##
-  # This +target+ is a link where the label is different from the URL
-  # <tt>label[url]</tt> or <tt>{long label}[url]</tt>
-
-  def handle_regexp_TIDYLINK(target)
-    text = target.text
-
-    if tidy_link_capturing?
-      return finish_tidy_link(text)
-    end
-
-    if text.start_with?('{') && !text.include?('}')
-      start_tidy_link text
-      return ''
-    end
-
-    convert_complete_tidy_link(text)
+  def handle_regexp_RDOCLINK(text)
+    handle_RDOCLINK text
   end
 
   # :section: Visitor
@@ -181,6 +387,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     @res = []
     @in_list_entry = []
     @list = []
+    @heading_ids = {}
   end
 
   ##
@@ -209,13 +416,22 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   def accept_paragraph(paragraph)
     @res << "\n<p>"
     text = paragraph.text @hard_break
-    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
+    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K(?:\r?\n)+(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
       defined?($2) && ' '
     }
     @res << to_html(text)
     @res << "</p>\n"
   end
 
+  # Generate syntax highlighted html for ruby-like text.
+
+  def parsable_text_to_html(text)
+    tokens = RDoc::Parser::RubyColorizer.colorize(text)
+    result = RDoc::TokenStream.to_html tokens
+    result = result + "\n" unless result.end_with?("\n")
+    result
+  end
+
   ##
   # Adds +verbatim+ to the output
 
@@ -223,29 +439,19 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     text = verbatim.text.rstrip
     format = verbatim.format
 
-    klass = nil
-
     # Apply Ruby syntax highlighting if
     # - explicitly marked as Ruby (via ruby? which accepts :ruby or :rb)
     # - no format specified but the text is parseable as Ruby
     # Otherwise, add language class when applicable and skip Ruby highlighting
-    content = if verbatim.ruby? || (format.nil? && parseable?(text))
-                begin
-                  tokens = RDoc::Parser::RipperStateLex.parse text
-                  klass  = ' class="ruby"'
-
-                  result = RDoc::TokenStream.to_html tokens
-                  result = result + "\n" unless "\n" == result[-1]
-                  result
-                rescue
-                  CGI.escapeHTML text
-                end
-              else
-                klass = " class=\"#{format}\"" if format
-                CGI.escapeHTML text
-              end
-
-    if @options.pipe then
+    if verbatim.ruby? || (format.nil? && parseable?(text))
+      content = parsable_text_to_html(text)
+      klass = ' class="ruby"'
+    else
+      content = CGI.escapeHTML text
+      klass = " class=\"#{format}\"" if format
+    end
+
+    if @pipe
       @res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"
     else
       @res << "\n<pre#{klass}>#{content}</pre>\n"
@@ -311,22 +517,22 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   def accept_heading(heading)
     level = [6, heading.level].min
 
-    label = heading.label @code_object
-    legacy_label = heading.legacy_label @code_object
+    label = deduplicate_heading_id(heading.label(@code_object))
+    legacy_label = deduplicate_heading_id(heading.legacy_label(@code_object))
 
     # Add legacy anchor before the heading for backward compatibility.
     # This allows old links with label- prefix to still work.
-    if @options.output_decoration && !@options.pipe
+    if @output_decoration && !@pipe
       @res << "\n<span id=\"#{legacy_label}\" class=\"legacy-anchor\"></span>"
     end
 
-    @res << if @options.output_decoration
+    @res << if @output_decoration
               "\n<h#{level} id=\"#{label}\">"
             else
               "\n<h#{level}>"
             end
 
-    if @options.pipe
+    if @pipe
       @res << to_html(heading.text)
     else
       @res << "<a href=\"##{label}\">#{to_html(heading.text)}</a>"
@@ -367,6 +573,20 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
   # :section: Utilities
 
+  ##
+  # Returns a unique heading ID, appending -1, -2, etc. for duplicates.
+  # Matches GitHub's behavior for duplicate heading anchors.
+
+  def deduplicate_heading_id(id)
+    if @heading_ids.key?(id)
+      @heading_ids[id] += 1
+      "#{id}-#{@heading_ids[id]}"
+    else
+      @heading_ids[id] = 0
+      id
+    end
+  end
+
   ##
   # CGI-escapes +text+
 
@@ -415,16 +635,6 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     tags[open_tag ? 0 : 1]
   end
 
-  ##
-  # Maps attributes to HTML tags
-
-  def init_tags
-    add_tag :BOLD,   "<strong>", "</strong>"
-    add_tag :TT,     "<code>",   "</code>"
-    add_tag :EM,     "<em>",     "</em>"
-    add_tag :STRIKE, "<del>",    "</del>"
-  end
-
   ##
   # Returns the HTML tag for +list_type+, possible using a label from
   # +list_item+
@@ -460,170 +670,13 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # Returns true if text is valid ruby syntax
 
   def parseable?(text)
-    verbose, $VERBOSE = $VERBOSE, nil
-    catch(:valid) do
-      eval("BEGIN { throw :valid, true }\n#{text}")
-    end
-  rescue SyntaxError
-    false
-  ensure
-    $VERBOSE = verbose
+    Prism.parse_success?(text)
   end
 
   ##
   # Converts +item+ to HTML using RDoc::Text#to_html
 
   def to_html(item)
-    super convert_flow @am.flow item
-  end
-
-  private
-
-  def convert_flow(flow_items)
-    res = []
-
-    flow_items.each do |item|
-      case item
-      when String
-        append_flow_fragment res, convert_string(item)
-      when RDoc::Markup::AttrChanger
-        off_tags res, item
-        on_tags  res, item
-      when RDoc::Markup::RegexpHandling
-        append_flow_fragment res, convert_regexp_handling(item)
-      else
-        raise "Unknown flow element: #{item.inspect}"
-      end
-    end
-
-    res.join
-  end
-
-  def append_flow_fragment(res, fragment)
-    return if fragment.nil? || fragment.empty?
-
-    emit_tidy_link_fragment(res, fragment)
-  end
-
-  def append_to_tidy_label(fragment)
-    @tidy_link_buffer << fragment
-  end
-
-  ##
-  # Matches an entire tidy link with a braced label "{label}[url]".
-  #
-  # Capture 1: label contents.
-  # Capture 2: URL text.
-  # Capture 3: trailing content.
-  TIDY_LINK_WITH_BRACES = /\A\{(.*?)\}\[(.*?)\](.*)\z/
-
-  ##
-  # Matches the tail of a braced tidy link when the opening brace was
-  # consumed earlier while accumulating the label text.
-  #
-  # Capture 1: remaining label content.
-  # Capture 2: URL text.
-  # Capture 3: trailing content.
-  TIDY_LINK_WITH_BRACES_TAIL = /\A(.*?)\}\[(.*?)\](.*)\z/
-
-  ##
-  # Matches a tidy link with a single-word label "label[url]".
-  #
-  # Capture 1: the single-word label (no whitespace).
-  # Capture 2: URL text between the brackets.
-  TIDY_LINK_SINGLE_WORD = /\A(\S+)\[(.*?)\](.*)\z/
-
-  def convert_complete_tidy_link(text)
-    return text unless
-      text =~ TIDY_LINK_WITH_BRACES or text =~ TIDY_LINK_SINGLE_WORD
-
-    label = $1
-    url   = CGI.escapeHTML($2)
-
-    label_html = if /^rdoc-image:/ =~ label
-                   handle_RDOCLINK(label)
-                 else
-                   render_tidy_link_label(label)
-                 end
-
-    gen_url url, label_html
-  end
-
-  def emit_tidy_link_fragment(res, fragment)
-    if tidy_link_capturing?
-      append_to_tidy_label fragment
-    else
-      res << fragment
-    end
-  end
-
-  def finish_tidy_link(text)
-    label_tail, url, trailing = extract_tidy_link_parts(text)
-    append_to_tidy_label CGI.escapeHTML(label_tail) unless label_tail.empty?
-
-    return '' unless url
-
-    label_html = @tidy_link_buffer
-    @tidy_link_buffer = nil
-    link = gen_url(url, label_html)
-
-    return link if trailing.empty?
-
-    link + CGI.escapeHTML(trailing)
-  end
-
-  def extract_tidy_link_parts(text)
-    if text =~ TIDY_LINK_WITH_BRACES
-      [$1, CGI.escapeHTML($2), $3]
-    elsif text =~ TIDY_LINK_WITH_BRACES_TAIL
-      [$1, CGI.escapeHTML($2), $3]
-    elsif text =~ TIDY_LINK_SINGLE_WORD
-      [$1, CGI.escapeHTML($2), $3]
-    else
-      [text, nil, '']
-    end
-  end
-
-  def on_tags(res, item)
-    each_attr_tag(item.turn_on) do |tag|
-      emit_tidy_link_fragment(res, annotate(tag.on))
-      @in_tt += 1 if tt? tag
-    end
-  end
-
-  def off_tags(res, item)
-    each_attr_tag(item.turn_off, true) do |tag|
-      emit_tidy_link_fragment(res, annotate(tag.off))
-      @in_tt -= 1 if tt? tag
-    end
-  end
-
-  def start_tidy_link(text)
-    @tidy_link_buffer = String.new
-    append_to_tidy_label CGI.escapeHTML(text.delete_prefix('{'))
-  end
-
-  def tidy_link_capturing?
-    !!@tidy_link_buffer
-  end
-
-  def render_tidy_link_label(label)
-    RDoc::Markup::LinkLabelToHtml.render(label, @options, @from_path)
-  end
-end
-
-##
-# Formatter dedicated to rendering tidy link labels without mutating the
-# calling formatter's state.
-
-class RDoc::Markup::LinkLabelToHtml < RDoc::Markup::ToHtml
-  def self.render(label, options, from_path)
-    new(options, from_path).to_html(label)
-  end
-
-  def initialize(options, from_path = nil)
-    super(options)
-
-    self.from_path = from_path if from_path
+    handle_inline(item)
   end
 end