rdoc 6.17.0 → 8.1.0

data/lib/rdoc/i18n/text.rb

@@ -1,126 +1,130 @@
 # frozen_string_literal: true
-##
-# An i18n supported text.
-#
-# This object provides the following two features:
-#
-#   * Extracts translation messages from wrapped raw text.
-#   * Translates wrapped raw text in specified locale.
-#
-# Wrapped raw text is one of String, RDoc::Comment or Array of them.
+module RDoc
+  module I18n
+    ##
+    # An i18n supported text.
+    #
+    # This object provides the following two features:
+    #
+    #   * Extracts translation messages from wrapped raw text.
+    #   * Translates wrapped raw text in specified locale.
+    #
+    # Wrapped raw text is one of String, RDoc::Comment or Array of them.
 
-class RDoc::I18n::Text
+    class Text
 
-  ##
-  # Creates a new i18n supported text for +raw+ text.
+      ##
+      # Creates a new i18n supported text for +raw+ text.
 
-  def initialize(raw)
-    @raw = raw
-  end
+      def initialize(raw)
+        @raw = raw
+      end
 
-  ##
-  # Extracts translation target messages and yields each message.
-  #
-  # Each yielded message is a Hash. It consists of the followings:
-  #
-  # :type      :: :paragraph
-  # :paragraph :: String (The translation target message itself.)
-  # :line_no   :: Integer (The line number of the :paragraph is started.)
-  #
-  # The above content may be added in the future.
+      ##
+      # Extracts translation target messages and yields each message.
+      #
+      # Each yielded message is a Hash. It consists of the followings:
+      #
+      # :type      :: :paragraph
+      # :paragraph :: String (The translation target message itself.)
+      # :line_no   :: Integer (The line number of the :paragraph is started.)
+      #
+      # The above content may be added in the future.
 
-  def extract_messages
-    parse do |part|
-      case part[:type]
-      when :empty_line
-        # ignore
-      when :paragraph
-        yield(part)
+      def extract_messages
+        parse do |part|
+          case part[:type]
+          when :empty_line
+            # ignore
+          when :paragraph
+            yield(part)
+          end
+        end
       end
-    end
-  end
 
-  # Translates raw text into +locale+.
-  def translate(locale)
-    translated_text = ''
-    parse do |part|
-      case part[:type]
-      when :paragraph
-        translated_text += locale.translate(part[:paragraph])
-      when :empty_line
-        translated_text += part[:line]
-      else
-        raise "should not reach here: unexpected type: #{type}"
+      # Translates raw text into +locale+.
+      def translate(locale)
+        translated_text = ''
+        parse do |part|
+          case part[:type]
+          when :paragraph
+            translated_text += locale.translate(part[:paragraph])
+          when :empty_line
+            translated_text += part[:line]
+          else
+            raise "should not reach here: unexpected type: #{type}"
+          end
+        end
+        translated_text
       end
-    end
-    translated_text
-  end
 
-  private
-  def parse(&block)
-    paragraph = ''
-    paragraph_start_line = 0
-    line_no = 0
+    private
+      def parse(&block)
+        paragraph = ''
+        paragraph_start_line = 0
+        line_no = 0
 
-    each_line(@raw) do |line|
-      line_no += 1
-      case line
-      when /\A\s*\z/
-        if paragraph.empty?
-          emit_empty_line_event(line, line_no, &block)
-        else
-          paragraph += line
-          emit_paragraph_event(paragraph, paragraph_start_line, line_no,
-                               &block)
-          paragraph = ''
+        each_line(@raw) do |line|
+          line_no += 1
+          case line
+          when /\A\s*\z/
+            if paragraph.empty?
+              emit_empty_line_event(line, line_no, &block)
+            else
+              paragraph += line
+              emit_paragraph_event(paragraph, paragraph_start_line, line_no,
+                                   &block)
+              paragraph = ''
+            end
+          else
+            paragraph_start_line = line_no if paragraph.empty?
+            paragraph += line
+          end
+        end
+
+        unless paragraph.empty?
+          emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
         end
-      else
-        paragraph_start_line = line_no if paragraph.empty?
-        paragraph += line
       end
-    end
 
-    unless paragraph.empty?
-      emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
-    end
-  end
+      def each_line(raw, &block)
+        case raw
+        when Comment
+          raw.text.each_line(&block)
+        when Hash
+          raw.each_value do |comments|
+            comments.each { |comment| each_line(comment, &block) }
+          end
+        else
+          raw.each_line(&block)
+        end
+      end
 
-  def each_line(raw, &block)
-    case raw
-    when RDoc::Comment
-      raw.text.each_line(&block)
-    when Array
-      raw.each do |comment, location|
-        each_line(comment, &block)
+      def emit_empty_line_event(line, line_no)
+        part = {
+          :type => :empty_line,
+          :line => line,
+          :line_no => line_no,
+        }
+        yield(part)
       end
-    else
-      raw.each_line(&block)
-    end
-  end
 
-  def emit_empty_line_event(line, line_no)
-    part = {
-      :type => :empty_line,
-      :line => line,
-      :line_no => line_no,
-    }
-    yield(part)
-  end
+      def emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
+        paragraph_part = {
+          :type => :paragraph,
+          :line_no => paragraph_start_line,
+        }
+        match_data = /(\s*)\z/.match(paragraph)
+        if match_data
+          paragraph_part[:paragraph] = match_data.pre_match
+          yield(paragraph_part)
+          emit_empty_line_event(match_data[1], line_no, &block)
+        else
+          paragraph_part[:paragraph] = paragraph
+          yield(paragraph_part)
+        end
+      end
 
-  def emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
-    paragraph_part = {
-      :type => :paragraph,
-      :line_no => paragraph_start_line,
-    }
-    match_data = /(\s*)\z/.match(paragraph)
-    if match_data
-      paragraph_part[:paragraph] = match_data.pre_match
-      yield(paragraph_part)
-      emit_empty_line_event(match_data[1], line_no, &block)
-    else
-      paragraph_part[:paragraph] = paragraph
-      yield(paragraph_part)
     end
   end
-
 end

data/lib/rdoc/i18n.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
-##
-# This module provides i18n related features.
+module RDoc
+  ##
+  # This module provides i18n related features.
 
-module RDoc::I18n
+  module I18n
 
-  autoload :Locale, "#{__dir__}/i18n/locale"
-  require_relative 'i18n/text'
+    autoload :Locale, "#{__dir__}/i18n/locale"
+    require_relative 'i18n/text'
 
+  end
 end

data/lib/rdoc/markdown/byte_runtime.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'strscan'
+
+module RDoc
+  class Markdown
+
+    ##
+    # Byte-offset replacements for the position helpers of the kpeg-generated
+    # parser runtime.
+    #
+    # The generated runtime addresses +@string+ by character index, which makes
+    # every position lookup scan the string from its beginning when the input
+    # contains non-ASCII characters, so parse time becomes quadratic in the
+    # input size.  Generated rule bodies only save and restore +pos+ without
+    # inspecting it, so replacing these helpers is enough to switch the whole
+    # parser to byte offsets.
+    #
+    # get_byte (the grammar's `.`) consumes one character and returns its
+    # codepoint, exactly like the character-index runtime, so positions always
+    # stay on character boundaries and the only observable difference is the
+    # position values themselves.  The input must be validly encoded in an
+    # ASCII-compatible encoding.
+    #
+    # The error-reporting helpers of the generated runtime (+current_line+,
+    # +current_column+, ...) are left as-is and would misreport locations when
+    # given byte offsets.  They are unreachable: markdown is deliberately
+    # designed to parse any input somehow rather than fail (the root rule
+    # `Doc = Block*` cannot fail), so a parse failure means a bug in the
+    # grammar itself, and nothing in RDoc invokes +raise_error+ or
+    # +show_error+.  Make these helpers byte-aware before using them for
+    # anything.
+
+    module ByteRuntime
+      def set_string(string, pos)
+        @string = string
+        @string_size = string ? string.bytesize : 0
+        @pos = pos
+        @position_line_offsets = nil
+        @scanner = string ? StringScanner.new(string) : nil
+      end
+
+      def scan(reg)
+        @scanner.pos = @pos
+        if @scanner.skip(reg)
+          @pos = @scanner.pos
+          true
+        end
+      end
+
+      def match_string(str)
+        len = str.bytesize
+        if @string.byteslice(@pos, len) == str
+          @pos += len
+          str
+        end
+      end
+
+      def get_byte
+        byte = @string.getbyte(@pos)
+        return nil unless byte
+
+        if byte < 0x80
+          @pos += 1
+          byte
+        else
+          @scanner.pos = @pos
+          # /./ interprets the character in the string's own encoding
+          char = @scanner.scan(/./m)
+          @pos = @scanner.pos
+          char.ord
+        end
+      end
+
+      def get_text(start)
+        @string.byteslice(start, @pos - start)
+      end
+    end
+  end
+end

data/lib/rdoc/markdown.kpeg

@@ -191,7 +191,9 @@
   require_relative 'markup/to_joined_paragraph'
   require_relative 'markdown/entities'
 
-  require_relative 'markdown/literals'
+  require_relative 'markdown/byte_runtime'
+
+  prepend ByteRuntime
 
   ##
   # Supported extensions
@@ -303,6 +305,20 @@
     end
   end
 
+  # Escape character that has special meaning in RDoc format.
+  # To allow rdoc-styled link used in markdown format for now, bracket and brace are not escaped.
+
+  def rdoc_escape(text)
+    text.gsub(/[*+<\\_]/) {|s| "\\#{s}" }
+  end
+
+  # Escape link url that contains brackets.
+  # Brackets needs escape because link url will be surrounded by `[]` in RDoc format.
+
+  def rdoc_link_url_escape(text)
+    text.gsub(/[\[\]\\]/) {|s| "\\#{s}" }
+  end
+
   ##
   # :category: Extensions
   #
@@ -389,6 +405,8 @@
   # the note order list for proper display at the end of the document.
 
   def note_for ref
+    return unless @note_order
+
     @note_order << ref
 
     label = @note_order.length
@@ -432,8 +450,12 @@
     if notes? then
       @footnotes       = {}
 
-      setup_parser markdown, @debug
-      peg_parse 'Notes'
+      # A note definition must contain "[^", so documents without one
+      # have nothing for the collecting pass to find
+      if markdown.include? '[^' then
+        setup_parser markdown, @debug
+        peg_parse 'Notes'
+      end
 
       # using note_order on the first pass would be a bug
       @note_order      = []
@@ -490,11 +512,7 @@
   # Wraps `text` in strike markup for rdoc inline formatting
 
   def strike text
-    if text =~ /\A[a-z\d.\/-]+\z/i then
-      "~#{text}~"
-    else
-      "<s>#{text}</s>"
-    end
+    "<del>#{text}</del>"
   end
 
   ##
@@ -551,7 +569,7 @@
 
 root = Doc
 
-Doc =       BOM? Block*:a { RDoc::Markup::Document.new(*a.compact) }
+Doc =       Block*:a { RDoc::Markup::Document.new(*a.compact) }
 
 Block =     @BlankLine*
             ( BlockQuote
@@ -607,8 +625,8 @@ BlockQuote = BlockQuoteRaw:a
 
 BlockQuoteRaw =  @StartList:a
                  (( ">" " "? Line:l { a << l } )
-                  ( !">" !@BlankLine Line:c { a << c } )*
-                  ( @BlankLine:n { a << n } )*
+                  ( !">" !@BlankLine !(AtxStart @Spacechar) !Bullet !Enumerator !( &{ github? } Ticks3 ) Line:c { a << c } )*
+                  ( ">" @BlankLine:n { a << n } )*
                  )+
                  { inner_parse a.join }
 
@@ -855,7 +873,7 @@ HtmlBlockInTags = HtmlAnchor
                 | HtmlBlockScript
                 | HtmlBlockHead
 
-HtmlBlock = < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) >
+HtmlBlock = &"<" < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) >
             @BlankLine+
             { if html? then
                 RDoc::Markup::Raw.new text
@@ -965,6 +983,7 @@ Inline  = Str
         | InlineNote
         | Code
         | RawHtml
+        | StrippedComment
         | Entity
         | EscapedChar
         | Symbol
@@ -973,11 +992,11 @@ Space = @Spacechar+ { " " }
 
 Str = @StartList:a
       < @NormalChar+ > { a = text }
-      ( StrChunk:c { a << c } )* { a }
+      ( StrChunk:c { a << c } )* { rdoc_escape(a) }
 
 StrChunk = < (@NormalChar | /_+/ &Alphanumeric)+ > { text }
 
-EscapedChar =   "\\" !@Newline < /[:\\`|*_{}\[\]()#+.!><-]/ > { text }
+EscapedChar =   "\\" !@Newline < /[:\\`|*_{}\[\]()#+.!><-]/ > { rdoc_escape(text) }
 
 Entity =    ( HexEntity | DecEntity | CharEntity ):a { a }
 
@@ -992,7 +1011,7 @@ TerminalEndline = @Sp @Newline @Eof
 LineBreak = "  " @NormalEndline { RDoc::Markup::HardBreak.new }
 
 Symbol = < @SpecialChar >
-         { text }
+         { rdoc_escape(text) }
 
 # This keeps the parser from getting bogged down on long strings of '*' or '_',
 # or strings of '*' or '_' with space on each side:
@@ -1057,7 +1076,7 @@ ReferenceLinkSingle = Label:content < (Spnl "[]")? >
                       { link_to content, content, text }
 
 ExplicitLink = ExplicitLinkWithLabel:a
-               { "{#{a[:label]}}[#{a[:link]}]" }
+               { "{#{a[:label]}}[#{rdoc_link_url_escape(a[:link])}]" }
 
 ExplicitLinkWithLabel = Label:label "(" @Sp Source:link Spnl Title @Sp ")"
                         { { label: label, link: link } }
@@ -1142,8 +1161,8 @@ Code = ( Ticks1 < (
        )
        { code text }
 
-RawHtml = < (HtmlComment | HtmlBlockScript | HtmlTag) >
-          { if html? then text else '' end }
+RawHtml = < HtmlBlockScript | HtmlTag > { html? ? text : '' }
+StrippedComment = HtmlComment { '' }
 
 BlankLine =     @Sp @Newline { "\n" }
 
@@ -1159,20 +1178,18 @@ SpecialChar =   /[~*_`&\[\]()<!#\\'"]/ | @ExtendedSpecialChar
 NormalChar =    !( @SpecialChar | @Spacechar | @Newline ) .
 Digit = [0-9]
 
-%literals         = RDoc::Markdown::Literals
-Alphanumeric      = %literals.Alphanumeric
-AlphanumericAscii = %literals.AlphanumericAscii
-BOM               = %literals.BOM
-Newline           = %literals.Newline
-Spacechar         = %literals.Spacechar
+Alphanumeric      = /\p{Word}/
+AlphanumericAscii = /[A-Za-z0-9]/
+Newline           = /\n|\r\n?|\p{Zl}|\p{Zp}/
+Spacechar         = /\t|\p{Zs}/
 
 HexEntity  = /&#x/i < /[0-9a-fA-F]+/ > ";"
-             { [text.to_i(16)].pack 'U' }
+             { rdoc_escape([text.to_i(16)].pack('U')) }
 DecEntity  = "&#"   < /[0-9]+/       > ";"
-             { [text.to_i].pack 'U' }
+             { rdoc_escape([text.to_i].pack('U')) }
 CharEntity = "&"    </[A-Za-z0-9]+/  > ";"
              { if entity = HTML_ENTITIES[text] then
-                 entity.pack 'U*'
+                 rdoc_escape(entity.pack('U*'))
                else
                  "&#{text};"
                end
@@ -1223,10 +1240,12 @@ InlineNote = &{ notes? }
              @StartList:a
              ( !"]" Inline:l { a << l } )+
              "]"
-             { ref = [:inline, @note_order.length]
-               @footnotes[ref] = paragraph a
+             { if @note_order
+                 ref = [:inline, @note_order.length]
+                 @footnotes[ref] = paragraph a
 
-               note_for ref
+                 note_for ref
+               end
              }
 
 Notes = ( Note | SkipBlock )*
@@ -1239,7 +1258,7 @@ RawNoteBlock = @StartList:a
 # Markdown extensions added by RDoc follow
 
 CodeFence = &{ github? }
-            Ticks3 (@Sp StrChunk:format)? Spnl < (
+            Ticks3 (@Sp StrChunk:format)? @Sp @Newline? < (
               ( !"`" Nonspacechar )+ |
               !Ticks3 /`+/ |
               Spacechar |
@@ -1260,13 +1279,13 @@ Table = &{ github? }
 TableHead = TableItem2+:items "|"? @Newline
             { items }
 
-TableRow = ( ( TableItem:item1 TableItem2*:items { [item1, *items] } ):row | TableItem2+:row ) "|"? @Newline
+TableRow = ( ( TableItem:item1 TableItem2+:items { [item1, *items] } ):row | TableItem2+:row ) "|"? @Newline
 	   { row }
 TableItem2 = "|" TableItem
 TableItem = < /(?:\\.|[^|\n])+/ >
 	    { text.strip.gsub(/\\([|])/, '\1')  }
 
-TableLine = ( ( TableAlign:align1 TableAlign2*:aligns {[align1, *aligns] } ):line | TableAlign2+:line ) "|"? @Newline
+TableLine = ( ( TableAlign:align1 TableAlign2+:aligns {[align1, *aligns] } ):line | TableAlign2+:line ) "|"? @Newline
 	    { line }
 TableAlign2 = "|" @Sp TableAlign
 TableAlign = < /:?-+:?/ > @Sp