rdoc 8.0.0 → 8.1.0

data/lib/rdoc/i18n/text.rb

@@ -1,126 +1,130 @@
 # frozen_string_literal: true
-##
-# An i18n supported text.
-#
-# This object provides the following two features:
-#
-#   * Extracts translation messages from wrapped raw text.
-#   * Translates wrapped raw text in specified locale.
-#
-# Wrapped raw text is one of String, RDoc::Comment or Array of them.
+module RDoc
+  module I18n
+    ##
+    # An i18n supported text.
+    #
+    # This object provides the following two features:
+    #
+    #   * Extracts translation messages from wrapped raw text.
+    #   * Translates wrapped raw text in specified locale.
+    #
+    # Wrapped raw text is one of String, RDoc::Comment or Array of them.
 
-class RDoc::I18n::Text
+    class Text
 
-  ##
-  # Creates a new i18n supported text for +raw+ text.
+      ##
+      # Creates a new i18n supported text for +raw+ text.
 
-  def initialize(raw)
-    @raw = raw
-  end
+      def initialize(raw)
+        @raw = raw
+      end
 
-  ##
-  # Extracts translation target messages and yields each message.
-  #
-  # Each yielded message is a Hash. It consists of the followings:
-  #
-  # :type      :: :paragraph
-  # :paragraph :: String (The translation target message itself.)
-  # :line_no   :: Integer (The line number of the :paragraph is started.)
-  #
-  # The above content may be added in the future.
+      ##
+      # Extracts translation target messages and yields each message.
+      #
+      # Each yielded message is a Hash. It consists of the followings:
+      #
+      # :type      :: :paragraph
+      # :paragraph :: String (The translation target message itself.)
+      # :line_no   :: Integer (The line number of the :paragraph is started.)
+      #
+      # The above content may be added in the future.
 
-  def extract_messages
-    parse do |part|
-      case part[:type]
-      when :empty_line
-        # ignore
-      when :paragraph
-        yield(part)
+      def extract_messages
+        parse do |part|
+          case part[:type]
+          when :empty_line
+            # ignore
+          when :paragraph
+            yield(part)
+          end
+        end
       end
-    end
-  end
 
-  # Translates raw text into +locale+.
-  def translate(locale)
-    translated_text = ''
-    parse do |part|
-      case part[:type]
-      when :paragraph
-        translated_text += locale.translate(part[:paragraph])
-      when :empty_line
-        translated_text += part[:line]
-      else
-        raise "should not reach here: unexpected type: #{type}"
+      # Translates raw text into +locale+.
+      def translate(locale)
+        translated_text = ''
+        parse do |part|
+          case part[:type]
+          when :paragraph
+            translated_text += locale.translate(part[:paragraph])
+          when :empty_line
+            translated_text += part[:line]
+          else
+            raise "should not reach here: unexpected type: #{type}"
+          end
+        end
+        translated_text
       end
-    end
-    translated_text
-  end
 
-  private
-  def parse(&block)
-    paragraph = ''
-    paragraph_start_line = 0
-    line_no = 0
+    private
+      def parse(&block)
+        paragraph = ''
+        paragraph_start_line = 0
+        line_no = 0
 
-    each_line(@raw) do |line|
-      line_no += 1
-      case line
-      when /\A\s*\z/
-        if paragraph.empty?
-          emit_empty_line_event(line, line_no, &block)
-        else
-          paragraph += line
-          emit_paragraph_event(paragraph, paragraph_start_line, line_no,
-                               &block)
-          paragraph = ''
+        each_line(@raw) do |line|
+          line_no += 1
+          case line
+          when /\A\s*\z/
+            if paragraph.empty?
+              emit_empty_line_event(line, line_no, &block)
+            else
+              paragraph += line
+              emit_paragraph_event(paragraph, paragraph_start_line, line_no,
+                                   &block)
+              paragraph = ''
+            end
+          else
+            paragraph_start_line = line_no if paragraph.empty?
+            paragraph += line
+          end
+        end
+
+        unless paragraph.empty?
+          emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
         end
-      else
-        paragraph_start_line = line_no if paragraph.empty?
-        paragraph += line
       end
-    end
 
-    unless paragraph.empty?
-      emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
-    end
-  end
+      def each_line(raw, &block)
+        case raw
+        when Comment
+          raw.text.each_line(&block)
+        when Hash
+          raw.each_value do |comments|
+            comments.each { |comment| each_line(comment, &block) }
+          end
+        else
+          raw.each_line(&block)
+        end
+      end
 
-  def each_line(raw, &block)
-    case raw
-    when RDoc::Comment
-      raw.text.each_line(&block)
-    when Hash
-      raw.each_value do |comments|
-        comments.each { |comment| each_line(comment, &block) }
+      def emit_empty_line_event(line, line_no)
+        part = {
+          :type => :empty_line,
+          :line => line,
+          :line_no => line_no,
+        }
+        yield(part)
       end
-    else
-      raw.each_line(&block)
-    end
-  end
 
-  def emit_empty_line_event(line, line_no)
-    part = {
-      :type => :empty_line,
-      :line => line,
-      :line_no => line_no,
-    }
-    yield(part)
-  end
+      def emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
+        paragraph_part = {
+          :type => :paragraph,
+          :line_no => paragraph_start_line,
+        }
+        match_data = /(\s*)\z/.match(paragraph)
+        if match_data
+          paragraph_part[:paragraph] = match_data.pre_match
+          yield(paragraph_part)
+          emit_empty_line_event(match_data[1], line_no, &block)
+        else
+          paragraph_part[:paragraph] = paragraph
+          yield(paragraph_part)
+        end
+      end
 
-  def emit_paragraph_event(paragraph, paragraph_start_line, line_no, &block)
-    paragraph_part = {
-      :type => :paragraph,
-      :line_no => paragraph_start_line,
-    }
-    match_data = /(\s*)\z/.match(paragraph)
-    if match_data
-      paragraph_part[:paragraph] = match_data.pre_match
-      yield(paragraph_part)
-      emit_empty_line_event(match_data[1], line_no, &block)
-    else
-      paragraph_part[:paragraph] = paragraph
-      yield(paragraph_part)
     end
   end
-
 end

data/lib/rdoc/i18n.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
-##
-# This module provides i18n related features.
+module RDoc
+  ##
+  # This module provides i18n related features.
 
-module RDoc::I18n
+  module I18n
 
-  autoload :Locale, "#{__dir__}/i18n/locale"
-  require_relative 'i18n/text'
+    autoload :Locale, "#{__dir__}/i18n/locale"
+    require_relative 'i18n/text'
 
+  end
 end

data/lib/rdoc/markdown/byte_runtime.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'strscan'
+
+module RDoc
+  class Markdown
+
+    ##
+    # Byte-offset replacements for the position helpers of the kpeg-generated
+    # parser runtime.
+    #
+    # The generated runtime addresses +@string+ by character index, which makes
+    # every position lookup scan the string from its beginning when the input
+    # contains non-ASCII characters, so parse time becomes quadratic in the
+    # input size.  Generated rule bodies only save and restore +pos+ without
+    # inspecting it, so replacing these helpers is enough to switch the whole
+    # parser to byte offsets.
+    #
+    # get_byte (the grammar's `.`) consumes one character and returns its
+    # codepoint, exactly like the character-index runtime, so positions always
+    # stay on character boundaries and the only observable difference is the
+    # position values themselves.  The input must be validly encoded in an
+    # ASCII-compatible encoding.
+    #
+    # The error-reporting helpers of the generated runtime (+current_line+,
+    # +current_column+, ...) are left as-is and would misreport locations when
+    # given byte offsets.  They are unreachable: markdown is deliberately
+    # designed to parse any input somehow rather than fail (the root rule
+    # `Doc = Block*` cannot fail), so a parse failure means a bug in the
+    # grammar itself, and nothing in RDoc invokes +raise_error+ or
+    # +show_error+.  Make these helpers byte-aware before using them for
+    # anything.
+
+    module ByteRuntime
+      def set_string(string, pos)
+        @string = string
+        @string_size = string ? string.bytesize : 0
+        @pos = pos
+        @position_line_offsets = nil
+        @scanner = string ? StringScanner.new(string) : nil
+      end
+
+      def scan(reg)
+        @scanner.pos = @pos
+        if @scanner.skip(reg)
+          @pos = @scanner.pos
+          true
+        end
+      end
+
+      def match_string(str)
+        len = str.bytesize
+        if @string.byteslice(@pos, len) == str
+          @pos += len
+          str
+        end
+      end
+
+      def get_byte
+        byte = @string.getbyte(@pos)
+        return nil unless byte
+
+        if byte < 0x80
+          @pos += 1
+          byte
+        else
+          @scanner.pos = @pos
+          # /./ interprets the character in the string's own encoding
+          char = @scanner.scan(/./m)
+          @pos = @scanner.pos
+          char.ord
+        end
+      end
+
+      def get_text(start)
+        @string.byteslice(start, @pos - start)
+      end
+    end
+  end
+end

data/lib/rdoc/markdown.kpeg

@@ -191,7 +191,9 @@
   require_relative 'markup/to_joined_paragraph'
   require_relative 'markdown/entities'
 
-  require_relative 'markdown/literals'
+  require_relative 'markdown/byte_runtime'
+
+  prepend ByteRuntime
 
   ##
   # Supported extensions
@@ -448,8 +450,12 @@
     if notes? then
       @footnotes       = {}
 
-      setup_parser markdown, @debug
-      peg_parse 'Notes'
+      # A note definition must contain "[^", so documents without one
+      # have nothing for the collecting pass to find
+      if markdown.include? '[^' then
+        setup_parser markdown, @debug
+        peg_parse 'Notes'
+      end
 
       # using note_order on the first pass would be a bug
       @note_order      = []
@@ -563,7 +569,7 @@
 
 root = Doc
 
-Doc =       BOM? Block*:a { RDoc::Markup::Document.new(*a.compact) }
+Doc =       Block*:a { RDoc::Markup::Document.new(*a.compact) }
 
 Block =     @BlankLine*
             ( BlockQuote
@@ -867,7 +873,7 @@ HtmlBlockInTags = HtmlAnchor
                 | HtmlBlockScript
                 | HtmlBlockHead
 
-HtmlBlock = < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) >
+HtmlBlock = &"<" < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) >
             @BlankLine+
             { if html? then
                 RDoc::Markup::Raw.new text
@@ -1172,12 +1178,10 @@ SpecialChar =   /[~*_`&\[\]()<!#\\'"]/ | @ExtendedSpecialChar
 NormalChar =    !( @SpecialChar | @Spacechar | @Newline ) .
 Digit = [0-9]
 
-%literals         = RDoc::Markdown::Literals
-Alphanumeric      = %literals.Alphanumeric
-AlphanumericAscii = %literals.AlphanumericAscii
-BOM               = %literals.BOM
-Newline           = %literals.Newline
-Spacechar         = %literals.Spacechar
+Alphanumeric      = /\p{Word}/
+AlphanumericAscii = /[A-Za-z0-9]/
+Newline           = /\n|\r\n?|\p{Zl}|\p{Zp}/
+Spacechar         = /\t|\p{Zs}/
 
 HexEntity  = /&#x/i < /[0-9a-fA-F]+/ > ";"
              { rdoc_escape([text.to_i(16)].pack('U')) }

data/lib/rdoc/markdown.rb

@@ -576,7 +576,9 @@ class RDoc::Markdown
   require_relative 'markup/to_joined_paragraph'
   require_relative 'markdown/entities'
 
-  require_relative 'markdown/literals'
+  require_relative 'markdown/byte_runtime'
+
+  prepend ByteRuntime
 
   ##
   # Supported extensions
@@ -833,8 +835,12 @@ class RDoc::Markdown
     if notes? then
       @footnotes       = {}
 
-      setup_parser markdown, @debug
-      peg_parse 'Notes'
+      # A note definition must contain "[^", so documents without one
+      # have nothing for the collecting pass to find
+      if markdown.include? '[^' then
+        setup_parser markdown, @debug
+        peg_parse 'Notes'
+      end
 
       # using note_order on the first pass would be a bug
       @note_order      = []
@@ -947,9 +953,7 @@ class RDoc::Markdown
 
 
   # :stopdoc:
-  def setup_foreign_grammar
-    @_grammar_literals = RDoc::Markdown::Literals.new(nil)
-  end
+  def setup_foreign_grammar; end
 
   # root = Doc
   def _root
@@ -958,21 +962,11 @@ class RDoc::Markdown
     return _tmp
   end
 
-  # Doc = BOM? Block*:a { RDoc::Markup::Document.new(*a.compact) }
+  # Doc = Block*:a { RDoc::Markup::Document.new(*a.compact) }
   def _Doc
 
     _save = self.pos
     while true # sequence
-      _save1 = self.pos
-      _tmp = apply(:_BOM)
-      unless _tmp
-        _tmp = true
-        self.pos = _save1
-      end
-      unless _tmp
-        self.pos = _save
-        break
-      end
       _ary = []
       while true
         _tmp = apply(:_Block)
@@ -8971,27 +8965,34 @@ class RDoc::Markdown
     return _tmp
   end
 
-  # HtmlBlock = < (HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) > @BlankLine+ { if html? then                 RDoc::Markup::Raw.new text               end }
+  # HtmlBlock = &"<" < (HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) > @BlankLine+ { if html? then                 RDoc::Markup::Raw.new text               end }
   def _HtmlBlock
 
     _save = self.pos
     while true # sequence
+      _save1 = self.pos
+      _tmp = match_string("<")
+      self.pos = _save1
+      unless _tmp
+        self.pos = _save
+        break
+      end
       _text_start = self.pos
 
-      _save1 = self.pos
+      _save2 = self.pos
       while true # choice
         _tmp = apply(:_HtmlBlockInTags)
         break if _tmp
-        self.pos = _save1
+        self.pos = _save2
         _tmp = apply(:_HtmlComment)
         break if _tmp
-        self.pos = _save1
+        self.pos = _save2
         _tmp = apply(:_HtmlBlockSelfClosing)
         break if _tmp
-        self.pos = _save1
+        self.pos = _save2
         _tmp = apply(:_HtmlUnclosed)
         break if _tmp
-        self.pos = _save1
+        self.pos = _save2
         break
       end # end choice
 
@@ -9002,7 +9003,7 @@ class RDoc::Markdown
         self.pos = _save
         break
       end
-      _save2 = self.pos
+      _save3 = self.pos
       _tmp = _BlankLine()
       if _tmp
         while true
@@ -9011,7 +9012,7 @@ class RDoc::Markdown
         end
         _tmp = true
       else
-        self.pos = _save2
+        self.pos = _save3
       end
       unless _tmp
         self.pos = _save
@@ -14752,37 +14753,30 @@ class RDoc::Markdown
     return _tmp
   end
 
-  # Alphanumeric = %literals.Alphanumeric
+  # Alphanumeric = /\p{Word}/
   def _Alphanumeric
-    _tmp = @_grammar_literals.external_invoke(self, :_Alphanumeric)
+    _tmp = scan(/\G(?-mix:\p{Word})/)
     set_failed_rule :_Alphanumeric unless _tmp
     return _tmp
   end
 
-  # AlphanumericAscii = %literals.AlphanumericAscii
+  # AlphanumericAscii = /[A-Za-z0-9]/
   def _AlphanumericAscii
-    _tmp = @_grammar_literals.external_invoke(self, :_AlphanumericAscii)
+    _tmp = scan(/\G(?-mix:[A-Za-z0-9])/)
     set_failed_rule :_AlphanumericAscii unless _tmp
     return _tmp
   end
 
-  # BOM = %literals.BOM
-  def _BOM
-    _tmp = @_grammar_literals.external_invoke(self, :_BOM)
-    set_failed_rule :_BOM unless _tmp
-    return _tmp
-  end
-
-  # Newline = %literals.Newline
+  # Newline = /\n|\r\n?|\p{Zl}|\p{Zp}/
   def _Newline
-    _tmp = @_grammar_literals.external_invoke(self, :_Newline)
+    _tmp = scan(/\G(?-mix:\n|\r\n?|\p{Zl}|\p{Zp})/)
     set_failed_rule :_Newline unless _tmp
     return _tmp
   end
 
-  # Spacechar = %literals.Spacechar
+  # Spacechar = /\t|\p{Zs}/
   def _Spacechar
-    _tmp = @_grammar_literals.external_invoke(self, :_Spacechar)
+    _tmp = scan(/\G(?-mix:\t|\p{Zs})/)
     set_failed_rule :_Spacechar unless _tmp
     return _tmp
   end
@@ -16613,7 +16607,7 @@ class RDoc::Markdown
 
   Rules = {}
   Rules[:_root] = rule_info("root", "Doc")
-  Rules[:_Doc] = rule_info("Doc", "BOM? Block*:a { RDoc::Markup::Document.new(*a.compact) }")
+  Rules[:_Doc] = rule_info("Doc", "Block*:a { RDoc::Markup::Document.new(*a.compact) }")
   Rules[:_Block] = rule_info("Block", "@BlankLine* (BlockQuote | Verbatim | CodeFence | Table | Note | Reference | HorizontalRule | Heading | OrderedList | BulletList | DefinitionList | HtmlBlock | StyleBlock | Para | Plain)")
   Rules[:_Para] = rule_info("Para", "@NonindentSpace Inlines:a @BlankLine+ { paragraph a }")
   Rules[:_Plain] = rule_info("Plain", "Inlines:a { paragraph a }")
@@ -16749,7 +16743,7 @@ class RDoc::Markdown
   Rules[:_HtmlBlockCloseHead] = rule_info("HtmlBlockCloseHead", "\"<\" Spnl \"/\" (\"head\" | \"HEAD\") Spnl \">\"")
   Rules[:_HtmlBlockHead] = rule_info("HtmlBlockHead", "HtmlBlockOpenHead (!HtmlBlockCloseHead .)* HtmlBlockCloseHead")
   Rules[:_HtmlBlockInTags] = rule_info("HtmlBlockInTags", "(HtmlAnchor | HtmlBlockAddress | HtmlBlockBlockquote | HtmlBlockCenter | HtmlBlockDir | HtmlBlockDiv | HtmlBlockDl | HtmlBlockFieldset | HtmlBlockForm | HtmlBlockH1 | HtmlBlockH2 | HtmlBlockH3 | HtmlBlockH4 | HtmlBlockH5 | HtmlBlockH6 | HtmlBlockMenu | HtmlBlockNoframes | HtmlBlockNoscript | HtmlBlockOl | HtmlBlockP | HtmlBlockPre | HtmlBlockTable | HtmlBlockUl | HtmlBlockDd | HtmlBlockDt | HtmlBlockFrameset | HtmlBlockLi | HtmlBlockTbody | HtmlBlockTd | HtmlBlockTfoot | HtmlBlockTh | HtmlBlockThead | HtmlBlockTr | HtmlBlockScript | HtmlBlockHead)")
-  Rules[:_HtmlBlock] = rule_info("HtmlBlock", "< (HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) > @BlankLine+ { if html? then                 RDoc::Markup::Raw.new text               end }")
+  Rules[:_HtmlBlock] = rule_info("HtmlBlock", "&\"<\" < (HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing | HtmlUnclosed) > @BlankLine+ { if html? then                 RDoc::Markup::Raw.new text               end }")
   Rules[:_HtmlUnclosed] = rule_info("HtmlUnclosed", "\"<\" Spnl HtmlUnclosedType Spnl HtmlAttribute* Spnl \">\"")
   Rules[:_HtmlUnclosedType] = rule_info("HtmlUnclosedType", "(\"HR\" | \"hr\")")
   Rules[:_HtmlBlockSelfClosing] = rule_info("HtmlBlockSelfClosing", "\"<\" Spnl HtmlBlockType Spnl HtmlAttribute* \"/\" Spnl \">\"")
@@ -16825,11 +16819,10 @@ class RDoc::Markdown
   Rules[:_SpecialChar] = rule_info("SpecialChar", "(/[~*_`&\\[\\]()<!\#\\\\'\"]/ | @ExtendedSpecialChar)")
   Rules[:_NormalChar] = rule_info("NormalChar", "!(@SpecialChar | @Spacechar | @Newline) .")
   Rules[:_Digit] = rule_info("Digit", "[0-9]")
-  Rules[:_Alphanumeric] = rule_info("Alphanumeric", "%literals.Alphanumeric")
-  Rules[:_AlphanumericAscii] = rule_info("AlphanumericAscii", "%literals.AlphanumericAscii")
-  Rules[:_BOM] = rule_info("BOM", "%literals.BOM")
-  Rules[:_Newline] = rule_info("Newline", "%literals.Newline")
-  Rules[:_Spacechar] = rule_info("Spacechar", "%literals.Spacechar")
+  Rules[:_Alphanumeric] = rule_info("Alphanumeric", "/\\p{Word}/")
+  Rules[:_AlphanumericAscii] = rule_info("AlphanumericAscii", "/[A-Za-z0-9]/")
+  Rules[:_Newline] = rule_info("Newline", "/\\n|\\r\\n?|\\p{Zl}|\\p{Zp}/")
+  Rules[:_Spacechar] = rule_info("Spacechar", "/\\t|\\p{Zs}/")
   Rules[:_HexEntity] = rule_info("HexEntity", "/&\#x/i < /[0-9a-fA-F]+/ > \";\" { rdoc_escape([text.to_i(16)].pack('U')) }")
   Rules[:_DecEntity] = rule_info("DecEntity", "\"&\#\" < /[0-9]+/ > \";\" { rdoc_escape([text.to_i].pack('U')) }")
   Rules[:_CharEntity] = rule_info("CharEntity", "\"&\" < /[A-Za-z0-9]+/ > \";\" { if entity = HTML_ENTITIES[text] then                  rdoc_escape(entity.pack('U*'))                else                  \"&\#{text};\"                end              }")

data/lib/rdoc/markup/block_quote.rb

@@ -1,14 +1,18 @@
 # frozen_string_literal: true
-##
-# A quoted section which contains markup items.
+module RDoc
+  class Markup
+    ##
+    # A quoted section which contains markup items.
 
-class RDoc::Markup::BlockQuote < RDoc::Markup::Raw
+    class BlockQuote < Markup::Raw
 
-  ##
-  # Calls #accept_block_quote on +visitor+
+      ##
+      # Calls #accept_block_quote on +visitor+
 
-  def accept(visitor)
-    visitor.accept_block_quote self
-  end
+      def accept(visitor)
+        visitor.accept_block_quote self
+      end
 
+    end
+  end
 end