rdoc 6.1.0 → 6.3.0

data/lib/rdoc/markdown/literals.rb

@@ -0,0 +1,417 @@
+# coding: UTF-8
+# frozen_string_literal: true
+# :markup: markdown
+
+##
+#--
+# This set of literals is for Ruby 1.9 regular expressions and gives full
+# unicode support.
+#
+# Unlike peg-markdown, this set of literals recognizes Unicode alphanumeric
+# characters, newlines and spaces.
+class RDoc::Markdown::Literals
+  # :stopdoc:
+
+    # This is distinct from setup_parser so that a standalone parser
+    # can redefine #initialize and still have access to the proper
+    # parser setup code.
+    def initialize(str, debug=false)
+      setup_parser(str, debug)
+    end
+
+
+
+    # Prepares for parsing +str+.  If you define a custom initialize you must
+    # call this method before #parse
+    def setup_parser(str, debug=false)
+      set_string str, 0
+      @memoizations = Hash.new { |h,k| h[k] = {} }
+      @result = nil
+      @failed_rule = nil
+      @failing_rule_offset = -1
+
+      setup_foreign_grammar
+    end
+
+    attr_reader :string
+    attr_reader :failing_rule_offset
+    attr_accessor :result, :pos
+
+    def current_column(target=pos)
+      if c = string.rindex("\n", target-1)
+        return target - c - 1
+      end
+
+      target + 1
+    end
+
+    def current_line(target=pos)
+      cur_offset = 0
+      cur_line = 0
+
+      string.each_line do |line|
+        cur_line += 1
+        cur_offset += line.size
+        return cur_line if cur_offset >= target
+      end
+
+      -1
+    end
+
+    def lines
+      lines = []
+      string.each_line { |l| lines << l }
+      lines
+    end
+
+
+
+    def get_text(start)
+      @string[start..@pos-1]
+    end
+
+    # Sets the string and current parsing position for the parser.
+    def set_string string, pos
+      @string = string
+      @string_size = string ? string.size : 0
+      @pos = pos
+    end
+
+    def show_pos
+      width = 10
+      if @pos < width
+        "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")"
+      else
+        "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")"
+      end
+    end
+
+    def failure_info
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'"
+      else
+        "line #{l}, column #{c}: failed rule '#{@failed_rule}'"
+      end
+    end
+
+    def failure_caret
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      line = lines[l-1]
+      "#{line}\n#{' ' * (c - 1)}^"
+    end
+
+    def failure_character
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+      lines[l-1][c-1, 1]
+    end
+
+    def failure_oneline
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      char = lines[l-1][c-1, 1]
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'"
+      else
+        "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'"
+      end
+    end
+
+    class ParseError < RuntimeError
+    end
+
+    def raise_error
+      raise ParseError, failure_oneline
+    end
+
+    def show_error(io=STDOUT)
+      error_pos = @failing_rule_offset
+      line_no = current_line(error_pos)
+      col_no = current_column(error_pos)
+
+      io.puts "On line #{line_no}, column #{col_no}:"
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')"
+      else
+        io.puts "Failed to match rule '#{@failed_rule}'"
+      end
+
+      io.puts "Got: #{string[error_pos,1].inspect}"
+      line = lines[line_no-1]
+      io.puts "=> #{line}"
+      io.print(" " * (col_no + 3))
+      io.puts "^"
+    end
+
+    def set_failed_rule(name)
+      if @pos > @failing_rule_offset
+        @failed_rule = name
+        @failing_rule_offset = @pos
+      end
+    end
+
+    attr_reader :failed_rule
+
+    def match_string(str)
+      len = str.size
+      if @string[pos,len] == str
+        @pos += len
+        return str
+      end
+
+      return nil
+    end
+
+    def scan(reg)
+      if m = reg.match(@string[@pos..-1])
+        width = m.end(0)
+        @pos += width
+        return true
+      end
+
+      return nil
+    end
+
+    if "".respond_to? :ord
+      def get_byte
+        if @pos >= @string_size
+          return nil
+        end
+
+        s = @string[@pos].ord
+        @pos += 1
+        s
+      end
+    else
+      def get_byte
+        if @pos >= @string_size
+          return nil
+        end
+
+        s = @string[@pos]
+        @pos += 1
+        s
+      end
+    end
+
+    def parse(rule=nil)
+      # We invoke the rules indirectly via apply
+      # instead of by just calling them as methods because
+      # if the rules use left recursion, apply needs to
+      # manage that.
+
+      if !rule
+        apply(:_root)
+      else
+        method = rule.gsub("-","_hyphen_")
+        apply :"_#{method}"
+      end
+    end
+
+    class MemoEntry
+      def initialize(ans, pos)
+        @ans = ans
+        @pos = pos
+        @result = nil
+        @set = false
+        @left_rec = false
+      end
+
+      attr_reader :ans, :pos, :result, :set
+      attr_accessor :left_rec
+
+      def move!(ans, pos, result)
+        @ans = ans
+        @pos = pos
+        @result = result
+        @set = true
+        @left_rec = false
+      end
+    end
+
+    def external_invoke(other, rule, *args)
+      old_pos = @pos
+      old_string = @string
+
+      set_string other.string, other.pos
+
+      begin
+        if val = __send__(rule, *args)
+          other.pos = @pos
+          other.result = @result
+        else
+          other.set_failed_rule "#{self.class}##{rule}"
+        end
+        val
+      ensure
+        set_string old_string, old_pos
+      end
+    end
+
+    def apply_with_args(rule, *args)
+      memo_key = [rule, args]
+      if m = @memoizations[memo_key][@pos]
+        @pos = m.pos
+        if !m.set
+          m.left_rec = true
+          return nil
+        end
+
+        @result = m.result
+
+        return m.ans
+      else
+        m = MemoEntry.new(nil, @pos)
+        @memoizations[memo_key][@pos] = m
+        start_pos = @pos
+
+        ans = __send__ rule, *args
+
+        lr = m.left_rec
+
+        m.move! ans, @pos, @result
+
+        # Don't bother trying to grow the left recursion
+        # if it's failing straight away (thus there is no seed)
+        if ans and lr
+          return grow_lr(rule, args, start_pos, m)
+        else
+          return ans
+        end
+      end
+    end
+
+    def apply(rule)
+      if m = @memoizations[rule][@pos]
+        @pos = m.pos
+        if !m.set
+          m.left_rec = true
+          return nil
+        end
+
+        @result = m.result
+
+        return m.ans
+      else
+        m = MemoEntry.new(nil, @pos)
+        @memoizations[rule][@pos] = m
+        start_pos = @pos
+
+        ans = __send__ rule
+
+        lr = m.left_rec
+
+        m.move! ans, @pos, @result
+
+        # Don't bother trying to grow the left recursion
+        # if it's failing straight away (thus there is no seed)
+        if ans and lr
+          return grow_lr(rule, nil, start_pos, m)
+        else
+          return ans
+        end
+      end
+    end
+
+    def grow_lr(rule, args, start_pos, m)
+      while true
+        @pos = start_pos
+        @result = m.result
+
+        if args
+          ans = __send__ rule, *args
+        else
+          ans = __send__ rule
+        end
+        return nil unless ans
+
+        break if @pos <= m.pos
+
+        m.move! ans, @pos, @result
+      end
+
+      @result = m.result
+      @pos = m.pos
+      return m.ans
+    end
+
+    class RuleInfo
+      def initialize(name, rendered)
+        @name = name
+        @rendered = rendered
+      end
+
+      attr_reader :name, :rendered
+    end
+
+    def self.rule_info(name, rendered)
+      RuleInfo.new(name, rendered)
+    end
+
+
+  # :startdoc:
+  # :stopdoc:
+  def setup_foreign_grammar; end
+
+  # Alphanumeric = /\p{Word}/
+  def _Alphanumeric
+    _tmp = scan(/\A(?-mix:\p{Word})/)
+    set_failed_rule :_Alphanumeric unless _tmp
+    return _tmp
+  end
+
+  # AlphanumericAscii = /[A-Za-z0-9]/
+  def _AlphanumericAscii
+    _tmp = scan(/\A(?-mix:[A-Za-z0-9])/)
+    set_failed_rule :_AlphanumericAscii unless _tmp
+    return _tmp
+  end
+
+  # BOM = "uFEFF"
+  def _BOM
+    _tmp = match_string("uFEFF")
+    set_failed_rule :_BOM unless _tmp
+    return _tmp
+  end
+
+  # Newline = /\n|\r\n?|\p{Zl}|\p{Zp}/
+  def _Newline
+    _tmp = scan(/\A(?-mix:\n|\r\n?|\p{Zl}|\p{Zp})/)
+    set_failed_rule :_Newline unless _tmp
+    return _tmp
+  end
+
+  # NonAlphanumeric = /\p{^Word}/
+  def _NonAlphanumeric
+    _tmp = scan(/\A(?-mix:\p{^Word})/)
+    set_failed_rule :_NonAlphanumeric unless _tmp
+    return _tmp
+  end
+
+  # Spacechar = /\t|\p{Zs}/
+  def _Spacechar
+    _tmp = scan(/\A(?-mix:\t|\p{Zs})/)
+    set_failed_rule :_Spacechar unless _tmp
+    return _tmp
+  end
+
+  Rules = {}
+  Rules[:_Alphanumeric] = rule_info("Alphanumeric", "/\\p{Word}/")
+  Rules[:_AlphanumericAscii] = rule_info("AlphanumericAscii", "/[A-Za-z0-9]/")
+  Rules[:_BOM] = rule_info("BOM", "\"uFEFF\"")
+  Rules[:_Newline] = rule_info("Newline", "/\\n|\\r\\n?|\\p{Zl}|\\p{Zp}/")
+  Rules[:_NonAlphanumeric] = rule_info("NonAlphanumeric", "/\\p{^Word}/")
+  Rules[:_Spacechar] = rule_info("Spacechar", "/\\t|\\p{Zs}/")
+  # :startdoc:
+end

data/lib/rdoc/markup.rb

@@ -849,8 +849,6 @@ https://github.com/ruby/rdoc/issues
 
   # Formatters
   autoload :Formatter,             'rdoc/markup/formatter'
-  autoload :FormatterTestCase,     'rdoc/markup/formatter_test_case'
-  autoload :TextFormatterTestCase, 'rdoc/markup/text_formatter_test_case'
 
   autoload :ToAnsi,                'rdoc/markup/to_ansi'
   autoload :ToBs,                  'rdoc/markup/to_bs'

data/lib/rdoc/markup/formatter.rb

@@ -156,7 +156,7 @@ class RDoc::Markup::Formatter
       method_name = "handle_regexp_#{name}"
 
       if respond_to? method_name then
-        target.text = send method_name, target
+        target.text = public_send method_name, target
         handled = true
       end
     end

data/lib/rdoc/markup/parser.rb

@@ -80,10 +80,6 @@ class RDoc::Markup::Parser
     @binary_input   = nil
     @current_token  = nil
     @debug          = false
-    @input          = nil
-    @input_encoding = nil
-    @line           = 0
-    @line_pos       = 0
     @s              = nil
     @tokens         = []
   end
@@ -319,13 +315,6 @@ class RDoc::Markup::Parser
     verbatim
   end
 
-  ##
-  # The character offset for the input string at the given +byte_offset+
-
-  def char_pos byte_offset
-    @input.byteslice(0, byte_offset).length
-  end
-
   ##
   # Pulls the next token from the stream.
 
@@ -424,15 +413,53 @@ class RDoc::Markup::Parser
     token
   end
 
+  ##
+  # A simple wrapper of StringScanner that is aware of the current column and lineno
+
+  class MyStringScanner
+    def initialize(input)
+      @line = @column = 0
+      @s = StringScanner.new input
+    end
+
+    def scan(re)
+      ret = @s.scan(re)
+      @column += ret.length if ret
+      ret
+    end
+
+    def unscan(s)
+      @s.pos -= s.bytesize
+      @column -= s.length
+    end
+
+    def pos
+      [@column, @line]
+    end
+
+    def newline!
+      @column = 0
+      @line += 1
+    end
+
+    def eos?
+      @s.eos?
+    end
+
+    def matched
+      @s.matched
+    end
+
+    def [](i)
+      @s[i]
+    end
+  end
+
   ##
   # Creates the StringScanner
 
   def setup_scanner input
-    @line     = 0
-    @line_pos = 0
-    @input    = input.dup
-
-    @s = StringScanner.new input
+    @s = MyStringScanner.new input
   end
 
   ##
@@ -467,31 +494,30 @@ class RDoc::Markup::Parser
       @tokens << case
                  # [CR]LF => :NEWLINE
                  when @s.scan(/\r?\n/) then
-                   token = [:NEWLINE, @s.matched, *token_pos(pos)]
-                   @line_pos = char_pos @s.pos
-                   @line += 1
+                   token = [:NEWLINE, @s.matched, *pos]
+                   @s.newline!
                    token
                  # === text => :HEADER then :TEXT
                  when @s.scan(/(=+)(\s*)/) then
                    level = @s[1].length
-                   header = [:HEADER, level, *token_pos(pos)]
+                   header = [:HEADER, level, *pos]
 
                    if @s[2] =~ /^\r?\n/ then
-                     @s.pos -= @s[2].length
+                     @s.unscan(@s[2])
                      header
                    else
                      pos = @s.pos
                      @s.scan(/.*/)
                      @tokens << header
-                     [:TEXT, @s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                     [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
                    end
                  # --- (at least 3) and nothing else on the line => :RULE
                  when @s.scan(/(-{3,}) *\r?$/) then
-                   [:RULE, @s[1].length - 2, *token_pos(pos)]
+                   [:RULE, @s[1].length - 2, *pos]
                  # * or - followed by white space and text => :BULLET
                  when @s.scan(/([*-]) +(\S)/) then
-                   @s.pos -= @s[2].bytesize # unget \S
-                   [:BULLET, @s[1], *token_pos(pos)]
+                   @s.unscan(@s[2])
+                   [:BULLET, @s[1], *pos]
                  # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
                  when @s.scan(/([a-z]|\d+)\. +(\S)/i) then
                    # FIXME if tab(s), the column will be wrong
@@ -500,7 +526,7 @@ class RDoc::Markup::Parser
                    # before (and provide a check for that at least in debug
                    # mode)
                    list_label = @s[1]
-                   @s.pos -= @s[2].bytesize # unget \S
+                   @s.unscan(@s[2])
                    list_type =
                      case list_label
                      when /[a-z]/ then :LALPHA
@@ -509,24 +535,24 @@ class RDoc::Markup::Parser
                      else
                        raise ParseError, "BUG token #{list_label}"
                      end
-                   [list_type, list_label, *token_pos(pos)]
+                   [list_type, list_label, *pos]
                  # [text] followed by spaces or end of line => :LABEL
                  when @s.scan(/\[(.*?)\]( +|\r?$)/) then
-                   [:LABEL, @s[1], *token_pos(pos)]
+                   [:LABEL, @s[1], *pos]
                  # text:: followed by spaces or end of line => :NOTE
                  when @s.scan(/(.*?)::( +|\r?$)/) then
-                   [:NOTE, @s[1], *token_pos(pos)]
+                   [:NOTE, @s[1], *pos]
                  # >>> followed by end of line => :BLOCKQUOTE
                  when @s.scan(/>>> *(\w+)?$/) then
-                   [:BLOCKQUOTE, @s[1], *token_pos(pos)]
+                   [:BLOCKQUOTE, @s[1], *pos]
                  # anything else: :TEXT
                  else
                    @s.scan(/(.*?)(  )?\r?$/)
-                   token = [:TEXT, @s[1], *token_pos(pos)]
+                   token = [:TEXT, @s[1], *pos]
 
                    if @s[2] then
                      @tokens << token
-                     [:BREAK, @s[2], *token_pos(pos + @s[1].length)]
+                     [:BREAK, @s[2], pos[0] + @s[1].length, pos[1]]
                    else
                      token
                    end
@@ -536,16 +562,6 @@ class RDoc::Markup::Parser
     self
   end
 
-  ##
-  # Calculates the column (by character) and line of the current token based
-  # on +byte_offset+.
-
-  def token_pos byte_offset
-    offset = char_pos byte_offset
-
-    [offset - @line_pos, @line]
-  end
-
   ##
   # Returns the current token to the token stream