rdoc 3.1 → 6.3.3

data/lib/rdoc/markup/parser.rb

@@ -1,5 +1,5 @@
+# frozen_string_literal: true
 require 'strscan'
-require 'rdoc/text'
 
 ##
 # A recursive-descent parser for RDoc markup.
@@ -9,8 +9,9 @@ require 'rdoc/text'
 # RDoc::Markup::ToHTML.
 #
 # The parser only handles the block-level constructs Paragraph, List,
-# ListItem, Heading, Verbatim, BlankLine and Rule.  Inline markup such as
-# <tt>\+blah\+</tt> is handled separately by RDoc::Markup::AttributeManager.
+# ListItem, Heading, Verbatim, BlankLine, Rule and BlockQuote.
+# Inline markup such as <tt>\+blah\+</tt> is handled separately by
+# RDoc::Markup::AttributeManager.
 #
 # To see what markup the Parser implements read RDoc.  To see how to use
 # RDoc markup to format text in your program read RDoc::Markup.
@@ -52,7 +53,9 @@ class RDoc::Markup::Parser
   attr_reader :tokens
 
   ##
-  # Parses +str+ into a Document
+  # Parses +str+ into a Document.
+  #
+  # Use RDoc::Markup#parse instead of this method.
 
   def self.parse str
     parser = new
@@ -74,23 +77,29 @@ class RDoc::Markup::Parser
   # Creates a new Parser.  See also ::parse
 
   def initialize
-    @tokens = []
-    @current_token = nil
-    @debug = false
-
-    @line = 0
-    @line_pos = 0
+    @binary_input   = nil
+    @current_token  = nil
+    @debug          = false
+    @s              = nil
+    @tokens         = []
   end
 
   ##
   # Builds a Heading of +level+
 
   def build_heading level
-    _, text, = get  # TEXT
-    heading = RDoc::Markup::Heading.new level, text
-    skip :NEWLINE
-
-    heading
+    type, text, = get
+
+    text = case type
+           when :TEXT then
+             skip :NEWLINE
+             text
+           else
+             unget
+             ''
+           end
+
+    RDoc::Markup::Heading.new level, text
   end
 
   ##
@@ -100,13 +109,13 @@ class RDoc::Markup::Parser
     p :list_start => margin if @debug
 
     list = RDoc::Markup::List.new
+    label = nil
 
     until @tokens.empty? do
       type, data, column, = get
 
       case type
-      when :BULLET, :LABEL, :LALPHA, :NOTE, :NUMBER, :UALPHA then
-
+      when *LIST_TOKENS then
         if column < margin || (list.type && list.type != type) then
           unget
           break
@@ -117,6 +126,8 @@ class RDoc::Markup::Parser
 
         case type
         when :NOTE, :LABEL then
+          label = [] unless label
+
           if peek_type == :NEWLINE then
             # description not on the same line as LABEL/NOTE
             # skip the trailing newline & any blank lines below
@@ -139,32 +150,35 @@ class RDoc::Markup::Parser
             # In all cases, we have an empty description.
             # In the last case only, we continue.
             if peek_type.nil? || column < margin then
-              empty = 1
+              empty = true
             elsif column == margin then
               case peek_type
               when type
-                empty = 2 # continue
+                empty = :continue
               when *LIST_TOKENS
-                empty = 1
+                empty = true
               else
-                empty = 0
+                empty = false
               end
             else
-              empty = 0
+              empty = false
             end
 
-            if empty > 0 then
-              item = RDoc::Markup::ListItem.new(data)
-              item << RDoc::Markup::BlankLine.new
-              list << item
-              break if empty == 1
-              next
+            if empty then
+              label << data
+              next if empty == :continue
+              break
             end
           end
         else
           data = nil
         end
 
+        if label then
+          data = label << data
+          label = nil
+        end
+
         list_item = RDoc::Markup::ListItem.new data
         parse list_item, column
         list << list_item
@@ -177,7 +191,13 @@ class RDoc::Markup::Parser
 
     p :list_end => margin if @debug
 
-    return nil if list.empty?
+    if list.empty? then
+      return nil unless label
+      return nil unless [:LABEL, :NOTE].include? list.type
+
+      list_item = RDoc::Markup::ListItem.new label, RDoc::Markup::BlankLine.new
+      list << list_item
+    end
 
     list
   end
@@ -193,15 +213,20 @@ class RDoc::Markup::Parser
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :TEXT && column == margin then
+      if type == :TEXT and column == margin then
         paragraph << data
-        skip :NEWLINE
+
+        break if peek_token.first == :BREAK
+
+        data << ' ' if skip :NEWLINE
       else
         unget
         break
       end
     end
 
+    paragraph.parts.last.sub!(/ \z/, '') # cleanup
+
     p :paragraph_end => margin if @debug
 
     paragraph
@@ -221,7 +246,7 @@ class RDoc::Markup::Parser
 
     min_indent = nil
     generate_leading_spaces = true
-    line = ''
+    line = ''.dup
 
     until @tokens.empty? do
       type, data, column, = get
@@ -229,7 +254,7 @@ class RDoc::Markup::Parser
       if type == :NEWLINE then
         line << data
         verbatim << line
-        line = ''
+        line = ''.dup
         generate_leading_spaces = true
         next
       end
@@ -260,7 +285,7 @@ class RDoc::Markup::Parser
         peek_column ||= column + width
         indent = peek_column - column - width
         line << ' ' * indent
-      when :TEXT then
+      when :BREAK, :TEXT then
         line << data
       else # *LIST_TOKENS
         list_marker = case type
@@ -314,14 +339,19 @@ class RDoc::Markup::Parser
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :NEWLINE then
+      case type
+      when :BREAK then
+        parent << RDoc::Markup::BlankLine.new
+        skip :NEWLINE, false
+        next
+      when :NEWLINE then
         # trailing newlines are skipped below, so this is a blank line
         parent << RDoc::Markup::BlankLine.new
         skip :NEWLINE, false
         next
       end
 
-      # indentation change: break or verbattim
+      # indentation change: break or verbatim
       if column < indent then
         unget
         break
@@ -340,7 +370,18 @@ class RDoc::Markup::Parser
         skip :NEWLINE
       when :TEXT then
         unget
-        parent << build_paragraph(indent)
+        parse_text parent, indent
+      when :BLOCKQUOTE then
+        type, _, column = get
+        if type == :NEWLINE
+          type, _, column = get
+        end
+        unget if type
+        bq = RDoc::Markup::BlockQuote.new
+        p :blockquote_start => [data, column] if @debug
+        parse bq, column
+        p :blockquote_end => indent if @debug
+        parent << bq
       when *LIST_TOKENS then
         unget
         parent << build_list(indent)
@@ -356,6 +397,13 @@ class RDoc::Markup::Parser
 
   end
 
+  ##
+  # Small hook that is overridden by RDoc::TomDoc
+
+  def parse_text parent, indent # :nodoc:
+    parent << build_paragraph(indent)
+  end
+
   ##
   # Returns the next token on the stream without modifying the stream
 
@@ -365,6 +413,55 @@ class RDoc::Markup::Parser
     token
   end
 
+  ##
+  # A simple wrapper of StringScanner that is aware of the current column and lineno
+
+  class MyStringScanner
+    def initialize(input)
+      @line = @column = 0
+      @s = StringScanner.new input
+    end
+
+    def scan(re)
+      ret = @s.scan(re)
+      @column += ret.length if ret
+      ret
+    end
+
+    def unscan(s)
+      @s.pos -= s.bytesize
+      @column -= s.length
+    end
+
+    def pos
+      [@column, @line]
+    end
+
+    def newline!
+      @column = 0
+      @line += 1
+    end
+
+    def eos?
+      @s.eos?
+    end
+
+    def matched
+      @s.matched
+    end
+
+    def [](i)
+      @s[i]
+    end
+  end
+
+  ##
+  # Creates the StringScanner
+
+  def setup_scanner input
+    @s = MyStringScanner.new input
+  end
+
   ##
   # Skips the next token if its type is +token_type+.
   #
@@ -382,52 +479,54 @@ class RDoc::Markup::Parser
   # Turns text +input+ into a stream of tokens
 
   def tokenize input
-    s = StringScanner.new input
-
-    @line = 0
-    @line_pos = 0
+    setup_scanner input
 
-    until s.eos? do
-      pos = s.pos
+    until @s.eos? do
+      pos = @s.pos
 
       # leading spaces will be reflected by the column of the next token
       # the only thing we loose are trailing spaces at the end of the file
-      next if s.scan(/ +/)
+      next if @s.scan(/ +/)
 
       # note: after BULLET, LABEL, etc.,
       # indent will be the column of the next non-newline token
 
       @tokens << case
                  # [CR]LF => :NEWLINE
-                 when s.scan(/\r?\n/) then
-                   token = [:NEWLINE, s.matched, *token_pos(pos)]
-                   @line_pos = s.pos
-                   @line += 1
+                 when @s.scan(/\r?\n/) then
+                   token = [:NEWLINE, @s.matched, *pos]
+                   @s.newline!
                    token
                  # === text => :HEADER then :TEXT
-                 when s.scan(/(=+)\s*/) then
-                   level = s[1].length
-                   level = 6 if level > 6
-                   @tokens << [:HEADER, level, *token_pos(pos)]
-                   pos = s.pos
-                   s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                 when @s.scan(/(=+)(\s*)/) then
+                   level = @s[1].length
+                   header = [:HEADER, level, *pos]
+
+                   if @s[2] =~ /^\r?\n/ then
+                     @s.unscan(@s[2])
+                     header
+                   else
+                     pos = @s.pos
+                     @s.scan(/.*/)
+                     @tokens << header
+                     [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
+                   end
                  # --- (at least 3) and nothing else on the line => :RULE
-                 when s.scan(/(-{3,}) *$/) then
-                   [:RULE, s[1].length - 2, *token_pos(pos)]
+                 when @s.scan(/(-{3,}) *\r?$/) then
+                   [:RULE, @s[1].length - 2, *pos]
                  # * or - followed by white space and text => :BULLET
-                 when s.scan(/([*-]) +(\S)/) then
-                   s.pos -= s[2].bytesize # unget \S
-                   [:BULLET, s[1], *token_pos(pos)]
+                 when @s.scan(/([*-]) +(\S)/) then
+                   @s.unscan(@s[2])
+                   [:BULLET, @s[1], *pos]
                  # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
-                 when s.scan(/([a-z]|\d+)\. +(\S)/i) then
+                 when @s.scan(/([a-z]|\d+)\. +(\S)/i) then
                    # FIXME if tab(s), the column will be wrong
                    # either support tabs everywhere by first expanding them to
                    # spaces, or assume that they will have been replaced
                    # before (and provide a check for that at least in debug
                    # mode)
-                   list_label = s[1]
-                   s.pos -= s[2].bytesize # unget \S
+                   list_label = @s[1]
+                   @s.unscan(@s[2])
                    list_type =
                      case list_label
                      when /[a-z]/ then :LALPHA
@@ -436,29 +535,33 @@ class RDoc::Markup::Parser
                      else
                        raise ParseError, "BUG token #{list_label}"
                      end
-                   [list_type, list_label, *token_pos(pos)]
+                   [list_type, list_label, *pos]
                  # [text] followed by spaces or end of line => :LABEL
-                 when s.scan(/\[(.*?)\]( +|$)/) then
-                   [:LABEL, s[1], *token_pos(pos)]
+                 when @s.scan(/\[(.*?)\]( +|\r?$)/) then
+                   [:LABEL, @s[1], *pos]
                  # text:: followed by spaces or end of line => :NOTE
-                 when s.scan(/(.*?)::( +|$)/) then
-                   [:NOTE, s[1], *token_pos(pos)]
+                 when @s.scan(/(.*?)::( +|\r?$)/) then
+                   [:NOTE, @s[1], *pos]
+                 # >>> followed by end of line => :BLOCKQUOTE
+                 when @s.scan(/>>> *(\w+)?$/) then
+                   [:BLOCKQUOTE, @s[1], *pos]
                  # anything else: :TEXT
-                 else s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                 else
+                   @s.scan(/(.*?)(  )?\r?$/)
+                   token = [:TEXT, @s[1], *pos]
+
+                   if @s[2] then
+                     @tokens << token
+                     [:BREAK, @s[2], pos[0] + @s[1].length, pos[1]]
+                   else
+                     token
+                   end
                  end
     end
 
     self
   end
 
-  ##
-  # Calculates the column and line of the current token based on +offset+.
-
-  def token_pos offset
-    [offset - @line_pos, @line]
-  end
-
   ##
   # Returns the current token to the token stream
 
@@ -470,14 +573,3 @@ class RDoc::Markup::Parser
   end
 
 end
-
-require 'rdoc/markup/blank_line'
-require 'rdoc/markup/document'
-require 'rdoc/markup/heading'
-require 'rdoc/markup/list'
-require 'rdoc/markup/list_item'
-require 'rdoc/markup/raw'
-require 'rdoc/markup/paragraph'
-require 'rdoc/markup/rule'
-require 'rdoc/markup/verbatim'
-