rdoc 3.12.2 → 4.0.0.preview2

data/lib/rdoc/markup/hard_break.rb

@@ -0,0 +1,31 @@
+##
+# A hard-break in the middle of a paragraph.
+
+class RDoc::Markup::HardBreak
+
+  @instance = new
+
+  ##
+  # RDoc::Markup::HardBreak is a singleton
+
+  def self.new
+    @instance
+  end
+
+  ##
+  # Calls #accept_hard_break on +visitor+
+
+  def accept visitor
+    visitor.accept_hard_break self
+  end
+
+  def == other # :nodoc:
+    self.class === other
+  end
+
+  def pretty_print q # :nodoc:
+    q.text "[break]"
+  end
+
+end
+

data/lib/rdoc/markup/heading.rb

@@ -23,7 +23,7 @@ class RDoc::Markup::Heading < Struct.new :level, :text
     markup = RDoc::Markup.new
     markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
 
-    @to_html = RDoc::Markup::ToHtml.new
+    @to_html = RDoc::Markup::ToHtml.new nil
 
     def @to_html.handle_special_CROSSREF special
       special.text.sub(/^\\/, '')

data/lib/rdoc/markup/indented_paragraph.rb

@@ -29,5 +29,19 @@ class RDoc::Markup::IndentedParagraph < RDoc::Markup::Raw
     visitor.accept_indented_paragraph self
   end
 
+  ##
+  # Joins the raw paragraph text and converts inline HardBreaks to the
+  # +hard_break+ text followed by the indent.
+
+  def text hard_break = nil
+    @parts.map do |part|
+      if RDoc::Markup::HardBreak === part then
+        '%1$s%3$*2$s' % [hard_break, @indent, ' '] if hard_break
+      else
+        part
+      end
+    end.join
+  end
+
 end
 

data/lib/rdoc/markup/list.rb

@@ -1,5 +1,24 @@
 ##
-# A List of ListItems
+# A List is a homogeneous set of ListItems.
+#
+# The supported list types include:
+#
+# :BULLET::
+#   An unordered list
+# :LABEL::
+#   An unordered definition list, but using an alternate RDoc::Markup syntax
+# :LALPHA::
+#   An ordered list using increasing lowercase English letters
+# :NOTE::
+#   An unordered definition list
+# :NUMBER::
+#   An ordered list using increasing Arabic numerals
+# :UALPHA::
+#   An ordered list using increasing uppercase English letters
+#
+# Definition lists behave like HTML definition lists.  Each list item can
+# describe multiple terms.  See RDoc::Markup::ListItem for how labels and
+# definition are stored as list items.
 
 class RDoc::Markup::List
 
@@ -15,12 +34,12 @@ class RDoc::Markup::List
 
   ##
   # Creates a new list of +type+ with +items+.  Valid list types are:
-  # +:BULLET+, +:LABEL+, +:LALPHA+, +:NOTE+, +:NUMBER+, +:UALPHA+ 
+  # +:BULLET+, +:LABEL+, +:LALPHA+, +:NOTE+, +:NUMBER+, +:UALPHA+
 
   def initialize type = nil, *items
     @type = type
     @items = []
-    @items.push(*items)
+    @items.concat items
   end
 
   ##
@@ -75,7 +94,7 @@ class RDoc::Markup::List
   # Appends +items+ to the list
 
   def push *items
-    @items.push(*items)
+    @items.concat items
   end
 
 end

data/lib/rdoc/markup/list_item.rb

@@ -1,5 +1,12 @@
 ##
 # An item within a List that contains paragraphs, headings, etc.
+#
+# For BULLET, NUMBER, LALPHA and UALPHA lists, the label will always be nil.
+# For NOTE and LABEL lists, the list label may contain:
+#
+# * a single String for a single label
+# * an Array of Strings for a list item with multiple terms
+# * nil for an extra description attached to a previously labeled list item
 
 class RDoc::Markup::ListItem
 
@@ -19,7 +26,7 @@ class RDoc::Markup::ListItem
   def initialize label = nil, *parts
     @label = label
     @parts = []
-    @parts.push(*parts)
+    @parts.concat parts
   end
 
   ##
@@ -64,8 +71,14 @@ class RDoc::Markup::ListItem
 
   def pretty_print q # :nodoc:
     q.group 2, '[item: ', ']' do
-      if @label then
-        q.text @label
+      case @label
+      when Array then
+        q.pp @label
+        q.text ';'
+        q.breakable
+      when String then
+        q.pp @label
+        q.text ';'
         q.breakable
       end
 
@@ -79,7 +92,7 @@ class RDoc::Markup::ListItem
   # Adds +parts+ to the ListItem
 
   def push *parts
-    @parts.push(*parts)
+    @parts.concat parts
   end
 
 end

data/lib/rdoc/markup/paragraph.rb

@@ -10,5 +10,19 @@ class RDoc::Markup::Paragraph < RDoc::Markup::Raw
     visitor.accept_paragraph self
   end
 
+  ##
+  # Joins the raw paragraph text and converts inline HardBreaks to the
+  # +hard_break+ text.
+
+  def text hard_break = ''
+    @parts.map do |part|
+      if RDoc::Markup::HardBreak === part then
+        hard_break
+      else
+        part
+      end
+    end.join
+  end
+
 end
 

data/lib/rdoc/markup/parser.rb

@@ -75,12 +75,17 @@ class RDoc::Markup::Parser
   # Creates a new Parser.  See also ::parse
 
   def initialize
-    @tokens = []
-    @current_token = nil
-    @debug = false
-
-    @line = 0
-    @line_pos = 0
+    @binary_input   = nil
+    @current_token  = nil
+    @debug          = false
+    @have_encoding  = Object.const_defined? :Encoding
+    @have_byteslice = ''.respond_to? :byteslice
+    @input          = nil
+    @input_encoding = nil
+    @line           = 0
+    @line_pos       = 0
+    @s              = nil
+    @tokens         = []
   end
 
   ##
@@ -108,13 +113,13 @@ class RDoc::Markup::Parser
     p :list_start => margin if @debug
 
     list = RDoc::Markup::List.new
+    label = nil
 
     until @tokens.empty? do
       type, data, column, = get
 
       case type
-      when :BULLET, :LABEL, :LALPHA, :NOTE, :NUMBER, :UALPHA then
-
+      when *LIST_TOKENS then
         if column < margin || (list.type && list.type != type) then
           unget
           break
@@ -125,6 +130,8 @@ class RDoc::Markup::Parser
 
         case type
         when :NOTE, :LABEL then
+          label = [] unless label
+
           if peek_type == :NEWLINE then
             # description not on the same line as LABEL/NOTE
             # skip the trailing newline & any blank lines below
@@ -147,32 +154,35 @@ class RDoc::Markup::Parser
             # In all cases, we have an empty description.
             # In the last case only, we continue.
             if peek_type.nil? || column < margin then
-              empty = 1
+              empty = true
             elsif column == margin then
               case peek_type
               when type
-                empty = 2 # continue
+                empty = :continue
               when *LIST_TOKENS
-                empty = 1
+                empty = true
               else
-                empty = 0
+                empty = false
               end
             else
-              empty = 0
+              empty = false
             end
 
-            if empty > 0 then
-              item = RDoc::Markup::ListItem.new(data)
-              item << RDoc::Markup::BlankLine.new
-              list << item
-              break if empty == 1
-              next
+            if empty then
+              label << data
+              next if empty == :continue
+              break
             end
           end
         else
           data = nil
         end
 
+        if label then
+          data = label << data
+          label = nil
+        end
+
         list_item = RDoc::Markup::ListItem.new data
         parse list_item, column
         list << list_item
@@ -185,7 +195,13 @@ class RDoc::Markup::Parser
 
     p :list_end => margin if @debug
 
-    return nil if list.empty?
+    if list.empty? then
+      return nil unless label
+      return nil unless [:LABEL, :NOTE].include? list.type
+
+      list_item = RDoc::Markup::ListItem.new label, RDoc::Markup::BlankLine.new
+      list << list_item
+    end
 
     list
   end
@@ -201,20 +217,20 @@ class RDoc::Markup::Parser
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :TEXT && column == margin then
+      if type == :TEXT and column == margin then
         paragraph << data
 
-        if peek_token[0] == :BREAK then
-          break
-        end
+        break if peek_token.first == :BREAK
 
-        skip :NEWLINE
+        data << ' ' if skip :NEWLINE
       else
         unget
         break
       end
     end
 
+    paragraph.parts.last.sub!(/ \z/, '') # cleanup
+
     p :paragraph_end => margin if @debug
 
     paragraph
@@ -303,6 +319,21 @@ class RDoc::Markup::Parser
     verbatim
   end
 
+  ##
+  # The character offset for the input string at the given +byte_offset+
+
+  def char_pos byte_offset
+    if @have_byteslice then
+      @input.byteslice(0, byte_offset).length
+    elsif @have_encoding then
+      matched = @binary_input[0, byte_offset]
+      matched.force_encoding @input_encoding
+      matched.length
+    else
+      byte_offset
+    end
+  end
+
   ##
   # Pulls the next token from the stream.
 
@@ -383,6 +414,22 @@ class RDoc::Markup::Parser
     token
   end
 
+  ##
+  # Creates the StringScanner
+
+  def setup_scanner input
+    @line     = 0
+    @line_pos = 0
+    @input    = input.dup
+
+    if @have_encoding and not @have_byteslice then
+      @input_encoding = @input.encoding
+      @binary_input   = @input.force_encoding Encoding::BINARY
+    end
+
+    @s = StringScanner.new input
+  end
+
   ##
   # Skips the next token if its type is +token_type+.
   #
@@ -400,58 +447,55 @@ class RDoc::Markup::Parser
   # Turns text +input+ into a stream of tokens
 
   def tokenize input
-    s = StringScanner.new input
+    setup_scanner input
 
-    @line = 0
-    @line_pos = 0
-
-    until s.eos? do
-      pos = s.pos
+    until @s.eos? do
+      pos = @s.pos
 
       # leading spaces will be reflected by the column of the next token
       # the only thing we loose are trailing spaces at the end of the file
-      next if s.scan(/ +/)
+      next if @s.scan(/ +/)
 
       # note: after BULLET, LABEL, etc.,
       # indent will be the column of the next non-newline token
 
       @tokens << case
                  # [CR]LF => :NEWLINE
-                 when s.scan(/\r?\n/) then
-                   token = [:NEWLINE, s.matched, *token_pos(pos)]
-                   @line_pos = s.pos
+                 when @s.scan(/\r?\n/) then
+                   token = [:NEWLINE, @s.matched, *token_pos(pos)]
+                   @line_pos = char_pos @s.pos
                    @line += 1
                    token
                  # === text => :HEADER then :TEXT
-                 when s.scan(/(=+)(\s*)/) then
-                   level = s[1].length
+                 when @s.scan(/(=+)(\s*)/) then
+                   level = @s[1].length
                    header = [:HEADER, level, *token_pos(pos)]
 
-                   if s[2] =~ /^\r?\n/ then
-                     s.pos -= s[2].length
+                   if @s[2] =~ /^\r?\n/ then
+                     @s.pos -= @s[2].length
                      header
                    else
-                     pos = s.pos
-                     s.scan(/.*/)
+                     pos = @s.pos
+                     @s.scan(/.*/)
                      @tokens << header
-                     [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                     [:TEXT, @s.matched.sub(/\r$/, ''), *token_pos(pos)]
                    end
                  # --- (at least 3) and nothing else on the line => :RULE
-                 when s.scan(/(-{3,}) *\r?$/) then
-                   [:RULE, s[1].length - 2, *token_pos(pos)]
+                 when @s.scan(/(-{3,}) *\r?$/) then
+                   [:RULE, @s[1].length - 2, *token_pos(pos)]
                  # * or - followed by white space and text => :BULLET
-                 when s.scan(/([*-]) +(\S)/) then
-                   s.pos -= s[2].bytesize # unget \S
-                   [:BULLET, s[1], *token_pos(pos)]
+                 when @s.scan(/([*-]) +(\S)/) then
+                   @s.pos -= @s[2].bytesize # unget \S
+                   [:BULLET, @s[1], *token_pos(pos)]
                  # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
-                 when s.scan(/([a-z]|\d+)\. +(\S)/i) then
+                 when @s.scan(/([a-z]|\d+)\. +(\S)/i) then
                    # FIXME if tab(s), the column will be wrong
                    # either support tabs everywhere by first expanding them to
                    # spaces, or assume that they will have been replaced
                    # before (and provide a check for that at least in debug
                    # mode)
-                   list_label = s[1]
-                   s.pos -= s[2].bytesize # unget \S
+                   list_label = @s[1]
+                   @s.pos -= @s[2].bytesize # unget \S
                    list_type =
                      case list_label
                      when /[a-z]/ then :LALPHA
@@ -462,18 +506,18 @@ class RDoc::Markup::Parser
                      end
                    [list_type, list_label, *token_pos(pos)]
                  # [text] followed by spaces or end of line => :LABEL
-                 when s.scan(/\[(.*?)\]( +|\r?$)/) then
-                   [:LABEL, s[1], *token_pos(pos)]
+                 when @s.scan(/\[(.*?)\]( +|\r?$)/) then
+                   [:LABEL, @s[1], *token_pos(pos)]
                  # text:: followed by spaces or end of line => :NOTE
-                 when s.scan(/(.*?)::( +|\r?$)/) then
-                   [:NOTE, s[1], *token_pos(pos)]
+                 when @s.scan(/(.*?)::( +|\r?$)/) then
+                   [:NOTE, @s[1], *token_pos(pos)]
                  # anything else: :TEXT
-                 else s.scan(/(.*?)(  )?\r?$/)
-                   token = [:TEXT, s[1], *token_pos(pos)]
+                 else @s.scan(/(.*?)(  )?\r?$/)
+                   token = [:TEXT, @s[1], *token_pos(pos)]
 
-                   if s[2] then
+                   if @s[2] then
                      @tokens << token
-                     [:BREAK, s[2], *token_pos(pos + s[1].length)]
+                     [:BREAK, @s[2], *token_pos(pos + @s[1].length)]
                    else
                      token
                    end
@@ -484,9 +528,12 @@ class RDoc::Markup::Parser
   end
 
   ##
-  # Calculates the column and line of the current token based on +offset+.
+  # Calculates the column (by character) and line of the current token from
+  # +scanner+ based on +byte_offset+.
+
+  def token_pos byte_offset
+    offset = char_pos byte_offset
 
-  def token_pos offset
     [offset - @line_pos, @line]
   end