rdoc 2.0.0

data/lib/rdoc/markup/attribute_manager.rb

@@ -0,0 +1,274 @@
+require 'rdoc/markup/inline'
+
+class RDoc::Markup::AttributeManager
+
+  NULL = "\000".freeze
+
+  ##
+  # We work by substituting non-printing characters in to the text. For now
+  # I'm assuming that I can substitute a character in the range 0..8 for a 7
+  # bit character without damaging the encoded string, but this might be
+  # optimistic
+
+  A_PROTECT  = 004
+  PROTECT_ATTR  = A_PROTECT.chr
+
+  ##
+  # This maps delimiters that occur around words (such as *bold* or +tt+)
+  # where the start and end delimiters and the same. This lets us optimize
+  # the regexp
+
+  MATCHING_WORD_PAIRS = {}
+
+  ##
+  # And this is used when the delimiters aren't the same. In this case the
+  # hash maps a pattern to the attribute character
+
+  WORD_PAIR_MAP = {}
+
+  ##
+  # This maps HTML tags to the corresponding attribute char
+
+  HTML_TAGS = {}
+
+  ##
+  # And this maps _special_ sequences to a name. A special sequence is
+  # something like a WikiWord
+
+  SPECIAL = {}
+
+  ##
+  # Return an attribute object with the given turn_on and turn_off bits set
+
+  def attribute(turn_on, turn_off)
+    RDoc::Markup::AttrChanger.new turn_on, turn_off
+  end
+
+  def change_attribute(current, new)
+    diff = current ^ new
+    attribute(new & diff, current & diff)
+  end
+
+  def changed_attribute_by_name(current_set, new_set)
+    current = new = 0
+    current_set.each do |name|
+      current |= RDoc::Markup::Attribute.bitmap_for(name)
+    end
+
+    new_set.each do |name|
+      new |= RDoc::Markup::Attribute.bitmap_for(name)
+    end
+
+    change_attribute(current, new)
+  end
+
+  def copy_string(start_pos, end_pos)
+    res = @str[start_pos...end_pos]
+    res.gsub!(/\000/, '')
+    res
+  end
+
+  ##
+  # Map attributes like <b>text</b>to the sequence
+  # \001\002<char>\001\003<char>, where <char> is a per-attribute specific
+  # character
+
+  def convert_attrs(str, attrs)
+    # first do matching ones
+    tags = MATCHING_WORD_PAIRS.keys.join("")
+
+    re = /(^|\W)([#{tags}])([#:\\]?[\w.\/-]+?\S?)\2(\W|$)/
+
+    1 while str.gsub!(re) do
+      attr = MATCHING_WORD_PAIRS[$2]
+      attrs.set_attrs($`.length + $1.length + $2.length, $3.length, attr)
+      $1 + NULL * $2.length + $3 + NULL * $2.length + $4
+    end
+
+    # then non-matching
+    unless WORD_PAIR_MAP.empty? then
+      WORD_PAIR_MAP.each do |regexp, attr|
+        str.gsub!(regexp) {
+          attrs.set_attrs($`.length + $1.length, $2.length, attr)
+          NULL * $1.length + $2 + NULL * $3.length
+        }
+      end
+    end
+  end
+
+  def convert_html(str, attrs)
+    tags = HTML_TAGS.keys.join '|'
+
+    1 while str.gsub!(/<(#{tags})>(.*?)<\/\1>/i) {
+      attr = HTML_TAGS[$1.downcase]
+      html_length = $1.length + 2
+      seq = NULL * html_length
+      attrs.set_attrs($`.length + html_length, $2.length, attr)
+      seq + $2 + seq + NULL
+    }
+  end
+
+  def convert_specials(str, attrs)
+    unless SPECIAL.empty?
+      SPECIAL.each do |regexp, attr|
+        str.scan(regexp) do
+          attrs.set_attrs($`.length, $&.length,
+                          attr | RDoc::Markup::Attribute::SPECIAL)
+        end
+      end
+    end
+  end
+
+  ##
+  # A \ in front of a character that would normally be processed turns off
+  # processing. We do this by turning \< into <#{PROTECT}
+
+  PROTECTABLE = %w[<\\]
+
+  def mask_protected_sequences
+    protect_pattern = Regexp.new("\\\\([#{Regexp.escape(PROTECTABLE.join(''))}])")
+    @str.gsub!(protect_pattern, "\\1#{PROTECT_ATTR}")
+  end
+
+  def unmask_protected_sequences
+    @str.gsub!(/(.)#{PROTECT_ATTR}/, "\\1\000")
+  end
+
+  def initialize
+    add_word_pair("*", "*", :BOLD)
+    add_word_pair("_", "_", :EM)
+    add_word_pair("+", "+", :TT)
+
+    add_html("em", :EM)
+    add_html("i",  :EM)
+    add_html("b",  :BOLD)
+    add_html("tt",   :TT)
+    add_html("code", :TT)
+
+    add_special(/<!--(.*?)-->/, :COMMENT)
+  end
+
+  def add_word_pair(start, stop, name)
+    raise ArgumentError, "Word flags may not start with '<'" if
+      start[0,1] == '<'
+
+    bitmap = RDoc::Markup::Attribute.bitmap_for name
+
+    if start == stop then
+      MATCHING_WORD_PAIRS[start] = bitmap
+    else
+      pattern = /(#{Regexp.escape start})(\S+)(#{Regexp.escape stop})/
+      WORD_PAIR_MAP[pattern] = bitmap
+    end
+
+    PROTECTABLE << start[0,1]
+    PROTECTABLE.uniq!
+  end
+
+  def add_html(tag, name)
+    HTML_TAGS[tag.downcase] = RDoc::Markup::Attribute.bitmap_for name
+  end
+
+  def add_special(pattern, name)
+    SPECIAL[pattern] = RDoc::Markup::Attribute.bitmap_for name
+  end
+
+  def flow(str)
+    @str = str
+
+    puts("Before flow, str='#{@str.dump}'") if $DEBUG_RDOC
+    mask_protected_sequences
+
+    @attrs = RDoc::Markup::AttrSpan.new @str.length
+
+    puts("After protecting, str='#{@str.dump}'") if $DEBUG_RDOC
+
+    convert_attrs(@str, @attrs)
+    convert_html(@str, @attrs)
+    convert_specials(str, @attrs)
+
+    unmask_protected_sequences
+
+    puts("After flow, str='#{@str.dump}'") if $DEBUG_RDOC
+
+    return split_into_flow
+  end
+
+  def display_attributes
+    puts
+    puts @str.tr(NULL, "!")
+    bit = 1
+    16.times do |bno|
+      line = ""
+      @str.length.times do |i|
+        if (@attrs[i] & bit) == 0
+          line << " "
+        else
+          if bno.zero?
+            line << "S"
+          else
+            line << ("%d" % (bno+1))
+          end
+        end
+      end
+      puts(line) unless line =~ /^ *$/
+      bit <<= 1
+    end
+  end
+
+  def split_into_flow
+    display_attributes if $DEBUG_RDOC
+
+    res = []
+    current_attr = 0
+    str = ""
+
+    str_len = @str.length
+
+    # skip leading invisible text
+    i = 0
+    i += 1 while i < str_len and @str[i].chr == "\0"
+    start_pos = i
+
+    # then scan the string, chunking it on attribute changes
+    while i < str_len
+      new_attr = @attrs[i]
+      if new_attr != current_attr
+        if i > start_pos
+          res << copy_string(start_pos, i)
+          start_pos = i
+        end
+
+        res << change_attribute(current_attr, new_attr)
+        current_attr = new_attr
+
+        if (current_attr & RDoc::Markup::Attribute::SPECIAL) != 0 then
+          i += 1 while
+            i < str_len and (@attrs[i] & RDoc::Markup::Attribute::SPECIAL) != 0
+
+          res << RDoc::Markup::Special.new(current_attr,
+                                           copy_string(start_pos, i))
+          start_pos = i
+          next
+        end
+      end
+
+      # move on, skipping any invisible characters
+      begin
+        i += 1
+      end while i < str_len and @str[i].chr == "\0"
+    end
+
+    # tidy up trailing text
+    if start_pos < str_len
+      res << copy_string(start_pos, str_len)
+    end
+
+    # and reset to all attributes off
+    res << change_attribute(current_attr, 0) if current_attr != 0
+
+    return res
+  end
+
+end
+

data/lib/rdoc/markup/formatter.rb

@@ -0,0 +1,14 @@
+require 'rdoc/markup'
+
+class RDoc::Markup::Formatter
+
+  def initialize
+    @markup = RDoc::Markup.new
+  end
+
+  def convert(content)
+    @markup.convert content, self
+  end
+
+end
+

data/lib/rdoc/markup/fragments.rb

@@ -0,0 +1,337 @@
+require 'rdoc/markup'
+require 'rdoc/markup/lines'
+
+class RDoc::Markup
+
+  ##
+  # A Fragment is a chunk of text, subclassed as a paragraph, a list
+  # entry, or verbatim text.
+
+  class Fragment
+    attr_reader   :level, :param, :txt
+    attr_accessor :type
+
+    ######
+    # This is a simple factory system that lets us associate fragement
+    # types (a string) with a subclass of fragment
+
+    TYPE_MAP = {}
+
+    def self.type_name(name)
+      TYPE_MAP[name] = self
+    end
+
+    def self.for(line)
+      klass =  TYPE_MAP[line.type] ||
+        raise("Unknown line type: '#{line.type.inspect}:' '#{line.text}'")
+      return klass.new(line.level, line.param, line.flag, line.text)
+    end
+
+    def initialize(level, param, type, txt)
+      @level = level
+      @param = param
+      @type  = type
+      @txt   = ""
+      add_text(txt) if txt
+    end
+
+    def add_text(txt)
+      @txt << " " if @txt.length > 0
+      @txt << txt.tr_s("\n ", "  ").strip
+    end
+
+    def to_s
+      "L#@level: #{self.class.name.split('::')[-1]}\n#@txt"
+    end
+
+  end
+
+  ##
+  # A paragraph is a fragment which gets wrapped to fit. We remove all
+  # newlines when we're created, and have them put back on output.
+
+  class Paragraph < Fragment
+    type_name :PARAGRAPH
+  end
+
+  class BlankLine < Paragraph
+    type_name :BLANK
+  end
+
+  class Heading < Paragraph
+    type_name :HEADING
+
+    def head_level
+      @param.to_i
+    end
+  end
+
+  ##
+  # A List is a fragment with some kind of label
+
+  class ListBase < Paragraph
+    LIST_TYPES = [
+      :BULLET,
+      :NUMBER,
+      :UPPERALPHA,
+      :LOWERALPHA,
+      :LABELED,
+      :NOTE,
+    ]
+  end
+
+  class ListItem < ListBase
+    type_name :LIST
+
+    def to_s
+      text = if [:NOTE, :LABELED].include? type then
+               "#{@param}: #{@txt}"
+             else
+               @txt
+             end
+
+      "L#@level: #{type} #{self.class.name.split('::')[-1]}\n#{text}"
+    end
+
+  end
+
+  class ListStart < ListBase
+    def initialize(level, param, type)
+      super(level, param, type, nil)
+    end
+  end
+
+  class ListEnd < ListBase
+    def initialize(level, type)
+      super(level, "", type, nil)
+    end
+  end
+
+  ##
+  # Verbatim code contains lines that don't get wrapped.
+
+  class Verbatim < Fragment
+    type_name  :VERBATIM
+
+    def add_text(txt)
+      @txt << txt.chomp << "\n"
+    end
+
+  end
+
+  ##
+  # A horizontal rule
+
+  class Rule < Fragment
+    type_name :RULE
+  end
+
+  ##
+  # Collect groups of lines together. Each group will end up containing a flow
+  # of text.
+
+  class LineCollection
+
+    def initialize
+      @fragments = []
+    end
+
+    def add(fragment)
+      @fragments << fragment
+    end
+
+    def each(&b)
+      @fragments.each(&b)
+    end
+
+    def to_a # :nodoc:
+      @fragments.map {|fragment| fragment.to_s}
+    end
+
+    ##
+    # Factory for different fragment types
+
+    def fragment_for(*args)
+      Fragment.for(*args)
+    end
+
+    ##
+    # Tidy up at the end
+
+    def normalize
+      change_verbatim_blank_lines
+      add_list_start_and_ends
+      add_list_breaks
+      tidy_blank_lines
+    end
+
+    def to_s
+      @fragments.join("\n----\n")
+    end
+
+    def accept(am, visitor)
+      visitor.start_accepting
+
+      @fragments.each do |fragment|
+        case fragment
+        when Verbatim
+          visitor.accept_verbatim(am, fragment)
+        when Rule
+          visitor.accept_rule(am, fragment)
+        when ListStart
+          visitor.accept_list_start(am, fragment)
+        when ListEnd
+          visitor.accept_list_end(am, fragment)
+        when ListItem
+          visitor.accept_list_item(am, fragment)
+        when BlankLine
+          visitor.accept_blank_line(am, fragment)
+        when Heading
+          visitor.accept_heading(am, fragment)
+        when Paragraph
+          visitor.accept_paragraph(am, fragment)
+        end
+      end
+
+      visitor.end_accepting
+    end
+
+    private
+
+    # If you have:
+    #
+    #    normal paragraph text.
+    #
+    #       this is code
+    #   
+    #       and more code
+    #
+    # You'll end up with the fragments Paragraph, BlankLine, Verbatim,
+    # BlankLine, Verbatim, BlankLine, etc.
+    #
+    # The BlankLine in the middle of the verbatim chunk needs to be changed to
+    # a real verbatim newline, and the two verbatim blocks merged
+
+    def change_verbatim_blank_lines
+      frag_block = nil
+      blank_count = 0
+      @fragments.each_with_index do |frag, i|
+        if frag_block.nil?
+          frag_block = frag if Verbatim === frag
+        else
+          case frag
+          when Verbatim
+            blank_count.times { frag_block.add_text("\n") }
+            blank_count = 0
+            frag_block.add_text(frag.txt)
+            @fragments[i] = nil    # remove out current fragment
+          when BlankLine
+            if frag_block
+              blank_count += 1
+              @fragments[i] = nil
+            end
+          else
+            frag_block = nil
+            blank_count = 0
+          end
+        end
+      end
+      @fragments.compact!
+    end
+
+    ##
+    # List nesting is implicit given the level of indentation. Make it
+    # explicit, just to make life a tad easier for the output processors
+
+    def add_list_start_and_ends
+      level = 0
+      res = []
+      type_stack = []
+
+      @fragments.each do |fragment|
+        # $stderr.puts "#{level} : #{fragment.class.name} : #{fragment.level}"
+        new_level = fragment.level
+        while (level < new_level)
+          level += 1
+          type = fragment.type
+          res << ListStart.new(level, fragment.param, type) if type
+          type_stack.push type
+          # $stderr.puts "Start: #{level}"
+        end
+
+        while level > new_level
+          type = type_stack.pop
+          res << ListEnd.new(level, type) if type
+          level -= 1
+          # $stderr.puts "End: #{level}, #{type}"
+        end
+
+        res << fragment
+        level = fragment.level
+      end
+      level.downto(1) do |i|
+        type = type_stack.pop
+        res << ListEnd.new(i, type) if type
+      end
+
+      @fragments = res
+    end
+
+    ##
+    # Inserts start/ends between list entries at the same level that have
+    # different element types
+
+    def add_list_breaks
+      res = @fragments
+
+      @fragments = []
+      list_stack = []
+
+      res.each do |fragment|
+        case fragment
+        when ListStart
+          list_stack.push fragment
+        when ListEnd
+          start = list_stack.pop
+          fragment.type = start.type
+        when ListItem
+          l = list_stack.last
+          if fragment.type != l.type
+            @fragments << ListEnd.new(l.level, l.type)
+            start = ListStart.new(l.level, fragment.param, fragment.type)
+            @fragments << start
+            list_stack.pop
+            list_stack.push start
+          end
+        else
+          ;
+        end
+        @fragments << fragment
+      end
+    end
+
+    ##
+    # Tidy up the blank lines:
+    # * change Blank/ListEnd into ListEnd/Blank
+    # * remove blank lines at the front
+
+    def tidy_blank_lines
+      (@fragments.size - 1).times do |i|
+        if BlankLine === @fragments[i] and ListEnd === @fragments[i+1] then
+          @fragments[i], @fragments[i+1] = @fragments[i+1], @fragments[i]
+        end
+      end
+
+      # remove leading blanks
+      @fragments.each_with_index do |f, i|
+        break unless f.kind_of? BlankLine
+        @fragments[i] = nil
+      end
+
+      @fragments.compact!
+    end
+
+  end
+
+end
+