kramdown 0.8.0 → 0.9.0

data/lib/kramdown/document.rb

@@ -27,6 +27,7 @@ require 'kramdown/error'
 require 'kramdown/parser'
 require 'kramdown/converter'
 require 'kramdown/options'
+require 'kramdown/utils'
 
 module Kramdown
 
@@ -92,6 +93,7 @@ module Kramdown
       @options = Options.merge(options)
       @warnings = []
       @parse_infos = {}
+      @parse_infos[:encoding] = source.encoding if RUBY_VERSION >= '1.9'
       @conversion_infos = {}
       parser = (options[:input] || 'kramdown').to_s
       parser = parser[0..0].upcase + parser[1..-1]

data/lib/kramdown/options.rb

@@ -210,21 +210,6 @@ footnote.
 
 Default: 1
 Used by: HTML converter
-EOF
-
-    define(:filter_html, Array, [], <<EOF)
-NOTE: This option is deprecated and will be removed in a future release!
-
-An array of HTML tags that should be filtered from the output
-
-The value can either be specified as array or as a space separated
-string (which will be converted to an array). All HTML tags that are
-listed in the array will be filtered from the output, i.e. only their
-contents is used. This applies only to HTML tags found in the initial
-document.
-
-Default: []
-Used by: HTML converter
 EOF
 
     define(:coderay_wrap, Symbol, :div, <<EOF)
@@ -275,6 +260,22 @@ styles are directly applied to the code elements).
 
 Default: style
 Used by: HTML converter
+EOF
+
+    define(:numeric_entities, Boolean, false, <<EOF)
+Defines whether entities are output using names or numeric values
+
+Default: false
+Used by: HTML converter, kramdown converter
+EOF
+
+    define(:toc_depth, Integer, 0, <<EOF)
+Defines the maximum level of headers which will be used to generate the table of
+contents. For instance, with a value of 2, toc entries will be generated for h1
+and h2 headers but not for h3, h4, etc. A value of 0 uses all header levels.
+
+Default: 0
+Used by: HTML/Latex converter
 EOF
 
   end

data/lib/kramdown/parser/base.rb

@@ -34,7 +34,6 @@ module Kramdown
       # Initialize the parser with the given Kramdown document +doc+.
       def initialize(doc)
         @doc = doc
-        @doc.parse_infos.clear
         @text_type = :text
       end
       private_class_method(:new, :allocate)

data/lib/kramdown/parser/html.rb

@@ -161,16 +161,23 @@ module Kramdown
       class ElementConverter
 
         include Constants
+        include ::Kramdown::Utils::Entities
 
         REMOVE_TEXT_CHILDREN =  %w{html head hgroup ol ul dl table colgroup tbody thead tfoot tr select optgroup}
+        WRAP_TEXT_CHILDREN = %w{body section nav article aside header footer address div li dd blockquote figure
+                                figcaption fieldset form}
         REMOVE_WHITESPACE_CHILDREN = %w{body section nav article aside header footer address
                                         div li dd blockquote figure figcaption td th fieldset form}
         STRIP_WHITESPACE = %w{address article aside blockquote body caption dd div dl dt fieldset figcaption form footer
                               header h1 h2 h3 h4 h5 h6 legend li nav p section td th}
         SIMPLE_ELEMENTS = %w{em strong blockquote hr br a img p thead tbody tfoot tr td th ul ol dl li dl dt dd}
 
+        def initialize(doc)
+          @doc = doc
+        end
+
         # Convert the element +el+ and its children.
-        def process(el, convert_simple = true, parent = nil)
+        def process(el, do_conversion = true, preserve_text = false, parent = nil)
           case el.type
           when :xml_comment, :xml_pi, :html_doctype
             ptype = if parent.nil?
@@ -194,25 +201,26 @@ module Kramdown
           remove_text_children(el) if REMOVE_TEXT_CHILDREN.include?(type)
 
           mname = "convert_#{el.value}"
-          if self.class.method_defined?(mname)
+          if do_conversion && self.class.method_defined?(mname)
             send(mname, el)
-          elsif convert_simple && SIMPLE_ELEMENTS.include?(type)
+          elsif do_conversion && SIMPLE_ELEMENTS.include?(type)
             set_basics(el, type.intern, HTML_SPAN_ELEMENTS.include?(type) ? :span : :block)
-            process_children(el, convert_simple)
+            process_children(el, do_conversion, preserve_text)
           else
-            process_html_element(el, convert_simple)
+            process_html_element(el, do_conversion, preserve_text)
           end
 
           strip_whitespace(el) if STRIP_WHITESPACE.include?(type)
           remove_whitespace_children(el) if REMOVE_WHITESPACE_CHILDREN.include?(type)
+          wrap_text_children(el) if WRAP_TEXT_CHILDREN.include?(type)
         end
 
-        def process_children(el, convert_simple = true)
+        def process_children(el, do_conversion = true, preserve_text = false)
           el.children.map! do |c|
             if c.type == :text
-              process_text(c.value)
+              process_text(c.value, preserve_text)
             else
-              process(c, convert_simple, el)
+              process(c, do_conversion, preserve_text, el)
               c
             end
           end.flatten!
@@ -234,7 +242,7 @@ module Kramdown
                         elsif %w{mdash ndash hellip laquo raquo}.include?(val)
                           Element.new(:typographic_sym, val.intern)
                         else
-                          Element.new(:entity, val)
+                          Element.new(:entity, entity(val))
                         end
             else
               result << Element.new(:text, src.scan(/.*/m))
@@ -243,18 +251,39 @@ module Kramdown
           result
         end
 
-        def process_html_element(el, convert_simple = true)
+        def process_html_element(el, do_conversion = true, preserve_text = false)
           el.options = {:category => HTML_SPAN_ELEMENTS.include?(el.value) ? :span : :block,
             :parse_type => HTML_PARSE_AS[el.value],
             :attr => el.options[:attr]
           }
-          process_children(el, convert_simple)
+          process_children(el, do_conversion, preserve_text)
         end
 
         def remove_text_children(el)
           el.children.delete_if {|c| c.type == :text}
         end
 
+        SPAN_ELEMENTS = [:em, :strong, :br, :a, :img, :codespan, :entity, :smart_quote, :typographic_sym, :math]
+
+        def wrap_text_children(el)
+          tmp = []
+          last_is_p = false
+          el.children.each do |c|
+            if c.options[:category] != :block || c.type == :text
+              if !last_is_p
+                tmp << Element.new(:p, nil, :transparent => true)
+                last_is_p = true
+              end
+              tmp.last.children << c
+              tmp
+            else
+              tmp << c
+              last_is_p = false
+            end
+          end
+          el.children = tmp
+        end
+
         def strip_whitespace(el)
           return if el.children.empty?
           if el.children.first.type == :text
@@ -291,20 +320,42 @@ module Kramdown
           extract_text(el, el.options[:raw_text] = '')
           process_children(el)
         end
-        %w{h2 h3 h4 h5 h6}.each {|i| alias_method("convert_#{i}".intern, :convert_h1)}
+        %w{h2 h3 h4 h5 h6}.each do |i|
+          alias_method("convert_#{i}".to_sym, :convert_h1)
+        end
 
         def convert_code(el)
-          if el.value == 'code'
-            set_basics(el, :codespan, :span)
-          else
-            set_basics(el, :codeblock, :block)
-          end
           raw = ''
           extract_text(el, raw)
           result = process_text(raw, true)
+          begin
+            str = result.inject('') do |mem, c|
+              if c.type == :text
+                mem << c.value
+              elsif c.type == :entity
+                if RUBY_VERSION >= '1.9'
+                  mem << c.value.char.encode(@doc.parse_infos[:encoding])
+                elsif [60, 62, 34, 38].include?(c.value.code_point)
+                  mem << c.value.code_point.chr
+                end
+              elsif c.type == :smart_quote || c.type == :typographic_sym
+                mem << entity(c.value.to_s).char.encode(@doc.parse_infos[:encoding])
+              else
+                raise "Bug - please report"
+              end
+            end
+            result.clear
+            result << Element.new(:text, str)
+          rescue
+          end
           if result.length > 1 || result.first.type != :text
-            el.children = result
+            process_html_element(el, false, true)
           else
+            if el.value == 'code'
+              set_basics(el, :codespan, :span)
+            else
+              set_basics(el, :codeblock, :block)
+            end
             el.value = result.first.value
           end
         end
@@ -318,16 +369,20 @@ module Kramdown
           process_children(el)
           set_basics(el, :table, :block)
           el.options[:alignment] = []
-          helper = lambda do |c|
+          calc_alignment = lambda do |c|
             if c.type == :tr && el.options[:alignment].empty?
               el.options[:alignment] = [:default] * c.children.length
               break
             else
-              c.children.each {|cc| helper.call(cc)}
+              c.children.each {|cc| calc_alignment.call(cc)}
             end
           end
-          helper.call(el)
-          true
+          calc_alignment.call(el)
+          if el.children.first.type == :tr
+            tbody = Element.new(:tbody, nil, :category => :block)
+            tbody.children = el.children
+            el.children = [tbody]
+          end
         end
 
         def is_simple_table?(el)
@@ -336,17 +391,56 @@ module Kramdown
               (cc.type == :text || !HTML_BLOCK_ELEMENTS.include?(cc.value)) && only_phrasing_content.call(cc)
             end
           end
-          helper = Proc.new do |c|
+          check_cells = Proc.new do |c|
             if c.value == 'th' || c.value == 'td'
               return false if !only_phrasing_content.call(c)
             else
-              c.children.each {|cc| helper.call(cc)}
+              c.children.each {|cc| check_cells.call(cc)}
             end
           end
-          helper.call(el)
-          true
+          check_cells.call(el)
+
+          check_rows = lambda do |t, type|
+            t.children.all? {|r| (r.value == 'tr' || r.type == :text) && r.children.all? {|c| c.value == type || c.type == :text}}
+          end
+          check_rows.call(el, 'td') ||
+            (el.children.all? do |t|
+               t.type == :text || (t.value == 'thead' && check_rows.call(t, 'th')) ||
+                 ((t.value == 'tfoot' || t.value == 'tbody') && check_rows.call(t, 'td'))
+             end && el.children.any? {|t| t.value == 'tbody'})
+        end
+
+        def convert_div(el)
+          if !is_math_tag?(el)
+            process_html_element(el)
+          else
+            handle_math_tag(el)
+          end
         end
+        alias :convert_span :convert_div
 
+        def is_math_tag?(el)
+          el.options[:attr] && el.options[:attr]['class'].to_s =~ /\bmath\b/ &&
+            el.children.size == 1 && el.children.first.type == :text
+        end
+
+        def handle_math_tag(el)
+          set_basics(el, :math, (el.value == 'div' ? :block : :span))
+          el.value = el.children.shift.value
+          if el.options[:attr]['class'] =~ /^\s*math\s*$/
+            el.options[:attr].delete('class')
+          else
+            el.options[:attr]['class'].sub!(/\s?math/, '')
+          end
+          el.value.gsub!(/&(amp|quot|gt|lt);/) do |m|
+            case m
+            when '&amp;'   then '&'
+            when '&quot;'  then '"'
+            when '&gt;'    then '>'
+            when '&lt;'    then '<'
+            end
+          end
+        end
       end
 
       include Parser
@@ -374,7 +468,7 @@ module Kramdown
         end
         parse_raw_html(@tree, &tag_handler)
 
-        ec = ElementConverter.new
+        ec = ElementConverter.new(@doc)
         @tree.children.each {|c| ec.process(c)}
         ec.remove_whitespace_children(@tree)
         @tree
@@ -385,3 +479,4 @@ module Kramdown
   end
 
 end
+

data/lib/kramdown/parser/kramdown.rb

@@ -174,6 +174,8 @@ module Kramdown
             @src = StringScanner.new(child.value)
             parse_spans(child)
             child.children
+          elsif child.type == :eob
+            []
           else
             update_tree(child)
             update_attr_with_ial(child.options[:attr] ||= {}, child.options[:ial]) if child.options[:ial]

data/lib/kramdown/parser/kramdown/attribute_list.rb

@@ -62,6 +62,7 @@ module Kramdown
       def parse_ald
         @src.pos += @src.matched_size
         parse_attribute_list(@src[2], @doc.parse_infos[:ald][@src[1]] ||= {})
+        @tree.children << Element.new(:eob)
         true
       end
       define_parser(:ald, ALD_START)
@@ -77,6 +78,7 @@ module Kramdown
         else
           parse_attribute_list(@src[1], @block_ial = {})
         end
+        @tree.children << Element.new(:eob) unless @src.check(IAL_BLOCK_START)
         true
       end
       define_parser(:block_ial, IAL_BLOCK_START)

data/lib/kramdown/parser/kramdown/extension.rb

@@ -44,6 +44,7 @@ module Kramdown
           stop_re = (type == :block ? /#{EXT_BLOCK_STOP_STR % ext}/ : /#{EXT_STOP_STR % ext}/)
           if result = @src.scan_until(stop_re)
             body = result.sub!(stop_re, '')
+            body.chomp! if type == :block
           else
             warning("No stop tag for extension '#{ext}' found - treating it as extension without body")
           end
@@ -55,7 +56,7 @@ module Kramdown
       def handle_extension(name, opts, body, type)
         case name
         when 'comment'
-          # nothing to do
+          @tree.children << Element.new(:comment, body, :category => type) if body.kind_of?(String)
         when 'nomarkdown'
           @tree.children << Element.new(:raw, body, :category => type) if body.kind_of?(String)
         when 'options'

data/lib/kramdown/parser/kramdown/html.rb

@@ -98,7 +98,7 @@ module Kramdown
           if result = @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1])
             @src.pos += @src.matched_size
             handle_html_start_tag(&method(:handle_kramdown_html_tag))
-            Kramdown::Parser::Html::ElementConverter.new.process(@tree.children.last) if @doc.options[:html_to_native]
+            Kramdown::Parser::Html::ElementConverter.new(@doc).process(@tree.children.last) if @doc.options[:html_to_native]
             true
           elsif result = @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1])
             @src.pos += @src.matched_size
@@ -161,7 +161,7 @@ module Kramdown
               add_text(@src.scan(/.*/m), el)
             end
           end
-          Kramdown::Parser::Html::ElementConverter.new.process(el) if @doc.options[:html_to_native]
+          Kramdown::Parser::Html::ElementConverter.new(@doc).process(el) if @doc.options[:html_to_native]
         else
           add_text(@src.scan(/./))
         end

data/lib/kramdown/parser/kramdown/html_entity.rb

@@ -29,7 +29,7 @@ module Kramdown
       # Parse the HTML entity at the current location.
       def parse_html_entity
         @src.pos += @src.matched_size
-        @tree.children << Element.new(:entity, @src[1] || (@src[2] && @src[2].to_i) || @src[3].hex)
+        @tree.children << Element.new(:entity, ::Kramdown::Utils::Entities.entity(@src[1] || (@src[2] && @src[2].to_i) || @src[3].hex))
       end
       define_parser(:html_entity, Kramdown::Parser::Html::Constants::HTML_ENTITY_RE, '&')
 

data/lib/kramdown/parser/kramdown/link.rb

@@ -86,7 +86,7 @@ module Kramdown
           end
           count - el.children.select {|c| c.type == :img}.size == 0
         end
-        if !found || el.children.empty?
+        if !found || (link_type == :a && el.children.empty?)
           @src.pos = reset_pos
           add_text(result)
           return
@@ -98,7 +98,10 @@ module Kramdown
         # reference style link or no link url
         if @src.scan(LINK_INLINE_ID_RE) || !@src.check(/\(/)
           link_id = (@src[1] || conv_link_id).downcase
-          if @doc.parse_infos[:link_defs].has_key?(link_id)
+          if link_id.empty?
+            @src.pos = reset_pos
+            add_text(result)
+          elsif @doc.parse_infos[:link_defs].has_key?(link_id)
             add_link(el, @doc.parse_infos[:link_defs][link_id].first, @doc.parse_infos[:link_defs][link_id].last, alt_text)
           else
             warning("No link definition for link ID '#{link_id}' found")

data/lib/kramdown/parser/kramdown/list.rb

@@ -32,7 +32,7 @@ module Kramdown
       # Used for parsing the first line of a list item or a definition, i.e. the line with list item
       # marker or the definition marker.
       def parse_first_list_line(indentation, content)
-        if content =~ /^\s*\n/
+        if content =~ /^\s*(#{IAL_SPAN_START})?\s*\n/
           indentation = 4
         else
           while content =~ /^ *\t/
@@ -75,7 +75,7 @@ module Kramdown
             item.value, indentation, content_re, indent_re = parse_first_list_line(@src[1].length, @src[2])
             list.children << item
 
-            item.value.sub!(/^#{IAL_SPAN_START}/) do |match|
+            item.value.sub!(/^#{IAL_SPAN_START}\s*/) do |match|
               parse_attribute_list($~[1], item.options[:ial] ||= {})
               ''
             end
@@ -119,10 +119,10 @@ module Kramdown
 
           if it.children.first.type == :p && (it.children.length < 2 || it.children[1].type != :blank ||
                                               (it == list.children.last && it.children.length == 2 && !eob_found)) &&
-              (list.children.last != it || list.children.size == 1 || list.children[0..-2].any? {|cit| cit.children.first.type != :p})
-            text = it.children.shift.children.first
-            text.value += "\n" if !it.children.empty? && it.children[0].type != :blank
-            it.children.unshift(text)
+              (list.children.last != it || list.children.size == 1 ||
+               list.children[0..-2].any? {|cit| cit.children.first.type != :p || cit.children.first.options[:transparent]})
+            it.children.first.children.first.value += "\n" if it.children.size > 1 && it.children[1].type != :blank
+            it.children.first.options[:transparent] = true
           end
 
           if it.children.last.type == :blank
@@ -173,6 +173,11 @@ module Kramdown
             item.value, indentation, content_re, indent_re = parse_first_list_line(@src[1].length, @src[2])
             deflist.children << item
 
+            item.value.sub!(/^#{IAL_SPAN_START}\s*/) do |match|
+              parse_attribute_list($~[1], item.options[:ial] ||= {})
+              ''
+            end
+
             def_start_re = /^( {0,#{[3, indentation - 1].min}}:)([\t| ].*?\n)/
             first_as_para = false
           elsif result = @src.scan(content_re)
@@ -202,9 +207,8 @@ module Kramdown
             last = nil
           end
           if it.children.first.type == :p && !it.options.delete(:first_as_para)
-            text = it.children.shift.children.first
-            text.value += "\n" if !it.children.empty?
-            it.children.unshift(text)
+            it.children.first.children.first.value += "\n" if it.children.size > 1
+            it.children.first.options[:transparent] = true
           end
         end