kramdown 2.0.0 → 2.3.1

data/lib/kramdown/parser/kramdown/header.rb

@@ -8,6 +8,7 @@
 #
 
 require 'kramdown/parser/kramdown/block_boundary'
+require 'rexml/xmltokens'
 
 module Kramdown
   module Parser
@@ -31,7 +32,7 @@ module Kramdown
       def parse_atx_header
         return false unless after_block_boundary?
         text, id = parse_header_contents
-        text.sub!(/[\t ]#+\z/, '') && text.rstrip!
+        text.sub!(/(?<!\\)#+\z/, '') && text.rstrip!
         return false if text.empty?
         add_header(@src["level"].length, text, id)
         true
@@ -40,7 +41,7 @@ module Kramdown
 
       protected
 
-      HEADER_ID = /[\t ]{#(?<id>[A-Za-z][\w:-]*)}\z/
+      HEADER_ID = /[\t ]{#(?<id>#{REXML::XMLTokens::NAME_START_CHAR}#{REXML::XMLTokens::NAME_CHAR}*)}\z/
 
       # Returns header text and optional ID.
       def parse_header_contents

data/lib/kramdown/parser/kramdown/html.rb

@@ -65,7 +65,7 @@ module Kramdown
         end
       end
 
-      HTML_BLOCK_START = /^#{OPT_SPACE}<(#{REXML::Parsers::BaseParser::UNAME_STR}|\?|!--|\/)/
+      HTML_BLOCK_START = /^#{OPT_SPACE}<(#{REXML::Parsers::BaseParser::UNAME_STR}|!--|\/)/
 
       # Parse the HTML at the current position as block-level HTML.
       def parse_block_html
@@ -74,17 +74,13 @@ module Kramdown
           @tree.children << Element.new(:xml_comment, result, nil, category: :block, location: line)
           @src.scan(TRAILING_WHITESPACE)
           true
-        elsif (result = @src.scan(HTML_INSTRUCTION_RE))
-          @tree.children << Element.new(:xml_pi, result, nil, category: :block, location: line)
-          @src.scan(TRAILING_WHITESPACE)
-          true
         else
-          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             @src.pos += @src.matched_size
             handle_html_start_tag(line, &method(:handle_kramdown_html_tag))
             Kramdown::Parser::Html::ElementConverter.convert(@root, @tree.children.last) if @options[:html_to_native]
             true
-          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             name = @src[1].downcase
 
             if @tree.type == :html_element && @tree.value == name
@@ -100,15 +96,13 @@ module Kramdown
       end
       define_parser(:block_html, HTML_BLOCK_START)
 
-      HTML_SPAN_START = /<(#{REXML::Parsers::BaseParser::UNAME_STR}|\?|!--|\/)/
+      HTML_SPAN_START = /<(#{REXML::Parsers::BaseParser::UNAME_STR}|!--|\/)/
 
       # Parse the HTML at the current position as span-level HTML.
       def parse_span_html
         line = @src.current_line_number
         if (result = @src.scan(HTML_COMMENT_RE))
           @tree.children << Element.new(:xml_comment, result, nil, category: :span, location: line)
-        elsif (result = @src.scan(HTML_INSTRUCTION_RE))
-          @tree.children << Element.new(:xml_pi, result, nil, category: :span, location: line)
         elsif (result = @src.scan(HTML_TAG_CLOSE_RE))
           warning("Found invalidly used HTML closing tag for '#{@src[1]}' on line #{line}")
           add_text(result)

data/lib/kramdown/parser/kramdown/link.rb

@@ -92,8 +92,9 @@ module Kramdown
           emit_warning = !@src[1]
           link_id = normalize_link_id(@src[1] || alt_text)
           if @link_defs.key?(link_id)
-            add_link(el, @link_defs[link_id][0], @link_defs[link_id][1], alt_text,
-                     @link_defs[link_id][2] && @link_defs[link_id][2].options[:ial])
+            link_def = @link_defs[link_id]
+            add_link(el, link_def[0], link_def[1], alt_text,
+                     link_def[2] && link_def[2].options[:ial])
           else
             if emit_warning
               warning("No link definition for link ID '#{link_id}' found on line #{start_line_number}")

data/lib/kramdown/parser/kramdown/list.rb

@@ -44,8 +44,10 @@ module Kramdown
         [content, indentation, *PARSE_FIRST_LIST_LINE_REGEXP_CACHE[indentation]]
       end
 
-      LIST_START_UL = /^(#{OPT_SPACE}[+*-])([\t| ].*?\n)/
-      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)([\t| ].*?\n)/
+      PATTERN_TAIL = /[\t| ].*?\n/
+
+      LIST_START_UL = /^(#{OPT_SPACE}[+*-])(#{PATTERN_TAIL})/
+      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)(#{PATTERN_TAIL})/
       LIST_START = /#{LIST_START_UL}|#{LIST_START_OL}/
 
       # Parse the ordered or unordered list at the current location.
@@ -72,16 +74,12 @@ module Kramdown
               parse_first_list_line(@src[1].length, @src[2])
             list.children << item
 
-            item.value.sub!(self.class::LIST_ITEM_IAL) do |_match|
+            item.value.sub!(self.class::LIST_ITEM_IAL) do
               parse_attribute_list($1, item.options[:ial] ||= {})
               ''
             end
 
-            list_start_re = if type == :ul
-                              /^( {0,#{[3, indentation - 1].min}}[+*-])([\t| ].*?\n)/
-                            else
-                              /^( {0,#{[3, indentation - 1].min}}\d+\.)([\t| ].*?\n)/
-                            end
+            list_start_re = fetch_pattern(type, indentation)
             nested_list_found = (item.value =~ LIST_START)
             last_is_blank = false
             item.value = [item.value]
@@ -122,22 +120,24 @@ module Kramdown
 
           it.children = temp.children
           it.value = nil
-          next if it.children.empty?
+
+          it_children = it.children
+          next if it_children.empty?
 
           # Handle the case where an EOB marker is inserted by a block IAL for the first paragraph
-          it.children.delete_at(1) if it.children.first.type == :p &&
-            it.children.length >= 2 && it.children[1].type == :eob && it.children.first.options[:ial]
+          it_children.delete_at(1) if it_children.first.type == :p &&
+            it_children.length >= 2 && it_children[1].type == :eob && it_children.first.options[:ial]
 
-          if it.children.first.type == :p &&
-              (it.children.length < 2 || it.children[1].type != :blank ||
-               (it == list.children.last && it.children.length == 2 && !eob_found)) &&
+          if it_children.first.type == :p &&
+              (it_children.length < 2 || it_children[1].type != :blank ||
+              (it == list.children.last && it_children.length == 2 && !eob_found)) &&
               (list.children.last != it || list.children.size == 1 ||
-               list.children[0..-2].any? {|cit| !cit.children.first || cit.children.first.type != :p || cit.children.first.options[:transparent] })
-            it.children.first.children.first.value << "\n" if it.children.size > 1 && it.children[1].type != :blank
-            it.children.first.options[:transparent] = true
+              list.children[0..-2].any? {|cit| !cit.children.first || cit.children.first.type != :p || cit.children.first.options[:transparent] })
+            it_children.first.children.first.value << "\n" if it_children.size > 1 && it_children[1].type != :blank
+            it_children.first.options[:transparent] = true
           end
 
-          last = (it.children.last.type == :blank ? it.children.pop : nil)
+          last = (it_children.last.type == :blank ? it_children.pop : nil)
         end
 
         @tree.children << last if !last.nil? && !eob_found
@@ -146,7 +146,7 @@ module Kramdown
       end
       define_parser(:list, LIST_START)
 
-      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)([\t| ].*?\n)/
+      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)(#{PATTERN_TAIL})/
 
       # Parse the ordered or unordered list at the current location.
       def parse_definition_list
@@ -196,7 +196,7 @@ module Kramdown
               ''
             end
 
-            def_start_re = /^( {0,#{[3, indentation - 1].min}}:)([\t| ].*?\n)/
+            def_start_re = fetch_pattern(:dl, indentation)
             first_as_para = false
             last_is_blank = false
           elsif @src.check(EOB_MARKER)
@@ -222,32 +222,63 @@ module Kramdown
 
           parse_blocks(it, it.value)
           it.value = nil
-          next if it.children.empty?
+          it_children = it.children
+          next if it_children.empty?
 
-          last = (it.children.last.type == :blank ? it.children.pop : nil)
+          last = (it_children.last.type == :blank ? it_children.pop : nil)
 
-          if it.children.first && it.children.first.type == :p && !it.options.delete(:first_as_para)
-            it.children.first.children.first.value << "\n" if it.children.size > 1
-            it.children.first.options[:transparent] = true
+          if it_children.first && it_children.first.type == :p && !it.options.delete(:first_as_para)
+            it_children.first.children.first.value << "\n" if it_children.size > 1
+            it_children.first.options[:transparent] = true
           end
         end
 
-        if @tree.children.length >= 1 && @tree.children.last.type == :dl
-          @tree.children[-1].children.concat(deflist.children)
-        elsif @tree.children.length >= 2 && @tree.children[-1].type == :blank &&
-            @tree.children[-2].type == :dl
-          @tree.children.pop
-          @tree.children[-1].children.concat(deflist.children)
+        children = @tree.children
+        if children.length >= 1 && children.last.type == :dl
+          children[-1].children.concat(deflist.children)
+        elsif children.length >= 2 && children[-1].type == :blank &&
+            children[-2].type == :dl
+          children.pop
+          children[-1].children.concat(deflist.children)
         else
-          @tree.children << deflist
+          children << deflist
         end
 
-        @tree.children << last if last
+        children << last if last
 
         true
       end
       define_parser(:definition_list, DEFINITION_LIST_START)
 
+      private
+
+      # precomputed patterns for indentations 1..4 and fallback expression
+      # to compute pattern when indentation is outside the 1..4 range.
+      def fetch_pattern(type, indentation)
+        if type == :ul
+          case indentation
+          when 1 then %r/^( {0}[+*-])(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}[+*-])(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}[+*-])(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}[+*-])(#{PATTERN_TAIL})/o
+          end
+        elsif type == :ol
+          case indentation
+          when 1 then %r/^( {0}\d+\.)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}\d+\.)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}\d+\.)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}\d+\.)(#{PATTERN_TAIL})/o
+          end
+        elsif type == :dl
+          case indentation
+          when 1 then %r/^( {0}:)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}:)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}:)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}:)(#{PATTERN_TAIL})/o
+          end
+        end
+      end
+
     end
   end
 end

data/lib/kramdown/parser/kramdown/math.rb

@@ -21,7 +21,7 @@ module Kramdown
         if !after_block_boundary?
           return false
         elsif @src[1]
-          @src.scan(/^#{OPT_SPACE}\\/) if @src[3]
+          @src.scan(/^#{OPT_SPACE}\\/o) if @src[3]
           return false
         end
 

data/lib/kramdown/parser/kramdown/paragraph.rb

@@ -36,13 +36,13 @@ module Kramdown
           result << @src.scan(PARAGRAPH_MATCH)
         end
         result.rstrip!
-        if @tree.children.last && @tree.children.last.type == :p
-          last_item_in_para = @tree.children.last.children.last
+        if (last_child = @tree.children.last) && last_child.type == :p
+          last_item_in_para = last_child.children.last
           if last_item_in_para && last_item_in_para.type == @text_type
             joiner = (extract_string((pos - 3)...pos, @src) == "  \n" ? "  \n" : "\n")
             last_item_in_para.value << joiner << result
           else
-            add_text(result, @tree.children.last)
+            add_text(result, last_child)
           end
         else
           @tree.children << new_block_el(:p, nil, nil, location: start_line_number)

data/lib/kramdown/parser/kramdown/table.rb

@@ -81,7 +81,7 @@ module Kramdown
                     cells.concat(l)
                   else
                     delim = (c.value.scan(/`+/).max || '') + '`'
-                    tmp = "#{delim}#{' ' if delim.size > 1}#{c.value}#{' ' if delim.size > 1}#{delim}"
+                    tmp = +"#{delim}#{' ' if delim.size > 1}#{c.value}#{' ' if delim.size > 1}#{delim}"
                     (cells.empty? ? cells : cells.last) << tmp
                   end
                 end
@@ -132,8 +132,8 @@ module Kramdown
               pipe_on_line = false
             end
           else
-            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/
-            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/)
+            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/o
+            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/o)
           end
         end
         @src.revert_pos(saved_pos) and return false unless pipe_on_line

data/lib/kramdown/utils/html.rb

@@ -42,6 +42,8 @@ module Kramdown
 
       # Return the HTML representation of the attributes +attr+.
       def html_attributes(attr)
+        return '' if attr.empty?
+
         attr.map do |k, v|
           v.nil? || (k == 'id' && v.strip.empty?) ? '' : " #{k}=\"#{escape_html(v.to_s, :attribute)}\""
         end.join('')
@@ -68,6 +70,13 @@ module Kramdown
         str.gsub(ESCAPE_RE_FROM_TYPE[type]) {|m| ESCAPE_MAP[m] || m }
       end
 
+      REDUNDANT_LINE_BREAK_REGEX = /([\p{Han}\p{Hiragana}\p{Katakana}]+)\n([\p{Han}\p{Hiragana}\p{Katakana}]+)/u
+      def fix_cjk_line_break(str)
+        while str.gsub!(REDUNDANT_LINE_BREAK_REGEX, '\1\2')
+        end
+        str
+      end
+
     end
 
   end

data/lib/kramdown/version.rb

@@ -10,6 +10,6 @@
 module Kramdown
 
   # The kramdown version.
-  VERSION = '2.0.0'
+  VERSION = '2.3.1'
 
 end

data/man/man1/kramdown.1

@@ -1,5 +1,5 @@
 .\" generated by kramdown
-.TH "KRAMDOWN" "1" "November 2016"
+.TH "KRAMDOWN" "1" "January 2019"
 .SH NAME
 kramdown \- a fast, pure\-Ruby Markdown\-superset converter
 .SH "SYNOPSIS"
@@ -20,6 +20,13 @@ Specify the input format\. Available input formats: \fIkramdown\fP (this is the
 \fB\-o\fP \fIFORMAT\fP, \fB\-\-output\fP \fIFORMAT\fP
 Specify one or more output formats separated by commas: \fIhtml\fP (default), \fIkramdown\fP, \fIlatex\fP, \fIman\fP or \fIremove_html_tags\fP\&\. The converter \fIpdf\fP is available through the \fBkramdown\-converter\-pdf\fP gem\.
 .TP
+\fB\-x\fP \fIEXT\fP, \fB\-\-extension\fP \fIEXT\fP
+Load one or more extensions\. The name of the extension should not include the \fBkramdown\-\fP prefix, e\.g\. just \fBparser\-gfm\fP\&\. Multiple extensions can be loaded by separating them with commas\.
+.RS
+.P
+Note: This option has to be used before any other options that rely on the extension already being loaded\.
+.RE
+.TP
 \fB\-\-no\-config\-file\fP
 Do not read any configuration file\. Default behavior is to check for a configuration file and read it if it exists\.
 .TP
@@ -111,6 +118,13 @@ This option can be used to set a prefix for footnote IDs\. This is useful when r
 Default: \[u2018]\[u2019] Used by: HTML
 .RE
 .TP
+\fB\-\-forbidden\-inline\-options\fP \fIARG\fP
+Defines the options that may not be set using the {::options} extension
+.RS
+.P
+Default: template Used by: HTML converter
+.RE
+.TP
 \fB\-\-header\-offset\fP \fIARG\fP
 Sets the output offset for headers
 .RS
@@ -207,6 +221,13 @@ If this option is \fBtrue\fP, the RemoveHtmlTags converter removes block HTML ta
 Default: true Used by: RemoveHtmlTags converter
 .RE
 .TP
+\fB\-\-[no\-]remove\-line\-breaks\-for\-cjk\fP
+Specifies whether line breaks should be removed between CJK characters
+.RS
+.P
+Default: false Used by: HTML converter
+.RE
+.TP
 \fB\-\-[no\-]remove\-span\-html\-tags\fP
 Remove span HTML tags
 .RS

data/test/test_files.rb

@@ -11,6 +11,7 @@ require 'minitest/autorun'
 require 'kramdown'
 require 'yaml'
 require 'tmpdir'
+require 'open3'
 
 begin
   require 'kramdown/converter/syntax_highlighter/rouge'
@@ -20,16 +21,20 @@ begin
   end
 
   # custom formatter for tests
-  class RougeHTMLFormatters < Kramdown::Converter::SyntaxHighlighter::Rouge.formatter_class
+  module Rouge
+    module Formatters
+      class RougeHTMLFormatters < Kramdown::Converter::SyntaxHighlighter::Rouge.formatter_class
 
-    tag 'rouge_html_formatters'
+        tag 'rouge_html_formatters'
 
-    def stream(tokens, &b)
-      yield %(<div class="custom-class">)
-      super
-      yield %(</div>)
-    end
+        def stream(tokens, &b)
+          yield %(<div class="custom-class">)
+          super
+          yield %(</div>)
+        end
 
+      end
+    end
   end
 rescue LoadError, SyntaxError, NameError
 end
@@ -72,6 +77,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/rouge/multiple.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/highlighting.html', # bc of span elements inside code element
       'test/testcases/block/06_codeblock/highlighting-opts.html', # bc of span elements inside code element
+      'test/testcases/block/06_codeblock/guess_lang_css_class.html', # bc of double surrounding <div>
       'test/testcases/block/12_extension/options3.html', # bc of rouge
       'test/testcases/block/14_table/empty_tag_in_cell.html', # bc of tidy
       'test/testcases/block/15_math/mathjax_preview.html', # bc of mathjax preview
@@ -110,14 +116,10 @@ class TestFiles < Minitest::Test
   end
 
   def tidy_output(out)
-    cmd = "tidy -q --doctype omit #{RUBY_VERSION >= '1.9' ? '-utf8' : '-raw'} 2>/dev/null"
-    result = IO.popen(cmd, 'r+') do |io|
-      io.write(out)
-      io.close_write
-      io.read
-    end
-    if $?.exitstatus == 2
-      raise "Problem using tidy"
+    cmd = "tidy -q --doctype omit -utf8"
+    result, error, status = Open3.capture3(cmd, stdin_data: out)
+    if status.exitstatus == 2
+      raise "Problem using tidy: #{error}"
     end
     result
   end
@@ -135,6 +137,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_name_with_dash.text',
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_any_char.text',
       'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text', # latex unicode support
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_LATEX_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -179,6 +182,9 @@ class TestFiles < Minitest::Test
       'test/testcases/block/09_html/xml.text', # bc of tidy
       'test/testcases/span/05_html/xml.text', # bc of tidy
       'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text',
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
+      'test/testcases/span/abbreviations/abbrev_in_html.text', # bc of invalid abbr tag in SVG
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_TEXT_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -211,6 +217,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/whitespace.html', # bc of entity to char conversion
       'test/testcases/block/06_codeblock/rouge/simple.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/rouge/multiple.html', # bc of double surrounding <div>
+      'test/testcases/block/06_codeblock/guess_lang_css_class.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/highlighting.html', # bc of span elements inside code element
       'test/testcases/block/06_codeblock/highlighting-opts.html', # bc of span elements inside code element
       'test/testcases/block/11_ial/simple.html',           # bc of change of ordering of attributes in header
@@ -220,14 +227,16 @@ class TestFiles < Minitest::Test
       'test/testcases/block/16_toc/toc_exclude.html',      # bc of different attribute ordering
       'test/testcases/span/autolinks/url_links.html',      # bc of quot entity being converted to char
       'test/testcases/block/14_table/empty_tag_in_cell.html', # bc of tidy
-      'test/testcases/block/15_math/mathjax_preview.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_simple.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_as_code.html', # bc of mathjax preview
       'test/testcases/span/01_link/link_defs_with_ial.html', # bc of attribute ordering
       'test/testcases/span/05_html/mark_element.html', # bc of tidy
       'test/testcases/block/09_html/xml.html', # bc of tidy
       'test/testcases/span/05_html/xml.html', # bc of tidy
       'test/testcases/block/03_paragraph/standalone_image.html', # bc of standalone image
+      'test/testcases/block/15_math/normal.html', # bc of mathjax and HTML parser
+      'test/testcases/block/15_math/gh_128.html', # bc of mathjax and HTML parser
+      'test/testcases/span/04_footnote/backlink_inline.html', # bc of mathjax
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
+      'test/testcases/block/09_html/processing_instruction.html', # bc of PI
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.html'].each do |html_file|
       next if EXCLUDE_HTML_KD_FILES.any? {|f| html_file =~ /#{f}$/ }