kramdown 2.0.0.beta2 → 2.3.0

data/lib/kramdown/parser/kramdown/html.rb

@@ -65,7 +65,7 @@ module Kramdown
         end
       end
 
-      HTML_BLOCK_START = /^#{OPT_SPACE}<(#{REXML::Parsers::BaseParser::UNAME_STR}|\?|!--|\/)/
+      HTML_BLOCK_START = /^#{OPT_SPACE}<(#{REXML::Parsers::BaseParser::UNAME_STR}|!--|\/)/
 
       # Parse the HTML at the current position as block-level HTML.
       def parse_block_html
@@ -74,17 +74,13 @@ module Kramdown
           @tree.children << Element.new(:xml_comment, result, nil, category: :block, location: line)
           @src.scan(TRAILING_WHITESPACE)
           true
-        elsif (result = @src.scan(HTML_INSTRUCTION_RE))
-          @tree.children << Element.new(:xml_pi, result, nil, category: :block, location: line)
-          @src.scan(TRAILING_WHITESPACE)
-          true
         else
-          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             @src.pos += @src.matched_size
             handle_html_start_tag(line, &method(:handle_kramdown_html_tag))
             Kramdown::Parser::Html::ElementConverter.convert(@root, @tree.children.last) if @options[:html_to_native]
             true
-          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             name = @src[1].downcase
 
             if @tree.type == :html_element && @tree.value == name
@@ -100,15 +96,13 @@ module Kramdown
       end
       define_parser(:block_html, HTML_BLOCK_START)
 
-      HTML_SPAN_START = /<(#{REXML::Parsers::BaseParser::UNAME_STR}|\?|!--|\/)/
+      HTML_SPAN_START = /<(#{REXML::Parsers::BaseParser::UNAME_STR}|!--|\/)/
 
       # Parse the HTML at the current position as span-level HTML.
       def parse_span_html
         line = @src.current_line_number
         if (result = @src.scan(HTML_COMMENT_RE))
           @tree.children << Element.new(:xml_comment, result, nil, category: :span, location: line)
-        elsif (result = @src.scan(HTML_INSTRUCTION_RE))
-          @tree.children << Element.new(:xml_pi, result, nil, category: :span, location: line)
         elsif (result = @src.scan(HTML_TAG_CLOSE_RE))
           warning("Found invalidly used HTML closing tag for '#{@src[1]}' on line #{line}")
           add_text(result)

data/lib/kramdown/parser/kramdown/link.rb

@@ -92,8 +92,9 @@ module Kramdown
           emit_warning = !@src[1]
           link_id = normalize_link_id(@src[1] || alt_text)
           if @link_defs.key?(link_id)
-            add_link(el, @link_defs[link_id][0], @link_defs[link_id][1], alt_text,
-                     @link_defs[link_id][2] && @link_defs[link_id][2].options[:ial])
+            link_def = @link_defs[link_id]
+            add_link(el, link_def[0], link_def[1], alt_text,
+                     link_def[2] && link_def[2].options[:ial])
           else
             if emit_warning
               warning("No link definition for link ID '#{link_id}' found on line #{start_line_number}")

data/lib/kramdown/parser/kramdown/list.rb

@@ -44,8 +44,10 @@ module Kramdown
         [content, indentation, *PARSE_FIRST_LIST_LINE_REGEXP_CACHE[indentation]]
       end
 
-      LIST_START_UL = /^(#{OPT_SPACE}[+*-])([\t| ].*?\n)/
-      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)([\t| ].*?\n)/
+      PATTERN_TAIL = /[\t| ].*?\n/
+
+      LIST_START_UL = /^(#{OPT_SPACE}[+*-])(#{PATTERN_TAIL})/
+      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)(#{PATTERN_TAIL})/
       LIST_START = /#{LIST_START_UL}|#{LIST_START_OL}/
 
       # Parse the ordered or unordered list at the current location.
@@ -72,16 +74,12 @@ module Kramdown
               parse_first_list_line(@src[1].length, @src[2])
             list.children << item
 
-            item.value.sub!(self.class::LIST_ITEM_IAL) do |_match|
+            item.value.sub!(self.class::LIST_ITEM_IAL) do
               parse_attribute_list($1, item.options[:ial] ||= {})
               ''
             end
 
-            list_start_re = if type == :ul
-                              /^( {0,#{[3, indentation - 1].min}}[+*-])([\t| ].*?\n)/
-                            else
-                              /^( {0,#{[3, indentation - 1].min}}\d+\.)([\t| ].*?\n)/
-                            end
+            list_start_re = fetch_pattern(type, indentation)
             nested_list_found = (item.value =~ LIST_START)
             last_is_blank = false
             item.value = [item.value]
@@ -122,22 +120,24 @@ module Kramdown
 
           it.children = temp.children
           it.value = nil
-          next if it.children.empty?
+
+          it_children = it.children
+          next if it_children.empty?
 
           # Handle the case where an EOB marker is inserted by a block IAL for the first paragraph
-          it.children.delete_at(1) if it.children.first.type == :p &&
-            it.children.length >= 2 && it.children[1].type == :eob && it.children.first.options[:ial]
+          it_children.delete_at(1) if it_children.first.type == :p &&
+            it_children.length >= 2 && it_children[1].type == :eob && it_children.first.options[:ial]
 
-          if it.children.first.type == :p &&
-              (it.children.length < 2 || it.children[1].type != :blank ||
-               (it == list.children.last && it.children.length == 2 && !eob_found)) &&
+          if it_children.first.type == :p &&
+              (it_children.length < 2 || it_children[1].type != :blank ||
+              (it == list.children.last && it_children.length == 2 && !eob_found)) &&
               (list.children.last != it || list.children.size == 1 ||
-               list.children[0..-2].any? {|cit| !cit.children.first || cit.children.first.type != :p || cit.children.first.options[:transparent] })
-            it.children.first.children.first.value << "\n" if it.children.size > 1 && it.children[1].type != :blank
-            it.children.first.options[:transparent] = true
+              list.children[0..-2].any? {|cit| !cit.children.first || cit.children.first.type != :p || cit.children.first.options[:transparent] })
+            it_children.first.children.first.value << "\n" if it_children.size > 1 && it_children[1].type != :blank
+            it_children.first.options[:transparent] = true
           end
 
-          last = (it.children.last.type == :blank ? it.children.pop : nil)
+          last = (it_children.last.type == :blank ? it_children.pop : nil)
         end
 
         @tree.children << last if !last.nil? && !eob_found
@@ -146,7 +146,7 @@ module Kramdown
       end
       define_parser(:list, LIST_START)
 
-      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)([\t| ].*?\n)/
+      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)(#{PATTERN_TAIL})/
 
       # Parse the ordered or unordered list at the current location.
       def parse_definition_list
@@ -196,7 +196,7 @@ module Kramdown
               ''
             end
 
-            def_start_re = /^( {0,#{[3, indentation - 1].min}}:)([\t| ].*?\n)/
+            def_start_re = fetch_pattern(:dl, indentation)
             first_as_para = false
             last_is_blank = false
           elsif @src.check(EOB_MARKER)
@@ -222,32 +222,63 @@ module Kramdown
 
           parse_blocks(it, it.value)
           it.value = nil
-          next if it.children.empty?
+          it_children = it.children
+          next if it_children.empty?
 
-          last = (it.children.last.type == :blank ? it.children.pop : nil)
+          last = (it_children.last.type == :blank ? it_children.pop : nil)
 
-          if it.children.first && it.children.first.type == :p && !it.options.delete(:first_as_para)
-            it.children.first.children.first.value << "\n" if it.children.size > 1
-            it.children.first.options[:transparent] = true
+          if it_children.first && it_children.first.type == :p && !it.options.delete(:first_as_para)
+            it_children.first.children.first.value << "\n" if it_children.size > 1
+            it_children.first.options[:transparent] = true
           end
         end
 
-        if @tree.children.length >= 1 && @tree.children.last.type == :dl
-          @tree.children[-1].children.concat(deflist.children)
-        elsif @tree.children.length >= 2 && @tree.children[-1].type == :blank &&
-            @tree.children[-2].type == :dl
-          @tree.children.pop
-          @tree.children[-1].children.concat(deflist.children)
+        children = @tree.children
+        if children.length >= 1 && children.last.type == :dl
+          children[-1].children.concat(deflist.children)
+        elsif children.length >= 2 && children[-1].type == :blank &&
+            children[-2].type == :dl
+          children.pop
+          children[-1].children.concat(deflist.children)
         else
-          @tree.children << deflist
+          children << deflist
         end
 
-        @tree.children << last if last
+        children << last if last
 
         true
       end
       define_parser(:definition_list, DEFINITION_LIST_START)
 
+      private
+
+      # precomputed patterns for indentations 1..4 and fallback expression
+      # to compute pattern when indentation is outside the 1..4 range.
+      def fetch_pattern(type, indentation)
+        if type == :ul
+          case indentation
+          when 1 then %r/^( {0}[+*-])(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}[+*-])(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}[+*-])(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}[+*-])(#{PATTERN_TAIL})/o
+          end
+        elsif type == :ol
+          case indentation
+          when 1 then %r/^( {0}\d+\.)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}\d+\.)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}\d+\.)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}\d+\.)(#{PATTERN_TAIL})/o
+          end
+        elsif type == :dl
+          case indentation
+          when 1 then %r/^( {0}:)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}:)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}:)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}:)(#{PATTERN_TAIL})/o
+          end
+        end
+      end
+
     end
   end
 end

data/lib/kramdown/parser/kramdown/math.rb

@@ -21,7 +21,7 @@ module Kramdown
         if !after_block_boundary?
           return false
         elsif @src[1]
-          @src.scan(/^#{OPT_SPACE}\\/) if @src[3]
+          @src.scan(/^#{OPT_SPACE}\\/o) if @src[3]
           return false
         end
 

data/lib/kramdown/parser/kramdown/paragraph.rb

@@ -36,13 +36,13 @@ module Kramdown
           result << @src.scan(PARAGRAPH_MATCH)
         end
         result.rstrip!
-        if @tree.children.last && @tree.children.last.type == :p
-          last_item_in_para = @tree.children.last.children.last
+        if (last_child = @tree.children.last) && last_child.type == :p
+          last_item_in_para = last_child.children.last
           if last_item_in_para && last_item_in_para.type == @text_type
             joiner = (extract_string((pos - 3)...pos, @src) == "  \n" ? "  \n" : "\n")
             last_item_in_para.value << joiner << result
           else
-            add_text(result, @tree.children.last)
+            add_text(result, last_child)
           end
         else
           @tree.children << new_block_el(:p, nil, nil, location: start_line_number)

data/lib/kramdown/parser/kramdown/table.rb

@@ -81,7 +81,7 @@ module Kramdown
                     cells.concat(l)
                   else
                     delim = (c.value.scan(/`+/).max || '') + '`'
-                    tmp = "#{delim}#{' ' if delim.size > 1}#{c.value}#{' ' if delim.size > 1}#{delim}"
+                    tmp = +"#{delim}#{' ' if delim.size > 1}#{c.value}#{' ' if delim.size > 1}#{delim}"
                     (cells.empty? ? cells : cells.last) << tmp
                   end
                 end
@@ -132,8 +132,8 @@ module Kramdown
               pipe_on_line = false
             end
           else
-            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/
-            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/)
+            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/o
+            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/o)
           end
         end
         @src.revert_pos(saved_pos) and return false unless pipe_on_line

data/lib/kramdown/utils/html.rb

@@ -42,6 +42,8 @@ module Kramdown
 
       # Return the HTML representation of the attributes +attr+.
       def html_attributes(attr)
+        return '' if attr.empty?
+
         attr.map do |k, v|
           v.nil? || (k == 'id' && v.strip.empty?) ? '' : " #{k}=\"#{escape_html(v.to_s, :attribute)}\""
         end.join('')
@@ -68,6 +70,13 @@ module Kramdown
         str.gsub(ESCAPE_RE_FROM_TYPE[type]) {|m| ESCAPE_MAP[m] || m }
       end
 
+      REDUNDANT_LINE_BREAK_REGEX = /([\p{Han}\p{Hiragana}\p{Katakana}]+)\n([\p{Han}\p{Hiragana}\p{Katakana}]+)/u
+      def fix_cjk_line_break(str)
+        while str.gsub!(REDUNDANT_LINE_BREAK_REGEX, '\1\2')
+        end
+        str
+      end
+
     end
 
   end

data/lib/kramdown/version.rb

@@ -10,6 +10,6 @@
 module Kramdown
 
   # The kramdown version.
-  VERSION = '2.0.0.beta2'
+  VERSION = '2.3.0'
 
 end

data/man/man1/kramdown.1

@@ -1,5 +1,5 @@
 .\" generated by kramdown
-.TH "KRAMDOWN" "1" "November 2016"
+.TH "KRAMDOWN" "1" "January 2019"
 .SH NAME
 kramdown \- a fast, pure\-Ruby Markdown\-superset converter
 .SH "SYNOPSIS"
@@ -20,6 +20,13 @@ Specify the input format\. Available input formats: \fIkramdown\fP (this is the
 \fB\-o\fP \fIFORMAT\fP, \fB\-\-output\fP \fIFORMAT\fP
 Specify one or more output formats separated by commas: \fIhtml\fP (default), \fIkramdown\fP, \fIlatex\fP, \fIman\fP or \fIremove_html_tags\fP\&\. The converter \fIpdf\fP is available through the \fBkramdown\-converter\-pdf\fP gem\.
 .TP
+\fB\-x\fP \fIEXT\fP, \fB\-\-extension\fP \fIEXT\fP
+Load one or more extensions\. The name of the extension should not include the \fBkramdown\-\fP prefix, e\.g\. just \fBparser\-gfm\fP\&\. Multiple extensions can be loaded by separating them with commas\.
+.RS
+.P
+Note: This option has to be used before any other options that rely on the extension already being loaded\.
+.RE
+.TP
 \fB\-\-no\-config\-file\fP
 Do not read any configuration file\. Default behavior is to check for a configuration file and read it if it exists\.
 .TP
@@ -102,6 +109,22 @@ This option can be used to specify the number that is used for the first footnot
 Default: 1 Used by: HTML converter
 .RE
 .TP
+\fB\-\-footnote\-prefix\fP \fIARG\fP
+Prefix used for footnote IDs
+.RS
+.P
+This option can be used to set a prefix for footnote IDs\. This is useful when rendering multiple documents into the same output file to avoid duplicate IDs\. The prefix should only contain characters that are valid in an ID!
+.P
+Default: \[u2018]\[u2019] Used by: HTML
+.RE
+.TP
+\fB\-\-forbidden\-inline\-options\fP \fIARG\fP
+Defines the options that may not be set using the {::options} extension
+.RS
+.P
+Default: template Used by: HTML converter
+.RE
+.TP
 \fB\-\-header\-offset\fP \fIARG\fP
 Sets the output offset for headers
 .RS
@@ -198,6 +221,13 @@ If this option is \fBtrue\fP, the RemoveHtmlTags converter removes block HTML ta
 Default: true Used by: RemoveHtmlTags converter
 .RE
 .TP
+\fB\-\-[no\-]remove\-line\-breaks\-for\-cjk\fP
+Specifies whether line breaks should be removed between CJK characters
+.RS
+.P
+Default: false Used by: HTML converter
+.RE
+.TP
 \fB\-\-[no\-]remove\-span\-html\-tags\fP
 Remove span HTML tags
 .RS

data/test/test_files.rb

@@ -11,6 +11,7 @@ require 'minitest/autorun'
 require 'kramdown'
 require 'yaml'
 require 'tmpdir'
+require 'open3'
 
 begin
   require 'kramdown/converter/syntax_highlighter/rouge'
@@ -72,6 +73,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/rouge/multiple.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/highlighting.html', # bc of span elements inside code element
       'test/testcases/block/06_codeblock/highlighting-opts.html', # bc of span elements inside code element
+      'test/testcases/block/06_codeblock/guess_lang_css_class.html', # bc of double surrounding <div>
       'test/testcases/block/12_extension/options3.html', # bc of rouge
       'test/testcases/block/14_table/empty_tag_in_cell.html', # bc of tidy
       'test/testcases/block/15_math/mathjax_preview.html', # bc of mathjax preview
@@ -110,14 +112,10 @@ class TestFiles < Minitest::Test
   end
 
   def tidy_output(out)
-    cmd = "tidy -q --doctype omit #{RUBY_VERSION >= '1.9' ? '-utf8' : '-raw'} 2>/dev/null"
-    result = IO.popen(cmd, 'r+') do |io|
-      io.write(out)
-      io.close_write
-      io.read
-    end
-    if $?.exitstatus == 2
-      raise "Problem using tidy"
+    cmd = "tidy -q --doctype omit -utf8"
+    result, error, status = Open3.capture3(cmd, stdin_data: out)
+    if status.exitstatus == 2
+      raise "Problem using tidy: #{error}"
     end
     result
   end
@@ -134,6 +132,8 @@ class TestFiles < Minitest::Test
       'test/testcases/span/04_footnote/markers.text', # bc of footnote in header
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_name_with_dash.text',
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_any_char.text',
+      'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text', # latex unicode support
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_LATEX_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -177,6 +177,9 @@ class TestFiles < Minitest::Test
       'test/testcases/span/05_html/mark_element.text', # bc of tidy
       'test/testcases/block/09_html/xml.text', # bc of tidy
       'test/testcases/span/05_html/xml.text', # bc of tidy
+      'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text',
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_TEXT_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -209,6 +212,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/whitespace.html', # bc of entity to char conversion
       'test/testcases/block/06_codeblock/rouge/simple.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/rouge/multiple.html', # bc of double surrounding <div>
+      'test/testcases/block/06_codeblock/guess_lang_css_class.html', # bc of double surrounding <div>
       'test/testcases/block/06_codeblock/highlighting.html', # bc of span elements inside code element
       'test/testcases/block/06_codeblock/highlighting-opts.html', # bc of span elements inside code element
       'test/testcases/block/11_ial/simple.html',           # bc of change of ordering of attributes in header
@@ -218,13 +222,16 @@ class TestFiles < Minitest::Test
       'test/testcases/block/16_toc/toc_exclude.html',      # bc of different attribute ordering
       'test/testcases/span/autolinks/url_links.html',      # bc of quot entity being converted to char
       'test/testcases/block/14_table/empty_tag_in_cell.html', # bc of tidy
-      'test/testcases/block/15_math/mathjax_preview.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_simple.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_as_code.html', # bc of mathjax preview
       'test/testcases/span/01_link/link_defs_with_ial.html', # bc of attribute ordering
       'test/testcases/span/05_html/mark_element.html', # bc of tidy
       'test/testcases/block/09_html/xml.html', # bc of tidy
       'test/testcases/span/05_html/xml.html', # bc of tidy
+      'test/testcases/block/03_paragraph/standalone_image.html', # bc of standalone image
+      'test/testcases/block/15_math/normal.html', # bc of mathjax and HTML parser
+      'test/testcases/block/15_math/gh_128.html', # bc of mathjax and HTML parser
+      'test/testcases/span/04_footnote/backlink_inline.html', # bc of mathjax
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
+      'test/testcases/block/09_html/processing_instruction.html', # bc of PI
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.html'].each do |html_file|
       next if EXCLUDE_HTML_KD_FILES.any? {|f| html_file =~ /#{f}$/ }

data/test/test_location.rb

@@ -18,7 +18,7 @@ describe 'location' do
   def check_element_for_location(element)
     if (match = /^line-(\d+)/.match(element.attr['class'] || ''))
       expected_line = match[1].to_i
-      element.options[:location].must_equal(expected_line)
+      assert_equal(expected_line, element.options[:location])
     end
     element.children.each do |child|
       check_element_for_location(child)
@@ -187,7 +187,7 @@ describe 'location' do
 *[duplicate]: The second definition
     )
     doc = Kramdown::Document.new(test_string.strip)
-    doc.warnings.must_equal ["Duplicate abbreviation ID 'duplicate' on line 4 - overwriting"]
+    assert_equal(["Duplicate abbreviation ID 'duplicate' on line 4 - overwriting"], doc.warnings)
   end
 
   it 'handles abbreviations' do