kramdown 2.1.0 → 2.2.0

data/lib/kramdown/parser/kramdown.rb

@@ -79,6 +79,8 @@ module Kramdown
         @span_parsers =  [:emphasis, :codespan, :autolink, :span_html, :footnote_marker, :link,
                           :smart_quotes, :inline_math, :span_extensions, :html_entity,
                           :typographic_syms, :line_break, :escaped_chars]
+
+        @span_pattern_cache ||= Hash.new { |h, k| h[k] = {} }
       end
       private_class_method(:new, :allocate)
 
@@ -195,6 +197,11 @@ module Kramdown
         end.flatten!
       end
 
+      def span_pattern_cache(stop_re, span_start)
+        @span_pattern_cache[stop_re][span_start] ||= /(?=#{Regexp.union(stop_re, span_start)})/
+      end
+      private :span_pattern_cache
+
       # Parse all span-level elements in the source string of @src into +el+.
       #
       # If the parameter +stop_re+ (a regexp) is used, parsing is immediately stopped if the regexp
@@ -213,7 +220,7 @@ module Kramdown
         span_start, span_start_re = span_parser_regexps(parsers) if parsers
         parsers ||= @span_parsers
 
-        used_re = (stop_re.nil? ? span_start_re : /(?=#{Regexp.union(stop_re, span_start)})/)
+        used_re = (stop_re.nil? ? span_start_re : span_pattern_cache(stop_re, span_start))
         stop_re_found = false
         while !@src.eos? && !stop_re_found
           if (result = @src.scan_until(used_re))

data/lib/kramdown/parser/kramdown/autolink.rb

@@ -11,8 +11,8 @@ module Kramdown
   module Parser
     class Kramdown
 
-      ACHARS = '[[:alnum:]]_'
-      AUTOLINK_START_STR = "<((mailto|https?|ftps?):.+?|[-.#{ACHARS}]+@[-#{ACHARS}]+(?:\.[-#{ACHARS}]+)*\.[a-z]+)>"
+      ACHARS = '[[:alnum:]]-_.'
+      AUTOLINK_START_STR = "<((mailto|https?|ftps?):.+?|[#{ACHARS}]+?@[#{ACHARS}]+?)>"
       AUTOLINK_START = /#{AUTOLINK_START_STR}/u
 
       # Parse the autolink at the current location.

data/lib/kramdown/parser/kramdown/codespan.rb

@@ -25,8 +25,18 @@ module Kramdown
           return
         end
 
-        if (text = @src.scan_until(/#{result}/))
-          text.sub!(/#{result}\Z/, '')
+        # assign static regex to avoid allocating the same on every instance
+        # where +result+ equals a single-backtick. Interpolate otherwise.
+        if result == '`'
+          scan_pattern = /`/
+          str_sub_pattern = /`\Z/
+        else
+          scan_pattern = /#{result}/
+          str_sub_pattern = /#{result}\Z/
+        end
+
+        if (text = @src.scan_until(scan_pattern))
+          text.sub!(str_sub_pattern, '')
           unless simple
             text = text[1..-1] if text[0..0] == ' '
             text = text[0..-2] if text[-1..-1] == ' '

data/lib/kramdown/parser/kramdown/emphasis.rb

@@ -22,7 +22,7 @@ module Kramdown
         element = (result.length == 2 ? :strong : :em)
         type = result[0..0]
 
-        if (type == '_' && @src.pre_match =~ /[[:alpha:]-]\z/) || @src.check(/\s/) ||
+        if (type == '_' && @src.pre_match =~ /[[:alpha:]]-?[[:alpha:]]*\z/) || @src.check(/\s/) ||
             @tree.type == element || @stack.any? {|el, _| el.type == element }
           add_text(result)
           return

data/lib/kramdown/parser/kramdown/header.rb

@@ -31,7 +31,7 @@ module Kramdown
       def parse_atx_header
         return false unless after_block_boundary?
         text, id = parse_header_contents
-        text.sub!(/[\t ]#+\z/, '') && text.rstrip!
+        text.sub!(/(?<!\\)#+\z/, '') && text.rstrip!
         return false if text.empty?
         add_header(@src["level"].length, text, id)
         true

data/lib/kramdown/parser/kramdown/html.rb

@@ -79,12 +79,12 @@ module Kramdown
           @src.scan(TRAILING_WHITESPACE)
           true
         else
-          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          if @src.check(/^#{OPT_SPACE}#{HTML_TAG_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             @src.pos += @src.matched_size
             handle_html_start_tag(line, &method(:handle_kramdown_html_tag))
             Kramdown::Parser::Html::ElementConverter.convert(@root, @tree.children.last) if @options[:html_to_native]
             true
-          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
+          elsif @src.check(/^#{OPT_SPACE}#{HTML_TAG_CLOSE_RE}/o) && !HTML_SPAN_ELEMENTS.include?(@src[1].downcase)
             name = @src[1].downcase
 
             if @tree.type == :html_element && @tree.value == name

data/lib/kramdown/parser/kramdown/list.rb

@@ -44,8 +44,10 @@ module Kramdown
         [content, indentation, *PARSE_FIRST_LIST_LINE_REGEXP_CACHE[indentation]]
       end
 
-      LIST_START_UL = /^(#{OPT_SPACE}[+*-])([\t| ].*?\n)/
-      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)([\t| ].*?\n)/
+      PATTERN_TAIL = /[\t| ].*?\n/
+
+      LIST_START_UL = /^(#{OPT_SPACE}[+*-])(#{PATTERN_TAIL})/
+      LIST_START_OL = /^(#{OPT_SPACE}\d+\.)(#{PATTERN_TAIL})/
       LIST_START = /#{LIST_START_UL}|#{LIST_START_OL}/
 
       # Parse the ordered or unordered list at the current location.
@@ -77,11 +79,7 @@ module Kramdown
               ''
             end
 
-            list_start_re = if type == :ul
-                              /^( {0,#{[3, indentation - 1].min}}[+*-])([\t| ].*?\n)/
-                            else
-                              /^( {0,#{[3, indentation - 1].min}}\d+\.)([\t| ].*?\n)/
-                            end
+            list_start_re = fetch_pattern(type, indentation)
             nested_list_found = (item.value =~ LIST_START)
             last_is_blank = false
             item.value = [item.value]
@@ -148,7 +146,7 @@ module Kramdown
       end
       define_parser(:list, LIST_START)
 
-      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)([\t| ].*?\n)/
+      DEFINITION_LIST_START = /^(#{OPT_SPACE}:)(#{PATTERN_TAIL})/
 
       # Parse the ordered or unordered list at the current location.
       def parse_definition_list
@@ -198,7 +196,7 @@ module Kramdown
               ''
             end
 
-            def_start_re = /^( {0,#{[3, indentation - 1].min}}:)([\t| ].*?\n)/
+            def_start_re = fetch_pattern(:dl, indentation)
             first_as_para = false
             last_is_blank = false
           elsif @src.check(EOB_MARKER)
@@ -252,6 +250,35 @@ module Kramdown
       end
       define_parser(:definition_list, DEFINITION_LIST_START)
 
+      private
+
+      # precomputed patterns for indentations 1..4 and fallback expression
+      # to compute pattern when indentation is outside the 1..4 range.
+      def fetch_pattern(type, indentation)
+        if type == :ul
+          case indentation
+          when 1 then %r/^( {0}[+*-])(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}[+*-])(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}[+*-])(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}[+*-])(#{PATTERN_TAIL})/o
+          end
+        elsif type == :ol
+          case indentation
+          when 1 then %r/^( {0}\d+\.)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}\d+\.)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}\d+\.)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}\d+\.)(#{PATTERN_TAIL})/o
+          end
+        elsif type == :dl
+          case indentation
+          when 1 then %r/^( {0}:)(#{PATTERN_TAIL})/o
+          when 2 then %r/^( {0,1}:)(#{PATTERN_TAIL})/o
+          when 3 then %r/^( {0,2}:)(#{PATTERN_TAIL})/o
+          else %r/^( {0,3}:)(#{PATTERN_TAIL})/o
+          end
+        end
+      end
+
     end
   end
 end

data/lib/kramdown/parser/kramdown/math.rb

@@ -21,7 +21,7 @@ module Kramdown
         if !after_block_boundary?
           return false
         elsif @src[1]
-          @src.scan(/^#{OPT_SPACE}\\/) if @src[3]
+          @src.scan(/^#{OPT_SPACE}\\/o) if @src[3]
           return false
         end
 

data/lib/kramdown/parser/kramdown/table.rb

@@ -132,8 +132,8 @@ module Kramdown
               pipe_on_line = false
             end
           else
-            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/
-            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/)
+            break if lines.size > 1 && !pipe_on_line && lines.first !~ /^#{TABLE_PIPE_CHECK}/o
+            pipe_on_line = (lines.size > 1 ? false : pipe_on_line) || (lines.last =~ /^#{TABLE_PIPE_CHECK}/o)
           end
         end
         @src.revert_pos(saved_pos) and return false unless pipe_on_line

data/lib/kramdown/utils/html.rb

@@ -68,6 +68,13 @@ module Kramdown
         str.gsub(ESCAPE_RE_FROM_TYPE[type]) {|m| ESCAPE_MAP[m] || m }
       end
 
+      REDUNDANT_LINE_BREAK_REGEX = /([\p{Han}\p{Hiragana}\p{Katakana}]+)\n([\p{Han}\p{Hiragana}\p{Katakana}]+)/u
+      def fix_cjk_line_break(str)
+        while str.gsub!(REDUNDANT_LINE_BREAK_REGEX, '\1\2')
+        end
+        str
+      end
+
     end
 
   end

data/lib/kramdown/version.rb

@@ -10,6 +10,6 @@
 module Kramdown
 
   # The kramdown version.
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 
 end

data/man/man1/kramdown.1

@@ -214,6 +214,13 @@ If this option is \fBtrue\fP, the RemoveHtmlTags converter removes block HTML ta
 Default: true Used by: RemoveHtmlTags converter
 .RE
 .TP
+\fB\-\-[no\-]remove\-line\-breaks\-for\-cjk\fP
+Specifies whether line breaks should be removed between CJK characters
+.RS
+.P
+Default: false Used by: HTML converter
+.RE
+.TP
 \fB\-\-[no\-]remove\-span\-html\-tags\fP
 Remove span HTML tags
 .RS

data/test/test_files.rb

@@ -11,6 +11,7 @@ require 'minitest/autorun'
 require 'kramdown'
 require 'yaml'
 require 'tmpdir'
+require 'open3'
 
 begin
   require 'kramdown/converter/syntax_highlighter/rouge'
@@ -111,14 +112,10 @@ class TestFiles < Minitest::Test
   end
 
   def tidy_output(out)
-    cmd = "tidy -q --doctype omit #{RUBY_VERSION >= '1.9' ? '-utf8' : '-raw'} 2>/dev/null"
-    result = IO.popen(cmd, 'r+') do |io|
-      io.write(out)
-      io.close_write
-      io.read
-    end
-    if $?.exitstatus == 2
-      raise "Problem using tidy"
+    cmd = "tidy -q --doctype omit -utf8"
+    result, error, status = Open3.capture3(cmd, stdin_data: out)
+    if status.exitstatus == 2
+      raise "Problem using tidy: #{error}"
     end
     result
   end
@@ -136,6 +133,7 @@ class TestFiles < Minitest::Test
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_name_with_dash.text',
       'test/testcases/block/06_codeblock/with_lang_in_fenced_block_any_char.text',
       'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text', # latex unicode support
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_LATEX_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -180,6 +178,8 @@ class TestFiles < Minitest::Test
       'test/testcases/block/09_html/xml.text', # bc of tidy
       'test/testcases/span/05_html/xml.text', # bc of tidy
       'test/testcases/block/03_paragraph/standalone_image.text', # bc of standalone image
+      'test/testcases/cjk-line-break.text',
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.text'].each do |text_file|
       next if EXCLUDE_TEXT_FILES.any? {|f| text_file =~ /#{f}$/ }
@@ -222,14 +222,15 @@ class TestFiles < Minitest::Test
       'test/testcases/block/16_toc/toc_exclude.html',      # bc of different attribute ordering
       'test/testcases/span/autolinks/url_links.html',      # bc of quot entity being converted to char
       'test/testcases/block/14_table/empty_tag_in_cell.html', # bc of tidy
-      'test/testcases/block/15_math/mathjax_preview.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_simple.html', # bc of mathjax preview
-      'test/testcases/block/15_math/mathjax_preview_as_code.html', # bc of mathjax preview
       'test/testcases/span/01_link/link_defs_with_ial.html', # bc of attribute ordering
       'test/testcases/span/05_html/mark_element.html', # bc of tidy
       'test/testcases/block/09_html/xml.html', # bc of tidy
       'test/testcases/span/05_html/xml.html', # bc of tidy
       'test/testcases/block/03_paragraph/standalone_image.html', # bc of standalone image
+      'test/testcases/block/15_math/normal.html', # bc of mathjax and HTML parser
+      'test/testcases/block/15_math/gh_128.html', # bc of mathjax and HTML parser
+      'test/testcases/span/04_footnote/backlink_inline.html', # bc of mathjax
+      'test/testcases/block/09_html/standalone_image_in_div.html', # bc of standalone image
     ].compact
     Dir[File.dirname(__FILE__) + '/testcases/**/*.html'].each do |html_file|
       next if EXCLUDE_HTML_KD_FILES.any? {|f| html_file =~ /#{f}$/ }

data/test/testcases/block/04_header/atx_header.html

@@ -26,6 +26,12 @@
 ### not a header</p>
 </blockquote>
 
+<h1>header</h1>
+
+<h1>header</h1>
+
+<h1>header</h1>
+
 <h1>header #</h1>
 
 <h1>header</h1>

data/test/testcases/block/04_header/atx_header.text

@@ -23,6 +23,12 @@ paragraph
 > blockquote
 ### not a header
 
+# header #
+
+# header#
+
+#header#
+
 # header \#
 
 # header

data/test/testcases/block/09_html/standalone_image_in_div.htmlinput

@@ -0,0 +1,7 @@
+<div class="example-1">
+  <img src="src.png" alt="inside" />
+</div>
+
+<div class="example-2">
+  <a href="website.html">text</a>
+</div>

data/test/testcases/block/09_html/standalone_image_in_div.text

@@ -0,0 +1,8 @@
+<div class="example-1" markdown="1">
+![inside](src.png)
+</div>
+
+<div class="example-2" markdown="1">
+[text](website.html)
+</div>
+

data/test/testcases/block/12_extension/options.html

@@ -10,12 +10,12 @@ some <span>*para*</span>
   <p>some <span><em>para</em></span></p>
 </div>
 
-<p>Some text<sup id="fnref:ab"><a href="#fn:ab" class="footnote">10</a></sup>.</p>
+<p>Some text<sup id="fnref:ab" role="doc-noteref"><a href="#fn:ab" class="footnote">10</a></sup>.</p>
 
-<div class="footnotes">
+<div class="footnotes" role="doc-endnotes">
   <ol start="10">
-    <li id="fn:ab">
-      <p>Some text. <a href="#fnref:ab" class="reversefootnote">&#8617;</a></p>
+    <li id="fn:ab" role="doc-endnote">
+      <p>Some text. <a href="#fnref:ab" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
     </li>
   </ol>
 </div>

data/test/testcases/block/12_extension/options2.html

@@ -1,10 +1,10 @@
 
-<p>Some text<sup id="fnref:ab"><a href="#fn:ab" class="footnote">1</a></sup>.</p>
+<p>Some text<sup id="fnref:ab" role="doc-noteref"><a href="#fn:ab" class="footnote">1</a></sup>.</p>
 
-<div class="footnotes">
+<div class="footnotes" role="doc-endnotes">
   <ol>
-    <li id="fn:ab">
-      <p>Some text. <a href="#fnref:ab" class="reversefootnote">&#8617;</a></p>
+    <li id="fn:ab" role="doc-endnote">
+      <p>Some text. <a href="#fnref:ab" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
     </li>
   </ol>
 </div>

data/test/testcases/block/14_table/table_with_footnote.html

@@ -1,7 +1,7 @@
 <table>
   <tbody>
     <tr>
-      <td>this is <sup id="fnref:1"><a href="#fn:1" class="footnote">1</a></sup></td>
+      <td>this is <sup id="fnref:1" role="doc-noteref"><a href="#fn:1" class="footnote">1</a></sup></td>
       <td>a table</td>
     </tr>
     <tr>
@@ -11,15 +11,15 @@
   </tbody>
 </table>
 
-<div class="footnotes">
+<div class="footnotes" role="doc-endnotes">
   <ol>
-    <li id="fn:1">
+    <li id="fn:1" role="doc-endnote">
       <p>Something</p>
 
       <blockquote>
         <p>special here</p>
       </blockquote>
-      <p><a href="#fnref:1" class="reversefootnote">&#8617;</a></p>
+      <p><a href="#fnref:1" class="reversefootnote" role="doc-backlink">&#8617;</a></p>
     </li>
   </ol>
 </div>

data/test/testcases/block/15_math/gh_128.html

@@ -1,2 +1 @@
-<script type="math/tex; mode=display">% <![CDATA[
-alert('a') alert('b<') %]]></script>
+\[&lt;script&gt;alert('a')&lt;/script&gt; &lt;script&gt;alert('b&lt;')&lt;/script&gt;\]

data/test/testcases/block/15_math/normal.html

@@ -1,29 +1,30 @@
 <p>This is a para.
-<script type="math/tex">\text{LaTeX} \lambda_5</script></p>
+\(\text{LaTeX} \lambda_5\)</p>
 
-<script type="math/tex; mode=display">\lambda_5 = \alpha + 4</script>
+\[\lambda_5 = \alpha + 4\]
 
-<p><script type="math/tex">\lambda_\alpha > 5</script>
+<p>\(\lambda_\alpha &gt; 5\)
 This is a para.</p>
 
-<script type="math/tex; mode=display">% <![CDATA[
-\begin{align*}
-&=5 \\
-&=6 \\
-\end{align*} %]]></script>
+\[\begin{align*}
+&amp;=5 \\
+&amp;=6 \\
+\end{align*}\]
 
-<script type="math/tex; mode=display">5+5</script>
+\[5+5\]
 
-<script type="math/tex; mode=display">5+5</script>
+\[5+5\]
 
-<script type="math/tex; mode=display">5+5</script>
+\[5+5\]
 
-<script type="math/tex; mode=display">5+5</script>
+\[5+5\]
 
 <pre><code>$$5+5$$
 </code></pre>
 
-<script type="math/tex; mode=display">5+5</script>
-<script type="math/tex; mode=display">5+5</script>
+<div class="cls">\[5+5\]
+</div>
+<div class="cls">\[5+5\]
+</div>
 
-<script type="math/tex; mode=display">|x| = 5</script>
+\[|x| = 5\]