kramdown 2.0.0.beta2 → 2.3.0

data/lib/kramdown/converter/syntax_highlighter.rb

@@ -42,7 +42,7 @@ module Kramdown
     #
     # == Special Implementation Details
     #
-    # HTML converter:: If the syntax highlighter is used with a HTML converter, it should return
+    # HTML converter:: If the syntax highlighter is used with an HTML converter, it should return
     #                  :block type text correctly wrapped (i.e. normally inside a pre-tag, but may
     #                  also be a table-tag or just a div-tag) but :span type text *without* a
     #                  code-tag!

data/lib/kramdown/converter/syntax_highlighter/rouge.rb

@@ -24,8 +24,11 @@ module Kramdown::Converter::SyntaxHighlighter
     def self.call(converter, text, lang, type, call_opts)
       opts = options(converter, type)
       call_opts[:default_lang] = opts[:default_lang]
+      return nil unless lang || opts[:default_lang] || opts[:guess_lang]
+
       lexer = ::Rouge::Lexer.find_fancy(lang || opts[:default_lang], text)
-      return nil if opts[:disable] || !lexer || lexer.tag == "plaintext"
+      return nil if opts[:disable] || !lexer || (lexer.tag == "plaintext" && !opts[:guess_lang])
+
       opts[:css_class] ||= 'highlight' # For backward compatibility when using Rouge 2.0
       formatter = formatter_class(opts).new(opts)
       formatter.format(lexer.lex(text))
@@ -42,18 +45,26 @@ module Kramdown::Converter::SyntaxHighlighter
       cache = converter.data[:syntax_highlighter_rouge] = {}
 
       opts = converter.options[:syntax_highlighter_opts].dup
-      span_opts = (opts.delete(:span) || {}).dup
-      block_opts = (opts.delete(:block) || {}).dup
-      [span_opts, block_opts].each do |hash|
-        hash.keys.each do |k|
-          hash[k.kind_of?(String) ? Kramdown::Options.str_to_sym(k) : k] = hash.delete(k)
-        end
-      end
 
-      cache[:span] = opts.merge(span_opts).update(wrap: false)
+      span_opts = opts.delete(:span)&.dup || {}
+      block_opts = opts.delete(:block)&.dup || {}
+      normalize_keys(span_opts)
+      normalize_keys(block_opts)
+
+      cache[:span] = opts.merge(span_opts)
+      cache[:span][:wrap] = false
+
       cache[:block] = opts.merge(block_opts)
     end
 
+    def self.normalize_keys(hash)
+      return if hash.empty?
+
+      hash.keys.each do |k|
+        hash[k.kind_of?(String) ? Kramdown::Options.str_to_sym(k) : k] = hash.delete(k)
+      end
+    end
+
     def self.formatter_class(opts = {})
       case formatter = opts[:formatter]
       when Class

data/lib/kramdown/element.rb

@@ -14,6 +14,14 @@ module Kramdown
   # kramdown only uses this one class for representing all available elements in an element tree
   # (paragraphs, headers, emphasis, ...). The type of element can be set via the #type accessor.
   #
+  # The root of a kramdown element tree has to be an element of type :root. It needs to have certain
+  # option keys set so that conversions work correctly. If only a part of a tree should be
+  # converted, duplicate the root node and assign the #children appropriately, e.g:
+  #
+  #   root = doc.root
+  #   new_root = root.dup
+  #   new_root.children = [root.children[0]]  # assign new array with elements to convert
+  #
   # Following is a description of all supported element types.
   #
   # Note that the option :location may contain the start line number of an element in the source
@@ -42,6 +50,7 @@ module Kramdown
   #
   # :options:: This key may be used to store options that were set during parsing of the document.
   #
+  # :footnote_count:: This key stores the number of actually referenced footnotes of the document.
   #
   # === :blank
   #
@@ -500,9 +509,11 @@ module Kramdown
     end
 
     def inspect #:nodoc:
-      "<kd:#{@type}#{@value.nil? ? '' : ' ' + @value.inspect} " \
-        "#{@attr.inspect}#{options.empty? ? '' : ' ' + @options.inspect} " \
-        "#{@children.empty? ? '' : ' ' + @children.inspect}>"
+      "<kd:#{@type}" \
+        "#{value.nil? ? '' : ' value=' + value.inspect}" \
+        "#{attr.empty? ? '' : ' attr=' + attr.inspect}" \
+        "#{options.empty? ? '' : ' options=' + options.inspect}" \
+        "#{children.empty? ? '' : ' children=' + children.inspect}>"
     end
 
     CATEGORY = {} # :nodoc:
@@ -519,6 +530,22 @@ module Kramdown
       CATEGORY[el.type] || el.options[:category]
     end
 
+    # syntactic sugar to simplify calls such as +Kramdown::Element.category(el) == :block+ with
+    # +el.block?+.
+    #
+    # Returns boolean true or false.
+    def block?
+      (CATEGORY[type] || options[:category]) == :block
+    end
+
+    # syntactic sugar to simplify calls such as +Kramdown::Element.category(el) == :span+ with
+    # +el.span?+.
+    #
+    # Returns boolean true or false.
+    def span?
+      (CATEGORY[type] || options[:category]) == :span
+    end
+
   end
 
 end

data/lib/kramdown/options.rb

@@ -328,7 +328,11 @@ module Kramdown
       Used by: HTML converter, kramdown converter
     EOF
 
-    define(:toc_levels, Object, (1..6).to_a, <<~EOF) do |val|
+    TOC_LEVELS_RANGE = (1..6).freeze
+    TOC_LEVELS_ARRAY = TOC_LEVELS_RANGE.to_a.freeze
+    private_constant :TOC_LEVELS_RANGE, :TOC_LEVELS_ARRAY
+
+    define(:toc_levels, Object, TOC_LEVELS_ARRAY, <<~EOF) do |val|
       Defines the levels that are used for the table of contents
 
       The individual levels can be specified by separating them with commas
@@ -347,12 +351,20 @@ module Kramdown
         else
           raise Kramdown::Error, "Invalid syntax for option toc_levels"
         end
-      when Array, Range
-        val = val.map(&:to_i).uniq
+      when Array
+        unless val.eql?(TOC_LEVELS_ARRAY)
+          val = val.map(&:to_i).uniq
+        end
+      when Range
+        if val.eql?(TOC_LEVELS_RANGE)
+          val = TOC_LEVELS_ARRAY
+        else
+          val = val.map(&:to_i).uniq
+        end
       else
         raise Kramdown::Error, "Invalid type #{val.class} for option toc_levels"
       end
-      if val.any? {|i| !(1..6).cover?(i) }
+      if val.any? {|i| !TOC_LEVELS_RANGE.cover?(i) }
         raise Kramdown::Error, "Level numbers for option toc_levels have to be integers from 1 to 6"
       end
       val
@@ -377,7 +389,11 @@ module Kramdown
       simple_array_validator(val, :latex_headers, 6)
     end
 
-    define(:smart_quotes, Object, %w[lsquo rsquo ldquo rdquo], <<~EOF) do |val|
+    SMART_QUOTES_ENTITIES = %w[lsquo rsquo ldquo rdquo].freeze
+    SMART_QUOTES_STR = SMART_QUOTES_ENTITIES.join(',').freeze
+    private_constant :SMART_QUOTES_ENTITIES, :SMART_QUOTES_STR
+
+    define(:smart_quotes, Object, SMART_QUOTES_ENTITIES, <<~EOF) do |val|
       Defines the HTML entity names or code points for smart quote output
 
       The entities identified by entity name or code point that should be
@@ -388,9 +404,13 @@ module Kramdown
       Default: lsquo,rsquo,ldquo,rdquo
       Used by: HTML/Latex converter
     EOF
-      val = simple_array_validator(val, :smart_quotes, 4)
-      val.map! {|v| Integer(v) rescue v }
-      val
+      if val == SMART_QUOTES_STR || val == SMART_QUOTES_ENTITIES
+        SMART_QUOTES_ENTITIES
+      else
+        val = simple_array_validator(val, :smart_quotes, 4)
+        val.map! {|v| Integer(v) rescue v }
+        val
+      end
     end
 
     define(:typographic_symbols, Object, {}, <<~EOF) do |val|
@@ -550,6 +570,35 @@ module Kramdown
       Used by: HTML converter
     EOF
 
+    define(:footnote_prefix, String, '', <<~EOF)
+      Prefix used for footnote IDs
+
+      This option can be used to set a prefix for footnote IDs. This is useful
+      when rendering multiple documents into the same output file to avoid
+      duplicate IDs. The prefix should only contain characters that are valid
+      in an ID!
+
+      Default: ''
+      Used by: HTML
+    EOF
+
+    define(:remove_line_breaks_for_cjk, Boolean, false, <<~EOF)
+      Specifies whether line breaks should be removed between CJK characters
+
+      Default: false
+      Used by: HTML converter
+    EOF
+
+    define(:forbidden_inline_options, Object, %w[template], <<~EOF) do |val|
+      Defines the options that may not be set using the {::options} extension
+
+      Default: template
+      Used by: HTML converter
+    EOF
+      val.map! {|item| item.kind_of?(String) ? str_to_sym(item) : item }
+      simple_array_validator(val, :forbidden_inline_options)
+    end
+
   end
 
 end

data/lib/kramdown/parser/base.rb

@@ -93,7 +93,9 @@ module Kramdown
           raise "The source text contains invalid characters for the used encoding #{source.encoding}"
         end
         source = source.encode('UTF-8')
-        source.gsub(/\r\n?/, "\n").chomp + "\n"
+        source.gsub!(/\r\n?/, "\n")
+        source.chomp!
+        source << "\n"
       end
 
       # This helper method adds the given +text+ either to the last element in the +tree+ if it is a

data/lib/kramdown/parser/html.rb

@@ -16,7 +16,7 @@ module Kramdown
 
   module Parser
 
-    # Used for parsing a HTML document.
+    # Used for parsing an HTML document.
     #
     # The parsing code is in the Parser module that can also be used by other parsers.
     class Html < Base
@@ -286,7 +286,7 @@ module Kramdown
           src = Kramdown::Utils::StringScanner.new(raw)
           result = []
           until src.eos?
-            if (tmp = src.scan_until(/(?=#{HTML_ENTITY_RE})/))
+            if (tmp = src.scan_until(/(?=#{HTML_ENTITY_RE})/o))
               result << Element.new(:text, tmp)
               src.scan(HTML_ENTITY_RE)
               val = src[1] || (src[2]&.to_i) || src[3].hex
@@ -324,7 +324,7 @@ module Kramdown
           tmp = []
           last_is_p = false
           el.children.each do |c|
-            if Element.category(c) != :block || c.type == :text
+            if !c.block? || c.type == :text
               unless last_is_p
                 tmp << Element.new(:p, nil, nil, transparent: true)
                 last_is_p = true
@@ -354,8 +354,8 @@ module Kramdown
           el.children = el.children.reject do |c|
             i += 1
             c.type == :text && c.value.strip.empty? &&
-              (i == 0 || i == el.children.length - 1 || (Element.category(el.children[i - 1]) == :block &&
-                                                         Element.category(el.children[i + 1]) == :block))
+              (i == 0 || i == el.children.length - 1 || ((el.children[i - 1]).block? &&
+                                                         (el.children[i + 1]).block?))
           end
         end
 
@@ -581,11 +581,11 @@ module Kramdown
         @src = Kramdown::Utils::StringScanner.new(adapt_source(source))
 
         while true
-          if (result = @src.scan(/\s*#{HTML_INSTRUCTION_RE}/))
+          if (result = @src.scan(/\s*#{HTML_INSTRUCTION_RE}/o))
             @tree.children << Element.new(:xml_pi, result.strip, nil, category: :block)
-          elsif (result = @src.scan(/\s*#{HTML_DOCTYPE_RE}/))
+          elsif (result = @src.scan(/\s*#{HTML_DOCTYPE_RE}/o))
             # ignore the doctype
-          elsif (result = @src.scan(/\s*#{HTML_COMMENT_RE}/))
+          elsif (result = @src.scan(/\s*#{HTML_COMMENT_RE}/o))
             @tree.children << Element.new(:xml_comment, result.strip, nil, category: :block)
           else
             break

data/lib/kramdown/parser/kramdown.rb

@@ -79,6 +79,8 @@ module Kramdown
         @span_parsers =  [:emphasis, :codespan, :autolink, :span_html, :footnote_marker, :link,
                           :smart_quotes, :inline_math, :span_extensions, :html_entity,
                           :typographic_syms, :line_break, :escaped_chars]
+
+        @span_pattern_cache ||= Hash.new { |h, k| h[k] = {} }
       end
       private_class_method(:new, :allocate)
 
@@ -93,11 +95,13 @@ module Kramdown
           update_tree(data[:content])
           replace_abbreviations(data[:content])
         end
+        footnote_count = 0
         @footnotes.each do |name, data|
-          next if data.key?(:marker)
+          (footnote_count += 1; next) if data.key?(:marker)
           line = data[:content].options[:location]
           warning("Footnote definition for '#{name}' on line #{line} is unreferenced - ignoring")
         end
+        @root.options[:footnote_count] = footnote_count
       end
 
       protected
@@ -193,6 +197,11 @@ module Kramdown
         end.flatten!
       end
 
+      def span_pattern_cache(stop_re, span_start)
+        @span_pattern_cache[stop_re][span_start] ||= /(?=#{Regexp.union(stop_re, span_start)})/
+      end
+      private :span_pattern_cache
+
       # Parse all span-level elements in the source string of @src into +el+.
       #
       # If the parameter +stop_re+ (a regexp) is used, parsing is immediately stopped if the regexp
@@ -211,7 +220,7 @@ module Kramdown
         span_start, span_start_re = span_parser_regexps(parsers) if parsers
         parsers ||= @span_parsers
 
-        used_re = (stop_re.nil? ? span_start_re : /(?=#{Regexp.union(stop_re, span_start)})/)
+        used_re = (stop_re.nil? ? span_start_re : span_pattern_cache(stop_re, span_start))
         stop_re_found = false
         while !@src.eos? && !stop_re_found
           if (result = @src.scan_until(used_re))

data/lib/kramdown/parser/kramdown/autolink.rb

@@ -11,8 +11,8 @@ module Kramdown
   module Parser
     class Kramdown
 
-      ACHARS = '[[:alnum:]]_'
-      AUTOLINK_START_STR = "<((mailto|https?|ftps?):.+?|[-.#{ACHARS}]+@[-#{ACHARS}]+(?:\.[-#{ACHARS}]+)*\.[a-z]+)>"
+      ACHARS = '[[:alnum:]]-_.'
+      AUTOLINK_START_STR = "<((mailto|https?|ftps?):.+?|[#{ACHARS}]+?@[#{ACHARS}]+?)>"
       AUTOLINK_START = /#{AUTOLINK_START_STR}/u
 
       # Parse the autolink at the current location.

data/lib/kramdown/parser/kramdown/blank_line.rb

@@ -16,8 +16,8 @@ module Kramdown
       # Parse the blank line at the current postition.
       def parse_blank_line
         @src.pos += @src.matched_size
-        if @tree.children.last && @tree.children.last.type == :blank
-          @tree.children.last.value << @src.matched
+        if (last_child = @tree.children.last) && last_child.type == :blank
+          last_child.value << @src.matched
         else
           @tree.children << new_block_el(:blank, @src.matched)
         end

data/lib/kramdown/parser/kramdown/block_boundary.rb

@@ -19,8 +19,9 @@ module Kramdown
 
       # Return +true+ if we are after a block boundary.
       def after_block_boundary?
-        !@tree.children.last || @tree.children.last.type == :blank ||
-          (@tree.children.last.type == :eob && @tree.children.last.value.nil?) || @block_ial
+        last_child = @tree.children.last
+        !last_child || last_child.type == :blank ||
+          (last_child.type == :eob && last_child.value.nil?) || @block_ial
       end
 
       # Return +true+ if we are before a block boundary.

data/lib/kramdown/parser/kramdown/codespan.rb

@@ -25,8 +25,18 @@ module Kramdown
           return
         end
 
-        if (text = @src.scan_until(/#{result}/))
-          text.sub!(/#{result}\Z/, '')
+        # assign static regex to avoid allocating the same on every instance
+        # where +result+ equals a single-backtick. Interpolate otherwise.
+        if result == '`'
+          scan_pattern = /`/
+          str_sub_pattern = /`\Z/
+        else
+          scan_pattern = /#{result}/
+          str_sub_pattern = /#{result}\Z/
+        end
+
+        if (text = @src.scan_until(scan_pattern))
+          text.sub!(str_sub_pattern, '')
           unless simple
             text = text[1..-1] if text[0..0] == ' '
             text = text[0..-2] if text[-1..-1] == ' '

data/lib/kramdown/parser/kramdown/emphasis.rb

@@ -22,7 +22,7 @@ module Kramdown
         element = (result.length == 2 ? :strong : :em)
         type = result[0..0]
 
-        if (type == '_' && @src.pre_match =~ /[[:alpha:]-]\z/) || @src.check(/\s/) ||
+        if (type == '_' && @src.pre_match =~ /[[:alpha:]]-?[[:alpha:]]*\z/) || @src.check(/\s/) ||
             @tree.type == element || @stack.any? {|el, _| el.type == element }
           add_text(result)
           return

data/lib/kramdown/parser/kramdown/extensions.rb

@@ -39,7 +39,7 @@ module Kramdown
 
       # Update the +ial+ with the information from the inline attribute list +opts+.
       def update_ial_with_ial(ial, opts)
-        (ial[:refs] ||= []) << opts[:refs]
+        (ial[:refs] ||= []).concat(opts[:refs]) if opts.key?(:refs)
         opts.each do |k, v|
           if k == IAL_CLASS_ATTR
             ial[k] = "#{ial[k]} #{v}".lstrip
@@ -110,6 +110,12 @@ module Kramdown
           opts.select do |k, v|
             k = k.to_sym
             if Kramdown::Options.defined?(k)
+              if @options[:forbidden_inline_options].include?(k) ||
+                  k == :forbidden_inline_options
+                warning("Option #{k} may not be set inline")
+                next false
+              end
+
               begin
                 val = Kramdown::Options.parse(k, v)
                 @options[k] = val
@@ -163,10 +169,10 @@ module Kramdown
         elsif @src.check(EXT_BLOCK_START)
           parse_extension_start_tag(:block)
         elsif @src.scan(IAL_BLOCK_START)
-          if @tree.children.last && @tree.children.last.type != :blank &&
-              (@tree.children.last.type != :eob ||
-               [:link_def, :abbrev_def, :footnote_def].include?(@tree.children.last.value))
-            parse_attribute_list(@src[1], @tree.children.last.options[:ial] ||= {})
+          if (last_child = @tree.children.last) && last_child.type != :blank &&
+              (last_child.type != :eob ||
+               [:link_def, :abbrev_def, :footnote_def].include?(last_child.value))
+            parse_attribute_list(@src[1], last_child.options[:ial] ||= {})
             @tree.children << new_block_el(:eob, :ial) unless @src.check(IAL_BLOCK_START)
           else
             parse_attribute_list(@src[1], @block_ial ||= {})
@@ -187,12 +193,12 @@ module Kramdown
         if @src.check(EXT_SPAN_START)
           parse_extension_start_tag(:span)
         elsif @src.check(IAL_SPAN_START)
-          if @tree.children.last && @tree.children.last.type != :text
+          if (last_child = @tree.children.last) && last_child.type != :text
             @src.pos += @src.matched_size
             attr = {}
             parse_attribute_list(@src[1], attr)
-            update_ial_with_ial(@tree.children.last.options[:ial] ||= {}, attr)
-            update_attr_with_ial(@tree.children.last.attr, attr)
+            update_ial_with_ial(last_child.options[:ial] ||= {}, attr)
+            update_attr_with_ial(last_child.attr, attr)
           else
             warning("Found span IAL after text - ignoring it")
             add_text(@src.getch)

data/lib/kramdown/parser/kramdown/header.rb

@@ -8,6 +8,7 @@
 #
 
 require 'kramdown/parser/kramdown/block_boundary'
+require 'rexml/xmltokens'
 
 module Kramdown
   module Parser
@@ -31,7 +32,7 @@ module Kramdown
       def parse_atx_header
         return false unless after_block_boundary?
         text, id = parse_header_contents
-        text.sub!(/[\t ]#+\z/, '') && text.rstrip!
+        text.sub!(/(?<!\\)#+\z/, '') && text.rstrip!
         return false if text.empty?
         add_header(@src["level"].length, text, id)
         true
@@ -40,7 +41,7 @@ module Kramdown
 
       protected
 
-      HEADER_ID = /[\t ]{#(?<id>[A-Za-z][\w:-]*)}\z/
+      HEADER_ID = /[\t ]{#(?<id>#{REXML::XMLTokens::NAME_START_CHAR}#{REXML::XMLTokens::NAME_CHAR}*)}\z/
 
       # Returns header text and optional ID.
       def parse_header_contents