motion-markdown-it 12.0.6 → 13.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74ae9a8bfdbc637f5e2127d34919298661b7d4f32b55830053a0c30012cd1ddf
-  data.tar.gz: 5787ec92937c5e8ef19fead547c0fae14b0de4507c0098cb5e8a79720dab611e
+  metadata.gz: 527ad2365d9f847959cb068d6483944ece97a6f25198b159c342fd09d340e7f1
+  data.tar.gz: de08e3d6331f1fe3e359df0e935cead71a259bb211885084000e6b93ed9cd909
 SHA512:
-  metadata.gz: 72214978791be3e608c627bfa80db56a1ca30df9650a1df85bef0c2a99fb68b822a1464261595779f16385a9e69cf9898fbd15966ba5e00ad969d4a0568c028a
-  data.tar.gz: 5e2ede784f3386d45b930789abb2c0a05ee8bc2dffb707d736c437e48fe722b9084dc75f2032bea6a8d47b723d0daac43428fb059e1ac1f7c6fa92dfad4a83e7
+  metadata.gz: 7abfc3bd88c7f30e93826a1ae27a00d95c8f47f6f9e634a739ddf597c7a0ffdc73fd295fa353f0e306861e5301b0190aba2f8fd1e56e4ffee5d3c61383be7f45
+  data.tar.gz: 10896d6c58b7d9675b52833ef975c77441fb4fa7c295aa6f38325226dd10de80609c892a1969f431bb0ce4b12565feb36a9ab6bdbfe4277cd116f5e43379dd08

data/README.md

@@ -7,7 +7,7 @@ Ruby/RubyMotion version of Markdown-it (CommonMark compliant and extendable)
 
 This gem is a port of the [markdown-it Javascript package](https://github.com/markdown-it/markdown-it) by Vitaly Puzrin and Alex Kocharin. 
 
-_Currently synced with markdown-it 12.0.6_
+_Currently synced with markdown-it 13.0.1_
 
 ---
 
@@ -185,7 +185,7 @@ var md = require('markdown-it')({
   highlight: function (str, lang) {
     if (lang && hljs.getLanguage(lang)) {
       try {
-        return hljs.highlight(lang, str).value;
+        return hljs.highlight(str, { language: lang }).value;
       } catch (__) {}
     }
 
@@ -205,7 +205,7 @@ var md = require('markdown-it')({
     if (lang && hljs.getLanguage(lang)) {
       try {
         return '<pre class="hljs"><code>' +
-               hljs.highlight(lang, str, true).value +
+               hljs.highlight(str, { language: lang, ignoreIllegals: true }).value +
                '</code></pre>';
       } catch (__) {}
     }
@@ -260,7 +260,7 @@ By default all rules are enabled, but can be restricted by options. On plugin
 load all its rules are enabled automatically.
 
 ```js
-// Activate/deactivate rules, with curring
+// Activate/deactivate rules, with currying
 var md = require('markdown-it')()
             .disable([ 'link', 'image' ])
             .enable([ 'link' ])

data/lib/motion-markdown-it/index.rb

@@ -190,7 +190,7 @@ module MarkdownIt
     #   highlight: function (str, lang) {
     #     if (lang && hljs.getLanguage(lang)) {
     #       try {
-    #         return hljs.highlight(lang, str, true).value;
+    #         return hljs.highlight(str, { language: lang, ignoreIllegals: true }).value;
     #       } catch (__) {}
     #     }
     #
@@ -210,7 +210,7 @@ module MarkdownIt
     #     if (lang && hljs.getLanguage(lang)) {
     #       try {
     #         return '<pre class="hljs"><code>' +
-    #                hljs.highlight(lang, str, true).value +
+    #                hljs.highlight(str, { language: lang, ignoreIllegals: true }).value +
     #                '</code></pre>';
     #       } catch (__) {}
     #     }

data/lib/motion-markdown-it/parser_core.rb

@@ -10,12 +10,15 @@ module MarkdownIt
     attr_accessor   :ruler
 
     RULES = [
-      [ 'normalize',      lambda { |state| RulesCore::Normalize.normalize(state) }         ],
+      [ 'normalize',      lambda { |state| RulesCore::Normalize.normalize(state) }      ],
       [ 'block',          lambda { |state| RulesCore::Block.block(state) }              ],
       [ 'inline',         lambda { |state| RulesCore::Inline.inline(state) }            ],
       [ 'linkify',        lambda { |state| RulesCore::Linkify.linkify(state) }          ],
       [ 'replacements',   lambda { |state| RulesCore::Replacements.replace(state) }     ],
       [ 'smartquotes',    lambda { |state| RulesCore::Smartquotes.smartquotes(state) }  ],
+      # `text_join` finds `text_special` tokens (for escape sequences)
+      # and joins them with the rest of the text
+      [ 'text_join',      lambda { |state| RulesCore::TextJoin.text_join(state) }       ],
     ]
 
 

data/lib/motion-markdown-it/parser_inline.rb

@@ -13,6 +13,7 @@ module MarkdownIt
 
     RULES = [
       [ 'text',            lambda { |state, startLine| RulesInline::Text.text(state, startLine) } ],
+      [ 'linkify',         lambda { |state, silent| RulesInline::Linkify.linkify(state, silent) } ],
       [ 'newline',         lambda { |state, startLine| RulesInline::Newline.newline(state, startLine) } ],
       [ 'escape',          lambda { |state, startLine| RulesInline::Escape.escape(state, startLine) } ],
       [ 'backticks',       lambda { |state, startLine| RulesInline::Backticks.backtick(state, startLine) } ],
@@ -25,11 +26,19 @@ module MarkdownIt
       [ 'entity',          lambda { |state, startLine| RulesInline::Entity.entity(state, startLine) } ],
     ]
 
+    # `rule2` ruleset was created specifically for emphasis/strikethrough
+    # post-processing and may be changed in the future.
+    #
+    # Don't use this for anything except pairs (plugins working with `balance_pairs`).
+    #
     RULES2 = [
       [ 'balance_pairs',   lambda { |state| RulesInline::BalancePairs.link_pairs(state) } ],
       [ 'strikethrough',   lambda { |state| RulesInline::Strikethrough.postProcess(state) } ],
       [ 'emphasis',        lambda { |state| RulesInline::Emphasis.postProcess(state) } ],
-      [ 'text_collapse',   lambda { |state| RulesInline::TextCollapse.text_collapse(state) } ]
+      # [ 'text_collapse',   lambda { |state| RulesInline::TextCollapse.text_collapse(state) } ]
+      # rules for pairs separate '**' into its own text tokens, which may be left unused,
+      # rule below merges unused segments back with the rest of the text
+      [ 'fragments_join',  lambda { |state| RulesInline::FragmentsJoin.fragments_join(state) } ]
     ];
 
     #------------------------------------------------------------------------------
@@ -154,4 +163,4 @@ module MarkdownIt
       end
     end
   end
-end
+end

data/lib/motion-markdown-it/presets/commonmark.rb

@@ -39,7 +39,8 @@ module MarkdownIt
               rules: [
                 'normalize',
                 'block',
-                'inline'
+                'inline',
+                'text_join'
               ]
             },
 
@@ -74,7 +75,7 @@ module MarkdownIt
               rules2: [
                 'balance_pairs',
                 'emphasis',
-                'text_collapse'
+                'fragments_join'
               ]
             }
           }

data/lib/motion-markdown-it/presets/zero.rb

@@ -40,7 +40,8 @@ module MarkdownIt
               rules: [
                 'normalize',
                 'block',
-                'inline'
+                'inline',
+                'text_join'
               ]
             },
 
@@ -56,7 +57,7 @@ module MarkdownIt
               ],
               rules2: [
                 'balance_pairs',
-                'text_collapse'
+                'fragments_join'
               ]
             }
           }
@@ -64,4 +65,4 @@ module MarkdownIt
       end
     end
   end
-end
+end

data/lib/motion-markdown-it/renderer.rb

@@ -245,7 +245,7 @@ module MarkdownIt
 
 
     # Renderer.renderInline(tokens, options, env) -> String
-    # - tokens (Array): list on block tokens to renter
+    # - tokens (Array): list on block tokens to render
     # - options (Object): params of parser instance
     # - env (Object): additional data from parsed input (references, for example)
     #
@@ -271,7 +271,7 @@ module MarkdownIt
 
     # internal
     # Renderer.renderInlineAsText(tokens, options, env) -> String
-    # - tokens (Array): list on block tokens to renter
+    # - tokens (Array): list on block tokens to render
     # - options (Object): params of parser instance
     # - env (Object): additional data from parsed input (references, for example)
     #
@@ -297,7 +297,7 @@ module MarkdownIt
 
 
     # Renderer.render(tokens, options, env) -> String
-    # - tokens (Array): list on block tokens to renter
+    # - tokens (Array): list on block tokens to render
     # - options (Object): params of parser instance
     # - env (Object): additional data from parsed input (references, for example)
     #

data/lib/motion-markdown-it/rules_block/code.rb

@@ -26,7 +26,7 @@ module MarkdownIt
         state.line    = last
 
         token         = state.push('code_block', 'code', 0)
-        token.content = state.getLines(startLine, last, 4 + state.blkIndent, true)
+        token.content = state.getLines(startLine, last, 4 + state.blkIndent, false) + "\n"
         token.map     = [ startLine, state.line ]
         return true
       end

data/lib/motion-markdown-it/rules_block/html_block.rb

@@ -11,7 +11,7 @@ module MarkdownIt
       # last argument defines whether it can terminate a paragraph or not
       #
       HTML_SEQUENCES = [
-        [ /^<(script|pre|style)(?=(\s|>|$))/i, /<\/(script|pre|style)>/i, true ],
+        [ /^<(script|pre|style|textarea)(?=(\s|>|$))/i, /<\/(script|pre|style|textarea)>/i, true ],
         [ /^<!--/,        /-->/,   true ],
         [ /^<\?/,         /\?>/,   true ],
         [ /^<![A-Z]/,     />/,     true ],

data/lib/motion-markdown-it/rules_block/list.rb

@@ -129,7 +129,7 @@ module MarkdownIt
           # This code can fail if plugins use blkIndent as well as lists,
           # but I hope the spec gets fixed long before that happens.
           #
-          if state.tShift[startLine] >= state.blkIndent
+          if state.sCount[startLine] >= state.blkIndent
             isTerminatingParagraph = true
           end
         end
@@ -138,7 +138,7 @@ module MarkdownIt
         if ((posAfterMarker = skipOrderedListMarker(state, startLine)) >= 0)
           isOrdered   = true
           start       = state.bMarks[startLine] + state.tShift[startLine]
-          markerValue = state.src[start, posAfterMarker - start - 1].to_i
+          markerValue = state.src[start, posAfterMarker - 1].to_i
 
           # If we're starting a new ordered list right after
           # a paragraph, it should start with 1.
@@ -231,6 +231,9 @@ module MarkdownIt
           token        = state.push('list_item_open', 'li', 1)
           token.markup = markerCharCode.chr
           token.map    = itemLines = [ startLine, 0 ]
+          if (isOrdered)
+            token.info = state.src.slice(start...posAfterMarker - 1)
+          end
 
           # change current state, then restore it after parser subcall
           oldTight                = state.tight
@@ -307,6 +310,7 @@ module MarkdownIt
           if (isOrdered)
             posAfterMarker = skipOrderedListMarker(state, nextLine)
             break if (posAfterMarker < 0)
+            start = state.bMarks[nextLine] + state.tShift[nextLine]
           else
             posAfterMarker = skipBulletListMarker(state, nextLine)
             break if (posAfterMarker < 0)

data/lib/motion-markdown-it/rules_block/table.rb

@@ -10,7 +10,7 @@ module MarkdownIt
         pos = state.bMarks[line] + state.tShift[line]
         max = state.eMarks[line]
 
-        return state.src[pos, max - pos]
+        return state.src[pos...max]
       end
 
       #------------------------------------------------------------------------------

data/lib/motion-markdown-it/rules_core/linkify.rb

@@ -66,6 +66,16 @@ module MarkdownIt
               level   = currentToken.level
               lastPos = 0
               
+              # forbid escape sequence at the start of the string,
+              # this avoids http\://example.com/ from being linkified as
+              # http:<a href="//example.com/">//example.com/</a>
+              if (links.length > 0 &&
+                  links[0].index == 0 &&
+                  i > 0 &&
+                  tokens[i - 1].type == 'text_special')
+                links = links.slice(1..-1)
+              end
+
               (0...links.length).each do |ln|
                 url = links[ln].url
                 fullUrl = state.md.normalizeLink.call(url)
@@ -133,4 +143,4 @@ module MarkdownIt
 
     end
   end
-end
+end

data/lib/motion-markdown-it/rules_core/replacements.rb

@@ -15,15 +15,14 @@ module MarkdownIt
 
       # TODO (from original)
       # - fractionals 1/2, 1/4, 3/4 -> ½, ¼, ¾
-      # - miltiplication 2 x 4 -> 2 × 4
+      # - multiplications 2 x 4 -> 2 × 4
 
       RARE_RE = /\+-|\.\.|\?\?\?\?|!!!!|,,|--/
 
-      SCOPED_ABBR_RE = /\((c|tm|r|p)\)/i
+      SCOPED_ABBR_RE = /\((c|tm|r)\)/i
       SCOPED_ABBR = {
         'c' => '©',
         'r' => '®',
-        'p' => '§',
         'tm' => '™'
       }
 

data/lib/motion-markdown-it/rules_core/text_join.rb

@@ -0,0 +1,51 @@
+# Join raw text tokens with the rest of the text
+#
+# This is set as a separate rule to provide an opportunity for plugins
+# to run text replacements after text join, but before escape join.
+#
+# For example, `\:)` shouldn't be replaced with an emoji.
+#
+module MarkdownIt
+  module RulesCore
+    class TextJoin
+      def self.text_join(state)
+        blockTokens = state.tokens
+
+        (0...blockTokens.length).each do |j|
+          next if (blockTokens[j].type != 'inline')
+      
+          tokens = blockTokens[j].children
+          max = tokens.length
+      
+          (0...max).each do |curr|
+            if (tokens[curr].type == 'text_special')
+              tokens[curr].type = 'text'
+            end
+          end
+
+          last = 0
+          curr = 0
+          while curr < max
+            if (tokens[curr].type == 'text' &&
+                curr + 1 < max &&
+                tokens[curr + 1].type == 'text')
+      
+              # collapse two adjacent text nodes
+              tokens[curr + 1].content = tokens[curr].content + tokens[curr + 1].content
+            else
+               tokens[last] = tokens[curr] if (curr != last)
+      
+              last += 1
+            end
+            
+            curr += 1
+          end
+      
+          if (curr != last)
+            tokens.pop(tokens.length - last)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/balance_pairs.rb

@@ -9,9 +9,28 @@ module MarkdownIt
         openersBottom = {}
         max = delimiters.length
         
+        return if (!max)
+      
+        # headerIdx is the first delimiter of the current (where closer is) delimiter run
+        headerIdx = 0
+        lastTokenIdx = -2 # needs any value lower than -1
+        jumps = []
+
         0.upto(max - 1) do |closerIdx|
           closer = delimiters[closerIdx]
       
+          jumps.push(0)
+      
+          # markers belong to same delimiter run if:
+          #  - they have adjacent tokens
+          #  - AND markers are the same
+          #
+          if (delimiters[headerIdx][:marker] != closer[:marker] || lastTokenIdx != closer[:token] - 1)
+            headerIdx = closerIdx
+          end
+      
+          lastTokenIdx = closer[:token]
+      
           # Length is only used for emphasis-specific "rule of 3",
           # if it's not defined (in strikethrough or 3rd party plugins),
           # we can default it to 0 to disable those checks.
@@ -21,24 +40,23 @@ module MarkdownIt
           next if (!closer[:close])
       
           # Previously calculated lower bounds (previous fails)
-          # for each marker and each delimiter length modulo 3.
+          # for each marker, each delimiter length modulo 3,
+          # and for whether this closer can be an opener;
+          # https://github.com/commonmark/cmark/commit/34250e12ccebdc6372b8b49c44fab57c72443460
           unless openersBottom[closer[:marker]]
-            openersBottom[closer[:marker]] = [ -1, -1, -1 ]
+            openersBottom[closer[:marker]] = [ -1, -1, -1, -1, -1, -1 ]
           end
       
-          minOpenerIdx = openersBottom[closer[:marker]][closer[:length] % 3]
+          minOpenerIdx = openersBottom[closer[:marker]][(closer[:open] ? 3 : 0) + (closer[:length] % 3)]
       
-          openerIdx = closerIdx - closer[:jump] - 1
-
-          # avoid crash if `closer.jump` is pointing outside of the array, see #742
-          openerIdx = -1 if (openerIdx < -1)
+          openerIdx = headerIdx - jumps[headerIdx] - 1
 
           newMinOpenerIdx = openerIdx
 
           while openerIdx > minOpenerIdx
             opener = delimiters[openerIdx]
       
-            (openerIdx -= opener[:jump] + 1) && next if (opener[:marker] != closer[:marker])
+            (openerIdx -= jumps[openerIdx] + 1) && next if (opener[:marker] != closer[:marker])
       
             if (opener[:open] && opener[:end] < 0)
       
@@ -65,19 +83,23 @@ module MarkdownIt
                 # sure algorithm has linear complexity (see *_*_*_*_*_... case).
                 #
                 lastJump = openerIdx > 0 && !delimiters[openerIdx - 1][:open] ?
-                  delimiters[openerIdx - 1][:jump] + 1 : 0
+                  jumps[openerIdx - 1] + 1 : 0
       
-                closer[:jump]  = closerIdx - openerIdx + lastJump
+                jumps[closerIdx] = closerIdx - openerIdx + lastJump
+                jumps[openerIdx] = lastJump
+                
                 closer[:open]  = false
                 opener[:end]   = closerIdx
-                opener[:jump]  = lastJump
                 opener[:close] = false
                 newMinOpenerIdx = -1
+                # treat next token as start of run,
+                # it optimizes skips in **<...>**a**<...>** pathological case
+                lastTokenIdx = -2
                 break
               end
             end
             
-            openerIdx -= opener[:jump] + 1
+            openerIdx -= jumps[openerIdx] + 1
           end
       
           if (newMinOpenerIdx != -1)
@@ -88,7 +110,7 @@ module MarkdownIt
             # See details here:
             # https://github.com/commonmark/cmark/issues/178#issuecomment-270417442
             #
-            openersBottom[closer[:marker]][(closer[:length] || 0) % 3] = newMinOpenerIdx
+            openersBottom[closer[:marker]][(closer[:open] ? 3 : 0) + ((closer[:length] || 0) % 3)] = newMinOpenerIdx
           end
         end
       end

data/lib/motion-markdown-it/rules_inline/emphasis.rb

@@ -30,15 +30,6 @@ module MarkdownIt
             #
             length: scanned[:length],
 
-            # An amount of characters before this one that's equivalent to
-            # current one. In plain English: if this delimiter does not open
-            # an emphasis, neither do previous `jump` characters.
-            #
-            # Used to skip sequences like "*****" in one step, for 1st asterisk
-            # value will be 0, for 2nd it's 1 and so on.
-            #
-            jump:   i,
-
             # A position of the token this delimiter corresponds to.
             #
             token:  state.tokens.length - 1,
@@ -82,9 +73,11 @@ module MarkdownIt
           #
           isStrong = i > 0 &&
                      delimiters[i - 1][:end] == startDelim[:end] + 1 &&
+                     # check that first two markers match and adjacent
+                     delimiters[i - 1][:marker] == startDelim[:marker] &&
                      delimiters[i - 1][:token] == startDelim[:token] - 1 &&
-                     delimiters[startDelim[:end] + 1][:token] == endDelim[:token] + 1 &&
-                     delimiters[i - 1][:marker] == startDelim[:marker]
+                     # check that last two markers are adjacent (we can safely assume they match)
+                     delimiters[startDelim[:end] + 1][:token] == endDelim[:token] + 1
 
           ch = fromCodePoint(startDelim[:marker])
 

data/lib/motion-markdown-it/rules_inline/entity.rb

@@ -8,44 +8,49 @@ module MarkdownIt
       DIGITAL_RE = /^&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));/i
       NAMED_RE   = /^&([a-z][a-z0-9]{1,31});/i
 
-
       #------------------------------------------------------------------------------
       def self.entity(state, silent)
         pos = state.pos
         max = state.posMax
 
-        return false if charCodeAt(state.src, pos) != 0x26    # &
+        return false if charCodeAt(state.src, pos) != 0x26 # &
+
+        return false if pos + 1 >= max
 
-        if pos + 1 < max
-          ch = charCodeAt(state.src, pos + 1)
+        ch = charCodeAt(state.src, pos + 1)
 
-          if ch == 0x23     # '#'
-            match = state.src[pos..-1].match(DIGITAL_RE)
-            if match
+        if ch == 0x23     # '#'
+          match = state.src[pos..-1].match(DIGITAL_RE)
+          if match
+            if !silent
+              code = match[1][0].downcase == 'x' ? match[1][1..-1].to_i(16) : match[1].to_i
+
+              token         = state.push('text_special', '', 0)
+              token.content = isValidEntityCode(code) ? fromCodePoint(code) : fromCodePoint(0xFFFD)
+              token.markup  = match[0]
+              token.info    = 'entity'
+            end
+            state.pos += match[0].length
+            return true
+          end
+        else
+          match = state.src[pos..-1].match(NAMED_RE)
+          if match
+            if MarkdownIt::HTMLEntities::MAPPINGS[match[1]]
               if !silent
-                code = match[1][0].downcase == 'x' ? match[1][1..-1].to_i(16) : match[1].to_i
-                state.pending += isValidEntityCode(code) ? fromCodePoint(code) : fromCodePoint(0xFFFD)
+                token          = state.push('text_special', '', 0)
+                token.content += fromCodePoint(MarkdownIt::HTMLEntities::MAPPINGS[match[1]])
+                token.markup  = match[0]
+                token.info    = 'entity'
               end
               state.pos += match[0].length
               return true
             end
-          else
-            match = state.src[pos..-1].match(NAMED_RE)
-            if match
-              if MarkdownIt::HTMLEntities::MAPPINGS[match[1]]
-                state.pending += fromCodePoint(MarkdownIt::HTMLEntities::MAPPINGS[match[1]]) if !silent
-                state.pos     += match[0].length
-                return true
-              end
-            end
           end
         end
 
-        state.pending += '&' if !silent
-        state.pos += 1
-        return true
+        return false
       end
-
     end
   end
 end

data/lib/motion-markdown-it/rules_inline/escape.rb

@@ -11,7 +11,6 @@ module MarkdownIt
 
       '\\!"#$%&\'()*+,./:;<=>?@[]^_`{|}~-'.split('').each { |ch| ESCAPED[ch.ord] = 1 }
 
-
       #------------------------------------------------------------------------------
       def self.escape(state, silent)
         pos = state.pos
@@ -21,35 +20,55 @@ module MarkdownIt
 
         pos += 1
 
-        if pos < max
-          ch = charCodeAt(state.src, pos)
+        # '\' at the end of the inline block
+        return false if pos >= max
+
+        ch1 = charCodeAt(state.src, pos)
+
+        if ch1 == 0x0A
+          if !silent
+            state.push('hardbreak', 'br', 0)
+          end
 
-          if ch < 256 && ESCAPED[ch] != 0
-            state.pending += state.src[pos] if !silent
-            state.pos     += 2
-            return true
+          pos += 1
+          # skip leading whitespaces from next line
+          while pos < max
+            ch1 = charCodeAt(state.src, pos)
+            break if !isSpace(ch1)
+            pos += 1
           end
 
-          if ch == 0x0A
-            if !silent
-              state.push('hardbreak', 'br', 0)
-            end
+          state.pos = pos
+          return true
+        end
+
+        escapedStr = state.src[pos]
 
+        if (ch1 >= 0xD800 && ch1 <= 0xDBFF && pos + 1 < max)
+          ch2 = charCodeAt(state.src, pos + 1)
+      
+          if (ch2 >= 0xDC00 && ch2 <= 0xDFFF)
+            escapedStr += state.src[pos + 1]
             pos += 1
-            # skip leading whitespaces from next line
-            while pos < max
-              ch = charCodeAt(state.src, pos)
-              break if !isSpace(ch)
-              pos += 1
-            end
-
-            state.pos = pos
-            return true
           end
         end
 
-        state.pending += '\\' if !silent
-        state.pos += 1
+        origStr = '\\' + escapedStr
+
+        if (!silent)
+          token = state.push('text_special', '', 0)
+
+          if ch1 < 256 && ESCAPED[ch1] != 0
+            token.content = escapedStr
+          else
+            token.content = origStr
+          end
+  
+          token.markup = origStr
+          token.info   = 'escape'
+        end
+
+        state.pos = pos + 1
         return true
       end
     end

data/lib/motion-markdown-it/rules_inline/{text_collapse.rb → fragments_join.rb}

@@ -1,47 +1,48 @@
 # Clean up tokens after emphasis and strikethrough postprocessing:
-# Merge adjacent text nodes into one, and re-calculate all token levels
+# merge adjacent text nodes into one and re-calculate all token levels
 #
 # This is necessary because initially emphasis delimiter markers (*, _, ~)
 # are treated as their own separate text tokens. Then emphasis rule either
 # leaves them as text (needed to merge with adjacent text) or turns them
 # into opening/closing tags (which messes up levels inside).
-#------------------------------------------------------------------------------
+#
 module MarkdownIt
   module RulesInline
-    class TextCollapse
+    class FragmentsJoin
+      extend Common::Utils
 
-      #------------------------------------------------------------------------------
-      def self.text_collapse(state)
-        level   = 0
-        tokens  = state.tokens
-        max     = state.tokens.length
+      def self.fragments_join(state)
+        level = 0
+        tokens = state.tokens
+        max = state.tokens.length
 
-        last = curr = 0
+        last = 0      
+        curr = 0
         while curr < max
           # re-calculate levels after emphasis/strikethrough turns some text nodes
           # into opening/closing tags
-          level -= 1 if tokens[curr].nesting < 0 # closing tag
+          level -= 1 if (tokens[curr].nesting < 0) # closing tag
           tokens[curr].level = level
-          level += 1 if tokens[curr].nesting > 0 # opening tag
-
-          if tokens[curr].type == 'text' &&
+          level +=1 if (tokens[curr].nesting > 0) # opening tag
+      
+          if (tokens[curr].type == 'text' &&
               curr + 1 < max &&
-              tokens[curr + 1].type == 'text'
-
+              tokens[curr + 1].type == 'text')
+      
             # collapse two adjacent text nodes
             tokens[curr + 1].content = tokens[curr].content + tokens[curr + 1].content
           else
-            tokens[last] = tokens[curr] if curr != last
-
+            tokens[last] = tokens[curr] if (curr != last)
+      
             last += 1
           end
-
+          
           curr += 1
         end
-
-        if curr != last
-          tokens.slice!(last..max)
-        end
+      
+          if (curr != last)
+            tokens.pop(tokens.length - last)
+          end
       end
     end
   end

data/lib/motion-markdown-it/rules_inline/html_inline.rb

@@ -6,6 +6,14 @@ module MarkdownIt
       extend Common::Utils
       include MarkdownIt::Common::HtmlRe
 
+      #------------------------------------------------------------------------------
+      def self.isLinkOpen(str)
+        return !(/^<a[>\s]/i =~ str).nil?
+      end
+      def self.isLinkClose(str)
+        return !(/^<\/a\s*>/i =~ str).nil?
+      end
+
       #------------------------------------------------------------------------------
       def self.isLetter(ch)
         lc = ch | 0x20    # to lower case
@@ -39,6 +47,9 @@ module MarkdownIt
         if !silent
           token         = state.push('html_inline', '', 0)
           token.content = state.src.slice(pos...(pos + match[0].length))
+
+          state.linkLevel += 1 if (isLinkOpen(token.content))
+          state.linkLevel -= 1 if (isLinkClose(token.content))
         end
         state.pos += match[0].length
         return true

data/lib/motion-markdown-it/rules_inline/link.rb

@@ -132,9 +132,11 @@ module MarkdownIt
             attrs.push([ 'title', title ])
           end
 
+          state.linkLevel += 1
           state.md.inline.tokenize(state)
+          state.linkLevel -= 1
 
-          token        = state.push('link_close', 'a', -1)
+          token = state.push('link_close', 'a', -1)
         end
 
         state.pos     = pos

data/lib/motion-markdown-it/rules_inline/linkify.rb

@@ -0,0 +1,60 @@
+# Process links like https://example.org/
+module MarkdownIt
+  module RulesInline
+    class Linkify
+      extend Common::Utils
+
+      # RFC3986: scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+      SCHEME_RE = /(?:^|[^a-z0-9.+-])([a-z][a-z0-9.+-]*)$/i
+
+      #------------------------------------------------------------------------------
+      def self.linkify(state, silent)
+        return false if (!state.md.options[:linkify])
+        return false if (state.linkLevel > 0)
+      
+        pos = state.pos
+        max = state.posMax
+
+        return false if (pos + 3 > max)
+        return false if (charCodeAt(state.src, pos) != 0x3A) # :
+        return false if (charCodeAt(state.src, pos + 1) != 0x2F) # /
+        return false if (charCodeAt(state.src, pos + 2) != 0x2F) # /
+      
+        match = state.pending.match(SCHEME_RE)
+        return false if (!match)
+      
+        proto = match[1]
+      
+        link = state.md.linkify.matchAtStart(state.src.slice((pos - proto.length)..-1))
+        return false if (!link)
+      
+        url = link.url
+      
+        # disallow '*' at the end of the link (conflicts with emphasis)
+        url = url.sub(/\*+$/, '')
+      
+        fullUrl = state.md.normalizeLink.call(url)
+        return false if (!state.md.validateLink.call(fullUrl))
+      
+        if (!silent)
+          state.pending = state.pending[0...-proto.length]
+      
+          token         = state.push('link_open', 'a', 1)
+          token.attrs   = [ [ 'href', fullUrl ] ]
+          token.markup  = 'linkify'
+          token.info    = 'auto'
+      
+          token         = state.push('text', '', 0)
+          token.content = state.md.normalizeLinkText.call(url)
+      
+          token         = state.push('link_close', 'a', -1)
+          token.markup  = 'linkify'
+          token.info    = 'auto'
+        end
+      
+        state.pos += url.length - proto.length
+        return true
+      end
+    end
+  end
+end

data/lib/motion-markdown-it/rules_inline/newline.rb

@@ -20,7 +20,13 @@ module MarkdownIt
         if !silent
           if pmax >= 0 && charCodeAt(state.pending, pmax) == 0x20
             if pmax >= 1 && charCodeAt(state.pending, pmax - 1) == 0x20
-              state.pending = state.pending.sub(/ +$/, '')
+              # Find whitespaces tail of pending chars.
+              ws = pmax - 1
+              while (ws >= 1 && charCodeAt(state.pending, ws - 1) == 0x20)
+                ws -= 1
+              end
+
+              state.pending = state.pending.slice(0...ws)
               state.push('hardbreak', 'br', 0)
             else
               state.pending = state.pending.slice(0...-1)

data/lib/motion-markdown-it/rules_inline/state_inline.rb

@@ -7,7 +7,7 @@ module MarkdownIt
 
       attr_accessor :src, :env, :md, :tokens, :pos, :posMax, :level, :tokens_meta
       attr_accessor :pending, :pendingLevel, :cache, :delimiters
-      attr_accessor :backticks, :backticksScanned
+      attr_accessor :backticks, :backticksScanned, :linkLevel
 
       #------------------------------------------------------------------------------
       def initialize(src, md, env, outTokens)
@@ -36,6 +36,10 @@ module MarkdownIt
         # backtick length => last seen position
         @backticks = {}
         @backticksScanned = false
+
+        # Counter used to disable inline linkify-it execution
+        # inside <a> and markdown links
+        @linkLevel = 0
       end
 
 

data/lib/motion-markdown-it/rules_inline/strikethrough.rb

@@ -35,7 +35,6 @@ module MarkdownIt
           state.delimiters.push({
             marker: marker,
             length: 0,     # disable "rule of 3" length checks meant for emphasis
-            jump:   i / 2, # for `~~` 1 marker = 2 characters
             token:  state.tokens.length - 1,
             end:    -1,
             open:   scanned[:can_open],

data/lib/motion-markdown-it/token.rb

@@ -67,6 +67,7 @@ module MarkdownIt
        # *
        # * - Info string for "fence" tokens
        # * - The value "auto" for autolink "link_open" and "link_close" tokens
+       # * - The string value of the item marker for ordered-list "list_item_open" tokens
       @info     = ''
 
        # * Token#meta -> Object

data/lib/motion-markdown-it/version.rb

@@ -1,3 +1,3 @@
 module MotionMarkdownIt
-  VERSION = '12.0.6'
+  VERSION = '13.0.1'
 end

data/lib/motion-markdown-it.rb

@@ -40,6 +40,7 @@ else
   require 'motion-markdown-it/rules_core/replacements'
   require 'motion-markdown-it/rules_core/smartquotes'
   require 'motion-markdown-it/rules_core/state_core'
+  require 'motion-markdown-it/rules_core/text_join'
   require 'motion-markdown-it/rules_block/blockquote'
   require 'motion-markdown-it/rules_block/code'
   require 'motion-markdown-it/rules_block/fence'
@@ -58,17 +59,18 @@ else
   require 'motion-markdown-it/rules_inline/emphasis'
   require 'motion-markdown-it/rules_inline/entity'
   require 'motion-markdown-it/rules_inline/escape'
+  require 'motion-markdown-it/rules_inline/fragments_join'
   require 'motion-markdown-it/rules_inline/html_inline'
   require 'motion-markdown-it/rules_inline/image'
   require 'motion-markdown-it/rules_inline/link'
+  require 'motion-markdown-it/rules_inline/linkify'
   require 'motion-markdown-it/rules_inline/newline'
   require 'motion-markdown-it/rules_inline/state_inline'
   require 'motion-markdown-it/rules_inline/strikethrough'
-  require 'motion-markdown-it/rules_inline/text_collapse'
   require 'motion-markdown-it/rules_inline/text'
 
   require 'motion-markdown-it/ruler'
   require 'motion-markdown-it/token'
   require 'motion-markdown-it/index'
 
-end
+end

data/spec/motion-markdown-it/misc_spec.rb

@@ -195,6 +195,9 @@ describe 'Misc' do
 
     expect(md.render('123')).to eq "<p>123</p>\n"
     expect(md.render("123\n")).to eq "<p>123</p>\n"
+
+    expect(md.render('    codeblock')).to eq "<pre><code>codeblock\n</code></pre>\n"
+    expect(md.render("    codeblock\n")).to eq "<pre><code>codeblock\n</code></pre>\n"
   end
 
   #------------------------------------------------------------------------------
@@ -273,6 +276,14 @@ describe 'Misc' do
   #   );
   # end
 
+  # TODO ------------------------------------------------------------------------------
+  # it 'Should escape surrogate pairs (coverage)' do
+  #   md = MarkdownIt::Parser.new({})
+  # 
+  #   expect(md.render("\\\uD835\uDC9C")).to eq "<p>\\\uD835\uDC9C</p>\n"
+  #   expect(md.render("\\\uD835x")).to eq "<p>\\\uD835x</p>\n"
+  #   expect(md.render("\\\uD835")).to eq "<p>\\\uD835</p>\n"
+  # end
 end
 
 
@@ -371,6 +382,42 @@ describe 'smartquotes' do
   end
 end
 
+describe 'Ordered list info' do
+  md = MarkdownIt::Parser.new
+
+  def type_filter(tokens, type)
+    return tokens.select { |t| t.type === type }
+  end
+
+  it 'Should mark ordered list item tokens with info' do
+    tokens = md.parse("1. Foo\n2. Bar\n20. Fuzz", {})
+    expect(type_filter(tokens, 'ordered_list_open').length).to eq 1
+
+    tokens = type_filter(tokens, 'list_item_open')
+    expect(tokens.length).to eq 3
+    expect(tokens[0].info).to eq '1'
+    expect(tokens[0].markup).to eq '.'
+    expect(tokens[1].info).to eq '2'
+    expect(tokens[1].markup).to eq '.'
+    expect(tokens[2].info).to eq '20'
+    expect(tokens[2].markup).to eq '.'
+
+    tokens = md.parse(" 1. Foo\n2. Bar\n  20. Fuzz\n 199. Flp", {})
+    expect(type_filter(tokens, 'ordered_list_open').length).to eq 1
+
+    tokens = type_filter(tokens, 'list_item_open')
+    expect(tokens.length).to eq 4
+    expect(tokens[0].info).to eq '1'
+    expect(tokens[0].markup).to eq '.'
+    expect(tokens[1].info).to eq '2'
+    expect(tokens[1].markup).to eq '.'
+    expect(tokens[2].info).to eq '20'
+    expect(tokens[2].markup).to eq '.'
+    expect(tokens[3].info).to eq '199'
+    expect(tokens[3].markup).to eq '.'
+  end
+end
+
 #------------------------------------------------------------------------------
 describe 'Token attributes' do
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: motion-markdown-it
 version: !ruby/object:Gem::Version
-  version: 12.0.6
+  version: 13.0.1
 platform: ruby
 authors:
 - Brett Walker
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-17 00:00:00.000000000 Z
+date: 2023-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mdurl-rb
@@ -113,20 +113,22 @@ files:
 - lib/motion-markdown-it/rules_core/replacements.rb
 - lib/motion-markdown-it/rules_core/smartquotes.rb
 - lib/motion-markdown-it/rules_core/state_core.rb
+- lib/motion-markdown-it/rules_core/text_join.rb
 - lib/motion-markdown-it/rules_inline/autolink.rb
 - lib/motion-markdown-it/rules_inline/backticks.rb
 - lib/motion-markdown-it/rules_inline/balance_pairs.rb
 - lib/motion-markdown-it/rules_inline/emphasis.rb
 - lib/motion-markdown-it/rules_inline/entity.rb
 - lib/motion-markdown-it/rules_inline/escape.rb
+- lib/motion-markdown-it/rules_inline/fragments_join.rb
 - lib/motion-markdown-it/rules_inline/html_inline.rb
 - lib/motion-markdown-it/rules_inline/image.rb
 - lib/motion-markdown-it/rules_inline/link.rb
+- lib/motion-markdown-it/rules_inline/linkify.rb
 - lib/motion-markdown-it/rules_inline/newline.rb
 - lib/motion-markdown-it/rules_inline/state_inline.rb
 - lib/motion-markdown-it/rules_inline/strikethrough.rb
 - lib/motion-markdown-it/rules_inline/text.rb
-- lib/motion-markdown-it/rules_inline/text_collapse.rb
 - lib/motion-markdown-it/token.rb
 - lib/motion-markdown-it/version.rb
 - spec/motion-markdown-it/commonmark_spec.rb