@depup/markdown-it 14.1.1-depup.1 → 14.3.0-depup.13

package/lib/rules_core/linkify.mjs

@@ -93,36 +93,36 @@ export default function linkify (state) {
           const pos = links[ln].index
 
           if (pos > lastPos) {
-            const token   = new state.Token('text', '', 0)
+            const token = new state.Token('text', '', 0)
             token.content = text.slice(lastPos, pos)
-            token.level   = level
+            token.level = level
             nodes.push(token)
           }
 
-          const token_o   = new state.Token('link_open', 'a', 1)
-          token_o.attrs   = [['href', fullUrl]]
-          token_o.level   = level++
-          token_o.markup  = 'linkify'
-          token_o.info    = 'auto'
+          const token_o = new state.Token('link_open', 'a', 1)
+          token_o.attrs = [['href', fullUrl]]
+          token_o.level = level++
+          token_o.markup = 'linkify'
+          token_o.info = 'auto'
           nodes.push(token_o)
 
-          const token_t   = new state.Token('text', '', 0)
+          const token_t = new state.Token('text', '', 0)
           token_t.content = urlText
-          token_t.level   = level
+          token_t.level = level
           nodes.push(token_t)
 
-          const token_c   = new state.Token('link_close', 'a', -1)
-          token_c.level   = --level
-          token_c.markup  = 'linkify'
-          token_c.info    = 'auto'
+          const token_c = new state.Token('link_close', 'a', -1)
+          token_c.level = --level
+          token_c.markup = 'linkify'
+          token_c.info = 'auto'
           nodes.push(token_c)
 
           lastPos = links[ln].lastIndex
         }
         if (lastPos < text.length) {
-          const token   = new state.Token('text', '', 0)
+          const token = new state.Token('text', '', 0)
           token.content = text.slice(lastPos)
-          token.level   = level
+          token.level = level
           nodes.push(token)
         }
 

package/lib/rules_core/normalize.mjs

@@ -1,8 +1,8 @@
 // Normalize input string
 
 // https://spec.commonmark.org/0.29/#line-ending
-const NEWLINES_RE  = /\r\n?|\n/g
-const NULL_RE      = /\0/g
+const NEWLINES_RE = /\r\n?|\n/g
+const NULL_RE = /\0/g
 
 export default function normalize (state) {
   let str

package/lib/rules_core/smartquotes.mjs

@@ -1,20 +1,42 @@
 // Convert straight quotation marks to typographic ones
 //
 
-import { isWhiteSpace, isPunctChar, isMdAsciiPunct } from '../common/utils.mjs'
+import { isWhiteSpace, isPunctCharCode, isMdAsciiPunct } from '../common/utils.mjs'
 
 const QUOTE_TEST_RE = /['"]/
 const QUOTE_RE = /['"]/g
 const APOSTROPHE = '\u2019' /* ’ */
 
-function replaceAt (str, index, ch) {
-  return str.slice(0, index) + ch + str.slice(index + 1)
+function addReplacement (replacements, tokenIdx, pos, ch) {
+  if (!replacements[tokenIdx]) {
+    replacements[tokenIdx] = []
+  }
+
+  replacements[tokenIdx].push({ pos, ch })
+}
+
+function applyReplacements (str, replacements) {
+  let result = ''
+  let lastPos = 0
+
+  replacements.sort((a, b) => a.pos - b.pos)
+
+  for (let i = 0; i < replacements.length; i++) {
+    const replacement = replacements[i]
+
+    result += str.slice(lastPos, replacement.pos) + replacement.ch
+    lastPos = replacement.pos + 1
+  }
+
+  return result + str.slice(lastPos)
 }
 
 function process_inlines (tokens, state) {
   let j
 
   const stack = []
+  // token index -> list of replacements in the original token content
+  const replacements = {}
 
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i]
@@ -28,9 +50,9 @@ function process_inlines (tokens, state) {
 
     if (token.type !== 'text') { continue }
 
-    let text = token.content
+    const text = token.content
     let pos = 0
-    let max = text.length
+    const max = text.length
 
     /* eslint no-labels:0,block-scoped-var:0 */
     OUTER:
@@ -78,8 +100,8 @@ function process_inlines (tokens, state) {
         }
       }
 
-      const isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(String.fromCharCode(lastChar))
-      const isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(String.fromCharCode(nextChar))
+      const isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctCharCode(lastChar)
+      const isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctCharCode(nextChar)
 
       const isLastWhiteSpace = isWhiteSpace(lastChar)
       const isNextWhiteSpace = isWhiteSpace(nextChar)
@@ -122,7 +144,7 @@ function process_inlines (tokens, state) {
       if (!canOpen && !canClose) {
         // middle of word
         if (isSingle) {
-          token.content = replaceAt(token.content, t.index, APOSTROPHE)
+          addReplacement(replacements, i, t.index, APOSTROPHE)
         }
         continue
       }
@@ -145,18 +167,8 @@ function process_inlines (tokens, state) {
               closeQuote = state.md.options.quotes[1]
             }
 
-            // replace token.content *before* tokens[item.token].content,
-            // because, if they are pointing at the same token, replaceAt
-            // could mess up indices when quote length != 1
-            token.content = replaceAt(token.content, t.index, closeQuote)
-            tokens[item.token].content = replaceAt(
-              tokens[item.token].content, item.pos, openQuote)
-
-            pos += closeQuote.length - 1
-            if (item.token === i) { pos += openQuote.length - 1 }
-
-            text = token.content
-            max = text.length
+            addReplacement(replacements, i, t.index, closeQuote)
+            addReplacement(replacements, item.token, item.pos, openQuote)
 
             stack.length = j
             continue OUTER
@@ -172,10 +184,14 @@ function process_inlines (tokens, state) {
           level: thisLevel
         })
       } else if (canClose && isSingle) {
-        token.content = replaceAt(token.content, t.index, APOSTROPHE)
+        addReplacement(replacements, i, t.index, APOSTROPHE)
       }
     }
   }
+
+  Object.keys(replacements).forEach(function (tokenIdx) {
+    tokens[tokenIdx].content = applyReplacements(tokens[tokenIdx].content, replacements[tokenIdx])
+  })
 }
 
 export default function smartquotes (state) {

package/lib/rules_inline/autolink.mjs

@@ -1,7 +1,7 @@
 // Process autolinks '<protocol:...>'
 
 /* eslint max-len:0 */
-const EMAIL_RE    = /^([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)$/
+const EMAIL_RE = /^([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)$/
 /* eslint-disable-next-line no-control-regex */
 const AUTOLINK_RE = /^([a-zA-Z][a-zA-Z0-9+.-]{1,31}):([^<>\x00-\x20]*)$/
 
@@ -29,17 +29,17 @@ export default function autolink (state, silent) {
     if (!state.md.validateLink(fullUrl)) { return false }
 
     if (!silent) {
-      const token_o   = state.push('link_open', 'a', 1)
-      token_o.attrs   = [['href', fullUrl]]
-      token_o.markup  = 'autolink'
-      token_o.info    = 'auto'
+      const token_o = state.push('link_open', 'a', 1)
+      token_o.attrs = [['href', fullUrl]]
+      token_o.markup = 'autolink'
+      token_o.info = 'auto'
 
-      const token_t   = state.push('text', '', 0)
+      const token_t = state.push('text', '', 0)
       token_t.content = state.md.normalizeLinkText(url)
 
-      const token_c   = state.push('link_close', 'a', -1)
-      token_c.markup  = 'autolink'
-      token_c.info    = 'auto'
+      const token_c = state.push('link_close', 'a', -1)
+      token_c.markup = 'autolink'
+      token_c.info = 'auto'
     }
 
     state.pos += url.length + 2
@@ -51,17 +51,17 @@ export default function autolink (state, silent) {
     if (!state.md.validateLink(fullUrl)) { return false }
 
     if (!silent) {
-      const token_o   = state.push('link_open', 'a', 1)
-      token_o.attrs   = [['href', fullUrl]]
-      token_o.markup  = 'autolink'
-      token_o.info    = 'auto'
+      const token_o = state.push('link_open', 'a', 1)
+      token_o.attrs = [['href', fullUrl]]
+      token_o.markup = 'autolink'
+      token_o.info = 'auto'
 
-      const token_t   = state.push('text', '', 0)
+      const token_t = state.push('text', '', 0)
       token_t.content = state.md.normalizeLinkText(url)
 
-      const token_c   = state.push('link_close', 'a', -1)
-      token_c.markup  = 'autolink'
-      token_c.info    = 'auto'
+      const token_c = state.push('link_close', 'a', -1)
+      token_c.markup = 'autolink'
+      token_c.info = 'auto'
     }
 
     state.pos += url.length + 2

package/lib/rules_inline/balance_pairs.mjs

@@ -85,8 +85,8 @@ function processDelimiters (delimiters) {
           jumps[closerIdx] = closerIdx - openerIdx + lastJump
           jumps[openerIdx] = lastJump
 
-          closer.open  = false
-          opener.end   = closerIdx
+          closer.open = false
+          opener.end = closerIdx
           opener.close = false
           newMinOpenerIdx = -1
           // treat next token as start of run,

package/lib/rules_inline/emphasis.mjs

@@ -80,18 +80,18 @@ function postProcess (state, delimiters) {
 
     const ch = String.fromCharCode(startDelim.marker)
 
-    const token_o   = state.tokens[startDelim.token]
-    token_o.type    = isStrong ? 'strong_open' : 'em_open'
-    token_o.tag     = isStrong ? 'strong' : 'em'
+    const token_o = state.tokens[startDelim.token]
+    token_o.type = isStrong ? 'strong_open' : 'em_open'
+    token_o.tag = isStrong ? 'strong' : 'em'
     token_o.nesting = 1
-    token_o.markup  = isStrong ? ch + ch : ch
+    token_o.markup = isStrong ? ch + ch : ch
     token_o.content = ''
 
-    const token_c   = state.tokens[endDelim.token]
-    token_c.type    = isStrong ? 'strong_close' : 'em_close'
-    token_c.tag     = isStrong ? 'strong' : 'em'
+    const token_c = state.tokens[endDelim.token]
+    token_c.type = isStrong ? 'strong_close' : 'em_close'
+    token_c.tag = isStrong ? 'strong' : 'em'
     token_c.nesting = -1
-    token_c.markup  = isStrong ? ch + ch : ch
+    token_c.markup = isStrong ? ch + ch : ch
     token_c.content = ''
 
     if (isStrong) {

package/lib/rules_inline/entity.mjs

@@ -1,10 +1,10 @@
 // Process html entity - {, ¯, ", ...
 
-import { decodeHTML } from 'entities'
+import { decodeHTMLStrict } from 'entities'
 import { isValidEntityCode, fromCodePoint } from '../common/utils.mjs'
 
 const DIGITAL_RE = /^&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));/i
-const NAMED_RE   = /^&([a-z][a-z0-9]{1,31});/i
+const NAMED_RE = /^&([a-z][a-z0-9]{1,31});/i
 
 export default function entity (state, silent) {
   const pos = state.pos
@@ -22,10 +22,10 @@ export default function entity (state, silent) {
       if (!silent) {
         const code = match[1][0].toLowerCase() === 'x' ? parseInt(match[1].slice(1), 16) : parseInt(match[1], 10)
 
-        const token   = state.push('text_special', '', 0)
+        const token = state.push('text_special', '', 0)
         token.content = isValidEntityCode(code) ? fromCodePoint(code) : fromCodePoint(0xFFFD)
-        token.markup  = match[0]
-        token.info    = 'entity'
+        token.markup = match[0]
+        token.info = 'entity'
       }
       state.pos += match[0].length
       return true
@@ -33,13 +33,13 @@ export default function entity (state, silent) {
   } else {
     const match = state.src.slice(pos).match(NAMED_RE)
     if (match) {
-      const decoded = decodeHTML(match[0])
+      const decoded = decodeHTMLStrict(match[0])
       if (decoded !== match[0]) {
         if (!silent) {
-          const token   = state.push('text_special', '', 0)
+          const token = state.push('text_special', '', 0)
           token.content = decoded
-          token.markup  = match[0]
-          token.info    = 'entity'
+          token.markup = match[0]
+          token.info = 'entity'
         }
         state.pos += match[0].length
         return true

package/lib/rules_inline/escape.mjs

@@ -38,6 +38,20 @@ export default function escape (state, silent) {
     return true
   }
 
+  // '\' before a space is a literal backslash. Don't consume the space, so a
+  // trailing two-space hard line break is still detected by the newline rule.
+  if (ch1 === 0x20) {
+    if (!silent) {
+      const token = state.push('text_special', '', 0)
+      token.content = '\\'
+      token.markup = '\\'
+      token.info = 'escape'
+    }
+
+    state.pos = pos
+    return true
+  }
+
   let escapedStr = state.src[pos]
 
   if (ch1 >= 0xD800 && ch1 <= 0xDBFF && pos + 1 < max) {
@@ -61,7 +75,7 @@ export default function escape (state, silent) {
     }
 
     token.markup = origStr
-    token.info   = 'escape'
+    token.info = 'escape'
   }
 
   state.pos = pos + 1

package/lib/rules_inline/html_inline.mjs

@@ -42,7 +42,7 @@ export default function html_inline (state, silent) {
     const token = state.push('html_inline', '', 0)
     token.content = match[0]
 
-    if (isLinkOpen(token.content))  state.linkLevel++
+    if (isLinkOpen(token.content)) state.linkLevel++
     if (isLinkClose(token.content)) state.linkLevel--
   }
   state.pos += match[0].length

package/lib/rules_inline/link.mjs

@@ -121,7 +121,7 @@ export default function link (state, silent) {
 
     const token_o = state.push('link_open', 'a', 1)
     const attrs = [['href', href]]
-    token_o.attrs  = attrs
+    token_o.attrs = attrs
     if (title) {
       attrs.push(['title', title])
     }

package/lib/rules_inline/state_inline.mjs

@@ -1,7 +1,7 @@
 // Inline parser state
 
 import Token from '../token.mjs'
-import { isWhiteSpace, isPunctChar, isMdAsciiPunct } from '../common/utils.mjs'
+import { isWhiteSpace, isPunctCharCode, isMdAsciiPunct } from '../common/utils.mjs'
 
 function StateInline (src, md, env, outTokens) {
   this.src = src
@@ -89,8 +89,30 @@ StateInline.prototype.scanDelims = function (start, canSplitWord) {
   const max = this.posMax
   const marker = this.src.charCodeAt(start)
 
-  // treat beginning of the line as a whitespace
-  const lastChar = start > 0 ? this.src.charCodeAt(start - 1) : 0x20
+  // Astral characters below are combined manually, because .codePointAt()
+  // does not guarantee numeric type output. And we don't wish JIT cache issues.
+  // The broken surrogate pairs are evaluated as U+FFFD to prevent possible
+  // crashes.
+
+  let lastChar
+  if (start === 0) {
+    // treat beginning of the line as a whitespace
+    lastChar = 0x20
+  } else if (start === 1) {
+    lastChar = this.src.charCodeAt(0)
+    if ((lastChar & 0xF800) === 0xD800) { lastChar = 0xFFFD }
+  } else {
+    lastChar = this.src.charCodeAt(start - 1)
+    if ((lastChar & 0xFC00) === 0xDC00) {
+      // low surrogate => add high one, replace broken pair with U+FFFD
+      const highSurr = this.src.charCodeAt(start - 2)
+      lastChar = (highSurr & 0xFC00) === 0xD800
+        ? 0x10000 + ((highSurr - 0xD800) << 10) + (lastChar - 0xDC00)
+        : 0xFFFD
+    } else if ((lastChar & 0xFC00) === 0xD800) {
+      lastChar = 0xFFFD
+    }
+  }
 
   let pos = start
   while (pos < max && this.src.charCodeAt(pos) === marker) { pos++ }
@@ -98,10 +120,19 @@ StateInline.prototype.scanDelims = function (start, canSplitWord) {
   const count = pos - start
 
   // treat end of the line as a whitespace
-  const nextChar = pos < max ? this.src.charCodeAt(pos) : 0x20
+  let nextChar = pos < max ? this.src.charCodeAt(pos) : 0x20
+  if ((nextChar & 0xFC00) === 0xD800) {
+    // high surrogate => add low one, replace broken pair with U+FFFD
+    const lowSurr = this.src.charCodeAt(pos + 1)
+    nextChar = (lowSurr & 0xFC00) === 0xDC00
+      ? 0x10000 + ((nextChar - 0xD800) << 10) + (lowSurr - 0xDC00)
+      : 0xFFFD
+  } else if ((nextChar & 0xFC00) === 0xDC00) {
+    nextChar = 0xFFFD
+  }
 
-  const isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(String.fromCharCode(lastChar))
-  const isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(String.fromCharCode(nextChar))
+  const isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctCharCode(lastChar)
+  const isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctCharCode(nextChar)
 
   const isLastWhiteSpace = isWhiteSpace(lastChar)
   const isNextWhiteSpace = isWhiteSpace(nextChar)
@@ -111,8 +142,8 @@ StateInline.prototype.scanDelims = function (start, canSplitWord) {
   const right_flanking =
     !isLastWhiteSpace && (!isLastPunctChar || isNextWhiteSpace || isNextPunctChar)
 
-  const can_open  = left_flanking  && (canSplitWord || !right_flanking || isLastPunctChar)
-  const can_close = right_flanking && (canSplitWord || !left_flanking  || isNextPunctChar)
+  const can_open = left_flanking && (canSplitWord || !right_flanking || isLastPunctChar)
+  const can_close = right_flanking && (canSplitWord || !left_flanking || isNextPunctChar)
 
   return { can_open, can_close, length: count }
 }

package/lib/rules_inline/strikethrough.mjs

@@ -20,13 +20,13 @@ function strikethrough_tokenize (state, silent) {
   let token
 
   if (len % 2) {
-    token         = state.push('text', '', 0)
+    token = state.push('text', '', 0)
     token.content = ch
     len--
   }
 
   for (let i = 0; i < len; i += 2) {
-    token         = state.push('text', '', 0)
+    token = state.push('text', '', 0)
     token.content = ch + ch
 
     state.delimiters.push({
@@ -62,18 +62,18 @@ function postProcess (state, delimiters) {
 
     const endDelim = delimiters[startDelim.end]
 
-    token         = state.tokens[startDelim.token]
-    token.type    = 's_open'
-    token.tag     = 's'
+    token = state.tokens[startDelim.token]
+    token.type = 's_open'
+    token.tag = 's'
     token.nesting = 1
-    token.markup  = '~~'
+    token.markup = '~~'
     token.content = ''
 
-    token         = state.tokens[endDelim.token]
-    token.type    = 's_close'
-    token.tag     = 's'
+    token = state.tokens[endDelim.token]
+    token.type = 's_close'
+    token.tag = 's'
     token.nesting = -1
-    token.markup  = '~~'
+    token.markup = '~~'
     token.content = ''
 
     if (state.tokens[endDelim.token - 1].type === 'text' &&

package/lib/token.mjs

@@ -15,28 +15,28 @@ function Token (type, tag, nesting) {
    *
    * Type of the token (string, e.g. "paragraph_open")
    **/
-  this.type     = type
+  this.type = type
 
   /**
    * Token#tag -> String
    *
    * html tag name, e.g. "p"
    **/
-  this.tag      = tag
+  this.tag = tag
 
   /**
    * Token#attrs -> Array
    *
    * Html attributes. Format: `[ [ name1, value1 ], [ name2, value2 ] ]`
    **/
-  this.attrs    = null
+  this.attrs = null
 
   /**
    * Token#map -> Array
    *
    * Source map info. Format: `[ line_begin, line_end ]`
    **/
-  this.map      = null
+  this.map = null
 
   /**
    * Token#nesting -> Number
@@ -47,14 +47,14 @@ function Token (type, tag, nesting) {
    * -  `0` means the tag is self-closing
    * - `-1` means the tag is closing
    **/
-  this.nesting  = nesting
+  this.nesting = nesting
 
   /**
    * Token#level -> Number
    *
    * nesting level, the same as `state.level`
    **/
-  this.level    = 0
+  this.level = 0
 
   /**
    * Token#children -> Array
@@ -69,14 +69,14 @@ function Token (type, tag, nesting) {
    * In a case of self-closing tag (code, html, fence, etc.),
    * it has contents of this tag.
    **/
-  this.content  = ''
+  this.content = ''
 
   /**
    * Token#markup -> String
    *
    * '*' or '_' for emphasis, fence string for fence, etc.
    **/
-  this.markup   = ''
+  this.markup = ''
 
   /**
    * Token#info -> String
@@ -87,14 +87,14 @@ function Token (type, tag, nesting) {
    * - The value "auto" for autolink "link_open" and "link_close" tokens
    * - The string value of the item marker for ordered-list "list_item_open" tokens
    **/
-  this.info     = ''
+  this.info = ''
 
   /**
    * Token#meta -> Object
    *
    * A place for plugins to store an arbitrary data
    **/
-  this.meta     = null
+  this.meta = null
 
   /**
    * Token#block -> Boolean
@@ -102,7 +102,7 @@ function Token (type, tag, nesting) {
    * True for block-level tokens, false for inline tokens.
    * Used in renderer to calculate line breaks
    **/
-  this.block    = false
+  this.block = false
 
   /**
    * Token#hidden -> Boolean
@@ -110,7 +110,7 @@ function Token (type, tag, nesting) {
    * If it's true, ignore this element when rendering. Used for tight lists
    * to hide paragraphs.
    **/
-  this.hidden   = false
+  this.hidden = false
 }
 
 /**