@peaceroad/markdown-it-strong-ja 0.9.0 → 0.9.2

package/README.md

@@ -357,6 +357,83 @@ Supporting visuals:
 - `aggressive`:  
   `<p>broken **tail <a href="https://x.test">aa<strong>aa</strong><em>Text</em><strong>and<em>More</em>bb</strong>bb</a> after</p>`
 
+## Compatibility Notes
+
+### `markdown-it-attrs` 5.x parity
+
+When `markdown-it-attrs` is installed, strong-ja follows the token stream produced by that plugin and does not reinterpret where `{...}` attributes should be attached. This is intentional: strong-ja should not make attribute syntax mean something different from `markdown-it-attrs` alone.
+
+One edge case to be aware of is a tight list item followed by an emphasized line:
+
+```markdown
+- e {.li-style}
+*{.ul-style}*
+```
+
+With `markdown-it-attrs` 5.x, the first attribute block is consumed as a block-level attribute on the hidden `paragraph_open` inside the tight list. Because that paragraph token is hidden by markdown-it's tight-list rendering, the class is not visible in the final HTML. The second `{.ul-style}` is inside emphasis text, not a suffix after a closed inline token, so it remains literal text:
+
+```html
+<ul>
+<li>e
+<em>{.ul-style}</em></li>
+</ul>
+```
+
+This output matches `markdown-it-attrs` alone. To attach attributes intentionally, use the syntax owned by `markdown-it-attrs`, for example:
+
+```markdown
+- e
+{.ul-style}
+```
+
+```html
+<ul class="ul-style">
+<li>e</li>
+</ul>
+```
+
+or attach inline attributes after the closing inline token:
+
+```markdown
+- e
+*x*{.ul-style}
+```
+
+```html
+<ul>
+<li>e
+<em class="ul-style">x</em></li>
+</ul>
+```
+
+strong-ja keeps this as dependency parity rather than adding a local workaround.
+
+### `markdown-it` 14.2 astral delimiter policy
+
+`markdown-it` 14.2 recognizes astral characters (surrogate pairs) as full Unicode code points when scanning emphasis delimiters. strong-ja keeps `compatible` mode aligned with that upstream behavior.
+
+In Japanese modes, strong-ja still only adds its own delimiter relaxation when Japanese/CJK context is present. Astral Han characters, such as CJK Extension B, are treated as CJK context:
+
+```markdown
+*𠀋?*abc*
+```
+
+```html
+<p><em>𠀋?</em>abc*</p>
+```
+
+Emoji or symbol-only English contexts remain aligned with `markdown-it` and are not promoted just because they are astral characters:
+
+```markdown
+*😀?*abc*
+```
+
+```html
+<p>*😀?<em>abc</em></p>
+```
+
+Symbols inside Japanese prose may still be emphasized by the existing Japanese-context rule, for example `**😀**です` can render as `<p><strong>😀</strong>です</p>`. Use `mode: 'compatible'` when exact `markdown-it` 14.2 delimiter behavior is required.
+
 ## Options
 
 ### `mode`

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@peaceroad/markdown-it-strong-ja",
   "description": "Extends asterisk emphasis handling for Japanese text while keeping markdown-it behavior as close as practical.",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "main": "index.js",
   "type": "module",
   "files": [
@@ -26,6 +26,7 @@
     "test:all": "node test/test-all.js",
     "bench:scan": "node test/material/perf-scan-delims.mjs",
     "bench:postprocess": "node test/material/perf-postprocess.mjs",
+    "bench:isolated": "node test/material/bench-isolated.mjs",
     "analyze:postprocess-calls": "node test/material/analyze-postprocess-calls.mjs",
     "analyze:fastpath": "node test/material/analyze-fastpath-hits.mjs"
   },
@@ -33,17 +34,16 @@
   "author": "peaceroad <peaceroad@gmail.com>",
   "license": "MIT",
   "dependencies": {
-    "markdown-it": "^14.1.0"
+    "markdown-it": "^14.2.0"
   },
   "devDependencies": {
-    "@peaceroad/markdown-it-cjk-breaks-mod": "^0.1.10",
-    "@peaceroad/markdown-it-hr-sandwiched-semantic-container": "^0.11.0",
-    "@peaceroad/markdown-it-renderer-image": "^0.12.0",
-    "@peaceroad/markdown-it-renderer-inline-text": "^0.8.0",
-    "markdown-it-attrs": "^4.3.1",
+    "@peaceroad/markdown-it-cjk-breaks-mod": "^0.1.11",
+    "@peaceroad/markdown-it-hr-sandwiched-semantic-container": "^0.12.0",
+    "@peaceroad/markdown-it-renderer-image": "^0.16.0",
+    "@peaceroad/markdown-it-renderer-inline-text": "^0.8.1",
+    "markdown-it-attrs": "^5.0.0",
     "markdown-it-sub": "^2.0.0",
     "markdown-it-sup": "^2.0.0",
-    "p7d-markdown-it-p-captions": "^0.21.0"
+    "p7d-markdown-it-p-captions": "^0.23.0"
   }
 }
-

package/src/token-compat.js

@@ -2,6 +2,9 @@ import Token from 'markdown-it/lib/token.mjs'
 import {
   REG_ATTRS,
   isJapaneseChar,
+  isAsciiWordCode,
+  codePointAtSafe,
+  codePointBeforeSafe,
   hasCjkBreaksRule,
   isCjkBreaksRuleName,
   getRuntimeOpt,
@@ -9,12 +12,6 @@ import {
   moveRuleAfter
 } from './token-utils.js'
 
-const isAsciiWordCode = (code) => {
-  return (code >= 0x30 && code <= 0x39) ||
-    (code >= 0x41 && code <= 0x5A) ||
-    (code >= 0x61 && code <= 0x7A)
-}
-
 const trimTrailingSpaceTab = (text) => {
   if (!text) return text
   let end = text.length
@@ -124,8 +121,8 @@ const registerTokenCompat = (md, baseOpt) => {
             if (!prevToken || !nextToken) continue
             if (prevToken.type !== 'text' || !prevToken.content) continue
             if (nextToken.type !== 'text' || !nextToken.content) continue
-            const prevCharCode = prevToken.content.charCodeAt(prevToken.content.length - 1)
-            const nextCharCode = nextToken.content.charCodeAt(0)
+            const prevCharCode = codePointBeforeSafe(prevToken.content, prevToken.content.length, 0)
+            const nextCharCode = codePointAtSafe(nextToken.content, 0, 0)
             const isAsciiWord = isAsciiWordCode(nextCharCode)
             const shouldReplace = isAsciiWord &&
               isJapaneseChar(prevCharCode) && !isJapaneseChar(nextCharCode)
@@ -143,8 +140,8 @@ const registerTokenCompat = (md, baseOpt) => {
           for (let idx = 0; idx < child.content.length; idx++) {
             const ch = child.content[idx]
             if (ch === '\n') {
-              const prevCharCode = idx > 0 ? child.content.charCodeAt(idx - 1) : 0
-              const nextCharCode = idx + 1 < child.content.length ? child.content.charCodeAt(idx + 1) : 0
+              const prevCharCode = codePointBeforeSafe(child.content, idx, 0)
+              const nextCharCode = codePointAtSafe(child.content, idx + 1, 0)
               const isAsciiWord = isAsciiWordCode(nextCharCode)
               const shouldReplace = isAsciiWord &&
                 isJapaneseChar(prevCharCode) && !isJapaneseChar(nextCharCode)
@@ -192,7 +189,7 @@ const registerTokenCompat = (md, baseOpt) => {
             if (!prevTextCharCode || !isJapaneseChar(prevTextCharCode)) continue
             const next = children[j + 1]
             if (!next || next.type !== 'text' || !next.content) continue
-            const nextCharCode = next.content.charCodeAt(0)
+            const nextCharCode = codePointAtSafe(next.content, 0, 0)
             if (nextCharCode !== 0x7B) continue
             child.type = 'softbreak'
             child.tag = ''
@@ -201,7 +198,7 @@ const registerTokenCompat = (md, baseOpt) => {
             child.info = ''
             continue
           }
-          prevTextCharCode = child.content.charCodeAt(child.content.length - 1)
+          prevTextCharCode = codePointBeforeSafe(child.content, child.content.length, 0)
         }
       }
     }

package/src/token-core.js

@@ -2,16 +2,19 @@ import { isWhiteSpace } from 'markdown-it/lib/common/utils.mjs'
 import Token from 'markdown-it/lib/token.mjs'
 import {
   CHAR_ASTERISK,
-  CHAR_SPACE,
-  CHAR_TAB,
   CHAR_NEWLINE,
-  CHAR_IDEOGRAPHIC_SPACE,
+  codePointAtSafe,
+  codePointBeforeSafe,
+  codePointStartBefore,
+  codePointSize,
   isJapaneseChar,
+  isAsciiWordCode,
+  isSoftSpaceCode,
   MODE_FLAG_COMPATIBLE,
   MODE_FLAG_AGGRESSIVE,
   MODE_FLAG_JAPANESE_PLUS,
-  getRuntimeOpt,
-  hasRuntimeOverride
+  hasRuntimeOverride,
+  getRuntimeOpt
 } from './token-utils.js'
 
 const SCAN_DELIMS_PATCHED = Symbol.for('strongJaTokenScanDelimsPatched')
@@ -20,10 +23,6 @@ const PREV_STAR_HAS_OPENER = 1
 const PREV_STAR_HAS_JP_BETWEEN = 2
 const SCAN_DELIMS_LOOKUP_KEY = Symbol.for('strongJaTokenScanDelimsLookup')
 
-const isSoftSpaceCode = (code) => {
-  return code === CHAR_SPACE || code === CHAR_TAB || code === CHAR_IDEOGRAPHIC_SPACE
-}
-
 const isPlusQuoteWrapperOpen = (code) => {
   return code === 0x2018 || // ‘
     code === 0x201C || // “
@@ -256,12 +255,6 @@ const isSingleStarClosingBoundary = (code) => {
     isClosingBracketLike(code)
 }
 
-const isAsciiAlphaNum = (code) => {
-  return (code >= 0x30 && code <= 0x39) ||
-    (code >= 0x41 && code <= 0x5A) ||
-    (code >= 0x61 && code <= 0x7A)
-}
-
 const isAsciiGuardOpenWrapper = (code) => {
   return code === 0x22 || // "
     code === 0x27 || // '
@@ -291,24 +284,34 @@ const buildScanDelimsLookupCache = (src) => {
 
   let prev = -1
   for (let i = 0; i < len; i++) {
-    const code = src.charCodeAt(i)
+    const code = codePointAtSafe(src, i)
     if (code === CHAR_NEWLINE) {
       prev = -1
       continue
     }
+    const size = codePointSize(code)
     if (!isSoftSpaceCode(code)) prev = i
     prevNonSpaceSameLine[i] = prev
+    if (size === 2 && i + 1 < len) {
+      prevNonSpaceSameLine[i + 1] = prev
+      i++
+    }
   }
 
   let next = -1
   for (let i = len - 1; i >= 0; i--) {
-    const code = src.charCodeAt(i)
+    const cpStart = codePointStartBefore(src, i + 1)
+    const code = cpStart === -1 ? 0 : codePointAtSafe(src, cpStart)
     if (code === CHAR_NEWLINE) {
       next = -1
       continue
     }
-    if (!isSoftSpaceCode(code)) next = i
+    if (!isSoftSpaceCode(code)) next = cpStart
     nextNonSpaceSameLine[i] = next
+    if (cpStart !== i) {
+      nextNonSpaceSameLine[cpStart] = next
+      i = cpStart
+    }
   }
 
   return {
@@ -334,11 +337,13 @@ const findPrevNonSpaceIndex = (src, start, lookupCache = null) => {
       start < lookupCache.prevNonSpaceSameLine.length) {
     return lookupCache.prevNonSpaceSameLine[start]
   }
-  for (let i = start; i >= 0; i--) {
-    const code = src.charCodeAt(i)
+  for (let i = start; i >= 0;) {
+    const cpStart = codePointStartBefore(src, i + 1)
+    if (cpStart === -1) return -1
+    const code = codePointAtSafe(src, cpStart)
     if (code === CHAR_NEWLINE) return -1
-    if (isSoftSpaceCode(code)) continue
-    return i
+    if (!isSoftSpaceCode(code)) return cpStart
+    i = cpStart - 1
   }
   return -1
 }
@@ -351,11 +356,11 @@ const findNextNonSpaceIndex = (src, start, max, lookupCache = null) => {
     const next = lookupCache.nextNonSpaceSameLine[start]
     return next !== -1 && next < max ? next : -1
   }
-  for (let i = start; i < max; i++) {
-    const code = src.charCodeAt(i)
+  for (let i = start; i < max;) {
+    const code = codePointAtSafe(src, i)
     if (code === CHAR_NEWLINE) return -1
-    if (isSoftSpaceCode(code)) continue
-    return i
+    if (!isSoftSpaceCode(code)) return i
+    i += codePointSize(code)
   }
   return -1
 }
@@ -364,26 +369,26 @@ const hasAsciiStartAfterOptionalOpenWrappers = (src, index, max, lookupCache = n
   let i = index
   // Two wrappers are enough for common shapes: * [ "word" ]*
   for (let wrappers = 0; wrappers < 2 && i >= 0 && i < max; wrappers++) {
-    const code = src.charCodeAt(i)
+    const code = codePointAtSafe(src, i)
     if (!isAsciiGuardOpenWrapper(code)) break
     i = findNextNonSpaceIndex(src, i + 1, max, lookupCache)
     if (i === -1) return false
   }
   if (i < 0 || i >= max) return false
-  return isAsciiAlphaNum(src.charCodeAt(i))
+  return isAsciiWordCode(codePointAtSafe(src, i))
 }
 
 const hasAsciiEndBeforeOptionalCloseWrappers = (src, index, lookupCache = null) => {
   let i = index
   // Two wrappers are enough for common shapes: *["word"] *
   for (let wrappers = 0; wrappers < 2 && i >= 0; wrappers++) {
-    const code = src.charCodeAt(i)
+    const code = codePointAtSafe(src, i)
     if (!isAsciiGuardCloseWrapper(code)) break
     i = findPrevNonSpaceIndex(src, i - 1, lookupCache)
     if (i === -1) return false
   }
   if (i < 0) return false
-  return isAsciiAlphaNum(src.charCodeAt(i))
+  return isAsciiWordCode(codePointAtSafe(src, i))
 }
 
 const isMarkdownStructuralOpenWrapper = (code) => {
@@ -420,18 +425,20 @@ const findPrevNonSpaceLimited = (src, start, maxLook, lookupCache = null) => {
       start < lookupCache.prevNonSpaceSameLine.length) {
     const prev = lookupCache.prevNonSpaceSameLine[start]
     if (prev !== -1 && (start - prev) < maxLook) {
-      return src.charCodeAt(prev)
+      return codePointAtSafe(src, prev)
     }
     return 0
   }
   let looked = 0
-  for (let i = start; i >= 0; i--) {
+  for (let i = start; i >= 0;) {
     if (looked >= maxLook) break
-    const code = src.charCodeAt(i)
-    looked++
+    const cpStart = codePointStartBefore(src, i + 1)
+    if (cpStart === -1) break
+    const code = codePointAtSafe(src, cpStart)
+    looked += i - cpStart + 1
     if (code === CHAR_NEWLINE) return 0
-    if (isSoftSpaceCode(code)) continue
-    return code
+    if (!isSoftSpaceCode(code)) return code
+    i = cpStart - 1
   }
   return 0
 }
@@ -443,18 +450,19 @@ const findNextNonSpaceLimited = (src, start, max, maxLook, lookupCache = null) =
       start < lookupCache.nextNonSpaceSameLine.length) {
     const next = lookupCache.nextNonSpaceSameLine[start]
     if (next !== -1 && next < max && (next - start) < maxLook) {
-      return src.charCodeAt(next)
+      return codePointAtSafe(src, next)
     }
     return 0
   }
   let looked = 0
-  for (let i = start; i < max; i++) {
+  for (let i = start; i < max;) {
     if (looked >= maxLook) break
-    const code = src.charCodeAt(i)
-    looked++
+    const code = codePointAtSafe(src, i)
+    const size = codePointSize(code)
+    looked += size
     if (code === CHAR_NEWLINE) return 0
-    if (isSoftSpaceCode(code)) continue
-    return code
+    if (!isSoftSpaceCode(code)) return code
+    i += size
   }
   return 0
 }
@@ -473,8 +481,8 @@ const hasJapaneseContextForBracketWrapper = (src, start, pos, max, lastChar, nex
 
 const scanPrevSingleStarContextFlags = (src, start) => {
   let hasJapaneseBetween = false
-  for (let i = start - 1; i >= 0; i--) {
-    const code = src.charCodeAt(i)
+  for (let i = codePointStartBefore(src, start); i >= 0; i = codePointStartBefore(src, i)) {
+    const code = codePointAtSafe(src, i)
     if (code === CHAR_NEWLINE) break
     if (isSentenceBoundaryStop(code) && i < start - 1) break
     if (code !== CHAR_ASTERISK) {
@@ -486,8 +494,8 @@ const scanPrevSingleStarContextFlags = (src, start) => {
       backslashCount++
     }
     if ((backslashCount % 2) === 1) continue
-    const prevCode = i > 0 ? src.charCodeAt(i - 1) : 0
-    const nextCode = i + 1 < src.length ? src.charCodeAt(i + 1) : 0
+    const prevCode = codePointBeforeSafe(src, i, 0)
+    const nextCode = codePointAtSafe(src, i + 1, 0)
     if (prevCode === CHAR_ASTERISK || nextCode === CHAR_ASTERISK) continue
     return hasJapaneseBetween ? PREV_STAR_HAS_OPENER | PREV_STAR_HAS_JP_BETWEEN : PREV_STAR_HAS_OPENER
   }
@@ -789,12 +797,12 @@ const patchScanDelims = (md) => {
     const aggressiveMode = (modeFlags & MODE_FLAG_AGGRESSIVE) !== 0
     const max = this.posMax
     let lookupCache = null
-    const lastChar = start > 0 ? src.charCodeAt(start - 1) : 0x20
+    const lastChar = codePointBeforeSafe(src, start, 0x20)
 
     const count = base && base.length ? base.length : 1
     const pos = start + count
 
-    const nextChar = pos < max ? src.charCodeAt(pos) : 0x20
+    const nextChar = codePointAtSafe(src, pos, 0x20)
     let prevStarFlags = -1
 
     const leftJapanese = isJapaneseChar(lastChar)
@@ -830,7 +838,7 @@ const patchScanDelims = (md) => {
         lookupCache || (lookupCache = getScanDelimsLookupCache(this))
       )
       if (prevNonSpaceIdx !== -1) {
-        const prevNonSpaceLocal = src.charCodeAt(prevNonSpaceIdx)
+        const prevNonSpaceLocal = codePointAtSafe(src, prevNonSpaceIdx)
         const plusStrictAsciiBoundary = plusMode &&
           hasAsciiEndBeforeOptionalCloseWrappers(src, prevNonSpaceIdx, lookupCache)
         if (prevNonSpaceLocal !== CHAR_ASTERISK && !plusStrictAsciiBoundary) {
@@ -846,7 +854,7 @@ const patchScanDelims = (md) => {
         lookupCache || (lookupCache = getScanDelimsLookupCache(this))
       )
       if (nextNonSpaceIdx !== -1) {
-        const nextNonSpace = src.charCodeAt(nextNonSpaceIdx)
+        const nextNonSpace = codePointAtSafe(src, nextNonSpaceIdx)
         const plusStrictAsciiBoundary = plusMode &&
           hasAsciiStartAfterOptionalOpenWrappers(src, nextNonSpaceIdx, max, lookupCache)
         if (nextNonSpace !== CHAR_ASTERISK && !plusStrictAsciiBoundary) {
@@ -870,8 +878,10 @@ const patchScanDelims = (md) => {
     if (!aggressiveMode && count === 1) {
       // Keep local directionality to avoid degrading markdown-it-valid runs,
       // e.g. `[。*a**](u)` where the first `*` should remain opener-only.
-      const rightIsBoundary = isSingleStarClosingBoundary(nextChar) || isWrapperOpenLike(nextChar)
-      const leftIsBoundary = isSingleStarBoundary(lastChar) || isWrapperCloseLike(lastChar)
+      const rightIsOpenWrapper = isWrapperOpenLike(nextChar)
+      const leftIsCloseWrapper = isWrapperCloseLike(lastChar)
+      const rightIsBoundary = isSingleStarClosingBoundary(nextChar) || rightIsOpenWrapper
+      const leftIsBoundary = isSingleStarBoundary(lastChar) || leftIsCloseWrapper
       if (leftJapanese && !rightJapanese && !rightIsBoundary) {
         prevStarFlags = ensurePrevStarFlags(src, start, prevStarFlags)
         if ((prevStarFlags & PREV_STAR_HAS_OPENER) === 0) {
@@ -880,28 +890,30 @@ const patchScanDelims = (md) => {
       } else if (!leftJapanese && rightJapanese && !leftIsBoundary) {
         relaxedOpen = false
       }
-      const rightIsOpenWrapper = isWrapperOpenLike(nextChar)
-      const leftIsCloseWrapper = isWrapperCloseLike(lastChar)
-      prevStarFlags = ensurePrevStarFlags(src, start, prevStarFlags)
-      const hasPrevJapaneseOpener = (prevStarFlags & PREV_STAR_HAS_OPENER) !== 0
-      const hasJapaneseSincePrevStar = (prevStarFlags & PREV_STAR_HAS_JP_BETWEEN) !== 0
       const leftIsExtraClosePunct = isExtraSingleStarClosePunct(lastChar)
-      const canForceCloseByPunct = leftIsExtraClosePunct && hasJapaneseSincePrevStar
-      if (leftJapanese &&
-          rightIsOpenWrapper &&
-          !hasPrevJapaneseOpener &&
-          !isMarkdownStructuralOpenWrapper(nextChar)) {
-        forceOpen = true
-        forceClose = false
-      } else if (leftIsCloseWrapper && rightJapanese && hasPrevJapaneseOpener) {
-        forceOpen = false
-        forceClose = true
-      } else if ((leftIsCloseWrapper || canForceCloseByPunct) &&
-          !rightJapanese &&
-          !rightIsBoundary &&
-          hasPrevJapaneseOpener) {
-        forceOpen = false
-        forceClose = true
+      const canCheckForceOpen =
+        leftJapanese && rightIsOpenWrapper && !isMarkdownStructuralOpenWrapper(nextChar)
+      const canCheckForceClose =
+        (leftIsCloseWrapper && rightJapanese) ||
+        ((leftIsCloseWrapper || leftIsExtraClosePunct) && !rightJapanese && !rightIsBoundary)
+      if (canCheckForceOpen || canCheckForceClose) {
+        prevStarFlags = ensurePrevStarFlags(src, start, prevStarFlags)
+        const hasPrevJapaneseOpener = (prevStarFlags & PREV_STAR_HAS_OPENER) !== 0
+        const hasJapaneseSincePrevStar = (prevStarFlags & PREV_STAR_HAS_JP_BETWEEN) !== 0
+        const canForceCloseByPunct = leftIsExtraClosePunct && hasJapaneseSincePrevStar
+        if (canCheckForceOpen && !hasPrevJapaneseOpener) {
+          forceOpen = true
+          forceClose = false
+        } else if (leftIsCloseWrapper && rightJapanese && hasPrevJapaneseOpener) {
+          forceOpen = false
+          forceClose = true
+        } else if ((leftIsCloseWrapper || canForceCloseByPunct) &&
+            !rightJapanese &&
+            !rightIsBoundary &&
+            hasPrevJapaneseOpener) {
+          forceOpen = false
+          forceClose = true
+        }
       }
     }
     const finalOpen = forceOpen === null ? ((base && base.can_open) || relaxedOpen) : forceOpen

package/src/token-link-utils.js

@@ -1,6 +1,6 @@
 import Token from 'markdown-it/lib/token.mjs'
 import { isWhiteSpace } from 'markdown-it/lib/common/utils.mjs'
-import { getReferenceCount } from './token-utils.js'
+import { cloneMap, getReferenceCount } from './token-utils.js'
 
 const CHAR_OPEN_BRACKET = 0x5B // [
 const CHAR_CLOSE_BRACKET = 0x5D // ]
@@ -80,11 +80,6 @@ const getNormalizeRef = (state) => {
 }
 
 
-const cloneMap = (map) => {
-  if (!map || !Array.isArray(map)) return null
-  return [map[0], map[1]]
-}
-
 const getMapFromTokenRange = (tokens, startIdx, endIdx) => {
   if (!tokens || startIdx > endIdx) return null
   let startLine = null