@ai-react-markdown/engine 2.8.0 → 2.8.2

package/README.md

@@ -90,7 +90,7 @@ for (const frame of ['# Hello', '# Hello\n\nworld', '# Hello\n\nworld and more']
 
 ## Verification
 
-The incremental engine ships with a five-layer equivalence stack (fixture pins, fuzz arbiter, direction battery, exhaustive census, arbiter-sensitivity meta-suite) plus a release-gate soak (`scripts/run-soak.sh`); the full record lives in `src/experiments/prefixFreeze/README.md`. Every reachable divergence found so far is pinned as a deterministic test.
+The incremental engine ships with a five-layer equivalence stack (fixture pins, fuzz arbiter, direction battery, exhaustive census, arbiter-sensitivity meta-suite) plus a five-leg release-gate soak (`scripts/soak/fiveleg.sh`, from the repo root, with a fresh seed base); the full record lives in `src/experiments/prefixFreeze/README.md`. Every reachable divergence found so far is pinned as a deterministic test.
 
 ## Runtime support
 

package/dist/index.cjs

@@ -48,7 +48,6 @@ __export(src_exports, {
   collectDefLabels: () => collectDefLabels,
   computeFreezeBoundary: () => computeFreezeBoundary,
   createDefLabelScanner: () => createDefLabelScanner,
-  createFile: () => createFile,
   createIncrementalLatexPreprocessor: () => createIncrementalLatexPreprocessor,
   createProcessor: () => createProcessor,
   createRegistry: () => createRegistry,
@@ -65,10 +64,8 @@ __export(src_exports, {
   hasLoneSurrogate: () => hasLoneSurrogate,
   highlight: () => highlight,
   isFootnoteSection: () => isFootnoteSection,
-  isWhitespaceText: () => isWhitespaceText,
   lastMeaningfulIdx: () => lastMeaningfulIdx,
   measureStage: () => measureStage,
-  mergeClassNameAllowlist: () => mergeClassNameAllowlist,
   normalizeForMatch: () => normalizeForMatch,
   normalizeId: () => normalizeId,
   pangu: () => pangu,
@@ -84,7 +81,6 @@ __export(src_exports, {
   shortenDocumentId: () => shortenDocumentId,
   smartypants: () => smartypants,
   sourceIdFromFootnoteLiId: () => sourceIdFromFootnoteLiId,
-  splitByProtectedRegions: () => splitByProtectedRegions,
   subscribeStageTimings: () => subscribeStageTimings,
   transformStage: () => transformStage,
   withDefs: () => withDefs
@@ -594,6 +590,7 @@ var isType7Line = (line) => {
   return i >= t.length;
 };
 var mdHtml = (b, type) => b.kind === "html" && b.type === type;
+var mdType1RawText = (b) => b.kind === "html" && b.type === 1 && b.raw;
 var mdHtml25 = (b) => b.kind === "html" && b.type >= 2 && b.type <= 5;
 var commentEitherOpen = (md, p5) => mdHtml(md, 2) || p5.kind === "comment";
 var inRawTextTok = (t) => t.kind === "rawText" || t.kind === "script";
@@ -615,11 +612,13 @@ var VOID_TAGS = /* @__PURE__ */ new Set([
   "wbr"
 ]);
 var LIST_MARKER_RE = /^ {0,3}(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$)/;
+var CONTAINER_MARKER_RE = /^ {0,3}(?:>|(?:[-*+]|\d{1,9}[.)])[ \t]+\S)/;
 var ATX_HEADING_RE = /^#{1,6}(?:[ \t]|$)/;
 var THEMATIC_BREAK_RE = /^(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$/;
 var BARE_MARKER_RE = /^(?:[-*+]|\d{1,9}[.)])[ \t]*$/;
 var SETEXT_LEFTOVER_RE = /^(?:=+|--)[ \t]*$/;
 var DEF_LIST_DD_RE = /^ {0,3}:[ \t]/;
+var CLOSE_TAG_ONLY_RE = /^[ \t]*<\/[A-Za-z][A-Za-z0-9-]*[ \t]*>[ \t\r]*$/;
 var FENCE_RE = /^ {0,3}(`{3,}|~{3,})/;
 var MATH_RUN_RE = /^ {0,3}(\$\$+)/;
 var TAG_OR_COMMENT_RE = /<(\/?)([A-Za-z][A-Za-z0-9-]*)(?=[\s/>])([^>]*)>|<!--|-->|--!>/g;
@@ -730,9 +729,12 @@ function freshCheckpoint(defListEnabled, mathFlow, referenceTaint) {
     prevLineWasValidDef: false,
     prevLineOpenContent: false,
     tableMaybeOpen: false,
+    containerMaybeOpen: false,
     paragraphHasUnpairedRun: false,
     openBracket: null,
     p5SealPending: false,
+    defBlockMaybeOpen: false,
+    fnDefResumable: false,
     phasePoisonedAt: Infinity,
     pendingTruncatedTags: [],
     pendingTruncatedCloses: [],
@@ -860,9 +862,10 @@ function processConfirmedLine(cp, ln, text) {
   }
   const isBlockStart = cp.prevLineBlank;
   if (cp.p5SealPending && !ln.blank && cp.mdBlock.kind !== "html" && !(cp.p5Tok.kind === "comment" || cp.p5Tok.kind === "bogus")) {
-    const defShapedLine = DEF_RE.test(ln.text) || FOOTNOTE_DEF_RE.test(ln.text);
+    const defShapedLine = DEF_RE.test(ln.text) || FOOTNOTE_DEF_RE.test(ln.text) || cp.defBlockMaybeOpen || cp.fnDefResumable && !(isBlockStart && ln.indent <= 3);
     const commentOnly = ln.text.replace(/<!--[\s\S]*?-->/g, " ").replace(/<!--[\s\S]*$/, " ").replace(/[ \t\r]/g, "") === "";
-    if (!defShapedLine && !commentOnly) {
+    const closeTagOnly = CLOSE_TAG_ONLY_RE.test(ln.text);
+    if (!defShapedLine && !commentOnly && !closeTagOnly) {
       cp.p5SealPending = false;
     }
   }
@@ -914,7 +917,7 @@ function processConfirmedLine(cp, ln, text) {
       cp.openTotal += 1;
     }
   };
-  const definitelyInsideTable = () => (cp.tagBalance.get("table") ?? 0) > 0;
+  const definitelyInsideTable = () => (cp.tagBalance.get("table") ?? 0) > cp.pendingTruncatedTags.filter((t) => t === "table").length;
   const strayTablePart = (tag) => TABLE_PART_NAMES.has(tag) && !definitelyInsideTable();
   const commentOpenAtLineStart = commentEitherOpen(cp.mdBlock, cp.p5Tok);
   const bothCommentsOpenAtLineStart = mdHtml(cp.mdBlock, 2) && cp.p5Tok.kind === "comment";
@@ -936,6 +939,7 @@ function processConfirmedLine(cp, ln, text) {
     cp.prevLineWasValidDef = false;
     cp.prevLineOpenContent = false;
     cp.tableMaybeOpen = false;
+    cp.containerMaybeOpen = false;
     return;
   }
   if (cp.mdBlock.kind !== "math" && !rawOpenAtLineStart) {
@@ -957,6 +961,7 @@ function processConfirmedLine(cp, ln, text) {
       cp.prevLineWasValidDef = false;
       cp.prevLineOpenContent = false;
       cp.tableMaybeOpen = false;
+      cp.containerMaybeOpen = false;
       return;
     }
   }
@@ -973,6 +978,7 @@ function processConfirmedLine(cp, ln, text) {
     cp.prevLineWasValidDef = false;
     cp.prevLineOpenContent = false;
     cp.tableMaybeOpen = false;
+    cp.containerMaybeOpen = false;
     return;
   }
   const mathRun = cp.mathFlow && !rawOpenAtLineStart ? MATH_RUN_RE.exec(ln.text) : null;
@@ -995,6 +1001,7 @@ function processConfirmedLine(cp, ln, text) {
         cp.prevLineWasValidDef = false;
         cp.prevLineOpenContent = false;
         cp.tableMaybeOpen = false;
+        cp.containerMaybeOpen = false;
         return;
       }
     }
@@ -1043,8 +1050,10 @@ function processConfirmedLine(cp, ln, text) {
     cp.prevLineBlank = true;
     cp.prevLineWasText = false;
     cp.prevLineWasValidDef = false;
+    cp.defBlockMaybeOpen = false;
     cp.prevLineOpenContent = false;
     cp.tableMaybeOpen = false;
+    cp.containerMaybeOpen = false;
     return;
   }
   if (isBlockStart) {
@@ -1056,21 +1065,27 @@ function processConfirmedLine(cp, ln, text) {
   const tagStart = ln.indent <= 3 ? /^<\/?([A-Za-z][A-Za-z0-9-]*)/.exec(mdTrimStart(ln.text)) : null;
   if (tagStart) {
     const noRealBlockOpen = cp.mdBlock.kind !== "html";
-    if (noRealBlockOpen && TYPE1_START_RE.test(mdTrimStart(ln.text))) cp.mdBlock = { kind: "html", type: 1 };
+    const t1 = noRealBlockOpen ? TYPE1_START_RE.exec(mdTrimStart(ln.text)) : null;
+    if (t1) cp.mdBlock = { kind: "html", type: 1, raw: RAW_TEXT_ELEMENTS.has(t1[1].toLowerCase()), indent: ln.indent };
     if (cp.mdBlock.kind !== "html") {
       const t = mdTrimStart(ln.text);
       const t6 = TYPE6_START_RE.exec(t);
       const realT6 = t6 !== null && TYPE6_NAMES.has(t6[1].toLowerCase());
-      if (realT6 || TYPE1_START_RE.test(t) || // Type 7 cannot interrupt CONTENT (micromark's paragraph/definition
+      const t1Line = TYPE1_START_RE.test(t);
+      const type7Shaped = !realT6 && !t1Line && isType7Line(t);
+      if (realT6 || t1Line || // Type 7 cannot interrupt CONTENT (micromark's paragraph/definition
       // construct — `prevLineOpenContent`, the exact interrupt input; the
       // old `prevLineWasText` gate refused after headings, terminator
       // lines and fence closes, where micromark measurably opens). The
       // classifier itself is exact too (isType7Line) — including closing
       // raw-text names (`</style>` alone is type 7, measured) and
       // quoted-`>` attribute values.
-      !cp.prevLineOpenContent && isType7Line(t)) {
-        if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: realT6 ? 6 : 7 };
-      } else if (cp.prevLineOpenContent && cp.tableMaybeOpen && isType7Line(t)) {
+      !cp.prevLineOpenContent && type7Shaped) {
+        if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: realT6 ? 6 : 7, indent: ln.indent };
+      } else if (cp.prevLineOpenContent && cp.tableMaybeOpen && type7Shaped) {
+        cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
+      }
+      if (type7Shaped && cp.containerMaybeOpen) {
         cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
       }
     }
@@ -1136,7 +1151,12 @@ function processConfirmedLine(cp, ln, text) {
       pos = c + 1;
       continue;
     }
-    if (commentOpenAtLineStart || inRawTextTok(cp.p5Tok) || mdHtml(cp.mdBlock, 1)) break;
+    if (commentOpenAtLineStart || inRawTextTok(cp.p5Tok) || mdType1RawText(cp.mdBlock)) {
+      if (inRawTextTok(cp.p5Tok) && cp.p5Tok.openedInline && (tailCarriesRetroactive(ln.text) || /<template(?![a-z0-9-])/i.test(ln.text))) {
+        cp.phasePoisonedAt = 0;
+      }
+      break;
+    }
     const pi = scanText.indexOf("<?", pos);
     const cd = scanText.indexOf("<![CDATA[", pos);
     const dm = scanText.slice(pos).search(/<![A-Za-z]/);
@@ -1153,7 +1173,7 @@ function processConfirmedLine(cp, ln, text) {
       pos = bogus + 2;
     } else if (first === cd) {
       rawSpans.push([cd, cd + 9]);
-      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 5 };
+      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 5, indent: ln.indent };
       if (cp.p5Tok.kind === "data") cp.p5Tok = { kind: "bogus" };
       if (!isMdBlank(scanText.slice(0, cd)) || ln.indent > 3) inlineRawOpenerIdx = cd;
       pos = cd + 9;
@@ -1165,14 +1185,14 @@ function processConfirmedLine(cp, ln, text) {
         continue;
       }
       rawSpans.push([pi, pi + 2]);
-      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 3 };
+      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 3, indent: ln.indent };
       if (cp.p5Tok.kind === "data") cp.p5Tok = { kind: "bogus" };
       if (!isMdBlank(scanText.slice(0, pi)) || ln.indent > 3) inlineRawOpenerIdx = pi;
       pos = pi + 2;
     } else {
       rawSpans.push([decl, decl + 2]);
       if (ln.indent <= 3 && /^doctype/i.test(scanText.slice(decl + 2))) cp.phasePoisonedAt = 0;
-      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 4 };
+      if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 4, indent: ln.indent };
       if (cp.p5Tok.kind === "data") cp.p5Tok = { kind: "bogus" };
       if (!isMdBlank(scanText.slice(0, decl)) || ln.indent > 3) inlineRawOpenerIdx = decl;
       pos = decl + 2;
@@ -1231,7 +1251,7 @@ function processConfirmedLine(cp, ln, text) {
         if (next.startsWith(">") || next === "->") {
           continue;
         }
-        if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 2 };
+        if (cp.mdBlock.kind === "none") cp.mdBlock = { kind: "html", type: 2, indent: ln.indent };
         if (cp.p5Tok.kind === "data") cp.p5Tok = { kind: "comment" };
         lastCommentOpenerIdx = m.index;
         continue;
@@ -1373,17 +1393,37 @@ function processConfirmedLine(cp, ln, text) {
       openContent = false;
     } else if (ln.indent >= 4) {
       openContent = cp.prevLineOpenContent;
-    } else if (ATX_HEADING_RE.test(tt) || THEMATIC_BREAK_RE.test(tt) || BARE_MARKER_RE.test(tt)) {
+    } else if (ATX_HEADING_RE.test(tt) || THEMATIC_BREAK_RE.test(tt)) {
       openContent = false;
+    } else if (BARE_MARKER_RE.test(tt)) {
+      openContent = tt[0] === "-" ? false : cp.prevLineOpenContent;
     } else if (SETEXT_LEFTOVER_RE.test(tt)) {
-      openContent = !cp.prevLineOpenContent;
+      openContent = cp.tableMaybeOpen && cp.prevLineOpenContent ? true : !cp.prevLineOpenContent;
     } else {
       openContent = true;
     }
     cp.prevLineOpenContent = openContent;
-    cp.tableMaybeOpen = ln.text.includes("|") || cp.tableMaybeOpen && openContent;
+    cp.tableMaybeOpen = !htmlOwnedLine && ln.text.includes("|") || cp.tableMaybeOpen && openContent;
+    const containerMarker = CONTAINER_MARKER_RE.test(ln.text) || FOOTNOTE_DEF_RE.test(ln.text) || cp.defListEnabled && DEF_LIST_DD_RE.test(ln.text);
+    cp.containerMaybeOpen = openContent && (containerMarker || cp.containerMaybeOpen);
   }
   cp.prevLineWasValidDef = validLinkDef;
+  cp.defBlockMaybeOpen = cp.defBlockMaybeOpen || DEF_RE.test(ln.text) || FOOTNOTE_DEF_RE.test(ln.text);
+  if (FOOTNOTE_DEF_RE.test(ln.text)) cp.fnDefResumable = true;
+  else if (isBlockStart && ln.indent <= 3) cp.fnDefResumable = false;
+  const isBarrier = (name) => SCOPE_BARRIER_NAMES.has(name);
+  const topLevelHtmlBlock = cp.mdBlock.kind === "html" && cp.mdBlock.indent === 0;
+  const confirmedBarriers = cp.openStack.filter(isBarrier).length - cp.pendingTruncatedTags.filter(isBarrier).length;
+  if (!topLevelHtmlBlock && confirmedBarriers > 0) {
+    cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
+  }
+  const maskUnbacked = mdType1RawText(cp.mdBlock) && !inRawTextTok(cp.p5Tok);
+  if (maskUnbacked) {
+    cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
+    if (tailCarriesRetroactive(ln.text) || /<template(?![a-z0-9-])/i.test(ln.text)) {
+      cp.phasePoisonedAt = 0;
+    }
+  }
 }
 
 // src/components/incrementalParse/spliceParse.ts
@@ -1898,7 +1938,7 @@ function alignPrefixCut(prefixMdast, cutRegion, tailWrapVisible) {
   if (sepBuffer.length !== trailingGaps && sepBuffer.length !== trailingGaps + 1) return null;
   for (let j = pairIdx + 1; j < visibles.length; j++) {
     const v = visibles[j];
-    if (v.type === "html" && !/^\s*<[!?]/.test(v.value)) return null;
+    if (v.type === "html" && !isExactSanitizeStrippedConstruct(v.value)) return null;
   }
   for (let i = 0; i < trailingGaps + seam; i++) {
     out.push({ type: "text", value: "\n" });
@@ -1948,7 +1988,7 @@ function tailLeadingTextIsHoist(tailMdastChildren, tailHastChildren) {
   if (!isSeparatorText(firstText)) {
     if (firstText.type === "text" && firstText.position === void 0 && firstVisible?.type === "html") {
       if (/^\s*<\/[A-Za-z][A-Za-z0-9-]*\s*>/.test(firstVisible.value)) return true;
-      if (isCompleteRawConstruct(firstVisible.value)) return false;
+      if (isSanitizeStrippedConstruct(firstVisible.value)) return false;
       return null;
     }
     return false;
@@ -1964,12 +2004,26 @@ function tailLeadingTextIsHoist(tailMdastChildren, tailHastChildren) {
     }
   }
   if (firstVisible.type === "math") return false;
-  if (firstVisible.type === "html" && isCompleteRawConstruct(firstVisible.value)) return false;
+  if (firstVisible.type === "html" && isSanitizeStrippedConstruct(firstVisible.value)) return false;
   return null;
 }
-function isCompleteRawConstruct(value) {
+function isSanitizeStrippedConstruct(value) {
   const v = value.trim();
-  return v.startsWith("<!--") && v.endsWith("-->") || v.startsWith("<?") && v.endsWith("?>") || v.startsWith("<![CDATA[") && v.endsWith("]]>") || /^<![A-Za-z]/.test(v) && v.endsWith(">");
+  return v.startsWith("<!--") && v.endsWith("-->") || v.startsWith("<?") && v.endsWith("?>") || v.startsWith("<![CDATA[") && v.endsWith("]]>") || /^<![A-Za-z]/.test(v) && !/^<!doctype/i.test(v) && v.endsWith(">");
+}
+function isExactSanitizeStrippedConstruct(value) {
+  if (value.startsWith("<!--")) {
+    if (value.length < 7 || !value.endsWith("-->")) return false;
+    const close = value.indexOf("-->", 4);
+    const bangClose = value.indexOf("--!>", 4);
+    if (close !== value.length - 3) return false;
+    return bangClose === -1 || bangClose > close;
+  }
+  if (/^<!doctype/i.test(value)) return false;
+  if (value.startsWith("<?") || /^<!(?:[A-Za-z]|\[CDATA\[)/.test(value)) {
+    return value.indexOf(">") === value.length - 1;
+  }
+  return false;
 }
 function isSeparatorText(node) {
   return node.type === "text" && node.position === void 0 && node.value.trim() === "";
@@ -5123,7 +5177,6 @@ function createRemendPreprocessor(options) {
   collectDefLabels,
   computeFreezeBoundary,
   createDefLabelScanner,
-  createFile,
   createIncrementalLatexPreprocessor,
   createProcessor,
   createRegistry,
@@ -5140,10 +5193,8 @@ function createRemendPreprocessor(options) {
   hasLoneSurrogate,
   highlight,
   isFootnoteSection,
-  isWhitespaceText,
   lastMeaningfulIdx,
   measureStage,
-  mergeClassNameAllowlist,
   normalizeForMatch,
   normalizeId,
   pangu,
@@ -5159,7 +5210,6 @@ function createRemendPreprocessor(options) {
   shortenDocumentId,
   smartypants,
   sourceIdFromFootnoteLiId,
-  splitByProtectedRegions,
   subscribeStageTimings,
   transformStage,
   withDefs