@ai-react-markdown/engine 2.11.0 → 2.12.0

package/dist/index.cjs

@@ -79,6 +79,7 @@ __export(src_exports, {
   rehypeRebaseHashLinks: () => rehypeRebaseHashLinks_default,
   rehypeVerifyEngineTags: () => rehypeVerifyEngineTags,
   removeComments: () => removeComments,
+  resolveCrossChunkReference: () => resolveCrossChunkReference,
   sanitizeCrossChunkUrl: () => sanitizeCrossChunkUrl,
   sanitizeSchema: () => sanitizeSchema,
   shortenDocumentId: () => shortenDocumentId,
@@ -2223,6 +2224,38 @@ function normalizeForMatch(s) {
   return (0, import_micromark_util_normalize_identifier3.normalizeIdentifier)(s);
 }
 
+// src/components/blankLineScanner.ts
+function createBlankLineScanner() {
+  let end = 0;
+  let newline = false;
+  let cr = false;
+  return (source, from = 0) => {
+    if (from === 0) {
+      end = 0;
+      newline = false;
+      cr = false;
+    }
+    for (let i = from; i < source.length; i++) {
+      const c = source[i];
+      if (c === "\n") {
+        if (newline) {
+          end = i + 1;
+          newline = false;
+        } else newline = true;
+        cr = false;
+      } else if (newline) {
+        if ((c === " " || c === "	") && !cr) continue;
+        if (c === "\r" && !cr) cr = true;
+        else {
+          newline = false;
+          cr = false;
+        }
+      }
+    }
+    return end;
+  };
+}
+
 // src/components/collectDefLabels.ts
 var SCANNER_BOUNDARY_PROFILE = { defListEnabled: false, mathFlow: false, referenceTaint: false };
 function buildProcessor() {
@@ -2254,19 +2287,12 @@ var setsEqual = (a, b) => {
   for (const v of a) if (!b.has(v)) return false;
   return true;
 };
-var BLANK_LINE_RE = /\r?\n[ \t]*\r?\n/g;
-function lastRegionStart(source) {
-  BLANK_LINE_RE.lastIndex = 0;
-  let start = 0;
-  for (let m = BLANK_LINE_RE.exec(source); m !== null; m = BLANK_LINE_RE.exec(source)) {
-    start = m.index + m[0].length;
-  }
-  return start;
-}
 var DEF_LINE_START_RE = /^[ \t>*+\d.)-]*\[(?:[^\]\\]|\\[\s\S])*\]:/m;
 function createDefLabelScanner(parse = collectDefLabels) {
   let prevSource = null;
   let prevLabels = null;
+  const scanBlankLines = createBlankLineScanner();
+  let regionStart = 0;
   let frozenEnd = 0;
   let frozenFootnotes = /* @__PURE__ */ new Set();
   let frozenLinks = /* @__PURE__ */ new Set();
@@ -2279,13 +2305,16 @@ function createDefLabelScanner(parse = collectDefLabels) {
   };
   return {
     scan(source) {
+      if (source === prevSource && prevLabels !== null) return prevLabels;
+      const previousRegionStart = regionStart;
+      const appended = prevSource !== null && source.startsWith(prevSource);
+      regionStart = scanBlankLines(source, appended ? prevSource.length : 0);
       let isAppend = false;
       if (prevSource !== null && prevLabels !== null) {
         if (source === prevSource) return prevLabels;
         if (source.startsWith(prevSource)) {
           isAppend = true;
-          const regionStart = lastRegionStart(prevSource);
-          const region = prevSource.slice(regionStart) + source.slice(prevSource.length);
+          const region = source.slice(previousRegionStart);
           if (!DEF_LINE_START_RE.test(region)) {
             prevSource = source;
             return prevLabels;
@@ -2518,8 +2547,47 @@ function* extractContributions(mdast, options = {}) {
   for (const c of out) yield c;
 }
 
+// src/components/registryIndex.ts
+function buildRegistryIndex(registry) {
+  const index = {
+    footnotes: /* @__PURE__ */ new Map(),
+    links: /* @__PURE__ */ new Map(),
+    numbers: /* @__PURE__ */ new Map(),
+    counts: /* @__PURE__ */ new Map(),
+    occurrences: /* @__PURE__ */ new Map()
+  };
+  for (const sym of registry.chunkOrder) {
+    const data = registry.chunkData.get(sym);
+    if (!data) continue;
+    for (const label of data.defs.keys()) if (!index.footnotes.has(label)) index.footnotes.set(label, sym);
+    for (const label of data.linkDefs.keys()) if (!index.links.has(label)) index.links.set(label, sym);
+    const local = /* @__PURE__ */ new Map();
+    index.occurrences.set(sym, local);
+    for (const ref of data.refs) {
+      if (ref.kind !== "footnote") continue;
+      const label = ref.label;
+      if (!index.numbers.has(label)) index.numbers.set(label, index.numbers.size + 1);
+      const total = (index.counts.get(label) ?? 0) + 1;
+      index.counts.set(label, total);
+      const prior = local.get(label);
+      if (prior) prior.count++;
+      else local.set(label, { start: total, count: 1 });
+    }
+  }
+  return index;
+}
+
 // src/components/documentRegistry.ts
 function createRegistry(onEmpty) {
+  let index;
+  let indexedVersion = -1;
+  const getIndex = () => {
+    if (!index || indexedVersion !== reg.version) {
+      index = buildRegistryIndex(reg);
+      indexedVersion = reg.version;
+    }
+    return index;
+  };
   const reg = {
     chunkOrder: [],
     chunkData: /* @__PURE__ */ new Map(),
@@ -2671,72 +2739,25 @@ function createRegistry(onEmpty) {
       };
     },
     canonicalFootnoteFor(label) {
-      const id = normalizeId(label);
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (data?.defs.has(id)) return sym;
-      }
-      return null;
+      return getIndex().footnotes.get(normalizeId(label)) ?? null;
     },
     canonicalLinkFor(label) {
-      const id = normalizeId(label);
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (data?.linkDefs.has(id)) return sym;
-      }
-      return null;
+      return getIndex().links.get(normalizeId(label)) ?? null;
     },
     globalNumber(label) {
-      const id = normalizeId(label);
-      let n = 0;
-      const seen = /* @__PURE__ */ new Set();
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        for (const ref of data.refs) {
-          if (ref.kind !== "footnote") continue;
-          if (!seen.has(ref.label)) {
-            seen.add(ref.label);
-            n++;
-            if (ref.label === id) return n;
-          }
-        }
-      }
-      return null;
+      return getIndex().numbers.get(normalizeId(label)) ?? null;
     },
     resolveLinkDef(label) {
-      const sym = this.canonicalLinkFor(label);
-      if (!sym) return null;
-      return this.chunkData.get(sym)?.linkDefs.get(normalizeId(label)) ?? null;
+      const id = normalizeId(label);
+      const sym = getIndex().links.get(id);
+      return sym ? this.chunkData.get(sym)?.linkDefs.get(id) ?? null : null;
     },
     getRefsForLabel(label) {
-      const id = normalizeId(label);
-      let n = 0;
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        for (const ref of data.refs) {
-          if (ref.kind === "footnote" && ref.label === id) n++;
-        }
-      }
-      return n;
+      return getIndex().counts.get(normalizeId(label)) ?? 0;
     },
     globalOccurrenceForRef(chunkSym, label, localOccurrence) {
-      const id = normalizeId(label);
-      let global2 = 0;
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        let localCount = 0;
-        for (const ref of data.refs) {
-          if (ref.kind !== "footnote") continue;
-          if (ref.label !== id) continue;
-          localCount++;
-          global2++;
-          if (sym === chunkSym && localCount === localOccurrence) return global2;
-        }
-      }
-      return null;
+      const range = getIndex().occurrences.get(chunkSym)?.get(normalizeId(label));
+      return range && Number.isInteger(localOccurrence) && localOccurrence > 0 && localOccurrence <= range.count ? range.start + localOccurrence - 1 : null;
     },
     _notify() {
       this.version++;
@@ -2801,7 +2822,7 @@ var ENGINE_PLACEHOLDER_TAGS = /* @__PURE__ */ new Set([
   "cross-chunk-image"
 ]);
 var ENGINE_PROVENANCE_PROPERTY = "engineProvenance";
-function walk(parent, provenance) {
+function walk(parent, provenance, ancestors) {
   const children = parent.children;
   let i = 0;
   while (i < children.length) {
@@ -2812,21 +2833,32 @@ function walk(parent, provenance) {
       const genuine = provenance !== "" && typeof stamped === "string" && stamped === provenance;
       if (genuine) {
         delete props[ENGINE_PROVENANCE_PROPERTY];
-        walk(node, provenance);
+        if (ancestors && (node.tagName === "cross-chunk-link" || node.tagName === "cross-chunk-image")) {
+          (node.data ??= {}).referenceAncestors = ancestors.slice();
+        }
+        ancestors?.push(
+          node.tagName === "cross-chunk-link" ? "a" : node.tagName === "cross-chunk-image" ? "img" : node.tagName
+        );
+        walk(node, provenance, ancestors);
+        ancestors?.pop();
         i += 1;
       } else {
         children.splice(i, 1, ...node.children);
       }
       continue;
     }
-    if (node.type === "element") walk(node, provenance);
+    if (node.type === "element") {
+      ancestors?.push(node.tagName);
+      walk(node, provenance, ancestors);
+      ancestors?.pop();
+    }
     i += 1;
   }
 }
 function rehypeVerifyEngineTags(options) {
   const provenance = options?.provenance ?? "";
   return function transformer(tree) {
-    walk(tree, provenance);
+    walk(tree, provenance, options?.referenceAncestors ? [] : void 0);
   };
 }
 
@@ -2847,16 +2879,18 @@ var import_remark_remove_comments = __toESM(require("remark-remove-comments"), 1
 // src/components/rehypeRebaseHashLinks.ts
 var import_unist_util_visit5 = require("unist-util-visit");
 var DEFAULT_PREFIX = "user-content-";
+function rebaseHashHref(href, prefix) {
+  const hashPrefix = "#" + prefix;
+  return href.startsWith("#") && !href.startsWith(hashPrefix) ? hashPrefix + href.slice(1) : href;
+}
 var rehypeRebaseHashLinks = (options) => {
   const prefix = options?.prefix ?? DEFAULT_PREFIX;
-  const hashPrefix = "#" + prefix;
   return (tree) => {
     (0, import_unist_util_visit5.visit)(tree, "element", (node) => {
       if (node.tagName !== "a") return;
       const href = node.properties?.href;
       if (typeof href !== "string" || !href.startsWith("#")) return;
-      if (href.startsWith(hashPrefix)) return;
-      node.properties.href = hashPrefix + href.slice(1);
+      node.properties.href = rebaseHashHref(href, prefix);
     });
   };
 };
@@ -2942,7 +2976,15 @@ function buildCoreRehypePlugins(sanitizeSchema2, clobberPrefix, options) {
     [import_rehype_raw.default, { passThrough: [] }],
     // Unwrap forged engine placeholders BEFORE sanitize admits their tag
     // names. Only when the caller holds a credential (see the option's doc).
-    ...options ? [[rehypeVerifyEngineTags, { provenance: options.provenance }]] : [],
+    ...options ? [
+      [
+        rehypeVerifyEngineTags,
+        {
+          provenance: options.provenance,
+          ...sanitizeSchema2.ancestors?.a || sanitizeSchema2.ancestors?.img ? { referenceAncestors: true } : {}
+        }
+      ]
+    ] : [],
     // Sanitize HTML while allowing <mark> (highlight), KaTeX class names,
     // and any extra protocols the caller permitted via the `sanitizeSchema`
     // prop. Override `clobberPrefix` with the instance-scoped value — the
@@ -3016,11 +3058,9 @@ function buildCrossChunkHandlers() {
         type: "element",
         tagName: "cross-chunk-link",
         properties: {
-          // `label` is the ORIGINAL source text (mdast's `label` field), NOT
-          // the normalized `identifier`. The placeholder uses it to construct
-          // hrefs that line up with mdast-util-to-hast's default `<li id>`
-          // which also preserves source case. Registry lookups normalize
-          // internally, so cross-chunk case-insensitive matching still works.
+          // Display labels decode escapes; registry identifiers must retain
+          // those bytes. Never use the display label as the lookup key.
+          identifier: node.identifier,
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           documentId: s.options.documentId,
@@ -3039,6 +3079,7 @@ function buildCrossChunkHandlers() {
         type: "element",
         tagName: "cross-chunk-image",
         properties: {
+          identifier: node.identifier,
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           alt: node.alt ?? "",
@@ -4301,8 +4342,8 @@ var sanitizeSchema = deepFreeze(
     attributes: {
       ...import_rehype_sanitize2.defaultSchema.attributes,
       code: mergeClassNameAllowlist(import_rehype_sanitize2.defaultSchema.attributes?.code, ["math-inline", "math-display"]),
-      "cross-chunk-link": ["label", "referenceType", "documentId", "localUrl", "localTitle"],
-      "cross-chunk-image": ["label", "referenceType", "documentId", "alt", "localUrl", "localTitle"],
+      "cross-chunk-link": ["identifier", "label", "referenceType", "documentId", "localUrl", "localTitle"],
+      "cross-chunk-image": ["identifier", "label", "referenceType", "documentId", "alt", "localUrl", "localTitle"],
       "footnote-sup": ["label", "localOccurrence", "localNumber", "documentId"]
     },
     strip: [.../* @__PURE__ */ new Set([...import_rehype_sanitize2.defaultSchema.strip || [], ...STRIPPED_TAGS])]
@@ -4338,6 +4379,49 @@ function sanitizeCrossChunkUrl(rawUrl, key, tagName, urlTransform, schema) {
   return String(transformed);
 }
 
+// src/components/resolveCrossChunkReference.ts
+var import_rehype_sanitize3 = __toESM(require("rehype-sanitize"), 1);
+var import_micromark_util_sanitize_uri3 = require("micromark-util-sanitize-uri");
+function resolveCrossChunkReference(input, schema, urlTransform, clobberPrefix) {
+  const key = input.tagName === "a" ? "href" : "src";
+  const element = {
+    type: "element",
+    tagName: input.tagName,
+    properties: {
+      [key]: (0, import_micromark_util_sanitize_uri3.normalizeUri)(input.url),
+      ...input.tagName === "img" ? { alt: input.alt ?? "" } : {},
+      ...input.title !== void 0 ? { title: input.title } : {}
+    },
+    children: input.tagName === "a" ? [{ type: "text", value: "__reference_children__" }] : []
+  };
+  const requiredAncestors = schema.ancestors?.[input.tagName];
+  const recordedAncestors = input.node?.data?.referenceAncestors;
+  let finalSchema = schema;
+  if (requiredAncestors && Array.isArray(recordedAncestors) && requiredAncestors.some((tag) => recordedAncestors.includes(tag))) {
+    const ancestors = { ...schema.ancestors };
+    delete ancestors[input.tagName];
+    finalSchema = { ...schema, ancestors };
+  }
+  const root2 = (0, import_rehype_sanitize3.default)({ ...finalSchema, clobberPrefix })({ type: "root", children: [element] });
+  const node = root2.children[0];
+  if (node?.type !== "element") return { element: null, keepChildren: node?.type === "text" };
+  if (node.tagName === "a" && typeof node.properties.href === "string") {
+    node.properties.href = rebaseHashHref(node.properties.href, clobberPrefix);
+  }
+  node.children = input.node?.children ?? [];
+  if (input.node?.position) node.position = input.node.position;
+  buildTransform({
+    allowedElements: void 0,
+    disallowedElements: void 0,
+    allowElement: void 0,
+    skipHtml: void 0,
+    unwrapDisallowed: void 0,
+    urlTransform
+  })(node, 0, root2);
+  node.children = [];
+  return { element: node, keepChildren: false };
+}
+
 // src/plugins/defs.ts
 function getEnginePluginInternals(plugin) {
   const candidate = plugin;
@@ -4542,6 +4626,8 @@ var createSmoothStreamController = (options = {}) => {
   let source = "";
   let visibleEnd = 0;
   let pending = [];
+  let pendingHead = 0;
+  const pendingCount = () => pending.length - pendingHead;
   let tentativeEnd = 0;
   let finished = false;
   let seam;
@@ -4586,7 +4672,7 @@ var createSmoothStreamController = (options = {}) => {
     cancelFrame = void 0;
   };
   const ensureScheduled = () => {
-    if (disposed || cancelFrame || pending.length === 0) return;
+    if (disposed || cancelFrame || pendingCount() === 0) return;
     lastTickAt = now();
     credit = 0;
     cancelFrame = schedule(tick);
@@ -4600,31 +4686,38 @@ var createSmoothStreamController = (options = {}) => {
     const params = resolveParams();
     let rate;
     if (finished && drainDeadlineAt !== void 0) {
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, drainDeadlineAt - t));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, drainDeadlineAt - t));
     } else if (gapWindow.length > 0 && lastArrivalAt !== void 0) {
       const gaps = gapWindow.map((s) => s.gap).sort((a, b) => a - b);
       const intervalQ = gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * INTERVAL_QUANTILE))];
       const horizon = Math.max(16, Math.min(params.bufferFactor * intervalQ + HORIZON_PAD_MS, params.maxLagMs));
       const deadline = Math.max(lastArrivalAt + horizon, t + DEADLINE_FLOOR_MS);
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, deadline - t));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, deadline - t));
     } else {
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, params.correctionTauMs));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, params.correctionTauMs));
     }
     credit += rate * dt / 1e3;
-    const reveal = Math.min(Math.floor(credit), pending.length);
+    const reveal = Math.min(Math.floor(credit), pendingCount());
     if (reveal > 0) {
-      visibleEnd = pending[reveal - 1];
-      pending = pending.slice(reveal);
-      credit = pending.length > 0 ? credit - reveal : 0;
+      visibleEnd = pending[pendingHead + reveal - 1];
+      pendingHead += reveal;
+      if (pendingHead === pending.length) {
+        pending = [];
+        pendingHead = 0;
+      } else if (pendingHead >= 1024 && pendingHead * 2 >= pending.length) {
+        pending = pending.slice(pendingHead);
+        pendingHead = 0;
+      }
+      credit = pendingCount() > 0 ? credit - reveal : 0;
       notify();
     }
-    if (!disposed && !cancelFrame && pending.length > 0) cancelFrame = schedule(tick);
+    if (!disposed && !cancelFrame && pendingCount() > 0) cancelFrame = schedule(tick);
   };
   const resegmentTail = () => {
-    if (seam !== void 0 && seam < source.length && pending[pending.length - 1] === seam) {
+    if (pendingCount() > 0 && seam !== void 0 && seam < source.length && pending[pending.length - 1] === seam) {
       pending.pop();
     }
-    const from = pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
+    const from = pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd;
     let anchor = from;
     if (seam !== void 0 && from <= seam) {
       anchor = Math.max(0, from - RESUME_LOOKBACK);
@@ -4658,6 +4751,7 @@ var createSmoothStreamController = (options = {}) => {
     visibleEnd = next.length;
     tentativeEnd = next.length;
     pending = [];
+    pendingHead = 0;
     seam = next.length;
     credit = 0;
     cancelScheduled();
@@ -4689,7 +4783,7 @@ var createSmoothStreamController = (options = {}) => {
       if (finished) return;
       finished = true;
       lastArrivalAt = void 0;
-      if (tentativeEnd > (pending.length > 0 ? pending[pending.length - 1] : visibleEnd)) {
+      if (tentativeEnd > (pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd)) {
         pending.push(tentativeEnd);
       }
       const params = resolveParams();
@@ -4715,10 +4809,11 @@ var createSmoothStreamController = (options = {}) => {
     snap,
     flush() {
       disposed = false;
-      const target = finished ? source.length : pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
+      const target = finished ? source.length : pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd;
       if (target <= visibleEnd) return;
       visibleEnd = target;
       pending = [];
+      pendingHead = 0;
       credit = 0;
       cancelScheduled();
       notify();
@@ -5569,6 +5664,7 @@ function createRemendPreprocessor(options) {
   rehypeRebaseHashLinks,
   rehypeVerifyEngineTags,
   removeComments,
+  resolveCrossChunkReference,
   sanitizeCrossChunkUrl,
   sanitizeSchema,
   shortenDocumentId,