@ai-react-markdown/engine 2.3.0 → 2.3.2

package/dist/index.d.cts

@@ -1393,14 +1393,51 @@ declare function normalizeForMatch(s: string): string;
  *
  * @module components/shortenDocumentId
  */
+/**
+ * Whether `id` is ill-formed UTF-16 (contains at least one unpaired
+ * surrogate — typically the fallout of an upstream pipeline slicing a
+ * string mid-emoji). Exported as the single owner of the detection
+ * semantics so consumers (e.g. core's dev-mode documentId warning) cannot
+ * drift from the branch condition inside `shortenDocumentId`.
+ */
+declare function hasLoneSurrogate(id: string): boolean;
 /**
  * Shorten a `documentId` for use inside an HTML id prefix.
  *
+ * Well-formed ids keep the original contract byte-for-byte: at or below
+ * `threshold` they pass through verbatim; above it they hash via
+ * MurmurHash3 over UTF-8 bytes to a 1–6 char Base62 form.
+ *
+ * Ill-formed UTF-16 ids (unpaired surrogates, e.g. from an upstream
+ * pipeline slicing a string mid-emoji) are ALWAYS hashed, regardless of
+ * length — over their raw UTF-16LE code units, seeded with
+ * `ILL_FORMED_SEED`. Rationale (issue #32):
+ *
+ *   - Returned verbatim, a lone surrogate makes the caller's
+ *     `encodeURIComponent` throw `URIError: URI malformed` synchronously,
+ *     aborting the whole render before any markdown is parsed.
+ *   - Hashing the WHATWG lossy projection (lone surrogates → U+FFFD, what
+ *     `TextEncoder`/`toWellFormed()` do) was rejected: it silently merges
+ *     distinct corrupted ids — and merges them with a well-formed id
+ *     containing a literal U+FFFD — turning the old loud error into a
+ *     silent cross-document anchor collision.
+ *   - Escaping into a well-formed string before hashing was rejected too:
+ *     a naive marker scheme is not injective (marker-lookalike content
+ *     forges it), and an escape-the-escape scheme is strictly more code
+ *     than hashing the raw code units, which is injective for free.
+ *
+ * Distinct ids therefore produce distinct prefixes up to the generic 2^32
+ * birthday bound — the same guarantee long well-formed ids always had —
+ * with no equivalence-class carve-out for corrupted input.
+ *
  * @param id - The raw documentId (consumer-supplied or `useId()` fallback).
- * @param threshold - Only ids strictly longer than this get hashed.
- *   Defaults to `16`, which leaves short hand-picked ids and React's
- *   `useId()` outputs unchanged but catches UUIDs (36 chars) and nanoids.
- * @returns Either the original `id` (when short) or a 1–6 char Base62 hash.
+ *   Any JavaScript string is accepted, including ill-formed UTF-16.
+ * @param threshold - Only well-formed ids strictly longer than this get
+ *   hashed. Defaults to `16`, which leaves short hand-picked ids and
+ *   React's `useId()` outputs unchanged but catches UUIDs (36 chars) and
+ *   nanoids. Ill-formed ids ignore the threshold — they always hash.
+ * @returns The original `id` (well-formed and short) or a 1–6 char Base62
+ *   hash (well-formed long, or ill-formed of any length).
  */
 declare function shortenDocumentId(id: string, threshold?: number): string;
 
@@ -1860,4 +1897,4 @@ type RemendPreprocessorOptions = Omit<RemendOptions, 'katex' | 'inlineKatex'>;
  */
 declare function createRemendPreprocessor(options?: RemendPreprocessorOptions): AIMDContentPreprocessor;
 
-export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, DEF_LINE_START_RE, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, getEnginePluginInternals, highlight, isFootnoteSection, lastRegionStart, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };
+export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, DEF_LINE_START_RE, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, lastRegionStart, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };

package/dist/index.d.ts

@@ -1393,14 +1393,51 @@ declare function normalizeForMatch(s: string): string;
  *
  * @module components/shortenDocumentId
  */
+/**
+ * Whether `id` is ill-formed UTF-16 (contains at least one unpaired
+ * surrogate — typically the fallout of an upstream pipeline slicing a
+ * string mid-emoji). Exported as the single owner of the detection
+ * semantics so consumers (e.g. core's dev-mode documentId warning) cannot
+ * drift from the branch condition inside `shortenDocumentId`.
+ */
+declare function hasLoneSurrogate(id: string): boolean;
 /**
  * Shorten a `documentId` for use inside an HTML id prefix.
  *
+ * Well-formed ids keep the original contract byte-for-byte: at or below
+ * `threshold` they pass through verbatim; above it they hash via
+ * MurmurHash3 over UTF-8 bytes to a 1–6 char Base62 form.
+ *
+ * Ill-formed UTF-16 ids (unpaired surrogates, e.g. from an upstream
+ * pipeline slicing a string mid-emoji) are ALWAYS hashed, regardless of
+ * length — over their raw UTF-16LE code units, seeded with
+ * `ILL_FORMED_SEED`. Rationale (issue #32):
+ *
+ *   - Returned verbatim, a lone surrogate makes the caller's
+ *     `encodeURIComponent` throw `URIError: URI malformed` synchronously,
+ *     aborting the whole render before any markdown is parsed.
+ *   - Hashing the WHATWG lossy projection (lone surrogates → U+FFFD, what
+ *     `TextEncoder`/`toWellFormed()` do) was rejected: it silently merges
+ *     distinct corrupted ids — and merges them with a well-formed id
+ *     containing a literal U+FFFD — turning the old loud error into a
+ *     silent cross-document anchor collision.
+ *   - Escaping into a well-formed string before hashing was rejected too:
+ *     a naive marker scheme is not injective (marker-lookalike content
+ *     forges it), and an escape-the-escape scheme is strictly more code
+ *     than hashing the raw code units, which is injective for free.
+ *
+ * Distinct ids therefore produce distinct prefixes up to the generic 2^32
+ * birthday bound — the same guarantee long well-formed ids always had —
+ * with no equivalence-class carve-out for corrupted input.
+ *
  * @param id - The raw documentId (consumer-supplied or `useId()` fallback).
- * @param threshold - Only ids strictly longer than this get hashed.
- *   Defaults to `16`, which leaves short hand-picked ids and React's
- *   `useId()` outputs unchanged but catches UUIDs (36 chars) and nanoids.
- * @returns Either the original `id` (when short) or a 1–6 char Base62 hash.
+ *   Any JavaScript string is accepted, including ill-formed UTF-16.
+ * @param threshold - Only well-formed ids strictly longer than this get
+ *   hashed. Defaults to `16`, which leaves short hand-picked ids and
+ *   React's `useId()` outputs unchanged but catches UUIDs (36 chars) and
+ *   nanoids. Ill-formed ids ignore the threshold — they always hash.
+ * @returns The original `id` (well-formed and short) or a 1–6 char Base62
+ *   hash (well-formed long, or ill-formed of any length).
  */
 declare function shortenDocumentId(id: string, threshold?: number): string;
 
@@ -1860,4 +1897,4 @@ type RemendPreprocessorOptions = Omit<RemendOptions, 'katex' | 'inlineKatex'>;
  */
 declare function createRemendPreprocessor(options?: RemendPreprocessorOptions): AIMDContentPreprocessor;
 
-export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, DEF_LINE_START_RE, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, getEnginePluginInternals, highlight, isFootnoteSection, lastRegionStart, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };
+export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, DEF_LINE_START_RE, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, lastRegionStart, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };

package/dist/index.dev.cjs

@@ -62,6 +62,7 @@ __export(src_exports, {
   extractContributions: () => extractContributions,
   extractDefBodiesFromHast: () => extractDefBodiesFromHast,
   getEnginePluginInternals: () => getEnginePluginInternals,
+  hasLoneSurrogate: () => hasLoneSurrogate,
   highlight: () => highlight,
   isFootnoteSection: () => isFootnoteSection,
   lastRegionStart: () => lastRegionStart,
@@ -3319,7 +3320,9 @@ var C1 = 3432918353;
 var C2 = 461845907;
 var UTF8_ENCODER = /* @__PURE__ */ new TextEncoder();
 function murmur3_32(s, seed = 0) {
-  const bytes = UTF8_ENCODER.encode(s);
+  return murmur3_32_bytes(UTF8_ENCODER.encode(s), seed);
+}
+function murmur3_32_bytes(bytes, seed = 0) {
   const len = bytes.length;
   let h1 = seed >>> 0;
   const nblocks = len >>> 2;
@@ -3353,7 +3356,22 @@ function murmur3_32(s, seed = 0) {
   h1 ^= h1 >>> 16;
   return h1 >>> 0;
 }
+var LONE_SURROGATE_RE = /[\uD800-\uDFFF]/u;
+function hasLoneSurrogate(id) {
+  return LONE_SURROGATE_RE.test(id);
+}
+var ILL_FORMED_SEED = 1;
+function utf16leBytes(s) {
+  const bytes = new Uint8Array(s.length * 2);
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i);
+    bytes[i * 2] = c & 255;
+    bytes[i * 2 + 1] = c >>> 8;
+  }
+  return bytes;
+}
 function shortenDocumentId(id, threshold = 16) {
+  if (hasLoneSurrogate(id)) return toBase62(murmur3_32_bytes(utf16leBytes(id), ILL_FORMED_SEED));
   if (id.length <= threshold) return id;
   return toBase62(murmur3_32(id));
 }
@@ -4123,6 +4141,7 @@ function createRemendPreprocessor(options) {
   extractContributions,
   extractDefBodiesFromHast,
   getEnginePluginInternals,
+  hasLoneSurrogate,
   highlight,
   isFootnoteSection,
   lastRegionStart,