@ai-react-markdown/engine 2.7.0 → 2.8.1

package/dist/index.d.cts

@@ -88,13 +88,6 @@ interface Deprecation {
  * `.parse()` and `.runSync()` (or `.run()`) on it.
  */
 declare function createProcessor(options: Readonly<PipelineOptions>): Processor<Root, Root, Root$1, undefined, undefined>;
-/**
- * Wrap the markdown string in a VFile so plugins that consume `file.value`
- * work. Mirrors react-markdown: in dev `unreachable` throws an AssertionError
- * for non-string `children`; in prod it silently no-ops, leaving `file.value`
- * undefined and unified treating the input as empty.
- */
-declare function createFile(options: Readonly<PipelineOptions>): VFile;
 
 /**
  * The pure pipeline stages, lifted verbatim from core's Markdown.tsx
@@ -227,7 +220,7 @@ declare const defaultUrlTransform: UrlTransform;
  *    rest of that line as raw content too (`<!-- c --> tail`) — same seam.
  * 7. **Phase poison** (`phasePoisonedAt`) — points where this line-level
  *    model may have DIVERGED from micromark and provably cannot resync:
- *    a fence/math open suppressed by `mayBeRawToMicromark` (only certainly
+ *    a fence/math open suppressed inside an html-flow run (only certainly
  *    swallowed at top level — in a container it really opens and the
  *    open/close phase inverts permanently), and a paragraph-inline `<!--`
  *    that fails to close by end of line (literal text to micromark, but
@@ -533,7 +526,8 @@ declare function advanceIncrementalParse(prev: IncrementalParseState | null, con
 declare function attributeHastChildren(mdast: Root, hast: Root$1, stopAt?: number): number[];
 
 /**
- * TEST/STORY helper (not exported from the package barrel): prefix
+ * TEST/STORY helper (exported from the package barrel for the Storybook
+ * streams): prefix
  * snapshots sliced at CODE-POINT granularity, so a frame boundary never
  * splits a surrogate pair. Every streaming verifier in the repo — the
  * splice-equivalence arbiter, the prefixFreeze experiment harness, and the
@@ -1296,13 +1290,6 @@ declare const defaultEnginePlugins: readonly AIMarkdownEnginePlugin[];
  * `tagName === 'section'` AND presence of the `dataFootnotes` property.
  */
 declare function isFootnoteSection(node: Element): boolean;
-/**
- * Whitespace-only text node. `mdast-util-to-hast`'s `state.wrap(content,
- * true)` interleaves `\n` text nodes between (and around) the block-level
- * children of an `<li>`, so any code reasoning about a list item's real
- * children has to look past them.
- */
-declare function isWhitespaceText(c: ElementContent): boolean;
 /**
  * Index of the last child that is not a whitespace-only text node, or -1 if
  * there is none.
@@ -1500,24 +1487,6 @@ declare function measureStage<T>(stage: PipelineStage, fn: () => T, instanceId?:
  */
 
 type Schema = typeof defaultSchema;
-type AttributeEntry = NonNullable<NonNullable<Schema['attributes']>[string]>[number];
-/**
- * Extend the allowlist for a tag's `className` attribute with extra class
- * names while preserving all other default entries.
- *
- * `findDefinition` in hast-util-sanitize returns the *first* matching entry
- * for a given property name, so appending a second `className` entry would be
- * ignored. Instead, merge the allowed values into the existing entry.
- *
- * Edge cases:
- * - `existing` is `undefined` → returns a single new `['className', ...extra]`
- * - `existing` has no `className` entry → appends one with just the extras
- * - `existing` has a bare-string `'className'` entry (hast-util-sanitize's
- *   "allow all values" form) → would be narrowed to an allow-list. This is a
- *   semantics change, but the current `defaultSchema.attributes.code` entry
- *   is always tuple-form, so this branch is defensive only.
- */
-declare function mergeClassNameAllowlist(existing: ReadonlyArray<AttributeEntry> | undefined, extraClassNames: readonly string[]): AttributeEntry[];
 declare const sanitizeSchema: Schema;
 
 /**
@@ -1728,49 +1697,6 @@ type AIMDContentPreprocessor = (content: string) => string;
  */
 declare function preprocessAIMDContent(content: string, extraPreprocessors?: AIMDContentPreprocessor[], latexPreprocessor?: AIMDContentPreprocessor): string;
 
-/**
- * LaTeX preprocessing pipeline.
- *
- * Normalizes raw markdown so that LaTeX expressions survive the remark/rehype
- * rendering pipeline intact. The main entry point is {@link preprocessLaTeX},
- * which splits content into protected regions (code blocks, inline code, HTML
- * tags) and applies a sequence of transformations to the unprotected text:
- *
- * 1. Escape mhchem commands (`\ce`, `\pu`)
- * 2. Escape currency dollar signs (e.g. `$100`, `$1,000.50`)
- * 3. Convert bracket delimiters (`\[...\]`, `\(...\)`) to dollar delimiters
- * 4. Escape pipes inside closed LaTeX blocks to prevent GFM table interference
- * 5. Escape pipes inside unclosed LaTeX blocks (streaming partial content)
- * 6. Escape underscores inside `\text{...}` commands
- * 7. Convert single-dollar delimiters to double-dollar delimiters
- * 8. Truncate trailing unclosed LaTeX blocks (streaming protection)
- *
- * Thanks to the implementations from the following repositories:
- * - https://github.com/lobehub/lobe-ui/blob/master/src/hooks/useMarkdown/latex.ts
- * - https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts
- *
- * @module preprocessors/latex
- */
-interface Segment {
-    text: string;
-    isCode: boolean;
-}
-/**
- * Split content into alternating text and protected segments.
- * Protected segments (isCode: true) are excluded from LaTeX processing:
- * - fenced multiline code blocks: 3+ backticks or tildes at the *start of a
- *   line* (any indentation — container-relative limits are not modelled).
- *   Mid-line runs are never fence openers. INDENTED code blocks (4+ spaces
- *   after a blank line, outside any container) are NOT modelled: without a
- *   container model they cannot be told from a list item's continuation
- *   paragraph, and protecting them would silence math in nested lists.
- *   Known limitation — `$` inside an indented code block may be rewritten.
- * - inline code spans: a run of N backticks closed by another run of exactly
- *   N backticks. May span newlines. Multi-backtick forms (e.g. `` `` `x` ``)
- *   are supported so literal backtick characters can appear inside.
- * - HTML tags (e.g. `<span>$</span>` where `$` should not be treated as LaTeX).
- */
-declare function splitByProtectedRegions(content: string): Segment[];
 /**
  * Main LaTeX preprocessor entry point.
  *
@@ -1879,4 +1805,4 @@ type RemendPreprocessorOptions = Omit<RemendOptions, 'katex' | 'inlineKatex'>;
  */
 declare function createRemendPreprocessor(options?: RemendPreprocessorOptions): AIMDContentPreprocessor;
 
-export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, footnoteSafeId, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, isWhitespaceText, lastMeaningfulIdx, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, phantomSuffixCloser, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };
+export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, footnoteSafeId, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, lastMeaningfulIdx, measureStage, normalizeForMatch, normalizeId, pangu, parseStage, phantomSuffixCloser, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, subscribeStageTimings, transformStage, withDefs };

package/dist/index.d.ts

@@ -88,13 +88,6 @@ interface Deprecation {
  * `.parse()` and `.runSync()` (or `.run()`) on it.
  */
 declare function createProcessor(options: Readonly<PipelineOptions>): Processor<Root, Root, Root$1, undefined, undefined>;
-/**
- * Wrap the markdown string in a VFile so plugins that consume `file.value`
- * work. Mirrors react-markdown: in dev `unreachable` throws an AssertionError
- * for non-string `children`; in prod it silently no-ops, leaving `file.value`
- * undefined and unified treating the input as empty.
- */
-declare function createFile(options: Readonly<PipelineOptions>): VFile;
 
 /**
  * The pure pipeline stages, lifted verbatim from core's Markdown.tsx
@@ -227,7 +220,7 @@ declare const defaultUrlTransform: UrlTransform;
  *    rest of that line as raw content too (`<!-- c --> tail`) — same seam.
  * 7. **Phase poison** (`phasePoisonedAt`) — points where this line-level
  *    model may have DIVERGED from micromark and provably cannot resync:
- *    a fence/math open suppressed by `mayBeRawToMicromark` (only certainly
+ *    a fence/math open suppressed inside an html-flow run (only certainly
  *    swallowed at top level — in a container it really opens and the
  *    open/close phase inverts permanently), and a paragraph-inline `<!--`
  *    that fails to close by end of line (literal text to micromark, but
@@ -533,7 +526,8 @@ declare function advanceIncrementalParse(prev: IncrementalParseState | null, con
 declare function attributeHastChildren(mdast: Root, hast: Root$1, stopAt?: number): number[];
 
 /**
- * TEST/STORY helper (not exported from the package barrel): prefix
+ * TEST/STORY helper (exported from the package barrel for the Storybook
+ * streams): prefix
  * snapshots sliced at CODE-POINT granularity, so a frame boundary never
  * splits a surrogate pair. Every streaming verifier in the repo — the
  * splice-equivalence arbiter, the prefixFreeze experiment harness, and the
@@ -1296,13 +1290,6 @@ declare const defaultEnginePlugins: readonly AIMarkdownEnginePlugin[];
  * `tagName === 'section'` AND presence of the `dataFootnotes` property.
  */
 declare function isFootnoteSection(node: Element): boolean;
-/**
- * Whitespace-only text node. `mdast-util-to-hast`'s `state.wrap(content,
- * true)` interleaves `\n` text nodes between (and around) the block-level
- * children of an `<li>`, so any code reasoning about a list item's real
- * children has to look past them.
- */
-declare function isWhitespaceText(c: ElementContent): boolean;
 /**
  * Index of the last child that is not a whitespace-only text node, or -1 if
  * there is none.
@@ -1500,24 +1487,6 @@ declare function measureStage<T>(stage: PipelineStage, fn: () => T, instanceId?:
  */
 
 type Schema = typeof defaultSchema;
-type AttributeEntry = NonNullable<NonNullable<Schema['attributes']>[string]>[number];
-/**
- * Extend the allowlist for a tag's `className` attribute with extra class
- * names while preserving all other default entries.
- *
- * `findDefinition` in hast-util-sanitize returns the *first* matching entry
- * for a given property name, so appending a second `className` entry would be
- * ignored. Instead, merge the allowed values into the existing entry.
- *
- * Edge cases:
- * - `existing` is `undefined` → returns a single new `['className', ...extra]`
- * - `existing` has no `className` entry → appends one with just the extras
- * - `existing` has a bare-string `'className'` entry (hast-util-sanitize's
- *   "allow all values" form) → would be narrowed to an allow-list. This is a
- *   semantics change, but the current `defaultSchema.attributes.code` entry
- *   is always tuple-form, so this branch is defensive only.
- */
-declare function mergeClassNameAllowlist(existing: ReadonlyArray<AttributeEntry> | undefined, extraClassNames: readonly string[]): AttributeEntry[];
 declare const sanitizeSchema: Schema;
 
 /**
@@ -1728,49 +1697,6 @@ type AIMDContentPreprocessor = (content: string) => string;
  */
 declare function preprocessAIMDContent(content: string, extraPreprocessors?: AIMDContentPreprocessor[], latexPreprocessor?: AIMDContentPreprocessor): string;
 
-/**
- * LaTeX preprocessing pipeline.
- *
- * Normalizes raw markdown so that LaTeX expressions survive the remark/rehype
- * rendering pipeline intact. The main entry point is {@link preprocessLaTeX},
- * which splits content into protected regions (code blocks, inline code, HTML
- * tags) and applies a sequence of transformations to the unprotected text:
- *
- * 1. Escape mhchem commands (`\ce`, `\pu`)
- * 2. Escape currency dollar signs (e.g. `$100`, `$1,000.50`)
- * 3. Convert bracket delimiters (`\[...\]`, `\(...\)`) to dollar delimiters
- * 4. Escape pipes inside closed LaTeX blocks to prevent GFM table interference
- * 5. Escape pipes inside unclosed LaTeX blocks (streaming partial content)
- * 6. Escape underscores inside `\text{...}` commands
- * 7. Convert single-dollar delimiters to double-dollar delimiters
- * 8. Truncate trailing unclosed LaTeX blocks (streaming protection)
- *
- * Thanks to the implementations from the following repositories:
- * - https://github.com/lobehub/lobe-ui/blob/master/src/hooks/useMarkdown/latex.ts
- * - https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts
- *
- * @module preprocessors/latex
- */
-interface Segment {
-    text: string;
-    isCode: boolean;
-}
-/**
- * Split content into alternating text and protected segments.
- * Protected segments (isCode: true) are excluded from LaTeX processing:
- * - fenced multiline code blocks: 3+ backticks or tildes at the *start of a
- *   line* (any indentation — container-relative limits are not modelled).
- *   Mid-line runs are never fence openers. INDENTED code blocks (4+ spaces
- *   after a blank line, outside any container) are NOT modelled: without a
- *   container model they cannot be told from a list item's continuation
- *   paragraph, and protecting them would silence math in nested lists.
- *   Known limitation — `$` inside an indented code block may be rewritten.
- * - inline code spans: a run of N backticks closed by another run of exactly
- *   N backticks. May span newlines. Multi-backtick forms (e.g. `` `` `x` ``)
- *   are supported so literal backtick characters can appear inside.
- * - HTML tags (e.g. `<span>$</span>` where `$` should not be treated as LaTeX).
- */
-declare function splitByProtectedRegions(content: string): Segment[];
 /**
  * Main LaTeX preprocessor entry point.
  *
@@ -1879,4 +1805,4 @@ type RemendPreprocessorOptions = Omit<RemendOptions, 'katex' | 'inlineKatex'>;
  */
 declare function createRemendPreprocessor(options?: RemendPreprocessorOptions): AIMDContentPreprocessor;
 
-export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createFile, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, footnoteSafeId, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, isWhitespaceText, lastMeaningfulIdx, measureStage, mergeClassNameAllowlist, normalizeForMatch, normalizeId, pangu, parseStage, phantomSuffixCloser, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, splitByProtectedRegions, subscribeStageTimings, transformStage, withDefs };
+export { type AIMDContentPreprocessor, type AIMarkdownEnginePlugin, type AIMarkdownEnginePluginName, type AdvanceOptions, type AdvanceResult, type AllowElement, type ChunkData, type Contribution, type CrossChunkHandlerOptions, DEFAULT_PAYLOAD, type DefLabelScanner, type DefLabels, type Deprecation, type EnginePluginInternals, type EnginePluginStage, type ExtractContributionsOptions, type FootnoteDef, type FreezeBoundaryOptions, type IncrementalParseState, type IncrementalStage, type LinkDef, PIPELINE_STAGES, type ParsedMarkdown, type PhantomLabels, type PipelineOptions, type PipelineStage, type RefKind, type RefRecord, type Registry, type RegistryInternal, type RehypePlugins, type RehypeRebaseHashLinksOptions, type RemarkPlugins, type RemarkRehypeOptions, type RemendPreprocessorOptions, SENTINEL_FN_CONTENT, SENTINEL_LINK_URL, SMOOTH_STREAM_PACING_PRESETS, STAGE_MEASURE_PREFIX, type SanitizeSchema, type SmoothStreamController, type SmoothStreamOptions, type SmoothStreamPacing, type SmoothStreamPacingParams, type TransformContext, type UrlAttrKey, type UrlAttrTag, type UrlTransform, advanceIncrementalParse, attributeHastChildren, buildCoreRehypePlugins, buildCoreRemarkPlugins, buildCoreRemarkRehypeOptions, buildCrossChunkHandlers, buildPhantomSuffix, buildTransform, codePointSnapshots, collectDefLabels, computeFreezeBoundary, createDefLabelScanner, createIncrementalLatexPreprocessor, createProcessor, createRegistry, createRemendPreprocessor, createSmoothStreamController, defaultEnginePlugins, defaultUrlTransform, definitionList, extendSanitizeSchema, extractContributions, extractDefBodiesFromHast, footnoteSafeId, getEnginePluginInternals, hasLoneSurrogate, highlight, isFootnoteSection, lastMeaningfulIdx, measureStage, normalizeForMatch, normalizeId, pangu, parseStage, phantomSuffixCloser, preprocessAIMDContent, preprocessLaTeX, rehypeFooterAdorn, rehypeRebaseHashLinks, removeComments, sanitizeCrossChunkUrl, sanitizeSchema, shortenDocumentId, smartypants, sourceIdFromFootnoteLiId, subscribeStageTimings, transformStage, withDefs };