@ai-matrx/content-ir 0.11.7 → 0.13.0

package/dist/source.d.cts

@@ -0,0 +1,374 @@
+/**
+ * THE OFFSET-PRESERVING SOURCE TOKENIZER (rich-content PLAN decision 6).
+ *
+ * Cuts stored markdown into blocks that cover the text EXACTLY: the
+ * concatenation of every block's `raw` is the input, always, byte for byte.
+ * Nothing is trimmed, normalized, or re-serialized here — the tokenizer only
+ * draws boundaries. Every custom construct the platform stores (kinds and
+ * `__kind` JSON, typed JSON blocks, special fences, XML sections including
+ * attribute-bearing, mid-line and unknown containers, `{{variables}}`,
+ * `<matrxcite/>`, math, raw HTML, code fences, HTML comments and pinned
+ * anchors `<!--@a:…-->`) is an ISLAND: an atomic span no editor may parse
+ * into its own model. Islands that sit inside running prose (a variable, a
+ * citation, inline math) stay inside their paragraph as `inlines`, so a
+ * visual editor keeps the paragraph whole and holds the island as an atom.
+ *
+ * Block kinds:
+ *   - `prose`  — a markdown paragraph run (headings, lists, tables…): the only
+ *                thing a visual editor may re-serialize, and only when edited.
+ *   - `island` — a protected construct; `islandType` says which.
+ *   - `gap`    — the line terminator(s) and blank lines between blocks. Never
+ *                edited; they keep the document's exact spacing.
+ *
+ * Streaming-tolerant: never throws on partial input. A construct whose
+ * closer has not arrived yet becomes an island running to the end of the text
+ * with `complete: false` — the same ownership rule the renderer's splitter
+ * applies, so a half-streamed fence or XML section is never reflowed.
+ *
+ * Boundaries follow matrx-frontend's renderer splitter (`content-splitter-v2.ts`)
+ * wherever it decides where a construct ends, so the editor and the renderer
+ * agree on what is locked. Offsets are UTF-16 (`start`/`end`) and code points
+ * (`startCp`/`endCp`).
+ */
+type SourceBlockKind = "prose" | "island" | "gap";
+/** Island types that own whole blocks. */
+type BlockIslandType = 
+/** ``` / ~~~ fenced code of any language (meta.lang, meta.special). */
+"fence"
+/** A bare JSON object block — typed JSON, `__kind` instances, directives. */
+ | "json"
+/** A known attribute-less XML region (`<thinking>`, `<flashcards>`, …). */
+ | "xml_region"
+/** An attribute-bearing XML element (`<artifact …>`, `<decision …>`, …). */
+ | "xml_attr"
+/** Any other line-leading, balanced XML element. */
+ | "xml_container"
+/** Allow-listed raw HTML block. */
+ | "html_block"
+/** `<!-- … -->` comment. */
+ | "html_comment"
+/** A pinned annotation anchor `<!--@a:id-->`. */
+ | "anchor"
+/** `$$ … $$` display math on its own lines. */
+ | "math_block"
+/** YAML front matter at the start of the text. */
+ | "front_matter"
+/** A box-drawing tree diagram. */
+ | "tree";
+/** Island types that live inside a prose block. */
+type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref";
+type IslandType = BlockIslandType | InlineIslandType;
+type IslandMeta = Readonly<Record<string, string | number | boolean>>;
+interface SourceSpan {
+    /** UTF-16 offset of the first code unit. */
+    readonly start: number;
+    /** UTF-16 offset just past the last code unit. */
+    readonly end: number;
+    /** Code-point offset of `start`. */
+    readonly startCp: number;
+    /** Code-point offset of `end`. */
+    readonly endCp: number;
+    /** Exactly `text.slice(start, end)`. */
+    readonly raw: string;
+}
+interface SourceIsland extends SourceSpan {
+    readonly islandType: IslandType;
+    /** True when the island sits inside a prose block. */
+    readonly inline: boolean;
+    /** False when the construct's closer never arrived (streaming / malformed). */
+    readonly complete: boolean;
+    readonly meta: IslandMeta;
+}
+interface SourceBlock extends SourceSpan {
+    readonly kind: SourceBlockKind;
+    /** Set only on `island` blocks. */
+    readonly islandType: BlockIslandType | null;
+    readonly complete: boolean;
+    readonly meta: IslandMeta;
+    /** Inline islands of a `prose` block, in document order. Empty otherwise. */
+    readonly inlines: readonly SourceIsland[];
+}
+/**
+ * Cut `text` into blocks whose raw concatenation is exactly `text`.
+ * Never throws; unclosed constructs are islands with `complete: false`.
+ */
+declare function tokenizeSource(text: string): SourceBlock[];
+/** The exact source the blocks were cut from. */
+declare function joinSource(blocks: readonly SourceBlock[]): string;
+/** Every island — block-level and inline — in document order. */
+declare function listIslands(blocks: readonly SourceBlock[]): SourceIsland[];
+/** The block containing UTF-16 offset `offset` (the later block at a boundary). */
+declare function blockAt(blocks: readonly SourceBlock[], offset: number): SourceBlock | null;
+
+/**
+ * SAVE = SPLICE (rich-content PLAN decision 6).
+ *
+ * An editor never hands back a whole re-serialized document. It hands back
+ * per-block replacements for exactly the blocks the person changed; every
+ * other byte of the stored text is written back as it was. This module
+ * applies those replacements and returns:
+ *
+ *   - the new text,
+ *   - a change set (old range → new range, in UTF-16 AND code points,
+ *     narrowed to the bytes that actually differ) that the annotation layer
+ *     uses to carry every live anchor through the save
+ *     (STORE-DESIGN §3.19 — the ProseMirror / Google Docs position-mapping
+ *     model), and
+ *   - an integrity guarantee: every island not explicitly replaced must still
+ *     be the same island, at its mapped position, in the new text. An edit
+ *     that opens an unclosed fence and swallows a kind below it is REFUSED.
+ *
+ * ISLANDS CHANGE ONLY THROUGH `islandEdit`. A prose/range edit must hand back
+ * every island inside its range byte-identical and in order (a visual editor
+ * holds them as atoms); if one is altered or dropped the save throws
+ * `SourceSpliceError("island_edit")` naming the island. A kind editor or code
+ * view changes an island with `islandEdit(island, newRaw)`.
+ *
+ * A no-edit save (or edits whose text equals the original block) returns the
+ * original string itself — byte-identical, zero changes.
+ */
+
+/**
+ * Replace `[start, end)` (UTF-16) of the original with `text`. A plain edit
+ * starts and ends on block boundaries; an island edit (`island: true`, made by
+ * `islandEdit`) names exactly one island's span.
+ */
+interface SourceEdit {
+    readonly start: number;
+    readonly end: number;
+    readonly text: string;
+    readonly island?: boolean;
+}
+interface SourceChange {
+    /** The block-aligned range the edit named, in the ORIGINAL text. */
+    readonly blockStart: number;
+    readonly blockEnd: number;
+    /** The narrowed range that actually differs: old text … */
+    readonly oldStart: number;
+    readonly oldEnd: number;
+    /** … and where its replacement sits in the NEW text. */
+    readonly newStart: number;
+    readonly newEnd: number;
+    readonly oldStartCp: number;
+    readonly oldEndCp: number;
+    readonly newStartCp: number;
+    readonly newEndCp: number;
+}
+interface DisturbedIsland {
+    readonly islandType: IslandType;
+    readonly inline: boolean;
+    /** Where the island was in the original (UTF-16). */
+    readonly start: number;
+    readonly end: number;
+    /** Where it should have been in the new text. */
+    readonly mappedStart: number;
+}
+interface SpliceIntegrity {
+    /** True when every untouched island survived intact and outside bytes are identical. */
+    readonly ok: boolean;
+    /** Always true by construction; checked anyway because it is THE guarantee. */
+    readonly bytesOutsideEditsIdentical: boolean;
+    readonly disturbed: readonly DisturbedIsland[];
+}
+interface SpliceResult {
+    readonly text: string;
+    readonly changed: boolean;
+    readonly changes: readonly SourceChange[];
+    readonly integrity: SpliceIntegrity;
+    /** Tokenization of the new text. */
+    readonly blocks: readonly SourceBlock[];
+}
+type SourceSpliceErrorCode = "out_of_range" | "misaligned" | "overlap" | "stale_blocks" | "island_edit" | "integrity";
+declare class SourceSpliceError extends Error {
+    readonly code: SourceSpliceErrorCode;
+    constructor(code: SourceSpliceErrorCode, message: string);
+}
+interface SpliceOptions {
+    /** The tokenization of `original`, when the caller already holds it. */
+    readonly blocks?: readonly SourceBlock[];
+    /**
+     * Default `true`: a save that disturbs any island it did not explicitly
+     * replace throws `SourceSpliceError("integrity")`. `false` is for TOOLING
+     * ONLY (diagnostic scripts that want the report instead of the refusal) —
+     * no editor or persistence path may pass it.
+     */
+    readonly requireIntegrity?: boolean;
+}
+/** An edit that replaces one block's raw text. */
+declare function blockEdit(block: SourceBlock, text: string): SourceEdit;
+/** The one door that changes an island: replace exactly its span with `text`. */
+declare function islandEdit(island: {
+    readonly start: number;
+    readonly end: number;
+}, text: string): SourceEdit;
+/** An edit that replaces a contiguous run of blocks `[first, last]`. */
+declare function blockRangeEdit(first: SourceBlock, last: SourceBlock, text: string): SourceEdit;
+/**
+ * Apply block-aligned replacements to `original`. Bytes outside the edits are
+ * copied, never rebuilt; an edit equal to its block changes nothing.
+ */
+declare function spliceSave(original: string, edits: readonly SourceEdit[], options?: SpliceOptions): SpliceResult;
+interface MapOptions {
+    /** -1 keeps a position at an insertion point before the inserted text; 1 (default) moves it after. */
+    readonly assoc?: -1 | 1;
+    /** Offset unit of `pos` and of the result. Default `utf16`. */
+    readonly unit?: "utf16" | "codepoint";
+}
+interface MappedPosition {
+    readonly pos: number;
+    /** True when the position sat strictly inside replaced text. */
+    readonly deleted: boolean;
+}
+/**
+ * Carry a position in the old text through a change set (ProseMirror StepMap
+ * semantics): positions before a change stay, positions after shift by the
+ * size difference, a position strictly inside replaced text is `deleted` and
+ * lands on the replacement's edge chosen by `assoc`.
+ */
+declare function mapPosition(changes: readonly SourceChange[], pos: number, options?: MapOptions): MappedPosition;
+interface MappedRange {
+    readonly start: number;
+    readonly end: number;
+    /** True when a change overlapped the range's interior. */
+    readonly touched: boolean;
+    /** True when the whole range was replaced away. */
+    readonly collapsed: boolean;
+}
+/** Carry an anchor range through a change set; it stays inside its text when edits happen around it. */
+declare function mapRange(changes: readonly SourceChange[], start: number, end: number, unit?: "utf16" | "codepoint"): MappedRange;
+
+/**
+ * UTF-16 ⇄ code-point offset conversion.
+ *
+ * JavaScript strings are indexed in UTF-16 code units; Postgres `position()`,
+ * `substring()` and `length()` count code points. Every offset this module
+ * emits is given in BOTH units so an anchor captured on either side of the
+ * wire can be mapped without re-deriving the text.
+ */
+/** Indexes the surrogate pairs of one text so conversion is O(log pairs). */
+interface CodePointIndex {
+    /** Code-point offset of a UTF-16 offset. */
+    toCodePoint(utf16: number): number;
+    /** UTF-16 offset of a code-point offset. */
+    toUtf16(codePoint: number): number;
+    /** Code-point length of the text. */
+    readonly lengthCp: number;
+}
+/** True when `offset` sits between the two halves of a surrogate pair. */
+declare function splitsSurrogatePair(text: string, offset: number): boolean;
+declare function buildCodePointIndex(text: string): CodePointIndex;
+/** One-shot conversion; build an index when converting many offsets. */
+declare function toCodePointOffset(text: string, utf16: number): number;
+/** One-shot conversion; build an index when converting many offsets. */
+declare function toUtf16Offset(text: string, codePoint: number): number;
+
+/**
+ * THE single-dollar math rule — the one predicate that decides whether
+ * `$…$` is inline math or currency.
+ *
+ * Moved here from matrx-frontend `components/markdown-core/math-normalizer.ts`
+ * (2026-09-23) so the renderer's normalizer and the source tokenizer can never
+ * disagree about which bytes are math: the normalizer imports this function.
+ *
+ * Pandoc's rule — the opening `$` is not followed by whitespace, the closing
+ * `$` is not preceded by whitespace and not followed by a digit — plus a math
+ * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), or a
+ * lone variable (`$x$`, `$n$`, `$x'$`). Content that opens with a digit needs
+ * a real TeX signal, so a price range can never read as math.
+ */
+declare function isSingleDollarMath(content: string, after: string | undefined): boolean;
+/**
+ * Where a single-dollar math span opened at `open` (a lone `$`) closes, or -1.
+ * Same scan the normalizer uses: same line, backslash-escapes skipped, the
+ * closer is a `$` not followed by another `$`, and the content passes
+ * {@link isSingleDollarMath}. Returns the offset just past the closing `$`.
+ */
+declare function singleDollarMathEnd(text: string, open: number, limit?: number): number;
+
+/**
+ * XML tag reading + the generic-container balance scanner.
+ *
+ * Ported VERBATIM (behaviour) from matrx-frontend
+ * `components/mardown-display/blocks/xml/readXmlTag.ts` and the
+ * `XmlContainerTracker` in
+ * `components/mardown-display/markdown-classification/processors/utils/content-splitter-v2.ts`
+ * so the source tokenizer ends a generic XML section exactly where the
+ * renderer does. The renderer copies stay for rendering (PLAN decision 6);
+ * a divergence between the two is a defect in whichever changed last.
+ */
+interface ReadXmlTag {
+    tagName: string;
+    raw: string;
+    isClosing: boolean;
+    isSelfClosing: boolean;
+    attributes: Array<{
+        name: string;
+        value: string;
+    }>;
+}
+/**
+ * `find(needle, from)` must behave like `content.indexOf(needle, from)`; the
+ * tokenizer passes a cached occurrence index so a text full of unclosed
+ * attribute quotes stays linear. Behaviour is identical either way.
+ */
+declare function readXmlTag(content: string, start: number, find?: (needle: string, from: number) => number): ReadXmlTag | null;
+/**
+ * Balances ONE unknown root tag line by line, ignoring lookalike tags inside
+ * fenced code, inline code spans, comments and CDATA. `consumeLine` returns
+ * the offset (within that line) just past the root's closing tag, or null.
+ */
+declare class XmlContainerTracker {
+    readonly rootTag: string;
+    private depth;
+    private inComment;
+    private inCdata;
+    private inlineTicks;
+    private fenceMarker;
+    constructor(rootTag: string);
+    consumeLine(line: string, startOffset?: number): number | null;
+}
+
+/** Fence languages whose body is itself a markdown document. */
+declare const NESTING_FENCE_LANGUAGES: ReadonlySet<string>;
+/** True when a fence opened with this info-string language nests inner fences. */
+declare function fenceNestsInnerFences(language: string | undefined | null): boolean;
+type InnerFenceLine = 
+/** Not a fence line — ordinary content of the outer fence. */
+"content"
+/** ```lang inside a markdown fence — content, and one level deeper. */
+ | "open-nested"
+/** Bare ``` that closes the innermost nested fence — content, one level up. */
+ | "close-nested"
+/** Bare ``` that closes the OUTER fence. */
+ | "close-outer";
+/**
+ * Classify one TRIMMED line inside an open backtick fence.
+ *
+ * @param trimmed     the line with surrounding whitespace removed
+ * @param openTicks   backtick count of the outer fence's opener
+ * @param nests       whether the outer fence follows the nesting rule
+ * @param nestedDepth how many nested fences are open right now
+ */
+declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
+
+/**
+ * `$$` pairing — the core's rule for which `$$` delimiters are real math.
+ *
+ * Ported from `@ai-matrx/kit/delimiter-guard` (`guardMathDelimiters`), which
+ * every math-capable renderer runs before remark-math. Its decisions:
+ *   - `$$` tokens outside code (``` / ~~~ fences, `inline code`) pair in
+ *     document order, open → close;
+ *   - a pair whose inside reads as prose or carries markdown structure is NOT
+ *     math: the renderer neutralizes its opener and the closer is retried as
+ *     the next opener;
+ *   - an unpaired `$$` is inert — it renders as literal text.
+ * So the tokenizer protects exactly the pairs the renderer shows as math, and
+ * a lone `$$` (a stray closer, or an opener that never closed) never locks
+ * anything. Linear time: one regex pass per code shape, one sweep.
+ */
+/** The guard's `looksLikeMath`, verbatim. */
+declare function looksLikeDisplayMath(inner: string): boolean;
+/** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
+declare function pairDisplayMath(text: string): Map<number, number>;
+
+export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };