@ai-matrx/content-ir 0.11.6 → 0.12.0

package/dist/source.d.ts

@@ -0,0 +1,306 @@
+/**
+ * THE OFFSET-PRESERVING SOURCE TOKENIZER (rich-content PLAN decision 6).
+ *
+ * Cuts stored markdown into blocks that cover the text EXACTLY: the
+ * concatenation of every block's `raw` is the input, always, byte for byte.
+ * Nothing is trimmed, normalized, or re-serialized here — the tokenizer only
+ * draws boundaries. Every custom construct the platform stores (kinds and
+ * `__kind` JSON, typed JSON blocks, special fences, XML sections including
+ * attribute-bearing, mid-line and unknown containers, `{{variables}}`,
+ * `<matrxcite/>`, math, raw HTML, code fences, HTML comments and pinned
+ * anchors `<!--@a:…-->`) is an ISLAND: an atomic span no editor may parse
+ * into its own model. Islands that sit inside running prose (a variable, a
+ * citation, inline math) stay inside their paragraph as `inlines`, so a
+ * visual editor keeps the paragraph whole and holds the island as an atom.
+ *
+ * Block kinds:
+ *   - `prose`  — a markdown paragraph run (headings, lists, tables…): the only
+ *                thing a visual editor may re-serialize, and only when edited.
+ *   - `island` — a protected construct; `islandType` says which.
+ *   - `gap`    — the line terminator(s) and blank lines between blocks. Never
+ *                edited; they keep the document's exact spacing.
+ *
+ * Streaming-tolerant: never throws on partial input. A construct whose
+ * closer has not arrived yet becomes an island running to the end of the text
+ * with `complete: false` — the same ownership rule the renderer's splitter
+ * applies, so a half-streamed fence or XML section is never reflowed.
+ *
+ * Boundaries follow matrx-frontend's renderer splitter (`content-splitter-v2.ts`)
+ * wherever it decides where a construct ends, so the editor and the renderer
+ * agree on what is locked. Offsets are UTF-16 (`start`/`end`) and code points
+ * (`startCp`/`endCp`).
+ */
+type SourceBlockKind = "prose" | "island" | "gap";
+/** Island types that own whole blocks. */
+type BlockIslandType = 
+/** ``` / ~~~ fenced code of any language (meta.lang, meta.special). */
+"fence"
+/** A bare JSON object block — typed JSON, `__kind` instances, directives. */
+ | "json"
+/** A known attribute-less XML region (`<thinking>`, `<flashcards>`, …). */
+ | "xml_region"
+/** An attribute-bearing XML element (`<artifact …>`, `<decision …>`, …). */
+ | "xml_attr"
+/** Any other line-leading, balanced XML element. */
+ | "xml_container"
+/** Allow-listed raw HTML block. */
+ | "html_block"
+/** `<!-- … -->` comment. */
+ | "html_comment"
+/** A pinned annotation anchor `<!--@a:id-->`. */
+ | "anchor"
+/** `$$ … $$` display math on its own lines. */
+ | "math_block"
+/** YAML front matter at the start of the text. */
+ | "front_matter"
+/** A box-drawing tree diagram. */
+ | "tree";
+/** Island types that live inside a prose block. */
+type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref";
+type IslandType = BlockIslandType | InlineIslandType;
+type IslandMeta = Readonly<Record<string, string | number | boolean>>;
+interface SourceSpan {
+    /** UTF-16 offset of the first code unit. */
+    readonly start: number;
+    /** UTF-16 offset just past the last code unit. */
+    readonly end: number;
+    /** Code-point offset of `start`. */
+    readonly startCp: number;
+    /** Code-point offset of `end`. */
+    readonly endCp: number;
+    /** Exactly `text.slice(start, end)`. */
+    readonly raw: string;
+}
+interface SourceIsland extends SourceSpan {
+    readonly islandType: IslandType;
+    /** True when the island sits inside a prose block. */
+    readonly inline: boolean;
+    /** False when the construct's closer never arrived (streaming / malformed). */
+    readonly complete: boolean;
+    readonly meta: IslandMeta;
+}
+interface SourceBlock extends SourceSpan {
+    readonly kind: SourceBlockKind;
+    /** Set only on `island` blocks. */
+    readonly islandType: BlockIslandType | null;
+    readonly complete: boolean;
+    readonly meta: IslandMeta;
+    /** Inline islands of a `prose` block, in document order. Empty otherwise. */
+    readonly inlines: readonly SourceIsland[];
+}
+/**
+ * Cut `text` into blocks whose raw concatenation is exactly `text`.
+ * Never throws; unclosed constructs are islands with `complete: false`.
+ */
+declare function tokenizeSource(text: string): SourceBlock[];
+/** The exact source the blocks were cut from. */
+declare function joinSource(blocks: readonly SourceBlock[]): string;
+/** Every island — block-level and inline — in document order. */
+declare function listIslands(blocks: readonly SourceBlock[]): SourceIsland[];
+/** The block containing UTF-16 offset `offset` (the later block at a boundary). */
+declare function blockAt(blocks: readonly SourceBlock[], offset: number): SourceBlock | null;
+
+/**
+ * SAVE = SPLICE (rich-content PLAN decision 6).
+ *
+ * An editor never hands back a whole re-serialized document. It hands back
+ * per-block replacements for exactly the blocks the person changed; every
+ * other byte of the stored text is written back as it was. This module
+ * applies those replacements and returns:
+ *
+ *   - the new text,
+ *   - a change set (old range → new range, in UTF-16 AND code points,
+ *     narrowed to the bytes that actually differ) that the annotation layer
+ *     uses to carry every live anchor through the save
+ *     (STORE-DESIGN §3.19 — the ProseMirror / Google Docs position-mapping
+ *     model), and
+ *   - an integrity report: every island outside the edited ranges must still
+ *     be the same island, at its mapped position, in the new text. An edit
+ *     that opens an unclosed fence and swallows a kind below it is caught
+ *     here instead of being saved silently.
+ *
+ * A no-edit save (or edits whose text equals the original block) returns the
+ * original string itself — byte-identical, zero changes.
+ */
+
+/** Replace `[start, end)` (UTF-16, on block boundaries) of the original with `text`. */
+interface SourceEdit {
+    readonly start: number;
+    readonly end: number;
+    readonly text: string;
+}
+interface SourceChange {
+    /** The block-aligned range the edit named, in the ORIGINAL text. */
+    readonly blockStart: number;
+    readonly blockEnd: number;
+    /** The narrowed range that actually differs: old text … */
+    readonly oldStart: number;
+    readonly oldEnd: number;
+    /** … and where its replacement sits in the NEW text. */
+    readonly newStart: number;
+    readonly newEnd: number;
+    readonly oldStartCp: number;
+    readonly oldEndCp: number;
+    readonly newStartCp: number;
+    readonly newEndCp: number;
+}
+interface DisturbedIsland {
+    readonly islandType: IslandType;
+    readonly inline: boolean;
+    /** Where the island was in the original (UTF-16). */
+    readonly start: number;
+    readonly end: number;
+    /** Where it should have been in the new text. */
+    readonly mappedStart: number;
+}
+interface SpliceIntegrity {
+    /** True when every untouched island survived intact and outside bytes are identical. */
+    readonly ok: boolean;
+    /** Always true by construction; checked anyway because it is THE guarantee. */
+    readonly bytesOutsideEditsIdentical: boolean;
+    readonly disturbed: readonly DisturbedIsland[];
+}
+interface SpliceResult {
+    readonly text: string;
+    readonly changed: boolean;
+    readonly changes: readonly SourceChange[];
+    readonly integrity: SpliceIntegrity;
+    /** Tokenization of the new text. */
+    readonly blocks: readonly SourceBlock[];
+}
+type SourceSpliceErrorCode = "out_of_range" | "misaligned" | "overlap" | "stale_blocks" | "integrity";
+declare class SourceSpliceError extends Error {
+    readonly code: SourceSpliceErrorCode;
+    constructor(code: SourceSpliceErrorCode, message: string);
+}
+interface SpliceOptions {
+    /** The tokenization of `original`, when the caller already holds it. */
+    readonly blocks?: readonly SourceBlock[];
+    /** Throw `SourceSpliceError("integrity")` instead of reporting a disturbed island. */
+    readonly requireIntegrity?: boolean;
+}
+/** An edit that replaces one block's raw text. */
+declare function blockEdit(block: SourceBlock, text: string): SourceEdit;
+/** An edit that replaces a contiguous run of blocks `[first, last]`. */
+declare function blockRangeEdit(first: SourceBlock, last: SourceBlock, text: string): SourceEdit;
+/**
+ * Apply block-aligned replacements to `original`. Bytes outside the edits are
+ * copied, never rebuilt; an edit equal to its block changes nothing.
+ */
+declare function spliceSave(original: string, edits: readonly SourceEdit[], options?: SpliceOptions): SpliceResult;
+interface MapOptions {
+    /** -1 keeps a position at an insertion point before the inserted text; 1 (default) moves it after. */
+    readonly assoc?: -1 | 1;
+    /** Offset unit of `pos` and of the result. Default `utf16`. */
+    readonly unit?: "utf16" | "codepoint";
+}
+interface MappedPosition {
+    readonly pos: number;
+    /** True when the position sat strictly inside replaced text. */
+    readonly deleted: boolean;
+}
+/**
+ * Carry a position in the old text through a change set (ProseMirror StepMap
+ * semantics): positions before a change stay, positions after shift by the
+ * size difference, a position strictly inside replaced text is `deleted` and
+ * lands on the replacement's edge chosen by `assoc`.
+ */
+declare function mapPosition(changes: readonly SourceChange[], pos: number, options?: MapOptions): MappedPosition;
+interface MappedRange {
+    readonly start: number;
+    readonly end: number;
+    /** True when a change overlapped the range's interior. */
+    readonly touched: boolean;
+    /** True when the whole range was replaced away. */
+    readonly collapsed: boolean;
+}
+/** Carry an anchor range through a change set; it stays inside its text when edits happen around it. */
+declare function mapRange(changes: readonly SourceChange[], start: number, end: number, unit?: "utf16" | "codepoint"): MappedRange;
+
+/**
+ * UTF-16 ⇄ code-point offset conversion.
+ *
+ * JavaScript strings are indexed in UTF-16 code units; Postgres `position()`,
+ * `substring()` and `length()` count code points. Every offset this module
+ * emits is given in BOTH units so an anchor captured on either side of the
+ * wire can be mapped without re-deriving the text.
+ */
+/** Indexes the surrogate pairs of one text so conversion is O(log pairs). */
+interface CodePointIndex {
+    /** Code-point offset of a UTF-16 offset. */
+    toCodePoint(utf16: number): number;
+    /** UTF-16 offset of a code-point offset. */
+    toUtf16(codePoint: number): number;
+    /** Code-point length of the text. */
+    readonly lengthCp: number;
+}
+/** True when `offset` sits between the two halves of a surrogate pair. */
+declare function splitsSurrogatePair(text: string, offset: number): boolean;
+declare function buildCodePointIndex(text: string): CodePointIndex;
+/** One-shot conversion; build an index when converting many offsets. */
+declare function toCodePointOffset(text: string, utf16: number): number;
+/** One-shot conversion; build an index when converting many offsets. */
+declare function toUtf16Offset(text: string, codePoint: number): number;
+
+/**
+ * THE single-dollar math rule — the one predicate that decides whether
+ * `$…$` is inline math or currency.
+ *
+ * Moved here from matrx-frontend `components/markdown-core/math-normalizer.ts`
+ * (2026-09-23) so the renderer's normalizer and the source tokenizer can never
+ * disagree about which bytes are math: the normalizer imports this function.
+ *
+ * Pandoc's rule — the opening `$` is not followed by whitespace, the closing
+ * `$` is not preceded by whitespace and not followed by a digit — plus a math
+ * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), or a
+ * lone variable (`$x$`, `$n$`, `$x'$`). Content that opens with a digit needs
+ * a real TeX signal, so a price range can never read as math.
+ */
+declare function isSingleDollarMath(content: string, after: string | undefined): boolean;
+/**
+ * Where a single-dollar math span opened at `open` (a lone `$`) closes, or -1.
+ * Same scan the normalizer uses: same line, backslash-escapes skipped, the
+ * closer is a `$` not followed by another `$`, and the content passes
+ * {@link isSingleDollarMath}. Returns the offset just past the closing `$`.
+ */
+declare function singleDollarMathEnd(text: string, open: number, limit?: number): number;
+
+/**
+ * XML tag reading + the generic-container balance scanner.
+ *
+ * Ported VERBATIM (behaviour) from matrx-frontend
+ * `components/mardown-display/blocks/xml/readXmlTag.ts` and the
+ * `XmlContainerTracker` in
+ * `components/mardown-display/markdown-classification/processors/utils/content-splitter-v2.ts`
+ * so the source tokenizer ends a generic XML section exactly where the
+ * renderer does. The renderer copies stay for rendering (PLAN decision 6);
+ * a divergence between the two is a defect in whichever changed last.
+ */
+interface ReadXmlTag {
+    tagName: string;
+    raw: string;
+    isClosing: boolean;
+    isSelfClosing: boolean;
+    attributes: Array<{
+        name: string;
+        value: string;
+    }>;
+}
+declare function readXmlTag(content: string, start: number): ReadXmlTag | null;
+/**
+ * Balances ONE unknown root tag line by line, ignoring lookalike tags inside
+ * fenced code, inline code spans, comments and CDATA. `consumeLine` returns
+ * the offset (within that line) just past the root's closing tag, or null.
+ */
+declare class XmlContainerTracker {
+    readonly rootTag: string;
+    private depth;
+    private inComment;
+    private inCdata;
+    private inlineTicks;
+    private fenceMarker;
+    constructor(rootTag: string);
+    consumeLine(line: string, startOffset?: number): number | null;
+}
+
+export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, type InlineIslandType, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, isSingleDollarMath, joinSource, listIslands, mapPosition, mapRange, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };