@ai-matrx/content-ir 0.18.0 → 0.18.2

package/dist/source.d.cts

@@ -350,17 +350,6 @@ declare function looksLikeOpenInlineMath(partial: string): boolean;
  */
 declare function singleDollarMathEnd(text: string, open: number, limit?: number): number;
 
-/**
- * XML tag reading + the generic-container balance scanner.
- *
- * Ported VERBATIM (behaviour) from matrx-frontend
- * `components/mardown-display/blocks/xml/readXmlTag.ts` and the
- * `XmlContainerTracker` in
- * `components/mardown-display/markdown-classification/processors/utils/content-splitter-v2.ts`
- * so the source tokenizer ends a generic XML section exactly where the
- * renderer does. The renderer copies stay for rendering (PLAN decision 6);
- * a divergence between the two is a defect in whichever changed last.
- */
 interface ReadXmlTag {
     tagName: string;
     raw: string;
@@ -388,7 +377,8 @@ declare class XmlContainerTracker {
     private inComment;
     private inCdata;
     private inlineTicks;
-    private fenceMarker;
+    /** The open fence, read by THE one closer (source/code-ranges.ts). */
+    private fence;
     constructor(rootTag: string);
     consumeLine(line: string, startOffset?: number): number | null;
 }
@@ -454,12 +444,108 @@ interface CodeRange {
     end: number;
     kind: "fence" | "span";
 }
+interface FenceOpener {
+    char: "`" | "~";
+    /** Length of the opening run. */
+    ticks: number;
+    /** The info string's first word ("" when none). */
+    lang: string;
+}
+/**
+ * THE opener rule for one line's content (container prefixes already
+ * removed; for a tilde fence, `content` starts at the container's column).
+ */
+declare function fenceOpenerOf(content: string): FenceOpener | null;
+/** The lines a fence is read from. */
+interface FenceLines {
+    count: number;
+    /** Line `k`'s content (no terminator), or null when the fence's container has ended before it. */
+    line(k: number): string | null;
+    /** Streaming: line `k` has not ended yet, so it closes nothing. */
+    openTail?(k: number): boolean;
+}
+interface FenceEnd {
+    /** The closing line when `closed`; otherwise the fence's last line (text end, or the line before its container ended). */
+    line: number;
+    closed: boolean;
+}
+/**
+ * THE closer rule, read line by line — the one implementation behind
+ * `closeFence` and every streaming walker. Feed each following line
+ * (container prefixes removed); `feed` returns true on the line that closes
+ * the fence. A backtick fence closes by the nested-fence rule or the
+ * renderer's line-end closer (`code````), and inside ```json a backtick run in
+ * a JSON string is text; a tilde fence closes on `~~~` (≥ its length, ≤ 3
+ * spaces). `nesting: false` forces strict CommonMark (the retry `closeFence`
+ * makes when a nested fence is left open at the end; a streaming reader
+ * cannot look ahead, so it reads with nesting and never retries).
+ */
+declare class FenceReader {
+    readonly opener: FenceOpener;
+    private state;
+    private depth;
+    /** A nested fence was opened at some point. */
+    sawNested: boolean;
+    private readonly isJson;
+    private readonly nesting;
+    private readonly tildeCloser;
+    constructor(opener: FenceOpener, options?: {
+        nesting?: boolean;
+    });
+    /** How many nested fences are open right now. */
+    get nestedDepth(): number;
+    feed(line: string): boolean;
+}
+/** Where the fence opened on line `open` ends. */
+declare function closeFence(src: FenceLines, open: number, opener: FenceOpener): FenceEnd;
+/**
+ * THE code-span pairing table for one paragraph, text[from, to): a backtick
+ * run closes at the first LATER run of exactly its length; a run with none is
+ * literal text. `findCodeRanges` pairs runs in order; the tokenizer's inline
+ * scanner asks `next` in the renderer's leftmost-first inline order.
+ */
+declare class BacktickRuns {
+    /** Every run, in order: [start, length]. */
+    readonly all: Array<[number, number]>;
+    private readonly byLength;
+    private readonly indexOf;
+    constructor(text: string, from: number, to: number);
+    /** Start of the first run of exactly `n` backticks at or after `after`, or -1. */
+    next(n: number, after: number): number;
+    /** Index of the run starting at `start` in `all`. */
+    index(start: number): number;
+}
 /**
  * Every code range in the text — fences (by the renderer's rule above) and
  * inline code spans — sorted by start. Linear: one pass over lines, one over
  * each paragraph's backtick runs.
  */
 declare function findCodeRanges(text: string): CodeRange[];
+/**
+ * Rewrite every code range (fences and spans, by THE rule) through `fn`; text
+ * outside code is kept byte for byte. The one way to strip, replace or
+ * placeholder code in markdown — never a private /```…```/ regex.
+ */
+declare function mapCodeRanges(text: string, fn: (range: CodeRange, raw: string) => string): string;
+/** A code span's text (CommonMark): the backtick runs off, then one padding space each side when both are there and the content is not all spaces. */
+declare function codeSpanText(raw: string): string;
+/**
+ * A fence range's parts: its opener, its body (the lines between the opener
+ * and the closer, blockquote prefixes kept), and whether it closed. Reads
+ * `raw` exactly as `findCodeRanges` returned it.
+ */
+declare function fenceParts(raw: string): {
+    opener: FenceOpener;
+    body: string;
+    closed: boolean;
+} | null;
+type FenceLineKind = "prose" | "open" | "body" | "close";
+/**
+ * Where each line (the text split on "\n") sits relative to the fences
+ * `findCodeRanges` finds: its opener, its body, its closer, or prose. For a
+ * line walker that must skip code — one rule, never a private `inFence` toggle.
+ */
+declare function fenceLineKinds(text: string): FenceLineKind[];
 
 /**
  * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
@@ -486,4 +572,4 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.d.ts

@@ -350,17 +350,6 @@ declare function looksLikeOpenInlineMath(partial: string): boolean;
  */
 declare function singleDollarMathEnd(text: string, open: number, limit?: number): number;
 
-/**
- * XML tag reading + the generic-container balance scanner.
- *
- * Ported VERBATIM (behaviour) from matrx-frontend
- * `components/mardown-display/blocks/xml/readXmlTag.ts` and the
- * `XmlContainerTracker` in
- * `components/mardown-display/markdown-classification/processors/utils/content-splitter-v2.ts`
- * so the source tokenizer ends a generic XML section exactly where the
- * renderer does. The renderer copies stay for rendering (PLAN decision 6);
- * a divergence between the two is a defect in whichever changed last.
- */
 interface ReadXmlTag {
     tagName: string;
     raw: string;
@@ -388,7 +377,8 @@ declare class XmlContainerTracker {
     private inComment;
     private inCdata;
     private inlineTicks;
-    private fenceMarker;
+    /** The open fence, read by THE one closer (source/code-ranges.ts). */
+    private fence;
     constructor(rootTag: string);
     consumeLine(line: string, startOffset?: number): number | null;
 }
@@ -454,12 +444,108 @@ interface CodeRange {
     end: number;
     kind: "fence" | "span";
 }
+interface FenceOpener {
+    char: "`" | "~";
+    /** Length of the opening run. */
+    ticks: number;
+    /** The info string's first word ("" when none). */
+    lang: string;
+}
+/**
+ * THE opener rule for one line's content (container prefixes already
+ * removed; for a tilde fence, `content` starts at the container's column).
+ */
+declare function fenceOpenerOf(content: string): FenceOpener | null;
+/** The lines a fence is read from. */
+interface FenceLines {
+    count: number;
+    /** Line `k`'s content (no terminator), or null when the fence's container has ended before it. */
+    line(k: number): string | null;
+    /** Streaming: line `k` has not ended yet, so it closes nothing. */
+    openTail?(k: number): boolean;
+}
+interface FenceEnd {
+    /** The closing line when `closed`; otherwise the fence's last line (text end, or the line before its container ended). */
+    line: number;
+    closed: boolean;
+}
+/**
+ * THE closer rule, read line by line — the one implementation behind
+ * `closeFence` and every streaming walker. Feed each following line
+ * (container prefixes removed); `feed` returns true on the line that closes
+ * the fence. A backtick fence closes by the nested-fence rule or the
+ * renderer's line-end closer (`code````), and inside ```json a backtick run in
+ * a JSON string is text; a tilde fence closes on `~~~` (≥ its length, ≤ 3
+ * spaces). `nesting: false` forces strict CommonMark (the retry `closeFence`
+ * makes when a nested fence is left open at the end; a streaming reader
+ * cannot look ahead, so it reads with nesting and never retries).
+ */
+declare class FenceReader {
+    readonly opener: FenceOpener;
+    private state;
+    private depth;
+    /** A nested fence was opened at some point. */
+    sawNested: boolean;
+    private readonly isJson;
+    private readonly nesting;
+    private readonly tildeCloser;
+    constructor(opener: FenceOpener, options?: {
+        nesting?: boolean;
+    });
+    /** How many nested fences are open right now. */
+    get nestedDepth(): number;
+    feed(line: string): boolean;
+}
+/** Where the fence opened on line `open` ends. */
+declare function closeFence(src: FenceLines, open: number, opener: FenceOpener): FenceEnd;
+/**
+ * THE code-span pairing table for one paragraph, text[from, to): a backtick
+ * run closes at the first LATER run of exactly its length; a run with none is
+ * literal text. `findCodeRanges` pairs runs in order; the tokenizer's inline
+ * scanner asks `next` in the renderer's leftmost-first inline order.
+ */
+declare class BacktickRuns {
+    /** Every run, in order: [start, length]. */
+    readonly all: Array<[number, number]>;
+    private readonly byLength;
+    private readonly indexOf;
+    constructor(text: string, from: number, to: number);
+    /** Start of the first run of exactly `n` backticks at or after `after`, or -1. */
+    next(n: number, after: number): number;
+    /** Index of the run starting at `start` in `all`. */
+    index(start: number): number;
+}
 /**
  * Every code range in the text — fences (by the renderer's rule above) and
  * inline code spans — sorted by start. Linear: one pass over lines, one over
  * each paragraph's backtick runs.
  */
 declare function findCodeRanges(text: string): CodeRange[];
+/**
+ * Rewrite every code range (fences and spans, by THE rule) through `fn`; text
+ * outside code is kept byte for byte. The one way to strip, replace or
+ * placeholder code in markdown — never a private /```…```/ regex.
+ */
+declare function mapCodeRanges(text: string, fn: (range: CodeRange, raw: string) => string): string;
+/** A code span's text (CommonMark): the backtick runs off, then one padding space each side when both are there and the content is not all spaces. */
+declare function codeSpanText(raw: string): string;
+/**
+ * A fence range's parts: its opener, its body (the lines between the opener
+ * and the closer, blockquote prefixes kept), and whether it closed. Reads
+ * `raw` exactly as `findCodeRanges` returned it.
+ */
+declare function fenceParts(raw: string): {
+    opener: FenceOpener;
+    body: string;
+    closed: boolean;
+} | null;
+type FenceLineKind = "prose" | "open" | "body" | "close";
+/**
+ * Where each line (the text split on "\n") sits relative to the fences
+ * `findCodeRanges` finds: its opener, its body, its closer, or prose. For a
+ * line walker that must skip code — one rule, never a private `inFence` toggle.
+ */
+declare function fenceLineKinds(text: string): FenceLineKind[];
 
 /**
  * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
@@ -486,4 +572,4 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.js

@@ -392,7 +392,7 @@ function scan(text) {
   }
   const count = starts.length;
   const raw = (k) => text.slice(starts[k], ends[k]);
-  const out = { ranges: [], paragraphs: [], endsInFence: false, pendingPara: -1 };
+  const out = { ranges: [], fences: [], paragraphs: [], endsInFence: false, pendingPara: -1 };
   const rawLines = { count, line: raw };
   let segment = 0;
   for (let k = 0; k < count; k++) {
@@ -401,6 +401,7 @@ function scan(text) {
     prose(text, starts, ends, raw, segment, k, out);
     const end = closeFence(rawLines, k, opener);
     out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    out.fences.push([k, end.line, end.closed]);
     out.pendingPara = -1;
     if (!end.closed && end.line === count - 1) out.endsInFence = true;
     k = end.line;
@@ -466,6 +467,7 @@ function prose(text, starts, ends, raw, from, to, out) {
     };
     const end = closeFence(src, k, opener);
     out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    out.fences.push([k, end.line, end.closed]);
     if (!end.closed && end.line === starts.length - 1) out.endsInFence = true;
     k = end.line;
   }
@@ -529,6 +531,45 @@ function findCodeRanges(text) {
   for (const [from, to] of s.paragraphs) codeSpans(text, from, to, ranges);
   return ranges.sort((a, b) => a.start - b.start);
 }
+function mapCodeRanges(text, fn) {
+  let out = "";
+  let at = 0;
+  for (const r of findCodeRanges(text)) {
+    if (r.start < at) continue;
+    out += text.slice(at, r.start) + fn(r, text.slice(r.start, r.end));
+    at = r.end;
+  }
+  return out + text.slice(at);
+}
+function codeSpanText(raw) {
+  const ticks = backtickRunLength(raw, 0);
+  const inner = raw.slice(ticks, raw.length - ticks);
+  return inner.length >= 2 && inner.startsWith(" ") && inner.endsWith(" ") && inner.trim() !== "" ? inner.slice(1, -1) : inner;
+}
+function fenceParts(raw) {
+  const lines = raw.split("\n").map((l) => l.endsWith("\r") ? l.slice(0, -1) : l);
+  const { offset } = quotePrefix(lines[0] ?? "");
+  const opener = fenceOpenerOf((lines[0] ?? "").slice(offset).replace(/^ *(?:[-*+]|\d{1,9}[.)])(?= |\t|$) ?/, ""));
+  if (!opener) return null;
+  const rest = lines.slice(1);
+  const closed = rest.length > 0 && new FenceReader(opener, { nesting: false }).feed(rest[rest.length - 1]);
+  if (!closed) return { opener, body: rest.join("\n"), closed };
+  const last = rest[rest.length - 1];
+  const before = trimFenceLine(last).startsWith("```") ? "" : last.slice(0, last.indexOf("```"));
+  return { opener, body: [...rest.slice(0, -1), ...before.trim() ? [before] : []].join("\n"), closed };
+}
+function fenceLineKinds(text) {
+  const s = scan(text);
+  let count = 1;
+  for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) count++;
+  const kinds = new Array(count).fill("prose");
+  for (const [open, last, closed] of s.fences) {
+    kinds[open] = "open";
+    for (let k = open + 1; k <= last; k++) kinds[k] = "body";
+    if (closed && last > open) kinds[last] = "close";
+  }
+  return kinds;
+}
 function pendingCodeSpanStart(text) {
   const n = text.length;
   if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
@@ -827,25 +868,20 @@ var XmlContainerTracker = class {
   inComment = false;
   inCdata = false;
   inlineTicks = 0;
-  fenceMarker = null;
+  /** The open fence, read by THE one closer (source/code-ranges.ts). */
+  fence = null;
   consumeLine(line, startOffset = 0) {
-    const fence = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);
-    if (!this.inComment && !this.inCdata && this.inlineTicks === 0 && fence) {
-      const marker = fence[1];
-      const char = marker[0];
-      const suffix = fence[2];
-      if (marker.length >= 3) {
-        if (!this.fenceMarker) {
-          this.fenceMarker = { char, ticks: marker.length };
-          return null;
-        }
-        if (char === this.fenceMarker.char && marker.length >= this.fenceMarker.ticks && suffix.trim() === "") {
-          this.fenceMarker = null;
-        }
+    if (this.fence) {
+      if (this.fence.feed(line)) this.fence = null;
+      return null;
+    }
+    if (!this.inComment && !this.inCdata && this.inlineTicks === 0) {
+      const opener = fenceOpenerOf(line);
+      if (opener) {
+        this.fence = new FenceReader(opener);
         return null;
       }
     }
-    if (this.fenceMarker) return null;
     for (let i = startOffset; i < line.length; i++) {
       if (this.inlineTicks > 0) {
         if (line[i] === "`") {
@@ -2335,6 +2371,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { BacktickRuns, CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, FenceReader, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map