@ai-matrx/content-ir 0.19.0 → 0.19.2

package/dist/source.d.cts

@@ -607,4 +607,46 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+/**
+ * THE GFM table rule — one splitter, one cell-pipe rule, one "where does a
+ * table start and run" — for every surface that reads a markdown table: the
+ * chat splitter and live stream, the table renderers and editors, the rich
+ * editor's writer, the print package, and (by shared vectors) the Python block
+ * detector. verify-RC-B4 R4/R5/R6.
+ *
+ * - A `|` is a cell boundary unless an ODD run of backslashes precedes it
+ *   (`\|` is a pipe inside the cell; `\\|` is an escaped backslash and then a
+ *   boundary). Code spans get no special treatment: GFM splits rows before
+ *   inline parsing, so an unescaped `|` inside backticks is a boundary too.
+ * - A cell's `\|` shows as `|` — inside code spans too (spec example 200).
+ * - A table starts at a header holding an unescaped pipe whose NEXT line is a
+ *   delimiter row (`--- | :-:`, edge pipes optional); a header without a leading
+ *   pipe must match the delimiter row's width. It runs to a blank line or a
+ *   line that starts another block (`- x`, `> y`, `# z`, `1.`, a fence, a rule).
+ *
+ * Pure TS, no dependencies.
+ */
+/** The row's bytes between boundaries (edge segments included). */
+declare function splitRowSegments(line: string): string[];
+/** A row's cell texts as GFM reads them (source bytes, trimmed; edge pipes dropped). */
+declare function rowCells(line: string): string[];
+/** GFM's table-level unescape: a cell's `\|` is a pipe, in code spans too. */
+declare function unescapeCellPipes(cell: string): string;
+/** A GFM delimiter row: every cell `---`, `:--`, `--:` or `:-:`, with at least one pipe. */
+declare function isGfmDelimiterRow(line: string): boolean;
+/** A pipe-led row: `|` first, and another `|` after it. */
+declare function isPipeLedRow(line: string): boolean;
+/** True when `header` + `delimiter` open a table whose header has NO leading pipe. */
+declare function startsPipelessTable(header: string, delimiter: string | undefined): boolean;
+/** A line that continues an open table: a pipe-led row, or a pipe-less line with an unescaped pipe that starts no block. */
+declare function continuesTable(line: string): boolean;
+/** Line `index` opens a table: a pipe-led row, or a pipe-less GFM header over its delimiter row. */
+declare function opensTable(lines: readonly string[], index: number): boolean;
+/** A WHOLE table header at `index`: a table opens here AND its next line is a GFM delimiter row. */
+declare function tableStartsAt(lines: readonly string[], index: number): boolean;
+/** Index of the first line at or after `from` that opens a table; -1 when none. */
+declare function findTableStart(lines: readonly string[], from?: number): number;
+/** Index just past the table whose header is at `start` (header, delimiter, then continuation rows). */
+declare function findTableEnd(lines: readonly string[], start: number): number;
+
+export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };

package/dist/source.d.ts

@@ -607,4 +607,46 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+/**
+ * THE GFM table rule — one splitter, one cell-pipe rule, one "where does a
+ * table start and run" — for every surface that reads a markdown table: the
+ * chat splitter and live stream, the table renderers and editors, the rich
+ * editor's writer, the print package, and (by shared vectors) the Python block
+ * detector. verify-RC-B4 R4/R5/R6.
+ *
+ * - A `|` is a cell boundary unless an ODD run of backslashes precedes it
+ *   (`\|` is a pipe inside the cell; `\\|` is an escaped backslash and then a
+ *   boundary). Code spans get no special treatment: GFM splits rows before
+ *   inline parsing, so an unescaped `|` inside backticks is a boundary too.
+ * - A cell's `\|` shows as `|` — inside code spans too (spec example 200).
+ * - A table starts at a header holding an unescaped pipe whose NEXT line is a
+ *   delimiter row (`--- | :-:`, edge pipes optional); a header without a leading
+ *   pipe must match the delimiter row's width. It runs to a blank line or a
+ *   line that starts another block (`- x`, `> y`, `# z`, `1.`, a fence, a rule).
+ *
+ * Pure TS, no dependencies.
+ */
+/** The row's bytes between boundaries (edge segments included). */
+declare function splitRowSegments(line: string): string[];
+/** A row's cell texts as GFM reads them (source bytes, trimmed; edge pipes dropped). */
+declare function rowCells(line: string): string[];
+/** GFM's table-level unescape: a cell's `\|` is a pipe, in code spans too. */
+declare function unescapeCellPipes(cell: string): string;
+/** A GFM delimiter row: every cell `---`, `:--`, `--:` or `:-:`, with at least one pipe. */
+declare function isGfmDelimiterRow(line: string): boolean;
+/** A pipe-led row: `|` first, and another `|` after it. */
+declare function isPipeLedRow(line: string): boolean;
+/** True when `header` + `delimiter` open a table whose header has NO leading pipe. */
+declare function startsPipelessTable(header: string, delimiter: string | undefined): boolean;
+/** A line that continues an open table: a pipe-led row, or a pipe-less line with an unescaped pipe that starts no block. */
+declare function continuesTable(line: string): boolean;
+/** Line `index` opens a table: a pipe-led row, or a pipe-less GFM header over its delimiter row. */
+declare function opensTable(lines: readonly string[], index: number): boolean;
+/** A WHOLE table header at `index`: a table opens here AND its next line is a GFM delimiter row. */
+declare function tableStartsAt(lines: readonly string[], index: number): boolean;
+/** Index of the first line at or after `from` that opens a table; -1 when none. */
+declare function findTableStart(lines: readonly string[], from?: number): number;
+/** Index just past the table whose header is at `start` (header, delimiter, then continuation rows). */
+declare function findTableEnd(lines: readonly string[], start: number): number;
+
+export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };

package/dist/source.js

@@ -586,7 +586,7 @@ function pendingCodeSpanStart(text) {
   const n = text.length;
   if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
   const lastLine = text.slice(text.lastIndexOf("\n") + 1);
-  if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
+  if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t\r]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
   const s = scan(text);
   const para = s.pendingPara;
   if (s.endsInFence || para === -1) return -1;
@@ -750,7 +750,7 @@ function isVariableChain(content, after) {
   const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
   if (!run) return false;
   if (after.length === 1) return false;
-  if (/^[A-Za-z_][A-Za-z0-9_]*[.@-]$/.test(content)) return true;
+  if (/^[A-Za-z_][A-Za-z0-9_]*(?:[.@-]|->|::)$/.test(content)) return true;
   return IDENTIFIER.test(content) && !MATH_SUFFIX.test(run);
 }
 var FUNCTION_NAMES = /* @__PURE__ */ new Set([
@@ -1155,7 +1155,7 @@ var Tokenizer = class {
           if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
         }
       }
-      for (const pad of ["`", "``", "```", "~~~"]) {
+      for (const pad of ["`", "``", "```", "~~~", "$"]) {
         const alt = pairDisplayMath(text + pad);
         for (const [open, close] of this.mathPairs) {
           if (alt.get(open) !== close) this.provisionalFrom = Math.min(this.provisionalFrom, open);
@@ -1187,19 +1187,6 @@ var Tokenizer = class {
   provisionalFrom = Number.POSITIVE_INFINITY;
   /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
   pendingCR = false;
-  /**
-   * Streaming: an open last line that begins with `{` is undecided — each new
-   * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — so it is provisional until it ends (G5).
-   * (A lone HTML tag on that line is held by {@link holdUndecidedTail}.)
-   */
-  markOpenJsonTail() {
-    const last = this.lineStarts.length - 1;
-    if (!this.isOpenTail(last)) return;
-    if (this.lineText(last).trim().startsWith("{")) {
-      this.provisionalFrom = Math.min(this.provisionalFrom, this.lineStarts[last]);
-    }
-  }
   /**
    * `text.indexOf(needle, from)` in O(log k): the first query for a needle
    * indexes all of its occurrences once, so repeated searches for a closer
@@ -1234,7 +1221,7 @@ var Tokenizer = class {
     const rest = this.n - pe;
     if (rest === 0) return true;
     const tail = this.text.slice(pe);
-    return /^(?:\r\n|\r|\n)[ \t]*$/.test(tail);
+    return /^(?:\r\n|\r|\n)[ \t\r]*$/.test(tail);
   }
   /**
    * Streaming: an open block that may still turn back into prose — a `$$`
@@ -1243,7 +1230,9 @@ var Tokenizer = class {
    * or any opener that is itself still arriving on the open last line.
    */
   mayDissolve(block) {
-    if (!this.streaming || block.complete) return false;
+    if (!this.streaming) return false;
+    if (block.start >= this.provisionalFrom) return true;
+    if (block.complete) return false;
     if (block.type === "math_block" || block.type === "json") return true;
     return this.isOpenTail(this.lineOf(block.start));
   }
@@ -1282,7 +1271,6 @@ var Tokenizer = class {
     return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
   }
   run() {
-    if (this.streaming) this.markOpenJsonTail();
     let pos = 0;
     while (pos < this.n) {
       const li = this.lineOf(pos);
@@ -1520,7 +1508,7 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    const fence = this.lineText(li);
+    const fence = this.lineText(li).replace(/^\uFEFF/, "");
     if (fence !== "---" && fence !== "+++" || t !== fence) return null;
     const toml = fence === "+++";
     let sawContent = false;
@@ -1531,9 +1519,13 @@ var Tokenizer = class {
       }
       if (!NOT_WHITESPACE.test(line)) continue;
       const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
-      if (!shaped) return null;
+      if (!shaped) {
+        if (this.isOpenTail(k)) break;
+        return null;
+      }
       sawContent = true;
     }
+    if (this.streaming) return this.island("front_matter", 0, this.n, false, { format: toml ? "toml" : "yaml" });
     return null;
   }
   /**
@@ -1934,7 +1926,7 @@ var Tokenizer = class {
       const block = this.detectBlock(this.lineStarts[k], k, false);
       if (block) {
         breakLine = k;
-        if (this.mayDissolve(block) || this.streaming && block.start >= this.provisionalFrom) {
+        if (this.mayDissolve(block)) {
           this.undecidedBreaks.add(end);
         }
         break;
@@ -2565,6 +2557,76 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { AFTER_WINDOW, BacktickRuns, CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, FenceReader, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+// source/gfm-table.ts
+function splitRowSegments(line) {
+  const segments = [];
+  let start = 0;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === "\\") {
+      i += 1;
+      continue;
+    }
+    if (ch === "|") {
+      segments.push(line.slice(start, i));
+      start = i + 1;
+    }
+  }
+  segments.push(line.slice(start));
+  return segments;
+}
+function rowCells(line) {
+  const cells = splitRowSegments(line).map((cell) => cell.trim());
+  if (cells.length > 1 && cells[0] === "") cells.shift();
+  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
+  else if (cells.length === 1 && cells[0] === "") cells.pop();
+  return cells;
+}
+function unescapeCellPipes(cell) {
+  return cell.replace(/\\\|/g, "|");
+}
+var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|```|~~~|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$)/;
+var DELIMITER_CELL = /^:?-+:?$/;
+function isGfmDelimiterRow(line) {
+  const trimmed = line.trim();
+  if (!trimmed.includes("|")) return false;
+  const cells = rowCells(trimmed);
+  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
+}
+function isPipeLedRow(line) {
+  const trimmed = line.trim();
+  return trimmed.startsWith("|") && trimmed.includes("|", 1);
+}
+function startsPipelessTable(header, delimiter) {
+  const head = header.trim();
+  if (!head || head.startsWith("|") || BLOCK_START.test(head)) return false;
+  if (splitRowSegments(head).length < 2) return false;
+  if (delimiter === void 0 || !isGfmDelimiterRow(delimiter)) return false;
+  return rowCells(head).length === rowCells(delimiter.trim()).length;
+}
+function continuesTable(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return false;
+  if (trimmed.startsWith("|")) return trimmed.includes("|", 1);
+  return splitRowSegments(trimmed).length > 1 && !BLOCK_START.test(trimmed);
+}
+function opensTable(lines, index) {
+  const line = lines[index] ?? "";
+  return isPipeLedRow(line) || startsPipelessTable(line, lines[index + 1]);
+}
+function tableStartsAt(lines, index) {
+  return opensTable(lines, index) && isGfmDelimiterRow(lines[index + 1] ?? "");
+}
+function findTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
+  return -1;
+}
+function findTableEnd(lines, start) {
+  let end = Math.min(lines.length, start + 2);
+  while (end < lines.length && continuesTable(lines[end] ?? "")) end += 1;
+  return end;
+}
+
+export { AFTER_WINDOW, BacktickRuns, CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, FenceReader, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map