@ai-matrx/content-ir 0.19.1 → 0.19.3

package/dist/source.d.cts

@@ -607,4 +607,46 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+/**
+ * THE GFM table rule — one splitter, one cell-pipe rule, one "where does a
+ * table start and run" — for every surface that reads a markdown table: the
+ * chat splitter and live stream, the table renderers and editors, the rich
+ * editor's writer, the print package, and (by shared vectors) the Python block
+ * detector. verify-RC-B4 R4/R5/R6.
+ *
+ * - A `|` is a cell boundary unless an ODD run of backslashes precedes it
+ *   (`\|` is a pipe inside the cell; `\\|` is an escaped backslash and then a
+ *   boundary). Code spans get no special treatment: GFM splits rows before
+ *   inline parsing, so an unescaped `|` inside backticks is a boundary too.
+ * - A cell's `\|` shows as `|` — inside code spans too (spec example 200).
+ * - A table starts at a header holding an unescaped pipe whose NEXT line is a
+ *   delimiter row (`--- | :-:`, edge pipes optional); a header without a leading
+ *   pipe must match the delimiter row's width. It runs to a blank line or a
+ *   line that starts another block (`- x`, `> y`, `# z`, `1.`, a fence, a rule).
+ *
+ * Pure TS, no dependencies.
+ */
+/** The row's bytes between boundaries (edge segments included). */
+declare function splitRowSegments(line: string): string[];
+/** A row's cell texts as GFM reads them (source bytes, trimmed; edge pipes dropped). */
+declare function rowCells(line: string): string[];
+/** GFM's table-level unescape: a cell's `\|` is a pipe, in code spans too. */
+declare function unescapeCellPipes(cell: string): string;
+/** A GFM delimiter row: every cell `---`, `:--`, `--:` or `:-:`, with at least one pipe. */
+declare function isGfmDelimiterRow(line: string): boolean;
+/** A pipe-led row: `|` first, and another `|` after it. */
+declare function isPipeLedRow(line: string): boolean;
+/** True when `header` + `delimiter` open a table whose header has NO leading pipe. */
+declare function startsPipelessTable(header: string, delimiter: string | undefined): boolean;
+/** A line that continues an open table: a pipe-led row, or a pipe-less line with an unescaped pipe that starts no block. */
+declare function continuesTable(line: string): boolean;
+/** Line `index` opens a table: a pipe-led row, or a pipe-less GFM header over its delimiter row. */
+declare function opensTable(lines: readonly string[], index: number): boolean;
+/** A WHOLE table header at `index`: a table opens here AND its next line is a GFM delimiter row. */
+declare function tableStartsAt(lines: readonly string[], index: number): boolean;
+/** Index of the first line at or after `from` that opens a table; -1 when none. */
+declare function findTableStart(lines: readonly string[], from?: number): number;
+/** Index just past the table whose header is at `start` (header, delimiter, then continuation rows). */
+declare function findTableEnd(lines: readonly string[], start: number): number;
+
+export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };

package/dist/source.d.ts

@@ -607,4 +607,46 @@ declare function pendingDisplayMathOpener(text: string): number;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+/**
+ * THE GFM table rule — one splitter, one cell-pipe rule, one "where does a
+ * table start and run" — for every surface that reads a markdown table: the
+ * chat splitter and live stream, the table renderers and editors, the rich
+ * editor's writer, the print package, and (by shared vectors) the Python block
+ * detector. verify-RC-B4 R4/R5/R6.
+ *
+ * - A `|` is a cell boundary unless an ODD run of backslashes precedes it
+ *   (`\|` is a pipe inside the cell; `\\|` is an escaped backslash and then a
+ *   boundary). Code spans get no special treatment: GFM splits rows before
+ *   inline parsing, so an unescaped `|` inside backticks is a boundary too.
+ * - A cell's `\|` shows as `|` — inside code spans too (spec example 200).
+ * - A table starts at a header holding an unescaped pipe whose NEXT line is a
+ *   delimiter row (`--- | :-:`, edge pipes optional); a header without a leading
+ *   pipe must match the delimiter row's width. It runs to a blank line or a
+ *   line that starts another block (`- x`, `> y`, `# z`, `1.`, a fence, a rule).
+ *
+ * Pure TS, no dependencies.
+ */
+/** The row's bytes between boundaries (edge segments included). */
+declare function splitRowSegments(line: string): string[];
+/** A row's cell texts as GFM reads them (source bytes, trimmed; edge pipes dropped). */
+declare function rowCells(line: string): string[];
+/** GFM's table-level unescape: a cell's `\|` is a pipe, in code spans too. */
+declare function unescapeCellPipes(cell: string): string;
+/** A GFM delimiter row: every cell `---`, `:--`, `--:` or `:-:`, with at least one pipe. */
+declare function isGfmDelimiterRow(line: string): boolean;
+/** A pipe-led row: `|` first, and another `|` after it. */
+declare function isPipeLedRow(line: string): boolean;
+/** True when `header` + `delimiter` open a table whose header has NO leading pipe. */
+declare function startsPipelessTable(header: string, delimiter: string | undefined): boolean;
+/** A line that continues an open table: a pipe-led row, or a pipe-less line with an unescaped pipe that starts no block. */
+declare function continuesTable(line: string): boolean;
+/** Line `index` opens a table: a pipe-led row, or a pipe-less GFM header over its delimiter row. */
+declare function opensTable(lines: readonly string[], index: number): boolean;
+/** A WHOLE table header at `index`: a table opens here AND its next line is a GFM delimiter row. */
+declare function tableStartsAt(lines: readonly string[], index: number): boolean;
+/** Index of the first line at or after `from` that opens a table; -1 when none. */
+declare function findTableStart(lines: readonly string[], from?: number): number;
+/** Index just past the table whose header is at `start` (header, delimiter, then continuation rows). */
+declare function findTableEnd(lines: readonly string[], start: number): number;
+
+export { AFTER_WINDOW, BacktickRuns, type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type FenceEnd, type FenceLineKind, type FenceLines, type FenceOpener, FenceReader, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };

package/dist/source.js

@@ -745,6 +745,7 @@ var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
 var MATH_SUFFIX = /^(?:s|es|th|st|nd|rd)$/i;
 function isVariableChain(content, after) {
   if (/^\{[A-Za-z_][\w.]*\}$/.test(content)) return true;
+  if (/^\(\(.*\)\)$/.test(content) || /^_[.[]/.test(content)) return true;
   if (!after) return false;
   if (after[0] === "{") return IDENTIFIER.test(content);
   const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
@@ -1316,6 +1317,16 @@ var Tokenizer = class {
   holdUndecidedTail() {
     const last = this.lineStarts.length - 1;
     if (last < 0) return;
+    if (this.provisionalFrom < this.n) {
+      for (let b = 0; b < this.blocks.length; b++) {
+        const block = this.blocks[b];
+        if (block.end <= this.provisionalFrom) continue;
+        if (block.kind === "prose") block.inlines = block.inlines.filter((inline) => inline.end <= this.provisionalFrom);
+        else if (block.island?.fromParagraph && block.start < this.provisionalFrom) {
+          this.blocks[b] = { kind: "prose", start: block.start, end: block.end, island: null, inlines: [] };
+        }
+      }
+    }
     if (this.isOpenTail(last)) {
       let b = this.blocks.length - 1;
       while (b >= 0 && this.blocks[b].kind === "gap") b--;
@@ -1613,6 +1624,9 @@ var Tokenizer = class {
       }
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
       if (FOOTNOTE_DEF.test(line.trimStart())) break;
+      if (this.streaming && TREE_GLYPHS.test(line) && this.connectorRunReachesEnd(k)) {
+        return this.heldFootnote(p, open[1]);
+      }
       const block = this.detectBlock(this.lineStarts[k], k, false);
       if (block && this.mayDissolve(block)) return this.heldFootnote(p, open[1]);
       if (block) break;
@@ -1620,8 +1634,22 @@ var Tokenizer = class {
     }
     const end = this.contentEnds[last];
     if (this.paragraphMayGrow(end)) return this.heldFootnote(p, open[1]);
+    if (this.streaming && end > this.provisionalFrom) return this.heldFootnote(p, open[1]);
     return this.island("footnote_def", p, end, true, { id: open[1] });
   }
+  /** Every line from `k` to the end is a connector line or blank (a tree may still form there). */
+  connectorRunReachesEnd(k) {
+    if (this.lastNonConnectorLine === void 0) {
+      let j = this.lineCount - 1;
+      for (; j >= 0; j--) {
+        this.tick(1);
+        if (!this.lineIsBlank(j) && !TREE_GLYPHS.test(this.lineText(j))) break;
+      }
+      this.lastNonConnectorLine = j;
+    }
+    return k > this.lastNonConnectorLine;
+  }
+  lastNonConnectorLine = void 0;
   /** A footnote whose continuation is still arriving: open to the end of the text. */
   heldFootnote(p, id) {
     return this.island("footnote_def", p, this.n, false, { id });
@@ -1886,7 +1914,6 @@ var Tokenizer = class {
       if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
         return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
       }
-      this.provisionalFrom = Math.min(this.provisionalFrom, p);
       return null;
     }
     if (net !== 0) return null;
@@ -2557,6 +2584,76 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { AFTER_WINDOW, BacktickRuns, CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, FenceReader, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, isEscapedBracketMath, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarAfter, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+// source/gfm-table.ts
+function splitRowSegments(line) {
+  const segments = [];
+  let start = 0;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === "\\") {
+      i += 1;
+      continue;
+    }
+    if (ch === "|") {
+      segments.push(line.slice(start, i));
+      start = i + 1;
+    }
+  }
+  segments.push(line.slice(start));
+  return segments;
+}
+function rowCells(line) {
+  const cells = splitRowSegments(line).map((cell) => cell.trim());
+  if (cells.length > 1 && cells[0] === "") cells.shift();
+  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
+  else if (cells.length === 1 && cells[0] === "") cells.pop();
+  return cells;
+}
+function unescapeCellPipes(cell) {
+  return cell.replace(/\\\|/g, "|");
+}
+var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|```|~~~|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$)/;
+var DELIMITER_CELL = /^:?-+:?$/;
+function isGfmDelimiterRow(line) {
+  const trimmed = line.trim();
+  if (!trimmed.includes("|")) return false;
+  const cells = rowCells(trimmed);
+  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
+}
+function isPipeLedRow(line) {
+  const trimmed = line.trim();
+  return trimmed.startsWith("|") && trimmed.includes("|", 1);
+}
+function startsPipelessTable(header, delimiter) {
+  const head = header.trim();
+  if (!head || head.startsWith("|") || BLOCK_START.test(head)) return false;
+  if (splitRowSegments(head).length < 2) return false;
+  if (delimiter === void 0 || !isGfmDelimiterRow(delimiter)) return false;
+  return rowCells(head).length === rowCells(delimiter.trim()).length;
+}
+function continuesTable(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return false;
+  if (trimmed.startsWith("|")) return trimmed.includes("|", 1);
+  return splitRowSegments(trimmed).length > 1 && !BLOCK_START.test(trimmed);
+}
+function opensTable(lines, index) {
+  const line = lines[index] ?? "";
+  return isPipeLedRow(line) || startsPipelessTable(line, lines[index + 1]);
+}
+function tableStartsAt(lines, index) {
+  return opensTable(lines, index) && isGfmDelimiterRow(lines[index + 1] ?? "");
+}
+function findTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
+  return -1;
+}
+function findTableEnd(lines, start) {
+  let end = Math.min(lines.length, start + 2);
+  while (end < lines.length && continuesTable(lines[end] ?? "")) end += 1;
+  return end;
+}
+
+export { AFTER_WINDOW, BacktickRuns, CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, FenceReader, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, closeFence, codeSpanText, continuesTable, fenceLineKinds, fenceNestsInnerFences, fenceOpenerOf, fenceParts, findCodeRanges, findTableEnd, findTableStart, isEscapedBracketMath, isGfmDelimiterRow, isMathPlaceholder, isPipeLedRow, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, looksLikeOpenInlineMath, mapCodeRanges, mapPosition, mapRange, opensTable, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, rowCells, singleDollarAfter, singleDollarMathEnd, spliceSave, splitRowSegments, splitsSurrogatePair, startsPipelessTable, tableStartsAt, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, unescapeCellPipes };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map