@ai-matrx/content-ir 0.19.1 → 0.19.3

package/dist/source.cjs

@@ -747,6 +747,7 @@ var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
 var MATH_SUFFIX = /^(?:s|es|th|st|nd|rd)$/i;
 function isVariableChain(content, after) {
   if (/^\{[A-Za-z_][\w.]*\}$/.test(content)) return true;
+  if (/^\(\(.*\)\)$/.test(content) || /^_[.[]/.test(content)) return true;
   if (!after) return false;
   if (after[0] === "{") return IDENTIFIER.test(content);
   const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
@@ -1318,6 +1319,16 @@ var Tokenizer = class {
   holdUndecidedTail() {
     const last = this.lineStarts.length - 1;
     if (last < 0) return;
+    if (this.provisionalFrom < this.n) {
+      for (let b = 0; b < this.blocks.length; b++) {
+        const block = this.blocks[b];
+        if (block.end <= this.provisionalFrom) continue;
+        if (block.kind === "prose") block.inlines = block.inlines.filter((inline) => inline.end <= this.provisionalFrom);
+        else if (block.island?.fromParagraph && block.start < this.provisionalFrom) {
+          this.blocks[b] = { kind: "prose", start: block.start, end: block.end, island: null, inlines: [] };
+        }
+      }
+    }
     if (this.isOpenTail(last)) {
       let b = this.blocks.length - 1;
       while (b >= 0 && this.blocks[b].kind === "gap") b--;
@@ -1615,6 +1626,9 @@ var Tokenizer = class {
       }
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
       if (FOOTNOTE_DEF.test(line.trimStart())) break;
+      if (this.streaming && TREE_GLYPHS.test(line) && this.connectorRunReachesEnd(k)) {
+        return this.heldFootnote(p, open[1]);
+      }
       const block = this.detectBlock(this.lineStarts[k], k, false);
       if (block && this.mayDissolve(block)) return this.heldFootnote(p, open[1]);
       if (block) break;
@@ -1622,8 +1636,22 @@ var Tokenizer = class {
     }
     const end = this.contentEnds[last];
     if (this.paragraphMayGrow(end)) return this.heldFootnote(p, open[1]);
+    if (this.streaming && end > this.provisionalFrom) return this.heldFootnote(p, open[1]);
     return this.island("footnote_def", p, end, true, { id: open[1] });
   }
+  /** Every line from `k` to the end is a connector line or blank (a tree may still form there). */
+  connectorRunReachesEnd(k) {
+    if (this.lastNonConnectorLine === void 0) {
+      let j = this.lineCount - 1;
+      for (; j >= 0; j--) {
+        this.tick(1);
+        if (!this.lineIsBlank(j) && !TREE_GLYPHS.test(this.lineText(j))) break;
+      }
+      this.lastNonConnectorLine = j;
+    }
+    return k > this.lastNonConnectorLine;
+  }
+  lastNonConnectorLine = void 0;
   /** A footnote whose continuation is still arriving: open to the end of the text. */
   heldFootnote(p, id) {
     return this.island("footnote_def", p, this.n, false, { id });
@@ -1888,7 +1916,6 @@ var Tokenizer = class {
       if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
         return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
       }
-      this.provisionalFrom = Math.min(this.provisionalFrom, p);
       return null;
     }
     if (net !== 0) return null;
@@ -2559,6 +2586,76 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
+// source/gfm-table.ts
+function splitRowSegments(line) {
+  const segments = [];
+  let start = 0;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === "\\") {
+      i += 1;
+      continue;
+    }
+    if (ch === "|") {
+      segments.push(line.slice(start, i));
+      start = i + 1;
+    }
+  }
+  segments.push(line.slice(start));
+  return segments;
+}
+function rowCells(line) {
+  const cells = splitRowSegments(line).map((cell) => cell.trim());
+  if (cells.length > 1 && cells[0] === "") cells.shift();
+  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
+  else if (cells.length === 1 && cells[0] === "") cells.pop();
+  return cells;
+}
+function unescapeCellPipes(cell) {
+  return cell.replace(/\\\|/g, "|");
+}
+var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|```|~~~|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$)/;
+var DELIMITER_CELL = /^:?-+:?$/;
+function isGfmDelimiterRow(line) {
+  const trimmed = line.trim();
+  if (!trimmed.includes("|")) return false;
+  const cells = rowCells(trimmed);
+  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
+}
+function isPipeLedRow(line) {
+  const trimmed = line.trim();
+  return trimmed.startsWith("|") && trimmed.includes("|", 1);
+}
+function startsPipelessTable(header, delimiter) {
+  const head = header.trim();
+  if (!head || head.startsWith("|") || BLOCK_START.test(head)) return false;
+  if (splitRowSegments(head).length < 2) return false;
+  if (delimiter === void 0 || !isGfmDelimiterRow(delimiter)) return false;
+  return rowCells(head).length === rowCells(delimiter.trim()).length;
+}
+function continuesTable(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return false;
+  if (trimmed.startsWith("|")) return trimmed.includes("|", 1);
+  return splitRowSegments(trimmed).length > 1 && !BLOCK_START.test(trimmed);
+}
+function opensTable(lines, index) {
+  const line = lines[index] ?? "";
+  return isPipeLedRow(line) || startsPipelessTable(line, lines[index + 1]);
+}
+function tableStartsAt(lines, index) {
+  return opensTable(lines, index) && isGfmDelimiterRow(lines[index + 1] ?? "");
+}
+function findTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
+  return -1;
+}
+function findTableEnd(lines, start) {
+  let end = Math.min(lines.length, start + 2);
+  while (end < lines.length && continuesTable(lines[end] ?? "")) end += 1;
+  return end;
+}
+
 exports.AFTER_WINDOW = AFTER_WINDOW;
 exports.BacktickRuns = BacktickRuns;
 exports.CALLOUT_NAMES = CALLOUT_NAMES;
@@ -2575,13 +2672,18 @@ exports.buildCodePointIndex = buildCodePointIndex;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
 exports.closeFence = closeFence;
 exports.codeSpanText = codeSpanText;
+exports.continuesTable = continuesTable;
 exports.fenceLineKinds = fenceLineKinds;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
 exports.fenceOpenerOf = fenceOpenerOf;
 exports.fenceParts = fenceParts;
 exports.findCodeRanges = findCodeRanges;
+exports.findTableEnd = findTableEnd;
+exports.findTableStart = findTableStart;
 exports.isEscapedBracketMath = isEscapedBracketMath;
+exports.isGfmDelimiterRow = isGfmDelimiterRow;
 exports.isMathPlaceholder = isMathPlaceholder;
+exports.isPipeLedRow = isPipeLedRow;
 exports.isSingleDollarMath = isSingleDollarMath;
 exports.islandEdit = islandEdit;
 exports.joinSource = joinSource;
@@ -2591,17 +2693,23 @@ exports.looksLikeOpenInlineMath = looksLikeOpenInlineMath;
 exports.mapCodeRanges = mapCodeRanges;
 exports.mapPosition = mapPosition;
 exports.mapRange = mapRange;
+exports.opensTable = opensTable;
 exports.pairDisplayMath = pairDisplayMath;
 exports.parseFenceOpener = parseFenceOpener;
 exports.pendingDisplayMathOpener = pendingDisplayMathOpener;
 exports.readXmlTag = readXmlTag;
+exports.rowCells = rowCells;
 exports.singleDollarAfter = singleDollarAfter;
 exports.singleDollarMathEnd = singleDollarMathEnd;
 exports.spliceSave = spliceSave;
+exports.splitRowSegments = splitRowSegments;
 exports.splitsSurrogatePair = splitsSurrogatePair;
+exports.startsPipelessTable = startsPipelessTable;
+exports.tableStartsAt = tableStartsAt;
 exports.toCodePointOffset = toCodePointOffset;
 exports.toUtf16Offset = toUtf16Offset;
 exports.tokenizeSource = tokenizeSource;
 exports.trimFenceLine = trimFenceLine;
+exports.unescapeCellPipes = unescapeCellPipes;
 //# sourceMappingURL=source.cjs.map
 //# sourceMappingURL=source.cjs.map