@ai-matrx/content-ir 0.20.0 → 0.21.0

package/dist/source.cjs

@@ -2877,20 +2877,443 @@ function resolveReferenceLinks(fragment, defs) {
   return resolved.replace(new RegExp(`${PLACEHOLDER}(\\d+)${PLACEHOLDER}`, "g"), (_m, i) => spans[Number(i)] ?? "");
 }
 
+// source/code-range-edits.ts
+function unwrapCodeSpans(text) {
+  if (!text.includes("`")) return text;
+  return mapCodeRanges(text, (range, raw) => range.kind === "span" ? codeSpanText(raw) : raw);
+}
+function removeCodeSpans(text) {
+  if (!text.includes("`")) return text;
+  return mapCodeRanges(text, (range, raw) => range.kind === "span" ? "" : raw);
+}
+function replaceFences(text, fn) {
+  if (!text.includes("```") && !text.includes("~~~")) return text;
+  return mapCodeRanges(text, (range, raw) => {
+    if (range.kind !== "fence") return raw;
+    const parts = fenceParts(raw);
+    return fn({ lang: parts?.opener.lang ?? "", body: parts?.body ?? raw, closed: parts?.closed ?? false, raw });
+  });
+}
+function soleFence(text, options = {}) {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("`") && !trimmed.startsWith("~")) return null;
+  const ranges = findCodeRanges(trimmed);
+  const only = ranges.length === 1 ? ranges[0] : void 0;
+  if (!only || only.kind !== "fence" || only.start !== 0 || only.end !== trimmed.length) return null;
+  const parts = fenceParts(trimmed);
+  if (!parts || !parts.closed && !options.allowUnclosed) return null;
+  return { lang: parts.opener.lang, body: parts.body, closed: parts.closed };
+}
+
+// source/plain-text.ts
+var INLINE_PATTERNS = [
+  // [label](href) → label. Runs first so emphasis inside a label still unwraps.
+  [/\[([^\]]+)\]\([^)]*\)/g, "$1"],
+  [/\*\*\*(?=\S)([\s\S]*?\S)\*\*\*/g, "$1"],
+  [/___(?=\S)([\s\S]*?\S)___/g, "$1"],
+  [/\*\*(?=\S)([\s\S]*?\S)\*\*/g, "$1"],
+  [/__(?=\S)([\s\S]*?\S)__/g, "$1"],
+  [/~~(?=\S)([\s\S]*?\S)~~/g, "$1"],
+  [/\*(?=\S)([^*\n]*?\S)\*/g, "$1"],
+  // Underscore italics only between word boundaries: snake_case_names are not
+  // emphasis, and mangling an identifier is a worse defect than a stray marker.
+  [/(^|[\s(])_(?=\S)([^_\n]*?\S)_(?=[\s).,;:!?]|$)/g, "$1$2"]
+];
+var MATH_STRUCTURAL = [
+  [/\\(?:d|t)?frac\s*\{([^{}]*)\}\s*\{([^{}]*)\}/g, "$1/$2"],
+  [/\\sqrt\s*\{([^{}]*)\}/g, "\u221A($1)"]
+];
+var MATH_PATTERNS = [
+  // Environments (\begin{aligned} … \end{aligned}) carry no meaning in a
+  // one-line preview; drop the wrapper, keep the contents. Without this the
+  // catch-all below turned them into the word "aligned" twice.
+  [/\\(?:begin|end)\s*\{[^{}]*\}/g, " "],
+  // A TeX line break inside a preview is just a space.
+  [/\\\\/g, " "],
+  // TeX spacing commands (\, \; \: thin/medium space, \! negative space).
+  [/\\[,;:]/g, " "],
+  [/\\!/g, ""],
+  // \left\{ … \right\} — the escaped-brace forms, which the character-class
+  // rule below cannot reach because the brace is preceded by a backslash.
+  [/\\(?:left|right)\s*\\?([([\]|){}])/g, "$1"],
+  // NOTE: every command below ends with (?![a-zA-Z]) and NOT \b. `_` is a WORD
+  // character, so \b fails after `\sum_` — which silently left every
+  // summation, integral and limit card reading as "sum_" / "int_".
+  [/\\times(?![a-zA-Z])/g, "\xD7"],
+  [/\\div(?![a-zA-Z])/g, "\xF7"],
+  [/\\pm(?![a-zA-Z])/g, "\xB1"],
+  [/\\cdot(?![a-zA-Z])/g, "\xB7"],
+  [/\\leq(?![a-zA-Z])/g, "\u2264"],
+  [/\\geq(?![a-zA-Z])/g, "\u2265"],
+  [/\\neq(?![a-zA-Z])/g, "\u2260"],
+  [/\\approx(?![a-zA-Z])/g, "\u2248"],
+  [/\\infty(?![a-zA-Z])/g, "\u221E"],
+  [/\\pi(?![a-zA-Z])/g, "\u03C0"],
+  [/\\theta(?![a-zA-Z])/g, "\u03B8"],
+  [/\\alpha(?![a-zA-Z])/g, "\u03B1"],
+  [/\\beta(?![a-zA-Z])/g, "\u03B2"],
+  [/\\Delta(?![a-zA-Z])/g, "\u0394"],
+  [/\\delta(?![a-zA-Z])/g, "\u03B4"],
+  [/\\sum(?![a-zA-Z])/g, "\u03A3"],
+  [/\\prod(?![a-zA-Z])/g, "\u220F"],
+  [/\\int(?![a-zA-Z])/g, "\u222B"],
+  [/\\lim(?![a-zA-Z])/g, "lim"],
+  [/\\(?:rightarrow|to)(?![a-zA-Z])/g, "\u2192"],
+  // Braces that only ever grouped a script or argument: ^{2} → ^2, _{i} → _i.
+  [/([_^])\s*\{([^{}]*)\}/g, "$1$2"],
+  // Any remaining single-argument command: \text{x} / \mathrm{x} → x.
+  [/\\[a-zA-Z]+\s*\{([^{}]*)\}/g, "$1"],
+  // Bare commands we did not translate: drop the backslash, keep the word.
+  [/\\([a-zA-Z]+)/g, "$1"]
+];
+var MATH_REGIONS = [
+  /\\\[([\s\S]*?)\\\]/g,
+  /\\\(([\s\S]*?)\\\)/g,
+  /\$\$([\s\S]*?)\$\$/g
+];
+function mathToReadable(expression) {
+  let text = expression;
+  for (let pass = 0; pass < 8; pass += 1) {
+    const before = text;
+    for (const [pattern, replacement] of MATH_STRUCTURAL) {
+      text = text.replace(pattern, replacement);
+    }
+    if (text === before) break;
+  }
+  for (const [pattern, replacement] of MATH_PATTERNS) {
+    text = text.replace(pattern, replacement);
+  }
+  return text.trim();
+}
+function markdownToPlainText(value) {
+  if (!value) return "";
+  const mathChunks = [];
+  let text = value;
+  for (const region of MATH_REGIONS) {
+    text = text.replace(region, (_match, inner) => {
+      mathChunks.push(mathToReadable(inner));
+      return `\uE000MATH${mathChunks.length - 1}\uE001`;
+    });
+  }
+  for (const [pattern, replacement] of INLINE_PATTERNS) {
+    text = text.replace(pattern, replacement);
+  }
+  text = unwrapCodeSpans(text);
+  text = text.replace(
+    /\uE000MATH(\d+)\uE001/g,
+    (_m, index) => mathChunks[Number(index)] ?? ""
+  );
+  return text.replace(/[ \t]{2,}/g, " ").trim();
+}
+function unwrapKindEnvelopes(value) {
+  if (!value) return "";
+  if (!value.includes("<artifact")) return value;
+  return tokenizeSource(value).map((block) => {
+    if (block.kind !== "island" || block.islandType !== "xml_attr" && block.islandType !== "xml_container") {
+      return block.raw;
+    }
+    const match = /^\s*<artifact\b[^>]*>([\s\S]*?)<\/artifact>\s*$/.exec(block.raw);
+    return match ? (match[1] ?? "").replace(/^\n/, "").replace(/\n$/, "") : block.raw;
+  }).join("");
+}
+
+// source/clean-markdown.ts
+var cleanMarkdown = (text) => {
+  if (!text || typeof text !== "string") return "";
+  let cleanedText = text;
+  const lines = cleanedText.split("\n");
+  const processedLines = [];
+  for (let line of lines) {
+    let processedLine = line;
+    if (processedLine.trim() === "") {
+      processedLines.push(processedLine);
+      continue;
+    }
+    processedLine = processedLine.replace(/^#{1,6}\s+(.+)$/, "$1");
+    processedLine = processedLine.replace(/(\*\*|__)(.*?)\1/, "$2");
+    processedLine = processedLine.replace(/(\*|_)(.*?)\1/, "$2");
+    processedLine = unwrapCodeSpans(processedLine);
+    processedLine = processedLine.replace(/\[([^\]]+)\]\(([^)]+)\)/, "$1");
+    processedLine = processedLine.replace(/!\[([^\]]*)\]\(([^)]+)\)/, "$1");
+    processedLine = processedLine.replace(/^>\s*(.+)$/, "$1");
+    processedLine = processedLine.replace(/^(\*{3,}|-{3,}|_{3,})$/, "");
+    processedLine = processedLine.replace(/^(\s*[-+*]\s+|\s*\d+\.\s+)(.+)$/, "$2");
+    processedLine = processedLine.replace(/<\/?[^>]+(>|$)/g, "");
+    processedLines.push(processedLine);
+  }
+  cleanedText = processedLines.join("\n");
+  cleanedText = replaceFences(cleanedText, ({ body }) => body);
+  return cleanedText;
+};
+
+// source/fence-meta.ts
+var FENCE_META_KEY = "fenceMeta";
+function splitFenceInfo(info) {
+  const trimmed = info.trim();
+  if (!trimmed) return {};
+  const firstSpace = trimmed.search(/\s/);
+  let token = firstSpace === -1 ? trimmed : trimmed.slice(0, firstSpace);
+  let rest = firstSpace === -1 ? "" : trimmed.slice(firstSpace).trim();
+  const braceAt = token.indexOf("{");
+  if (braceAt > 0) {
+    rest = `${token.slice(braceAt)} ${rest}`.trim();
+    token = token.slice(0, braceAt);
+  }
+  const colonAt = token.indexOf(":");
+  if (colonAt > 0 && colonAt < token.length - 1) {
+    const filename = token.slice(colonAt + 1);
+    rest = `title="${filename}" ${rest}`.trim();
+    token = token.slice(0, colonAt);
+  }
+  if (token.startsWith("{")) {
+    rest = `${token} ${rest}`.trim();
+    token = "";
+  }
+  const out = {};
+  if (token) out.language = token;
+  if (rest) out.meta = rest;
+  return out;
+}
+var TITLE_RE = /(?:^|\s)(?:title|filename|file)=(?:"([^"]*)"|'([^']*)'|(\S+))/i;
+var RANGES_RE = /(?:^|\s)\{([\d\s,-]+)\}/;
+var LINE_NUMBERS_RE = /(?:^|\s)showLineNumbers(?:\{(\d+)\})?(?=\s|$)/;
+var MAX_HIGHLIGHT_LINES = 1e4;
+function parseFenceMeta(meta) {
+  const result = { highlightLines: [] };
+  if (!meta) return result;
+  const title = TITLE_RE.exec(meta);
+  if (title) {
+    const value = (title[1] ?? title[2] ?? title[3] ?? "").trim();
+    if (value) result.title = value;
+  }
+  const ranges = RANGES_RE.exec(meta);
+  if (ranges) result.highlightLines = parseLineRanges(ranges[1] ?? "");
+  const lineNumbers = LINE_NUMBERS_RE.exec(meta);
+  if (lineNumbers) {
+    result.showLineNumbers = true;
+    if (lineNumbers[1]) {
+      const start = Number.parseInt(lineNumbers[1], 10);
+      if (Number.isFinite(start) && start > 0) result.startLine = start;
+    }
+  }
+  return result;
+}
+function parseLineRanges(spec) {
+  const lines = /* @__PURE__ */ new Set();
+  for (const part of spec.split(",")) {
+    const piece = part.trim();
+    if (!piece) continue;
+    const range = /^(\d+)\s*-\s*(\d+)$/.exec(piece);
+    if (range) {
+      const from = Number.parseInt(range[1] ?? "", 10);
+      const to = Number.parseInt(range[2] ?? "", 10);
+      if (from < 1 || to < from) continue;
+      for (let n = from; n <= to && lines.size < MAX_HIGHLIGHT_LINES; n += 1) {
+        lines.add(n);
+      }
+      continue;
+    }
+    if (/^\d+$/.test(piece)) {
+      const n = Number.parseInt(piece, 10);
+      if (n >= 1) lines.add(n);
+    }
+  }
+  return [...lines].sort((a, b) => a - b);
+}
+
+// source/directive-container.ts
+var DIRECTIVE_CONTAINER_OPEN = /^[ \t]{0,3}(:{3,})[A-Za-z][\w-]*/;
+var CLOSE = /^[ \t]{0,3}(:{3,})[ \t]*$/;
+var DirectiveContainerTracker = class {
+  stack;
+  fence = null;
+  constructor(openingLine) {
+    const m = DIRECTIVE_CONTAINER_OPEN.exec(openingLine);
+    this.stack = [m ? m[1].length : 3];
+  }
+  /** Feed the next line (after the opener). True once the outermost container has closed on this line. */
+  consume(line) {
+    if (this.fence) {
+      if (this.fence.feed(line)) this.fence = null;
+      return false;
+    }
+    const opener = fenceOpenerOf(line);
+    if (opener) {
+      this.fence = new FenceReader(opener);
+      return false;
+    }
+    const open = DIRECTIVE_CONTAINER_OPEN.exec(line);
+    if (open) {
+      this.stack.push(open[1].length);
+      return false;
+    }
+    const close = CLOSE.exec(line);
+    if (close && close[1].length >= (this.stack[this.stack.length - 1] ?? 3)) {
+      this.stack.pop();
+      return this.stack.length === 0;
+    }
+    return false;
+  }
+};
+
+// source/image-figure.ts
+var TITLED_IMAGE_LINE = /^[ \t]{0,3}!\[[^\]\n]*\]\(\s*<?[^\s)>]+>?\s+(?:"[^"\n]+"|'[^'\n]+')\s*\)[ \t]*$/;
+
+// source/inline-code-span.ts
+function isInsideInlineCode(line, at) {
+  let i = 0;
+  while (i <= at && i < line.length) {
+    if (line[i] === "\\") {
+      i += 2;
+      continue;
+    }
+    if (line[i] !== "`") {
+      i += 1;
+      continue;
+    }
+    let run = 0;
+    while (line[i + run] === "`") run += 1;
+    let close = -1;
+    for (let j = i + run; j < line.length; ) {
+      if (line[j] !== "`") {
+        j += 1;
+        continue;
+      }
+      let r = 0;
+      while (line[j + r] === "`") r += 1;
+      if (r === run) {
+        close = j;
+        break;
+      }
+      j += r;
+    }
+    if (close === -1) {
+      i += run;
+      continue;
+    }
+    if (at > i && at < close + run) return true;
+    i = close + run;
+  }
+  return false;
+}
+function indexOutsideInlineCode(line, needle, from = 0) {
+  for (let idx = line.indexOf(needle, from); idx !== -1; idx = line.indexOf(needle, idx + 1)) {
+    if (!isInsideInlineCode(line, idx)) return idx;
+  }
+  return -1;
+}
+function hasUnclosedBacktickRun(text) {
+  let i = 0;
+  while (i < text.length) {
+    if (text[i] === "\\") {
+      i += 2;
+      continue;
+    }
+    if (text[i] !== "`") {
+      i += 1;
+      continue;
+    }
+    let run = 0;
+    while (text[i + run] === "`") run += 1;
+    let close = -1;
+    for (let j = i + run; j < text.length; ) {
+      if (text[j] !== "`") {
+        j += 1;
+        continue;
+      }
+      let r = 0;
+      while (text[j + r] === "`") r += 1;
+      if (r === run) {
+        close = j;
+        break;
+      }
+      j += r;
+    }
+    if (close === -1) return true;
+    i = close + run;
+  }
+  return false;
+}
+
+// source/xml-tag-balance.ts
+function initialXmlBalance() {
+  return { depth: 1, fence: null };
+}
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function findBalancedXmlClose(line, tagName, state, { trackFences = true, nest = true } = {}) {
+  if (state.fence) {
+    if (state.fence.feed(line)) state.fence = null;
+    return -1;
+  }
+  const opener = trackFences ? fenceOpenerOf(line) : null;
+  if (opener) {
+    state.fence = new FenceReader(opener);
+    return -1;
+  }
+  const tag = new RegExp(
+    `<(\\/?)${escapeRegExp(tagName)}(?=[\\s>/])[^<>]*?(\\/?)>`,
+    "g"
+  );
+  for (const match of line.matchAll(tag)) {
+    const closing = match[1] === "/";
+    const selfClosing = match[2] === "/";
+    if (closing) {
+      state.depth -= 1;
+      if (state.depth === 0) return match.index ?? -1;
+    } else if (nest && !selfClosing && line.slice(0, match.index).trim() === "") {
+      state.depth += 1;
+    }
+  }
+  return -1;
+}
+
+// source/gfm-table-lines.ts
+function findWholeTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i + 1 < lines.length; i += 1) if (tableStartsAt(lines, i)) return i;
+  return -1;
+}
+function startsLikeTableRow(line) {
+  return /^\s*\|/.test(line);
+}
+function isGrowingDelimiterRow(line) {
+  return /^[\s|:-]*-[\s|:-]*$/.test(line);
+}
+function trailingTableStart(lines, end = lines.length) {
+  const isTableLine = (line) => startsLikeTableRow(line) || continuesTable(line.trimStart());
+  let start = end;
+  while (start > 0 && isTableLine(lines[start - 1] ?? "")) start -= 1;
+  if (start === end && end > 1 && isGrowingDelimiterRow(lines[end - 1] ?? "") && continuesTable((lines[end - 2] ?? "").trimStart())) {
+    start = end - 1;
+    while (start > 0 && isTableLine(lines[start - 1] ?? "")) start -= 1;
+  }
+  for (let head = start; head < end; head += 1) {
+    if (startsLikeTableRow(lines[head] ?? "")) return head;
+    if (end - head > 1 && isGrowingDelimiterRow(lines[head + 1] ?? "")) return head;
+  }
+  return end;
+}
+
 exports.AFTER_WINDOW = AFTER_WINDOW;
 exports.BacktickRuns = BacktickRuns;
 exports.CALLOUT_NAMES = CALLOUT_NAMES;
+exports.DIRECTIVE_CONTAINER_OPEN = DIRECTIVE_CONTAINER_OPEN;
+exports.DirectiveContainerTracker = DirectiveContainerTracker;
+exports.FENCE_META_KEY = FENCE_META_KEY;
 exports.FENCE_OPENER_WHITESPACE = FENCE_OPENER_WHITESPACE;
 exports.FENCE_WHITESPACE = FENCE_WHITESPACE;
 exports.FenceReader = FenceReader;
 exports.NESTING_FENCE_LANGUAGES = NESTING_FENCE_LANGUAGES;
 exports.SourceSpliceError = SourceSpliceError;
+exports.TITLED_IMAGE_LINE = TITLED_IMAGE_LINE;
 exports.XmlContainerTracker = XmlContainerTracker;
 exports.blockAt = blockAt;
 exports.blockEdit = blockEdit;
 exports.blockRangeEdit = blockRangeEdit;
 exports.buildCodePointIndex = buildCodePointIndex;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
+exports.cleanMarkdown = cleanMarkdown;
 exports.closeFence = closeFence;
 exports.codeSpanText = codeSpanText;
 exports.collectLinkDefinitions = collectLinkDefinitions;
@@ -2899,14 +3322,21 @@ exports.fenceLineKinds = fenceLineKinds;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
 exports.fenceOpenerOf = fenceOpenerOf;
 exports.fenceParts = fenceParts;
+exports.findBalancedXmlClose = findBalancedXmlClose;
 exports.findCodeRanges = findCodeRanges;
 exports.findInlineCodeSpans = findInlineCodeSpans;
 exports.findLinkDefinitionLines = findLinkDefinitionLines;
 exports.findTableEnd = findTableEnd;
 exports.findTableStart = findTableStart;
+exports.findWholeTableStart = findWholeTableStart;
+exports.hasUnclosedBacktickRun = hasUnclosedBacktickRun;
+exports.indexOutsideInlineCode = indexOutsideInlineCode;
+exports.initialXmlBalance = initialXmlBalance;
 exports.isEscapedBracketMath = isEscapedBracketMath;
 exports.isGfmDelimiterRow = isGfmDelimiterRow;
+exports.isGrowingDelimiterRow = isGrowingDelimiterRow;
 exports.isHtmlBlockTagName = isHtmlBlockTagName;
+exports.isInsideInlineCode = isInsideInlineCode;
 exports.isMathPlaceholder = isMathPlaceholder;
 exports.isPipeLedRow = isPipeLedRow;
 exports.isSingleDollarMath = isSingleDollarMath;
@@ -2920,29 +3350,41 @@ exports.mapCodeRanges = mapCodeRanges;
 exports.mapInlineCodeSpans = mapInlineCodeSpans;
 exports.mapPosition = mapPosition;
 exports.mapRange = mapRange;
+exports.markdownToPlainText = markdownToPlainText;
+exports.mathToReadable = mathToReadable;
 exports.normalizeLinkLabel = normalizeLinkLabel;
 exports.opensStrippedHtmlBlock = opensStrippedHtmlBlock;
 exports.opensTable = opensTable;
 exports.pairDisplayMath = pairDisplayMath;
+exports.parseFenceMeta = parseFenceMeta;
 exports.parseFenceOpener = parseFenceOpener;
+exports.parseLineRanges = parseLineRanges;
 exports.pendingCodeSpanStart = pendingCodeSpanStart;
 exports.pendingDisplayMathOpener = pendingDisplayMathOpener;
 exports.readXmlTag = readXmlTag;
+exports.removeCodeSpans = removeCodeSpans;
+exports.replaceFences = replaceFences;
 exports.resolveReferenceLinks = resolveReferenceLinks;
 exports.rowCells = rowCells;
 exports.singleDollarAfter = singleDollarAfter;
 exports.singleDollarMathEnd = singleDollarMathEnd;
+exports.soleFence = soleFence;
 exports.spliceSave = spliceSave;
+exports.splitFenceInfo = splitFenceInfo;
 exports.splitRowSegments = splitRowSegments;
 exports.splitsSurrogatePair = splitsSurrogatePair;
 exports.startsHtmlBlock = startsHtmlBlock;
+exports.startsLikeTableRow = startsLikeTableRow;
 exports.startsPipelessTable = startsPipelessTable;
 exports.tableContainerIndent = tableContainerIndent;
 exports.tableStartsAt = tableStartsAt;
 exports.toCodePointOffset = toCodePointOffset;
 exports.toUtf16Offset = toUtf16Offset;
 exports.tokenizeSource = tokenizeSource;
+exports.trailingTableStart = trailingTableStart;
 exports.trimFenceLine = trimFenceLine;
 exports.unescapeCellPipes = unescapeCellPipes;
+exports.unwrapCodeSpans = unwrapCodeSpans;
+exports.unwrapKindEnvelopes = unwrapKindEnvelopes;
 //# sourceMappingURL=source.cjs.map
 //# sourceMappingURL=source.cjs.map