@swarmvaultai/engine 0.2.1 → 0.3.0

package/dist/index.js

@@ -1729,7 +1729,7 @@ import matter3 from "gray-matter";
 import ignore from "ignore";
 import { JSDOM as JSDOM2 } from "jsdom";
 import mime from "mime-types";
-import TurndownService from "turndown";
+import TurndownService2 from "turndown";
 
 // src/code-analysis.ts
 import fs6 from "fs/promises";
@@ -4481,9 +4481,10 @@ async function analyzeCodeSource(manifest, extractedText, schemaHash) {
   const language = manifest.language ?? inferCodeLanguage(manifest.originalPath ?? manifest.storedPath, manifest.mimeType) ?? "typescript";
   const { code, rationales } = language === "javascript" || language === "jsx" || language === "typescript" || language === "tsx" ? analyzeTypeScriptLikeCode(manifest, extractedText) : await analyzeTreeSitterCode(manifest, extractedText, language);
   return {
-    analysisVersion: 6,
+    analysisVersion: 7,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: manifest.title,
@@ -4503,8 +4504,10 @@ async function analyzeCodeSource(manifest, extractedText, schemaHash) {
 import fs7 from "fs/promises";
 import os from "os";
 import path7 from "path";
+import { parse as parseCsvSync } from "csv-parse/sync";
 import { strFromU8, unzipSync } from "fflate";
 import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
 import { z } from "zod";
 var imageVisionExtractionSchema = z.object({
   title: z.string().min(1).nullable().optional(),
@@ -4684,7 +4687,7 @@ function normalizePdfMetadata(raw) {
 function normalizeDocumentText(raw) {
   return raw.replace(/\r\n/g, "\n").split(/\n{2,}/).map((section) => normalizeWhitespace(section)).filter(Boolean).join("\n\n").trim();
 }
-function parseDocxCoreMetadata(bytes) {
+function parseOfficeCoreMetadata(bytes) {
   try {
     const archive = unzipSync(new Uint8Array(bytes));
     const coreXml = archive["docProps/core.xml"];
@@ -4724,6 +4727,122 @@ function parseDocxCoreMetadata(bytes) {
     return void 0;
   }
 }
+function decodeTextBytes(bytes) {
+  const text = bytes.toString("utf8");
+  return text.charCodeAt(0) === 65279 ? text.slice(1) : text;
+}
+function normalizeTableCell(value) {
+  return normalizeWhitespace(String(value ?? ""));
+}
+function isNumericCell(value) {
+  return value.length > 0 && Number.isFinite(Number(value));
+}
+function detectHeaderRow(rows) {
+  if (!rows.length) {
+    return { headers: [], bodyRows: [] };
+  }
+  const firstRow = rows[0] ?? [];
+  const nonEmpty = firstRow.filter(Boolean);
+  const unique = new Set(nonEmpty);
+  const nonNumeric = nonEmpty.filter((value) => !isNumericCell(value));
+  const looksLikeHeader = nonEmpty.length > 0 && unique.size === nonEmpty.length && nonNumeric.length >= Math.ceil(nonEmpty.length / 2) && rows.length > 1;
+  if (looksLikeHeader) {
+    return {
+      headers: firstRow.map((value, index) => value || `column_${index + 1}`),
+      bodyRows: rows.slice(1)
+    };
+  }
+  const columnCount = Math.max(...rows.map((row) => row.length), 0);
+  return {
+    headers: Array.from({ length: columnCount }, (_, index) => `column_${index + 1}`),
+    bodyRows: rows
+  };
+}
+function columnHints(headers, rows) {
+  return headers.map((header, index) => {
+    const values = rows.map((row) => row[index] ?? "").map(normalizeTableCell).filter(Boolean);
+    if (!values.length) {
+      return null;
+    }
+    const uniqueValues = [...new Set(values)];
+    if (values.every(isNumericCell)) {
+      return `- ${header}: numeric`;
+    }
+    if (uniqueValues.length <= 6 && values.length >= uniqueValues.length) {
+      return `- ${header}: low-cardinality (${uniqueValues.slice(0, 6).join(", ")})`;
+    }
+    return null;
+  }).filter((item) => Boolean(item));
+}
+function markdownTable(headers, rows, rowLimit = 20) {
+  if (!headers.length) {
+    return ["No tabular preview available."];
+  }
+  const width = headers.length;
+  const lines = [`| ${headers.join(" | ")} |`, `| ${headers.map(() => "---").join(" | ")} |`];
+  for (const row of rows.slice(0, rowLimit)) {
+    const normalized = Array.from({ length: width }, (_, index) => normalizeTableCell(row[index] ?? ""));
+    lines.push(`| ${normalized.join(" | ")} |`);
+  }
+  return lines;
+}
+function zipEntryText(archive, entryPath) {
+  const entry = archive[entryPath];
+  return entry ? strFromU8(entry) : void 0;
+}
+function parseXmlDocument(xml) {
+  return new JSDOM(xml, { contentType: "text/xml" }).window.document;
+}
+function zipDirname(value) {
+  const index = value.lastIndexOf("/");
+  return index === -1 ? "" : value.slice(0, index);
+}
+function resolveZipTarget(basePath, target) {
+  return path7.posix.normalize(path7.posix.join(zipDirname(basePath), target));
+}
+function relationshipTargets(xml, basePath) {
+  const document = parseXmlDocument(xml);
+  const map = /* @__PURE__ */ new Map();
+  for (const node of Array.from(document.getElementsByTagName("*"))) {
+    if (node.localName !== "Relationship") {
+      continue;
+    }
+    const id = node.getAttribute("Id")?.trim();
+    const target = node.getAttribute("Target")?.trim();
+    const type = node.getAttribute("Type")?.trim() ?? "";
+    if (!id || !target) {
+      continue;
+    }
+    map.set(id, { target: resolveZipTarget(basePath, target), type });
+  }
+  return map;
+}
+function xmlTextNodes(xml, localName) {
+  const document = parseXmlDocument(xml);
+  const values = [];
+  for (const node of Array.from(document.getElementsByTagName("*"))) {
+    if (node.localName !== localName) {
+      continue;
+    }
+    const text = normalizeWhitespace(node.textContent ?? "");
+    if (text) {
+      values.push(text);
+    }
+  }
+  return values;
+}
+function firstHtmlHeading(html) {
+  const dom = new JSDOM(html);
+  const heading = dom.window.document.querySelector("h1, h2, h3");
+  const title = normalizeWhitespace(heading?.textContent ?? "");
+  return title || void 0;
+}
+function htmlToMarkdown(html) {
+  const dom = new JSDOM(html);
+  const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+  const body = dom.window.document.body?.innerHTML ?? html;
+  return turndown.turndown(body).trim();
+}
 async function extractPdfText(input) {
   try {
     const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
@@ -4781,7 +4900,7 @@ async function extractDocxText(input) {
     const warnings = result.messages.map((message) => normalizeWhitespace(message.message)).filter(Boolean).map((message) => truncate(message, 240));
     const artifact = {
       ...extractionMetadata("docx", input.mimeType, "docx_text"),
-      metadata: parseDocxCoreMetadata(input.bytes),
+      metadata: parseOfficeCoreMetadata(input.bytes),
       warnings: warnings.length ? warnings : void 0
     };
     if (!extractedText) {
@@ -4800,6 +4919,258 @@ async function extractDocxText(input) {
     };
   }
 }
+async function extractCsvText(input) {
+  try {
+    const rawText = decodeTextBytes(input.bytes);
+    const delimiter = input.fileName?.toLowerCase().endsWith(".tsv") || input.mimeType.includes("tab-separated") ? "	" : ",";
+    const parsed = parseCsvSync(rawText, {
+      delimiter,
+      relax_column_count: true,
+      skip_empty_lines: true,
+      trim: true
+    });
+    const rows = parsed.map((row) => row.map((value) => normalizeTableCell(value)));
+    const { headers, bodyRows } = detectHeaderRow(rows);
+    const hintLines = columnHints(headers, bodyRows);
+    const title = input.fileName ? path7.basename(input.fileName, path7.extname(input.fileName)) : void 0;
+    const extractedText = [
+      title ? `# ${title}` : null,
+      `Format: ${delimiter === "	" ? "TSV" : "CSV"}`,
+      `Rows: ${bodyRows.length}`,
+      `Columns: ${headers.length}`,
+      headers.length ? `Headers: ${headers.join(", ")}` : null,
+      "",
+      hintLines.length ? "## Column Hints" : null,
+      hintLines.length ? hintLines.join("\n") : null,
+      hintLines.length ? "" : null,
+      "## Preview",
+      ...markdownTable(headers, bodyRows)
+    ].filter((item) => Boolean(item)).join("\n").trim();
+    const artifact = {
+      ...extractionMetadata("csv", input.mimeType, "csv_text"),
+      metadata: {
+        format: delimiter === "	" ? "tsv" : "csv",
+        row_count: String(bodyRows.length),
+        column_count: String(headers.length),
+        headers: headers.join(", ")
+      }
+    };
+    return {
+      title,
+      extractedText,
+      artifact
+    };
+  } catch (error) {
+    return {
+      artifact: {
+        ...extractionMetadata("csv", input.mimeType, "csv_text"),
+        warnings: [`CSV extraction failed: ${error instanceof Error ? truncate(error.message, 240) : "unknown error"}`]
+      }
+    };
+  }
+}
+async function extractXlsxText(input) {
+  try {
+    const XLSX = await import("xlsx");
+    const workbook = XLSX.read(input.bytes, { type: "buffer", cellFormula: false, cellHTML: false, cellStyles: false });
+    const allSheetNames = workbook.SheetNames;
+    const sheetNames = allSheetNames.slice(0, 10);
+    const sheetSections = [];
+    const metadata = {
+      sheet_count: String(allSheetNames.length),
+      sheet_names: allSheetNames.join(", ")
+    };
+    for (const sheetName of sheetNames) {
+      const sheet = workbook.Sheets[sheetName];
+      if (!sheet) {
+        continue;
+      }
+      const rows = XLSX.utils.sheet_to_json(sheet, {
+        header: 1,
+        raw: false,
+        defval: ""
+      }).map((row) => row.map((value) => normalizeTableCell(value)));
+      const { headers, bodyRows } = detectHeaderRow(rows);
+      sheetSections.push(`## Sheet: ${sheetName}`);
+      sheetSections.push(`Rows: ${bodyRows.length}`);
+      sheetSections.push(`Columns: ${headers.length}`);
+      sheetSections.push(...markdownTable(headers, bodyRows));
+      sheetSections.push("");
+    }
+    const title = normalizeWhitespace(String(workbook.Props?.Title ?? "")) || (input.fileName ? path7.basename(input.fileName, path7.extname(input.fileName)) : void 0);
+    const extractedText = [
+      title ? `# ${title}` : null,
+      `Sheets: ${allSheetNames.length}`,
+      allSheetNames.length ? `Sheet Names: ${allSheetNames.join(", ")}` : null,
+      "",
+      ...sheetSections
+    ].filter((item) => Boolean(item)).join("\n").trim();
+    const warnings = allSheetNames.length > sheetNames.length ? ["Workbook preview truncated to the first 10 sheets."] : void 0;
+    return {
+      title,
+      extractedText,
+      artifact: {
+        ...extractionMetadata("xlsx", input.mimeType, "xlsx_text"),
+        metadata,
+        warnings
+      }
+    };
+  } catch (error) {
+    return {
+      artifact: {
+        ...extractionMetadata("xlsx", input.mimeType, "xlsx_text"),
+        warnings: [`XLSX extraction failed: ${error instanceof Error ? truncate(error.message, 240) : "unknown error"}`]
+      }
+    };
+  }
+}
+async function extractPptxText(input) {
+  try {
+    const archive = unzipSync(new Uint8Array(input.bytes));
+    const presentationXml = zipEntryText(archive, "ppt/presentation.xml");
+    if (!presentationXml) {
+      throw new Error("Missing ppt/presentation.xml");
+    }
+    const relsXml = zipEntryText(archive, "ppt/_rels/presentation.xml.rels");
+    if (!relsXml) {
+      throw new Error("Missing ppt/_rels/presentation.xml.rels");
+    }
+    const rels = relationshipTargets(relsXml, "ppt/presentation.xml");
+    const document = parseXmlDocument(presentationXml);
+    const slideTargets = Array.from(document.getElementsByTagName("*")).filter((node) => node.localName === "sldId").map((node) => node.getAttribute("r:id")?.trim()).filter((value) => Boolean(value)).map((relationshipId) => rels.get(relationshipId)?.target).filter((value) => Boolean(value)).slice(0, 60);
+    const slideSections = [];
+    for (let index = 0; index < slideTargets.length; index += 1) {
+      const slidePath = slideTargets[index];
+      const slideXml = zipEntryText(archive, slidePath);
+      if (!slideXml) {
+        continue;
+      }
+      const slideTexts = xmlTextNodes(slideXml, "t");
+      const slideTitle = slideTexts[0] ?? `Slide ${index + 1}`;
+      slideSections.push(`## Slide ${index + 1}: ${slideTitle}`);
+      if (slideTexts.length) {
+        slideSections.push(slideTexts.join("\n"));
+      }
+      const slideRelsPath = `${zipDirname(slidePath)}/_rels/${path7.posix.basename(slidePath)}.rels`;
+      const slideRelsXml = zipEntryText(archive, slideRelsPath);
+      if (slideRelsXml) {
+        const slideRels = relationshipTargets(slideRelsXml, slidePath);
+        const notesTarget = [...slideRels.values()].find((entry) => entry.type.endsWith("/notesSlide"))?.target;
+        if (notesTarget) {
+          const notesXml = zipEntryText(archive, notesTarget);
+          const noteTexts = notesXml ? xmlTextNodes(notesXml, "t") : [];
+          if (noteTexts.length) {
+            slideSections.push("Notes:");
+            slideSections.push(noteTexts.join("\n"));
+          }
+        }
+      }
+      slideSections.push("");
+    }
+    const metadata = parseOfficeCoreMetadata(input.bytes);
+    const title = metadata?.title || (input.fileName ? path7.basename(input.fileName, path7.extname(input.fileName)) : void 0);
+    const extractedText = [title ? `# ${title}` : null, `Slides: ${slideTargets.length}`, "", ...slideSections].filter((item) => Boolean(item)).join("\n").trim();
+    return {
+      title,
+      extractedText,
+      artifact: {
+        ...extractionMetadata("pptx", input.mimeType, "pptx_text"),
+        metadata: {
+          ...metadata ?? {},
+          slide_count: String(slideTargets.length)
+        },
+        warnings: Array.from(document.getElementsByTagName("*")).filter((node) => node.localName === "sldId").length > slideTargets.length ? ["Slide extraction truncated to the first 60 slides."] : void 0
+      }
+    };
+  } catch (error) {
+    return {
+      artifact: {
+        ...extractionMetadata("pptx", input.mimeType, "pptx_text"),
+        warnings: [`PPTX extraction failed: ${error instanceof Error ? truncate(error.message, 240) : "unknown error"}`]
+      }
+    };
+  }
+}
+async function extractEpubChapters(input) {
+  try {
+    const archive = unzipSync(new Uint8Array(input.bytes));
+    const containerXml = zipEntryText(archive, "META-INF/container.xml");
+    if (!containerXml) {
+      throw new Error("Missing META-INF/container.xml");
+    }
+    const container = parseXmlDocument(containerXml);
+    const rootfile = Array.from(container.getElementsByTagName("*")).find((node) => node.localName === "rootfile");
+    const packagePath = rootfile?.getAttribute("full-path")?.trim();
+    if (!packagePath) {
+      throw new Error("EPUB container did not declare a package document.");
+    }
+    const packageXml = zipEntryText(archive, packagePath);
+    if (!packageXml) {
+      throw new Error(`Missing EPUB package document: ${packagePath}`);
+    }
+    const packageDocument = parseXmlDocument(packageXml);
+    const manifestEntries = new Map(
+      Array.from(packageDocument.getElementsByTagName("*")).filter((node) => node.localName === "item").map(
+        (node) => [
+          node.getAttribute("id")?.trim() ?? "",
+          {
+            href: node.getAttribute("href")?.trim() ?? "",
+            mediaType: node.getAttribute("media-type")?.trim() ?? "",
+            properties: node.getAttribute("properties")?.trim() ?? ""
+          }
+        ]
+      ).filter(([id, item]) => Boolean(id && item.href))
+    );
+    const spineIds = Array.from(packageDocument.getElementsByTagName("*")).filter((node) => node.localName === "itemref").map((node) => node.getAttribute("idref")?.trim()).filter((value) => Boolean(value));
+    const bookTitle = xmlTextNodes(packageXml, "title")[0] || (input.fileName ? path7.basename(input.fileName, path7.extname(input.fileName)) : void 0);
+    const author = xmlTextNodes(packageXml, "creator")[0];
+    const chapters = [];
+    for (const spineId of spineIds) {
+      const item = manifestEntries.get(spineId);
+      if (!item || !item.mediaType.includes("html") && !item.mediaType.includes("xhtml")) {
+        continue;
+      }
+      if (item.properties.split(/\s+/).includes("nav")) {
+        continue;
+      }
+      const entryPath = resolveZipTarget(packagePath, item.href);
+      const html = zipEntryText(archive, entryPath);
+      if (!html) {
+        continue;
+      }
+      const markdown = htmlToMarkdown(html);
+      if (!markdown) {
+        continue;
+      }
+      const chapterTitle = firstHtmlHeading(html) || markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() || item.href;
+      const normalizedTitle = normalizeWhitespace(chapterTitle);
+      if (!normalizedTitle || /^table of contents$/i.test(normalizedTitle)) {
+        continue;
+      }
+      chapters.push({
+        partKey: item.href,
+        title: normalizedTitle,
+        markdown,
+        metadata: {
+          book_title: bookTitle ?? "",
+          chapter_title: normalizedTitle,
+          author: author ?? ""
+        }
+      });
+    }
+    return {
+      title: bookTitle,
+      author,
+      chapters,
+      warnings: chapters.length ? void 0 : ["EPUB extraction completed but found no chapter-like spine entries."]
+    };
+  } catch (error) {
+    return {
+      chapters: [],
+      warnings: [`EPUB extraction failed: ${error instanceof Error ? truncate(error.message, 240) : "unknown error"}`]
+    };
+  }
+}
 
 // src/logs.ts
 import fs8 from "fs/promises";
@@ -5208,6 +5579,17 @@ var HARD_REPO_IGNORES = /* @__PURE__ */ new Set([".git", ".venv"]);
 var PROGRESS_FILE_THRESHOLD = 150;
 var PROGRESS_UPDATE_INTERVAL = 100;
 var RST_HEADING_MARKERS = /* @__PURE__ */ new Set(["=", "-", "~", "^", '"', "#", "*", "+"]);
+var MARKDOWN_SEMANTIC_FRONTMATTER_KEYS = [
+  "title",
+  "summary",
+  "description",
+  "aliases",
+  "tags",
+  "authors",
+  "published_at",
+  "canonical_url",
+  "source_type"
+];
 function uniqueStrings(values) {
   return [...new Set(values.filter(Boolean))];
 }
@@ -5224,15 +5606,27 @@ function inferKind(mimeType, filePath) {
   if (mimeType.includes("html")) {
     return "html";
   }
-  if (mimeType.startsWith("text/")) {
-    return "text";
-  }
   if (mimeType === "application/pdf" || filePath.toLowerCase().endsWith(".pdf")) {
     return "pdf";
   }
   if (mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || filePath.toLowerCase().endsWith(".docx")) {
     return "docx";
   }
+  if (mimeType === "application/epub+zip" || filePath.toLowerCase().endsWith(".epub")) {
+    return "epub";
+  }
+  if (mimeType === "text/csv" || mimeType === "text/tab-separated-values" || filePath.toLowerCase().endsWith(".csv") || filePath.toLowerCase().endsWith(".tsv")) {
+    return "csv";
+  }
+  if (mimeType.startsWith("text/")) {
+    return "text";
+  }
+  if (mimeType === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" || filePath.toLowerCase().endsWith(".xlsx")) {
+    return "xlsx";
+  }
+  if (mimeType === "application/vnd.openxmlformats-officedocument.presentationml.presentation" || filePath.toLowerCase().endsWith(".pptx")) {
+    return "pptx";
+  }
   if (mimeType.startsWith("image/")) {
     return "image";
   }
@@ -5258,6 +5652,10 @@ function guessMimeType(target) {
   }
   return mime.lookup(target) || "application/octet-stream";
 }
+function sourceGroupIdFor(prepared) {
+  const originKey = prepared.originType === "url" ? prepared.url ?? prepared.title : prepared.originalPath ?? prepared.title;
+  return `${slugify(prepared.title)}-${sha256(originKey).slice(0, 8)}`;
+}
 function rstAdornmentLine(line) {
   const trimmed = line.trim();
   if (trimmed.length < 3) {
@@ -5345,6 +5743,65 @@ function extractedTextForPlainSource(filePath, sourceKind, content) {
   }
   return content;
 }
+function normalizeSemanticMarkdownScalar(value) {
+  if (typeof value !== "string") {
+    return void 0;
+  }
+  const normalized = normalizeWhitespace(value.trim());
+  return normalized || void 0;
+}
+function normalizeSemanticMarkdownList(value) {
+  if (!Array.isArray(value)) {
+    return void 0;
+  }
+  const items = uniqueStrings(
+    value.flatMap((item) => typeof item === "string" ? [normalizeWhitespace(item.trim())] : []).filter(Boolean)
+  );
+  return items.length ? items : void 0;
+}
+function semanticMarkdownTitle(fallback, content, filePath) {
+  const parsed = matter3(content);
+  const frontmatterTitle = normalizeSemanticMarkdownScalar(parsed.data.title);
+  if (frontmatterTitle) {
+    return frontmatterTitle;
+  }
+  return titleFromText(fallback, parsed.content, filePath);
+}
+function semanticMarkdownContent(content) {
+  const parsed = matter3(content);
+  const body = parsed.content.replace(/\r\n?/g, "\n").trim();
+  const semanticFrontmatter = Object.fromEntries(
+    MARKDOWN_SEMANTIC_FRONTMATTER_KEYS.flatMap((key) => {
+      const value = key === "aliases" || key === "tags" || key === "authors" ? normalizeSemanticMarkdownList(parsed.data[key]) : normalizeSemanticMarkdownScalar(parsed.data[key]);
+      return value === void 0 ? [] : [[key, value]];
+    })
+  );
+  const semanticLines = Object.entries(semanticFrontmatter).map(
+    ([key, value]) => `${key}: ${Array.isArray(value) ? value.join(", ") : value}`
+  );
+  const extractedText = [...semanticLines, ...semanticLines.length && body ? [""] : [], body].filter(Boolean).join("\n").trim();
+  return {
+    extractedText,
+    semanticHash: sha256(
+      JSON.stringify({
+        body,
+        frontmatter: semanticFrontmatter
+      })
+    )
+  };
+}
+function finalizePreparedInput(prepared) {
+  if (prepared.sourceKind !== "markdown") {
+    return prepared;
+  }
+  const semantic = semanticMarkdownContent(prepared.payloadBytes.toString("utf8"));
+  return {
+    ...prepared,
+    extractedText: semantic.extractedText,
+    extractionHash: buildExtractionHash(semantic.extractedText, prepared.extractionArtifact),
+    semanticHash: semantic.semanticHash
+  };
+}
 function shouldEmitProgress(totalItems) {
   return totalItems >= PROGRESS_FILE_THRESHOLD && Boolean(process.stderr?.isTTY);
 }
@@ -5511,7 +5968,7 @@ function markdownFrontmatter(value) {
   return matter3.stringify("", normalized).trimEnd().split("\n").concat([""]);
 }
 function prepareCapturedMarkdownInput(input) {
-  return {
+  return finalizePreparedInput({
     title: input.title,
     originType: "url",
     sourceKind: "markdown",
@@ -5523,7 +5980,7 @@ function prepareCapturedMarkdownInput(input) {
     extractedText: input.markdown,
     attachments: input.attachments,
     logDetails: input.logDetails
-  };
+  });
 }
 function isPrivateIp(ip) {
   if (ip === "::1" || ip.startsWith("fc") || ip.startsWith("fd")) return true;
@@ -5773,6 +6230,9 @@ function manifestMatchesOrigin(manifest, prepared) {
   }
   return Boolean(prepared.originalPath && manifest.originalPath && toPosix(manifest.originalPath) === toPosix(prepared.originalPath));
 }
+function manifestMatchesOriginPart(manifest, prepared) {
+  return manifestMatchesOrigin(manifest, prepared) && (manifest.sourcePartKey ?? "") === (prepared.sourcePartKey ?? "");
+}
 function buildCompositeHash(payloadBytes, attachments = []) {
   if (!attachments.length) {
     return sha256(payloadBytes);
@@ -5870,7 +6330,7 @@ function extractMarkdownImageReferences(content, baseUrl) {
 async function convertHtmlToMarkdown(html, url) {
   const dom = new JSDOM2(html, { url });
   const article = new Readability(dom.window.document).parse();
-  const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+  const turndown = new TurndownService2({ headingStyle: "atx", codeBlockStyle: "fenced" });
   const body = article?.content ?? dom.window.document.body.innerHTML;
   const markdown = turndown.turndown(body);
   return {
@@ -5886,23 +6346,34 @@ async function readManifestByHash(manifestsDir, contentHash) {
     }
     const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
     if (manifest?.contentHash === contentHash) {
-      return manifest;
+      return {
+        ...manifest,
+        semanticHash: manifest.semanticHash ?? manifest.contentHash
+      };
     }
   }
   return null;
 }
-async function readManifestByOrigin(manifestsDir, prepared) {
+async function readManifestsByOrigin(manifestsDir, prepared) {
   const entries = await fs11.readdir(manifestsDir, { withFileTypes: true }).catch(() => []);
+  const manifests = [];
   for (const entry of entries) {
     if (!entry.isFile() || !entry.name.endsWith(".json")) {
       continue;
     }
     const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
     if (manifest && manifestMatchesOrigin(manifest, prepared)) {
-      return manifest;
+      manifests.push({
+        ...manifest,
+        semanticHash: manifest.semanticHash ?? manifest.contentHash
+      });
     }
   }
-  return null;
+  return manifests;
+}
+async function readManifestByOrigin(manifestsDir, prepared) {
+  const manifests = await readManifestsByOrigin(manifestsDir, prepared);
+  return manifests.find((manifest) => manifestMatchesOriginPart(manifest, prepared)) ?? null;
 }
 async function loadGitignoreMatcher(repoRoot, enabled) {
   if (!enabled) {
@@ -6148,10 +6619,11 @@ async function persistPreparedInput(rootDir, prepared, paths) {
   await ensureDir(paths.extractsDir);
   const attachments = prepared.attachments ?? [];
   const contentHash = prepared.contentHash ?? buildCompositeHash(prepared.payloadBytes, attachments);
+  const semanticHash = prepared.semanticHash ?? contentHash;
   const extractionHash = prepared.extractionHash ?? buildExtractionHash(prepared.extractedText, prepared.extractionArtifact);
   const existingByOrigin = await readManifestByOrigin(paths.manifestsDir, prepared);
-  const existingByHash = existingByOrigin ? null : await readManifestByHash(paths.manifestsDir, contentHash);
-  if (existingByOrigin && existingByOrigin.contentHash === contentHash && existingByOrigin.extractionHash === extractionHash && existingByOrigin.title === prepared.title && existingByOrigin.sourceKind === prepared.sourceKind && existingByOrigin.sourceType === prepared.sourceType && existingByOrigin.sourceClass === prepared.sourceClass && existingByOrigin.language === prepared.language && existingByOrigin.mimeType === prepared.mimeType && existingByOrigin.repoRelativePath === prepared.repoRelativePath) {
+  const existingByHash = existingByOrigin || prepared.sourcePartKey ? null : await readManifestByHash(paths.manifestsDir, contentHash);
+  if (existingByOrigin && existingByOrigin.contentHash === contentHash && existingByOrigin.semanticHash === semanticHash && existingByOrigin.extractionHash === extractionHash && existingByOrigin.title === prepared.title && existingByOrigin.sourceKind === prepared.sourceKind && existingByOrigin.sourceType === prepared.sourceType && existingByOrigin.sourceClass === prepared.sourceClass && existingByOrigin.language === prepared.language && existingByOrigin.mimeType === prepared.mimeType && existingByOrigin.repoRelativePath === prepared.repoRelativePath && existingByOrigin.sourceGroupId === prepared.sourceGroupId && existingByOrigin.sourceGroupTitle === prepared.sourceGroupTitle && existingByOrigin.sourcePartKey === prepared.sourcePartKey && existingByOrigin.partIndex === prepared.partIndex && existingByOrigin.partCount === prepared.partCount && existingByOrigin.partTitle === prepared.partTitle && JSON.stringify(existingByOrigin.details ?? {}) === JSON.stringify(prepared.details ?? {})) {
     return { manifest: existingByOrigin, isNew: false, wasUpdated: false };
   }
   if (existingByHash) {
@@ -6209,6 +6681,14 @@ async function persistPreparedInput(rootDir, prepared, paths) {
     extractionHash,
     mimeType: prepared.mimeType,
     contentHash,
+    semanticHash,
+    sourceGroupId: prepared.sourceGroupId,
+    sourceGroupTitle: prepared.sourceGroupTitle,
+    sourcePartKey: prepared.sourcePartKey,
+    partIndex: prepared.partIndex,
+    partCount: prepared.partCount,
+    partTitle: prepared.partTitle,
+    details: prepared.details,
     createdAt: previous?.createdAt ?? now,
     updatedAt: now,
     attachments: manifestAttachments.length ? manifestAttachments : void 0
@@ -6230,6 +6710,42 @@ async function persistPreparedInput(rootDir, prepared, paths) {
   }
   return { manifest, isNew: !previous, wasUpdated: Boolean(previous) };
 }
+async function persistPreparedInputs(rootDir, input, preparedInputs, paths) {
+  const template = preparedInputs[0];
+  const existingByOrigin = template ? await readManifestsByOrigin(paths.manifestsDir, template) : [];
+  const created = [];
+  const updated = [];
+  const unchanged = [];
+  const removed = [];
+  const seenSourceIds = /* @__PURE__ */ new Set();
+  for (const prepared of preparedInputs) {
+    const result = await persistPreparedInput(rootDir, prepared, paths);
+    if (result.isNew) {
+      created.push(result.manifest);
+    } else if (result.wasUpdated) {
+      updated.push(result.manifest);
+    } else {
+      unchanged.push(result.manifest);
+    }
+    seenSourceIds.add(result.manifest.sourceId);
+  }
+  for (const manifest of existingByOrigin) {
+    if (seenSourceIds.has(manifest.sourceId)) {
+      continue;
+    }
+    await removeManifestArtifacts(rootDir, manifest, paths);
+    removed.push(manifest);
+  }
+  return {
+    input,
+    scannedCount: preparedInputs.length,
+    created,
+    updated,
+    unchanged,
+    removed,
+    skipped: []
+  };
+}
 async function removeManifestArtifacts(rootDir, manifest, paths) {
   await fs11.rm(path12.join(paths.manifestsDir, `${manifest.sourceId}.json`), { force: true });
   await fs11.rm(path12.resolve(rootDir, manifest.storedPath), { force: true });
@@ -6256,10 +6772,10 @@ function repoSyncWorkspaceIgnorePaths(rootDir, paths, repoRoot) {
   return candidates.map((candidate) => path12.resolve(candidate)).filter((candidate, index, items) => items.indexOf(candidate) === index).filter((candidate) => withinRoot(repoRoot, candidate));
 }
 function preparedMatchesManifest(manifest, prepared, contentHash) {
-  return manifest.contentHash === contentHash && manifest.extractionHash === (prepared.extractionHash ?? buildExtractionHash(prepared.extractedText, prepared.extractionArtifact)) && manifest.title === prepared.title && manifest.sourceKind === prepared.sourceKind && manifest.sourceType === prepared.sourceType && manifest.sourceClass === prepared.sourceClass && manifest.language === prepared.language && manifest.mimeType === prepared.mimeType && manifest.repoRelativePath === prepared.repoRelativePath;
+  return manifest.contentHash === contentHash && manifest.extractionHash === (prepared.extractionHash ?? buildExtractionHash(prepared.extractedText, prepared.extractionArtifact)) && manifest.semanticHash === (prepared.semanticHash ?? contentHash) && manifest.title === prepared.title && manifest.sourceKind === prepared.sourceKind && manifest.sourceType === prepared.sourceType && manifest.sourceClass === prepared.sourceClass && manifest.language === prepared.language && manifest.mimeType === prepared.mimeType && manifest.repoRelativePath === prepared.repoRelativePath && manifest.sourceGroupId === prepared.sourceGroupId && manifest.sourceGroupTitle === prepared.sourceGroupTitle && manifest.sourcePartKey === prepared.sourcePartKey && manifest.partIndex === prepared.partIndex && manifest.partCount === prepared.partCount && manifest.partTitle === prepared.partTitle && JSON.stringify(manifest.details ?? {}) === JSON.stringify(prepared.details ?? {});
 }
 function shouldDeferWatchSemanticRefresh(sourceKind) {
-  return sourceKind === "markdown" || sourceKind === "text" || sourceKind === "html" || sourceKind === "pdf" || sourceKind === "docx" || sourceKind === "image";
+  return sourceKind === "markdown" || sourceKind === "text" || sourceKind === "html" || sourceKind === "pdf" || sourceKind === "docx" || sourceKind === "epub" || sourceKind === "csv" || sourceKind === "xlsx" || sourceKind === "pptx" || sourceKind === "image";
 }
 function pendingSemanticRefreshId(changeType, repoRoot, relativePath) {
   return `pending:${changeType}:${sha256(`${toPosix(repoRoot)}:${relativePath}`).slice(0, 12)}`;
@@ -6325,13 +6841,16 @@ async function syncTrackedRepos(rootDir, options, repoRoots) {
     const currentPaths = new Set(files.map((absolutePath) => path12.resolve(absolutePath)));
     for (const absolutePath of files) {
       const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
-      const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
-      const result = await persistPreparedInput(rootDir, prepared, paths);
-      if (result.isNew) {
-        imported.push(result.manifest);
-      } else if (result.wasUpdated) {
-        updated.push(result.manifest);
-      }
+      const preparedInputs = await prepareFileInputs(
+        rootDir,
+        absolutePath,
+        repoRoot,
+        sourceClassForRelativePath(relativePath, normalizedOptions)
+      );
+      const result = await persistPreparedInputs(rootDir, absolutePath, preparedInputs, paths);
+      imported.push(...result.created);
+      updated.push(...result.updated);
+      removed.push(...result.removed);
       progress.tick();
     }
     progress.finish(`repo=${toPosix(path12.relative(rootDir, repoRoot)) || "."}`);
@@ -6390,9 +6909,6 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
   let scannedCount = 0;
   for (const repoRoot of uniqueRoots) {
     const repoManifests = manifestsByRepoRoot.get(repoRoot) ?? [];
-    const manifestsByOriginalPath = new Map(
-      repoManifests.filter((manifest) => manifest.originalPath).map((manifest) => [path12.resolve(manifest.originalPath), manifest])
-    );
     if (!await fileExists(repoRoot)) {
       for (const manifest of repoManifests) {
         if (shouldDeferWatchSemanticRefresh(manifest.sourceKind)) {
@@ -6428,38 +6944,50 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
     const currentPaths = new Set(files.map((absolutePath) => path12.resolve(absolutePath)));
     for (const absolutePath of files) {
       const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
-      const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
-      if (shouldDeferWatchSemanticRefresh(prepared.sourceKind)) {
-        const existing = manifestsByOriginalPath.get(path12.resolve(absolutePath));
-        const contentHash = buildCompositeHash(prepared.payloadBytes, prepared.attachments);
-        const changed = !existing || !preparedMatchesManifest(existing, prepared, contentHash);
+      const preparedInputs = await prepareFileInputs(
+        rootDir,
+        absolutePath,
+        repoRoot,
+        sourceClassForRelativePath(relativePath, normalizedOptions)
+      );
+      const firstPrepared = preparedInputs[0];
+      if (firstPrepared && shouldDeferWatchSemanticRefresh(firstPrepared.sourceKind)) {
+        const existing = repoManifests.filter(
+          (manifest) => manifest.originalPath && path12.resolve(manifest.originalPath) === path12.resolve(absolutePath)
+        );
+        const existingByPartKey = new Map(existing.map((manifest) => [manifest.sourcePartKey ?? "__single__", manifest]));
+        const changed = existing.length !== preparedInputs.length || preparedInputs.some((prepared) => {
+          const match = existingByPartKey.get(prepared.sourcePartKey ?? "__single__");
+          const contentHash = buildCompositeHash(prepared.payloadBytes, prepared.attachments);
+          return !match || !preparedMatchesManifest(match, prepared, contentHash);
+        }) || existing.some(
+          (manifest) => !preparedInputs.some((prepared) => (prepared.sourcePartKey ?? "") === (manifest.sourcePartKey ?? ""))
+        );
         if (changed) {
           pendingSemanticRefresh.push({
             id: pendingSemanticRefreshId(
-              existing ? "modified" : "added",
+              existing.length ? "modified" : "added",
               repoRoot,
-              prepared.repoRelativePath ?? toPosix(path12.relative(repoRoot, absolutePath))
+              firstPrepared.repoRelativePath ?? toPosix(path12.relative(repoRoot, absolutePath))
             ),
             repoRoot,
             path: toPosix(path12.relative(rootDir, absolutePath)),
-            changeType: existing ? "modified" : "added",
+            changeType: existing.length ? "modified" : "added",
             detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
-            sourceId: existing?.sourceId,
-            sourceKind: prepared.sourceKind
+            sourceId: existing[0]?.sourceId,
+            sourceKind: firstPrepared.sourceKind
           });
-          if (existing?.sourceId) {
-            staleSourceIds.add(existing.sourceId);
+          for (const manifest of existing) {
+            staleSourceIds.add(manifest.sourceId);
           }
         }
         progress.tick();
         continue;
       }
-      const result = await persistPreparedInput(rootDir, prepared, paths);
-      if (result.isNew) {
-        imported.push(result.manifest);
-      } else if (result.wasUpdated) {
-        updated.push(result.manifest);
-      }
+      const result = await persistPreparedInputs(rootDir, absolutePath, preparedInputs, paths);
+      imported.push(...result.created);
+      updated.push(...result.updated);
+      removed.push(...result.removed);
       progress.tick();
     }
     progress.finish(`repo=${toPosix(path12.relative(rootDir, repoRoot)) || "."}`);
@@ -6513,7 +7041,7 @@ async function syncTrackedReposForWatch(rootDir, options, repoRoots) {
     staleSourceIds: [...staleSourceIds]
   };
 }
-async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
+async function prepareFileInputs(rootDir, absoluteInput, repoRoot, sourceClass) {
   const payloadBytes = await fs11.readFile(absoluteInput);
   const mimeType = guessMimeType(absoluteInput);
   const sourceKind = inferKind(mimeType, absoluteInput);
@@ -6523,14 +7051,15 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
   let extractedText;
   let extractionArtifact;
   if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-    extractedText = extractedTextForPlainSource(absoluteInput, sourceKind, payloadBytes.toString("utf8"));
-    title = titleFromText(path12.basename(absoluteInput, path12.extname(absoluteInput)), extractedText, absoluteInput);
+    const rawText = payloadBytes.toString("utf8");
+    extractedText = sourceKind === "markdown" ? semanticMarkdownContent(rawText).extractedText : extractedTextForPlainSource(absoluteInput, sourceKind, rawText);
+    title = sourceKind === "markdown" ? semanticMarkdownTitle(path12.basename(absoluteInput, path12.extname(absoluteInput)), rawText, absoluteInput) : titleFromText(path12.basename(absoluteInput, path12.extname(absoluteInput)), extractedText, absoluteInput);
     extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
   } else if (sourceKind === "html") {
     const html = payloadBytes.toString("utf8");
     const converted = await convertHtmlToMarkdown(html, pathToFileURL(absoluteInput).toString());
     title = converted.title;
-    extractedText = converted.markdown;
+    extractedText = semanticMarkdownContent(converted.markdown).extractedText;
     extractionArtifact = createHtmlReadabilityExtractionArtifact(sourceKind, mimeType);
   } else if (sourceKind === "pdf") {
     title = path12.basename(absoluteInput, path12.extname(absoluteInput));
@@ -6543,6 +7072,94 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
     title = extracted.artifact.metadata?.title?.trim() || title;
     extractedText = extracted.extractedText;
     extractionArtifact = extracted.artifact;
+  } else if (sourceKind === "csv") {
+    title = path12.basename(absoluteInput, path12.extname(absoluteInput));
+    const extracted = await extractCsvText({ mimeType, bytes: payloadBytes, fileName: absoluteInput });
+    title = extracted.title?.trim() || title;
+    extractedText = extracted.extractedText;
+    extractionArtifact = extracted.artifact;
+  } else if (sourceKind === "xlsx") {
+    title = path12.basename(absoluteInput, path12.extname(absoluteInput));
+    const extracted = await extractXlsxText({ mimeType, bytes: payloadBytes, fileName: absoluteInput });
+    title = extracted.title?.trim() || title;
+    extractedText = extracted.extractedText;
+    extractionArtifact = extracted.artifact;
+  } else if (sourceKind === "pptx") {
+    title = path12.basename(absoluteInput, path12.extname(absoluteInput));
+    const extracted = await extractPptxText({ mimeType, bytes: payloadBytes, fileName: absoluteInput });
+    title = extracted.title?.trim() || title;
+    extractedText = extracted.extractedText;
+    extractionArtifact = extracted.artifact;
+  } else if (sourceKind === "epub") {
+    title = path12.basename(absoluteInput, path12.extname(absoluteInput));
+    const extracted = await extractEpubChapters({ mimeType, bytes: payloadBytes, fileName: absoluteInput });
+    title = extracted.title?.trim() || title;
+    const groupId = sourceGroupIdFor({
+      title,
+      originType: "file",
+      originalPath: toPosix(absoluteInput)
+    });
+    if (extracted.chapters.length) {
+      return extracted.chapters.map(
+        (chapter, index) => finalizePreparedInput({
+          title: `${title} - ${chapter.title}`,
+          originType: "file",
+          sourceKind: "epub",
+          sourceClass,
+          originalPath: toPosix(absoluteInput),
+          repoRelativePath: repoRelativePathFor(absoluteInput, repoRoot),
+          mimeType: "text/markdown",
+          storedExtension: ".md",
+          payloadBytes: Buffer.from(chapter.markdown, "utf8"),
+          extractedText: chapter.markdown,
+          extractionArtifact: {
+            extractor: "epub_text",
+            sourceKind: "epub",
+            mimeType,
+            producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+            metadata: {
+              ...chapter.metadata,
+              chapter_index: String(index + 1),
+              chapter_count: String(extracted.chapters.length)
+            },
+            warnings: extracted.warnings
+          },
+          extractionHash: buildExtractionHash(chapter.markdown, {
+            extractor: "epub_text",
+            sourceKind: "epub",
+            mimeType,
+            producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+            metadata: {
+              ...chapter.metadata,
+              chapter_index: String(index + 1),
+              chapter_count: String(extracted.chapters.length)
+            },
+            warnings: extracted.warnings
+          }),
+          sourceGroupId: groupId,
+          sourceGroupTitle: title,
+          sourcePartKey: chapter.partKey,
+          partIndex: index + 1,
+          partCount: extracted.chapters.length,
+          partTitle: chapter.title,
+          details: {
+            book_title: title,
+            chapter_title: chapter.title,
+            chapter_index: String(index + 1),
+            chapter_count: String(extracted.chapters.length),
+            ...extracted.author ? { author: extracted.author } : {}
+          }
+        })
+      );
+    }
+    extractedText = void 0;
+    extractionArtifact = {
+      extractor: "epub_text",
+      sourceKind: "epub",
+      mimeType,
+      producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      warnings: extracted.warnings ?? ["EPUB extraction completed but produced no chapter content."]
+    };
   } else if (sourceKind === "image") {
     title = path12.basename(absoluteInput, path12.extname(absoluteInput));
     const extracted = await extractImageWithVision(rootDir, {
@@ -6556,23 +7173,33 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
   } else {
     title = path12.basename(absoluteInput, path12.extname(absoluteInput));
   }
-  return {
-    title,
-    originType: "file",
-    sourceKind,
-    sourceClass,
-    language,
-    originalPath: toPosix(absoluteInput),
-    repoRelativePath: repoRelativePathFor(absoluteInput, repoRoot),
-    mimeType,
-    storedExtension,
-    payloadBytes,
-    extractedText,
-    extractionArtifact,
-    extractionHash: buildExtractionHash(extractedText, extractionArtifact)
-  };
+  return [
+    finalizePreparedInput({
+      title,
+      originType: "file",
+      sourceKind,
+      sourceClass,
+      language,
+      originalPath: toPosix(absoluteInput),
+      repoRelativePath: repoRelativePathFor(absoluteInput, repoRoot),
+      mimeType,
+      storedExtension,
+      payloadBytes,
+      extractedText,
+      extractionArtifact,
+      extractionHash: buildExtractionHash(extractedText, extractionArtifact),
+      details: extractionArtifact?.metadata
+    })
+  ];
 }
-async function prepareUrlInput(rootDir, input, options) {
+async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
+  const prepared = await prepareFileInputs(rootDir, absoluteInput, repoRoot, sourceClass);
+  if (!prepared.length) {
+    throw new Error(`No ingestable sources were extracted from ${absoluteInput}.`);
+  }
+  return prepared[0];
+}
+async function prepareUrlInputs(rootDir, input, options) {
   await validateUrlSafety(input);
   const response = await fetch(input);
   if (!response.ok) {
@@ -6634,8 +7261,9 @@ async function prepareUrlInput(rootDir, input, options) {
     const extension = path12.extname(inputUrl.pathname);
     storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
     if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-      extractedText = extractedTextForPlainSource(inputUrl.pathname, sourceKind, payloadBytes.toString("utf8"));
-      title = titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
+      const rawText = payloadBytes.toString("utf8");
+      extractedText = sourceKind === "markdown" ? semanticMarkdownContent(rawText).extractedText : extractedTextForPlainSource(inputUrl.pathname, sourceKind, rawText);
+      title = sourceKind === "markdown" ? semanticMarkdownTitle(title || inputUrl.hostname, rawText, inputUrl.pathname) : titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
       extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
       if (sourceKind === "markdown" && options.includeAssets) {
         const { attachments: remoteAttachments, skippedCount } = await collectRemoteImageAttachments(
@@ -6666,6 +7294,88 @@ async function prepareUrlInput(rootDir, input, options) {
       title = extracted.artifact.metadata?.title?.trim() || title;
       extractedText = extracted.extractedText;
       extractionArtifact = extracted.artifact;
+    } else if (sourceKind === "csv") {
+      const extracted = await extractCsvText({ mimeType, bytes: payloadBytes, fileName: inputUrl.pathname });
+      title = extracted.title?.trim() || title;
+      extractedText = extracted.extractedText;
+      extractionArtifact = extracted.artifact;
+    } else if (sourceKind === "xlsx") {
+      const extracted = await extractXlsxText({ mimeType, bytes: payloadBytes, fileName: inputUrl.pathname });
+      title = extracted.title?.trim() || title;
+      extractedText = extracted.extractedText;
+      extractionArtifact = extracted.artifact;
+    } else if (sourceKind === "pptx") {
+      const extracted = await extractPptxText({ mimeType, bytes: payloadBytes, fileName: inputUrl.pathname });
+      title = extracted.title?.trim() || title;
+      extractedText = extracted.extractedText;
+      extractionArtifact = extracted.artifact;
+    } else if (sourceKind === "epub") {
+      const extracted = await extractEpubChapters({ mimeType, bytes: payloadBytes, fileName: inputUrl.pathname });
+      title = extracted.title?.trim() || title;
+      const groupId = sourceGroupIdFor({
+        title,
+        originType: "url",
+        url: finalUrl
+      });
+      if (extracted.chapters.length) {
+        return extracted.chapters.map(
+          (chapter, index) => finalizePreparedInput({
+            title: `${title} - ${chapter.title}`,
+            originType: "url",
+            sourceKind: "epub",
+            url: finalUrl,
+            mimeType: "text/markdown",
+            storedExtension: ".md",
+            payloadBytes: Buffer.from(chapter.markdown, "utf8"),
+            extractedText: chapter.markdown,
+            extractionArtifact: {
+              extractor: "epub_text",
+              sourceKind: "epub",
+              mimeType,
+              producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+              metadata: {
+                ...chapter.metadata,
+                chapter_index: String(index + 1),
+                chapter_count: String(extracted.chapters.length)
+              },
+              warnings: extracted.warnings
+            },
+            extractionHash: buildExtractionHash(chapter.markdown, {
+              extractor: "epub_text",
+              sourceKind: "epub",
+              mimeType,
+              producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+              metadata: {
+                ...chapter.metadata,
+                chapter_index: String(index + 1),
+                chapter_count: String(extracted.chapters.length)
+              },
+              warnings: extracted.warnings
+            }),
+            sourceGroupId: groupId,
+            sourceGroupTitle: title,
+            sourcePartKey: chapter.partKey,
+            partIndex: index + 1,
+            partCount: extracted.chapters.length,
+            partTitle: chapter.title,
+            details: {
+              book_title: title,
+              chapter_title: chapter.title,
+              chapter_index: String(index + 1),
+              chapter_count: String(extracted.chapters.length),
+              ...extracted.author ? { author: extracted.author } : {}
+            },
+            logDetails
+          })
+        );
+      }
+      extractionArtifact = {
+        extractor: "epub_text",
+        sourceKind: "epub",
+        mimeType,
+        producedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        warnings: extracted.warnings ?? ["EPUB extraction completed but produced no chapter content."]
+      };
     } else if (sourceKind === "image") {
       const extracted = await extractImageWithVision(rootDir, {
         title,
@@ -6677,22 +7387,32 @@ async function prepareUrlInput(rootDir, input, options) {
       extractionArtifact = extracted.artifact;
     }
   }
-  return {
-    title,
-    originType: "url",
-    sourceKind,
-    language,
-    url: finalUrl,
-    mimeType,
-    storedExtension,
-    payloadBytes,
-    extractedText,
-    extractionArtifact,
-    extractionHash: buildExtractionHash(extractedText, extractionArtifact),
-    attachments,
-    contentHash,
-    logDetails
-  };
+  return [
+    finalizePreparedInput({
+      title,
+      originType: "url",
+      sourceKind,
+      language,
+      url: finalUrl,
+      mimeType,
+      storedExtension,
+      payloadBytes,
+      extractedText,
+      extractionArtifact,
+      extractionHash: buildExtractionHash(extractedText, extractionArtifact),
+      attachments,
+      contentHash,
+      details: extractionArtifact?.metadata,
+      logDetails
+    })
+  ];
+}
+async function prepareUrlInput(rootDir, input, options) {
+  const prepared = await prepareUrlInputs(rootDir, input, options);
+  if (!prepared.length) {
+    throw new Error(`No ingestable sources were extracted from ${input}.`);
+  }
+  return prepared[0];
 }
 async function collectInboxAttachmentRefs(inputDir, files) {
   const refsBySource = /* @__PURE__ */ new Map();
@@ -6766,7 +7486,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
   );
   const rewrittenText = rewriteMarkdownReferences(originalText, replacements);
   const extractionArtifact = createPlainTextExtractionArtifact("markdown", "text/markdown");
-  return {
+  return finalizePreparedInput({
     title,
     originType: "file",
     sourceKind: "markdown",
@@ -6779,7 +7499,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
     extractionHash: buildExtractionHash(rewrittenText, extractionArtifact),
     attachments,
     contentHash
-  };
+  });
 }
 async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
   const originalBytes = await fs11.readFile(absolutePath);
@@ -6824,18 +7544,23 @@ async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
   };
 }
 function isSupportedInboxKind(sourceKind) {
-  return ["markdown", "text", "html", "pdf", "docx", "image"].includes(sourceKind);
+  return ["markdown", "text", "html", "pdf", "docx", "epub", "csv", "xlsx", "pptx", "image"].includes(sourceKind);
 }
 async function ingestInputDetailed(rootDir, input, options) {
   const { paths } = await initWorkspace(rootDir);
   const normalizedOptions = normalizeIngestOptions(options);
   const absoluteInput = path12.resolve(rootDir, input);
   const repoRoot = isHttpUrl(input) || normalizedOptions.repoRoot ? normalizedOptions.repoRoot : await findNearestGitRoot2(absoluteInput).then((value) => value ?? path12.dirname(absoluteInput));
-  const prepared = isHttpUrl(input) ? await prepareUrlInput(rootDir, input, normalizedOptions) : await prepareFileInput(rootDir, absoluteInput, repoRoot);
-  return await persistPreparedInput(rootDir, prepared, paths);
+  const prepared = isHttpUrl(input) ? await prepareUrlInputs(rootDir, input, normalizedOptions) : await prepareFileInputs(rootDir, absoluteInput, repoRoot);
+  return await persistPreparedInputs(rootDir, input, prepared, paths);
 }
 async function ingestInput(rootDir, input, options) {
-  return (await ingestInputDetailed(rootDir, input, options)).manifest;
+  const result = await ingestInputDetailed(rootDir, input, options);
+  const manifest = [...result.created, ...result.updated, ...result.unchanged][0];
+  if (!manifest) {
+    throw new Error(`No source manifests were created or updated for ${input}.`);
+  }
+  return manifest;
 }
 async function addInput(rootDir, input, options = {}) {
   const { paths } = await initWorkspace(rootDir);
@@ -6933,13 +7658,20 @@ async function ingestDirectory(rootDir, inputDir, options) {
   const progress = createProgressReporter("ingest", files.length);
   for (const absolutePath of files) {
     const relativePath = repoRelativePathFor(absolutePath, repoRoot) ?? toPosix(path12.relative(repoRoot, absolutePath));
-    const prepared = await prepareFileInput(rootDir, absolutePath, repoRoot, sourceClassForRelativePath(relativePath, normalizedOptions));
-    const result = await persistPreparedInput(rootDir, prepared, paths);
-    if (result.isNew) {
-      imported.push(result.manifest);
-    } else if (result.wasUpdated) {
-      updated.push(result.manifest);
-    } else {
+    const preparedInputs = await prepareFileInputs(
+      rootDir,
+      absolutePath,
+      repoRoot,
+      sourceClassForRelativePath(relativePath, normalizedOptions)
+    );
+    const result = await persistPreparedInputs(rootDir, absolutePath, preparedInputs, paths);
+    if (result.created.length) {
+      imported.push(...result.created);
+    }
+    if (result.updated.length) {
+      updated.push(...result.updated);
+    }
+    if (!result.created.length && !result.updated.length && !result.removed.length) {
       skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "duplicate_content" });
     }
     progress.tick();
@@ -6990,13 +7722,13 @@ async function importInbox(rootDir, inputDir) {
       continue;
     }
     const prepared = sourceKind === "markdown" && refsBySource.has(absolutePath) ? await prepareInboxMarkdownInput(absolutePath, refsBySource.get(absolutePath) ?? []) : sourceKind === "html" && refsBySource.has(absolutePath) ? await prepareInboxHtmlInput(absolutePath, refsBySource.get(absolutePath) ?? []) : await prepareFileInput(rootDir, absolutePath);
-    const result = await persistPreparedInput(rootDir, prepared, paths);
-    if (!result.isNew) {
+    const result = await persistPreparedInputs(rootDir, absolutePath, [prepared], paths);
+    if (!result.created.length) {
       skipped.push({ path: toPosix(path12.relative(rootDir, absolutePath)), reason: "duplicate_content" });
       continue;
     }
-    attachmentCount += result.manifest.attachments?.length ?? 0;
-    imported.push(result.manifest);
+    attachmentCount += result.created.reduce((total, manifest) => total + (manifest.attachments?.length ?? 0), 0);
+    imported.push(...result.created);
   }
   await appendLogEntry(rootDir, "inbox_import", toPosix(path12.relative(rootDir, effectiveInputDir)) || ".", [
     `scanned=${files.length}`,
@@ -7021,7 +7753,10 @@ async function listManifests(rootDir) {
   const manifests = await Promise.all(
     entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(path12.join(paths.manifestsDir, entry)))
   );
-  return manifests.filter((manifest) => Boolean(manifest));
+  return manifests.filter((manifest) => Boolean(manifest)).map((manifest) => ({
+    ...manifest,
+    semanticHash: manifest.semanticHash ?? manifest.contentHash
+  }));
 }
 async function removeManifestBySourceId(rootDir, sourceId) {
   const { paths } = await initWorkspace(rootDir);
@@ -7029,8 +7764,12 @@ async function removeManifestBySourceId(rootDir, sourceId) {
   if (!manifest) {
     return null;
   }
-  await removeManifestArtifacts(rootDir, manifest, paths);
-  return manifest;
+  const normalizedManifest = {
+    ...manifest,
+    semanticHash: manifest.semanticHash ?? manifest.contentHash
+  };
+  await removeManifestArtifacts(rootDir, normalizedManifest, paths);
+  return normalizedManifest;
 }
 async function readExtractedText(rootDir, manifest) {
   if (!manifest.extractedTextPath) {
@@ -7176,7 +7915,7 @@ import { z as z7 } from "zod";
 // src/analysis.ts
 import path14 from "path";
 import { z as z2 } from "zod";
-var ANALYSIS_FORMAT_VERSION = 6;
+var ANALYSIS_FORMAT_VERSION = 7;
 var sourceAnalysisSchema = z2.object({
   title: z2.string().min(1),
   summary: z2.string().min(1),
@@ -7281,6 +8020,7 @@ function heuristicAnalysis(manifest, text, schemaHash) {
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: deriveTitle(manifest, text),
@@ -7331,6 +8071,7 @@ ${truncate(text, 18e3)}`
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash: schema.hash,
     title: parsed.title,
@@ -7367,6 +8108,7 @@ function analysisFromVisionExtraction(manifest, extraction, schemaHash) {
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: extraction.vision.title?.trim() || manifest.title,
@@ -7405,7 +8147,7 @@ function extractionWarningSummary(manifest, extraction) {
 async function analyzeSource(manifest, extractedText, provider, paths, schema) {
   const cachePath = path14.join(paths.analysesDir, `${manifest.sourceId}.json`);
   const cached = await readJsonFile(cachePath);
-  if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && cached.sourceHash === manifest.contentHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
+  if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && (cached.semanticHash ?? cached.sourceHash) === manifest.semanticHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
     return cached;
   }
   const extraction = await readExtractionArtifact(paths.rootDir, manifest);
@@ -7422,6 +8164,7 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
         analysisVersion: ANALYSIS_FORMAT_VERSION,
         sourceId: manifest.sourceId,
         sourceHash: manifest.contentHash,
+        semanticHash: manifest.semanticHash,
         extractionHash: manifest.extractionHash,
         schemaHash: schema.hash,
         title: manifest.title,
@@ -7448,6 +8191,7 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
       analysisVersion: ANALYSIS_FORMAT_VERSION,
       sourceId: manifest.sourceId,
       sourceHash: manifest.contentHash,
+      semanticHash: manifest.semanticHash,
       extractionHash: manifest.extractionHash,
       schemaHash: schema.hash,
       title: manifest.title,
@@ -8231,7 +8975,9 @@ async function resolveEmbeddingProvider(rootDir) {
     }
     const provider2 = await createProvider(explicitProviderId, providerConfig, rootDir);
     if (!provider2.capabilities.has("embeddings") || typeof provider2.embedTexts !== "function") {
-      throw new Error(`Provider ${provider2.id} does not support required capability "embeddings".`);
+      throw new Error(
+        `Provider ${provider2.id} does not support required capability "embeddings". Configure tasks.embeddingProvider to use an embedding-capable backend such as ollama or another openai-compatible embedding service.`
+      );
     }
     return provider2;
   }
@@ -9127,6 +9873,18 @@ function uniqueStrings2(values) {
 function safeFrontmatter(value) {
   return JSON.parse(JSON.stringify(value));
 }
+function sourceHashesForManifest(manifest) {
+  return {
+    sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+    sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash }
+  };
+}
+function sourceHashFrontmatter(sourceHashes, sourceSemanticHashes) {
+  return {
+    source_hashes: sourceHashes,
+    source_semantic_hashes: sourceSemanticHashes
+  };
+}
 function decoratedTags(baseTags, decorations) {
   return uniqueStrings2([
     ...baseTags,
@@ -9190,6 +9948,7 @@ function relatedOutputsSection(relatedOutputs) {
 function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutputs = [], modulePage, decorations) {
   const relativePath = pagePathFor("source", manifest.sourceId);
   const pageId = `source:${manifest.sourceId}`;
+  const { sourceHashes, sourceSemanticHashes } = sourceHashesForManifest(manifest);
   const moduleNodeIds = analysis.code ? [analysis.code.moduleId, ...analysis.code.symbols.map((symbol) => symbol.id)] : [];
   const nodeIds = [
     `source:${manifest.sourceId}`,
@@ -9222,17 +9981,25 @@ function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutput
     managed_by: metadata.managedBy,
     backlinks,
     schema_hash: schemaHash,
-    source_hashes: {
-      [manifest.sourceId]: manifest.contentHash
-    }
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes)
   };
   const body = [
     `# ${analysis.title}`,
     "",
     `Source ID: \`${manifest.sourceId}\``,
+    `Source Kind: \`${manifest.sourceKind}\``,
     manifest.url ? `Source URL: ${manifest.url}` : `Source Path: \`${manifest.originalPath ?? manifest.storedPath}\``,
     ...manifest.sourceType ? [`Source Type: \`${manifest.sourceType}\``, ""] : [""],
     ...manifest.sourceClass ? [`Source Class: \`${manifest.sourceClass}\``, ""] : [],
+    ...manifest.sourceGroupTitle ? [`Source Group: ${manifest.sourceGroupTitle}`] : [],
+    ...manifest.partTitle ? [`Part: ${manifest.partIndex ?? "?"}/${manifest.partCount ?? "?"} - ${manifest.partTitle}`] : [],
+    ...manifest.details && Object.keys(manifest.details).length ? [
+      "",
+      "## Source Details",
+      "",
+      ...Object.entries(manifest.details).map(([key, value]) => `- ${key.replace(/_/g, " ")}: ${value}`),
+      ""
+    ] : [],
     "",
     "## Summary",
     "",
@@ -9287,7 +10054,8 @@ function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutput
       confidence: metadata.confidence,
       backlinks,
       schemaHash,
-      sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: [...modulePage ? [modulePage.id] : [], ...relatedOutputs.map((page) => page.id)],
       relatedNodeIds: moduleNodeIds,
       relatedSourceIds: [],
@@ -9312,6 +10080,7 @@ function buildModulePage(input) {
   const localModuleBacklinks = input.localModules.map((moduleRef) => moduleRef.page.id);
   const relatedOutputs = input.relatedOutputs ?? [];
   const backlinks = uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]);
+  const { sourceHashes, sourceSemanticHashes } = sourceHashesForManifest(manifest);
   const importsSection = code.imports.length ? code.imports.map((item) => {
     const localModule = item.resolvedSourceId ? input.localModules.find((moduleRef) => moduleRef.sourceId === item.resolvedSourceId && moduleRef.reExport === item.reExport) : void 0;
     const importedBits = [
@@ -9355,9 +10124,7 @@ function buildModulePage(input) {
     managed_by: metadata.managedBy,
     backlinks,
     schema_hash: schemaHash,
-    source_hashes: {
-      [manifest.sourceId]: manifest.contentHash
-    },
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes),
     related_page_ids: uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]),
     related_node_ids: [],
     related_source_ids: uniqueStrings2([
@@ -9433,7 +10200,8 @@ function buildModulePage(input) {
       confidence: metadata.confidence,
       backlinks,
       schemaHash,
-      sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]),
       relatedNodeIds: [],
       relatedSourceIds: uniqueStrings2([
@@ -9449,7 +10217,7 @@ function buildModulePage(input) {
     content: matter5.stringify(body, frontmatter)
   };
 }
-function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, schemaHash, metadata, relativePath, relatedOutputs = [], decorations) {
+function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, sourceSemanticHashes, schemaHash, metadata, relativePath, relatedOutputs = [], decorations) {
   const slug = slugify(name);
   const pageId = `${kind}:${slug}`;
   const sourceIds = sourceAnalyses.map((item) => item.sourceId);
@@ -9473,7 +10241,7 @@ function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHash
     managed_by: metadata.managedBy,
     backlinks: otherPages,
     schema_hash: schemaHash,
-    source_hashes: sourceHashes
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes)
   };
   const body = [
     `# ${name}`,
@@ -9511,6 +10279,7 @@ function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHash
       backlinks: otherPages,
       schemaHash,
       sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: relatedOutputs.map((page) => page.id),
       relatedNodeIds: [],
       relatedSourceIds: [],
@@ -9551,6 +10320,7 @@ function buildIndexPage(pages, schemaHash, metadata, projectPages = []) {
     "backlinks: []",
     `schema_hash: ${schemaHash}`,
     "source_hashes: {}",
+    "source_semantic_hashes: {}",
     "---",
     "",
     "# SwarmVault Index",
@@ -9614,7 +10384,8 @@ function buildSectionIndex(kind, pages, schemaHash, metadata, projectIds = []) {
       managed_by: metadata.managedBy,
       backlinks: [],
       schema_hash: schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -9910,6 +10681,7 @@ function buildGraphReportPage(input) {
     backlinks: [],
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds
@@ -10025,6 +10797,7 @@ function buildGraphReportPage(input) {
       backlinks: [],
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10068,6 +10841,7 @@ function buildCommunitySummaryPage(input) {
     backlinks: ["graph:report"],
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: uniqueStrings2(["graph:report", ...communityPageIds]),
     related_node_ids: input.community.nodeIds,
     related_source_ids: relatedSourceIds
@@ -10107,6 +10881,7 @@ function buildCommunitySummaryPage(input) {
       backlinks: ["graph:report"],
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds: uniqueStrings2(["graph:report", ...communityPageIds]),
       relatedNodeIds: input.community.nodeIds,
       relatedSourceIds,
@@ -10143,7 +10918,8 @@ function buildProjectsIndex(projectPages, schemaHash, metadata) {
       managed_by: metadata.managedBy,
       backlinks: [],
       schema_hash: schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -10195,7 +10971,8 @@ function buildProjectIndex(input) {
       managed_by: input.metadata.managedBy,
       backlinks: [],
       schema_hash: input.schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -10226,6 +11003,7 @@ function buildOutputPage(input) {
     backlinks,
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds,
@@ -10250,6 +11028,7 @@ function buildOutputPage(input) {
       backlinks,
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10352,6 +11131,7 @@ function buildExploreHubPage(input) {
     backlinks,
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds,
@@ -10376,6 +11156,7 @@ function buildExploreHubPage(input) {
       backlinks,
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10674,6 +11455,9 @@ function normalizeSourceHashes(value) {
     Object.entries(value).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
   );
 }
+function normalizeSourceSemanticHashes(value) {
+  return normalizeSourceHashes(value);
+}
 function normalizePageStatus(value, fallback = "active") {
   return value === "draft" || value === "candidate" || value === "active" || value === "archived" ? value : fallback;
 }
@@ -10802,6 +11586,7 @@ function parseStoredPage(relativePath, content, defaults = {}) {
     backlinks,
     schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
     sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+    sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
     relatedPageIds,
     relatedNodeIds,
     relatedSourceIds,
@@ -10855,6 +11640,7 @@ async function loadInsightPages(wikiDir) {
         backlinks,
         schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
         sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+        sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
         relatedPageIds,
         relatedNodeIds,
         relatedSourceIds,
@@ -10955,6 +11741,7 @@ async function loadSavedOutputPages(wikiDir) {
           backlinks,
           schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
           sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+          sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
           relatedPageIds,
           relatedNodeIds,
           relatedSourceIds,
@@ -12431,11 +13218,13 @@ function aggregateItems(analyses, kind) {
         name: item.name,
         descriptions: [],
         sourceAnalyses: [],
-        sourceHashes: {}
+        sourceHashes: {},
+        sourceSemanticHashes: {}
       };
       existing.descriptions.push(item.description);
       existing.sourceAnalyses.push(analysis);
       existing.sourceHashes[analysis.sourceId] = analysis.sourceHash;
+      existing.sourceSemanticHashes[analysis.sourceId] = analysis.semanticHash;
       grouped.set(key, existing);
     }
   }
@@ -12457,6 +13246,7 @@ function emptyGraphPage(input) {
     backlinks: [],
     schemaHash: input.schemaHash,
     sourceHashes: input.sourceHashes,
+    sourceSemanticHashes: input.sourceSemanticHashes ?? {},
     relatedPageIds: [],
     relatedNodeIds: [],
     relatedSourceIds: [],
@@ -12621,6 +13411,7 @@ async function syncVaultArtifacts(rootDir, input) {
       nodeIds: [analysis.code.moduleId, ...analysis.code.symbols.map((symbol) => symbol.id)],
       schemaHash: sourceSchemaHash,
       sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash },
       confidence: 1
     }) : null;
     const preview = emptyGraphPage({
@@ -12639,6 +13430,7 @@ async function syncVaultArtifacts(rootDir, input) {
       ],
       schemaHash: sourceSchemaHash,
       sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash },
       confidence: 1
     });
     const sourceRecord = await buildManagedGraphPage(
@@ -12755,6 +13547,7 @@ async function syncVaultArtifacts(rootDir, input) {
           aggregate.descriptions,
           aggregate.sourceAnalyses,
           aggregate.sourceHashes,
+          aggregate.sourceSemanticHashes,
           schemaHash,
           metadata,
           relativePath,
@@ -13002,6 +13795,7 @@ async function syncVaultArtifacts(rootDir, input) {
     projectConfigHash: projectConfigHash(config),
     analyses: Object.fromEntries(input.analyses.map((analysis) => [analysis.sourceId, analysisSignature(analysis)])),
     sourceHashes: Object.fromEntries(input.manifests.map((manifest) => [manifest.sourceId, manifest.contentHash])),
+    sourceSemanticHashes: Object.fromEntries(input.manifests.map((manifest) => [manifest.sourceId, manifest.semanticHash])),
     sourceProjects: input.sourceProjects,
     outputHashes: input.outputHashes,
     insightHashes: input.insightHashes,
@@ -13467,6 +14261,7 @@ function emptyCompileState() {
     projectConfigHash: "",
     analyses: {},
     sourceHashes: {},
+    sourceSemanticHashes: {},
     sourceProjects: {},
     outputHashes: {},
     insightHashes: {},
@@ -13896,7 +14691,8 @@ async function initVault(rootDir, options = {}) {
         managed_by: "human",
         backlinks: [],
         schema_hash: "",
-        source_hashes: {}
+        source_hashes: {},
+        source_semantic_hashes: {}
       }
     )
   );
@@ -13919,7 +14715,8 @@ async function initVault(rootDir, options = {}) {
       managed_by: "system",
       backlinks: [],
       schema_hash: "",
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     })
   );
   await writeFileIfChanged(
@@ -13941,7 +14738,8 @@ async function initVault(rootDir, options = {}) {
       managed_by: "system",
       backlinks: [],
       schema_hash: "",
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     })
   );
   if (options.obsidian) {
@@ -13982,7 +14780,7 @@ async function compileVault(rootDir, options = {}) {
   );
   const nextProjectConfigHash = projectConfigHash(config);
   const projectConfigChanged = !previousState || previousState.projectConfigHash !== nextProjectConfigHash;
-  const previousSourceHashes = previousState?.sourceHashes ?? {};
+  const previousSourceHashes = previousState?.sourceSemanticHashes ?? previousState?.sourceHashes ?? {};
   const previousAnalyses = previousState?.analyses ?? {};
   const previousSourceProjects = previousState?.sourceProjects ?? {};
   const previousOutputHashes = previousState?.outputHashes ?? {};
@@ -13997,7 +14795,7 @@ async function compileVault(rootDir, options = {}) {
   const dirty = [];
   const clean = [];
   for (const manifest of manifests) {
-    const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.contentHash;
+    const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.semanticHash;
     const noAnalysis = !previousAnalyses[manifest.sourceId];
     const projectId = sourceProjects[manifest.sourceId] ?? null;
     const projectChanged = (previousSourceProjects[manifest.sourceId] ?? null) !== projectId;
@@ -14707,9 +15505,11 @@ function structuralLintFindings(_rootDir, paths, graph, schemas, manifests, sour
           relatedPageIds: [page.id]
         });
       }
-      for (const [sourceId, knownHash] of Object.entries(page.sourceHashes)) {
+      const freshnessHashes = Object.keys(page.sourceSemanticHashes).length ? page.sourceSemanticHashes : page.sourceHashes;
+      for (const [sourceId, knownHash] of Object.entries(freshnessHashes)) {
         const manifest = manifestMap.get(sourceId);
-        if (manifest && manifest.contentHash !== knownHash) {
+        const manifestHash = manifest?.semanticHash ?? manifest?.contentHash;
+        if (manifestHash && manifestHash !== knownHash) {
           findings.push({
             severity: "warning",
             code: "stale_page",
@@ -14848,7 +15648,7 @@ async function bootstrapDemo(rootDir, input) {
 }
 
 // src/mcp.ts
-var SERVER_VERSION = "0.2.1";
+var SERVER_VERSION = "0.3.0";
 async function createMcpServer(rootDir) {
   const server = new McpServer({
     name: "swarmvault",
@@ -15026,8 +15826,8 @@ async function createMcpServer(rootDir) {
       }
     },
     async ({ input }) => {
-      const manifest = await ingestInput(rootDir, input);
-      return asToolText(manifest);
+      const result = await ingestInputDetailed(rootDir, input);
+      return asToolText(result);
     }
   );
   server.registerTool(
@@ -15831,12 +16631,11 @@ async function syncCrawlSource(rootDir, entry, options) {
   let updatedCount = 0;
   for (const pageUrl of crawl.pages) {
     const persisted = await ingestInputDetailed(rootDir, pageUrl);
-    currentSourceIds.push(persisted.manifest.sourceId);
-    if (persisted.isNew) {
-      importedCount += 1;
-    } else if (persisted.wasUpdated) {
-      updatedCount += 1;
-    }
+    currentSourceIds.push(...persisted.created.map((manifest) => manifest.sourceId));
+    currentSourceIds.push(...persisted.updated.map((manifest) => manifest.sourceId));
+    currentSourceIds.push(...persisted.unchanged.map((manifest) => manifest.sourceId));
+    importedCount += persisted.created.length;
+    updatedCount += persisted.updated.length;
   }
   let removedCount = 0;
   for (const sourceId of previousSourceIds) {
@@ -16221,6 +17020,124 @@ import { promisify } from "util";
 import matter10 from "gray-matter";
 import mime2 from "mime-types";
 
+// src/graph-presentation.ts
+var OVERVIEW_THRESHOLD = 5e3;
+var OVERVIEW_NODE_BUDGET = 1500;
+function nodePriority(node, pinnedNodeIds) {
+  return [pinnedNodeIds.has(node.id) ? 0 : 1, -(node.degree ?? 0), -(node.bridgeScore ?? 0), node.label, node.id];
+}
+function compareTuples(left, right) {
+  const length = Math.max(left.length, right.length);
+  for (let index = 0; index < length; index += 1) {
+    const leftValue = left[index];
+    const rightValue = right[index];
+    if (leftValue === rightValue) {
+      continue;
+    }
+    if (typeof leftValue === "number" && typeof rightValue === "number") {
+      return leftValue - rightValue;
+    }
+    return String(leftValue ?? "").localeCompare(String(rightValue ?? ""));
+  }
+  return 0;
+}
+function survivingHyperedges(hyperedges, sampledNodeIds) {
+  return hyperedges.filter((hyperedge) => hyperedge.nodeIds.filter((nodeId) => sampledNodeIds.has(nodeId)).length >= 2);
+}
+function pinnedNodeIdsForReport(report) {
+  if (!report) {
+    return /* @__PURE__ */ new Set();
+  }
+  return /* @__PURE__ */ new Set([
+    ...report.godNodes.map((node) => node.nodeId),
+    ...report.bridgeNodes.map((node) => node.nodeId),
+    ...report.surprisingConnections.flatMap((connection) => [connection.sourceNodeId, connection.targetNodeId])
+  ]);
+}
+function sampleGraphNodes(graph, report, nodeBudget = OVERVIEW_NODE_BUDGET) {
+  const pinned = pinnedNodeIdsForReport(report);
+  const nodeById2 = new Map(graph.nodes.map((node) => [node.id, node]));
+  const selected = new Set([...pinned].filter((nodeId) => nodeById2.has(nodeId)));
+  const sortedCommunities2 = [...graph.communities ?? []].sort((left, right) => {
+    const leftNodes = left.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node));
+    const rightNodes = right.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node));
+    const leftFirstParty = leftNodes.filter((node) => node.sourceClass === "first_party").length;
+    const rightFirstParty = rightNodes.filter((node) => node.sourceClass === "first_party").length;
+    return compareTuples(
+      [-leftFirstParty, -leftNodes.length, left.label, left.id],
+      [-rightFirstParty, -rightNodes.length, right.label, right.id]
+    );
+  });
+  for (const community of sortedCommunities2) {
+    const communityNodes = community.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node)).sort((left, right) => compareTuples(nodePriority(left, pinned), nodePriority(right, pinned)));
+    for (const node of communityNodes) {
+      if (selected.size >= nodeBudget && !pinned.has(node.id)) {
+        break;
+      }
+      selected.add(node.id);
+    }
+    if (selected.size >= nodeBudget) {
+      break;
+    }
+  }
+  if (selected.size < nodeBudget) {
+    for (const node of [...graph.nodes].sort((left, right) => compareTuples(nodePriority(left, pinned), nodePriority(right, pinned)))) {
+      if (selected.size >= nodeBudget && !pinned.has(node.id)) {
+        break;
+      }
+      selected.add(node.id);
+    }
+  }
+  return selected;
+}
+function buildViewerGraphArtifact(graph, options = {}) {
+  const threshold = options.threshold ?? OVERVIEW_THRESHOLD;
+  const nodeBudget = options.nodeBudget ?? OVERVIEW_NODE_BUDGET;
+  const totalCommunities = graph.communities?.length ?? 0;
+  if (options.full || graph.nodes.length <= threshold) {
+    return {
+      ...graph,
+      presentation: {
+        mode: "full",
+        threshold,
+        nodeBudget,
+        totalNodes: graph.nodes.length,
+        displayedNodes: graph.nodes.length,
+        totalEdges: graph.edges.length,
+        displayedEdges: graph.edges.length,
+        totalCommunities,
+        displayedCommunities: totalCommunities
+      }
+    };
+  }
+  const sampledNodeIds = sampleGraphNodes(graph, options.report, nodeBudget);
+  const nodes = graph.nodes.filter((node) => sampledNodeIds.has(node.id));
+  const edges = graph.edges.filter((edge) => sampledNodeIds.has(edge.source) && sampledNodeIds.has(edge.target));
+  const hyperedges = survivingHyperedges(graph.hyperedges ?? [], sampledNodeIds);
+  const communities = (graph.communities ?? []).map((community) => ({
+    ...community,
+    nodeIds: community.nodeIds.filter((nodeId) => sampledNodeIds.has(nodeId))
+  })).filter((community) => community.nodeIds.length > 0);
+  return {
+    ...graph,
+    nodes,
+    edges,
+    hyperedges,
+    communities,
+    presentation: {
+      mode: "overview",
+      threshold,
+      nodeBudget,
+      totalNodes: graph.nodes.length,
+      displayedNodes: nodes.length,
+      totalEdges: graph.edges.length,
+      displayedEdges: edges.length,
+      totalCommunities,
+      displayedCommunities: communities.length
+    }
+  };
+}
+
 // src/watch.ts
 import path26 from "path";
 import process3 from "process";
@@ -16686,7 +17603,7 @@ async function ensureViewerDist(viewerDistDir) {
     await execFileAsync("pnpm", ["build"], { cwd: viewerProjectDir });
   }
 }
-async function startGraphServer(rootDir, port) {
+async function startGraphServer(rootDir, port, options = {}) {
   const { config, paths } = await loadVaultConfig(rootDir);
   const effectivePort = port ?? config.viewer.port;
   await ensureViewerDist(paths.viewerDistDir);
@@ -16698,8 +17615,16 @@ async function startGraphServer(rootDir, port) {
         response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
         return;
       }
+      const graph = await readJsonFile(paths.graphPath);
+      if (!graph) {
+        response.writeHead(404, { "content-type": "application/json" });
+        response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
+        return;
+      }
+      const reportPath = path27.join(paths.wikiDir, "graph", "report.json");
+      const report = await readJsonFile(reportPath) ?? null;
       response.writeHead(200, { "content-type": "application/json" });
-      response.end(await fs22.readFile(paths.graphPath, "utf8"));
+      response.end(JSON.stringify(buildViewerGraphArtifact(graph, { report, full: options.full ?? false })));
       return;
     }
     if (url.pathname === "/api/graph/query") {
@@ -16875,7 +17800,7 @@ async function startGraphServer(rootDir, port) {
     }
   };
 }
-async function exportGraphHtml(rootDir, outputPath) {
+async function exportGraphHtml(rootDir, outputPath, options = {}) {
   const { paths } = await loadVaultConfig(rootDir);
   const graph = await readJsonFile(paths.graphPath);
   if (!graph) {
@@ -16919,7 +17844,11 @@ async function exportGraphHtml(rootDir, outputPath) {
   const script = await fs22.readFile(scriptPath, "utf8");
   const style = stylePath && await fileExists(stylePath) ? await fs22.readFile(stylePath, "utf8") : "";
   const report = await readJsonFile(path27.join(paths.wikiDir, "graph", "report.json"));
-  const embeddedData = JSON.stringify({ graph, pages: pages.filter(Boolean), report }, null, 2).replace(/</g, "\\u003c");
+  const embeddedData = JSON.stringify(
+    { graph: buildViewerGraphArtifact(graph, { report, full: options.full ?? false }), pages: pages.filter(Boolean), report },
+    null,
+    2
+  ).replace(/</g, "\\u003c");
   const html = [
     "<!doctype html>",
     '<html lang="en">',
@@ -16968,6 +17897,7 @@ export {
   importInbox,
   ingestDirectory,
   ingestInput,
+  ingestInputDetailed,
   initVault,
   initWorkspace,
   installAgent,