@swarmvaultai/engine 0.2.1 → 0.2.2

package/dist/index.js

@@ -4481,9 +4481,10 @@ async function analyzeCodeSource(manifest, extractedText, schemaHash) {
   const language = manifest.language ?? inferCodeLanguage(manifest.originalPath ?? manifest.storedPath, manifest.mimeType) ?? "typescript";
   const { code, rationales } = language === "javascript" || language === "jsx" || language === "typescript" || language === "tsx" ? analyzeTypeScriptLikeCode(manifest, extractedText) : await analyzeTreeSitterCode(manifest, extractedText, language);
   return {
-    analysisVersion: 6,
+    analysisVersion: 7,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: manifest.title,
@@ -5208,6 +5209,17 @@ var HARD_REPO_IGNORES = /* @__PURE__ */ new Set([".git", ".venv"]);
 var PROGRESS_FILE_THRESHOLD = 150;
 var PROGRESS_UPDATE_INTERVAL = 100;
 var RST_HEADING_MARKERS = /* @__PURE__ */ new Set(["=", "-", "~", "^", '"', "#", "*", "+"]);
+var MARKDOWN_SEMANTIC_FRONTMATTER_KEYS = [
+  "title",
+  "summary",
+  "description",
+  "aliases",
+  "tags",
+  "authors",
+  "published_at",
+  "canonical_url",
+  "source_type"
+];
 function uniqueStrings(values) {
   return [...new Set(values.filter(Boolean))];
 }
@@ -5345,6 +5357,65 @@ function extractedTextForPlainSource(filePath, sourceKind, content) {
   }
   return content;
 }
+function normalizeSemanticMarkdownScalar(value) {
+  if (typeof value !== "string") {
+    return void 0;
+  }
+  const normalized = normalizeWhitespace(value.trim());
+  return normalized || void 0;
+}
+function normalizeSemanticMarkdownList(value) {
+  if (!Array.isArray(value)) {
+    return void 0;
+  }
+  const items = uniqueStrings(
+    value.flatMap((item) => typeof item === "string" ? [normalizeWhitespace(item.trim())] : []).filter(Boolean)
+  );
+  return items.length ? items : void 0;
+}
+function semanticMarkdownTitle(fallback, content, filePath) {
+  const parsed = matter3(content);
+  const frontmatterTitle = normalizeSemanticMarkdownScalar(parsed.data.title);
+  if (frontmatterTitle) {
+    return frontmatterTitle;
+  }
+  return titleFromText(fallback, parsed.content, filePath);
+}
+function semanticMarkdownContent(content) {
+  const parsed = matter3(content);
+  const body = parsed.content.replace(/\r\n?/g, "\n").trim();
+  const semanticFrontmatter = Object.fromEntries(
+    MARKDOWN_SEMANTIC_FRONTMATTER_KEYS.flatMap((key) => {
+      const value = key === "aliases" || key === "tags" || key === "authors" ? normalizeSemanticMarkdownList(parsed.data[key]) : normalizeSemanticMarkdownScalar(parsed.data[key]);
+      return value === void 0 ? [] : [[key, value]];
+    })
+  );
+  const semanticLines = Object.entries(semanticFrontmatter).map(
+    ([key, value]) => `${key}: ${Array.isArray(value) ? value.join(", ") : value}`
+  );
+  const extractedText = [...semanticLines, ...semanticLines.length && body ? [""] : [], body].filter(Boolean).join("\n").trim();
+  return {
+    extractedText,
+    semanticHash: sha256(
+      JSON.stringify({
+        body,
+        frontmatter: semanticFrontmatter
+      })
+    )
+  };
+}
+function finalizePreparedInput(prepared) {
+  if (prepared.sourceKind !== "markdown") {
+    return prepared;
+  }
+  const semantic = semanticMarkdownContent(prepared.payloadBytes.toString("utf8"));
+  return {
+    ...prepared,
+    extractedText: semantic.extractedText,
+    extractionHash: buildExtractionHash(semantic.extractedText, prepared.extractionArtifact),
+    semanticHash: semantic.semanticHash
+  };
+}
 function shouldEmitProgress(totalItems) {
   return totalItems >= PROGRESS_FILE_THRESHOLD && Boolean(process.stderr?.isTTY);
 }
@@ -5511,7 +5582,7 @@ function markdownFrontmatter(value) {
   return matter3.stringify("", normalized).trimEnd().split("\n").concat([""]);
 }
 function prepareCapturedMarkdownInput(input) {
-  return {
+  return finalizePreparedInput({
     title: input.title,
     originType: "url",
     sourceKind: "markdown",
@@ -5523,7 +5594,7 @@ function prepareCapturedMarkdownInput(input) {
     extractedText: input.markdown,
     attachments: input.attachments,
     logDetails: input.logDetails
-  };
+  });
 }
 function isPrivateIp(ip) {
   if (ip === "::1" || ip.startsWith("fc") || ip.startsWith("fd")) return true;
@@ -5886,7 +5957,10 @@ async function readManifestByHash(manifestsDir, contentHash) {
     }
     const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
     if (manifest?.contentHash === contentHash) {
-      return manifest;
+      return {
+        ...manifest,
+        semanticHash: manifest.semanticHash ?? manifest.contentHash
+      };
     }
   }
   return null;
@@ -5899,7 +5973,10 @@ async function readManifestByOrigin(manifestsDir, prepared) {
     }
     const manifest = await readJsonFile(path12.join(manifestsDir, entry.name));
     if (manifest && manifestMatchesOrigin(manifest, prepared)) {
-      return manifest;
+      return {
+        ...manifest,
+        semanticHash: manifest.semanticHash ?? manifest.contentHash
+      };
     }
   }
   return null;
@@ -6148,10 +6225,11 @@ async function persistPreparedInput(rootDir, prepared, paths) {
   await ensureDir(paths.extractsDir);
   const attachments = prepared.attachments ?? [];
   const contentHash = prepared.contentHash ?? buildCompositeHash(prepared.payloadBytes, attachments);
+  const semanticHash = prepared.semanticHash ?? contentHash;
   const extractionHash = prepared.extractionHash ?? buildExtractionHash(prepared.extractedText, prepared.extractionArtifact);
   const existingByOrigin = await readManifestByOrigin(paths.manifestsDir, prepared);
   const existingByHash = existingByOrigin ? null : await readManifestByHash(paths.manifestsDir, contentHash);
-  if (existingByOrigin && existingByOrigin.contentHash === contentHash && existingByOrigin.extractionHash === extractionHash && existingByOrigin.title === prepared.title && existingByOrigin.sourceKind === prepared.sourceKind && existingByOrigin.sourceType === prepared.sourceType && existingByOrigin.sourceClass === prepared.sourceClass && existingByOrigin.language === prepared.language && existingByOrigin.mimeType === prepared.mimeType && existingByOrigin.repoRelativePath === prepared.repoRelativePath) {
+  if (existingByOrigin && existingByOrigin.contentHash === contentHash && existingByOrigin.semanticHash === semanticHash && existingByOrigin.extractionHash === extractionHash && existingByOrigin.title === prepared.title && existingByOrigin.sourceKind === prepared.sourceKind && existingByOrigin.sourceType === prepared.sourceType && existingByOrigin.sourceClass === prepared.sourceClass && existingByOrigin.language === prepared.language && existingByOrigin.mimeType === prepared.mimeType && existingByOrigin.repoRelativePath === prepared.repoRelativePath) {
     return { manifest: existingByOrigin, isNew: false, wasUpdated: false };
   }
   if (existingByHash) {
@@ -6209,6 +6287,7 @@ async function persistPreparedInput(rootDir, prepared, paths) {
     extractionHash,
     mimeType: prepared.mimeType,
     contentHash,
+    semanticHash,
     createdAt: previous?.createdAt ?? now,
     updatedAt: now,
     attachments: manifestAttachments.length ? manifestAttachments : void 0
@@ -6523,14 +6602,15 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
   let extractedText;
   let extractionArtifact;
   if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-    extractedText = extractedTextForPlainSource(absoluteInput, sourceKind, payloadBytes.toString("utf8"));
-    title = titleFromText(path12.basename(absoluteInput, path12.extname(absoluteInput)), extractedText, absoluteInput);
+    const rawText = payloadBytes.toString("utf8");
+    extractedText = sourceKind === "markdown" ? semanticMarkdownContent(rawText).extractedText : extractedTextForPlainSource(absoluteInput, sourceKind, rawText);
+    title = sourceKind === "markdown" ? semanticMarkdownTitle(path12.basename(absoluteInput, path12.extname(absoluteInput)), rawText, absoluteInput) : titleFromText(path12.basename(absoluteInput, path12.extname(absoluteInput)), extractedText, absoluteInput);
     extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
   } else if (sourceKind === "html") {
     const html = payloadBytes.toString("utf8");
     const converted = await convertHtmlToMarkdown(html, pathToFileURL(absoluteInput).toString());
     title = converted.title;
-    extractedText = converted.markdown;
+    extractedText = semanticMarkdownContent(converted.markdown).extractedText;
     extractionArtifact = createHtmlReadabilityExtractionArtifact(sourceKind, mimeType);
   } else if (sourceKind === "pdf") {
     title = path12.basename(absoluteInput, path12.extname(absoluteInput));
@@ -6556,7 +6636,7 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
   } else {
     title = path12.basename(absoluteInput, path12.extname(absoluteInput));
   }
-  return {
+  return finalizePreparedInput({
     title,
     originType: "file",
     sourceKind,
@@ -6570,7 +6650,7 @@ async function prepareFileInput(rootDir, absoluteInput, repoRoot, sourceClass) {
     extractedText,
     extractionArtifact,
     extractionHash: buildExtractionHash(extractedText, extractionArtifact)
-  };
+  });
 }
 async function prepareUrlInput(rootDir, input, options) {
   await validateUrlSafety(input);
@@ -6634,8 +6714,9 @@ async function prepareUrlInput(rootDir, input, options) {
     const extension = path12.extname(inputUrl.pathname);
     storedExtension = extension || `.${mime.extension(mimeType) || "bin"}`;
     if (sourceKind === "markdown" || sourceKind === "text" || sourceKind === "code") {
-      extractedText = extractedTextForPlainSource(inputUrl.pathname, sourceKind, payloadBytes.toString("utf8"));
-      title = titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
+      const rawText = payloadBytes.toString("utf8");
+      extractedText = sourceKind === "markdown" ? semanticMarkdownContent(rawText).extractedText : extractedTextForPlainSource(inputUrl.pathname, sourceKind, rawText);
+      title = sourceKind === "markdown" ? semanticMarkdownTitle(title || inputUrl.hostname, rawText, inputUrl.pathname) : titleFromText(title || inputUrl.hostname, extractedText, inputUrl.pathname);
       extractionArtifact = createPlainTextExtractionArtifact(sourceKind, mimeType);
       if (sourceKind === "markdown" && options.includeAssets) {
         const { attachments: remoteAttachments, skippedCount } = await collectRemoteImageAttachments(
@@ -6677,7 +6758,7 @@ async function prepareUrlInput(rootDir, input, options) {
       extractionArtifact = extracted.artifact;
     }
   }
-  return {
+  return finalizePreparedInput({
     title,
     originType: "url",
     sourceKind,
@@ -6692,7 +6773,7 @@ async function prepareUrlInput(rootDir, input, options) {
     attachments,
     contentHash,
     logDetails
-  };
+  });
 }
 async function collectInboxAttachmentRefs(inputDir, files) {
   const refsBySource = /* @__PURE__ */ new Map();
@@ -6766,7 +6847,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
   );
   const rewrittenText = rewriteMarkdownReferences(originalText, replacements);
   const extractionArtifact = createPlainTextExtractionArtifact("markdown", "text/markdown");
-  return {
+  return finalizePreparedInput({
     title,
     originType: "file",
     sourceKind: "markdown",
@@ -6779,7 +6860,7 @@ async function prepareInboxMarkdownInput(absolutePath, attachmentRefs) {
     extractionHash: buildExtractionHash(rewrittenText, extractionArtifact),
     attachments,
     contentHash
-  };
+  });
 }
 async function prepareInboxHtmlInput(absolutePath, attachmentRefs) {
   const originalBytes = await fs11.readFile(absolutePath);
@@ -7021,7 +7102,10 @@ async function listManifests(rootDir) {
   const manifests = await Promise.all(
     entries.filter((entry) => entry.endsWith(".json")).map((entry) => readJsonFile(path12.join(paths.manifestsDir, entry)))
   );
-  return manifests.filter((manifest) => Boolean(manifest));
+  return manifests.filter((manifest) => Boolean(manifest)).map((manifest) => ({
+    ...manifest,
+    semanticHash: manifest.semanticHash ?? manifest.contentHash
+  }));
 }
 async function removeManifestBySourceId(rootDir, sourceId) {
   const { paths } = await initWorkspace(rootDir);
@@ -7029,8 +7113,12 @@ async function removeManifestBySourceId(rootDir, sourceId) {
   if (!manifest) {
     return null;
   }
-  await removeManifestArtifacts(rootDir, manifest, paths);
-  return manifest;
+  const normalizedManifest = {
+    ...manifest,
+    semanticHash: manifest.semanticHash ?? manifest.contentHash
+  };
+  await removeManifestArtifacts(rootDir, normalizedManifest, paths);
+  return normalizedManifest;
 }
 async function readExtractedText(rootDir, manifest) {
   if (!manifest.extractedTextPath) {
@@ -7176,7 +7264,7 @@ import { z as z7 } from "zod";
 // src/analysis.ts
 import path14 from "path";
 import { z as z2 } from "zod";
-var ANALYSIS_FORMAT_VERSION = 6;
+var ANALYSIS_FORMAT_VERSION = 7;
 var sourceAnalysisSchema = z2.object({
   title: z2.string().min(1),
   summary: z2.string().min(1),
@@ -7281,6 +7369,7 @@ function heuristicAnalysis(manifest, text, schemaHash) {
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: deriveTitle(manifest, text),
@@ -7331,6 +7420,7 @@ ${truncate(text, 18e3)}`
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash: schema.hash,
     title: parsed.title,
@@ -7367,6 +7457,7 @@ function analysisFromVisionExtraction(manifest, extraction, schemaHash) {
     analysisVersion: ANALYSIS_FORMAT_VERSION,
     sourceId: manifest.sourceId,
     sourceHash: manifest.contentHash,
+    semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
     title: extraction.vision.title?.trim() || manifest.title,
@@ -7405,7 +7496,7 @@ function extractionWarningSummary(manifest, extraction) {
 async function analyzeSource(manifest, extractedText, provider, paths, schema) {
   const cachePath = path14.join(paths.analysesDir, `${manifest.sourceId}.json`);
   const cached = await readJsonFile(cachePath);
-  if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && cached.sourceHash === manifest.contentHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
+  if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && (cached.semanticHash ?? cached.sourceHash) === manifest.semanticHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
     return cached;
   }
   const extraction = await readExtractionArtifact(paths.rootDir, manifest);
@@ -7422,6 +7513,7 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
         analysisVersion: ANALYSIS_FORMAT_VERSION,
         sourceId: manifest.sourceId,
         sourceHash: manifest.contentHash,
+        semanticHash: manifest.semanticHash,
         extractionHash: manifest.extractionHash,
         schemaHash: schema.hash,
         title: manifest.title,
@@ -7448,6 +7540,7 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
       analysisVersion: ANALYSIS_FORMAT_VERSION,
       sourceId: manifest.sourceId,
       sourceHash: manifest.contentHash,
+      semanticHash: manifest.semanticHash,
       extractionHash: manifest.extractionHash,
       schemaHash: schema.hash,
       title: manifest.title,
@@ -8231,7 +8324,9 @@ async function resolveEmbeddingProvider(rootDir) {
     }
     const provider2 = await createProvider(explicitProviderId, providerConfig, rootDir);
     if (!provider2.capabilities.has("embeddings") || typeof provider2.embedTexts !== "function") {
-      throw new Error(`Provider ${provider2.id} does not support required capability "embeddings".`);
+      throw new Error(
+        `Provider ${provider2.id} does not support required capability "embeddings". Configure tasks.embeddingProvider to use an embedding-capable backend such as ollama or another openai-compatible embedding service.`
+      );
     }
     return provider2;
   }
@@ -9127,6 +9222,18 @@ function uniqueStrings2(values) {
 function safeFrontmatter(value) {
   return JSON.parse(JSON.stringify(value));
 }
+function sourceHashesForManifest(manifest) {
+  return {
+    sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+    sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash }
+  };
+}
+function sourceHashFrontmatter(sourceHashes, sourceSemanticHashes) {
+  return {
+    source_hashes: sourceHashes,
+    source_semantic_hashes: sourceSemanticHashes
+  };
+}
 function decoratedTags(baseTags, decorations) {
   return uniqueStrings2([
     ...baseTags,
@@ -9190,6 +9297,7 @@ function relatedOutputsSection(relatedOutputs) {
 function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutputs = [], modulePage, decorations) {
   const relativePath = pagePathFor("source", manifest.sourceId);
   const pageId = `source:${manifest.sourceId}`;
+  const { sourceHashes, sourceSemanticHashes } = sourceHashesForManifest(manifest);
   const moduleNodeIds = analysis.code ? [analysis.code.moduleId, ...analysis.code.symbols.map((symbol) => symbol.id)] : [];
   const nodeIds = [
     `source:${manifest.sourceId}`,
@@ -9222,9 +9330,7 @@ function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutput
     managed_by: metadata.managedBy,
     backlinks,
     schema_hash: schemaHash,
-    source_hashes: {
-      [manifest.sourceId]: manifest.contentHash
-    }
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes)
   };
   const body = [
     `# ${analysis.title}`,
@@ -9287,7 +9393,8 @@ function buildSourcePage(manifest, analysis, schemaHash, metadata, relatedOutput
       confidence: metadata.confidence,
       backlinks,
       schemaHash,
-      sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: [...modulePage ? [modulePage.id] : [], ...relatedOutputs.map((page) => page.id)],
       relatedNodeIds: moduleNodeIds,
       relatedSourceIds: [],
@@ -9312,6 +9419,7 @@ function buildModulePage(input) {
   const localModuleBacklinks = input.localModules.map((moduleRef) => moduleRef.page.id);
   const relatedOutputs = input.relatedOutputs ?? [];
   const backlinks = uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]);
+  const { sourceHashes, sourceSemanticHashes } = sourceHashesForManifest(manifest);
   const importsSection = code.imports.length ? code.imports.map((item) => {
     const localModule = item.resolvedSourceId ? input.localModules.find((moduleRef) => moduleRef.sourceId === item.resolvedSourceId && moduleRef.reExport === item.reExport) : void 0;
     const importedBits = [
@@ -9355,9 +9463,7 @@ function buildModulePage(input) {
     managed_by: metadata.managedBy,
     backlinks,
     schema_hash: schemaHash,
-    source_hashes: {
-      [manifest.sourceId]: manifest.contentHash
-    },
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes),
     related_page_ids: uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]),
     related_node_ids: [],
     related_source_ids: uniqueStrings2([
@@ -9433,7 +9539,8 @@ function buildModulePage(input) {
       confidence: metadata.confidence,
       backlinks,
       schemaHash,
-      sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: uniqueStrings2([sourcePage.id, ...localModuleBacklinks, ...relatedOutputs.map((page) => page.id)]),
       relatedNodeIds: [],
       relatedSourceIds: uniqueStrings2([
@@ -9449,7 +9556,7 @@ function buildModulePage(input) {
     content: matter5.stringify(body, frontmatter)
   };
 }
-function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, schemaHash, metadata, relativePath, relatedOutputs = [], decorations) {
+function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHashes, sourceSemanticHashes, schemaHash, metadata, relativePath, relatedOutputs = [], decorations) {
   const slug = slugify(name);
   const pageId = `${kind}:${slug}`;
   const sourceIds = sourceAnalyses.map((item) => item.sourceId);
@@ -9473,7 +9580,7 @@ function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHash
     managed_by: metadata.managedBy,
     backlinks: otherPages,
     schema_hash: schemaHash,
-    source_hashes: sourceHashes
+    ...sourceHashFrontmatter(sourceHashes, sourceSemanticHashes)
   };
   const body = [
     `# ${name}`,
@@ -9511,6 +9618,7 @@ function buildAggregatePage(kind, name, descriptions, sourceAnalyses, sourceHash
       backlinks: otherPages,
       schemaHash,
       sourceHashes,
+      sourceSemanticHashes,
       relatedPageIds: relatedOutputs.map((page) => page.id),
       relatedNodeIds: [],
       relatedSourceIds: [],
@@ -9551,6 +9659,7 @@ function buildIndexPage(pages, schemaHash, metadata, projectPages = []) {
     "backlinks: []",
     `schema_hash: ${schemaHash}`,
     "source_hashes: {}",
+    "source_semantic_hashes: {}",
     "---",
     "",
     "# SwarmVault Index",
@@ -9614,7 +9723,8 @@ function buildSectionIndex(kind, pages, schemaHash, metadata, projectIds = []) {
       managed_by: metadata.managedBy,
       backlinks: [],
       schema_hash: schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -9910,6 +10020,7 @@ function buildGraphReportPage(input) {
     backlinks: [],
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds
@@ -10025,6 +10136,7 @@ function buildGraphReportPage(input) {
       backlinks: [],
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10068,6 +10180,7 @@ function buildCommunitySummaryPage(input) {
     backlinks: ["graph:report"],
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: uniqueStrings2(["graph:report", ...communityPageIds]),
     related_node_ids: input.community.nodeIds,
     related_source_ids: relatedSourceIds
@@ -10107,6 +10220,7 @@ function buildCommunitySummaryPage(input) {
       backlinks: ["graph:report"],
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds: uniqueStrings2(["graph:report", ...communityPageIds]),
       relatedNodeIds: input.community.nodeIds,
       relatedSourceIds,
@@ -10143,7 +10257,8 @@ function buildProjectsIndex(projectPages, schemaHash, metadata) {
       managed_by: metadata.managedBy,
       backlinks: [],
       schema_hash: schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -10195,7 +10310,8 @@ function buildProjectIndex(input) {
       managed_by: input.metadata.managedBy,
       backlinks: [],
       schema_hash: input.schemaHash,
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     }
   );
 }
@@ -10226,6 +10342,7 @@ function buildOutputPage(input) {
     backlinks,
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds,
@@ -10250,6 +10367,7 @@ function buildOutputPage(input) {
       backlinks,
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10352,6 +10470,7 @@ function buildExploreHubPage(input) {
     backlinks,
     schema_hash: input.schemaHash,
     source_hashes: {},
+    source_semantic_hashes: {},
     related_page_ids: relatedPageIds,
     related_node_ids: relatedNodeIds,
     related_source_ids: relatedSourceIds,
@@ -10376,6 +10495,7 @@ function buildExploreHubPage(input) {
       backlinks,
       schemaHash: input.schemaHash,
       sourceHashes: {},
+      sourceSemanticHashes: {},
       relatedPageIds,
       relatedNodeIds,
       relatedSourceIds,
@@ -10674,6 +10794,9 @@ function normalizeSourceHashes(value) {
     Object.entries(value).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
   );
 }
+function normalizeSourceSemanticHashes(value) {
+  return normalizeSourceHashes(value);
+}
 function normalizePageStatus(value, fallback = "active") {
   return value === "draft" || value === "candidate" || value === "active" || value === "archived" ? value : fallback;
 }
@@ -10802,6 +10925,7 @@ function parseStoredPage(relativePath, content, defaults = {}) {
     backlinks,
     schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
     sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+    sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
     relatedPageIds,
     relatedNodeIds,
     relatedSourceIds,
@@ -10855,6 +10979,7 @@ async function loadInsightPages(wikiDir) {
         backlinks,
         schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
         sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+        sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
         relatedPageIds,
         relatedNodeIds,
         relatedSourceIds,
@@ -10955,6 +11080,7 @@ async function loadSavedOutputPages(wikiDir) {
           backlinks,
           schemaHash: typeof parsed.data.schema_hash === "string" ? parsed.data.schema_hash : "",
           sourceHashes: normalizeSourceHashes(parsed.data.source_hashes),
+          sourceSemanticHashes: normalizeSourceSemanticHashes(parsed.data.source_semantic_hashes),
           relatedPageIds,
           relatedNodeIds,
           relatedSourceIds,
@@ -12431,11 +12557,13 @@ function aggregateItems(analyses, kind) {
         name: item.name,
         descriptions: [],
         sourceAnalyses: [],
-        sourceHashes: {}
+        sourceHashes: {},
+        sourceSemanticHashes: {}
       };
       existing.descriptions.push(item.description);
       existing.sourceAnalyses.push(analysis);
       existing.sourceHashes[analysis.sourceId] = analysis.sourceHash;
+      existing.sourceSemanticHashes[analysis.sourceId] = analysis.semanticHash;
       grouped.set(key, existing);
     }
   }
@@ -12457,6 +12585,7 @@ function emptyGraphPage(input) {
     backlinks: [],
     schemaHash: input.schemaHash,
     sourceHashes: input.sourceHashes,
+    sourceSemanticHashes: input.sourceSemanticHashes ?? {},
     relatedPageIds: [],
     relatedNodeIds: [],
     relatedSourceIds: [],
@@ -12621,6 +12750,7 @@ async function syncVaultArtifacts(rootDir, input) {
       nodeIds: [analysis.code.moduleId, ...analysis.code.symbols.map((symbol) => symbol.id)],
       schemaHash: sourceSchemaHash,
       sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash },
       confidence: 1
     }) : null;
     const preview = emptyGraphPage({
@@ -12639,6 +12769,7 @@ async function syncVaultArtifacts(rootDir, input) {
       ],
       schemaHash: sourceSchemaHash,
       sourceHashes: { [manifest.sourceId]: manifest.contentHash },
+      sourceSemanticHashes: { [manifest.sourceId]: manifest.semanticHash },
       confidence: 1
     });
     const sourceRecord = await buildManagedGraphPage(
@@ -12755,6 +12886,7 @@ async function syncVaultArtifacts(rootDir, input) {
           aggregate.descriptions,
           aggregate.sourceAnalyses,
           aggregate.sourceHashes,
+          aggregate.sourceSemanticHashes,
           schemaHash,
           metadata,
           relativePath,
@@ -13002,6 +13134,7 @@ async function syncVaultArtifacts(rootDir, input) {
     projectConfigHash: projectConfigHash(config),
     analyses: Object.fromEntries(input.analyses.map((analysis) => [analysis.sourceId, analysisSignature(analysis)])),
     sourceHashes: Object.fromEntries(input.manifests.map((manifest) => [manifest.sourceId, manifest.contentHash])),
+    sourceSemanticHashes: Object.fromEntries(input.manifests.map((manifest) => [manifest.sourceId, manifest.semanticHash])),
     sourceProjects: input.sourceProjects,
     outputHashes: input.outputHashes,
     insightHashes: input.insightHashes,
@@ -13467,6 +13600,7 @@ function emptyCompileState() {
     projectConfigHash: "",
     analyses: {},
     sourceHashes: {},
+    sourceSemanticHashes: {},
     sourceProjects: {},
     outputHashes: {},
     insightHashes: {},
@@ -13896,7 +14030,8 @@ async function initVault(rootDir, options = {}) {
         managed_by: "human",
         backlinks: [],
         schema_hash: "",
-        source_hashes: {}
+        source_hashes: {},
+        source_semantic_hashes: {}
       }
     )
   );
@@ -13919,7 +14054,8 @@ async function initVault(rootDir, options = {}) {
       managed_by: "system",
       backlinks: [],
       schema_hash: "",
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     })
   );
   await writeFileIfChanged(
@@ -13941,7 +14077,8 @@ async function initVault(rootDir, options = {}) {
       managed_by: "system",
       backlinks: [],
       schema_hash: "",
-      source_hashes: {}
+      source_hashes: {},
+      source_semantic_hashes: {}
     })
   );
   if (options.obsidian) {
@@ -13982,7 +14119,7 @@ async function compileVault(rootDir, options = {}) {
   );
   const nextProjectConfigHash = projectConfigHash(config);
   const projectConfigChanged = !previousState || previousState.projectConfigHash !== nextProjectConfigHash;
-  const previousSourceHashes = previousState?.sourceHashes ?? {};
+  const previousSourceHashes = previousState?.sourceSemanticHashes ?? previousState?.sourceHashes ?? {};
   const previousAnalyses = previousState?.analyses ?? {};
   const previousSourceProjects = previousState?.sourceProjects ?? {};
   const previousOutputHashes = previousState?.outputHashes ?? {};
@@ -13997,7 +14134,7 @@ async function compileVault(rootDir, options = {}) {
   const dirty = [];
   const clean = [];
   for (const manifest of manifests) {
-    const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.contentHash;
+    const hashChanged = previousSourceHashes[manifest.sourceId] !== manifest.semanticHash;
     const noAnalysis = !previousAnalyses[manifest.sourceId];
     const projectId = sourceProjects[manifest.sourceId] ?? null;
     const projectChanged = (previousSourceProjects[manifest.sourceId] ?? null) !== projectId;
@@ -14707,9 +14844,11 @@ function structuralLintFindings(_rootDir, paths, graph, schemas, manifests, sour
           relatedPageIds: [page.id]
         });
       }
-      for (const [sourceId, knownHash] of Object.entries(page.sourceHashes)) {
+      const freshnessHashes = Object.keys(page.sourceSemanticHashes).length ? page.sourceSemanticHashes : page.sourceHashes;
+      for (const [sourceId, knownHash] of Object.entries(freshnessHashes)) {
         const manifest = manifestMap.get(sourceId);
-        if (manifest && manifest.contentHash !== knownHash) {
+        const manifestHash = manifest?.semanticHash ?? manifest?.contentHash;
+        if (manifestHash && manifestHash !== knownHash) {
           findings.push({
             severity: "warning",
             code: "stale_page",
@@ -14848,7 +14987,7 @@ async function bootstrapDemo(rootDir, input) {
 }
 
 // src/mcp.ts
-var SERVER_VERSION = "0.2.1";
+var SERVER_VERSION = "0.2.2";
 async function createMcpServer(rootDir) {
   const server = new McpServer({
     name: "swarmvault",
@@ -16221,6 +16360,124 @@ import { promisify } from "util";
 import matter10 from "gray-matter";
 import mime2 from "mime-types";
 
+// src/graph-presentation.ts
+var OVERVIEW_THRESHOLD = 5e3;
+var OVERVIEW_NODE_BUDGET = 1500;
+function nodePriority(node, pinnedNodeIds) {
+  return [pinnedNodeIds.has(node.id) ? 0 : 1, -(node.degree ?? 0), -(node.bridgeScore ?? 0), node.label, node.id];
+}
+function compareTuples(left, right) {
+  const length = Math.max(left.length, right.length);
+  for (let index = 0; index < length; index += 1) {
+    const leftValue = left[index];
+    const rightValue = right[index];
+    if (leftValue === rightValue) {
+      continue;
+    }
+    if (typeof leftValue === "number" && typeof rightValue === "number") {
+      return leftValue - rightValue;
+    }
+    return String(leftValue ?? "").localeCompare(String(rightValue ?? ""));
+  }
+  return 0;
+}
+function survivingHyperedges(hyperedges, sampledNodeIds) {
+  return hyperedges.filter((hyperedge) => hyperedge.nodeIds.filter((nodeId) => sampledNodeIds.has(nodeId)).length >= 2);
+}
+function pinnedNodeIdsForReport(report) {
+  if (!report) {
+    return /* @__PURE__ */ new Set();
+  }
+  return /* @__PURE__ */ new Set([
+    ...report.godNodes.map((node) => node.nodeId),
+    ...report.bridgeNodes.map((node) => node.nodeId),
+    ...report.surprisingConnections.flatMap((connection) => [connection.sourceNodeId, connection.targetNodeId])
+  ]);
+}
+function sampleGraphNodes(graph, report, nodeBudget = OVERVIEW_NODE_BUDGET) {
+  const pinned = pinnedNodeIdsForReport(report);
+  const nodeById2 = new Map(graph.nodes.map((node) => [node.id, node]));
+  const selected = new Set([...pinned].filter((nodeId) => nodeById2.has(nodeId)));
+  const sortedCommunities2 = [...graph.communities ?? []].sort((left, right) => {
+    const leftNodes = left.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node));
+    const rightNodes = right.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node));
+    const leftFirstParty = leftNodes.filter((node) => node.sourceClass === "first_party").length;
+    const rightFirstParty = rightNodes.filter((node) => node.sourceClass === "first_party").length;
+    return compareTuples(
+      [-leftFirstParty, -leftNodes.length, left.label, left.id],
+      [-rightFirstParty, -rightNodes.length, right.label, right.id]
+    );
+  });
+  for (const community of sortedCommunities2) {
+    const communityNodes = community.nodeIds.map((nodeId) => nodeById2.get(nodeId)).filter((node) => Boolean(node)).sort((left, right) => compareTuples(nodePriority(left, pinned), nodePriority(right, pinned)));
+    for (const node of communityNodes) {
+      if (selected.size >= nodeBudget && !pinned.has(node.id)) {
+        break;
+      }
+      selected.add(node.id);
+    }
+    if (selected.size >= nodeBudget) {
+      break;
+    }
+  }
+  if (selected.size < nodeBudget) {
+    for (const node of [...graph.nodes].sort((left, right) => compareTuples(nodePriority(left, pinned), nodePriority(right, pinned)))) {
+      if (selected.size >= nodeBudget && !pinned.has(node.id)) {
+        break;
+      }
+      selected.add(node.id);
+    }
+  }
+  return selected;
+}
+function buildViewerGraphArtifact(graph, options = {}) {
+  const threshold = options.threshold ?? OVERVIEW_THRESHOLD;
+  const nodeBudget = options.nodeBudget ?? OVERVIEW_NODE_BUDGET;
+  const totalCommunities = graph.communities?.length ?? 0;
+  if (options.full || graph.nodes.length <= threshold) {
+    return {
+      ...graph,
+      presentation: {
+        mode: "full",
+        threshold,
+        nodeBudget,
+        totalNodes: graph.nodes.length,
+        displayedNodes: graph.nodes.length,
+        totalEdges: graph.edges.length,
+        displayedEdges: graph.edges.length,
+        totalCommunities,
+        displayedCommunities: totalCommunities
+      }
+    };
+  }
+  const sampledNodeIds = sampleGraphNodes(graph, options.report, nodeBudget);
+  const nodes = graph.nodes.filter((node) => sampledNodeIds.has(node.id));
+  const edges = graph.edges.filter((edge) => sampledNodeIds.has(edge.source) && sampledNodeIds.has(edge.target));
+  const hyperedges = survivingHyperedges(graph.hyperedges ?? [], sampledNodeIds);
+  const communities = (graph.communities ?? []).map((community) => ({
+    ...community,
+    nodeIds: community.nodeIds.filter((nodeId) => sampledNodeIds.has(nodeId))
+  })).filter((community) => community.nodeIds.length > 0);
+  return {
+    ...graph,
+    nodes,
+    edges,
+    hyperedges,
+    communities,
+    presentation: {
+      mode: "overview",
+      threshold,
+      nodeBudget,
+      totalNodes: graph.nodes.length,
+      displayedNodes: nodes.length,
+      totalEdges: graph.edges.length,
+      displayedEdges: edges.length,
+      totalCommunities,
+      displayedCommunities: communities.length
+    }
+  };
+}
+
 // src/watch.ts
 import path26 from "path";
 import process3 from "process";
@@ -16686,7 +16943,7 @@ async function ensureViewerDist(viewerDistDir) {
     await execFileAsync("pnpm", ["build"], { cwd: viewerProjectDir });
   }
 }
-async function startGraphServer(rootDir, port) {
+async function startGraphServer(rootDir, port, options = {}) {
   const { config, paths } = await loadVaultConfig(rootDir);
   const effectivePort = port ?? config.viewer.port;
   await ensureViewerDist(paths.viewerDistDir);
@@ -16698,8 +16955,16 @@ async function startGraphServer(rootDir, port) {
         response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
         return;
       }
+      const graph = await readJsonFile(paths.graphPath);
+      if (!graph) {
+        response.writeHead(404, { "content-type": "application/json" });
+        response.end(JSON.stringify({ error: "Graph artifact not found. Run `swarmvault compile` first." }));
+        return;
+      }
+      const reportPath = path27.join(paths.wikiDir, "graph", "report.json");
+      const report = await readJsonFile(reportPath) ?? null;
       response.writeHead(200, { "content-type": "application/json" });
-      response.end(await fs22.readFile(paths.graphPath, "utf8"));
+      response.end(JSON.stringify(buildViewerGraphArtifact(graph, { report, full: options.full ?? false })));
       return;
     }
     if (url.pathname === "/api/graph/query") {
@@ -16875,7 +17140,7 @@ async function startGraphServer(rootDir, port) {
     }
   };
 }
-async function exportGraphHtml(rootDir, outputPath) {
+async function exportGraphHtml(rootDir, outputPath, options = {}) {
   const { paths } = await loadVaultConfig(rootDir);
   const graph = await readJsonFile(paths.graphPath);
   if (!graph) {
@@ -16919,7 +17184,11 @@ async function exportGraphHtml(rootDir, outputPath) {
   const script = await fs22.readFile(scriptPath, "utf8");
   const style = stylePath && await fileExists(stylePath) ? await fs22.readFile(stylePath, "utf8") : "";
   const report = await readJsonFile(path27.join(paths.wikiDir, "graph", "report.json"));
-  const embeddedData = JSON.stringify({ graph, pages: pages.filter(Boolean), report }, null, 2).replace(/</g, "\\u003c");
+  const embeddedData = JSON.stringify(
+    { graph: buildViewerGraphArtifact(graph, { report, full: options.full ?? false }), pages: pages.filter(Boolean), report },
+    null,
+    2
+  ).replace(/</g, "\\u003c");
   const html = [
     "<!doctype html>",
     '<html lang="en">',