@swarmvaultai/engine 0.6.3 → 0.6.5

package/README.md

@@ -292,7 +292,7 @@ Compile and repo-refresh runs also keep benchmark artifacts current by default,
 ## Notes
 
 - The engine expects Node `>=24`
-- The local search layer currently uses the built-in `node:sqlite` module, which may emit an experimental warning in Node 24
+- The local search layer uses the built-in `node:sqlite` module on Node `>=24`; current CLI releases suppress the upstream experimental warning during normal runs
 - The viewer source lives in the companion `@swarmvaultai/viewer` package, and the built assets are bundled into the engine package for CLI installs
 
 ## Links

package/dist/index.d.ts

@@ -46,7 +46,7 @@ type Freshness = "fresh" | "stale";
 type ClaimStatus = "extracted" | "inferred" | "conflicted" | "stale";
 type EvidenceClass = "extracted" | "inferred" | "ambiguous";
 type Polarity = "positive" | "negative" | "neutral";
-type OutputOrigin = "query" | "explore";
+type OutputOrigin = "query" | "explore" | "source_brief" | "source_review" | "source_guide" | "source_session";
 type OutputFormat = "markdown" | "report" | "slides" | "chart" | "image";
 type OutputAssetRole = "primary" | "preview" | "manifest" | "poster";
 type GraphExportFormat = "html" | "svg" | "graphml" | "cypher";
@@ -359,6 +359,7 @@ interface ManagedSourceRecord {
     lastSyncStatus?: "success" | "error";
     lastSyncCounts?: ManagedSourceSyncCounts;
     lastError?: string;
+    changed?: boolean;
 }
 interface ManagedSourcesArtifact {
     version: 1;

package/dist/index.js

@@ -9099,6 +9099,7 @@ import { z as z7 } from "zod";
 
 // src/analysis.ts
 import path14 from "path";
+import { fromMarkdown } from "mdast-util-from-markdown";
 import { z as z2 } from "zod";
 var ANALYSIS_FORMAT_VERSION = 7;
 var sourceAnalysisSchema = z2.object({
@@ -9158,6 +9159,12 @@ var STOPWORDS = /* @__PURE__ */ new Set([
   "would",
   "your"
 ]);
+var HEURISTIC_SECTION_SOURCE_KINDS = /* @__PURE__ */ new Map([
+  ["transcript", "Transcript"],
+  ["chat_export", "Messages"],
+  ["email", "Message"],
+  ["calendar", "Description"]
+]);
 function extractTopTerms(text, count) {
   const frequency = /* @__PURE__ */ new Map();
   for (const token of text.toLowerCase().match(/[a-z][a-z0-9-]{3,}/g) ?? []) {
@@ -9184,18 +9191,112 @@ function detectPolarity(text) {
   }
   return "neutral";
 }
-function deriveTitle(manifest, text) {
-  const heading = text.match(/^#\s+(.+)$/m)?.[1]?.trim();
-  return heading || manifest.title;
+function parseMarkdownNodes(text) {
+  try {
+    const root = fromMarkdown(text);
+    return Array.isArray(root.children) ? root.children : [];
+  } catch {
+    return [];
+  }
+}
+function markdownNodeText(node) {
+  if (node.type === "text" || node.type === "inlineCode" || node.type === "code") {
+    return normalizeWhitespace(node.value ?? "");
+  }
+  if (node.type === "image") {
+    return normalizeWhitespace(node.alt ?? "");
+  }
+  if (node.type === "break" || node.type === "thematicBreak") {
+    return " ";
+  }
+  return normalizeWhitespace((node.children ?? []).map((child) => markdownNodeText(child)).join(" "));
+}
+function markdownNodesText(nodes) {
+  return normalizeWhitespace(nodes.map((node) => markdownNodeText(node)).join("\n"));
+}
+function stripLeadingTitleNodes(nodes, title) {
+  const normalizedTitle = normalizeWhitespace(title);
+  if (!normalizedTitle || !nodes.length) {
+    return nodes;
+  }
+  for (let index = 0; index < nodes.length; index += 1) {
+    const node = nodes[index];
+    if (!node) {
+      continue;
+    }
+    const nodeText2 = markdownNodeText(node);
+    if (node.type === "heading" && node.depth === 1 && nodeText2 === normalizedTitle) {
+      return nodes.slice(index + 1);
+    }
+    if (node.type === "paragraph" && nodeText2 === normalizedTitle) {
+      return nodes.slice(index + 1);
+    }
+    return nodes;
+  }
+  return nodes;
+}
+function markdownSectionNodes(nodes, heading) {
+  const normalizedHeading = normalizeWhitespace(heading);
+  for (let index = 0; index < nodes.length; index += 1) {
+    const node = nodes[index];
+    if (node?.type !== "heading" || node.depth !== 2) {
+      continue;
+    }
+    if (markdownNodeText(node) !== normalizedHeading) {
+      continue;
+    }
+    const sectionNodes = [];
+    for (let cursor = index + 1; cursor < nodes.length; cursor += 1) {
+      const candidate = nodes[cursor];
+      if (candidate?.type === "heading" && typeof candidate.depth === "number" && candidate.depth <= 2) {
+        break;
+      }
+      if (candidate) {
+        sectionNodes.push(candidate);
+      }
+    }
+    return sectionNodes;
+  }
+  return [];
+}
+function textForHeuristicAnalysis(manifest, text) {
+  const nodes = parseMarkdownNodes(text);
+  if (!nodes.length) {
+    return normalizeWhitespace(text);
+  }
+  const sectionHeading = HEURISTIC_SECTION_SOURCE_KINDS.get(manifest.sourceKind);
+  const scopedNodes = sectionHeading ? markdownSectionNodes(nodes, sectionHeading) : nodes;
+  const relevantNodes = scopedNodes.length ? scopedNodes : nodes;
+  const contentNodes = stripLeadingTitleNodes(relevantNodes, manifest.title);
+  const normalized = markdownNodesText(contentNodes.length ? contentNodes : relevantNodes);
+  return normalized || normalizeWhitespace(text);
+}
+function normalizeAnalysisTitle(manifest, candidate) {
+  if (manifest.sourceKind !== "code") {
+    return manifest.title;
+  }
+  const normalized = normalizeWhitespace(candidate.replace(/^#+\s+/, ""));
+  if (!normalized) {
+    return manifest.title;
+  }
+  if (normalized.length > 140 || normalized.includes(" ## ")) {
+    return manifest.title;
+  }
+  return normalized;
+}
+function normalizeSourceAnalysis(manifest, analysis) {
+  const title = normalizeAnalysisTitle(manifest, analysis.title);
+  return title === analysis.title ? analysis : { ...analysis, title };
 }
 function heuristicAnalysis(manifest, text, schemaHash) {
-  const normalized = normalizeWhitespace(text);
+  const analysisText = textForHeuristicAnalysis(manifest, text);
+  const normalized = normalizeWhitespace(analysisText);
   const concepts = extractTopTerms(normalized, 6).map((term) => ({
     id: `concept:${slugify(term)}`,
     name: term,
     description: `Frequently referenced concept in ${manifest.title}.`
   }));
-  const entities = extractEntities(text, 6).map((term) => ({
+  const entities = extractEntities(analysisText, 6).map((term) => ({
     id: `entity:${slugify(term)}`,
     name: term,
     description: `Named entity mentioned in ${manifest.title}.`
@@ -9208,7 +9309,7 @@ function heuristicAnalysis(manifest, text, schemaHash) {
     semanticHash: manifest.semanticHash,
     extractionHash: manifest.extractionHash,
     schemaHash,
-    title: deriveTitle(manifest, text),
+    title: manifest.title,
     summary: firstSentences(normalized, 3) || truncate(normalized, 280) || `Imported ${manifest.sourceKind} source.`,
     concepts,
     entities,
@@ -9333,7 +9434,11 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
   const cachePath = path14.join(paths.analysesDir, `${manifest.sourceId}.json`);
   const cached = await readJsonFile(cachePath);
   if (cached && cached.analysisVersion === ANALYSIS_FORMAT_VERSION && (cached.semanticHash ?? cached.sourceHash) === manifest.semanticHash && cached.extractionHash === manifest.extractionHash && cached.schemaHash === schema.hash) {
-    return cached;
+    const normalizedCached = normalizeSourceAnalysis(manifest, cached);
+    if (normalizedCached !== cached) {
+      await writeJsonFile(cachePath, normalizedCached);
+    }
+    return normalizedCached;
   }
   const extraction = await readExtractionArtifact(paths.rootDir, manifest);
   const content = normalizeWhitespace(extractedText ?? "");
@@ -9398,8 +9503,9 @@ async function analyzeSource(manifest, extractedText, provider, paths, schema) {
       analysis = heuristicAnalysis(manifest, content, schema.hash);
     }
   }
-  await writeJsonFile(cachePath, analysis);
-  return analysis;
+  const normalized = normalizeSourceAnalysis(manifest, analysis);
+  await writeJsonFile(cachePath, normalized);
+  return normalized;
 }
 function analysisSignature(analysis) {
   return sha256(JSON.stringify(analysis));
@@ -13019,8 +13125,37 @@ async function loadSavedOutputPages(wikiDir) {
 import fs17 from "fs/promises";
 import path21 from "path";
 import matter8 from "gray-matter";
+function warningMessage(warning) {
+  return warning instanceof Error ? warning.message : String(warning);
+}
+function warningType(warning, type) {
+  if (warning instanceof Error) {
+    return warning.name;
+  }
+  return typeof type === "string" ? type : void 0;
+}
+function isSqliteExperimentalWarning(warning, type) {
+  return warningType(warning, type) === "ExperimentalWarning" && warningMessage(warning).includes("SQLite is an experimental feature");
+}
+function withSuppressedSqliteExperimentalWarning(run) {
+  const originalEmitWarning = process.emitWarning.bind(process);
+  process.emitWarning = ((warning, options, ...args) => {
+    const type = typeof options === "string" ? options : typeof options?.type === "string" ? options.type ?? void 0 : void 0;
+    if (isSqliteExperimentalWarning(warning, type)) {
+      return;
+    }
+    return originalEmitWarning(warning, options, ...args);
+  });
+  try {
+    return run();
+  } finally {
+    process.emitWarning = originalEmitWarning;
+  }
+}
 function getDatabaseSync() {
-  const builtin = process.getBuiltinModule?.("node:sqlite");
+  const builtin = withSuppressedSqliteExperimentalWarning(
+    () => process.getBuiltinModule?.("node:sqlite")
+  );
   if (!builtin?.DatabaseSync) {
     throw new Error("node:sqlite is unavailable in this Node runtime.");
   }
@@ -13045,7 +13180,7 @@ function normalizeSourceClass2(value) {
 async function rebuildSearchIndex(dbPath, pages, wikiDir) {
   await ensureDir(path21.dirname(dbPath));
   const DatabaseSync = getDatabaseSync();
-  const db = new DatabaseSync(dbPath);
+  const db = withSuppressedSqliteExperimentalWarning(() => new DatabaseSync(dbPath));
   db.exec("PRAGMA journal_mode = WAL;");
   db.exec(`
     DROP TABLE IF EXISTS page_search;
@@ -13123,7 +13258,7 @@ function searchPages(dbPath, query, limitOrOptions = 5) {
     return [];
   }
   const DatabaseSync = getDatabaseSync();
-  const db = new DatabaseSync(dbPath, { readOnly: true });
+  const db = withSuppressedSqliteExperimentalWarning(() => new DatabaseSync(dbPath, { readOnly: true }));
   const clauses = ["page_search MATCH ?"];
   const params = [ftsQuery];
   if (options.kind && options.kind !== "all") {
@@ -17688,7 +17823,7 @@ async function bootstrapDemo(rootDir, input) {
 }
 
 // src/mcp.ts
-var SERVER_VERSION = "0.6.3";
+var SERVER_VERSION = "0.6.5";
 async function createMcpServer(rootDir) {
   const server = new McpServer({
     name: "swarmvault",
@@ -18365,6 +18500,15 @@ function uniqueStrings4(values) {
 function normalizeManagedStatus(value) {
   return value === "missing" || value === "error" ? value : "ready";
 }
+function emptyManagedSourceSyncCounts() {
+  return {
+    scannedCount: 0,
+    importedCount: 0,
+    updatedCount: 0,
+    removedCount: 0,
+    skippedCount: 0
+  };
+}
 function withinRoot2(rootPath, targetPath) {
   const relative = path26.relative(rootPath, targetPath);
   return relative === "" || !relative.startsWith("..") && !path26.isAbsolute(relative);
@@ -18740,6 +18884,7 @@ async function syncManagedSource(rootDir, entry, options) {
           updatedAt: now,
           lastSyncAt: now,
           lastSyncStatus: "error",
+          lastSyncCounts: emptyManagedSourceSyncCounts(),
           lastError: `Directory not found: ${entry.path}`,
           changed: false
         };
@@ -18756,6 +18901,7 @@ async function syncManagedSource(rootDir, entry, options) {
           updatedAt: now,
           lastSyncAt: now,
           lastSyncStatus: "error",
+          lastSyncCounts: emptyManagedSourceSyncCounts(),
           lastError: `File not found: ${entry.path}`,
           changed: false
         };
@@ -18785,6 +18931,7 @@ async function syncManagedSource(rootDir, entry, options) {
       updatedAt: now,
       lastSyncAt: now,
       lastSyncStatus: "error",
+      lastSyncCounts: emptyManagedSourceSyncCounts(),
       lastError: error instanceof Error ? error.message : String(error),
       changed: false
     };
@@ -18950,7 +19097,7 @@ async function writeSourceBriefForScope(rootDir, source) {
     relatedSourceIds: source.sourceIds,
     projectIds,
     extraTags: ["source-brief"],
-    origin: "query",
+    origin: "source_brief",
     slug: `source-briefs/${source.id}`,
     metadata: {
       status: "active",
@@ -19196,7 +19343,7 @@ async function buildSourceReviewStagedPage(rootDir, scope) {
     relatedSourceIds: scope.sourceIds,
     projectIds,
     extraTags: ["source-review"],
-    origin: "query",
+    origin: "source_review",
     slug: `source-reviews/${scope.id}`,
     metadata: {
       status: "draft",
@@ -19443,7 +19590,7 @@ async function buildSourceGuideStagedPage(rootDir, scope) {
     relatedSourceIds: scope.sourceIds,
     projectIds,
     extraTags: ["source-guide", "guided-ingest"],
-    origin: "query",
+    origin: "source_guide",
     slug: `source-guides/${scope.id}`,
     metadata: {
       status: "draft",
@@ -19597,7 +19744,7 @@ async function buildSourceSessionSavedPage(rootDir, scope, session) {
     relatedSourceIds: scope.sourceIds,
     projectIds,
     extraTags: ["source-session", "guided-session"],
-    origin: "query",
+    origin: "source_session",
     slug: `source-sessions/${scope.id}`,
     metadata: {
       status: "active",
@@ -19943,6 +20090,15 @@ async function resumeSourceSession(rootDir, id, options = {}) {
 function shouldCompile(changedSources, graphExists, compileRequested) {
   return compileRequested && (!graphExists || changedSources.length > 0);
 }
+async function shouldRefreshBriefForManagedSource(source, options) {
+  if (options.compilePerformed || options.changed) {
+    return true;
+  }
+  if (!source.briefPath) {
+    return true;
+  }
+  return !await fileExists(source.briefPath);
+}
 async function listManagedSourceRecords(rootDir) {
   await ensureManagedSourcesArtifact(rootDir);
   return await loadManagedSources(rootDir);
@@ -19974,12 +20130,15 @@ async function addManagedSource(rootDir, input, options = {}) {
   }
   const graphExists = await loadVaultConfig(rootDir).then(({ paths }) => fileExists(paths.graphPath));
   let compile;
-  if (shouldCompile([synced], graphExists, compileRequested)) {
+  if (shouldCompile(synced.changed ? [synced] : [], graphExists, compileRequested)) {
     compile = await compileVault(rootDir, {});
   }
   let briefGenerated = false;
   let briefPath;
-  if (compileRequested && briefRequested && synced.status === "ready") {
+  if (compileRequested && briefRequested && synced.status === "ready" && await shouldRefreshBriefForManagedSource(synced, {
+    compilePerformed: Boolean(compile),
+    changed: synced.changed
+  })) {
     const briefs = await generateBriefsForSources(rootDir, [synced]);
     briefPath = briefs.get(synced.id);
     briefGenerated = Boolean(briefPath);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swarmvaultai/engine",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "description": "Core engine for SwarmVault: ingest, compile, query, lint, and provider abstractions.",
   "type": "module",
   "main": "dist/index.js",
@@ -57,6 +57,7 @@
     "jsdom": "^27.0.0",
     "mailparser": "^3.9.8",
     "mammoth": "^1.12.0",
+    "mdast-util-from-markdown": "^2.0.3",
     "mime-types": "^3.0.1",
     "neo4j-driver": "^5.28.3",
     "node-ical": "^0.26.0",