@velvetmonkey/flywheel-memory 2.0.155 → 2.0.156

package/dist/index.js

@@ -6516,6 +6516,7 @@ import {
   loadFlywheelConfigFromDb,
   saveFlywheelConfigToDb
 } from "@velvetmonkey/vault-core";
+var DEFAULT_ENTITY_EXCLUDE_FOLDERS = ["node_modules", "templates", "attachments", "tmp"];
 var DEFAULT_CONFIG = {
   exclude: [],
   exclude_entity_folders: [],
@@ -7238,6 +7239,7 @@ function addNoteToIndex(index, note) {
 async function upsertNote(index, vaultPath2, notePath) {
   try {
     const existed = index.notes.has(notePath);
+    const oldNote = existed ? index.notes.get(notePath) : void 0;
     let releasedKeys = [];
     if (existed) {
       releasedKeys = removeNoteFromIndex(index, notePath);
@@ -7255,10 +7257,12 @@ async function upsertNote(index, vaultPath2, notePath) {
     if (releasedKeys.length > 0) {
       reconcileReleasedKeys(index, releasedKeys);
     }
+    const entityFieldChanged = !oldNote || (oldNote.title !== note.title || JSON.stringify(oldNote.aliases) !== JSON.stringify(note.aliases) || oldNote.frontmatter.type !== note.frontmatter.type);
     return {
       success: true,
       action: existed ? "updated" : "added",
-      path: notePath
+      path: notePath,
+      entityFieldChanged
     };
   } catch (error) {
     return {
@@ -7300,7 +7304,8 @@ async function processBatch(index, vaultPath2, batch, options = {}) {
       successful: 0,
       failed: 0,
       results: [],
-      durationMs: 0
+      durationMs: 0,
+      hasEntityRelevantChanges: false
     };
   }
   console.error(`[flywheel] Processing ${total} file events`);
@@ -7355,7 +7360,8 @@ async function processBatch(index, vaultPath2, batch, options = {}) {
     successful,
     failed,
     results,
-    durationMs
+    durationMs,
+    hasEntityRelevantChanges: results.some((r) => r.entityFieldChanged)
   };
 }
 
@@ -8427,6 +8433,7 @@ var PipelineRunner = class {
   entitiesAfter = [];
   entitiesBefore = [];
   hubBefore = /* @__PURE__ */ new Map();
+  hasEntityRelevantChanges = false;
   forwardLinkResults = [];
   linkDiffs = [];
   survivedLinks = [];
@@ -8498,6 +8505,7 @@ var PipelineRunner = class {
     if (!vaultIndex2) {
       const rebuilt = await p.buildVaultIndex(p.vp);
       p.updateVaultIndex(rebuilt);
+      this.hasEntityRelevantChanges = true;
       serverLog("watcher", `Index rebuilt (full): ${rebuilt.notes.size} notes, ${rebuilt.entities.size} entities`);
     } else {
       const absoluteBatch = {
@@ -8505,6 +8513,7 @@ var PipelineRunner = class {
         events: p.events.map((e) => ({ ...e, path: path15.join(p.vp, e.path) }))
       };
       const batchResult = await processBatch(vaultIndex2, p.vp, absoluteBatch);
+      this.hasEntityRelevantChanges = batchResult.hasEntityRelevantChanges;
       serverLog("watcher", `Incremental: ${batchResult.successful}/${batchResult.total} files in ${batchResult.durationMs}ms`);
     }
     p.updateIndexState("ready");
@@ -8551,7 +8560,7 @@ var PipelineRunner = class {
       for (const r of rows) this.hubBefore.set(r.name, r.hub_score);
     }
     const entityScanAgeMs = p.ctx.lastEntityScanAt > 0 ? Date.now() - p.ctx.lastEntityScanAt : Infinity;
-    if (entityScanAgeMs < 5 * 60 * 1e3) {
+    if (entityScanAgeMs < 5 * 60 * 1e3 && !this.hasEntityRelevantChanges) {
       tracker.start("entity_scan", {});
       tracker.skip("entity_scan", `cache valid (${Math.round(entityScanAgeMs / 1e3)}s old)`);
       this.entitiesBefore = p.sd ? getAllEntitiesFromDb(p.sd) : [];
@@ -9329,7 +9338,7 @@ async function flushLogs() {
 
 // src/index.ts
 init_embeddings();
-import { openStateDb, scanVaultEntities as scanVaultEntities4, getAllEntitiesFromDb as getAllEntitiesFromDb5, loadContentHashes, saveContentHashBatch, renameContentHash, checkDbIntegrity as checkDbIntegrity2, safeBackupAsync as safeBackupAsync2, preserveCorruptedDb, deleteStateDbFiles, attemptSalvage } from "@velvetmonkey/vault-core";
+import { openStateDb, scanVaultEntities as scanVaultEntities4, getAllEntitiesFromDb as getAllEntitiesFromDb6, loadContentHashes, saveContentHashBatch, renameContentHash, checkDbIntegrity as checkDbIntegrity2, safeBackupAsync as safeBackupAsync2, preserveCorruptedDb, deleteStateDbFiles, attemptSalvage } from "@velvetmonkey/vault-core";
 
 // src/core/write/memory.ts
 init_wikilinkFeedback();
@@ -10723,6 +10732,20 @@ import { z } from "zod";
 // src/core/read/constants.ts
 var MAX_LIMIT = 200;
 
+// src/core/read/indexGuard.ts
+function requireIndex() {
+  const state2 = getIndexState();
+  if (state2 === "building") {
+    const { parsed, total } = getIndexProgress();
+    const progress = total > 0 ? ` (${parsed}/${total} files)` : "";
+    throw new Error(`Index building${progress}... try again shortly`);
+  }
+  if (state2 === "error") {
+    const error = getIndexError();
+    throw new Error(`Index failed to build: ${error?.message || "unknown error"}`);
+  }
+}
+
 // src/tools/read/query.ts
 init_embeddings();
 import {
@@ -11747,6 +11770,7 @@ function registerQueryTools(server2, getIndex, getVaultPath, getStateDb3) {
       consumer: z.enum(["llm", "human"]).default("llm").describe('Output format: "llm" applies sandwich ordering and strips scoring fields for context efficiency. "human" preserves score order and all scoring metadata for UI display.')
     },
     async ({ query, where, has_tag, has_any_tag, has_all_tags, include_children, folder, title_contains, modified_after, modified_before, sort_by, order, prefix, limit: requestedLimit, detail_count: requestedDetailCount, context_note, consumer }) => {
+      requireIndex();
       const limit = Math.min(requestedLimit ?? 10, MAX_LIMIT);
       const detailN = requestedDetailCount ?? 5;
       const index = getIndex();
@@ -12466,20 +12490,6 @@ function detectCycles(index, maxLength = 10, limit = 20) {
   return cycles;
 }
 
-// src/core/read/indexGuard.ts
-function requireIndex() {
-  const state2 = getIndexState();
-  if (state2 === "building") {
-    const { parsed, total } = getIndexProgress();
-    const progress = total > 0 ? ` (${parsed}/${total} files)` : "";
-    throw new Error(`Index building${progress}... try again shortly`);
-  }
-  if (state2 === "error") {
-    const error = getIndexError();
-    throw new Error(`Index failed to build: ${error?.message || "unknown error"}`);
-  }
-}
-
 // src/tools/read/graph.ts
 async function getContext(vaultPath2, sourcePath, line, contextLines = 1) {
   try {
@@ -14656,7 +14666,7 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig2 = () =>
 import * as fs15 from "fs";
 import * as path18 from "path";
 import { z as z6 } from "zod";
-import { scanVaultEntities as scanVaultEntities2, getEntityIndexFromDb as getEntityIndexFromDb2 } from "@velvetmonkey/vault-core";
+import { scanVaultEntities as scanVaultEntities2, getEntityIndexFromDb as getEntityIndexFromDb2, getAllEntitiesFromDb as getAllEntitiesFromDb3 } from "@velvetmonkey/vault-core";
 
 // src/core/read/aliasSuggestions.ts
 import { STOPWORDS_EN as STOPWORDS_EN3 } from "@velvetmonkey/vault-core";
@@ -14726,6 +14736,10 @@ function suggestEntityAliases(stateDb2, folder) {
 // src/tools/read/system.ts
 init_edgeWeights();
 init_embeddings();
+init_wikilinks();
+init_wikilinkFeedback();
+init_recency();
+init_cooccurrence();
 function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfig, getStateDb3) {
   const RefreshIndexOutputSchema = {
     success: z6.boolean().describe("Whether the refresh succeeded"),
@@ -14733,7 +14747,15 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
     entities_count: z6.number().describe("Number of entities (titles + aliases)"),
     fts5_notes: z6.number().describe("Number of notes in FTS5 search index"),
     edges_recomputed: z6.number().optional().describe("Number of edges with recomputed weights"),
+    hub_scores: z6.number().optional().describe("Number of hub scores exported"),
+    graph_snapshot: z6.boolean().optional().describe("Whether graph topology snapshot was recorded"),
+    suppression_list: z6.boolean().optional().describe("Whether wikilink suppression list was updated"),
+    task_cache: z6.boolean().optional().describe("Whether task cache was refreshed"),
     embeddings_refreshed: z6.number().optional().describe("Number of note embeddings updated"),
+    entity_embeddings_refreshed: z6.number().optional().describe("Number of entity embeddings updated"),
+    recency_rebuilt: z6.boolean().optional().describe("Whether recency index was rebuilt"),
+    cooccurrence_associations: z6.number().optional().describe("Number of co-occurrence associations rebuilt"),
+    index_cached: z6.boolean().optional().describe("Whether vault index cache was saved"),
     duration_ms: z6.number().describe("Time taken to rebuild index")
   };
   server2.registerTool(
@@ -14747,90 +14769,224 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
     async () => {
       const vaultPath2 = getVaultPath();
       const startTime = Date.now();
+      const tracker = createStepTracker();
       setIndexState("building");
       setIndexError(null);
       try {
+        tracker.start("vault_index", {});
         const newIndex = await buildVaultIndex(vaultPath2);
         setIndex(newIndex);
         setIndexState("ready");
+        tracker.end({ notes: newIndex.notes.size, entities: newIndex.entities.size });
         const stateDb2 = getStateDb3?.();
         if (stateDb2) {
+          tracker.start("entity_sync", {});
           try {
             const config = loadConfig(stateDb2);
+            const excludeFolders = config.exclude_entity_folders?.length ? config.exclude_entity_folders : DEFAULT_ENTITY_EXCLUDE_FOLDERS;
             const entityIndex2 = await scanVaultEntities2(vaultPath2, {
-              excludeFolders: [
-                "daily-notes",
-                "daily",
-                "weekly",
-                "weekly-notes",
-                "monthly",
-                "monthly-notes",
-                "quarterly",
-                "yearly-notes",
-                "periodic",
-                "journal",
-                "inbox",
-                "templates",
-                "attachments",
-                "tmp",
-                "clippings",
-                "readwise",
-                "articles",
-                "bookmarks",
-                "web-clips"
-              ],
+              excludeFolders,
               customCategories: config.custom_categories
             });
             stateDb2.replaceAllEntities(entityIndex2);
+            tracker.end({ entities: entityIndex2._metadata.total_entities });
             console.error(`[Flywheel] Updated ${entityIndex2._metadata.total_entities} entities in StateDb`);
           } catch (e) {
+            tracker.end({ error: String(e) });
             console.error("[Flywheel] Failed to update entities:", e);
           }
         }
+        let flywheelConfig2;
         if (setConfig) {
+          tracker.start("config_merge", {});
           const existing = loadConfig(stateDb2);
           const inferred = inferConfig(newIndex, vaultPath2);
           if (stateDb2) {
             saveConfig(stateDb2, inferred, existing);
           }
-          setConfig(loadConfig(stateDb2));
+          flywheelConfig2 = loadConfig(stateDb2);
+          setConfig(flywheelConfig2);
+          tracker.end({});
         }
         let fts5Notes = 0;
+        tracker.start("fts5_rebuild", {});
         try {
           const ftsState = await buildFTS5Index(vaultPath2);
           fts5Notes = ftsState.noteCount;
+          tracker.end({ notes: fts5Notes });
           console.error(`[Flywheel] FTS5 index rebuilt: ${fts5Notes} notes`);
         } catch (err) {
+          tracker.end({ error: String(err) });
           console.error("[Flywheel] FTS5 rebuild failed:", err);
         }
         let edgesRecomputed = 0;
         if (stateDb2) {
+          tracker.start("edge_weights", {});
           try {
             const edgeResult = recomputeEdgeWeights(stateDb2);
             edgesRecomputed = edgeResult.edges_updated;
+            tracker.end({ edges: edgeResult.edges_updated, duration_ms: edgeResult.duration_ms });
             console.error(`[Flywheel] Edge weights: ${edgeResult.edges_updated} edges in ${edgeResult.duration_ms}ms`);
           } catch (err) {
+            tracker.end({ error: String(err) });
             console.error("[Flywheel] Edge weight recompute failed:", err);
           }
         }
+        tracker.start("entity_index_init", {});
+        try {
+          await initializeEntityIndex(vaultPath2);
+          tracker.end({});
+          console.error("[Flywheel] Entity index initialized");
+        } catch (err) {
+          tracker.end({ error: String(err) });
+          console.error("[Flywheel] Entity index init failed:", err);
+        }
+        let hubScoresExported = 0;
+        tracker.start("hub_scores", {});
+        try {
+          hubScoresExported = await exportHubScores(newIndex, stateDb2);
+          tracker.end({ exported: hubScoresExported });
+          if (hubScoresExported > 0) {
+            console.error(`[Flywheel] Hub scores: ${hubScoresExported} entities`);
+          }
+        } catch (err) {
+          tracker.end({ error: String(err) });
+          console.error("[Flywheel] Hub score export failed:", err);
+        }
+        let graphSnapshotRecorded = false;
+        if (stateDb2) {
+          tracker.start("graph_snapshot", {});
+          try {
+            const graphMetrics = computeGraphMetrics(newIndex);
+            recordGraphSnapshot(stateDb2, graphMetrics);
+            graphSnapshotRecorded = true;
+            tracker.end({ recorded: true });
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Graph snapshot failed:", err);
+          }
+        }
+        let suppressionUpdated = false;
+        if (stateDb2) {
+          tracker.start("suppression_list", {});
+          try {
+            updateSuppressionList(stateDb2);
+            suppressionUpdated = true;
+            tracker.end({ updated: true });
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Suppression list update failed:", err);
+          }
+        }
+        let recencyRebuilt = false;
+        if (stateDb2) {
+          tracker.start("recency", {});
+          try {
+            const entities = getAllEntitiesFromDb3(stateDb2).map((e) => ({
+              name: e.name,
+              path: e.path,
+              aliases: e.aliases
+            }));
+            const recencyIndex2 = await buildRecencyIndex(vaultPath2, entities);
+            saveRecencyToStateDb(recencyIndex2, stateDb2);
+            recencyRebuilt = true;
+            tracker.end({ entities: recencyIndex2.lastMentioned.size });
+            console.error(`[Flywheel] Recency: rebuilt ${recencyIndex2.lastMentioned.size} entities`);
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Recency rebuild failed:", err);
+          }
+        }
+        let cooccurrenceAssociations;
+        if (stateDb2) {
+          tracker.start("cooccurrence", {});
+          try {
+            const entityNames = getAllEntitiesFromDb3(stateDb2).map((e) => e.name);
+            const cooccurrenceIdx = await mineCooccurrences(vaultPath2, entityNames);
+            setCooccurrenceIndex(cooccurrenceIdx);
+            saveCooccurrenceToStateDb(stateDb2, cooccurrenceIdx);
+            cooccurrenceAssociations = cooccurrenceIdx._metadata.total_associations;
+            tracker.end({ associations: cooccurrenceAssociations });
+            console.error(`[Flywheel] Co-occurrence: rebuilt ${cooccurrenceAssociations} associations`);
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Co-occurrence rebuild failed:", err);
+          }
+        }
+        let taskCacheRefreshed = false;
+        tracker.start("task_cache", {});
+        try {
+          if (!flywheelConfig2) {
+            flywheelConfig2 = loadConfig(stateDb2);
+          }
+          await buildTaskCache(vaultPath2, newIndex, getExcludeTags(flywheelConfig2));
+          taskCacheRefreshed = true;
+          tracker.end({ rebuilt: true });
+          console.error("[Flywheel] Task cache rebuilt");
+        } catch (err) {
+          tracker.end({ error: String(err) });
+          console.error("[Flywheel] Task cache rebuild failed:", err);
+        }
         let embeddingsRefreshed = 0;
         if (hasEmbeddingsIndex()) {
+          tracker.start("embeddings_sync", {});
           try {
             const progress = await buildEmbeddingsIndex(vaultPath2);
             embeddingsRefreshed = progress.total - progress.skipped;
+            tracker.end({ refreshed: embeddingsRefreshed });
             if (embeddingsRefreshed > 0) {
               console.error(`[Flywheel] Embeddings: ${embeddingsRefreshed} notes updated`);
             }
           } catch (err) {
+            tracker.end({ error: String(err) });
             console.error("[Flywheel] Embedding sync failed:", err);
           }
         }
+        let entityEmbeddingsRefreshed = 0;
+        if (stateDb2 && hasEntityEmbeddingsIndex()) {
+          tracker.start("entity_embeddings", {});
+          try {
+            const entities = getAllEntitiesFromDb3(stateDb2);
+            if (entities.length > 0) {
+              const entityMap = new Map(entities.map((e) => [e.name, {
+                name: e.name,
+                path: e.path,
+                category: e.category,
+                aliases: e.aliases
+              }]));
+              entityEmbeddingsRefreshed = await buildEntityEmbeddingsIndex(vaultPath2, entityMap);
+              loadEntityEmbeddingsToMemory();
+              tracker.end({ refreshed: entityEmbeddingsRefreshed });
+              if (entityEmbeddingsRefreshed > 0) {
+                console.error(`[Flywheel] Entity embeddings: ${entityEmbeddingsRefreshed} updated`);
+              }
+            } else {
+              tracker.end({ refreshed: 0 });
+            }
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Entity embedding sync failed:", err);
+          }
+        }
+        let indexCached = false;
+        if (stateDb2) {
+          tracker.start("index_cache", {});
+          try {
+            saveVaultIndexToCache(stateDb2, newIndex);
+            indexCached = true;
+            tracker.end({ cached: true });
+          } catch (err) {
+            tracker.end({ error: String(err) });
+            console.error("[Flywheel] Index cache save failed:", err);
+          }
+        }
         const duration = Date.now() - startTime;
         if (stateDb2) {
           recordIndexEvent(stateDb2, {
             trigger: "manual_refresh",
             duration_ms: duration,
-            note_count: newIndex.notes.size
+            note_count: newIndex.notes.size,
+            steps: tracker.steps
           });
         }
         const output = {
@@ -14839,7 +14995,15 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
           entities_count: newIndex.entities.size,
           fts5_notes: fts5Notes,
           edges_recomputed: edgesRecomputed,
+          hub_scores: hubScoresExported || void 0,
+          graph_snapshot: graphSnapshotRecorded || void 0,
+          suppression_list: suppressionUpdated || void 0,
+          task_cache: taskCacheRefreshed || void 0,
           embeddings_refreshed: embeddingsRefreshed || void 0,
+          entity_embeddings_refreshed: entityEmbeddingsRefreshed || void 0,
+          recency_rebuilt: recencyRebuilt || void 0,
+          cooccurrence_associations: cooccurrenceAssociations,
+          index_cached: indexCached || void 0,
           duration_ms: duration
         };
         return {
@@ -22728,7 +22892,7 @@ function registerSimilarityTools(server2, getIndex, getVaultPath, getStateDb3) {
 // src/tools/read/semantic.ts
 init_embeddings();
 import { z as z32 } from "zod";
-import { getAllEntitiesFromDb as getAllEntitiesFromDb3 } from "@velvetmonkey/vault-core";
+import { getAllEntitiesFromDb as getAllEntitiesFromDb4 } from "@velvetmonkey/vault-core";
 function registerSemanticTools(server2, getVaultPath, getStateDb3) {
   server2.registerTool(
     "init_semantic",
@@ -22792,7 +22956,7 @@ function registerSemanticTools(server2, getVaultPath, getStateDb3) {
       const embedded = progress.total - progress.skipped;
       let entityEmbedded = 0;
       try {
-        const allEntities = getAllEntitiesFromDb3(stateDb2);
+        const allEntities = getAllEntitiesFromDb4(stateDb2);
         const entityMap = /* @__PURE__ */ new Map();
         for (const e of allEntities) {
           entityMap.set(e.name, {
@@ -22844,7 +23008,7 @@ function registerSemanticTools(server2, getVaultPath, getStateDb3) {
 // src/tools/read/merges.ts
 init_levenshtein();
 import { z as z33 } from "zod";
-import { getAllEntitiesFromDb as getAllEntitiesFromDb4, getDismissedMergePairs, recordMergeDismissal } from "@velvetmonkey/vault-core";
+import { getAllEntitiesFromDb as getAllEntitiesFromDb5, getDismissedMergePairs, recordMergeDismissal } from "@velvetmonkey/vault-core";
 function normalizeName(name) {
   return name.toLowerCase().replace(/[.\-_]/g, "").replace(/js$/, "").replace(/ts$/, "");
 }
@@ -22862,7 +23026,7 @@ function registerMergeTools2(server2, getStateDb3) {
           content: [{ type: "text", text: JSON.stringify({ suggestions: [], error: "StateDb not available" }) }]
         };
       }
-      const entities = getAllEntitiesFromDb4(stateDb2);
+      const entities = getAllEntitiesFromDb5(stateDb2);
       if (entities.length === 0) {
         return {
           content: [{ type: "text", text: JSON.stringify({ suggestions: [] }) }]
@@ -25122,7 +25286,6 @@ async function main() {
     })();
   }
 }
-var DEFAULT_ENTITY_EXCLUDE_FOLDERS = ["node_modules", "templates", "attachments", "tmp"];
 async function updateEntitiesInStateDb(vp, sd) {
   const db4 = sd ?? stateDb;
   const vault = vp ?? vaultPath;
@@ -25293,7 +25456,7 @@ async function runPostIndexWork(ctx) {
             }
           });
           if (sd) {
-            const entities = getAllEntitiesFromDb5(sd);
+            const entities = getAllEntitiesFromDb6(sd);
             if (entities.length > 0) {
               const entityMap = new Map(entities.map((e) => [e.name, {
                 name: e.name,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@velvetmonkey/flywheel-memory",
-  "version": "2.0.155",
+  "version": "2.0.156",
   "description": "MCP tools that search, write, and auto-link your Obsidian vault — and learn from your edits.",
   "type": "module",
   "main": "dist/index.js",
@@ -54,7 +54,7 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "@modelcontextprotocol/sdk": "^1.25.1",
-    "@velvetmonkey/vault-core": "^2.0.155",
+    "@velvetmonkey/vault-core": "^2.0.156",
     "better-sqlite3": "^12.0.0",
     "chokidar": "^4.0.0",
     "gray-matter": "^4.0.3",