@velvetmonkey/flywheel-memory 2.0.22 → 2.0.24

package/dist/index.js

@@ -1839,6 +1839,7 @@ var MAX_FILE_SIZE2 = 5 * 1024 * 1024;
 var db = null;
 var pipeline = null;
 var initPromise = null;
+var embeddingsBuilding = false;
 var embeddingCache = /* @__PURE__ */ new Map();
 var EMBEDDING_CACHE_MAX = 500;
 var entityEmbeddingsMap = /* @__PURE__ */ new Map();
@@ -1894,6 +1895,7 @@ async function buildEmbeddingsIndex(vaultPath2, onProgress) {
   if (!db) {
     throw new Error("Embeddings database not initialized. Call setEmbeddingsDatabase() first.");
   }
+  embeddingsBuilding = true;
   await initEmbeddings();
   const files = await scanVault(vaultPath2);
   const indexable = files.filter((f) => shouldIndexFile(f.path));
@@ -1937,6 +1939,7 @@ async function buildEmbeddingsIndex(vaultPath2, onProgress) {
       deleteStmt.run(existingPath);
     }
   }
+  embeddingsBuilding = false;
   console.error(`[Semantic] Indexed ${progress.current - progress.skipped} notes, skipped ${progress.skipped}`);
   return progress;
 }
@@ -2034,6 +2037,12 @@ function reciprocalRankFusion(...lists) {
   }
   return scores;
 }
+function isEmbeddingsBuilding() {
+  return embeddingsBuilding;
+}
+function setEmbeddingsBuilding(value) {
+  embeddingsBuilding = value;
+}
 function hasEmbeddingsIndex() {
   if (!db) return false;
   try {
@@ -5010,6 +5019,15 @@ function getEntityIndexStats() {
       organizations: entityIndex.organizations?.length ?? 0,
       locations: entityIndex.locations?.length ?? 0,
       concepts: entityIndex.concepts?.length ?? 0,
+      animals: entityIndex.animals?.length ?? 0,
+      media: entityIndex.media?.length ?? 0,
+      events: entityIndex.events?.length ?? 0,
+      documents: entityIndex.documents?.length ?? 0,
+      vehicles: entityIndex.vehicles?.length ?? 0,
+      health: entityIndex.health?.length ?? 0,
+      finance: entityIndex.finance?.length ?? 0,
+      food: entityIndex.food?.length ?? 0,
+      hobbies: entityIndex.hobbies?.length ?? 0,
       other: entityIndex.other.length
     }
   };
@@ -5146,14 +5164,32 @@ var DEFAULT_STRICTNESS = "conservative";
 var TYPE_BOOST = {
   people: 5,
   // Names are high value for connections
+  animals: 3,
+  // Pets and animals are personal and specific
   projects: 3,
   // Projects provide context
   organizations: 2,
   // Companies/teams relevant
+  events: 2,
+  // Meetings, trips, milestones
+  media: 2,
+  // Movies, books, shows
+  health: 2,
+  // Medical, fitness — personal relevance
+  vehicles: 2,
+  // Cars, bikes — specific items
   locations: 1,
   // Geographic context
   concepts: 1,
   // Abstract concepts
+  documents: 1,
+  // Reports, guides
+  food: 1,
+  // Recipes, restaurants
+  hobbies: 1,
+  // Crafts, sports
+  finance: 1,
+  // Accounts, budgets
   technologies: 0,
   // Common, avoid over-suggesting
   acronyms: 0,
@@ -5197,14 +5233,24 @@ var CONTEXT_BOOST = {
   daily: {
     people: 5,
     // Daily notes often mention people
-    projects: 2
+    animals: 3,
+    // Pets in daily life
+    events: 3,
+    // Daily events and meetings
+    projects: 2,
     // Work updates reference projects
+    food: 2,
+    // Meals and recipes in daily logs
+    health: 2
+    // Fitness and wellness tracking
   },
   project: {
     projects: 5,
     // Project docs reference other projects
-    technologies: 2
+    technologies: 2,
     // Technical dependencies
+    documents: 2
+    // Reference documents
   },
   tech: {
     technologies: 5,
@@ -5722,9 +5768,18 @@ var EXCLUDED_DIRS3 = /* @__PURE__ */ new Set([
 ]);
 var MAX_INDEX_FILE_SIZE = 5 * 1024 * 1024;
 var STALE_THRESHOLD_MS = 60 * 60 * 1e3;
+function splitFrontmatter(raw) {
+  if (!raw.startsWith("---")) return { frontmatter: "", body: raw };
+  const end = raw.indexOf("\n---", 3);
+  if (end === -1) return { frontmatter: "", body: raw };
+  const yaml = raw.substring(4, end);
+  const values = yaml.split("\n").map((line) => line.replace(/^[\s-]*/, "").replace(/^[\w]+:\s*/, "")).filter((v) => v && !v.startsWith("[") && !v.startsWith("{")).join(" ");
+  return { frontmatter: values, body: raw.substring(end + 4) };
+}
 var db2 = null;
 var state = {
   ready: false,
+  building: false,
   lastBuilt: null,
   noteCount: 0,
   error: null
@@ -5740,6 +5795,7 @@ function setFTS5Database(database) {
       const countRow = db2.prepare("SELECT COUNT(*) as count FROM notes_fts").get();
       state = {
         ready: countRow.count > 0,
+        building: false,
         lastBuilt,
         noteCount: countRow.count,
         error: null
@@ -5755,6 +5811,7 @@ function shouldIndexFile2(filePath) {
 async function buildFTS5Index(vaultPath2) {
   try {
     state.error = null;
+    state.building = true;
     if (!db2) {
       throw new Error("FTS5 database not initialized. Call setFTS5Database() first.");
     }
@@ -5762,7 +5819,7 @@ async function buildFTS5Index(vaultPath2) {
     const files = await scanVault(vaultPath2);
     const indexableFiles = files.filter((f) => shouldIndexFile2(f.path));
     const insert = db2.prepare(
-      "INSERT INTO notes_fts (path, title, content) VALUES (?, ?, ?)"
+      "INSERT INTO notes_fts (path, title, frontmatter, content) VALUES (?, ?, ?, ?)"
     );
     const insertMany = db2.transaction((filesToIndex) => {
       let indexed2 = 0;
@@ -5772,9 +5829,10 @@ async function buildFTS5Index(vaultPath2) {
           if (stats.size > MAX_INDEX_FILE_SIZE) {
             continue;
           }
-          const content = fs7.readFileSync(file.absolutePath, "utf-8");
+          const raw = fs7.readFileSync(file.absolutePath, "utf-8");
+          const { frontmatter, body } = splitFrontmatter(raw);
           const title = file.path.replace(/\.md$/, "").split("/").pop() || file.path;
-          insert.run(file.path, title, content);
+          insert.run(file.path, title, frontmatter, body);
           indexed2++;
         } catch (err) {
           console.error(`[FTS5] Skipping ${file.path}:`, err);
@@ -5789,6 +5847,7 @@ async function buildFTS5Index(vaultPath2) {
     ).run("last_built", now.toISOString());
     state = {
       ready: true,
+      building: false,
       lastBuilt: now,
       noteCount: indexed,
       error: null
@@ -5798,6 +5857,7 @@ async function buildFTS5Index(vaultPath2) {
   } catch (err) {
     state = {
       ready: false,
+      building: false,
       lastBuilt: null,
       noteCount: 0,
       error: err instanceof Error ? err.message : String(err)
@@ -5832,10 +5892,10 @@ function searchFTS5(_vaultPath, query, limit = 10) {
       SELECT
         path,
         title,
-        snippet(notes_fts, 2, '[', ']', '...', 20) as snippet
+        snippet(notes_fts, 3, '<mark>', '</mark>', '...', 20) as snippet
       FROM notes_fts
       WHERE notes_fts MATCH ?
-      ORDER BY rank
+      ORDER BY bm25(notes_fts, 0.0, 5.0, 10.0, 1.0)
       LIMIT ?
     `);
     const results = stmt.all(query, limit);
@@ -6766,6 +6826,9 @@ function getActivitySummary(index, days) {
   };
 }
 
+// src/tools/read/health.ts
+import { SCHEMA_VERSION } from "@velvetmonkey/vault-core";
+
 // src/core/shared/indexActivity.ts
 function recordIndexEvent(stateDb2, event) {
   stateDb2.db.prepare(
@@ -6858,6 +6921,7 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig = () =>
   });
   const HealthCheckOutputSchema = {
     status: z3.enum(["healthy", "degraded", "unhealthy"]).describe("Overall health status"),
+    schema_version: z3.coerce.number().describe("StateDb schema version"),
     vault_accessible: z3.boolean().describe("Whether the vault path is accessible"),
     vault_path: z3.string().describe("The vault path being used"),
     index_state: z3.enum(["building", "ready", "error"]).describe("Current state of the vault index"),
@@ -6877,6 +6941,11 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig = () =>
       duration_ms: z3.number(),
       ago_seconds: z3.number()
     }).optional().describe("Most recent index rebuild event"),
+    fts5_ready: z3.boolean().describe("Whether the FTS5 keyword search index is ready"),
+    fts5_building: z3.boolean().describe("Whether the FTS5 keyword search index is currently building"),
+    embeddings_building: z3.boolean().describe("Whether semantic embeddings are currently building"),
+    embeddings_ready: z3.boolean().describe("Whether semantic embeddings have been built (enables hybrid keyword+semantic search)"),
+    embeddings_count: z3.coerce.number().describe("Number of notes with semantic embeddings"),
     recommendations: z3.array(z3.string()).describe("Suggested actions if any issues detected")
   };
   server2.registerTool(
@@ -6962,8 +7031,10 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig = () =>
         } catch {
         }
       }
+      const ftsState = getFTS5State();
       const output = {
         status,
+        schema_version: SCHEMA_VERSION,
         vault_accessible: vaultAccessible,
         vault_path: vaultPath2,
         index_state: indexState2,
@@ -6978,6 +7049,11 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig = () =>
         periodic_notes: periodicNotes && periodicNotes.length > 0 ? periodicNotes : void 0,
         config: configInfo,
         last_rebuild: lastRebuild,
+        fts5_ready: ftsState.ready,
+        fts5_building: ftsState.building,
+        embeddings_building: isEmbeddingsBuilding(),
+        embeddings_ready: hasEmbeddingsIndex(),
+        embeddings_count: getEmbeddingsCount(),
         recommendations
       };
       return {
@@ -7320,27 +7396,55 @@ function registerQueryTools(server2, getIndex, getVaultPath, getStateDb) {
           return { content: [{ type: "text", text: JSON.stringify({ error: "query is required for content search" }, null, 2) }] };
         }
         const ftsState = getFTS5State();
+        if (ftsState.building) {
+          return { content: [{ type: "text", text: JSON.stringify({
+            scope,
+            method: "fts5",
+            query,
+            building: true,
+            total_results: 0,
+            results: [],
+            message: "Search index is building, try again shortly"
+          }, null, 2) }] };
+        }
         if (!ftsState.ready || isIndexStale(vaultPath2)) {
           console.error("[FTS5] Index stale or missing, rebuilding...");
           await buildFTS5Index(vaultPath2);
         }
         const fts5Results = searchFTS5(vaultPath2, query, limit);
+        let entityResults = [];
+        if (scope === "all") {
+          const stateDb2 = getStateDb();
+          if (stateDb2) {
+            try {
+              entityResults = searchEntities(stateDb2, query, limit);
+            } catch {
+            }
+          }
+        }
         if (hasEmbeddingsIndex()) {
           try {
             const semanticResults = await semanticSearch(query, limit);
             const fts5Ranked = fts5Results.map((r) => ({ path: r.path, title: r.title, snippet: r.snippet }));
             const semanticRanked = semanticResults.map((r) => ({ path: r.path, title: r.title }));
-            const rrfScores = reciprocalRankFusion(fts5Ranked, semanticRanked);
-            const allPaths = /* @__PURE__ */ new Set([...fts5Results.map((r) => r.path), ...semanticResults.map((r) => r.path)]);
+            const entityRanked = entityResults.map((r) => ({ path: r.path, title: r.name }));
+            const rrfScores = reciprocalRankFusion(fts5Ranked, semanticRanked, entityRanked);
+            const allPaths = /* @__PURE__ */ new Set([
+              ...fts5Results.map((r) => r.path),
+              ...semanticResults.map((r) => r.path),
+              ...entityResults.map((r) => r.path)
+            ]);
             const fts5Map = new Map(fts5Results.map((r) => [r.path, r]));
             const semanticMap = new Map(semanticResults.map((r) => [r.path, r]));
+            const entityMap = new Map(entityResults.map((r) => [r.path, r]));
             const merged = Array.from(allPaths).map((p) => ({
               path: p,
-              title: fts5Map.get(p)?.title || semanticMap.get(p)?.title || p.replace(/\.md$/, "").split("/").pop() || p,
+              title: fts5Map.get(p)?.title || semanticMap.get(p)?.title || entityMap.get(p)?.name || p.replace(/\.md$/, "").split("/").pop() || p,
               snippet: fts5Map.get(p)?.snippet,
               rrf_score: Math.round((rrfScores.get(p) || 0) * 1e4) / 1e4,
               in_fts5: fts5Map.has(p),
-              in_semantic: semanticMap.has(p)
+              in_semantic: semanticMap.has(p),
+              in_entity: entityMap.has(p)
             }));
             merged.sort((a, b) => b.rrf_score - a.rrf_score);
             return { content: [{ type: "text", text: JSON.stringify({
@@ -7354,6 +7458,21 @@ function registerQueryTools(server2, getIndex, getVaultPath, getStateDb) {
             console.error("[Semantic] Hybrid search failed, falling back to FTS5:", err instanceof Error ? err.message : err);
           }
         }
+        if (entityResults.length > 0) {
+          const fts5Map = new Map(fts5Results.map((r) => [r.path, r]));
+          const entityRanked = entityResults.filter((r) => !fts5Map.has(r.path));
+          const merged = [
+            ...fts5Results.map((r) => ({ path: r.path, title: r.title, snippet: r.snippet, in_entity: fts5Map.has(r.path) && entityResults.some((e) => e.path === r.path) })),
+            ...entityRanked.map((r) => ({ path: r.path, title: r.name, snippet: void 0, in_entity: true }))
+          ];
+          return { content: [{ type: "text", text: JSON.stringify({
+            scope: "content",
+            method: "fts5",
+            query,
+            total_results: merged.length,
+            results: merged.slice(0, limit)
+          }, null, 2) }] };
+        }
         return { content: [{ type: "text", text: JSON.stringify({
           scope: "content",
           method: "fts5",
@@ -14147,11 +14266,11 @@ function findSimilarNotes(db3, vaultPath2, index, sourcePath, options = {}) {
       SELECT
         path,
         title,
-        bm25(notes_fts) as score,
-        snippet(notes_fts, 2, '[', ']', '...', 15) as snippet
+        bm25(notes_fts, 0.0, 5.0, 10.0, 1.0) as score,
+        snippet(notes_fts, 3, '[', ']', '...', 15) as snippet
       FROM notes_fts
       WHERE notes_fts MATCH ?
-      ORDER BY rank
+      ORDER BY bm25(notes_fts, 0.0, 5.0, 10.0, 1.0)
       LIMIT ?
     `).all(query, limit + 20);
     let filtered = results.filter((r) => r.path !== sourcePath);
@@ -14763,7 +14882,6 @@ async function main() {
     setEmbeddingsDatabase(stateDb.db);
     loadEntityEmbeddingsToMemory();
     setWriteStateDb(stateDb);
-    await initializeEntityIndex(vaultPath);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     console.error(`[Memory] StateDb initialization failed: ${msg}`);
@@ -14782,6 +14900,15 @@ async function main() {
   initializeLogger2(vaultPath).catch((err) => {
     console.error(`[Memory] Write logger initialization failed: ${err}`);
   });
+  if (isIndexStale(vaultPath)) {
+    buildFTS5Index(vaultPath).then(() => {
+      console.error("[Memory] FTS5 search index ready");
+    }).catch((err) => {
+      console.error("[Memory] FTS5 build failed:", err);
+    });
+  } else {
+    console.error("[Memory] FTS5 search index already fresh, skipping rebuild");
+  }
   let cachedIndex = null;
   if (stateDb) {
     try {
@@ -14879,6 +15006,7 @@ async function updateEntitiesInStateDb() {
 }
 async function runPostIndexWork(index) {
   await updateEntitiesInStateDb();
+  await initializeEntityIndex(vaultPath);
   await exportHubScores(index, stateDb);
   if (stateDb) {
     try {
@@ -14917,6 +15045,33 @@ async function runPostIndexWork(index) {
   if (flywheelConfig.vault_name) {
     console.error(`[Memory] Vault: ${flywheelConfig.vault_name}`);
   }
+  if (hasEmbeddingsIndex()) {
+    console.error("[Memory] Embeddings already built, skipping full scan");
+  } else {
+    setEmbeddingsBuilding(true);
+    buildEmbeddingsIndex(vaultPath, (p) => {
+      if (p.current % 100 === 0 || p.current === p.total) {
+        console.error(`[Semantic] ${p.current}/${p.total}`);
+      }
+    }).then(async () => {
+      if (stateDb) {
+        const entities = getAllEntitiesFromDb2(stateDb);
+        if (entities.length > 0) {
+          const entityMap = new Map(entities.map((e) => [e.name, {
+            name: e.name,
+            path: e.path,
+            category: e.category,
+            aliases: e.aliases
+          }]));
+          await buildEntityEmbeddingsIndex(vaultPath, entityMap);
+        }
+      }
+      loadEntityEmbeddingsToMemory();
+      console.error("[Memory] Embeddings refreshed");
+    }).catch((err) => {
+      console.error("[Memory] Embeddings refresh failed:", err);
+    });
+  }
   if (process.env.FLYWHEEL_WATCH !== "false") {
     const config = parseWatcherConfig();
     console.error(`[Memory] File watcher enabled (debounce: ${config.debounceMs}ms)`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@velvetmonkey/flywheel-memory",
-  "version": "2.0.22",
+  "version": "2.0.24",
   "description": "MCP server that gives Claude full read/write access to your Obsidian vault. 42 tools for search, backlinks, graph queries, mutations, and hybrid semantic search.",
   "type": "module",
   "main": "dist/index.js",
@@ -50,7 +50,7 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.1",
-    "@velvetmonkey/vault-core": "^2.0.22",
+    "@velvetmonkey/vault-core": "^2.0.24",
     "better-sqlite3": "^11.0.0",
     "chokidar": "^4.0.0",
     "gray-matter": "^4.0.3",