@owrede/vault-memory 0.8.3 → 0.9.1

package/README.md

@@ -6,7 +6,9 @@ Reads one or more Obsidian vaults, indexes them with local embeddings via Ollama
 
 ## Status
 
-**v0.8.1** — Phase 8 (real ONNX cross-encoder reranker) + search-quality fixes. See `_research/vault-memory-spec.md` in a consuming vault for the design contract, `_research/vault-memory-eval-v2-results.md` for retrieval-quality benchmarks, and `_research/vault-memory-eval-v3-spec.md` for the planned reranker eval.
+**v0.9.1** — Body-hash short-circuit (migration 006): frontmatter-only edits (the common case for `update_frontmatter`, `/log-fact`, `/import-person`) no longer trigger chunk + embedding regeneration. A new `body_hash` column on `notes` lets the watcher detect "body unchanged, frontmatter changed" and keep all existing chunks/embeddings in place — saving one Ollama roundtrip per chunk per frontmatter edit. Legacy rows pre-migration self-heal on next touch.
+
+Previous: **v0.9.0** — Agent-Compatibility & Self-Orientation: OB1-compatible `search`/`fetch` tools so ChatGPT Custom Connectors, Claude.ai, and Deep-Research modes can use vault-memory as a connector; `vault_stats` and `recent_notes` for agent self-orientation on first connect.
 
 ## Architecture in one paragraph
 
@@ -183,7 +185,7 @@ exclude_globs = [".obsidian/**", ".trash/**", "_research/**", ".claude/**"]
 # secondary_embedding_model = "qwen3-embedding:0.6b"
 ```
 
-## MCP tools (18)
+## MCP tools (22)
 
 **Discovery & Read:** `list_vaults`, `read_note`
 **Search:** `search_semantic`, `search_text`, `search_hybrid` — all support optional `exclude_paths` (glob) and an explicit `vaults` filter; responses include a `note` field when vaults were skipped (e.g. mid-indexing)
@@ -193,13 +195,26 @@ exclude_globs = [".obsidian/**", ".trash/**", "_research/**", ".claude/**"]
 **Audit:** `audit_log`, `index_runs`
 **Model management (Phase 7c):** `list_models`, `start_shadow_index`, `switch_active_model`
 **Maintenance (v0.7.3):** `vacuum_embeddings` — drop orphaned embedding rows whose chunk_id no longer exists
+**Agent-Compatibility (v0.9.0):** `search`, `fetch` — OB1-compatible flat-shape adapters for ChatGPT Custom Connectors, Claude.ai, and Deep-Research modes. Backed by the hybrid (semantic+BM25+RRF) retrieval pipeline, so connector users get vault-memory's full search quality through the standardized interface.
+**Agent self-orientation (v0.9.0):** `vault_stats`, `recent_notes` — vault overview (note count, top tags, top frontmatter keys, last index run) and recently-modified notes (mtime DESC). Use these on first connect to brief an agent on what's in the vault and what the user has been working on.
+
+### Connector compatibility (v0.9.0)
+
+`search`/`fetch` follow the flat-shape spec used by OB1 and adopted by ChatGPT Custom Connectors / Claude.ai / Deep-Research:
+
+```
+search({query, limit}) → { results: [{ id, title, url, snippet }] }
+fetch({id})            → { id, title, text, url, metadata }
+```
+
+`id` is the opaque format `<vault>:<vault-relative-path>`. `url` is an `obsidian://open?…` URL — connectors render it as a clickable link that opens the note locally. Use the richer `search_hybrid` / `read_note` tools when working with a vault-memory-aware client (Claude Code's MCP integration); use `search` / `fetch` when integrating with a connector ecosystem that expects the standard shape.
 
 ## Development
 
 ```bash
 npm install
 npm run dev          # MCP server on stdio with hot reload
-npm test             # 278 tests across 33 files (v0.8.1)
+npm test             # 324 tests across 35 files (v0.9.1)
 npm run build
 ```
 

package/dist/cli.js

@@ -302,7 +302,7 @@ function runMigration005(db) {
     }
   }
 }
-var INITIAL_SCHEMA, MIGRATION_002_ALIASES, MIGRATION_003_FIX_DELETE_FKS, MIGRATION_004_VARIABLE_DIMS, MIGRATIONS;
+var INITIAL_SCHEMA, MIGRATION_002_ALIASES, MIGRATION_003_FIX_DELETE_FKS, MIGRATION_004_VARIABLE_DIMS, MIGRATION_006_BODY_HASH, MIGRATIONS;
 var init_schema = __esm({
   "src/db/schema.ts"() {
     "use strict";
@@ -310,6 +310,9 @@ var init_schema = __esm({
     INITIAL_SCHEMA = `
 -- \u2500\u2500 3.1 Raw Layer \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
 
+-- Migration 006 adds body_hash to this table (kept out of v1 schema so
+-- the migration chain has historical accuracy and frequent DB-rebuild
+-- tests do not trip over duplicate-column errors).
 CREATE TABLE IF NOT EXISTS notes (
   id            INTEGER PRIMARY KEY AUTOINCREMENT,
   path          TEXT NOT NULL UNIQUE,
@@ -493,6 +496,10 @@ INSERT INTO embeddings_1024 (chunk_id, model_id, vector)
   SELECT chunk_id, model_id, vector FROM embeddings;
 
 DROP TABLE embeddings;
+`;
+    MIGRATION_006_BODY_HASH = `
+ALTER TABLE notes ADD COLUMN body_hash TEXT;
+CREATE INDEX IF NOT EXISTS idx_notes_body_hash ON notes(body_hash);
 `;
     MIGRATIONS = [
       {
@@ -519,6 +526,11 @@ DROP TABLE embeddings;
         version: 5,
         description: "add partition key on model_id (two models per dim can coexist)",
         run: runMigration005
+      },
+      {
+        version: 6,
+        description: "add body_hash for frontmatter-only-change short-circuit",
+        sql: MIGRATION_006_BODY_HASH
       }
     ];
   }
@@ -540,8 +552,8 @@ var init_notes = __esm({
           "SELECT * FROM notes WHERE id = ?"
         );
         this._insert = db.prepare(`
-      INSERT INTO notes (path, content, frontmatter, title, hash, mtime, word_count, created_at, updated_at)
-      VALUES (@path, @content, @frontmatter, @title, @hash, @mtime, @word_count, @now, @now)
+      INSERT INTO notes (path, content, frontmatter, title, hash, body_hash, mtime, word_count, created_at, updated_at)
+      VALUES (@path, @content, @frontmatter, @title, @hash, @body_hash, @mtime, @word_count, @now, @now)
     `);
         this._update = db.prepare(`
       UPDATE notes
@@ -549,6 +561,7 @@ var init_notes = __esm({
           frontmatter = @frontmatter,
           title = @title,
           hash = @hash,
+          body_hash = @body_hash,
           mtime = @mtime,
           word_count = @word_count,
           updated_at = @now
@@ -583,6 +596,7 @@ var init_notes = __esm({
             frontmatter: input.frontmatter,
             title: input.title,
             hash: input.hash,
+            body_hash: input.bodyHash,
             mtime: input.mtime,
             word_count: input.wordCount,
             now
@@ -595,6 +609,7 @@ var init_notes = __esm({
           frontmatter: input.frontmatter,
           title: input.title,
           hash: input.hash,
+          body_hash: input.bodyHash,
           mtime: input.mtime,
           word_count: input.wordCount,
           now
@@ -2297,6 +2312,9 @@ function canonicalJsonStringify(value) {
 function computeNoteHash(content, frontmatter) {
   return sha256(content + canonicalJsonStringify(frontmatter ?? {}));
 }
+function computeBodyHash(content) {
+  return sha256(content);
+}
 var init_hash = __esm({
   "src/reader/hash.ts"() {
     "use strict";
@@ -2536,6 +2554,7 @@ async function parseNote(absolutePath, vaultRoot) {
   const frontmatter = fmData !== void 0 && Object.keys(fmData).length > 0 ? fmData : null;
   const title = extractTitle(content) ?? path3.basename(absolutePath, ".md");
   const hash = computeNoteHash(content, frontmatter);
+  const bodyHash = computeBodyHash(content);
   const mtime = Math.floor(stat.mtimeMs);
   const bodyLinks = extractWikilinks(content);
   const frontmatterLinks = extractFrontmatterWikilinks(frontmatter);
@@ -2550,6 +2569,7 @@ async function parseNote(absolutePath, vaultRoot) {
     frontmatter,
     title,
     hash,
+    bodyHash,
     mtime,
     wikilinks,
     wordCount
@@ -3038,6 +3058,7 @@ async function indexVault(vault, options) {
         frontmatter: parsed.frontmatter ? JSON.stringify(parsed.frontmatter) : null,
         title: parsed.title,
         hash: parsed.hash,
+        bodyHash: parsed.bodyHash,
         mtime: parsed.mtime,
         wordCount: parsed.wordCount
       });
@@ -3499,6 +3520,7 @@ async function updateFrontmatter(input) {
         frontmatter: fmJson,
         title,
         hash: newHash,
+        bodyHash: computeBodyHash(content),
         mtime: Math.floor(stat.mtimeMs),
         wordCount
       });
@@ -3582,6 +3604,31 @@ async function indexNote(options) {
       isNew: false
     };
   }
+  if (existing && existing.body_hash && existing.body_hash === parsed.bodyHash) {
+    const upsert2 = vault.db.notes.upsertByPath({
+      path: parsed.relativePath,
+      content: parsed.content,
+      frontmatter: parsed.frontmatter ? JSON.stringify(parsed.frontmatter) : null,
+      title: parsed.title,
+      hash: parsed.hash,
+      bodyHash: parsed.bodyHash,
+      mtime: parsed.mtime,
+      wordCount: parsed.wordCount
+    });
+    vault.db.aliases.setForNote(
+      upsert2.id,
+      extractAliases(parsed.frontmatter)
+    );
+    vault.db.wikilinks.deleteByNote(upsert2.id);
+    insertWikilinks2(vault, upsert2.id, parsed.wikilinks);
+    return {
+      status: "indexed",
+      notePath: parsed.relativePath,
+      noteId: upsert2.id,
+      chunksCreated: 0,
+      isNew: false
+    };
+  }
   const activeModel = vault.db.models.getActive();
   if (!activeModel) {
     throw new Error(
@@ -3599,6 +3646,7 @@ async function indexNote(options) {
     frontmatter: parsed.frontmatter ? JSON.stringify(parsed.frontmatter) : null,
     title: parsed.title,
     hash: parsed.hash,
+    bodyHash: parsed.bodyHash,
     mtime: parsed.mtime,
     wordCount: parsed.wordCount
   });
@@ -4133,6 +4181,7 @@ async function writeNote(input) {
         frontmatter: written.frontmatter ? JSON.stringify(written.frontmatter) : null,
         title,
         hash: written.hash,
+        bodyHash: computeBodyHash(written.content),
         mtime: Math.floor(stat.mtimeMs),
         wordCount: countWords3(written.content)
       });
@@ -4652,7 +4701,13 @@ var init_watcher2 = __esm({
 // src/server.ts
 var server_exports = {};
 __export(server_exports, {
-  serve: () => serve
+  aggregateTopFrontmatterKeys: () => aggregateTopFrontmatterKeys,
+  aggregateTopTags: () => aggregateTopTags,
+  decodeNoteId: () => decodeNoteId,
+  encodeNoteId: () => encodeNoteId,
+  obsidianUrl: () => obsidianUrl,
+  serve: () => serve,
+  truncateSnippet: () => truncateSnippet
 });
 import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
@@ -5031,6 +5086,57 @@ async function serve() {
             limit: { type: "integer", minimum: 1, maximum: 200, default: 20 }
           }
         }
+      },
+      {
+        name: "search",
+        description: "OB1-compatible search adapter. Returns a flat list of {id, title, url, snippet} for connector ecosystems (ChatGPT Custom Connectors, Claude.ai, Deep-Research). Backed by hybrid (semantic+BM25+RRF) search. For richer output use search_hybrid.",
+        inputSchema: {
+          type: "object",
+          required: ["query"],
+          properties: {
+            query: { type: "string" },
+            limit: { type: "integer", minimum: 1, maximum: 50, default: 10 }
+          }
+        }
+      },
+      {
+        name: "fetch",
+        description: "OB1-compatible fetch adapter. Resolves an opaque id (from `search`) to {id, title, text, url, metadata}. Backed by read_note.",
+        inputSchema: {
+          type: "object",
+          required: ["id"],
+          properties: {
+            id: {
+              type: "string",
+              description: "Opaque id from `search` results, format: <vault>:<vault-relative-path>"
+            }
+          }
+        }
+      },
+      {
+        name: "vault_stats",
+        description: "Vault overview for agent self-orientation: note/word counts, top tags, top frontmatter keys, embedding model, last index run. Omit `vault` to get all configured vaults.",
+        inputSchema: {
+          type: "object",
+          properties: {
+            vault: { type: "string", description: "Optional. Omit for all vaults." }
+          }
+        }
+      },
+      {
+        name: "recent_notes",
+        description: "List recently modified notes (mtime DESC). Use for agent self-orientation: 'what has the user been working on lately?'. No vector search, just SQL.",
+        inputSchema: {
+          type: "object",
+          properties: {
+            vault: { type: "string", description: "Optional. Omit for all vaults." },
+            limit: { type: "integer", minimum: 1, maximum: 200, default: 20 },
+            since: {
+              type: "integer",
+              description: "Optional unix-ms threshold. Only notes with mtime > since."
+            }
+          }
+        }
       }
     ]
   }));
@@ -5211,6 +5317,39 @@ async function serve() {
           const runs = getIndexRuns({ vault, limit: parsed.limit });
           return ok({ runs, count: runs.length });
         }
+        case "search": {
+          const parsed = SearchCompatArgs.parse(args2 ?? {});
+          return ok(
+            await handleSearchCompat(
+              manager,
+              ollama,
+              defaultModel,
+              activeVault,
+              parsed.query,
+              parsed.limit,
+              reranker
+            )
+          );
+        }
+        case "fetch": {
+          const parsed = FetchCompatArgs.parse(args2 ?? {});
+          return ok(handleFetchCompat(manager, parsed.id));
+        }
+        case "vault_stats": {
+          const parsed = VaultStatsArgs.parse(args2 ?? {});
+          return ok(handleVaultStats(manager, parsed.vault));
+        }
+        case "recent_notes": {
+          const parsed = RecentNotesArgs.parse(args2 ?? {});
+          return ok(
+            handleRecentNotes(
+              manager,
+              parsed.vault,
+              parsed.limit,
+              parsed.since
+            )
+          );
+        }
         default:
           return errorResponse(`Unknown tool: ${name}`);
       }
@@ -5409,6 +5548,181 @@ async function handleSearchHybrid(manager, ollama, defaultModel, activeVault, qu
   }
   return out;
 }
+function encodeNoteId(vault, path5) {
+  return `${vault}:${path5}`;
+}
+function decodeNoteId(id) {
+  const idx = id.indexOf(":");
+  if (idx <= 0 || idx === id.length - 1) {
+    throw new Error(
+      `Invalid id: ${id}. Expected format <vault>:<vault-relative-path>.`
+    );
+  }
+  return { vault: id.slice(0, idx), path: id.slice(idx + 1) };
+}
+function obsidianUrl(vaultName, notePath) {
+  return `obsidian://open?vault=${encodeURIComponent(vaultName)}&file=${encodeURIComponent(notePath)}`;
+}
+async function handleSearchCompat(manager, ollama, defaultModel, activeVault, query, limit, reranker) {
+  const { targets, skipped } = resolveVaultTargets(manager, void 0, activeVault);
+  if (targets.length === 0) {
+    return {
+      results: [],
+      note: skipped.length > 0 ? `All eligible vaults are indexing; skipped: ${skipped.join(", ")}.` : "No vaults configured."
+    };
+  }
+  const hits = await hybridSearch({
+    query,
+    embeddingModel: defaultModel,
+    ollama,
+    vaults: targets,
+    topK: limit,
+    rrfK: 60,
+    includeBreakdown: false,
+    reranker
+  });
+  const seen = /* @__PURE__ */ new Set();
+  const results = [];
+  for (const h of hits) {
+    const noteKey = `${h.vault}:${h.notePath}`;
+    if (seen.has(noteKey)) continue;
+    seen.add(noteKey);
+    results.push({
+      id: encodeNoteId(h.vault, h.notePath),
+      title: h.noteTitle ?? h.notePath,
+      url: obsidianUrl(h.vault, h.notePath),
+      snippet: truncateSnippet(h.chunkText, 280)
+    });
+    if (results.length >= limit) break;
+  }
+  const out = { results };
+  if (skipped.length > 0) {
+    out.note = `Skipped vault(s) currently indexing: ${skipped.join(", ")}.`;
+  }
+  return out;
+}
+function truncateSnippet(text, max) {
+  const collapsed = text.replace(/\s+/g, " ").trim();
+  if (collapsed.length <= max) return collapsed;
+  return collapsed.slice(0, max - 1).trimEnd() + "\u2026";
+}
+function handleFetchCompat(manager, id) {
+  const { vault: vaultName, path: path5 } = decodeNoteId(id);
+  const vault = manager.require(vaultName);
+  const note = vault.db.notes.getByPath(path5);
+  if (!note) {
+    throw new Error(`Note not found: ${vaultName}/${path5}`);
+  }
+  const metadata = {
+    vault: vaultName,
+    path: note.path,
+    mtime: note.mtime,
+    hash: note.hash,
+    word_count: note.word_count
+  };
+  if (note.frontmatter) {
+    try {
+      metadata.frontmatter = JSON.parse(note.frontmatter);
+    } catch {
+    }
+  }
+  return {
+    id,
+    title: note.title ?? note.path,
+    text: note.content,
+    url: obsidianUrl(vaultName, note.path),
+    metadata
+  };
+}
+function handleVaultStats(manager, vaultFilter) {
+  const targets = vaultFilter ? [manager.require(vaultFilter)] : manager.list();
+  const stats = targets.map((v) => {
+    const total_notes = v.db.notes.countAll();
+    const wordRow = v.db.handle.prepare(
+      "SELECT SUM(word_count) AS total FROM notes"
+    ).get();
+    const lastRun = v.db.audit.listRuns(1)[0];
+    const activeModel = v.db.models.getActive();
+    return {
+      vault: v.config.name,
+      vault_path: v.config.path,
+      total_notes,
+      total_words: wordRow?.total ?? 0,
+      embedding_model: activeModel?.name ?? v.config.embedding_model ?? null,
+      indexed_at: lastRun?.finished_at ?? null,
+      top_tags: aggregateTopTags(v.db.handle, 10),
+      top_frontmatter_keys: aggregateTopFrontmatterKeys(v.db.handle, 10)
+    };
+  });
+  if (vaultFilter) {
+    return stats[0];
+  }
+  return { vaults: stats, count: stats.length };
+}
+function aggregateTopTags(db, limit) {
+  const rows = db.prepare(
+    `
+      SELECT je.value AS tag, COUNT(*) AS count
+      FROM notes
+      JOIN json_each(json_extract(notes.frontmatter, '$.tags')) AS je
+      WHERE notes.frontmatter IS NOT NULL
+        AND json_type(notes.frontmatter, '$.tags') = 'array'
+        AND typeof(je.value) = 'text'
+      GROUP BY je.value
+      ORDER BY count DESC, tag ASC
+      LIMIT ?
+    `
+  ).all(limit);
+  return rows;
+}
+function aggregateTopFrontmatterKeys(db, limit) {
+  const rows = db.prepare(
+    `
+      SELECT je.key AS key, COUNT(*) AS count
+      FROM notes
+      JOIN json_each(notes.frontmatter) AS je
+      WHERE notes.frontmatter IS NOT NULL
+        AND json_type(notes.frontmatter) = 'object'
+      GROUP BY je.key
+      ORDER BY count DESC, key ASC
+      LIMIT ?
+    `
+  ).all(limit);
+  return rows;
+}
+function handleRecentNotes(manager, vaultFilter, limit, since) {
+  const targets = vaultFilter ? [manager.require(vaultFilter)] : manager.list();
+  const all = [];
+  for (const v of targets) {
+    const rows = since !== void 0 ? v.db.handle.prepare(
+      "SELECT path, title, mtime, word_count, frontmatter FROM notes WHERE mtime > ? ORDER BY mtime DESC LIMIT ?"
+    ).all(since, limit) : v.db.handle.prepare(
+      "SELECT path, title, mtime, word_count, frontmatter FROM notes ORDER BY mtime DESC LIMIT ?"
+    ).all(limit);
+    for (const r of rows) {
+      let tags = null;
+      if (r.frontmatter) {
+        try {
+          const fm = JSON.parse(r.frontmatter);
+          if (Array.isArray(fm.tags)) {
+            tags = fm.tags.filter((t) => typeof t === "string");
+          }
+        } catch {
+        }
+      }
+      all.push({
+        vault: v.config.name,
+        path: r.path,
+        title: r.title,
+        mtime: r.mtime,
+        word_count: r.word_count,
+        tags
+      });
+    }
+  }
+  all.sort((a, b) => b.mtime - a.mtime);
+  return { notes: all.slice(0, limit), count: Math.min(all.length, limit) };
+}
 function ok(data) {
   return {
     content: [{ type: "text", text: JSON.stringify(data, null, 2) }]
@@ -5420,7 +5734,7 @@ function errorResponse(message) {
     content: [{ type: "text", text: message }]
   };
 }
-var VERSION, ReadNoteArgs, SearchArgs, HybridSearchArgs, VaultPathArgs, ForwardLinksArgs, FindBrokenLinksArgs, PredicateSchema, QueryFrontmatterArgs, WriteNoteArgs, UpdateFrontmatterArgs, DeleteNoteArgs, AuditLogArgs, IndexRunsArgs, ListModelsArgs, StartShadowIndexArgs, SwitchActiveModelArgs, VacuumEmbeddingsArgs;
+var VERSION, ReadNoteArgs, SearchArgs, HybridSearchArgs, VaultPathArgs, ForwardLinksArgs, FindBrokenLinksArgs, PredicateSchema, QueryFrontmatterArgs, WriteNoteArgs, UpdateFrontmatterArgs, DeleteNoteArgs, AuditLogArgs, IndexRunsArgs, ListModelsArgs, StartShadowIndexArgs, SwitchActiveModelArgs, VacuumEmbeddingsArgs, SearchCompatArgs, FetchCompatArgs, VaultStatsArgs, RecentNotesArgs;
 var init_server = __esm({
   "src/server.ts"() {
     "use strict";
@@ -5437,7 +5751,7 @@ var init_server = __esm({
     init_audit3();
     init_watcher2();
     init_indexer2();
-    VERSION = "0.8.1";
+    VERSION = "0.9.1";
     ReadNoteArgs = z3.object({
       vault: z3.string(),
       path: z3.string()
@@ -5528,6 +5842,21 @@ var init_server = __esm({
     VacuumEmbeddingsArgs = z3.object({
       vault: z3.string()
     });
+    SearchCompatArgs = z3.object({
+      query: z3.string().min(1),
+      limit: z3.number().int().positive().max(50).optional().default(10)
+    });
+    FetchCompatArgs = z3.object({
+      id: z3.string().min(1)
+    });
+    VaultStatsArgs = z3.object({
+      vault: z3.string().optional()
+    });
+    RecentNotesArgs = z3.object({
+      vault: z3.string().optional(),
+      limit: z3.number().int().positive().max(200).optional().default(20),
+      since: z3.number().int().nonnegative().optional()
+    });
   }
 });