@staix/vault-mem 0.3.1 → 0.3.3

package/src/vault-index.ts

@@ -1,10 +1,11 @@
+import { realPath } from "./platform.ts";
 import { Database } from "bun:sqlite";
 import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readSync, readdirSync, realpathSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import type { AnalysisSnapshot } from "./analysis-types.ts";
-import type { VaultSpec } from "./config.ts";
-import { parseFrontmatter } from "./frontmatter.ts";
+import type { VaultSpec, SpecSourceSpec } from "./config.ts";
+import { parseFrontmatter, type FrontmatterDocument } from "./frontmatter.ts";
 import { normalizeTarget, noteKey, proseLines, resolveLinks, type LinkNote, type ResolvedLink } from "./links.ts";
 
 export const MAX_NOTE_BYTES = 2 * 1024 * 1024;
@@ -41,13 +42,13 @@ function within(root: string, target: string): boolean {
   const rel = relative(root, target);
   return rel === "" || (!rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) && rel !== ".." && !isAbsolute(rel));
 }
-function canonicalLocation(location: string): string {
-  try { return realpathSync(location); }
+function canonicalLocation(location: string, canonicalize: (path: string) => string = realpathSync): string {
+  try { return canonicalize(location); }
   catch (error) {
     if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
     const parent = dirname(location);
     if (parent === location) throw error;
-    return join(canonicalLocation(parent), relative(parent, location));
+    return join(canonicalLocation(parent, canonicalize), relative(parent, location));
   }
 }
 export function resultLimit(value = 50): number {
@@ -61,6 +62,15 @@ function aliases(value: unknown): string[] {
 function metadata(stat: NonNullable<ReturnType<typeof lstatSync>>): string {
   return [stat.dev, stat.ino, stat.size, stat.mtimeMs, stat.ctimeMs].join(":");
 }
+/** Adapter file I/O gets path-specific guidance without rewriting unrelated Git or binding errors. */
+function specFileIO<T>(path: string, action: string, operation: () => T): T {
+  try { return operation(); }
+  catch (error) {
+    const code = (error as NodeJS.ErrnoException)?.code ?? "I/O failure";
+    const advice = action === "close" ? "retry the scan and inspect local file I/O if the failure repeats" : "restore read access and a stable file, then retry";
+    throw new Error(`spec discovery ${path}: cannot ${action} Markdown (${code}); ${advice}`, { cause: error });
+  }
+}
 function boundedText(value: string, bytes: number): { content: string; truncated: boolean } {
   const buffer = Buffer.from(value);
   if (buffer.length <= bytes) return { content: value, truncated: false };
@@ -77,7 +87,7 @@ function readPage(prefix: Buffer, maxBytes: number, offset: number, totalBytes:
   const truncated = offset + end < totalBytes;
   return { content: prefix.subarray(0, end).toString("utf8"), truncated, ...(truncated ? { nextOffset: offset + end } : {}) };
 }
-function h2Sections(body: string): { name: string; start: number; end: number }[] {
+export function h2Sections(body: string): { name: string; start: number; end: number }[] {
   const visible = proseLines(body), lines = body.match(/[^\n]*\n|[^\n]+$/g) ?? [];
   const sections: { name: string; start: number; end: number }[] = [];
   let offset = 0, active: number | undefined;
@@ -94,7 +104,7 @@ function h2Sections(body: string): { name: string; start: number; end: number }[
   }
   return sections;
 }
-function headingMetadata(sections: { name: string }[]): Pick<NoteReadResult, "headings" | "headingsTruncated"> {
+export function headingMetadata(sections: { name: string }[]): Pick<NoteReadResult, "headings" | "headingsTruncated"> {
   const headings: string[] = [];
   let bytes = 2, truncated = false;
   for (const { name } of sections) {
@@ -104,69 +114,91 @@ function headingMetadata(sections: { name: string }[]): Pick<NoteReadResult, "he
   }
   return { headings, ...(truncated ? { headingsTruncated: true } : {}) };
 }
+/** A source changes discovery and record selection, while sharing scan storage and BM25. */
+export interface IndexSourceAdapter {
+  validate(): void;
+  paths(): string[];
+  validatePath(path: string): void;
+  accept(path: string, document: FrontmatterDocument, hash: string): boolean;
+  invalid(path: string): boolean | void;
+  skipFile?(path: string, reason: "oversized" | "invalid_utf8", bytes: Buffer): boolean;
+}
+export class IndexCacheError extends Error {}
+export interface IndexOptions { memory?: boolean; adapter?: IndexSourceAdapter; reuseCache?: boolean }
 /** Derived cache only. No vault or application configuration files are written. */
-export class VaultIndex {
+export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
   private readonly db: Database;
   readonly file: string;
   private readonly root: string;
-  constructor(home: string, readonly vault: VaultSpec, vaultRoots: string[] = [vault.path], options: { memory?: boolean } = {}) {
-    this.root = realpathSync(vault.path);
-    if (this.root !== vault.path) throw new Error(`vault ${vault.id}: registered path changed`);
+  private readonly adapter?: IndexSourceAdapter;
+  constructor(home: string, readonly vault: Source, vaultRoots: string[] = [vault.path], options: IndexOptions = {}) {
+    this.adapter = options.adapter;
+    this.root = this.adapter ? realPath(vault.path) : realpathSync(vault.path);
+    if (this.root !== vault.path) throw new Error(this.adapter ? "spec index: worktree path changed; resolve the source again with resolveSpecWorktree" : `vault ${vault.id}: registered path changed`);
     const notes = join(this.root, "notes");
-    if (realpathSync(notes) !== notes || !lstatSync(notes).isDirectory()) throw new Error(`vault ${vault.id}: notes must be a plain directory`);
-    const cache = resolve(home, "indexes");
-    const canonical = canonicalLocation(cache);
-    for (const root of vaultRoots) {
-      if (within(canonicalLocation(resolve(root)), canonical)) throw new Error("index: VMEM_HOME cache must be outside every vault");
-    }
-    // Canonical home resolution is allowed, but a linked cache directory/file is not.
-    if (!options.memory) {
-      mkdirSync(cache, { recursive: true, mode: 0o700 });
-      if (lstatSync(cache).isSymbolicLink() || !lstatSync(cache).isDirectory()) throw new Error("index: expected plain cache directory");
-      this.file = join(cache, indexName(this.root));
-      try {
-        const fd = openSync(this.file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
-        closeSync(fd);
-      } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
-      if (!lstatSync(this.file).isFile() || lstatSync(this.file).isSymbolicLink()) throw new Error("index: expected plain database file");
-    } else this.file = ":memory:";
-    this.db = new Database(this.file);
-    this.db.run("PRAGMA busy_timeout=5000; PRAGMA journal_mode=DELETE;");
-    this.db.run(`CREATE TABLE IF NOT EXISTS notes (
-      path TEXT PRIMARY KEY, hash TEXT NOT NULL, metadata TEXT NOT NULL,
-      title TEXT NOT NULL, description TEXT NOT NULL, body TEXT NOT NULL,
-      type TEXT NOT NULL, domain TEXT NOT NULL, aliases TEXT NOT NULL, source TEXT NOT NULL);
-      CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(path UNINDEXED, title, description, body, tokenize='unicode61');
-      CREATE TABLE IF NOT EXISTS analysis_notes (path TEXT PRIMARY KEY, source TEXT NOT NULL, hash TEXT NOT NULL, metadata TEXT NOT NULL);
-      CREATE TABLE IF NOT EXISTS links (source TEXT NOT NULL, ordinal INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(source, ordinal));`);
-    // Dry runs reuse the derived cache via a readonly connection, then refresh only changed notes in memory.
-    if (options.memory) {
-      const file = join(cache, indexName(this.root));
-      let exists = false;
-      try { const stat = lstatSync(file); if (!stat.isFile() || stat.isSymbolicLink()) throw new Error("index: expected plain database file"); exists = true; }
-      catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
-      if (exists) {
-        if (realpathSync(cache) !== cache) throw new Error("index: linked cache directory");
-        const disk = new Database(file, { readonly: true });
+    if (this.adapter) this.adapter.validate();
+    else if (realpathSync(notes) !== notes || !lstatSync(notes).isDirectory()) throw new Error(`vault ${vault.id}: notes must be a plain directory`);
+    try {
+      const cacheName = this.adapter ? `spec-${indexName(this.root)}` : indexName(this.root);
+      const cache = resolve(home, "indexes");
+      const noDisk = options.memory && options.reuseCache === false;
+      const canonicalize = this.adapter ? realPath : realpathSync;
+      const canonical = noDisk ? cache : canonicalLocation(cache, canonicalize);
+      for (const root of noDisk ? [] : vaultRoots) {
+        if (within(canonicalLocation(resolve(root), canonicalize), canonical)) throw new Error(this.adapter ? "spec index: cache is inside a protected source; choose VMEM_HOME outside every repository and vault" : "index: VMEM_HOME cache must be outside every vault");
+      }
+      // Canonical home resolution is allowed, but a linked cache directory/file is not.
+      if (!options.memory) {
+        mkdirSync(cache, { recursive: true, mode: 0o700 });
+        if (lstatSync(cache).isSymbolicLink() || !lstatSync(cache).isDirectory()) throw new Error("index: expected plain cache directory");
+        this.file = join(cache, cacheName);
         try {
-          disk.transaction(() => {
-            const size = disk.query<{count:number;bytes:number}, []>("SELECT count(*) AS count, coalesce(sum(length(CAST(source AS BLOB))),0) AS bytes FROM notes").get()!;
-            if (size.count > 10000 || size.bytes > 64 * 1024 * 1024) throw new Error("analysis: cached snapshot exceeds bound");
-            for (const row of disk.query<Omit<StoredNote,"aliases"> & {aliases:string}, []>("SELECT * FROM notes").all()) {
-              this.db.query("INSERT INTO notes VALUES (?,?,?,?,?,?,?,?,?,?)").run(row.path,row.hash,row.metadata,row.title,row.description,row.body,row.type,row.domain,row.aliases,row.source);
-              this.db.query("INSERT INTO fts VALUES (?,?,?,?)").run(row.path,row.title,row.description,row.body);
-            }
-            for (const row of disk.query<{source:string;ordinal:number;data:string}, []>("SELECT * FROM links").all()) this.db.query("INSERT INTO links VALUES (?,?,?)").run(row.source,row.ordinal,row.data);
-            this.db.run(`PRAGMA user_version = ${Number(disk.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version ?? 0)}`);
-            const exists = disk.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name='analysis_notes'").get();
-            if (exists) {
-              const size = disk.query<{count:number;bytes:number}, []>("SELECT count(*) AS count, coalesce(sum(length(CAST(source AS BLOB))),0) AS bytes FROM analysis_notes").get()!;
+          const fd = openSync(this.file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
+          closeSync(fd);
+        } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
+        if (!lstatSync(this.file).isFile() || lstatSync(this.file).isSymbolicLink() || (this.adapter && lstatSync(this.file).nlink !== 1)) throw new Error("index: expected plain database file");
+      } else this.file = ":memory:";
+      this.db = new Database(this.file);
+      this.db.run("PRAGMA busy_timeout=5000; PRAGMA journal_mode=DELETE;");
+      this.db.run(`CREATE TABLE IF NOT EXISTS notes (
+        path TEXT PRIMARY KEY, hash TEXT NOT NULL, metadata TEXT NOT NULL,
+        title TEXT NOT NULL, description TEXT NOT NULL, body TEXT NOT NULL,
+        type TEXT NOT NULL, domain TEXT NOT NULL, aliases TEXT NOT NULL, source TEXT NOT NULL);
+        CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(path UNINDEXED, title, description, body, tokenize='unicode61');
+        CREATE TABLE IF NOT EXISTS analysis_notes (path TEXT PRIMARY KEY, source TEXT NOT NULL, hash TEXT NOT NULL, metadata TEXT NOT NULL);
+        CREATE TABLE IF NOT EXISTS links (source TEXT NOT NULL, ordinal INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(source, ordinal));`);
+      // Dry runs reuse the derived cache via a readonly connection, then refresh only changed notes in memory.
+      if (options.memory && options.reuseCache !== false) {
+        const file = join(cache, cacheName);
+        let exists = false;
+        try { const stat = lstatSync(file); if (!stat.isFile() || stat.isSymbolicLink() || (this.adapter && stat.nlink !== 1)) throw new Error("index: expected plain database file"); exists = true; }
+        catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+        if (exists) {
+          if (canonicalize(cache) !== cache) throw new Error("index: linked cache directory");
+          const disk = new Database(file, { readonly: true });
+          try {
+            disk.transaction(() => {
+              const size = disk.query<{count:number;bytes:number}, []>("SELECT count(*) AS count, coalesce(sum(length(CAST(source AS BLOB))),0) AS bytes FROM notes").get()!;
               if (size.count > 10000 || size.bytes > 64 * 1024 * 1024) throw new Error("analysis: cached snapshot exceeds bound");
-              for (const row of disk.query<{path:string;source:string;hash:string;metadata:string}, []>("SELECT * FROM analysis_notes").all()) this.db.query("INSERT INTO analysis_notes VALUES (?,?,?,?)").run(row.path,row.source,row.hash,row.metadata);
-            }
-          })();
-        } finally { disk.close(); }
+              for (const row of disk.query<Omit<StoredNote,"aliases"> & {aliases:string}, []>("SELECT * FROM notes").all()) {
+                this.db.query("INSERT INTO notes VALUES (?,?,?,?,?,?,?,?,?,?)").run(row.path,row.hash,row.metadata,row.title,row.description,row.body,row.type,row.domain,row.aliases,row.source);
+                this.db.query("INSERT INTO fts VALUES (?,?,?,?)").run(row.path,row.title,row.description,row.body);
+              }
+              for (const row of disk.query<{source:string;ordinal:number;data:string}, []>("SELECT * FROM links").all()) this.db.query("INSERT INTO links VALUES (?,?,?)").run(row.source,row.ordinal,row.data);
+              this.db.run(`PRAGMA user_version = ${Number(disk.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version ?? 0)}`);
+              const exists = disk.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name='analysis_notes'").get();
+              if (exists) {
+                const size = disk.query<{count:number;bytes:number}, []>("SELECT count(*) AS count, coalesce(sum(length(CAST(source AS BLOB))),0) AS bytes FROM analysis_notes").get()!;
+                if (size.count > 10000 || size.bytes > 64 * 1024 * 1024) throw new Error("analysis: cached snapshot exceeds bound");
+                for (const row of disk.query<{path:string;source:string;hash:string;metadata:string}, []>("SELECT * FROM analysis_notes").all()) this.db.query("INSERT INTO analysis_notes VALUES (?,?,?,?)").run(row.path,row.source,row.hash,row.metadata);
+              }
+            })();
+          } finally { disk.close(); }
+        }
       }
+    } catch (error) {
+      if (this.adapter) throw new IndexCacheError(error instanceof Error ? error.message : "cache initialization failed", { cause: error });
+      throw error;
     }
   }
   close(): void { this.db.close(); }
@@ -175,64 +207,111 @@ export class VaultIndex {
   }
   /** Scan metadata each call; hash only new or metadata-changed files. Atomic cache update. */
   scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
-    return this.db.transaction(() => {
+    // Discovery/binding Git calls must never hold a SQLite writer lock.
+    let preparedPaths: string[] | undefined;
+    if (this.adapter) { this.adapter.validate(); preparedPaths = this.adapter.paths(); }
+    const result = this.db.transaction(() => {
       const notesDir = join(this.root, "notes");
-      if (realpathSync(this.root) !== this.root || realpathSync(notesDir) !== notesDir || !lstatSync(notesDir).isDirectory()) throw new Error(`vault ${this.vault.id}: path or notes directory changed`);
-      const previous = new Map(this.db.query<{ path: string; hash: string; metadata: string }, []>(options.captureInvalidFrontmatter ? "SELECT path,hash,metadata FROM notes UNION SELECT path,hash,metadata FROM analysis_notes" : "SELECT path,hash,metadata FROM notes").all().map(note => [note.path, note]));
+      if (!this.adapter && (realpathSync(this.root) !== this.root || realpathSync(notesDir) !== notesDir || !lstatSync(notesDir).isDirectory())) throw new Error(`vault ${this.vault.id}: path or notes directory changed`);
+      const indexed = this.adapter ? new Set(this.db.query<{path:string}, []>("SELECT path FROM notes").all().map(row => row.path)) : undefined;
+      const previous = new Map(this.db.query<{ path: string; hash: string; metadata: string }, []>(this.adapter ? "SELECT path,hash,metadata FROM analysis_notes" : options.captureInvalidFrontmatter ? "SELECT path,hash,metadata FROM notes UNION SELECT path,hash,metadata FROM analysis_notes" : "SELECT path,hash,metadata FROM notes").all().map(note => [note.path, note]));
       if (rebuild) this.db.run("DELETE FROM fts");
       const live = new Set<string>(), present = new Set<string>();
       let changed = 0, analysisBytes = 0;
       const skipped: string[] = [], touched = new Set<string>(), keys = new Set<string>();
       // Links resolve by path only, so other notes re-resolve only when they target a touched path.
       const touch = (path: string) => { touched.add(path); keys.add(noteKey(path)); };
-      for (const entry of readdirSync(notesDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
-        if (!entry.name.endsWith(".md") || !entry.isFile()) continue;
-        const path = entry.name, file = join(notesDir, path), before = lstatSync(file);
-        if (!before.isFile() || before.isSymbolicLink()) throw new Error(`vault ${this.vault.id}: note path changed`);
-        if (before.size > MAX_NOTE_BYTES) throw new Error(`vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
+      const paths = preparedPaths ?? readdirSync(notesDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).filter(entry => entry.name.endsWith(".md") && entry.isFile()).map(entry => entry.name);
+      for (const path of paths) {
+        this.adapter?.validatePath(path);
+        const file = join(this.adapter ? this.root : notesDir, path);
+        const before = this.adapter ? specFileIO(path, "inspect", () => lstatSync(file)) : lstatSync(file);
+        if (!before.isFile() || before.isSymbolicLink()) throw new Error(this.adapter ? `spec discovery ${path}: selected path changed; restore a plain file and retry` : `vault ${this.vault.id}: note path changed`);
+        if (!this.adapter && before.size > MAX_NOTE_BYTES) throw new Error(`vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
         live.add(path); present.add(path);
-        if (options.captureInvalidFrontmatter) {
+        if (options.captureInvalidFrontmatter && !this.adapter) {
           analysisBytes += before.size;
-          if (present.size > 10000 || analysisBytes > 64 * 1024 * 1024) throw new Error("analysis: snapshot exceeds bound");
+          if (present.size > 10000 || analysisBytes > 64 * 1024 * 1024) throw new Error(this.adapter ? `spec discovery ${path}: more than 10000 Markdown files or 64 MiB of input; ignore large or vendored Markdown via .gitignore and untrack already-tracked excluded files with git rm --cached before rebuilding` : "analysis: snapshot exceeds bound");
+        }
+        if (this.adapter) {
+          analysisBytes += Math.min(before.size, MAX_NOTE_BYTES + 1);
+          if (present.size > 10000 || analysisBytes > 64 * 1024 * 1024) throw new Error(`spec discovery ${path}: more than 10000 Markdown inputs or 64 MiB of bounded reads; ignore vendored Markdown via .gitignore and untrack excluded files with git rm --cached`);
         }
         const old = previous.get(path), stamp = metadata(before);
-        if (!rebuild && old?.metadata === stamp && this.db.query("SELECT 1 FROM analysis_notes WHERE path=?").get(path)) continue;
-        const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
-        let buffer: Buffer;
-        try {
-          const stat = fstatSync(fd);
-          if (!stat.isFile() || stat.size > MAX_NOTE_BYTES || metadata(stat) !== stamp) throw new Error(`vault ${this.vault.id}, ${path}: note changed during scan`);
-          // Fixed bound even if a concurrent writer grows the file.
-          buffer = Buffer.alloc(MAX_NOTE_BYTES + 1);
-          let size = 0;
-          for (;;) {
-            const count = readSync(fd, buffer, size, buffer.length - size, null);
-            size += count;
-            if (!count || size === buffer.length) break;
-          }
-          buffer = buffer.subarray(0, size);
-          if (size > MAX_NOTE_BYTES || metadata(fstatSync(fd)) !== stamp) throw new Error(`vault ${this.vault.id}, ${path}: note changed during scan`);
-        } finally { closeSync(fd); }
-        const hash = digest(buffer);
-        this.db.query("INSERT OR REPLACE INTO analysis_notes VALUES (?,?,?,?)").run(path, buffer.toString("utf8"), hash, stamp);
-        if (!rebuild && old?.hash === hash) { this.db.query("UPDATE notes SET metadata=? WHERE path=?").run(stamp, path); continue; }
+        if (!this.adapter && !rebuild && old?.metadata === stamp && this.db.query("SELECT 1 FROM analysis_notes WHERE path=?").get(path)) continue;
+        const cached = this.adapter && before.size <= MAX_NOTE_BYTES && !rebuild && old?.metadata === stamp ? this.db.query<{source:string}, [string]>("SELECT source FROM analysis_notes WHERE path=?").get(path) : null;
+        let source: string, hash: string;
+        if (cached && old) { source = cached.source; hash = old.hash; }
+        else {
+          const fd = this.adapter ? specFileIO(path, "open", () => openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)) : openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+          let buffer: Buffer;
+          try {
+            const stat = this.adapter ? specFileIO(path, "inspect", () => fstatSync(fd)) : fstatSync(fd);
+            if (!stat.isFile() || (!this.adapter && stat.size > MAX_NOTE_BYTES) || metadata(stat) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
+            // Fixed bound even if a concurrent writer grows the file.
+            buffer = Buffer.alloc(MAX_NOTE_BYTES + 1);
+            let size = 0;
+            for (;;) {
+              const count = this.adapter ? specFileIO(path, "read", () => readSync(fd, buffer, size, buffer.length - size, null)) : readSync(fd, buffer, size, buffer.length - size, null);
+              size += count;
+              if (!count || size === buffer.length) break;
+            }
+            buffer = buffer.subarray(0, size);
+            if ((!this.adapter && size > MAX_NOTE_BYTES) || metadata(this.adapter ? specFileIO(path, "inspect", () => fstatSync(fd)) : fstatSync(fd)) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
+          } finally { if (this.adapter) specFileIO(path, "close", () => closeSync(fd)); else closeSync(fd); }
+          this.adapter?.validatePath(path);
+          if (this.adapter && metadata(specFileIO(path, "inspect", () => lstatSync(file))) !== stamp) throw new Error(`spec discovery ${path}: file changed during read; retry on a stable worktree`);
+          if (this.adapter) {
+            const skip = (reason: "oversized" | "invalid_utf8") => {
+              if (!this.adapter?.skipFile?.(path, reason, buffer.subarray(0, MAX_NOTE_BYTES))) return false;
+              live.delete(path); skipped.push(path);
+              this.db.query("DELETE FROM analysis_notes WHERE path=?").run(path);
+              return true;
+            };
+            if (before.size > MAX_NOTE_BYTES || buffer.length > MAX_NOTE_BYTES) {
+              if (skip("oversized")) continue;
+              throw new Error(`spec discovery ${path}: Markdown exceeds ${MAX_NOTE_BYTES} bytes; split the candidate before validating`);
+            }
+            try { source = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(buffer); }
+            catch {
+              if (skip("invalid_utf8")) continue;
+              throw new Error(`spec discovery ${path}: Markdown is not valid UTF-8; re-encode the candidate as UTF-8 before validating`);
+            }
+          } else source = buffer.toString("utf8");
+          hash = digest(buffer);
+          this.db.query("INSERT OR REPLACE INTO analysis_notes VALUES (?,?,?,?)").run(path, source, hash, stamp);
+        }
+        if (!this.adapter && !rebuild && old?.hash === hash) { this.db.query("UPDATE notes SET metadata=? WHERE path=?").run(stamp, path); continue; }
         let document: ReturnType<typeof parseFrontmatter>;
-        try { document = parseFrontmatter(buffer.toString("utf8")); }
+        try { document = parseFrontmatter(source); }
         catch {
-          if (!options.skipInvalidFrontmatter && !options.captureInvalidFrontmatter) throw new Error(`vault ${this.vault.id}, ${path}: invalid frontmatter`);
-          live.delete(path); skipped.push(path); continue;
+          if (!this.adapter && !options.skipInvalidFrontmatter && !options.captureInvalidFrontmatter) throw new Error(`vault ${this.vault.id}, ${path}: invalid frontmatter`);
+          const report = this.adapter?.invalid(path);
+          live.delete(path); if (report !== false) skipped.push(path); continue;
+        }
+        if (this.adapter) {
+          this.adapter.validatePath(path);
+          if (metadata(specFileIO(path, "inspect", () => lstatSync(file))) !== stamp) throw new Error(`spec discovery ${path}: file changed during scan; retry on a stable worktree`);
+          if (!this.adapter.accept(path, document, hash)) { live.delete(path); continue; }
+          const row = this.db.query<{hash:string}, [string]>("SELECT hash FROM notes WHERE path=?").get(path);
+          if (!rebuild && row?.hash === hash) { this.db.query("UPDATE notes SET metadata=? WHERE path=?").run(stamp, path); continue; }
         }
         const { data, body } = document;
         touch(path);
         this.db.query("INSERT OR REPLACE INTO notes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(path, hash, stamp,
-          text(data.title) || path.slice(0, -3), text(data.description), body, text(data.type), text(data.domain), JSON.stringify(aliases(data.aliases ?? data.alias)), buffer.toString("utf8"));
+          text(data.title) || path.slice(0, -3), text(data.description), body, text(data.type), text(data.domain), JSON.stringify(aliases(data.aliases ?? data.alias)), source);
         this.db.query("DELETE FROM fts WHERE path=?").run(path);
         this.db.query("INSERT INTO fts(path,title,description,body) SELECT path,title,description,body FROM notes WHERE path=?").run(path);
         changed++;
       }
       for (const row of this.db.query<{path:string}, []>("SELECT path FROM analysis_notes").all()) if (!present.has(row.path)) this.db.query("DELETE FROM analysis_notes WHERE path=?").run(row.path);
       let removed = 0;
-      for (const path of previous.keys()) if (!live.has(path)) { touch(path); this.db.query("DELETE FROM notes WHERE path=?").run(path); this.db.query("DELETE FROM fts WHERE path=?").run(path); removed++; }
+      for (const path of indexed ?? previous.keys()) if (!live.has(path)) { touch(path); this.db.query("DELETE FROM notes WHERE path=?").run(path); this.db.query("DELETE FROM fts WHERE path=?").run(path); removed++; }
+      if (this.adapter) {
+        // Spec links are resolved by frontmatter id in buildSpecGraph, never by vault paths.
+        this.db.run("DELETE FROM links");
+        return { notes: live.size, changed, removed, ...(skipped.length ? { skipped } : {}) };
+      }
       const stale = this.db.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version !== LINK_RESOLUTION;
       if (rebuild || stale) this.db.run("DELETE FROM links");
       else if (changed || removed) {
@@ -253,6 +332,8 @@ export class VaultIndex {
       if (stale) this.db.run(`PRAGMA user_version = ${LINK_RESOLUTION}`);
       return { notes: live.size, changed, removed, ...(skipped.length ? { skipped } : {}) };
     }).immediate();
+    this.adapter?.validate();
+    return result;
   }
   /** Consume cached note sources only. Auxiliary inputs are captured separately by analysisInput. */
   analysisSnapshot(inputs: Omit<AnalysisSnapshot, "notes">): AnalysisSnapshot {
@@ -261,15 +342,18 @@ export class VaultIndex {
     const notes = this.db.query<{path: string; source: string}, []>("SELECT path,source FROM analysis_notes ORDER BY path").all();
     return { ...inputs, notes };
   }
-  search(query: string, options: SearchOptions = {}): SearchResult[] {
+  search(query: string, options: SearchOptions = {}, snapshot?: ReadonlyMap<string, string>, snapshotOffset = 0, allSnapshotRows = false): SearchResult[] {
     const limit = resultLimit(options.limit);
     if (!query.trim() || Buffer.byteLength(query) > 1024) throw new Error("search: query must contain 1..1024 bytes");
     // Literal AND terms: users cannot inject FTS operators, columns or SQL.
     const terms = query.trim().split(/\s+/).map(term => `"${term.replaceAll('"', '""')}"`).join(" AND ");
-    return this.db.query<SearchResult, [string, string | null, string | null, string | null, string | null, number]>(`SELECT n.path, n.title, substr(n.description,1,512) AS description, n.type, n.domain,
-      bm25(fts,0,5,2,1) AS score, snippet(fts,3,'','',' ... ',32) AS snippet FROM fts JOIN notes n ON n.path=fts.path
-      WHERE fts MATCH ? AND (? IS NULL OR n.type=?) AND (? IS NULL OR n.domain=?) ORDER BY score,n.path LIMIT ?`).all(terms, options.type ?? null, options.type ?? null, options.domain ?? null, options.domain ?? null, limit)
-      .map(row => ({ ...row, title: boundedText(row.title, 512).content, snippet: boundedText(row.snippet, 1024).content, type: boundedText(row.type, 256).content, domain: boundedText(row.domain, 256).content }));
+    const hashColumn = snapshot ? ",n.hash AS sourceHash" : "";
+    return this.db.query<SearchResult & { sourceHash?: string }, [string, string | null, string | null, string | null, string | null, number]>(`SELECT n.path, n.title, substr(n.description,1,512) AS description, n.type, n.domain,
+      bm25(fts,0,5,2,1) AS score, snippet(fts,3,'','',' ... ',32) AS snippet${hashColumn} FROM fts JOIN notes n ON n.path=fts.path
+      WHERE fts MATCH ? AND (? IS NULL OR n.type=?) AND (? IS NULL OR n.domain=?) ORDER BY score,n.path LIMIT ?`).all(terms, options.type ?? null, options.type ?? null, options.domain ?? null, options.domain ?? null, snapshot ? 10000 : limit)
+      .filter(row => !snapshot || snapshot.get(row.path) === row.sourceHash)
+      .slice(snapshot ? snapshotOffset : 0, (snapshot ? snapshotOffset : 0) + (snapshot && allSnapshotRows ? 10000 : limit))
+      .map(({ sourceHash: _, ...row }) => ({ ...row, title: boundedText(row.title, 512).content, snippet: boundedText(row.snippet, 1024).content, type: boundedText(row.type, 256).content, domain: boundedText(row.domain, 256).content }));
   }
   private find(target: string): string {
     if (!target || Buffer.byteLength(target) > 1024) throw new Error("note: expected target <=1024 bytes");

package/src/write-commands.ts

@@ -1,3 +1,4 @@
+import { COMMANDS as COMMAND_USAGE } from "./commands-table.ts";
 import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
 import { canonicalPath, isDenied } from "./platform.ts";
 import { loadRegistry } from "./registry.ts";
@@ -44,7 +45,7 @@ export function runWriteCommand(command: string, args: string[], home: string, e
   for (let i = 0; i < args.length; i++) {
     const arg = args[i]!;
     if (!arg.startsWith("--")) { positional.push(arg); continue; }
-    if (Object.hasOwn(flags, arg) || (!values.includes(arg) && !booleans.includes(arg))) throw new Error(`${command}: unknown or repeated option`);
+    if (Object.hasOwn(flags, arg) || (!values.includes(arg) && !booleans.includes(arg))) throw new Error(`${command}: unknown or repeated option: ${arg}`);
     if (booleans.includes(arg)) flags[arg] = true;
     else {
       const value = args[++i];
@@ -54,8 +55,12 @@ export function runWriteCommand(command: string, args: string[], home: string, e
   }
   const required = (name: string): string => { if (typeof flags[name] !== "string") throw new Error(`${name}: required`); return flags[name] as string; };
   const optional = (name: string): string | undefined => typeof flags[name] === "string" ? flags[name] as string : undefined;
-  const count = ["move", "link", "supersede"].includes(command) ? 3 : command === "tidy" ? 1 : 2;
-  if (positional.length !== count) throw new Error(`${command}: requires vault id${count > 1 ? " and target(s)" : ""}`);
+  const usage = COMMAND_USAGE.find(item => item.name === command)!.usage;
+  const targets = usage.slice(command.length).split(/\s+(?:--|\[)/, 1)[0]!.match(/<[^>]+>/g) ?? [];
+  if (positional.length !== targets.length) {
+    const expected = targets.length === 2 ? `${targets[0]} and one ${targets[1]}` : targets.join(" and ");
+    throw new Error(`${command}: takes ${expected}; got ${Math.max(0, positional.length - 1)} targets`);
+  }
   const vault = positional[0]!, target = positional[1]!;
   if (command === "tidy") {
     const all = flags["--all"] === true, rules = TIDY_RULES.filter(rule => all || flags[`--${rule}`] === true);

package/src/write-operations.ts

@@ -1,3 +1,4 @@
+import { isDeepStrictEqual } from "node:util";
 import { normalizeSource } from "./source.ts";
 import { parseFrontmatter, emitFrontmatter } from "./frontmatter.ts";
 import { normalizeTarget, proseLines, rewriteLinkTargets, extractLinks, resolveLinks, type LinkNote } from "./links.ts";
@@ -45,6 +46,42 @@ export function h2Sections(body: string): { name: string; start: number; end: nu
   }
   return sections;
 }
+/** Replace the last prose `## Topics` section (never one inside a fence) with the given topics; append one if absent. */
+export function replaceTopicsSection(body: string, topics: string[]): string {
+  const prose = proseLines(body), raw = body.split("\n");
+  let start = -1;
+  for (let i = prose.length - 1; i >= 0; i--) if (/^ {0,3}##[ \t]+Topics[ \t]*#*[ \t]*$/i.test(prose[i]!)) { start = i; break; }
+  const section = ["## Topics", "", ...topics.map(topic => `- ${topic}`), ""];
+  if (start < 0) return `${body.replace(/\n*$/, "")}\n\n${section.join("\n")}`;
+  // The section ends at the next prose heading of level 1 or 2.
+  let end = raw.length;
+  for (let i = start + 1; i < prose.length; i++) if (/^ {0,3}#{1,2}[ \t]/.test(prose[i]!)) { end = i; break; }
+  const tail = raw.slice(end);
+  return [...raw.slice(0, start), ...section, ...(tail.length ? tail : [])].join("\n").replace(/\n*$/, "\n");
+}
+/** In-place refiling replaces only visible wiki-list lines, preserving all other bytes. */
+function replaceTopicsList(body: string, topics: string[], newline: string): string {
+  const visible = proseLines(body), raw = body.match(/[^\n]*\n|[^\n]+$/g) ?? [];
+  let start = -1;
+  for (let i = visible.length - 1; i >= 0; i--) if (/^ {0,3}##[ \t]+Topics(?:[ \t]+#+)?[ \t]*$/i.test(visible[i]!)) { start = i; break; }
+  const list = topics.map(topic => `- ${topic}`).join(newline);
+  if (start < 0) return body + (body && !body.endsWith("\n") ? newline : "") + newline + `## Topics${newline}${newline}${list}${newline}`;
+  const items = new Set<number>();
+  // The Topics list is the leading run of list lines as V05 sees them: blank and fenced lines continue it (fences are kept),
+  // the first visible heading or prose line ends it.
+  for (let i = start + 1; i < raw.length; i++) {
+    const line = visible[i] ?? "";
+    if (/^[ \t]*(?:[-*+][ \t]+)?(\[\[[^\]|#]+\]\])[ \t]*$/.test(line)) items.add(i);
+    else if (line.trim()) break;
+  }
+  const first = items.values().next().value;
+  if (first === undefined) {
+    const addition = (raw[start]!.endsWith("\n") ? "" : newline) + list + (list ? newline : "");
+    return raw.slice(0, start + 1).join("") + addition + raw.slice(start + 1).join("");
+  }
+  const ending = /\r?\n$/.exec(raw[first]!)?.[0] ?? "";
+  return raw.map((line, i) => i === first ? list + (list ? ending : "") : items.has(i) ? "" : line).join("");
+}
 export function appendSection(source: string, section: string, markdown: string, path: string): string {
   if (typeof section !== "string" || !section.trim() || section.length > 120 || /[\r\n]/.test(section) || section.trim().replace(/[ \t]+#+$/, "").trim().toLowerCase() === "topics") refuse("V05", "append cannot target Topics or an invalid section");
   if (typeof markdown !== "string" || Buffer.byteLength(markdown) > 16384 || proseLines(markdown).some(line => /^ {0,3}##[ \t]+Topics(?:[ \t]+#+)?[ \t]*$/i.test(line))) refuse("V05", "append cannot introduce a Topics section or exceed 16384 bytes");
@@ -125,7 +162,15 @@ export function planOperation(operation: WriteOp, notes: Map<string, string>, re
       if (!operation.set || typeof operation.set !== "object" || Array.isArray(operation.set) || (operation.unset !== undefined && (!Array.isArray(operation.unset) || operation.unset.some(key => typeof key !== "string")))) refuse("V01", "invalid frontmatter update");
       const changes = {...operation.set};
       for (const key of operation.unset ?? []) changes[key] = undefined;
-      try { set(path, emitFrontmatter(document(source, path), normalizedSourceChanges(changes))); } catch (error) { if (error instanceof WriteRefusal) throw error; refuse("V01", "cannot emit frontmatter", path); }
+      try {
+        const before = document(source, path);
+        let after = emitFrontmatter(before, normalizedSourceChanges(changes));
+        if (Object.hasOwn(operation.set, "topics") && !isDeepStrictEqual(changes.topics, before.data.topics) && Array.isArray(changes.topics) && changes.topics.every((value): value is string => typeof value === "string")) {
+          const doc = document(after, path);
+          after = (doc.frontmatter ?? "") + replaceTopicsList(doc.body, changes.topics, /\r?\n/.exec(source)?.[0] ?? "\n");
+        }
+        set(path, after);
+      } catch (error) { if (error instanceof WriteRefusal) throw error; refuse("V01", "cannot emit frontmatter", path); }
       break;
     }
     case "link": {

package/src/write-validation.ts

@@ -181,22 +181,34 @@ function topicIssues(note: ParsedNote, index: Map<string, Set<ParsedNote>>, seve
   }
   return [];
 }
-function topicPlacement(note: ParsedNote): WriteIssue[] {
-  if (note.data.type === "moc" && note.data.topics === undefined) return [];
+/** V05 has two independent problems: placement (one Topics H2, and it is the last H2) and content (the last Topics section lists exactly the frontmatter topics). */
+function topicProblems(note: ParsedNote): Set<"placement" | "content"> {
+  const problems = new Set<"placement" | "content">();
+  if (note.data.type === "moc" && note.data.topics === undefined) return problems;
   const lines = proseLines(note.body), h2s = lines.flatMap((line, index) => /^ {0,3}##[ \t]+/.test(line) ? [index] : []);
   const topics = h2s.filter(index => /^ {0,3}##[ \t]+Topics(?:[ \t]+#+)?[ \t]*$/i.test(lines[index]!));
-  let valid = topics.length === 1 && h2s[h2s.length - 1] === topics[0];
+  if (topics.length !== 1 || h2s[h2s.length - 1] !== topics[0]) problems.add("placement");
+  const last = topics[topics.length - 1], end = h2s.find(index => last !== undefined && index > last) ?? lines.length;
   const actual: string[] = [];
-  if (valid) for (const line of lines.slice(topics[0]! + 1)) {
+  let content = last !== undefined;
+  if (last !== undefined) for (const line of lines.slice(last + 1, end)) {
     if (!line.trim()) continue;
     const wiki = /^[ \t]*(?:[-*+][ \t]+)?(\[\[[^\]|#]+\]\])[ \t]*$/.exec(line);
-    if (!wiki) { valid = false; break; }
+    if (!wiki) { content = false; break; }
     actual.push(noteKey(wiki[1]!.slice(2, -2)));
   }
   const expected = Array.isArray(note.data.topics) ? note.data.topics.map(value => typeof value === "string" ? noteKey(value.slice(2, -2)) : "") : [];
   const sortedExpected = [...expected].sort();
-  if (actual.length !== expected.length || [...actual].sort().some((value, index) => value !== sortedExpected[index])) valid = false;
-  return valid ? [] : [issue("V05", "Topics must be the final H2 and list exactly the frontmatter topics", "warning", note.path)];
+  if (!content || actual.length !== expected.length || [...actual].sort().some((value, index) => value !== sortedExpected[index])) problems.add("content");
+  return problems;
+}
+/** An error unless every V05 problem of the new note was already present in the prior note (as V01 retains only unchanged findings). */
+function topicPlacement(note: ParsedNote, old: ParsedNote | undefined, context: ValidationContext): WriteIssue[] {
+  const problems = topicProblems(note);
+  if (!problems.size) return [];
+  const before = context.operation.op !== "create" && old !== undefined ? topicProblems(old) : new Set<string>();
+  const retained = [...problems].every(problem => before.has(problem));
+  return [issue("V05", "Topics must be the final H2 and list exactly the frontmatter topics", retained ? "warning" : "error", note.path)];
 }
 
 /** Validate a complete mutation plan against the final overlaid snapshot, without filesystem writes. */
@@ -226,7 +238,7 @@ export function validateWrite(changes: PlannedChange[], context: ValidationConte
     const note = newNotes.get(change.path);
     if (!note) { issues.push(issue("V01", "invalid frontmatter", "error", change.path)); continue; }
     const prior=context.operation.op==="move"&&noteKey(context.operation.to)===noteKey(note.path)?oldNotes.get(movedFrom(context.operation.from, context)):oldNotes.get(note.path);
-    issues.push(...schemaIssues(note,prior,context), ...topicPlacement(note));
+    issues.push(...schemaIssues(note,prior,context), ...topicPlacement(note,prior,context));
     const source = note.data.source;
     let sourceDenied=strings(source).some(entry=>isDenied(entry.replaceAll("\\","/"),context.config.denyPaths));
     if(source!==undefined&&!sourceDenied)try{