@staix/vault-mem 0.3.2 → 0.3.3

package/src/spec-validate.ts

@@ -0,0 +1,241 @@
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+import { realPath } from "./platform.ts";
+import { extractLinks } from "./links.ts";
+import { specReference, MAX_SPEC_INPUTS } from "./spec-graph.ts";
+import { discoverSpecs, type SpecCandidate, type SpecDiagnostic, type SpecSnapshot } from "./spec-source.ts";
+import { specFreshness, specMetadataIssues, type SpecFreshnessEntry } from "./spec-freshness.ts";
+
+export interface SpecValidationIssue {
+  code: string;
+  severity: "error" | "warning";
+  path: string;
+  id?: string;
+  field?: string;
+  target?: string;
+  message: string;
+}
+export interface SpecValidationOptions { env?: NodeJS.ProcessEnv; requireFresh?: boolean; limit?: number; git?: boolean }
+export interface SpecValidationReport {
+  ok: boolean;
+  errors: number;
+  warnings: number;
+  items: SpecValidationIssue[];
+  diagnostics: SpecDiagnostic[];
+  freshness: SpecFreshnessEntry[];
+  truncated: boolean;
+}
+export interface SpecRepoValidationOptions { env?: NodeJS.ProcessEnv; requireFresh?: boolean; limit?: number }
+export interface SpecRepoValidation { code: 0 | 1; report: SpecValidationReport }
+const RANK: Readonly<Record<string, number>> = { goal: 0, architecture: 1, module: 2, submodule: 3 };
+const STATUSES = new Set(["draft", "active", "deprecated"]);
+const MAX_OPERATIONS = 200000, MAX_ITEMS = 500, MAX_REPORT_BYTES = 128 * 1024;
+const compare = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
+const nonempty = (value: unknown): value is string => typeof value === "string" && !!value.trim();
+const candidateId = (candidate: SpecCandidate) => nonempty(candidate.frontmatter.id) ? candidate.frontmatter.id : undefined;
+interface ErrorOwner { kind: "old" | "new"; path: string }
+interface Collected { records: { issue: SpecValidationIssue; key: string }[]; freshness: SpecFreshnessEntry[] }
+
+/** Complete collection is separate from display limits, so the introduced-error gate never compares samples. */
+function collect(snapshot: SpecSnapshot, options: SpecValidationOptions, owners?: ReadonlyMap<string, ErrorOwner>): Collected {
+  const candidates = [...(snapshot.candidates ?? snapshot.specs)].sort((a, b) => compare(a.path.normalize("NFC"), b.path.normalize("NFC")) || compare(a.path, b.path));
+  if (new Set(candidates.map(candidate => candidate.path)).size !== candidates.length) throw new Error("spec validation: duplicate candidate paths; provide one candidate record per file");
+  if (candidates.length > MAX_SPEC_INPUTS) throw new Error("spec validation: more than 10000 candidates; reduce the candidate tree before validating");
+  let operations = 0;
+  const tick = () => { if (++operations > MAX_OPERATIONS) throw new Error("spec validation: more than 200000 checks; reduce repeated links or metadata lists before validating"); };
+  const hashes = new Map<string, string>();
+  const digest = (value: string): string => {
+    let hash = hashes.get(value);
+    if (hash === undefined) { hash = createHash("sha256").update(value).digest("hex"); hashes.set(value, hash); }
+    return hash;
+  };
+  const records: Collected["records"] = [], keys = new Set<string>();
+  const addIssue = (issue: SpecValidationIssue) => {
+    tick();
+    const correspondence = owners?.get(issue.path) ?? { kind: owners === undefined ? "old" : "new", path: issue.path };
+    const owner = `${correspondence.kind}:${digest(correspondence.path)}`;
+    // Hash values independently: a long owner id is never copied once per relation/error key.
+    const key = JSON.stringify([digest(snapshot.commonDir), issue.severity, issue.code, owner, issue.field ?? "", issue.target === undefined ? "" : digest(issue.target)]);
+    if (!keys.has(key)) { keys.add(key); records.push({ issue, key }); }
+  };
+  const add = (candidate: SpecCandidate, code: string, severity: SpecValidationIssue["severity"], field: string, message: string, target?: string) => {
+    const id = candidateId(candidate);
+    addIssue({ code, severity, path: candidate.path, ...(id === undefined ? {} : { id }), field, message, ...(target === undefined ? {} : { target }) });
+  };
+  const nodes = new Map<string, SpecCandidate>(), first = new Map<string, string>();
+  for (const candidate of candidates) {
+    const id = candidateId(candidate);
+    if (id !== undefined) {
+      const previous = first.get(id);
+      if (previous !== undefined) add(candidate, "duplicate_id", "error", "id", "duplicate id; give each candidate a unique id", previous);
+      else first.set(id, candidate.path);
+      if (nonempty(candidate.frontmatter.type) && !nodes.has(id)) nodes.set(id, candidate);
+    }
+  }
+  const covers = new Map<string, Map<string, SpecCandidate>>();
+  for (const candidate of candidates) {
+    tick();
+    const data = candidate.frontmatter, id = candidateId(candidate);
+    if (options.git === false) for (const issue of specMetadataIssues(candidate)) addIssue(issue);
+    for (const field of ["id", "type", "title"] as const) {
+      if (data[field] === undefined) add(candidate, `missing_${field}`, "error", field, `required ${field} is missing; add a nonempty string field`);
+      else if (!nonempty(data[field])) add(candidate, "wrong_shape", "error", field, `${field} must be a nonempty string; correct the frontmatter field`);
+    }
+    if (nonempty(data.type) && !Object.hasOwn(RANK, data.type)) add(candidate, "unknown_type", "error", "type", "unknown type; use goal, architecture, module or submodule");
+    if (data.status !== undefined && (typeof data.status !== "string" || !STATUSES.has(data.status))) add(candidate, "unknown_status", "error", "status", "unknown status; use draft, active or deprecated");
+    if (data.parent === undefined && data.type !== "goal") add(candidate, "orphan", "warning", "parent", "non-goal spec has no parent; link it to its owning higher-level spec");
+    if (data.status === "active" && data.paths === undefined) add(candidate, "active_without_paths", "warning", "paths", "active spec has no paths; add relative Git pathspecs for its source scope");
+    const checkLink = (field: string, value: unknown, shapeField = field): string | undefined => {
+      tick();
+      const target = specReference(value);
+      if (target === undefined) { add(candidate, "wrong_shape", "error", shapeField, "link must be one id or wiki link; correct the frontmatter value"); return undefined; }
+      if (!nodes.has(target)) add(candidate, "dangling_link", "error", field, "frontmatter link names no graph node; link to an existing spec id or remove it", target);
+      return target;
+    };
+    let parent: string | undefined, replacement: string | undefined;
+    if (data.parent !== undefined) parent = checkLink("parent", data.parent);
+    if (data["superseded-by"] !== undefined) {
+      replacement = checkLink("superseded-by", data["superseded-by"]);
+      if (data.status !== "deprecated") add(candidate, "superseded_status", "error", "superseded-by", "replacement link requires deprecated status; deprecate the spec or remove the link");
+      if (replacement !== undefined && replacement === id) add(candidate, "superseded_self", "error", "superseded-by", "replacement points to this spec; link to a different replacement", replacement);
+    } else if (data.status === "deprecated") add(candidate, "deprecated_without_replacement", "warning", "superseded-by", "deprecated spec has no replacement; add a superseded-by id when one exists");
+    if (data.status === "deprecated" && replacement !== undefined && replacement !== id && nodes.get(replacement)?.frontmatter.status === "deprecated") add(candidate, "deprecated_replacement", "warning", "superseded-by", "replacement is also deprecated; link to an active replacement", replacement);
+    for (const field of ["depends-on", "references", "implements"] as const) {
+      if (data[field] === undefined) continue;
+      const values = data[field];
+      if (!Array.isArray(values)) { add(candidate, "wrong_shape", "error", field, `${field} must be a list of links; wrap the value in a list`); continue; }
+      for (let i = 0; i < values.length; i++) checkLink(field, values[i], `${field}[${i}]`);
+    }
+    if (parent !== undefined) {
+      const parentType = nodes.get(parent)?.frontmatter.type;
+      if (typeof data.type === "string" && typeof parentType === "string" && Object.hasOwn(RANK, data.type) && Object.hasOwn(RANK, parentType) && RANK[data.type]! <= RANK[parentType]!) add(candidate, "type_rank", "error", "parent", "child rank must exceed parent rank; choose a higher-level parent (levels may be skipped)", parent);
+    }
+    for (const field of ["covers", "tags"] as const) {
+      const values = data[field];
+      if (values === undefined) continue;
+      let valid = Array.isArray(values);
+      if (Array.isArray(values)) for (const value of values) { tick(); if (!nonempty(value)) valid = false; }
+      if (!valid) {
+        add(candidate, "wrong_shape", "error", field, `${field} must be a list of nonempty strings; correct the frontmatter field`);
+        continue;
+      }
+      if (field === "covers") for (const value of values as string[]) {
+        const token = value.trim();
+        let owners = covers.get(token); if (!owners) covers.set(token, owners = new Map());
+        owners.set(candidate.path, candidate);
+      }
+    }
+    for (const link of extractLinks(candidate.body)) {
+      tick();
+      if (link.target && !nodes.has(link.target)) add(candidate, "unknown_body_link", "warning", "body", "body wiki link names no graph node; correct the target id or remove the link", link.target);
+    }
+  }
+  for (const [token, owners] of covers) if (owners.size > 1) for (const candidate of owners.values()) add(candidate, "duplicate_covers", "warning", "covers", "covers token is claimed by multiple specs; choose a single owning spec", token);
+  const goals = [...nodes.values()].filter(candidate => candidate.frontmatter.type === "goal");
+  if (goals.length > 1) for (const candidate of goals) add(candidate, "multiple_goals", "warning", "type", "multiple goal roots exist; consolidate or document separate goal trees");
+  // Iterative traversal handles a 10000-node chain without overflowing the JS stack.
+  const finished = new Set<string>();
+  for (const id of nodes.keys()) {
+    if (finished.has(id)) continue;
+    const chain: string[] = [], positions = new Map<string, number>();
+    let next: string | undefined = id;
+    while (next !== undefined && nodes.has(next) && !finished.has(next) && !positions.has(next)) {
+      tick(); positions.set(next, chain.length); chain.push(next);
+      next = specReference(nodes.get(next)!.frontmatter.parent);
+    }
+    if (next !== undefined && positions.has(next)) for (const member of chain.slice(positions.get(next)!)) add(nodes.get(member)!, "parent_cycle", "error", "parent", "parent cycle detected; remove a cycle link and point to a higher-level parent", specReference(nodes.get(member)!.frontmatter.parent));
+    for (const member of chain) finished.add(member);
+  }
+  for (const diagnostic of snapshot.diagnostics) {
+    if (diagnostic.code === "invalid_frontmatter") addIssue({ code: "invalid_frontmatter", severity: "error", path: diagnostic.path, field: "frontmatter", message: diagnostic.message });
+    else if (snapshot.candidates === undefined && diagnostic.code === "duplicate_id") addIssue({ code: "duplicate_id", severity: "error", path: diagnostic.path, id: diagnostic.id, field: "id", target: diagnostic.firstPath, message: diagnostic.message });
+    else if (snapshot.candidates === undefined && diagnostic.code === "invalid_identity") addIssue({ code: "wrong_shape", severity: "error", path: diagnostic.path, field: "id/type", message: diagnostic.message });
+  }
+  if (options.requireFresh && options.git === false) throw new Error("spec validation: requireFresh needs Git inspection; enable Git checks before requiring freshness");
+  const freshness = options.git === false ? { items: [], issues: [] } : specFreshness(snapshot, { env: options.env });
+  for (const issue of freshness.issues) addIssue(issue);
+  if (options.requireFresh) for (const item of freshness.items) if (item.state === "stale") addIssue({ code: "stale", severity: "error", path: item.path, id: item.id, field: "verified-against", message: "spec source scope is stale; review its paths and refresh verified-against after verification" });
+  records.sort((a, b) => (a.issue.severity === b.issue.severity ? 0 : a.issue.severity === "error" ? -1 : 1) || compare(a.issue.path.normalize("NFC"), b.issue.path.normalize("NFC")) || compare(a.issue.path, b.issue.path) || compare(a.issue.code, b.issue.code) || compare(a.issue.field ?? "", b.issue.field ?? "") || compare(a.key, b.key));
+  return { records, freshness: freshness.items };
+}
+
+/** Display is count/byte bounded; oversized optional identifiers are omitted, never silently renamed. */
+function display<T extends object>(values: readonly T[], limit: number, budget: { bytes: number; truncated: boolean }): T[] {
+  const out: T[] = [];
+  for (const original of values) {
+    if (out.length >= limit) { budget.truncated = true; break; }
+    const value = { ...original } as T & { id?: string; target?: string };
+    for (const field of ["id", "target"] as const) if (typeof value[field] === "string" && Buffer.byteLength(value[field]!) > 1024) { delete value[field]; budget.truncated = true; }
+    const bytes = Buffer.byteLength(JSON.stringify(value)) + 1;
+    if (budget.bytes + bytes > MAX_REPORT_BYTES) { budget.truncated = true; break; }
+    budget.bytes += bytes; out.push(value);
+  }
+  return out;
+}
+function reportLimit(value = MAX_ITEMS): number {
+  if (!Number.isSafeInteger(value) || value < 1 || value > MAX_ITEMS) throw new Error("spec validation: report limit must be 1..500; pass a smaller positive item limit");
+  return value;
+}
+export function validateSpecs(snapshot: SpecSnapshot, options: SpecValidationOptions = {}): SpecValidationReport {
+  const limit = reportLimit(options.limit), result = collect(snapshot, options), budget = { bytes: 1024, truncated: false };
+  const errors = result.records.filter(record => record.issue.severity === "error").length;
+  return { ok: errors === 0, errors, warnings: result.records.length - errors,
+    items: display(result.records.map(record => record.issue), limit, budget),
+    diagnostics: display(snapshot.diagnostics, limit, budget), freshness: display(result.freshness, limit, budget),
+    get truncated() { return budget.truncated; } };
+}
+/** Existing error identities are retained even when their display sample was truncated. */
+export function introducedSpecErrors(current: SpecSnapshot, prospective: SpecSnapshot, options: SpecValidationOptions = {}): SpecValidationIssue[] {
+  const limit = reportLimit(options.limit);
+  const pathsById = (input: SpecSnapshot) => {
+    const paths = new Map<string, Set<string>>();
+    const add = (id: string, path: string) => { let found = paths.get(id); if (!found) paths.set(id, found = new Set()); found.add(path); };
+    for (const candidate of input.candidates ?? input.specs) { const id = candidateId(candidate); if (id !== undefined) add(id, candidate.path); }
+    if (input.candidates === undefined) for (const diagnostic of input.diagnostics) if (diagnostic.code === "duplicate_id" && diagnostic.id !== undefined) add(diagnostic.id, diagnostic.path);
+    return paths;
+  };
+  const oldIds = pathsById(current), nextIds = pathsById(prospective), owners = new Map<string, ErrorOwner>(), reserved = new Set<string>();
+  for (const candidate of prospective.candidates ?? prospective.specs) {
+    const id = candidateId(candidate), previous = id === undefined ? undefined : oldIds.get(id);
+    if (previous?.size === 1 && nextIds.get(id!)?.size === 1) {
+      const path = previous.values().next().value!;
+      owners.set(candidate.path, { kind: "old", path }); reserved.add(path);
+    }
+  }
+  const previousPaths = new Set([...(current.candidates ?? current.specs).map(candidate => candidate.path), ...current.diagnostics.map(diagnostic => diagnostic.path)]);
+  const prospectivePaths = new Set([...(prospective.candidates ?? prospective.specs).map(candidate => candidate.path), ...prospective.diagnostics.map(diagnostic => diagnostic.path)]);
+  for (const path of prospectivePaths) if (!owners.has(path)) {
+    const retained = previousPaths.has(path) && !reserved.has(path);
+    owners.set(path, { kind: retained ? "old" : "new", path });
+    if (retained) reserved.add(path);
+  }
+  // Duplicate identity belongs to the id, even when a move changes the first file in path order.
+  // Keep collection keys unchanged so validation still reports each duplicate file.
+  const comparisonKey = (input: SpecSnapshot, record: Collected["records"][number]): string =>
+    record.issue.code === "duplicate_id" && record.issue.id !== undefined
+      ? JSON.stringify([createHash("sha256").update(input.commonDir).digest("hex"), "duplicate_id", createHash("sha256").update(record.issue.id).digest("hex")])
+      : record.key;
+  const old = new Map<string, number>();
+  for (const record of collect(current, options).records) if (record.issue.severity === "error") {
+    const key = comparisonKey(current, record); old.set(key, (old.get(key) ?? 0) + 1);
+  }
+  const prospectiveErrors = collect(prospective, options, owners).records.filter(record => record.issue.severity === "error");
+  // Retained owners consume existing debt before new or moved files do.
+  prospectiveErrors.sort((a, b) => Number(owners.get(b.issue.path)?.kind === "old") - Number(owners.get(a.issue.path)?.kind === "old"));
+  const next = prospectiveErrors.filter(record => {
+    const key = comparisonKey(prospective, record), count = old.get(key) ?? 0;
+    if (count === 0) return true;
+    old.set(key, count - 1); return false;
+  });
+  const result = display(next.map(record => record.issue), limit, { bytes: 1024, truncated: false });
+  if (next.length && !result.length) throw new Error("spec validation: introduced error metadata exceeds the report budget; reduce metadata before writing");
+  return result;
+}
+/** Registry-free CI logic; no home/config/cache creation and no authored freshness fields. */
+export function validateSpecRepo(repo: string, options: SpecRepoValidationOptions = {}): SpecRepoValidation {
+  let root: string;
+  try { root = realPath(resolve(repo)); } catch (error) { throw new Error("spec validation: repository root is unavailable; pass an existing checkout root", { cause: error }); }
+  const snapshot = discoverSpecs({ id: "repo", path: root, write: "none", pii: "deny" }, { env: options.env });
+  const report = validateSpecs(snapshot, options);
+  return { code: report.ok ? 0 : 1, report };
+}

package/src/vault-index.ts

@@ -87,7 +87,7 @@ function readPage(prefix: Buffer, maxBytes: number, offset: number, totalBytes:
   const truncated = offset + end < totalBytes;
   return { content: prefix.subarray(0, end).toString("utf8"), truncated, ...(truncated ? { nextOffset: offset + end } : {}) };
 }
-function h2Sections(body: string): { name: string; start: number; end: number }[] {
+export function h2Sections(body: string): { name: string; start: number; end: number }[] {
   const visible = proseLines(body), lines = body.match(/[^\n]*\n|[^\n]+$/g) ?? [];
   const sections: { name: string; start: number; end: number }[] = [];
   let offset = 0, active: number | undefined;
@@ -104,7 +104,7 @@ function h2Sections(body: string): { name: string; start: number; end: number }[
   }
   return sections;
 }
-function headingMetadata(sections: { name: string }[]): Pick<NoteReadResult, "headings" | "headingsTruncated"> {
+export function headingMetadata(sections: { name: string }[]): Pick<NoteReadResult, "headings" | "headingsTruncated"> {
   const headings: string[] = [];
   let bytes = 2, truncated = false;
   for (const { name } of sections) {
@@ -120,7 +120,8 @@ export interface IndexSourceAdapter {
   paths(): string[];
   validatePath(path: string): void;
   accept(path: string, document: FrontmatterDocument, hash: string): boolean;
-  invalid(path: string): void;
+  invalid(path: string): boolean | void;
+  skipFile?(path: string, reason: "oversized" | "invalid_utf8", bytes: Buffer): boolean;
 }
 export class IndexCacheError extends Error {}
 export interface IndexOptions { memory?: boolean; adapter?: IndexSourceAdapter; reuseCache?: boolean }
@@ -206,10 +207,12 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
   }
   /** Scan metadata each call; hash only new or metadata-changed files. Atomic cache update. */
   scan(rebuild = false, options: { skipInvalidFrontmatter?: boolean; captureInvalidFrontmatter?: boolean } = {}): { notes: number; changed: number; removed: number; skipped?: string[] } {
-    return this.db.transaction(() => {
+    // Discovery/binding Git calls must never hold a SQLite writer lock.
+    let preparedPaths: string[] | undefined;
+    if (this.adapter) { this.adapter.validate(); preparedPaths = this.adapter.paths(); }
+    const result = this.db.transaction(() => {
       const notesDir = join(this.root, "notes");
-      if (this.adapter) this.adapter.validate();
-      else if (realpathSync(this.root) !== this.root || realpathSync(notesDir) !== notesDir || !lstatSync(notesDir).isDirectory()) throw new Error(`vault ${this.vault.id}: path or notes directory changed`);
+      if (!this.adapter && (realpathSync(this.root) !== this.root || realpathSync(notesDir) !== notesDir || !lstatSync(notesDir).isDirectory())) throw new Error(`vault ${this.vault.id}: path or notes directory changed`);
       const indexed = this.adapter ? new Set(this.db.query<{path:string}, []>("SELECT path FROM notes").all().map(row => row.path)) : undefined;
       const previous = new Map(this.db.query<{ path: string; hash: string; metadata: string }, []>(this.adapter ? "SELECT path,hash,metadata FROM analysis_notes" : options.captureInvalidFrontmatter ? "SELECT path,hash,metadata FROM notes UNION SELECT path,hash,metadata FROM analysis_notes" : "SELECT path,hash,metadata FROM notes").all().map(note => [note.path, note]));
       if (rebuild) this.db.run("DELETE FROM fts");
@@ -218,21 +221,25 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
       const skipped: string[] = [], touched = new Set<string>(), keys = new Set<string>();
       // Links resolve by path only, so other notes re-resolve only when they target a touched path.
       const touch = (path: string) => { touched.add(path); keys.add(noteKey(path)); };
-      const paths = this.adapter ? this.adapter.paths() : readdirSync(notesDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).filter(entry => entry.name.endsWith(".md") && entry.isFile()).map(entry => entry.name);
+      const paths = preparedPaths ?? readdirSync(notesDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).filter(entry => entry.name.endsWith(".md") && entry.isFile()).map(entry => entry.name);
       for (const path of paths) {
         this.adapter?.validatePath(path);
         const file = join(this.adapter ? this.root : notesDir, path);
         const before = this.adapter ? specFileIO(path, "inspect", () => lstatSync(file)) : lstatSync(file);
         if (!before.isFile() || before.isSymbolicLink()) throw new Error(this.adapter ? `spec discovery ${path}: selected path changed; restore a plain file and retry` : `vault ${this.vault.id}: note path changed`);
-        if (before.size > MAX_NOTE_BYTES) throw new Error(this.adapter ? `spec discovery ${path}: Markdown exceeds ${MAX_NOTE_BYTES} bytes; split the file before rebuilding` : `vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
+        if (!this.adapter && before.size > MAX_NOTE_BYTES) throw new Error(`vault ${this.vault.id}, ${path}: note exceeds ${MAX_NOTE_BYTES} bytes`);
         live.add(path); present.add(path);
-        if (options.captureInvalidFrontmatter || this.adapter) {
+        if (options.captureInvalidFrontmatter && !this.adapter) {
           analysisBytes += before.size;
           if (present.size > 10000 || analysisBytes > 64 * 1024 * 1024) throw new Error(this.adapter ? `spec discovery ${path}: more than 10000 Markdown files or 64 MiB of input; ignore large or vendored Markdown via .gitignore and untrack already-tracked excluded files with git rm --cached before rebuilding` : "analysis: snapshot exceeds bound");
         }
+        if (this.adapter) {
+          analysisBytes += Math.min(before.size, MAX_NOTE_BYTES + 1);
+          if (present.size > 10000 || analysisBytes > 64 * 1024 * 1024) throw new Error(`spec discovery ${path}: more than 10000 Markdown inputs or 64 MiB of bounded reads; ignore vendored Markdown via .gitignore and untrack excluded files with git rm --cached`);
+        }
         const old = previous.get(path), stamp = metadata(before);
         if (!this.adapter && !rebuild && old?.metadata === stamp && this.db.query("SELECT 1 FROM analysis_notes WHERE path=?").get(path)) continue;
-        const cached = this.adapter && !rebuild && old?.metadata === stamp ? this.db.query<{source:string}, [string]>("SELECT source FROM analysis_notes WHERE path=?").get(path) : null;
+        const cached = this.adapter && before.size <= MAX_NOTE_BYTES && !rebuild && old?.metadata === stamp ? this.db.query<{source:string}, [string]>("SELECT source FROM analysis_notes WHERE path=?").get(path) : null;
         let source: string, hash: string;
         if (cached && old) { source = cached.source; hash = old.hash; }
         else {
@@ -240,7 +247,7 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
           let buffer: Buffer;
           try {
             const stat = this.adapter ? specFileIO(path, "inspect", () => fstatSync(fd)) : fstatSync(fd);
-            if (!stat.isFile() || stat.size > MAX_NOTE_BYTES || metadata(stat) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
+            if (!stat.isFile() || (!this.adapter && stat.size > MAX_NOTE_BYTES) || metadata(stat) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
             // Fixed bound even if a concurrent writer grows the file.
             buffer = Buffer.alloc(MAX_NOTE_BYTES + 1);
             let size = 0;
@@ -250,15 +257,28 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
               if (!count || size === buffer.length) break;
             }
             buffer = buffer.subarray(0, size);
-            if (size > MAX_NOTE_BYTES || metadata(this.adapter ? specFileIO(path, "inspect", () => fstatSync(fd)) : fstatSync(fd)) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
+            if ((!this.adapter && size > MAX_NOTE_BYTES) || metadata(this.adapter ? specFileIO(path, "inspect", () => fstatSync(fd)) : fstatSync(fd)) !== stamp) throw new Error(this.adapter ? `spec discovery ${path}: file changed during scan; retry on a stable worktree` : `vault ${this.vault.id}, ${path}: note changed during scan`);
           } finally { if (this.adapter) specFileIO(path, "close", () => closeSync(fd)); else closeSync(fd); }
+          this.adapter?.validatePath(path);
+          if (this.adapter && metadata(specFileIO(path, "inspect", () => lstatSync(file))) !== stamp) throw new Error(`spec discovery ${path}: file changed during read; retry on a stable worktree`);
           if (this.adapter) {
+            const skip = (reason: "oversized" | "invalid_utf8") => {
+              if (!this.adapter?.skipFile?.(path, reason, buffer.subarray(0, MAX_NOTE_BYTES))) return false;
+              live.delete(path); skipped.push(path);
+              this.db.query("DELETE FROM analysis_notes WHERE path=?").run(path);
+              return true;
+            };
+            if (before.size > MAX_NOTE_BYTES || buffer.length > MAX_NOTE_BYTES) {
+              if (skip("oversized")) continue;
+              throw new Error(`spec discovery ${path}: Markdown exceeds ${MAX_NOTE_BYTES} bytes; split the candidate before validating`);
+            }
             try { source = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(buffer); }
-            catch { throw new Error(`spec discovery ${path}: Markdown is not valid UTF-8; re-encode the file as UTF-8 before rebuilding`); }
+            catch {
+              if (skip("invalid_utf8")) continue;
+              throw new Error(`spec discovery ${path}: Markdown is not valid UTF-8; re-encode the candidate as UTF-8 before validating`);
+            }
           } else source = buffer.toString("utf8");
           hash = digest(buffer);
-          this.adapter?.validatePath(path);
-          if (this.adapter && metadata(specFileIO(path, "inspect", () => lstatSync(file))) !== stamp) throw new Error(`spec discovery ${path}: file changed during read; retry on a stable worktree`);
           this.db.query("INSERT OR REPLACE INTO analysis_notes VALUES (?,?,?,?)").run(path, source, hash, stamp);
         }
         if (!this.adapter && !rebuild && old?.hash === hash) { this.db.query("UPDATE notes SET metadata=? WHERE path=?").run(stamp, path); continue; }
@@ -266,8 +286,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
         try { document = parseFrontmatter(source); }
         catch {
           if (!this.adapter && !options.skipInvalidFrontmatter && !options.captureInvalidFrontmatter) throw new Error(`vault ${this.vault.id}, ${path}: invalid frontmatter`);
-          this.adapter?.invalid(path);
-          live.delete(path); skipped.push(path); continue;
+          const report = this.adapter?.invalid(path);
+          live.delete(path); if (report !== false) skipped.push(path); continue;
         }
         if (this.adapter) {
           this.adapter.validatePath(path);
@@ -290,7 +310,6 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
       if (this.adapter) {
         // Spec links are resolved by frontmatter id in buildSpecGraph, never by vault paths.
         this.db.run("DELETE FROM links");
-        this.adapter.validate();
         return { notes: live.size, changed, removed, ...(skipped.length ? { skipped } : {}) };
       }
       const stale = this.db.query<{ user_version: number }, []>("PRAGMA user_version").get()?.user_version !== LINK_RESOLUTION;
@@ -313,6 +332,8 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
       if (stale) this.db.run(`PRAGMA user_version = ${LINK_RESOLUTION}`);
       return { notes: live.size, changed, removed, ...(skipped.length ? { skipped } : {}) };
     }).immediate();
+    this.adapter?.validate();
+    return result;
   }
   /** Consume cached note sources only. Auxiliary inputs are captured separately by analysisInput. */
   analysisSnapshot(inputs: Omit<AnalysisSnapshot, "notes">): AnalysisSnapshot {
@@ -321,7 +342,7 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
     const notes = this.db.query<{path: string; source: string}, []>("SELECT path,source FROM analysis_notes ORDER BY path").all();
     return { ...inputs, notes };
   }
-  search(query: string, options: SearchOptions = {}, snapshot?: ReadonlyMap<string, string>): SearchResult[] {
+  search(query: string, options: SearchOptions = {}, snapshot?: ReadonlyMap<string, string>, snapshotOffset = 0, allSnapshotRows = false): SearchResult[] {
     const limit = resultLimit(options.limit);
     if (!query.trim() || Buffer.byteLength(query) > 1024) throw new Error("search: query must contain 1..1024 bytes");
     // Literal AND terms: users cannot inject FTS operators, columns or SQL.
@@ -329,8 +350,9 @@ export class VaultIndex<Source extends VaultSpec | SpecSourceSpec = VaultSpec> {
     const hashColumn = snapshot ? ",n.hash AS sourceHash" : "";
     return this.db.query<SearchResult & { sourceHash?: string }, [string, string | null, string | null, string | null, string | null, number]>(`SELECT n.path, n.title, substr(n.description,1,512) AS description, n.type, n.domain,
       bm25(fts,0,5,2,1) AS score, snippet(fts,3,'','',' ... ',32) AS snippet${hashColumn} FROM fts JOIN notes n ON n.path=fts.path
-      WHERE fts MATCH ? AND (? IS NULL OR n.type=?) AND (? IS NULL OR n.domain=?) ORDER BY score,n.path LIMIT ?`).all(terms, options.type ?? null, options.type ?? null, options.domain ?? null, options.domain ?? null, limit)
+      WHERE fts MATCH ? AND (? IS NULL OR n.type=?) AND (? IS NULL OR n.domain=?) ORDER BY score,n.path LIMIT ?`).all(terms, options.type ?? null, options.type ?? null, options.domain ?? null, options.domain ?? null, snapshot ? 10000 : limit)
       .filter(row => !snapshot || snapshot.get(row.path) === row.sourceHash)
+      .slice(snapshot ? snapshotOffset : 0, (snapshot ? snapshotOffset : 0) + (snapshot && allSnapshotRows ? 10000 : limit))
       .map(({ sourceHash: _, ...row }) => ({ ...row, title: boundedText(row.title, 512).content, snippet: boundedText(row.snippet, 1024).content, type: boundedText(row.type, 256).content, domain: boundedText(row.domain, 256).content }));
   }
   private find(target: string): string {