@dzhechkov/harness-core 0.3.74 → 0.3.78

package/src/vector-tier.ts

@@ -0,0 +1,957 @@
+/**
+ * Vector Tier — the SEMANTIC half of the dz self-learning pattern store
+ * (`features/dz-rvf-vector-bridge`, ADR-001 "Option A extended").
+ *
+ * One PORT ({@link VectorEngine}) with two adapters behind it:
+ *
+ * - **AgentdbVectorEngine** (default) — reuses the `.dz/agentdb.db` ReasoningBank store the
+ *   consolidate Option-C mirror already writes (`agentdb-index.ts`), so semantic recall reads
+ *   the vectors that exist today. Zero new dependencies: `agentdb`/`better-sqlite3` are
+ *   dynamically resolved from the PROJECT, never imported at module top level.
+ * - **RvfVectorEngine** (opt-in, `memory.vector.engine = "rvf"`) — the portable single-file
+ *   VECTOR form (`.dz/memory/patterns.rvf` + `.idmap.json`/`.manifest.json` sidecars) via a
+ *   lazily imported `@ruvector/rvf`. Never a `dependencies` entry (at most a documentation-only
+ *   `peerDependenciesMeta`).
+ *
+ * HONEST-ERROR CONTRACT (load-bearing, Invariant I-1): every function in this module returns an
+ * honest `{ …, error?: string }` receipt and NEVER throws or hangs — engine absence, a failed
+ * embed, a locked DB, or a timeout all degrade to today's exact lexical behavior. The lexical
+ * store (`patterns.sqlite`/`patterns.json`) is the SOURCE OF TRUTH; the vector tier is a
+ * best-effort MIRROR that can be rebuilt from it at any time (`dz consolidate` backfill), and a
+ * vector hit whose lexical twin is gone is DROPPED, never resurrected (Invariant V-1).
+ *
+ * Constraint 7 (QR-10, field-diagnosis discipline): when lexical and vector counts diverge,
+ * **rule out local DB corruption before blaming the mirror** — check `dz doctor`'s store health
+ * first; the mirror's own divergence line (`dz vector status`) reports BOTH counts plus the
+ * `dz consolidate` backfill hint, and is informational, never an error.
+ *
+ * Both engine calls (read `search` AND write `upsert`/`listIds`) are wall-time bounded
+ * ({@link DEFAULT_VECTOR_TIMEOUT_MS}); a write-side timeout (e.g. a first-run embedding-model
+ * download) lands the batch in `.dz/mirror-pending.json` and never blocks `dz teach` (NC1).
+ *
+ * @packageDocumentation
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, renameSync, appendFileSync, copyFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createRequire } from 'node:module';
+
+import { isNoiseInsight } from '@dzhechkov/memory';
+import type { DreamPattern, MemoryRecord } from '@dzhechkov/memory';
+
+import {
+  recallPatterns,
+  recordToPattern,
+  patternRecordId,
+  patternIdentityOf,
+  dreamRecordId,
+  loadStoreRecords,
+  type PatternRecord,
+  type RecallHit,
+} from './patterns.js';
+import { indexPatternsToAgentdb, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder } from './agentdb-index.js';
+
+/* ------------------------------------------------------------------ */
+/*  Types (04_domain_model §3.4 / §4.1)                                */
+/* ------------------------------------------------------------------ */
+
+/** Which adapter sits behind the port. */
+export type VectorEngineKind = 'agentdb' | 'rvf';
+
+/** `memory.vector.engine` config modes (`.dz/config.json`). Absent/corrupt ⇒ `auto`. */
+export type VectorEngineMode = 'auto' | 'agentdb' | 'rvf' | 'off';
+
+/** One record on its way into the vector store (the ACL between dz records and engines). */
+export interface VectorEntry {
+  /** Join key back to the lexical store — the canonical `MemoryRecord.id` (`teach:…`/`dream:…`). */
+  readonly dzId: string;
+  /** The text that gets embedded (as `${taskType}: ${text}`) and stored. */
+  readonly text: string;
+  /** REAL reward signal in [0,1] — never a fabricated 1.0. */
+  readonly score: number;
+  /** ReasoningBank task_type: `dz-teach` for taught patterns, `dz-learning` for consolidate dreams. */
+  readonly taskType: string;
+  readonly tags?: readonly string[] | undefined;
+  readonly metadata?: Record<string, unknown> | undefined;
+}
+
+/** One semantic search hit — a POINTER into the lexical store, never a pattern by itself. */
+export interface VectorHit {
+  readonly dzId: string;
+  /** Cosine similarity (or engine-native score), higher = closer. */
+  readonly similarity: number;
+  readonly text?: string | undefined;
+}
+
+/** Honest outcome of a mirror attempt. Never thrown — always returned. */
+export interface MirrorReceipt {
+  /** Entries newly written to the vector store this call. */
+  readonly mirrored: number;
+  /** Entries skipped by the noise gate or the dzId dedup (already mirrored — I-5). */
+  readonly skipped: number;
+  /** Entries parked in `.dz/mirror-pending.json` for the next consolidate to heal (I-3). */
+  readonly queued: number;
+  readonly engine?: VectorEngineKind | undefined;
+  readonly error?: string | undefined;
+}
+
+/** The engine PORT — both adapters implement exactly this surface (04 §4.1). */
+export interface VectorEngine {
+  readonly kind: VectorEngineKind;
+  upsert(entries: readonly VectorEntry[]): Promise<{ indexed: number; error?: string | undefined }>;
+  search(query: string, limit: number): Promise<{ hits: VectorHit[]; error?: string | undefined }>;
+  listIds(): Promise<{ ids: string[]; error?: string | undefined }>;
+  /** Portable single-file checkpoint (RVF adapter only — `dz vector export`). */
+  exportCheckpoint?(dest: string): Promise<{ error?: string | undefined }>;
+}
+
+/** Outcome of {@link resolveVectorEngine}: an engine, or an honest reason why not. */
+export interface ResolvedVectorEngine {
+  readonly engine?: VectorEngine | undefined;
+  readonly reason?: string | undefined;
+}
+
+/** Recall mode: `hybrid` (default), `semantic` (`--semantic`, 2× vector weight), `lexical` (`--no-semantic`). */
+export type HybridRecallMode = 'hybrid' | 'semantic' | 'lexical';
+
+/** One merged recall hit (RRF-scored). `pattern` ALWAYS comes from the lexical store (V-1). */
+export interface HybridHit {
+  readonly pattern: PatternRecord;
+  readonly backend: RecallHit['backend'];
+  /** Reciprocal-rank-fusion score (ranking only — NOT the pattern's reward). */
+  readonly score: number;
+}
+
+/** Outcome of {@link recallHybrid}. With no engine this is content-identical to `recallPatterns`. */
+export interface HybridRecall {
+  readonly hits: HybridHit[];
+  readonly lexicalBackend: 'sqlite' | 'json';
+  readonly vectorEngine: VectorEngineKind | 'none';
+  /** Why the vector tier did not participate (engine absent / disabled). */
+  readonly vectorReason?: string | undefined;
+  /** Engine was present but the search failed/timed out — lexical results returned instead. */
+  readonly vectorError?: string | undefined;
+}
+
+/** Field observability for `dz vector status` (I-2/I-5 in the field). */
+export interface VectorTierStatus {
+  readonly mode: VectorEngineMode;
+  readonly kind?: VectorEngineKind | undefined;
+  readonly available: boolean;
+  readonly reason?: string | undefined;
+  readonly lexicalTotal: number;
+  readonly lexicalMirrorable: number;
+  readonly mirrored?: number | undefined;
+  readonly pending: number;
+}
+
+/** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
+export const DEFAULT_VECTOR_TIMEOUT_MS = 10_000;
+
+/* ------------------------------------------------------------------ */
+/*  Timeout wrapper (both legs — NC1/QR-1)                             */
+/* ------------------------------------------------------------------ */
+
+/**
+ * Bound `promise` to `ms` wall-clock milliseconds. On timeout, resolve with `onTimeout()`
+ * instead — the underlying operation keeps running detached (its eventual write is later
+ * deduplicated by dzId), but the CALLER's latency is bounded. A rejection also resolves via
+ * `onTimeout()` (honest-error contract: this wrapper never throws). The timer is cleared /
+ * unref'd so a fast path never keeps the process alive.
+ */
+export async function withVectorTimeout<T>(promise: Promise<T>, ms: number, onTimeout: () => T): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      promise.catch(() => onTimeout()),
+      new Promise<T>((resolvePromise) => {
+        timer = setTimeout(() => resolvePromise(onTimeout()), ms);
+        timer.unref?.();
+      }),
+    ]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
+/**
+ * Invoke an engine call so that BOTH a synchronous throw and an async rejection surface as an
+ * honest `onError(message)` value (never as the timeout fallback — a throw and a timeout are
+ * different diagnoses in the field). Pairs with {@link withVectorTimeout} at every call site.
+ */
+function safeEngineCall<T>(fn: () => Promise<T>, onError: (message: string) => T): Promise<T> {
+  const msg = (err: unknown): string => (err instanceof Error ? err.message : String(err));
+  try {
+    return fn().then((v) => v, (err: unknown) => onError(msg(err)));
+  } catch (err) {
+    return Promise.resolve(onError(msg(err)));
+  }
+}
+
+/* ------------------------------------------------------------------ */
+/*  Noise gate + ACL mappers (I-6, V-3)                                */
+/* ------------------------------------------------------------------ */
+
+const TOOL_TELEMETRY_RE = /^Tool \S+ invoked during session$/;
+
+/** Text that must never be embedded: bare-approval echoes + tool telemetry (V-3 / ADR-002). */
+export function isVectorNoise(text: string): boolean {
+  return isNoiseInsight(text) || TOOL_TELEMETRY_RE.test(text);
+}
+
+/**
+ * ACL: taught {@link PatternRecord} → {@link VectorEntry}. Returns `undefined` for noise (the
+ * ingest gate — I-6). Score is the record's REAL reward, never a fabricated 1.0.
+ */
+export function patternVectorEntry(p: PatternRecord, source = 'dz-teach'): VectorEntry | undefined {
+  if (isVectorNoise(p.pattern)) return undefined;
+  const dzId = patternRecordId(p);
+  return {
+    dzId,
+    text: p.pattern,
+    score: p.reward,
+    taskType: 'dz-teach',
+    tags: ['dz-teach', p.type],
+    metadata: { dzId, source, ts: p.ts, domain: p.domain },
+  };
+}
+
+/**
+ * ACL: harvested {@link DreamPattern} → {@link VectorEntry}. Byte-compatible with the
+ * pre-feature consolidate mirror rows (same task_type `dz-learning`, same tags, `dreamId`
+ * preserved in metadata — NFR-7); `dzId` is additive.
+ */
+export function dreamVectorEntry(d: DreamPattern): VectorEntry | undefined {
+  if (isVectorNoise(d.insight)) return undefined;
+  const dzId = dreamRecordId(d);
+  return {
+    dzId,
+    text: d.insight,
+    score: d.score,
+    taskType: 'dz-learning',
+    tags: ['dz-consolidate', d.outcome],
+    metadata: { source: 'dz-consolidate', skillId: d.skillId, sessionFile: d.sessionFile, ts: d.timestamp, dreamId: dzId, dzId },
+  };
+}
+
+/** ACL: stored {@link MemoryRecord} → {@link VectorEntry} (the consolidate-backfill mapper). */
+export function memoryRecordVectorEntry(r: MemoryRecord): VectorEntry | undefined {
+  if (isVectorNoise(r.text)) return undefined;
+  return {
+    dzId: r.id,
+    text: r.text,
+    score: r.score,
+    taskType: r.id.startsWith('dream:') ? 'dz-learning' : 'dz-teach',
+    tags: ['dz-backfill', r.outcome],
+    metadata: { dzId: r.id, source: r.metadata?.['source'] ?? 'dz-backfill', ts: r.timestamp, skillId: r.skillId },
+  };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Config + engine resolution cascade (05 §2.1)                       */
+/* ------------------------------------------------------------------ */
+
+/** Read `memory.vector.engine` from `.dz/config.json`. Absent/corrupt ⇒ `auto` (never throws). */
+export function readVectorEngineMode(projectRoot: string): VectorEngineMode {
+  try {
+    const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8')) as {
+      memory?: { vector?: { engine?: string } };
+    };
+    const mode = cfg.memory?.vector?.engine;
+    return mode === 'off' || mode === 'agentdb' || mode === 'rvf' || mode === 'auto' ? mode : 'auto';
+  } catch {
+    return 'auto';
+  }
+}
+
+/**
+ * Should `dz teach` attempt the best-effort vector mirror at all? True when the project opted
+ * into the agentdb memory backend (`memory.backend === 'agentdb'`, the same gate consolidate
+ * uses — D3) or explicitly configured a vector engine. A fresh, unconfigured project returns
+ * `false`, so its `dz teach` output stays byte-identical to the pre-feature baseline (AC-1).
+ */
+export function vectorMirrorEnabled(projectRoot: string): boolean {
+  try {
+    const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8')) as {
+      memory?: { backend?: string; vector?: { engine?: string } };
+    };
+    if (cfg.memory?.backend === 'agentdb') return true;
+    const engine = cfg.memory?.vector?.engine;
+    return engine === 'agentdb' || engine === 'rvf';
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Engine selection cascade: config mode → `require.resolve` probe (NO module load — a
+ * lexical-only project pays zero engine cost, NFR-5) → adapter or an honest reason.
+ * Never throws. `auto` prefers agentdb (it reads the vectors consolidate already wrote),
+ * falling through to rvf.
+ */
+/**
+ * Is `pkgName` installed for this PROJECT? A pure filesystem probe: walk `node_modules` up the
+ * directory tree from `projectRoot` (the npm resolution chain) — deliberately NOT
+ * `require.resolve`, which also consults process-global paths (`NODE_PATH`/global folders) and
+ * would make a lexical-only project's engine availability depend on the HOST process instead
+ * of the project (the I-1 determinism leak). No module is loaded (NFR-5).
+ */
+function isPackageInstalled(projectRoot: string, pkgName: string): boolean {
+  let dir = projectRoot;
+  for (;;) {
+    if (existsSync(join(dir, 'node_modules', pkgName, 'package.json'))) return true;
+    const parent = dirname(dir);
+    if (parent === dir) return false;
+    dir = parent;
+  }
+}
+
+export function resolveVectorEngine(projectRoot: string): ResolvedVectorEngine {
+  const mode = readVectorEngineMode(projectRoot);
+  if (mode === 'off') return { reason: 'vector tier disabled (memory.vector.engine = "off")' };
+  const canResolve = (id: string): boolean => isPackageInstalled(projectRoot, id);
+  if (mode === 'agentdb' || mode === 'auto') {
+    if (canResolve('agentdb') && canResolve('better-sqlite3')) return { engine: agentdbVectorEngine(projectRoot) };
+    if (mode === 'agentdb') {
+      return { reason: 'agentdb/better-sqlite3 not installed in project (run: dz setup --memory agentdb)' };
+    }
+  }
+  if (mode === 'rvf' || mode === 'auto') {
+    if (canResolve('@ruvector/rvf')) return { engine: rvfVectorEngine(projectRoot) };
+    if (mode === 'rvf') {
+      return { reason: '@ruvector/rvf not installed in project (npm i @ruvector/rvf) — vector tier inactive' };
+    }
+  }
+  return { reason: 'no vector engine available: agentdb/better-sqlite3 not installed in project (run: dz setup --memory agentdb)' };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Pending-mirror queue (generalized — one file for every source)     */
+/* ------------------------------------------------------------------ */
+
+/** Queue entry: a {@link VectorEntry} plus a legacy-compatible `insight` alias of `text`. */
+type PendingEntry = VectorEntry & { readonly insight: string };
+
+function pendingPath(projectRoot: string): string {
+  return join(projectRoot, '.dz', 'mirror-pending.json');
+}
+
+function toPending(e: VectorEntry): PendingEntry {
+  return { ...e, insight: e.text };
+}
+
+/** Legacy queue rows (pre-port `DreamPattern[]`) are converted on read — nothing is lost. */
+function readVectorPending(projectRoot: string): PendingEntry[] {
+  try {
+    const arr = JSON.parse(readFileSync(pendingPath(projectRoot), 'utf-8')) as unknown[];
+    if (!Array.isArray(arr)) return [];
+    const out: PendingEntry[] = [];
+    for (const item of arr) {
+      if (typeof item !== 'object' || item === null) continue;
+      const rec = item as Record<string, unknown>;
+      if (typeof rec['dzId'] === 'string' && typeof rec['text'] === 'string') {
+        out.push(toPending(rec as unknown as VectorEntry));
+      } else if (typeof rec['insight'] === 'string' && typeof rec['timestamp'] === 'string') {
+        const entry = dreamVectorEntry(rec as unknown as DreamPattern);
+        if (entry !== undefined) out.push(toPending(entry));
+      }
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
+function writeVectorPending(projectRoot: string, entries: readonly PendingEntry[]): void {
+  try {
+    if (entries.length === 0) {
+      if (existsSync(pendingPath(projectRoot))) rmSync(pendingPath(projectRoot));
+    } else {
+      const path = pendingPath(projectRoot);
+      mkdirSync(dirname(path), { recursive: true });
+      // Atomic write: a concurrent teach/consolidate must never observe a torn file. Write to a
+      // temp sibling then rename() (atomic on POSIX). Recovery guarantee: even if a racing writer
+      // clobbers the queue, backfillVectorMirror re-derives the missing set from the lexical store,
+      // so a dropped entry is recovered on the next consolidate — no permanent loss.
+      const tmp = `${path}.tmp`;
+      writeFileSync(tmp, JSON.stringify(entries, null, 2));
+      renameSync(tmp, path);
+    }
+  } catch { /* best-effort */ }
+}
+
+/** Honest failure note next to the session telemetry (the detached SessionEnd path is silent). */
+function logMirrorNote(projectRoot: string, error: string, pending: number): void {
+  try {
+    appendFileSync(
+      join(projectRoot, '.dz', 'sessions.jsonl'),
+      JSON.stringify({ event: 'mirror', ts: new Date().toISOString(), error, pending }) + '\n',
+    );
+  } catch { /* best-effort */ }
+}
+
+/* ------------------------------------------------------------------ */
+/*  VectorMirrorService — the ONE write seam (QR-6)                    */
+/* ------------------------------------------------------------------ */
+
+/** Options shared by the mirror/recall services. `engine: null` force-disables (tests). */
+export interface VectorServiceOptions {
+  readonly engine?: VectorEngine | null | undefined;
+  readonly timeoutMs?: number | undefined;
+}
+
+function pickEngine(projectRoot: string, opts: VectorServiceOptions): ResolvedVectorEngine {
+  if (opts.engine === null) return { reason: 'vector engine disabled (injected)' };
+  if (opts.engine !== undefined) return { engine: opts.engine };
+  return resolveVectorEngine(projectRoot);
+}
+
+/**
+ * Mirror prepared {@link VectorEntry}s into the vector store — **the single write seam** that
+ * teach, `teach --from-json`, consolidate, and the backfill all route through (QR-6). The
+ * lexical write is ALWAYS already durable before this runs (I-3). Semantics:
+ *
+ * 1. noise-gate the entries (I-6), merge with the pending queue (dedup by dzId),
+ * 2. nothing to do ⇒ `{mirrored:0}` with NO error and no queue file,
+ * 3. engine absent ⇒ park the batch in the queue + honest reason (heals on the next consolidate),
+ * 4. dedup against `engine.listIds()` (I-5 idempotency — a re-mirror adds 0 rows),
+ * 5. time-bounded `engine.upsert` (NC1); failure/timeout ⇒ queue + `sessions.jsonl` note.
+ *
+ * NEVER throws; the caller's exit code is unaffected by any outcome here (I-1).
+ */
+export async function mirrorEntriesToVector(
+  projectRoot: string,
+  entries: readonly VectorEntry[],
+  opts: VectorServiceOptions = {},
+): Promise<MirrorReceipt> {
+  try {
+    let skipped = 0;
+    const gated: VectorEntry[] = [];
+    for (const e of entries) {
+      if (isVectorNoise(e.text)) skipped += 1;
+      else gated.push(e);
+    }
+    const byId = new Map<string, PendingEntry>();
+    for (const e of [...readVectorPending(projectRoot), ...gated.map(toPending)]) {
+      if (!byId.has(e.dzId)) byId.set(e.dzId, e);
+    }
+    const batch = [...byId.values()];
+    if (batch.length === 0) return { mirrored: 0, skipped, queued: 0 };
+
+    const resolved = pickEngine(projectRoot, opts);
+    if (resolved.engine === undefined) {
+      writeVectorPending(projectRoot, batch);
+      const error = resolved.reason ?? 'no vector engine available';
+      logMirrorNote(projectRoot, error, batch.length);
+      return { mirrored: 0, skipped, queued: batch.length, error };
+    }
+    const engine = resolved.engine;
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
+
+    // I-5 idempotency: skip what the store already holds (best-effort, time-bounded).
+    let toSend: PendingEntry[] = batch;
+    const listed = await withVectorTimeout(
+      safeEngineCall(() => engine.listIds(), (m) => ({ ids: [] as string[], error: `vector listIds failed: ${m}` })),
+      timeoutMs,
+      () => ({ ids: [] as string[], error: 'vector listIds timed out' }),
+    );
+    if (listed.error === undefined) {
+      const have = new Set(listed.ids);
+      const before = toSend.length;
+      toSend = toSend.filter((e) => !have.has(e.dzId));
+      skipped += before - toSend.length;
+    }
+    if (toSend.length === 0) {
+      writeVectorPending(projectRoot, []);
+      return { mirrored: 0, skipped, queued: 0, engine: engine.kind };
+    }
+
+    const up = await withVectorTimeout(
+      safeEngineCall(
+        () => engine.upsert(toSend.map(({ insight: _insight, ...entry }) => entry)),
+        (m) => ({ indexed: 0, error: `vector mirror failed: ${m}` }),
+      ),
+      timeoutMs,
+      () => ({ indexed: 0, error: `vector mirror timed out after ${timeoutMs}ms (batch queued for the next consolidate)` }),
+    );
+    if (up.error !== undefined) {
+      writeVectorPending(projectRoot, toSend);
+      logMirrorNote(projectRoot, up.error, toSend.length);
+      return { mirrored: up.indexed, skipped, queued: toSend.length, engine: engine.kind, error: up.error };
+    }
+    writeVectorPending(projectRoot, []);
+    return { mirrored: up.indexed, skipped, queued: 0, engine: engine.kind };
+  } catch (err) {
+    // Belt-and-braces: the mirror must NEVER take the caller down (I-1/I-3).
+    return { mirrored: 0, skipped: 0, queued: 0, error: `mirror failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}
+
+/** Convenience seam for taught patterns: ACL-map + delegate to {@link mirrorEntriesToVector}. */
+export async function mirrorPatternsToVector(
+  projectRoot: string,
+  patterns: readonly PatternRecord[],
+  source = 'dz-teach',
+  opts: VectorServiceOptions = {},
+): Promise<MirrorReceipt> {
+  const entries: VectorEntry[] = [];
+  let gatedOut = 0;
+  for (const p of patterns) {
+    const e = patternVectorEntry(p, source);
+    if (e !== undefined) entries.push(e);
+    else gatedOut += 1; // noise never maps (I-6) — reported honestly as skipped
+  }
+  const receipt = await mirrorEntriesToVector(projectRoot, entries, opts);
+  return gatedOut === 0 ? receipt : { ...receipt, skipped: receipt.skipped + gatedOut };
+}
+
+/**
+ * Eventual consistency (FR-2): diff `lexical dzIds ∖ engine.listIds()` and mirror the missing
+ * set (bounded batch) + drain the pending queue. Run by `dz consolidate` after the watermark
+ * write, so a teach-time mirror failure heals on the next consolidate (AC-3). Engine absent ⇒
+ * silent no-op (the absent tier is a state, not an error).
+ */
+export async function backfillVectorMirror(
+  projectRoot: string,
+  opts: VectorServiceOptions & { readonly batchLimit?: number | undefined } = {},
+): Promise<MirrorReceipt> {
+  try {
+    const resolved = pickEngine(projectRoot, opts);
+    if (resolved.engine === undefined) {
+      return { mirrored: 0, skipped: 0, queued: readVectorPending(projectRoot).length };
+    }
+    const engine = resolved.engine;
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
+    const listed = await withVectorTimeout(
+      safeEngineCall(() => engine.listIds(), (m) => ({ ids: [] as string[], error: `vector listIds failed: ${m}` })),
+      timeoutMs,
+      () => ({ ids: [] as string[], error: 'vector listIds timed out' }),
+    );
+    if (listed.error !== undefined) {
+      return { mirrored: 0, skipped: 0, queued: readVectorPending(projectRoot).length, engine: engine.kind, error: listed.error };
+    }
+    const have = new Set(listed.ids);
+    const limit = opts.batchLimit ?? 200;
+    const missing: VectorEntry[] = [];
+    for (const r of loadStoreRecords(projectRoot)) {
+      if (have.has(r.id)) continue;
+      const e = memoryRecordVectorEntry(r);
+      if (e === undefined) continue;
+      missing.push(e);
+      if (missing.length >= limit) break;
+    }
+    // The seam drains the pending queue too (it merges + dedups internally).
+    return mirrorEntriesToVector(projectRoot, missing, { engine, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) });
+  } catch (err) {
+    return { mirrored: 0, skipped: 0, queued: 0, error: `backfill failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}
+
+/* ------------------------------------------------------------------ */
+/*  HybridRecallService — the read seam (05 §2.3)                      */
+/* ------------------------------------------------------------------ */
+
+/** One ranked pattern feeding the RRF merge (exported so the merge is unit-testable pure). */
+export interface RankedPattern {
+  readonly id: string;
+  readonly pattern: PatternRecord;
+  readonly backend: RecallHit['backend'];
+}
+
+const RRF_K = 60;
+
+/**
+ * Reciprocal Rank Fusion merge: `score(p) = Σ 1/(60 + rank)` over the lists containing `p`
+ * (semantic ranks weighted by `semanticWeight`). Dedup by id; `backend: 'both'` when a pattern
+ * appears in both lists. DETERMINISTIC (AC-6): ties break on id, so fixed inputs always yield
+ * the same ordering. Pure — no I/O.
+ */
+export function mergeHybridHits(
+  lexical: readonly RankedPattern[],
+  semantic: readonly RankedPattern[],
+  opts: { readonly limit: number; readonly semanticWeight?: number | undefined },
+): HybridHit[] {
+  const weight = opts.semanticWeight ?? 1;
+  interface Acc { pattern: PatternRecord; lex?: RecallHit['backend']; sem: boolean; score: number }
+  const acc = new Map<string, Acc>();
+  lexical.forEach((h, rank) => {
+    const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
+    cur.lex = h.backend;
+    cur.score += 1 / (RRF_K + rank + 1);
+    acc.set(h.id, cur);
+  });
+  semantic.forEach((h, rank) => {
+    const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
+    cur.sem = true;
+    cur.score += weight / (RRF_K + rank + 1);
+    acc.set(h.id, cur);
+  });
+  return [...acc.entries()]
+    .sort((a, b) => b[1].score - a[1].score || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
+    .slice(0, opts.limit)
+    .map(([, v]) => ({
+      pattern: v.pattern,
+      backend: v.lex !== undefined && v.sem ? ('both' as const) : v.lex ?? ('vector' as const),
+      score: v.score,
+    }));
+}
+
+/**
+ * Hybrid recall (FR-3): lexical `recallPatterns` FIRST (always, sync, UNCHANGED — AC-5), then a
+ * time-bounded semantic leg merged via RRF. Degradation contract (I-1): with no engine — or on
+ * any engine error/timeout — the returned hits are CONTENT-IDENTICAL to plain `recallPatterns`
+ * output, with the honest `vectorReason`/`vectorError` alongside. A vector hit whose dzId no
+ * longer resolves in the lexical store is DROPPED (V-1 — pruned patterns never resurrect, QR-4).
+ */
+export async function recallHybrid(
+  projectRoot: string,
+  query: string,
+  opts: VectorServiceOptions & { readonly limit?: number | undefined; readonly mode?: HybridRecallMode | undefined } = {},
+): Promise<HybridRecall> {
+  // Config-surface note (QE P3, benign by design): recall resolves the engine directly, while teach
+  // only mirrors when the memory backend is agentdb (or an engine is explicit). In the window where
+  // the agentdb deps are INSTALLED but `memory.backend` hasn't been switched, the semantic leg reads a
+  // store teach never populated → empty/foreign hits. That degrades honestly (orphan dzIds are dropped
+  // against the lexical store, V-1) and lexical results are always returned, so it never misleads — it
+  // only spends a bounded, cached read. Not gated on purpose: a read-only recall must not depend on the
+  // write-side backend flag.
+  const limit = opts.limit ?? 10;
+  const mode = opts.mode ?? 'hybrid';
+  const lexical = recallPatterns(projectRoot, query, limit);
+  const lexicalBackend: 'sqlite' | 'json' = lexical[0]?.backend === 'sqlite' ? 'sqlite' : 'json';
+  const lexicalOnly = (extra: Partial<Pick<HybridRecall, 'vectorEngine' | 'vectorReason' | 'vectorError'>>): HybridRecall => ({
+    hits: lexical.map((h, rank) => ({ pattern: h.pattern, backend: h.backend, score: 1 / (RRF_K + rank + 1) })),
+    lexicalBackend,
+    vectorEngine: 'none',
+    ...extra,
+  });
+
+  if (mode === 'lexical') return lexicalOnly({});
+
+  let resolved: ResolvedVectorEngine;
+  try {
+    resolved = pickEngine(projectRoot, opts);
+  } catch (err) {
+    return lexicalOnly({ vectorReason: err instanceof Error ? err.message : String(err) });
+  }
+  if (resolved.engine === undefined) {
+    return lexicalOnly(resolved.reason !== undefined ? { vectorReason: resolved.reason } : {});
+  }
+  const engine = resolved.engine;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
+
+  const sr = await withVectorTimeout(
+    safeEngineCall(
+      () => engine.search(query, limit * 2),
+      (m) => ({ hits: [] as VectorHit[], error: `vector search failed: ${m}` }),
+    ),
+    timeoutMs,
+    () => ({ hits: [] as VectorHit[], error: `vector search timed out after ${timeoutMs}ms` }),
+  );
+  if (sr.error !== undefined) {
+    return { ...lexicalOnly({}), vectorEngine: engine.kind, vectorError: sr.error };
+  }
+
+  // Resolve dzId → the FULL lexical record (source of truth). Orphans are dropped (V-1/QR-4).
+  let records: MemoryRecord[];
+  try {
+    records = loadStoreRecords(projectRoot);
+  } catch {
+    records = [];
+  }
+  const idToPattern = new Map<string, PatternRecord>();
+  const identityToId = new Map<string, string>();
+  for (const r of records) {
+    const p = recordToPattern(r);
+    idToPattern.set(r.id, p);
+    identityToId.set(patternIdentityOf(p), r.id);
+  }
+  const semantic: RankedPattern[] = [];
+  const seen = new Set<string>();
+  for (const h of sr.hits) {
+    if (seen.has(h.dzId)) continue;
+    const p = idToPattern.get(h.dzId);
+    if (p === undefined) continue; // vector-only orphan — the store pruned/expired it; NEVER resurrect
+    seen.add(h.dzId);
+    semantic.push({ id: h.dzId, pattern: p, backend: 'vector' });
+  }
+  const lex: RankedPattern[] = lexical.map((h) => ({
+    id: identityToId.get(patternIdentityOf(h.pattern)) ?? patternRecordId(h.pattern),
+    pattern: h.pattern,
+    backend: h.backend,
+  }));
+  const hits = mergeHybridHits(lex, semantic, { limit, semanticWeight: mode === 'semantic' ? 2 : 1 });
+  return { hits, lexicalBackend, vectorEngine: engine.kind };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Status (dz vector status / dz doctor divergence line)              */
+/* ------------------------------------------------------------------ */
+
+/** Field observability: engine availability + mirrored-vs-lexical counts + queue size. */
+export async function vectorTierStatus(
+  projectRoot: string,
+  opts: VectorServiceOptions = {},
+): Promise<VectorTierStatus> {
+  const mode = readVectorEngineMode(projectRoot);
+  let records: MemoryRecord[];
+  try {
+    records = loadStoreRecords(projectRoot);
+  } catch {
+    records = [];
+  }
+  const lexicalMirrorable = records.filter((r) => !isVectorNoise(r.text)).length;
+  const pending = readVectorPending(projectRoot).length;
+  const resolved = pickEngine(projectRoot, opts);
+  if (resolved.engine === undefined) {
+    return {
+      mode,
+      available: false,
+      ...(resolved.reason !== undefined ? { reason: resolved.reason } : {}),
+      lexicalTotal: records.length,
+      lexicalMirrorable,
+      pending,
+    };
+  }
+  const engine = resolved.engine;
+  const listed = await withVectorTimeout(
+    safeEngineCall(() => engine.listIds(), (m) => ({ ids: [] as string[], error: `vector listIds failed: ${m}` })),
+    opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS,
+    () => ({ ids: [] as string[], error: 'vector listIds timed out' }),
+  );
+  return {
+    mode,
+    kind: engine.kind,
+    available: true,
+    ...(listed.error !== undefined ? { reason: listed.error } : {}),
+    lexicalTotal: records.length,
+    lexicalMirrorable,
+    mirrored: listed.error === undefined ? listed.ids.length : undefined,
+    pending,
+  };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Adapter A (default): AgentdbVectorEngine                           */
+/* ------------------------------------------------------------------ */
+
+/**
+ * Option A: the `.dz/agentdb.db` ReasoningBank store. `upsert` delegates to the very same
+ * {@link indexPatternsToAgentdb} rows the consolidate Option-C mirror writes today (schema
+ * unchanged — the `agentdb-memory` MCP skill keeps reading them, NFR-7); `search`/`listIds`
+ * are the new READONLY halves in `agentdb-index.ts`.
+ */
+function agentdbVectorEngine(projectRoot: string): VectorEngine {
+  return {
+    kind: 'agentdb',
+    async upsert(entries) {
+      const r = await indexPatternsToAgentdb(
+        projectRoot,
+        entries.map((e) => ({
+          taskType: e.taskType,
+          text: e.text,
+          score: e.score,
+          ...(e.tags !== undefined ? { tags: e.tags } : {}),
+          ...(e.metadata !== undefined ? { metadata: e.metadata } : {}),
+        })),
+      );
+      return { indexed: r.indexed, ...(r.error !== undefined ? { error: r.error } : {}) };
+    },
+    async search(query, limit) {
+      const r = await searchAgentdbPatterns(projectRoot, query, { limit });
+      const hits: VectorHit[] = [];
+      for (const h of r.hits) {
+        if (h.dzId !== undefined) hits.push({ dzId: h.dzId, similarity: h.similarity, text: h.text });
+      }
+      return { hits, ...(r.error !== undefined ? { error: r.error } : {}) };
+    },
+    async listIds() {
+      return listAgentdbDzIds(projectRoot);
+    },
+  };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Adapter B (opt-in): RvfVectorEngine                                */
+/* ------------------------------------------------------------------ */
+
+interface RvfIdmap {
+  version: 1;
+  /** rvf slot/label → dzId. */
+  slots: Record<string, string>;
+}
+
+function rvfBase(projectRoot: string): string {
+  return join(projectRoot, '.dz', 'memory', 'patterns.rvf');
+}
+
+function readRvfIdmap(projectRoot: string): RvfIdmap {
+  try {
+    const parsed = JSON.parse(readFileSync(`${rvfBase(projectRoot)}.idmap.json`, 'utf-8')) as RvfIdmap;
+    return typeof parsed === 'object' && parsed !== null && typeof parsed.slots === 'object' ? parsed : { version: 1, slots: {} };
+  } catch {
+    return { version: 1, slots: {} };
+  }
+}
+
+function writeRvfSidecars(projectRoot: string, idmap: RvfIdmap): void {
+  const base = rvfBase(projectRoot);
+  mkdirSync(dirname(base), { recursive: true });
+  writeFileSync(`${base}.idmap.json`, JSON.stringify(idmap, null, 2));
+  writeFileSync(`${base}.manifest.json`, JSON.stringify(
+    { model: 'Xenova/all-MiniLM-L6-v2', dim: 384, engine: '@ruvector/rvf', version: 1 },
+    null,
+    2,
+  ));
+}
+
+interface RvfStoreHandle {
+  ingest: (id: string, vec: Float32Array) => Promise<unknown> | unknown;
+  query: (vec: Float32Array, k: number) => Promise<unknown> | unknown;
+  close?: (() => Promise<void> | void) | undefined;
+  exportCheckpoint?: ((dest: string) => Promise<unknown> | unknown) | undefined;
+}
+
+/**
+ * Open a `@ruvector/rvf` store, pinned to the REAL published SDK surface (grounded in
+ * ruvector/npm/packages/rvf/src/index.ts + a live linux-x64 smoke against @ruvector/rvf@0.2.3):
+ * the canonical class is `RvfDatabase` with `create(path, { dimensions })` → `ingestBatch([{id,
+ * vector}])` → `query(vector, k)` returning `[{ id, distance }]` → `close()`. A few tolerant
+ * fallbacks (add/insert, search) keep older/alt shapes working; anything unrecognized returns an
+ * HONEST error (the D8 no-go evidence), never a throw. NOTE: RVF stores the vector under the `id`
+ * we pass (= the dzId), so no slot↔id mapping is needed — the query result's `id` IS the dzId.
+ */
+export async function openRvfStore(mod: Record<string, unknown>, path: string, dimensions: number): Promise<RvfStoreHandle | { error: string }> {
+  try {
+    const dflt = mod['default'] as Record<string, unknown> | undefined;
+    const cls = (mod['RvfDatabase'] ?? dflt?.['RvfDatabase'] ?? mod['RvfStore'] ?? mod['Store'] ?? dflt?.['RvfStore'] ?? dflt ?? mod) as {
+      create?: (p: string, o: { dimensions?: number; dimension?: number }) => unknown;
+      open?: (p: string, o: { dimensions?: number; dimension?: number }) => unknown;
+    };
+    let db: Record<string, unknown> | undefined;
+    if (typeof cls.create === 'function') db = (await cls.create(path, { dimensions, dimension: dimensions })) as Record<string, unknown>;
+    else if (typeof cls.open === 'function') db = (await cls.open(path, { dimensions, dimension: dimensions })) as Record<string, unknown>;
+    else if (typeof cls === 'function') db = new (cls as unknown as new (p: string, o: { dimensions: number }) => Record<string, unknown>)(path, { dimensions });
+    if (db === undefined) return { error: 'unsupported @ruvector/rvf API (no RvfDatabase.create/open/constructor) — record a D8 no-go' };
+    const ingestBatch = (db['ingestBatch'] ?? db['ingest'] ?? db['add'] ?? db['insert']) as ((rows: Array<{ id: string; vector: Float32Array }>) => unknown) | undefined;
+    const query = (db['query'] ?? db['search']) as ((vec: Float32Array, k: number) => unknown) | undefined;
+    if (typeof ingestBatch !== 'function' || typeof query !== 'function') {
+      return { error: 'unsupported @ruvector/rvf store surface (no ingestBatch/ingest + query/search) — record a D8 no-go' };
+    }
+    const close = db['close'];
+    const exp = db['exportCheckpoint'] ?? db['export_checkpoint'] ?? db['checkpoint'];
+    return {
+      ingest: (id, vec) => ingestBatch.call(db, [{ id, vector: vec }]),
+      query: (vec, k) => query.call(db, vec, k),
+      close: typeof close === 'function' ? (close as () => void).bind(db) : undefined,
+      exportCheckpoint: typeof exp === 'function' ? (exp as (d: string) => unknown).bind(db) : undefined,
+    };
+  } catch (err) {
+    return { error: `@ruvector/rvf store open failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}
+
+async function loadRvfModule(
+  projectRoot: string,
+): Promise<{ ok: true; mod: Record<string, unknown> } | { ok: false; error: string }> {
+  try {
+    const req = createRequire(join(projectRoot, 'package.json'));
+    const mod = (await import(pathToFileURL(req.resolve('@ruvector/rvf')).href)) as Record<string, unknown>;
+    return { ok: true, mod };
+  } catch (err) {
+    return { ok: false, error: `@ruvector/rvf failed to load: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}
+
+/**
+ * Option B: the portable single-file VECTOR form (`.rvf`, magic `0x52564653`) with
+ * `.idmap.json` (slot ↔ dzId) and `.manifest.json` (model/dim — Constraint 5 staleness
+ * detection) sidecars. Embeddings come from agentdb's `EmbeddingService` when resolvable —
+ * with NEITHER embedder the engine degrades gracefully with an honest reason (05 §3.6).
+ */
+function rvfVectorEngine(projectRoot: string): VectorEngine {
+  const noEmbedder = 'rvf engine present but no embedder — install agentdb (dz setup --memory agentdb)';
+  return {
+    kind: 'rvf',
+    async upsert(entries) {
+      const emb = await resolveAgentdbEmbedder(projectRoot);
+      if ('error' in emb) return { indexed: 0, error: noEmbedder };
+      const loaded = await loadRvfModule(projectRoot);
+      if (!loaded.ok) return { indexed: 0, error: loaded.error };
+      const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
+      if ('error' in store) return { indexed: 0, error: store.error };
+      try {
+        const idmap = readRvfIdmap(projectRoot);
+        let indexed = 0;
+        for (const e of entries) {
+          const vec = await emb.embed(`${e.taskType}: ${e.text}`);
+          await store.ingest(e.dzId, vec); // RVF stores the vector UNDER id = dzId (no slot mapping)
+          idmap.slots[e.dzId] = e.dzId; // sidecar keeps the dzId set for listIds/observability
+          indexed += 1;
+        }
+        await store.close?.();
+        writeRvfSidecars(projectRoot, idmap);
+        return { indexed };
+      } catch (err) {
+        return { indexed: 0, error: `rvf upsert failed: ${err instanceof Error ? err.message : String(err)}` };
+      }
+    },
+    async search(query, limit) {
+      const emb = await resolveAgentdbEmbedder(projectRoot);
+      if ('error' in emb) return { hits: [], error: noEmbedder };
+      const loaded = await loadRvfModule(projectRoot);
+      if (!loaded.ok) return { hits: [], error: loaded.error };
+      const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
+      if ('error' in store) return { hits: [], error: store.error };
+      try {
+        const idmap = readRvfIdmap(projectRoot);
+        const raw = await store.query(await emb.embed(query), limit);
+        await store.close?.();
+        const hits: VectorHit[] = [];
+        if (Array.isArray(raw)) {
+          for (const item of raw as Array<Record<string, unknown> | [unknown, unknown]>) {
+            const id = Array.isArray(item) ? item[0] : item['id'] ?? item['slot'] ?? item['label'];
+            const distance = Array.isArray(item) ? item[1] : item['distance'] ?? item['score'] ?? item['similarity'];
+            // RVF returns the id we ingested (= dzId); the sidecar is a safety join for alt shapes.
+            const dzId = idmap.slots[String(id)] ?? (typeof id === 'string' ? id : undefined);
+            // distance: lower = closer → negate so higher = better (RRF ranks by position regardless).
+            if (dzId !== undefined) hits.push({ dzId, similarity: typeof distance === 'number' ? -distance : 0 });
+          }
+        }
+        return { hits };
+      } catch (err) {
+        return { hits: [], error: `rvf search failed: ${err instanceof Error ? err.message : String(err)}` };
+      }
+    },
+    async listIds() {
+      // Sidecar-only read — no SDK load needed for observability/dedup.
+      return { ids: [...new Set(Object.values(readRvfIdmap(projectRoot).slots))] };
+    },
+    async exportCheckpoint(dest) {
+      try {
+        const base = rvfBase(projectRoot);
+        if (!existsSync(base)) return { error: `no ${base} yet — teach/consolidate with the rvf engine first` };
+        const loaded = await loadRvfModule(projectRoot);
+        let exported = false;
+        if (loaded.ok) {
+          const store = await openRvfStore(loaded.mod, base, 384);
+          if (!('error' in store) && store.exportCheckpoint !== undefined) {
+            await store.exportCheckpoint(dest);
+            await store.close?.();
+            exported = true;
+          } else if (!('error' in store)) {
+            await store.close?.();
+          }
+        }
+        if (!exported) copyFileSync(base, dest); // append-only format — a file copy IS a checkpoint
+        for (const sidecar of ['.idmap.json', '.manifest.json']) {
+          if (existsSync(`${base}${sidecar}`)) copyFileSync(`${base}${sidecar}`, `${dest}${sidecar}`);
+        }
+        return {};
+      } catch (err) {
+        return { error: `rvf export failed: ${err instanceof Error ? err.message : String(err)}` };
+      }
+    },
+  };
+}