@ngockhoale/ukit 3.0.8 → 3.0.10

package/src/core/memory/storeBackup.js

@@ -0,0 +1,226 @@
+// storeBackup — operator lane over whole v2 store docs (SPEC §5 FR-008/FR-009).
+// createBackup snapshots records.json copies + a sha256 manifest into a
+// timestamped dir under <memoryRoot>/backups/ (or explicit outDir). Every
+// restore write still commits through mutateMemory op 'restore', so guard
+// redaction, fingerprint dedupe, and tombstone precedence are never
+// duplicated here. diagnoseStore is strictly read-only — it never triggers
+// the quarantine path that a mutation would.
+//
+// Backup copies may contain secrets → mode 0600, same boundary as the
+// canonical store and quarantine copies (SPEC §14).
+
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { ensureDir, pathExists, readJsonIfExists, writeJson } from '../fileOps.js';
+import { buildRuntimePaths } from '../runtimePaths.js';
+import { buildUserPaths } from '../userPaths.js';
+import { mutateMemory } from './mutateMemory.js';
+import { readRecordStore } from './recordStore.js';
+
+const MANIFEST_VERSION = 1;
+const COPY_MODE = 0o600;
+
+function sha256(buf) {
+  return crypto.createHash('sha256').update(buf).digest('hex');
+}
+
+function resolveStores({ projectRoot, homeDir, includeUser } = {}) {
+  const stores = [];
+  if (projectRoot) {
+    const paths = buildRuntimePaths(projectRoot);
+    stores.push({
+      kind: 'project',
+      paths,
+      recordsPath: paths.memoryV2RecordsPath,
+      target: { kind: 'project', projectRoot },
+    });
+  }
+  if (includeUser || (homeDir && !projectRoot)) {
+    const paths = buildUserPaths({ homeDir });
+    stores.push({
+      kind: 'user',
+      paths,
+      recordsPath: paths.memoryV2RecordsPath,
+      target: { kind: 'user', homeDir },
+    });
+  }
+  return stores;
+}
+
+function backupFileName(kind) {
+  return `${kind}-records.json`;
+}
+
+function stateHint(state) {
+  switch (state) {
+    case 'missing': return 'no v2 store yet — records appear after the first memory write or migration';
+    case 'empty': return 'store exists but holds no usable records';
+    case 'corrupt': return 'records.json failed to parse — restore from a backup; a mutation will quarantine the file first';
+    case 'unsupported-schema': return 'written by a newer UKit build — upgrade before mutating';
+    default: return null;
+  }
+}
+
+/**
+ * createBackup({projectRoot, homeDir?, outDir?, includeUser?}) → {backupDir, manifest}
+ * Copies each store's records.json byte-identical (corrupt bytes included —
+ * the manifest marks state:'corrupt' so restore skips it). Default location
+ * `<projectMemoryRoot>/backups/<ts>/`; outDir is the explicit escape.
+ */
+export async function createBackup({ projectRoot, homeDir, outDir, includeUser } = {}) {
+  const stores = resolveStores({ projectRoot, homeDir, includeUser });
+  if (stores.length === 0) throw new Error('createBackup: projectRoot or homeDir required');
+
+  const stamp = `${new Date().toISOString().replace(/[:.]/g, '-')}-${crypto.randomBytes(3).toString('hex')}`;
+  const backupDir = outDir ?? path.join(stores[0].paths.memoryRoot, 'backups', stamp);
+  await ensureDir(backupDir);
+
+  const manifest = { version: MANIFEST_VERSION, createdAt: new Date().toISOString(), stores: [] };
+  for (const store of stores) {
+    const doc = await readRecordStore(store.recordsPath);
+    const entry = {
+      kind: store.kind,
+      state: doc.state,
+      generation: doc.generation,
+      recordCount: doc.records.length,
+      tombstoneCount: doc.tombstones.length,
+      sha256: null,
+      file: null,
+    };
+    if (doc.state !== 'missing' && await pathExists(store.recordsPath)) {
+      const bytes = await fs.readFile(store.recordsPath);
+      const dest = path.join(backupDir, backupFileName(store.kind));
+      await fs.writeFile(dest, bytes, { mode: COPY_MODE });
+      await fs.chmod(dest, COPY_MODE).catch(() => {});
+      entry.sha256 = sha256(bytes);
+      entry.file = backupFileName(store.kind);
+    }
+    manifest.stores.push(entry);
+  }
+  const manifestPath = path.join(backupDir, 'manifest.json');
+  await writeJson(manifestPath, manifest, { fsync: true });
+  await fs.chmod(manifestPath, COPY_MODE).catch(() => {});
+  return { backupDir, manifest };
+}
+
+/**
+ * restoreBackup(backupDir, {projectRoot, homeDir?, dryRun?}) →
+ *   {restored, skipped, plan[]}
+ * Each store file is verified against the manifest sha256 and parsed before
+ * any write; a tampered/malformed backup file is a plan 'skip', never a
+ * partial merge. Merges go through mutateMemory op 'restore' — a corrupt
+ * target is quarantined inside that path (first call returns 'corrupt' after
+ * the quarantine copy, the retry merges onto the now-missing doc).
+ */
+export async function restoreBackup(backupDir, { projectRoot, homeDir, dryRun } = {}) {
+  const plan = [];
+  let restored = 0;
+  let skipped = 0;
+
+  const manifest = await readJsonIfExists(path.join(backupDir, 'manifest.json'));
+  if (!manifest || !Array.isArray(manifest.stores)) {
+    return { restored, skipped, plan: [{ kind: 'manifest', action: 'skip', reason: 'manifest-missing-or-invalid' }] };
+  }
+
+  const stores = resolveStores({ projectRoot, homeDir, includeUser: true });
+  const byKind = new Map(stores.map((s) => [s.kind, s]));
+
+  for (const entry of manifest.stores) {
+    const store = byKind.get(entry.kind);
+    if (!store) {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: 'no-target' });
+      continue;
+    }
+    if (!entry.file) {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: `backup-state:${entry.state ?? 'missing'}` });
+      continue;
+    }
+    // Boundary (SPEC §14): a manifest file entry must resolve inside
+    // backupDir — a crafted '..' path is a skip, never a read outside.
+    const backupRoot = path.resolve(backupDir);
+    const filePath = path.resolve(backupRoot, String(entry.file));
+    if (filePath !== backupRoot && !filePath.startsWith(backupRoot + path.sep)) {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: 'backup-file-out-of-boundary' });
+      continue;
+    }
+    let bytes;
+    try {
+      bytes = await fs.readFile(filePath);
+    } catch {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: 'backup-file-missing' });
+      continue;
+    }
+    if (entry.sha256 && sha256(bytes) !== entry.sha256) {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: 'sha256-mismatch' });
+      continue;
+    }
+    let doc;
+    try {
+      doc = JSON.parse(bytes.toString('utf8'));
+    } catch {
+      doc = null;
+    }
+    if (!doc || typeof doc !== 'object' || !Array.isArray(doc.records)) {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: 'corrupt-backup' });
+      continue;
+    }
+    if (dryRun) {
+      plan.push({ kind: entry.kind, action: 'dry-run', records: doc.records.length });
+      continue;
+    }
+
+    let res = await mutateMemory(store.target, { op: 'restore', payload: { records: doc.records } });
+    if (res?.status === 'corrupt') {
+      // Quarantine already happened inside the locked RMW; retry merges onto
+      // the now-missing doc. Never loop — a second 'corrupt' is a real failure.
+      res = await mutateMemory(store.target, { op: 'restore', payload: { records: doc.records } });
+    }
+    if (res?.status === 'ok') {
+      restored += res.added ?? 0;
+      plan.push({ kind: entry.kind, action: (res.added ?? 0) > 0 ? 'restored' : 'noop', added: res.added ?? 0 });
+    } else {
+      skipped += 1;
+      plan.push({ kind: entry.kind, action: 'skip', reason: `restore-status:${res?.status ?? 'null'}` });
+    }
+  }
+  return { restored, skipped, plan };
+}
+
+/**
+ * diagnoseStore({projectRoot, homeDir?}) → {stores:[StoreDiag]}
+ * Read-only doctor: typed state + counts + quarantine listing + hint.
+ * Never mutates — uses readRecordStore directly, not the mutate path.
+ */
+export async function diagnoseStore({ projectRoot, homeDir } = {}) {
+  const stores = resolveStores({ projectRoot, homeDir, includeUser: Boolean(homeDir) });
+  const out = [];
+  for (const store of stores) {
+    const doc = await readRecordStore(store.recordsPath);
+    const quarantineDir = path.join(path.dirname(store.recordsPath), 'quarantine');
+    let quarantine = [];
+    try {
+      quarantine = (await fs.readdir(quarantineDir)).map((n) => path.join(quarantineDir, n));
+    } catch {
+      quarantine = [];
+    }
+    out.push({
+      kind: store.kind,
+      state: doc.state,
+      generation: doc.generation,
+      recordCount: doc.records.length,
+      tombstoneCount: doc.tombstones.length,
+      invalidSkipped: doc.invalidSkipped,
+      quarantine,
+      hint: stateHint(doc.state),
+    });
+  }
+  return { stores: out };
+}

package/src/core/memory/storeV2.js

@@ -5,19 +5,17 @@
 // v1→v2 migration (SPEC §4); migrate.js is lazy-imported to break the
 // storeV2↔migrate import cycle. The record-document read/write core lives in
 // recordStore.js (FR-007) so the user-level store (userMemory.js) shares it
-// without the migration flow; mutations go through mutateRecordStore, which
-// serializes read-modify-write under the shared file lock.
+// without the migration flow; all public mutations route through
+// mutateMemory (mutateMemory.js) — the single guarded write entry.
 
 import { buildRuntimePaths } from '../runtimePaths.js';
 import { loadRuntimeConfig } from '../runtimeConfig.js';
 import { createRecord, normalizeRecord } from './records.js';
 import {
   readRecordStore,
-  writeRecordStore,
-  applyRecordPatch,
   isRawRecord,
-  mutateRecordStore,
 } from './recordStore.js';
+import { mutateMemory } from './mutateMemory.js';
 
 // Per-process memoization for ensureMigrated — one in-flight migration per
 // projectRoot. The PROMISE is cached (not a done-flag): concurrent callers
@@ -31,9 +29,13 @@ const migrationPromises = new Map();
 async function ensureMigrated(projectRoot) {
   const key = String(projectRoot);
   const existing = migrationPromises.get(key);
-  if (existing) return existing;
+  if (existing) {
+    await existing.promise;
+    return { inFlight: true, failed: existing.failed };
+  }
 
-  const promise = (async () => {
+  const entry = { promise: null, failed: false };
+  entry.promise = (async () => {
     try {
       const config = await loadRuntimeConfig(projectRoot);
       if (config?.memoryV2?.autoMigrate === false) return;
@@ -47,22 +49,48 @@ async function ensureMigrated(projectRoot) {
       }
     } catch {
       // Lazy migration is best-effort: a failed auto-run must not break reads.
+      entry.failed = true;
     }
   })();
-  migrationPromises.set(key, promise);
+  migrationPromises.set(key, entry);
   try {
-    await promise;
+    await entry.promise;
   } finally {
     migrationPromises.delete(key);
   }
+  return { inFlight: false, failed: entry.failed };
+}
+
+/**
+ * loadRecordsDetailed(projectRoot) → { state, records }
+ * Typed loader states (M01-03): 'disabled' (memoryV2.enabled===false),
+ * 'migrating' (auto-migration in-flight or failed), otherwise the
+ * readRecordStore state ('ok'|'empty'|'missing'|'corrupt').
+ */
+export async function loadRecordsDetailed(projectRoot) {
+  try {
+    const config = await loadRuntimeConfig(projectRoot);
+    if (config?.memoryV2?.enabled === false) {
+      return { state: 'disabled', records: [] };
+    }
+  } catch {
+    // unreadable config → default enabled, keep going
+  }
+  const migration = await ensureMigrated(projectRoot);
+  if (migration.failed) {
+    return { state: 'migrating', records: [] };
+  }
+  // inFlight callers awaited the same migration promise — it has completed,
+  // so fall through and read the store like a normal caller.
+  const { records, state, generation } = await readStore(projectRoot);
+  return { state, records, generation };
 }
 
 /**
  * loadRecords(projectRoot) → record[] — tolerant: invalid entries skipped.
  */
 export async function loadRecords(projectRoot) {
-  await ensureMigrated(projectRoot);
-  const { records } = await readStore(projectRoot);
+  const { records } = await loadRecordsDetailed(projectRoot);
   return records;
 }
 
@@ -72,11 +100,18 @@ async function readStore(projectRoot) {
 }
 
 /**
- * saveRecords(projectRoot, records) → void — atomic write; drops unrecoverable entries.
+ * saveRecords(projectRoot, records) → void — explicit restore path
+ * (mutateMemory op:'restore'); drops unrecoverable entries.
  */
 export async function saveRecords(projectRoot, records) {
-  const paths = buildRuntimePaths(projectRoot);
-  await writeRecordStore(paths.memoryV2RecordsPath, records);
+  await ensureMigrated(projectRoot);
+  const res = await mutateMemory(
+    { kind: 'project', projectRoot },
+    { op: 'restore', payload: { records } },
+  );
+  if (res.status === 'rejected' || res.status === 'conflict' || res.status === 'corrupt') {
+    throw new Error(`saveRecords: restore ${res.status} — ${res.reason ?? 'no detail'}`);
+  }
 }
 
 /**
@@ -93,12 +128,13 @@ export async function addRecord(projectRoot, recordInput) {
     throw new Error('addRecord: input could not be normalized into a valid record');
   }
   await ensureMigrated(projectRoot);
-  const paths = buildRuntimePaths(projectRoot);
-  const stored = await mutateRecordStore(paths.memoryV2RecordsPath, (records) => ({
-    result: record,
-    records: [...records, record],
-  }));
-  return stored ?? record;
+  const res = await mutateMemory(
+    { kind: 'project', projectRoot },
+    { op: 'add', payload: record },
+  );
+  if (res.status === 'ok' || res.status === 'duplicate') return res.record ?? record;
+  if (res.status === 'disabled') return record;
+  throw new Error(`addRecord: ${res.status} — ${res.reason ?? 'no detail'}`);
 }
 
 /**
@@ -106,12 +142,13 @@ export async function addRecord(projectRoot, recordInput) {
  */
 export async function updateRecord(projectRoot, id, patch = {}) {
   await ensureMigrated(projectRoot);
-  const paths = buildRuntimePaths(projectRoot);
-  return mutateRecordStore(paths.memoryV2RecordsPath, (records) => {
-    const result = applyRecordPatch(records, id, patch);
-    if (!result) return null;
-    return { result: result.record, records: result.records };
-  });
+  const res = await mutateMemory(
+    { kind: 'project', projectRoot },
+    { op: 'update', payload: { id, patch } },
+  );
+  if (res.status === 'ok') return res.record;
+  if (res.status === 'not-found' || res.status === 'disabled') return null;
+  throw new Error(`updateRecord: ${res.status} — ${res.reason ?? 'no detail'}`);
 }
 
 /**

package/src/core/memory/storeV2Loader.js

@@ -1,22 +1,40 @@
 // Lazy/defensive bridge to storeV2.js (owned by TASK-002).
-// Real storeV2.queryRecords performs ensureMigrated auto-migration internally —
-// consumers must never call migration themselves; just call this and treat an
-// empty result as "fall back to the legacy memory path".
+// Real storeV2 performs ensureMigrated auto-migration internally —
+// consumers must never call migration themselves; just call this and treat a
+// non-'ok' state as "fall back to the legacy memory path".
+import { resolveMemoryStage } from './memoryFlags.js';
 
 /**
- * loadV2Records(projectRoot) → record[] | []
- * Returns [] when storeV2 is unavailable, disabled upstream, or the v2 store
- * holds no records yet. Callers decide fallback.
+ * loadV2Records(projectRoot, filter, { config, projectId } = {}) → { state, records }
+ * Typed loader states (M01-03): 'ok'|'empty'|'missing'|'corrupt'|'disabled'|
+ * 'migrating'. Dynamic-import failure or a missing storeV2 API → 'missing'.
+ * Callers decide fallback on non-'ok' states.
+ *
+ * M06 rollout flag (SPEC §5 FR-003): when a config is supplied, the
+ * eligibility stage gates the v2 lane — 'off' (incl. killSwitch or an
+ * unlisted canary project) reports 'missing' so callers take the legacy
+ * lane exactly as if the v2 store were absent.
  */
-export async function loadV2Records(projectRoot, filter = {}) {
+export async function loadV2Records(projectRoot, filter = {}, { config, projectId } = {}) {
+  if (config && resolveMemoryStage(config, 'eligibility', { projectId }) === 'off') {
+    return { state: 'missing', records: [] };
+  }
   try {
     const storeV2 = await import('./storeV2.js');
-    if (typeof storeV2?.queryRecords !== 'function') {
-      return [];
+    if (typeof storeV2?.queryRecords !== 'function'
+      || typeof storeV2?.loadRecordsDetailed !== 'function') {
+      return { state: 'missing', records: [] };
     }
-    const records = await storeV2.queryRecords(projectRoot, filter);
-    return Array.isArray(records) ? records : [];
+    const { state, records, generation } = await storeV2.loadRecordsDetailed(projectRoot);
+    const { type, scope, status, projectId } = filter ?? {};
+    const filtered = (Array.isArray(records) ? records : []).filter((r) => (
+      (type == null || r.type === type)
+      && (scope == null || r.scope === scope)
+      && (status == null || r.status === status)
+      && (projectId == null || r.project_id === projectId)
+    ));
+    return { state, records: filtered, generation: generation ?? 0 };
   } catch {
-    return [];
+    return { state: 'missing', records: [] };
   }
 }

package/src/core/memory/userMemory.js

@@ -8,11 +8,9 @@ import { buildUserPaths } from '../userPaths.js';
 import { createRecord, normalizeRecord } from './records.js';
 import {
   readRecordStore,
-  writeRecordStore,
-  applyRecordPatch,
   isRawRecord,
-  mutateRecordStore,
 } from './recordStore.js';
+import { mutateMemory } from './mutateMemory.js';
 import { loadRecords } from './storeV2.js';
 
 function userRecordsPath(homeDir) {
@@ -27,11 +25,27 @@ export async function loadUserRecords({ homeDir } = {}) {
   return records;
 }
 
+/**
+ * loadUserRecordsDetailed({homeDir}={}) → { state, records }
+ * Typed loader states (M01-03) via readRecordStore: 'ok'|'empty'|'missing'|
+ * 'corrupt'. No 'disabled'/'migrating' — the user layer has no v1 migration.
+ */
+export async function loadUserRecordsDetailed({ homeDir } = {}) {
+  const { records, state } = await readRecordStore(userRecordsPath(homeDir));
+  return { state, records };
+}
+
 /**
  * saveUserRecords(records, {homeDir}={}) → void — atomic write.
  */
 export async function saveUserRecords(records, { homeDir } = {}) {
-  await writeRecordStore(userRecordsPath(homeDir), records);
+  const res = await mutateMemory(
+    { kind: 'user', homeDir },
+    { op: 'restore', payload: { records } },
+  );
+  if (res.status === 'rejected' || res.status === 'conflict' || res.status === 'corrupt') {
+    throw new Error(`saveUserRecords: restore ${res.status} — ${res.reason ?? 'no detail'}`);
+  }
 }
 
 /**
@@ -45,22 +59,25 @@ export async function addUserRecord(recordInput, { homeDir } = {}) {
   if (!record) {
     throw new Error('addUserRecord: input could not be normalized into a valid record');
   }
-  const stored = await mutateRecordStore(userRecordsPath(homeDir), (records) => ({
-    result: record,
-    records: [...records, record],
-  }));
-  return stored ?? record;
+  const res = await mutateMemory(
+    { kind: 'user', homeDir },
+    { op: 'add', payload: record },
+  );
+  if (res.status === 'ok' || res.status === 'duplicate') return res.record ?? record;
+  throw new Error(`addUserRecord: ${res.status} — ${res.reason ?? 'no detail'}`);
 }
 
 /**
  * updateUserRecord(id, patch, {homeDir}={}) → record | null
  */
 export async function updateUserRecord(id, patch = {}, { homeDir } = {}) {
-  return mutateRecordStore(userRecordsPath(homeDir), (records) => {
-    const result = applyRecordPatch(records, id, patch);
-    if (!result) return null;
-    return { result: result.record, records: result.records };
-  });
+  const res = await mutateMemory(
+    { kind: 'user', homeDir },
+    { op: 'update', payload: { id, patch } },
+  );
+  if (res.status === 'ok') return res.record;
+  if (res.status === 'not-found') return null;
+  throw new Error(`updateUserRecord: ${res.status} — ${res.reason ?? 'no detail'}`);
 }
 
 /**
@@ -70,12 +87,13 @@ export async function updateUserRecord(id, patch = {}, { homeDir } = {}) {
  * or null when the id is absent.
  */
 export async function removeUserRecord(id, { homeDir } = {}) {
-  return mutateRecordStore(userRecordsPath(homeDir), (records) => {
-    const index = records.findIndex((r) => r.id === id);
-    if (index === -1) return null;
-    const removed = records[index];
-    return { result: removed, records: records.filter((r) => r.id !== id) };
-  });
+  const res = await mutateMemory(
+    { kind: 'user', homeDir },
+    { op: 'purge', payload: { id }, confirmScope: 'user' },
+  );
+  if (res.status === 'ok') return res.record;
+  if (res.status === 'not-found') return null;
+  throw new Error(`removeUserRecord: ${res.status} — ${res.reason ?? 'no detail'}`);
 }
 
 /**