@yemi33/minions 0.1.2115 → 0.1.2117

package/engine/shared.js

@@ -65,6 +65,16 @@ const COOLDOWNS_PATH = path.join(ENGINE_DIR, 'cooldowns.json');
 // process-lifetime (state/pid/ownerToken). See ENGINE_DEFAULTS.abandonedReconciliationVersion
 // for the first consumer.
 const ENGINE_STATE_PATH = path.join(ENGINE_DIR, 'state.json');
+// Phase 9.3: wrap raw safeWrite callers for kb-checkpoint, kb-swept,
+// kb-sweep-state, and test-results so they go through mutateJsonFileLocked
+// (the same path mutateControl / mutateEngineState / mutateCooldowns use).
+// This sets them up for SQL routing via _SMALL_STATE_MUTATE_ROUTES later, and
+// in the case of kb-sweep-state.json closes a real cross-process race (the
+// dashboard process and the detached kb-sweep-runner child both write it).
+const KB_CHECKPOINT_PATH = path.join(ENGINE_DIR, 'kb-checkpoint.json');
+const KB_SWEPT_PATH = path.join(ENGINE_DIR, 'kb-swept.json');
+const KB_SWEEP_STATE_PATH = path.join(ENGINE_DIR, 'kb-sweep-state.json');
+const TEST_RESULTS_PATH = path.join(ENGINE_DIR, 'test-results.json');
 const PR_LINKS_PATH = path.join(MINIONS_DIR, 'engine', 'pr-links.json');
 const PINNED_ITEMS_PATH = path.join(MINIONS_DIR, 'engine', 'kb-pins.json');
 const LOG_PATH = path.join(MINIONS_DIR, 'engine', 'log.json');
@@ -536,6 +546,22 @@ function _routeJsonReadToSql(p) {
       const store = require('./small-state-store');
       return { value: store.readPrLinks() };
     }
+    if (norm.endsWith('/engine/cooldowns.json')) {
+      const store = require('./small-state-store');
+      return { value: store.readCooldowns() };
+    }
+    if (norm.endsWith('/engine/pending-rebases.json')) {
+      const store = require('./small-state-store');
+      return { value: store.readPendingRebases() };
+    }
+    if (norm.endsWith('/engine/cc-sessions.json')) {
+      const store = require('./small-state-store');
+      return { value: store.readCcSessions() };
+    }
+    if (norm.endsWith('/engine/doc-sessions.json')) {
+      const store = require('./small-state-store');
+      return { value: store.readDocSessions() };
+    }
     // Per-project work-items.json — match `/projects/<name>/work-items.json`.
     // When SQL has no rows for the scope AND the JSON file is absent on
     // disk, preserve the legacy "file missing → null" semantic. This guards
@@ -565,9 +591,11 @@ function _routeJsonReadToSql(p) {
       const store = require('./pull-requests-store');
       return { value: store.readPullRequestsForScope('central') };
     }
-  } catch {
-    // SQLite unavailable / store load failure — fall through to disk read.
-    return null;
+  } catch (e) {
+    // Phase 9.4: store/load failures (not SQLite-unavailable — the CLI shim
+    // in bin/minions.js guarantees node:sqlite is loadable) propagate up so
+    // the caller can decide whether to retry or surface.
+    throw e;
   }
   return null;
 }
@@ -1323,13 +1351,64 @@ function withFileLock(lockPath, fn, {
   throw lastErr;
 }
 
+// Route table for small-state files migrated to SQL stores. Each entry
+// declares the SQL store function to invoke and the expected default JSON
+// shape (for the "ensure file exists" fallback below).
+const _SMALL_STATE_MUTATE_ROUTES = {
+  'cooldowns.json':       { fn: 'applyCooldownsMutation',       mirror: '_mirrorCooldownsJson',       defaultShape: 'object' },
+  'pending-rebases.json': { fn: 'applyPendingRebasesMutation',  mirror: '_mirrorPendingRebasesJson',  defaultShape: 'array'  },
+  'cc-sessions.json':     { fn: 'applyCcSessionsMutation',      mirror: '_mirrorCcSessionsJson',      defaultShape: 'array'  },
+  'doc-sessions.json':    { fn: 'applyDocSessionsMutation',     mirror: '_mirrorDocSessionsJson',     defaultShape: 'object' },
+  'pr-links.json':        { fn: 'applyPrLinksMutation',         mirror: '_mirrorPrLinksJson',         defaultShape: 'object' },
+};
+
 function _tryRouteMutateToSql(filePath, mutateFn, onWrote) {
   const baseName = path.basename(filePath);
-  if (baseName !== 'work-items.json' && baseName !== 'pull-requests.json') return null;
+  const smallRoute = _SMALL_STATE_MUTATE_ROUTES[baseName];
+  if (baseName !== 'work-items.json' && baseName !== 'pull-requests.json' && !smallRoute) return null;
   const fpNorm = String(filePath).replace(/\\/g, '/');
   const minionsNorm = String(MINIONS_DIR).replace(/\\/g, '/');
   const insideMinionsDir = fpNorm.startsWith(minionsNorm + '/') || fpNorm === minionsNorm + '/' + baseName;
   if (!insideMinionsDir) return null;
+
+  // Small-state files live exclusively under <MINIONS_DIR>/engine/<baseName>.
+  // Don't hijack writes to ad-hoc paths that happen to share the basename.
+  if (smallRoute) {
+    if (!fpNorm.endsWith('/engine/' + baseName)) return null;
+    const store = require('./small-state-store');
+    // Hold the JSON file lock across the SQL transaction AND the mirror
+    // write. The SQL transaction itself is cross-process serialized via
+    // BEGIN IMMEDIATE inside the SQL store, but the JSON mirror is a
+    // separate read-from-SQL → atomic-rename that can race across
+    // processes: an earlier mirror's stale snapshot can land AFTER a
+    // later mirror's complete snapshot and lose committed rows from the
+    // on-disk JSON file. Pre-Phase-9.4 this was masked by the JSON
+    // fallback (concurrent writers serialized through the same lock at
+    // the bottom of mutateJsonFileLocked); after the fallback removal,
+    // the bare mirror races. Locking around the SQL+mirror block puts
+    // those two operations back into one cross-process critical section.
+    const lockPath = `${filePath}.lock`;
+    return withFileLock(lockPath, () => {
+      const out = store[smallRoute.fn]((data) => {
+        const next = mutateFn(data);
+        if (onWrote) onWrote();
+        return next;
+      });
+      const result = out && Object.prototype.hasOwnProperty.call(out, 'result') ? out.result : undefined;
+      try { store[smallRoute.mirror](filePath); } catch { /* mirror best-effort */ }
+      if (!fs.existsSync(filePath)) {
+        try {
+          fs.mkdirSync(path.dirname(filePath), { recursive: true });
+          const fallback = smallRoute.defaultShape === 'array'
+            ? (Array.isArray(result) ? result : [])
+            : (result && typeof result === 'object' && !Array.isArray(result) ? result : {});
+          safeWrite(filePath, fallback);
+        } catch { /* best-effort */ }
+      }
+      return { routed: result };
+    });
+  }
+
   let result;
   if (baseName === 'work-items.json') {
     result = mutateWorkItems(filePath, (arr) => {
@@ -1451,6 +1530,45 @@ function mutateCooldowns(mutator) {
   }, { defaultValue: {}, skipWriteIfUnchanged: true });
 }
 
+// Phase 9.3 (W-mp48wxqw / SQL-canonicalization prep): lock-safe RMW wrappers
+// for the small JSON state files that were still going through raw safeWrite.
+// Pattern mirrors mutateControl / mutateEngineState / mutateCooldowns — so a
+// future SQL migration is a mechanical _SMALL_STATE_MUTATE_ROUTES addition.
+
+// KB classification checkpoint counter (consolidation.js post-classify).
+function mutateKbCheckpoint(mutator) {
+  return mutateJsonFileLocked(KB_CHECKPOINT_PATH, (data) => {
+    if (!data || typeof data !== 'object' || Array.isArray(data)) data = {};
+    return mutator(data) || data;
+  }, { defaultValue: {}, skipWriteIfUnchanged: true });
+}
+
+// KB last-sweep summary (kb-sweep.js post-sweep, dashboard reader).
+function mutateKbSwept(mutator) {
+  return mutateJsonFileLocked(KB_SWEPT_PATH, (data) => {
+    if (!data || typeof data !== 'object' || Array.isArray(data)) data = {};
+    return mutator(data) || data;
+  }, { defaultValue: {}, skipWriteIfUnchanged: true });
+}
+
+// KB sweep state machine. Written by BOTH the dashboard process AND the
+// detached kb-sweep-runner child — wrapping under mutateJsonFileLocked makes
+// cross-process writes file-locked (previously a real race via raw safeWrite).
+function mutateKbSweepState(mutator) {
+  return mutateJsonFileLocked(KB_SWEEP_STATE_PATH, (data) => {
+    if (!data || typeof data !== 'object' || Array.isArray(data)) data = {};
+    return mutator(data) || data;
+  }, { defaultValue: {}, skipWriteIfUnchanged: true });
+}
+
+// Test results history (capped at TEST_RESULTS_CAP by cleanup.js).
+function mutateTestResults(mutator) {
+  return mutateJsonFileLocked(TEST_RESULTS_PATH, (data) => {
+    if (!Array.isArray(data)) data = [];
+    return mutator(data) || data;
+  }, { defaultValue: [], skipWriteIfUnchanged: true });
+}
+
 let _uidCounter = 0;
 
 /**
@@ -3094,32 +3212,20 @@ const WATCH_ACTION_TYPE = {
 
 /**
  * Phase 7 — small state file mutators. Each routes through the
- * small-state-store, mirrors back to its JSON file, and emits a topic
- * event on real writes. SQLite failure falls through to the legacy
- * mutateJsonFileLocked path.
+ * small-state-store and mirrors back to its JSON file, then emits a topic
+ * event on real writes. SQL-canonical (Phase 9.4): SQLite failures
+ * propagate; the CLI shim in bin/minions.js guarantees `node:sqlite` is
+ * loadable on every supported Node version.
  */
-function _smallStateMutator({ filePath, applyMutation, mirror, topic, defaultValue }) {
+function _smallStateMutator({ filePath, applyMutation, mirror, topic }) {
   return (mutator) => {
-    try {
-      const store = require('./small-state-store');
-      const { wrote, result } = store[applyMutation]((obj) => mutator(obj) || obj);
-      if (wrote) {
-        try { if (mirror && typeof store[mirror] === 'function') store[mirror](filePath); } catch { /* mirror best-effort */ }
-        try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
-      }
-      return result;
-    } catch (e) {
-      if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) throw e;
+    const store = require('./small-state-store');
+    const { wrote, result } = store[applyMutation]((obj) => mutator(obj) || obj);
+    if (wrote) {
+      try { if (mirror && typeof store[mirror] === 'function') store[mirror](filePath); } catch { /* mirror best-effort */ }
+      try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
     }
-    return mutateJsonFileLocked(filePath, (data) => {
-      if (data == null) data = defaultValue();
-      return mutator(data) || data;
-    }, {
-      defaultValue: defaultValue(),
-      onWrote: () => {
-        try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
-      },
-    });
+    return result;
   };
 }
 
@@ -3175,31 +3281,18 @@ function _qaDualWriteEnabled() {
 function _qaMutator({ filePath, applyMutation, mirror, topic }) {
   return (mutator) => {
     return withFileLock(filePath + '.lock', () => {
-      try {
-        const store = require('./small-state-store');
-        const { wrote, result } = store[applyMutation]((arr) => {
-          if (!Array.isArray(arr)) arr = [];
-          return mutator(arr) || arr;
-        });
-        if (wrote) {
-          if (_qaDualWriteEnabled()) {
-            try { if (mirror && typeof store[mirror] === 'function') store[mirror](filePath); } catch { /* mirror best-effort */ }
-          }
-          try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
+      const store = require('./small-state-store');
+      const { wrote, result } = store[applyMutation]((arr) => {
+        if (!Array.isArray(arr)) arr = [];
+        return mutator(arr) || arr;
+      });
+      if (wrote) {
+        if (_qaDualWriteEnabled()) {
+          try { if (mirror && typeof store[mirror] === 'function') store[mirror](filePath); } catch { /* mirror best-effort */ }
         }
-        return result;
-      } catch (e) {
-        if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) throw e;
-        return mutateJsonFileLocked(filePath, (data) => {
-          if (!Array.isArray(data)) data = [];
-          return mutator(data) || data;
-        }, {
-          defaultValue: [],
-          onWrote: () => {
-            try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
-          },
-        });
+        try { require('./db-events').emitStateEvent(topic); } catch { /* optional */ }
       }
+      return result;
     }, { timeoutMs: 5000, retries: 3 });
   };
 }
@@ -3222,36 +3315,20 @@ const mutateQaSessions = _qaMutator({
  * Route a watches mutation through the SQL store. Same shape as
  * mutateWorkItems / mutatePullRequests: mutator receives the watches
  * array, mutates in place or returns a replacement, and the store
- * diffs by id. Falls back to the legacy mutateJsonFileLocked path on
- * SQLite failure.
+ * diffs by id. SQL-canonical (Phase 9.4); SQLite failures propagate.
  */
 function mutateWatches(mutator) {
   const watchesPath = path.join(MINIONS_DIR, 'engine', 'watches.json');
-  try {
-    const store = require('./watches-store');
-    const { wrote, result } = store.applyWatchesMutation((arr) => {
-      if (!Array.isArray(arr)) arr = [];
-      return mutator(arr) || arr;
-    });
-    if (wrote) {
-      try { store._mirrorJsonFromSql(watchesPath); } catch { /* mirror best-effort */ }
-      try { require('./db-events').emitStateEvent('watches'); } catch { /* optional */ }
-    }
-    return result;
-  } catch (e) {
-    if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
-      throw e;
-    }
-  }
-  return mutateJsonFileLocked(watchesPath, (data) => {
-    if (!Array.isArray(data)) data = [];
-    return mutator(data) || data;
-  }, {
-    defaultValue: [],
-    onWrote: () => {
-      try { require('./db-events').emitStateEvent('watches'); } catch { /* optional */ }
-    },
+  const store = require('./watches-store');
+  const { wrote, result } = store.applyWatchesMutation((arr) => {
+    if (!Array.isArray(arr)) arr = [];
+    return mutator(arr) || arr;
   });
+  if (wrote) {
+    try { store._mirrorJsonFromSql(watchesPath); } catch { /* mirror best-effort */ }
+    try { require('./db-events').emitStateEvent('watches'); } catch { /* optional */ }
+  }
+  return result;
 }
 
 /**
@@ -3259,37 +3336,20 @@ function mutateWatches(mutator) {
  * mirror. Same shape as mutateWorkItems / mutatePullRequests: mutator
  * receives the full legacy-shape metrics object, mutates in place or
  * returns a replacement, and the store diffs by (kind, key) row.
- *
- * Falls back to the legacy mutateJsonFileLocked path on SQLite failure
- * so a node:sqlite-broken install keeps recording metrics.
+ * SQL-canonical (Phase 9.4); SQLite failures propagate.
  */
 function mutateMetrics(mutator) {
   const metricsPath = path.join(MINIONS_DIR, 'engine', 'metrics.json');
-  try {
-    const store = require('./metrics-store');
-    const { wrote, result } = store.applyMetricsMutation((m) => {
-      if (!m || typeof m !== 'object') m = {};
-      return mutator(m) || m;
-    });
-    if (wrote) {
-      try { store._mirrorJsonFromSql(metricsPath); } catch { /* mirror best-effort */ }
-      try { require('./db-events').emitStateEvent('metrics'); } catch { /* optional */ }
-    }
-    return result;
-  } catch (e) {
-    if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
-      throw e;
-    }
-  }
-  return mutateJsonFileLocked(metricsPath, (metrics) => {
-    if (!metrics || typeof metrics !== 'object') metrics = {};
-    return mutator(metrics) || metrics;
-  }, {
-    defaultValue: {},
-    onWrote: () => {
-      try { require('./db-events').emitStateEvent('metrics'); } catch { /* optional */ }
-    },
+  const store = require('./metrics-store');
+  const { wrote, result } = store.applyMetricsMutation((m) => {
+    if (!m || typeof m !== 'object') m = {};
+    return mutator(m) || m;
   });
+  if (wrote) {
+    try { store._mirrorJsonFromSql(metricsPath); } catch { /* mirror best-effort */ }
+    try { require('./db-events').emitStateEvent('metrics'); } catch { /* optional */ }
+  }
+  return result;
 }
 
 /** Update per-agent review metrics (prsApproved/prsRejected). Only writes for configured agents. */
@@ -5055,23 +5115,11 @@ function addPrLink(prId, itemId, { project = null, url = '', prNumber = null } =
     links[effectivePrId] = [...mergedCurrent];
     return links;
   };
-  // Phase 9.1: pr-links is SQL-canonical via small-state-store; the JSON file
-  // is a dual-write mirror. SQLite failures fall through to the legacy JSON
-  // path so older installs without --experimental-sqlite still work.
-  let routedViaSql = false;
-  try {
-    const store = require('./small-state-store');
-    store.applyPrLinksMutation(mutator);
-    try { store._mirrorPrLinksJson(); } catch { /* mirror best-effort */ }
-    routedViaSql = true;
-  } catch (e) {
-    if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
-      throw e;
-    }
-  }
-  if (!routedViaSql) {
-    mutateJsonFileLocked(PR_LINKS_PATH, mutator, { defaultValue: {} });
-  }
+  // Phase 9.4: pr-links is SQL-only via small-state-store; the JSON file
+  // is a write-only mirror artifact for legacy direct-disk readers.
+  const store = require('./small-state-store');
+  store.applyPrLinksMutation(mutator);
+  try { store._mirrorPrLinksJson(); } catch { /* mirror best-effort */ }
 
   if (!project) return;
   const prPath = projectPrPath(project);
@@ -5463,42 +5511,27 @@ function listProcessReachable(rootPids, allProcesses = null) {
  * @param {Function} mutator - Receives the array, mutates in place or returns new value
  */
 function mutateWorkItems(filePath, mutator) {
-  // Phase 2 SQL path → Phase 9 SQL-only. Route through work-items-store so
-  // SQL is the canonical (and only) store; the legacy JSON mirror was
-  // retired in Phase 9.
-  //
-  // The SQL store identifies records by `scope` (central or project name)
-  // derived from the file path's last two segments. Ad-hoc file paths
-  // outside the MINIONS_DIR layout (e.g. tests using createTmpDir()) can't
-  // be mapped to a stable scope, so we short-circuit to the legacy
-  // JSON path for those. Production callers always use
-  // shared.projectWorkItemsPath(p) / MINIONS_DIR/work-items.json.
-  //
-  // SQLite failures fall through to the legacy JSON path below — keeps a
-  // node:sqlite-broken install fully functional.
+  // Phase 9.4 SQL-only. Route through work-items-store; SQL is the canonical
+  // (and only) store. The legacy JSON path below remains ONLY for ad-hoc
+  // file paths outside the MINIONS_DIR layout (e.g. tests using
+  // createTmpDir()) that can't be mapped to a stable scope. Production
+  // callers always use shared.projectWorkItemsPath(p) / MINIONS_DIR/work-items.json.
   const fpNorm = String(filePath).replace(/\\/g, '/');
   const minionsNorm = String(MINIONS_DIR).replace(/\\/g, '/');
   const insideMinionsDir = fpNorm.startsWith(minionsNorm + '/') || fpNorm === minionsNorm + '/work-items.json';
   if (insideMinionsDir) {
-    try {
-      const store = require('./work-items-store');
-      const scope = store.scopeForFilePath(filePath);
-      const { wrote, result } = store.applyWorkItemsMutation(scope, (items) => {
-        if (!Array.isArray(items)) items = [];
-        return mutator(items) || items;
-      });
-      if (wrote) {
-        try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror best-effort */ }
-        try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
-      }
-      try { require('./queries').invalidateWorkItemsCache(); } catch { /* queries not loaded */ }
-      return result;
-    } catch (e) {
-      if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
-        throw e;
-      }
-      // Fall through to the legacy JSON path on SQLite errors only.
+    const store = require('./work-items-store');
+    const scope = store.scopeForFilePath(filePath);
+    const { wrote, result } = store.applyWorkItemsMutation(scope, (items) => {
+      if (!Array.isArray(items)) items = [];
+      return mutator(items) || items;
+    });
+    if (wrote) {
+      try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror best-effort */ }
+      try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
     }
+    try { require('./queries').invalidateWorkItemsCache(); } catch { /* queries not loaded */ }
+    return result;
   }
 
   const result = mutateJsonFileLocked(filePath, (data) => {
@@ -5536,31 +5569,24 @@ function reopenWorkItem(wi) {
  * @param {Function} mutator - Receives the array, mutates in place or returns new value
  */
 function mutatePullRequests(filePath, mutator) {
-  // Phase 3 SQL path → Phase 9 SQL-only. Route through pull-requests-store
-  // when filePath sits under MINIONS_DIR. Ad-hoc tmp paths (legacy tests
-  // using createTmpDir) and SQLite failures fall through to the JSON path.
+  // Phase 9.4 SQL-only. Route through pull-requests-store when filePath sits
+  // under MINIONS_DIR. Ad-hoc tmp paths (legacy tests using createTmpDir)
+  // still fall through to the JSON path.
   const fpNorm = String(filePath).replace(/\\/g, '/');
   const minionsNorm = String(MINIONS_DIR).replace(/\\/g, '/');
   const insideMinionsDir = fpNorm.startsWith(minionsNorm + '/') || fpNorm === minionsNorm + '/pull-requests.json';
   if (insideMinionsDir) {
-    try {
-      const store = require('./pull-requests-store');
-      const scope = store.scopeForFilePath(filePath);
-      const { wrote, result } = store.applyPullRequestsMutation(scope, (prs) => {
-        if (!Array.isArray(prs)) prs = [];
-        return mutator(prs) || prs;
-      });
-      if (wrote) {
-        try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror best-effort */ }
-        try { require('./db-events').emitStateEvent('pull_requests'); } catch { /* optional */ }
-      }
-      return result;
-    } catch (e) {
-      if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
-        throw e;
-      }
-      // Fall through to legacy JSON path on SQLite errors only.
+    const store = require('./pull-requests-store');
+    const scope = store.scopeForFilePath(filePath);
+    const { wrote, result } = store.applyPullRequestsMutation(scope, (prs) => {
+      if (!Array.isArray(prs)) prs = [];
+      return mutator(prs) || prs;
+    });
+    if (wrote) {
+      try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror best-effort */ }
+      try { require('./db-events').emitStateEvent('pull_requests'); } catch { /* optional */ }
     }
+    return result;
   }
 
   return mutateJsonFileLocked(filePath, (data) => {
@@ -5975,6 +6001,10 @@ module.exports = {
   mutateEngineState, // W-mp60tw0u000j3931
   readEngineState,   // W-mp60tw0u000j3931
   mutateCooldowns,
+  mutateKbCheckpoint,   // Phase 9.3
+  mutateKbSwept,        // Phase 9.3
+  mutateKbSweepState,   // Phase 9.3
+  mutateTestResults,    // Phase 9.3
   mutateWorkItems,
   reopenWorkItem,
   mutatePullRequests,