@yemi33/minions 0.1.2378 → 0.1.2380

package/engine/cleanup.js

@@ -7,16 +7,16 @@ const fs = require('fs');
 const path = require('path');
 const shared = require('./shared');
 const queries = require('./queries');
+const smallStateStore = require('./small-state-store');
 
 const { exec, execAsync, execSilent, log, ts, ENGINE_DEFAULTS } = shared;
-const { safeJson, safeJsonArr, safeJsonNoRestore, safeWrite, safeReadDir, mutateCooldowns, mutateWorkItems, mutateJsonFileLocked, getProjects, projectWorkItemsPath, projectPrPath,
+const { safeJson, safeJsonArr, safeJsonNoRestore, safeWrite, safeReadDir, readWorkItems, readPullRequests, mutateCooldowns, mutateMetrics, mutateWorkItems, getProjects,
   sanitizeBranch, KB_CATEGORIES } = shared;
 const { getDispatch, getAgentStatus } = queries;
 
 const MINIONS_DIR = shared.MINIONS_DIR;
 const AGENTS_DIR = queries.AGENTS_DIR;
 const ENGINE_DIR = queries.ENGINE_DIR;
-const PRD_DIR = queries.PRD_DIR;
 const PLANS_DIR = queries.PLANS_DIR;
 
 // Lazy require to break circular dependency with engine.js
@@ -98,7 +98,7 @@ function localBranchWorktreeInUse(root, branch) {
 function collectPhantomBranchesForProject(project) {
   const branches = new Set();
   try {
-    const items = safeJsonArr(projectWorkItemsPath(project));
+    const items = readWorkItems(project);
     if (!Array.isArray(items)) return branches;
     for (const w of items) {
       if (w && w._phantomCompletion === true && w._phantomBranch) {
@@ -187,7 +187,7 @@ function cleanupMergedPrLocalBranch(root, project, pr) {
  * env var is unset, or that fail before cleanup, leave `.json` + `.json.backup`
  * sidecars behind). The engine's tick loop then re-discovers these fixtures
  * every tick, fails playbook validation ("missing required template variables:
- * agenda"), and spams log.json with the same error.
+ * agenda"), and spams the engine log with the same error.
  *
  * Filter is conservative: only files whose ID begins with `TEST-` AND whose
  * JSON either lacks an `agenda` or has an empty/whitespace-only one. Both the
@@ -206,41 +206,14 @@ function cleanupMergedPrLocalBranch(root, project, pr) {
 // `.json` — and only when SQLite is healthy so backups remain the legacy
 // fallback for users on Node < 22.5.
 //
-// SQL_BACKED maps every engine-dir state file with a known SQL backing
-// table to that table; safe-by-age files have no SQL backing but their
-// .backup sidecars are equally useless once SQLite is the authority.
-
-const ENGINE_SQL_BACKED_BACKUPS = [
-  { file: 'dispatch.json', table: 'dispatches' },
-  { file: 'metrics.json', table: 'metrics' },
-  { file: 'watches.json', table: 'watches' },
-  { file: 'schedule-runs.json', table: 'schedule_runs' },
-  { file: 'pipeline-runs.json', table: 'pipeline_runs' },
-  { file: 'managed-processes.json', table: 'managed_processes' },
-  { file: 'log.json', table: 'logs' },
-];
-
-const ENGINE_SAFE_BY_AGE_BACKUPS = [
-  'cc-sessions.json',
+const ENGINE_FILE_BACKED_BACKUPS = [
   'control.json',
-  'cooldowns.json',
   'dashboard-browser.json',
-  'doc-sessions.json',
   'kb-pins.json',
-  'pending-rebases.json',
-  'pr-links.json',
-  'qa-sessions.json',
 ];
 
 const MEETINGS_BACKUP_FRESH_WINDOW_MS = 24 * 60 * 60 * 1000;
 
-function _tableHasRow(db, table) {
-  try {
-    const row = db.prepare(`SELECT 1 FROM ${table} LIMIT 1`).get();
-    return !!row;
-  } catch { return false; }
-}
-
 function _tryPruneBackup(liveFile, pruned) {
   const backupPath = liveFile + '.backup';
   if (!fs.existsSync(backupPath)) return;
@@ -253,32 +226,7 @@ function _tryPruneBackup(liveFile, pruned) {
 
 function pruneStaleBackupSidecars(config) {
   const pruned = { count: 0 };
-  let db;
-  try { db = require('./db').getDb(); }
-  catch { return pruned; } // SQLite unavailable — keep backups as the legacy fallback
-
-  // (1a) SQL-backed engine state files — require the table to be populated.
-  for (const { file, table } of ENGINE_SQL_BACKED_BACKUPS) {
-    if (!_tableHasRow(db, table)) continue;
-    _tryPruneBackup(path.join(ENGINE_DIR, file), pruned);
-  }
-
-  // (1b) Per-project work-items.json + pull-requests.json — both share
-  // global SQL tables, so populate-check those tables once.
-  const wiHasRow = _tableHasRow(db, 'work_items');
-  const prHasRow = _tableHasRow(db, 'pull_requests');
-  if (wiHasRow || prHasRow) {
-    for (const project of getProjects(config)) {
-      try {
-        if (wiHasRow) _tryPruneBackup(projectWorkItemsPath(project), pruned);
-        if (prHasRow) _tryPruneBackup(projectPrPath(project), pruned);
-      } catch { /* malformed project entry — skip */ }
-    }
-  }
-
-  // (1c) Safe-by-age set — no SQL table backing, but the .backup sidecars
-  // are equally useless once SQLite is healthy.
-  for (const file of ENGINE_SAFE_BY_AGE_BACKUPS) {
+  for (const file of ENGINE_FILE_BACKED_BACKUPS) {
     _tryPruneBackup(path.join(ENGINE_DIR, file), pruned);
   }
 
@@ -528,7 +476,7 @@ async function runCleanup(config, verbose = false) {
         if (e?.tmpDir) activeDispatchTmpDirs.add(path.resolve(e.tmpDir));
       }
     }
-  } catch { /* dispatch.json may be empty */ }
+  } catch { /* dispatch state may be empty */ }
   for (const dir of scanDirs) {
     // Each directory gets its own try-catch so one failure doesn't abort other directories (Bug #27)
     let dirEntries;
@@ -600,11 +548,11 @@ async function runCleanup(config, verbose = false) {
   }
 
   // 2c. Reap orphan agents/temp-* dirs whose dispatch is no longer referenced
-  // anywhere in dispatch.json. Temp-agent dirs are created by the engine for
+  // anywhere in dispatch state. Temp-agent dirs are created by the engine for
   // ephemeral temp-<uid> agents; once the dispatch ages out of dispatch
   // history they're never touched again, accumulating MB of live-output.log
   // tails over weeks. 1h mtime gate prevents reaping a still-spawning temp
-  // agent that races dispatch.json visibility.
+  // agent that races dispatch-state visibility.
   cleaned.orphanTempAgentDirs = 0;
   try {
     const dispatch = getDispatch();
@@ -766,6 +714,12 @@ async function runCleanup(config, verbose = false) {
   // 3. Clean git worktrees for merged/abandoned PRs
   const _attemptedWorktreePaths = new Set(); // dedup across projects sharing a worktreeRoot
   const _reapedScratchRoots = new Set();     // reap each worktree root's .agent-temp once per cleanup
+  let activeSharedBranchPlans = [];
+  try {
+    activeSharedBranchPlans = require('./prd-store').listPrdRows()
+      .map(row => row.plan)
+      .filter(plan => plan?.branch_strategy === 'shared-branch' && plan.feature_branch && plan.status !== 'completed');
+  } catch { /* optional */ }
   cleaned.agentScratch = 0;
   for (const project of projects) {
     const root = project.localPath ? path.resolve(project.localPath) : null;
@@ -788,12 +742,7 @@ async function runCleanup(config, verbose = false) {
       try { cleaned.agentScratch += reapAgentScratch(worktreeRoot); } catch (_e) { /* best-effort */ }
     }
 
-    // Get PRs for this project
-    // Array.isArray guard: safeJson(p) || [] only substitutes [] for falsy values;
-    // a truthy non-array (e.g. {}, 42, true) bypasses the fallback and causes a
-    // TypeError in the for-of loop below. Array.isArray covers that case.
-    const _prsRaw = safeJson(projectPrPath(project));
-    const prs = Array.isArray(_prsRaw) ? _prsRaw : [];
+    const prs = readPullRequests(project);
     const mergedBranches = new Set();
     for (const pr of prs) {
       if (pr.status === shared.PR_STATUS.MERGED || pr.status === shared.PR_STATUS.ABANDONED) {
@@ -856,7 +805,7 @@ async function runCleanup(config, verbose = false) {
       // `cwd` inside the dispatch's worktree; the agent is gone so no
       // active dispatch references the branch and the >2h age sweep would
       // reap the worktree from under the live services. Build a list of
-      // normalized cwds from engine/managed-processes.json and protect any
+      // normalized cwd values from managed-process state and protect any
       // worktree dir that contains one. Cwd is optional per the schema, so
       // entries without cwd contribute nothing.
       const _managedSpawnCwds = [];
@@ -970,27 +919,18 @@ async function runCleanup(config, verbose = false) {
           } catch { /* optional */ }
         }
 
-        // Skip worktrees for active shared-branch plans (check both prd/ and plans/ for .json PRDs)
+        // Skip worktrees for active shared-branch plans.
         if (shouldClean || !isProtected) {
-          try {
-            for (const checkDir of [PRD_DIR, path.join(MINIONS_DIR, 'plans')]) {
-              if (!fs.existsSync(checkDir)) continue;
-              for (const pf of fs.readdirSync(checkDir).filter(f => f.endsWith('.json'))) {
-                const plan = safeJson(path.join(checkDir, pf));
-                if (plan?.branch_strategy === 'shared-branch' && plan?.feature_branch && plan?.status !== 'completed') {
-                  if (worktreeMatchesBranch(dirLower, plan.feature_branch, actualBranch)) {
-                    isProtected = true;
-                    if (shouldClean) {
-                      shouldClean = false;
-                      if (verbose) console.log(`  Skipping worktree ${dir}: active shared-branch plan`);
-                    }
-                    break;
-                  }
-                }
+          for (const plan of activeSharedBranchPlans) {
+            if (worktreeMatchesBranch(dirLower, plan.feature_branch, actualBranch)) {
+              isProtected = true;
+              if (shouldClean) {
+                shouldClean = false;
+                if (verbose) console.log(`  Skipping worktree ${dir}: active shared-branch plan`);
               }
-              if (isProtected) break;
+              break;
             }
-          } catch (e) { log('warn', 'check shared-branch protection: ' + e.message); }
+          }
         }
 
         wtEntries.push({ dir, wtPath, mtime, shouldClean, isProtected, actualBranch, matchedMergedBranch });
@@ -1011,10 +951,7 @@ async function runCleanup(config, verbose = false) {
       // Remove all marked worktrees
       // Re-read PR status immediately before deletion — a PR can be reopened between
       // the initial status check and the actual deletion (Bug #15: TOCTOU race)
-      // Array.isArray guard: same as the prs read above — non-array truthy JSON
-      // bypasses || [] and would throw inside the try-catch, silently skipping deletions.
-      const _freshPrsRaw = safeJson(projectPrPath(project));
-      const freshPrs = Array.isArray(_freshPrsRaw) ? _freshPrsRaw : [];
+      const freshPrs = readPullRequests(project);
       const freshMergedBranches = new Set();
       const freshMergedPrByBranch = new Map();
       for (const pr of freshPrs) {
@@ -1280,9 +1217,8 @@ async function runCleanup(config, verbose = false) {
   // race conditions with concurrent engine/dashboard/lifecycle writers.
   for (const project of projects) {
     try {
-      const wiPath = projectWorkItemsPath(project);
       let reconciled = 0;
-        mutateWorkItems(wiPath, items => {
+        mutateWorkItems(project, items => {
           for (const item of items) {
             if (item.status === shared.WI_STATUS.FAILED && item._pr) {
               item.status = shared.WI_STATUS.DONE;
@@ -1323,10 +1259,10 @@ async function runCleanup(config, verbose = false) {
     }
     return false;
   }
-  function _migrateLegacyItemsAt(wiPath, label) {
+  function _migrateLegacyItemsAt(scope, label) {
     try {
       let migrated = 0;
-      mutateWorkItems(wiPath, items => {
+      mutateWorkItems(scope, items => {
         for (const item of items) if (_migrateLegacyItem(item)) migrated++;
       });
       if (migrated > 0) {
@@ -1335,12 +1271,9 @@ async function runCleanup(config, verbose = false) {
         // gate ("legacy-done-aliases" removal after 30 consecutive days zero
         // migrations) measurable by bumping a rolling-daily counter under
         // `_engine.legacyStatusMigrations[YYYY-MM-DD]`.
-        // Inline mutateJsonFileLocked by design — promote to a shared
-        // `mutateMetrics(key, date)` helper if more N-day-silence gates land.
         try {
-          const metricsPath = path.join(ENGINE_DIR, 'metrics.json');
           const dateKey = new Date().toISOString().slice(0, 10);
-          mutateJsonFileLocked(metricsPath, (metrics) => {
+          mutateMetrics((metrics) => {
             metrics = metrics || {};
             if (!metrics._engine) metrics._engine = {};
             if (!metrics._engine.legacyStatusMigrations) metrics._engine.legacyStatusMigrations = {};
@@ -1354,15 +1287,14 @@ async function runCleanup(config, verbose = false) {
       }
     } catch (e) { log('warn', `migrate legacy statuses (${label}): ${e.message}`); }
   }
-  for (const project of projects) _migrateLegacyItemsAt(projectWorkItemsPath(project), `${project.name} work items`);
-  _migrateLegacyItemsAt(path.join(MINIONS_DIR, 'work-items.json'), 'central work items');
+  for (const project of projects) _migrateLegacyItemsAt(project, `${project.name} work items`);
+  _migrateLegacyItemsAt('central', 'central work items');
 
   // 6c. Strip stale retry metadata from completed work items
   cleaned.doneRetryCounts = 0;
   for (const project of projects) {
     try {
-      const wiPath = projectWorkItemsPath(project);
-      mutateWorkItems(wiPath, items => {
+      mutateWorkItems(project, items => {
         for (const item of items) {
           if (item.status === shared.WI_STATUS.DONE && item._retryCount !== undefined) {
             delete item._retryCount;
@@ -1373,8 +1305,7 @@ async function runCleanup(config, verbose = false) {
     } catch (e) { log('warn', 'cleanup done retry metadata: ' + e.message); }
   }
   try {
-    const centralPath = path.join(MINIONS_DIR, 'work-items.json');
-    mutateWorkItems(centralPath, items => {
+    mutateWorkItems('central', items => {
       for (const item of items) {
         if (item.status === shared.WI_STATUS.DONE && item._retryCount !== undefined) {
           delete item._retryCount;
@@ -1396,15 +1327,14 @@ async function runCleanup(config, verbose = false) {
     const prdStore = require('./prd-store');
     const prdFiles = prdStore.listPrdRows().filter(row => !row.archived).map(row => row.filename);
     for (const pf of prdFiles) {
-      const prdPath = path.join(PRD_DIR, pf);
       // Cheap out-of-transaction peek to skip PRDs that clearly need no work;
       // the authoritative decision (and the write) happens inside mutatePrd.
-      const peek = prdStore.readPrd(prdPath);
+      const peek = prdStore.readPrd(pf);
       const needsMigration = peek?.missing_features?.some(feat =>
         LEGACY_DONE_ALIASES.has(feat.status) || feat.status === LEGACY_NEEDS_REVIEW_STATUS);
       if (!needsMigration) continue;
       let migrated = 0;
-      prdStore.mutatePrd(prdPath, (prd) => {
+      prdStore.mutatePrd(pf, (prd) => {
         if (prd?.missing_features) {
           for (const feat of prd.missing_features) {
             if (LEGACY_DONE_ALIASES.has(feat.status)) {
@@ -1436,11 +1366,10 @@ async function runCleanup(config, verbose = false) {
     const prdStore = require('./prd-store');
     const orphanPrdEntries = prdStore.listPrdRows().filter(row => !row.archived).map(row => row.filename);
     for (const pf of orphanPrdEntries) {
-      const prdPath = path.join(PRD_DIR, pf);
-      const peek = prdStore.readPrd(prdPath);
+      const peek = prdStore.readPrd(pf);
       if (!peek?.missing_features) continue;
       let reset = 0;
-      prdStore.mutatePrd(prdPath, (prd) => {
+      prdStore.mutatePrd(pf, (prd) => {
         if (prd?.missing_features) {
           for (const feat of prd.missing_features) {
             if ((feat.status === shared.WI_STATUS.DISPATCHED || feat.status === shared.WI_STATUS.FAILED) && !wiIds.has(feat.id)) {
@@ -1458,39 +1387,28 @@ async function runCleanup(config, verbose = false) {
     }
   } catch (e) { log('warn', 'orphan PRD status reset: ' + e.message); }
 
-  // 10. CC tab sessions are non-expiring by design — they persist until the
-  // user explicitly closes the tab (which fires DELETE /api/cc-sessions/:id).
-  // Cleanup intentionally does NOT prune cc-sessions.json; doing so would
-  // silently invalidate live chat tabs the user expects to keep.
+  // 10. CC tab sessions persist until the user explicitly closes a tab.
   cleaned.ccSessions = 0;
 
   // 10b. Prune doc-chat sessions — cap at 100 entries, remove oldest beyond cap.
-  // P-c2sess-1d8e: read+sort+write must run atomically under the file lock so a
-  // concurrent dashboard persistDocSessions can't race the cap-trim.
   cleaned.docSessions = 0;
   try {
-    const docSessionsPath = path.join(ENGINE_DIR, 'doc-sessions.json');
     const DOC_SESSIONS_CAP = 100;
-    mutateJsonFileLocked(docSessionsPath, (docSessions) => {
+    smallStateStore.applyDocSessionsMutation((docSessions) => {
       if (!docSessions || typeof docSessions !== 'object' || Array.isArray(docSessions)) return docSessions;
       const entries = Object.entries(docSessions);
       if (entries.length <= DOC_SESSIONS_CAP) return docSessions;
       entries.sort((a, b) => new Date(b.lastActiveAt || 0) - new Date(a.lastActiveAt || 0));
       cleaned.docSessions = entries.length - DOC_SESSIONS_CAP;
       return Object.fromEntries(entries.slice(0, DOC_SESSIONS_CAP));
-    }, { defaultValue: {}, skipWriteIfUnchanged: true });
-  } catch (e) { log('warn', 'prune doc-sessions: ' + e.message); }
+    });
+  } catch (e) { log('warn', 'prune doc sessions: ' + e.message); }
 
-  // 11. Cap cooldowns.json — keep at most 500 entries (on top of 24h TTL in cooldown.js)
-  //     Also trim pendingContexts arrays to ENGINE_DEFAULTS.maxPendingContexts to prevent bloat.
+  // 11. Cap cooldown state and pending context arrays.
   cleaned.cooldowns = 0;
   cleaned.pendingContextsTrimmed = 0;
   try {
-    const cooldownPath = path.join(ENGINE_DIR, 'cooldowns.json');
-    // safeJsonNoRestore — same rationale as engine/cooldown.js loadCooldowns:
-    // resurrecting a stale .backup at cleanup time could re-introduce expired
-    // entries the active code already pruned (P-bfa1d-safejson-no-restore).
-    const cooldowns = safeJsonNoRestore(cooldownPath);
+    const cooldowns = smallStateStore.readCooldowns();
     if (cooldowns && typeof cooldowns === 'object') {
       let dirty = false;
       // Trim oversized pendingContexts arrays (one-time migration + ongoing cap)
@@ -1562,7 +1480,7 @@ async function runCleanup(config, verbose = false) {
     }
   } catch { /* optional — file may not exist */ }
 
-  // 14. Scrub stale temp agent keys from metrics.json
+  // 14. Scrub stale temp agent keys from metrics state
   try { scrubStaleMetrics(); } catch { /* best-effort cleanup */ }
 
   // 15. Evict old completion reports — keep reports durable beyond the capped
@@ -1660,12 +1578,11 @@ async function runCleanup(config, verbose = false) {
 
 // ─── Metrics Scrub ──────────────────────────────────────────────────────────
 
-/** Remove stale temp-*, agent1, and _test-* keys from metrics.json.
+/** Remove stale temp-*, agent1, and _test-* keys from metrics state.
  *  Mirrors the guard in lifecycle.js updateMetrics() — cleans up keys
  *  that were written before that guard existed. */
 function scrubStaleMetrics() {
-  const metricsPath = path.join(ENGINE_DIR, 'metrics.json');
-  mutateJsonFileLocked(metricsPath, metrics => {
+  mutateMetrics(metrics => {
     for (const key of Object.keys(metrics)) {
       if (key.startsWith('temp-') || key === 'agent1' || key.startsWith('_test')) {
         delete metrics[key];