@yemi33/minions 0.1.2066 → 0.1.2067

package/engine/dispatch-store.js

@@ -0,0 +1,316 @@
+// engine/dispatch-store.js — SQL-backed implementation of the
+// section-shaped dispatch object. Same external contract as the legacy
+// dispatch.json reader:
+//
+//   readDispatchSectioned()   -> { pending, active, completed, review }
+//   applyDispatchMutation(fn) -> runs fn against the section object,
+//                                computes a diff, applies INSERTs/UPDATEs/
+//                                DELETEs in one transaction, returns the
+//                                mutated object. Returns { wrote, result }
+//                                so callers can suppress side effects
+//                                (event emission, cache invalidation)
+//                                when nothing actually changed.
+//
+// All callers go through engine/dispatch.js#mutateDispatch and
+// engine/queries.js#getDispatch as before — those are thin wrappers
+// over the helpers here. Zero changes at any call site.
+
+const SECTIONS = ['pending', 'active', 'completed', 'review'];
+
+function _emptySectioned() {
+  return { pending: [], active: [], completed: [], review: [] };
+}
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function _parseRow(row) {
+  if (!row || !row.data) return null;
+  try { return JSON.parse(row.data); }
+  catch { return null; }
+}
+
+function readDispatchSectioned() {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return _readDispatchJsonFallback(); }  // SQLite unavailable
+
+  // All rows, all statuses. Matches the legacy dispatch.json reader's
+  // semantics — returns the complete dispatch state regardless of how old
+  // completed entries are. Cross-reference consumers like state-integrity
+  // need the full completed list to match against work-items, and the
+  // dashboard's slim slice handles its own payload sizing downstream.
+  const rows = db.prepare(`
+    SELECT data, status FROM dispatches
+    ORDER BY status, created_at
+  `).all();
+
+  if (rows.length === 0) {
+    // SQL has no live dispatches. Two scenarios:
+    //   (a) Production fresh install (or all pruned) — JSON file is empty/missing,
+    //       fallback returns the same empty sectioned object. No harm.
+    //   (b) Test that wrote dispatch.json directly (legacy test helpers in
+    //       timeout-behavioral / orphan-* / etc. — they bypass the proper
+    //       mutateDispatch API and seed via fs.writeFileSync). Picking up
+    //       the JSON keeps those tests working through the transition without
+    //       touching every helper. Phase 1.5 migrates them to the proper API.
+    const fallback = _readDispatchJsonFallback();
+    const hasContent = (
+      (fallback.pending && fallback.pending.length > 0) ||
+      (fallback.active && fallback.active.length > 0) ||
+      (fallback.completed && fallback.completed.length > 0) ||
+      (fallback.review && fallback.review.length > 0)
+    );
+    if (hasContent) return fallback;
+  }
+
+  const out = _emptySectioned();
+  for (const row of rows) {
+    const rec = _parseRow(row);
+    if (!rec) continue;
+    if (out[row.status]) out[row.status].push(rec);
+  }
+  return out;
+}
+
+function _readDispatchJsonFallback() {
+  const path = require('path');
+  const fs = require('fs');
+  const shared = require('./shared');
+  const dispatchPath = path.join(shared.MINIONS_DIR, 'engine', 'dispatch.json');
+  let raw;
+  try {
+    raw = fs.readFileSync(dispatchPath, 'utf8');
+  } catch {
+    // ENOENT / permission / etc. — file genuinely absent. Silently
+    // return an empty sectioned object; that's the legacy semantics.
+    return _emptySectioned();
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    const out = _emptySectioned();
+    for (const s of SECTIONS) {
+      if (Array.isArray(parsed[s])) out[s] = parsed[s];
+    }
+    return out;
+  } catch (e) {
+    // File exists but parse failed — that's user-visible corruption.
+    // Surface to logs the same way the legacy queries.getDispatch path
+    // did via shared.readJsonNoRestore. (Phase 1 contract test
+    // "getDispatch warns on corrupt dispatch.json".)
+    try {
+      // eslint-disable-next-line no-console
+      console.warn(`[dispatch-store] corrupt JSON in ${dispatchPath}: ${e.message}`);
+    } catch { /* console may be wrapped in tests */ }
+    return _emptySectioned();
+  }
+}
+
+// Internal: read the full (unfiltered) dispatch state from SQL.
+// Used by applyDispatchMutation so the diff sees ALL completed rows,
+// not just the recent window. Mutators that prune old completed
+// dispatches MUST see those rows or the prune is a no-op.
+function _readDispatchSectionedFull() {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return _emptySectioned(); }
+  const rows = db.prepare('SELECT data, status FROM dispatches').all();
+  const out = _emptySectioned();
+  for (const row of rows) {
+    const rec = _parseRow(row);
+    if (!rec) continue;
+    if (out[row.status]) out[row.status].push(rec);
+  }
+  return out;
+}
+
+// Build a flat { id -> { status, record } } map from a sectioned object.
+function _indexBySection(sectioned) {
+  const out = new Map();
+  for (const status of SECTIONS) {
+    const rows = sectioned[status];
+    if (!Array.isArray(rows)) continue;
+    for (const rec of rows) {
+      if (!rec || !rec.id) continue;
+      out.set(String(rec.id), { status, record: rec });
+    }
+  }
+  return out;
+}
+
+// Compute the diff between before and after sectioned states.
+// Returns { toUpsert: [{id, status, record}], toDelete: [id] }.
+// toUpsert covers brand-new rows, status flips, and field changes.
+// We compare JSON-stringified records to detect content edits; any
+// change to ANY field flips the comparison and emits an UPSERT.
+function _computeDispatchDiff(before, after) {
+  const beforeMap = _indexBySection(before);
+  const afterMap = _indexBySection(after);
+  const toUpsert = [];
+  const toDelete = [];
+
+  for (const [id, { status: afterStatus, record: afterRec }] of afterMap) {
+    const beforeEntry = beforeMap.get(id);
+    if (!beforeEntry) {
+      toUpsert.push({ id, status: afterStatus, record: afterRec });
+      continue;
+    }
+    const sameStatus = beforeEntry.status === afterStatus;
+    const sameContent = JSON.stringify(beforeEntry.record) === JSON.stringify(afterRec);
+    if (!sameStatus || !sameContent) {
+      toUpsert.push({ id, status: afterStatus, record: afterRec });
+    }
+  }
+  for (const [id] of beforeMap) {
+    if (!afterMap.has(id)) toDelete.push(id);
+  }
+  return { toUpsert, toDelete };
+}
+
+function _applyDispatchDiff(db, diff) {
+  if (diff.toUpsert.length === 0 && diff.toDelete.length === 0) return false;
+  const now = Date.now();
+  const upsertStmt = db.prepare(`
+    INSERT INTO dispatches (id, status, agent, type, created_at, started_at, completed_at, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      status        = excluded.status,
+      agent         = excluded.agent,
+      type          = excluded.type,
+      created_at    = excluded.created_at,
+      started_at    = excluded.started_at,
+      completed_at  = excluded.completed_at,
+      data          = excluded.data,
+      updated_at    = excluded.updated_at
+  `);
+  const deleteStmt = db.prepare('DELETE FROM dispatches WHERE id = ?');
+
+  for (const { id, status, record } of diff.toUpsert) {
+    upsertStmt.run(
+      String(id),
+      status,
+      record.agent || null,
+      record.type || null,
+      _toMs(record.created_at),
+      _toMs(record.started_at),
+      _toMs(record.completed_at),
+      JSON.stringify(record),
+      now,
+    );
+  }
+  for (const id of diff.toDelete) {
+    deleteStmt.run(String(id));
+  }
+  return true;
+}
+
+// Apply a mutation to the dispatch state. The mutator receives the
+// section-shaped object, mutates it in place (or returns a replacement),
+// and we diff against the pre-mutation snapshot to compute the SQL
+// operations. Wrapped in a single transaction so the diff and the apply
+// can't race against a concurrent reader/writer.
+//
+// Returns { wrote, result }:
+//   wrote  — true iff at least one INSERT/UPDATE/DELETE landed.
+//            Callers (mutateDispatch) gate event emission + cache
+//            invalidation on this flag so no-op mutators don't bump
+//            the cache version (mirrors the onWrote contract from
+//            mutateJsonFileLocked).
+//   result — the mutator's return value (or the original object if the
+//            mutator mutated in place + returned undefined). Same
+//            contract as the legacy mutateJsonFileLocked-based path.
+function applyDispatchMutation(mutator) {
+  const { getDb, withTransaction } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch (e) {
+    throw new Error(`engine/dispatch-store: SQLite unavailable (${e.message}); cannot mutate dispatch state`);
+  }
+
+  // withTransaction is nestable: shared.mutateWorkItems → mutator → cli
+  // codepath that itself calls mutateDispatch composes into a single
+  // outer BEGIN/COMMIT instead of throwing "transaction within transaction".
+  return withTransaction(db, () => {
+    const before = _readDispatchSectionedFull();
+    // Deep-clone before handing to the mutator so in-place mutations don't
+    // contaminate our snapshot. JSON round-trip is fine here — dispatch
+    // records are JSON-serializable by construction.
+    const beforeSnapshot = JSON.parse(JSON.stringify(before));
+    const next = mutator(before);
+    // Mutator may return a replacement, mutate in place + return undefined,
+    // or return a falsy value (legacy: same as undefined → use `before`).
+    const after = (next === undefined || next === null) ? before : next;
+    // Normalize: callers may return objects missing some sections (e.g.
+    // older code only touched pending). Fill in defaults so the diff
+    // doesn't mis-classify a missing section as "everything deleted".
+    for (const s of SECTIONS) {
+      if (!Array.isArray(after[s])) after[s] = beforeSnapshot[s] || [];
+    }
+    const diff = _computeDispatchDiff(beforeSnapshot, after);
+    const wrote = _applyDispatchDiff(db, diff);
+    return { wrote, result: after };
+  });
+}
+
+// Mirror the canonical SQL state back to dispatch.json after a successful
+// mutation. Phase 1 keeps the JSON file as a *read-only derivative* of SQL
+// — never an independent source of truth — so legacy callers that
+// readFileSync the file directly (a couple of internal paths + the broader
+// test suite) keep seeing consistent data. There's no drift opportunity
+// because:
+//
+//   - The mirror is always regenerated from the SQL table, never from
+//     in-memory state, so a write that lands transactionally in SQL but
+//     then crashes before the mirror is also written produces a stale
+//     mirror — but the SQL state is correct, and the next successful
+//     mutation overwrites the mirror.
+//   - No code path writes to dispatch.json independently anymore;
+//     mutateDispatch is the only writer.
+//
+// Phase 1.5 will remove the mirror once we've migrated the remaining
+// direct-readers (engine/queries.js fallback path, engine/routing.js,
+// the test infrastructure) to use readDispatchSectioned() / getDispatch().
+function _mirrorJsonFromSql() {
+  try {
+    const path = require('path');
+    const fs = require('fs');
+    const shared = require('./shared');
+    // The full sectioned view (not the 24h-windowed one) so the JSON
+    // mirror matches what a legacy reader would have seen before Phase 1.
+    const full = _readDispatchSectionedFull();
+    const dispatchPath = path.join(shared.MINIONS_DIR, 'engine', 'dispatch.json');
+    // safeWrite handles the atomic rename + .backup sidecar. Best-effort —
+    // failure here doesn't propagate (SQL is source of truth, mirror is
+    // optional refresh material).
+    shared.safeWrite(dispatchPath, full);
+    // Re-derive what the parent shared.safeWrite would have done so an
+    // operator copying the file out gets matched fs.statSync mtimes
+    // (the dashboard's slow-state tracker watches this file as part of
+    // the dispatch path list). No-op if safeWrite already advanced mtime.
+    try { fs.utimesSync(dispatchPath, new Date(), new Date()); } catch { /* best-effort */ }
+  } catch {
+    // Mirror failures are non-fatal: SQL has already committed. Tests +
+    // legacy readers will see the previous mirror until the next write.
+  }
+}
+
+// Test seam — clear the dispatches table. Production code never calls this.
+function _resetDispatchTableForTest() {
+  const { getDb } = require('./db');
+  try { getDb().exec('DELETE FROM dispatches'); } catch { /* not initialized */ }
+}
+
+module.exports = {
+  readDispatchSectioned,
+  applyDispatchMutation,
+  _readDispatchSectionedFull,
+  _computeDispatchDiff,
+  _mirrorJsonFromSql,
+  _resetDispatchTableForTest,
+};

package/engine/dispatch.js

@@ -58,7 +58,57 @@ function _sidecarChangedPrompts(dispatch, prevSnap) {
   }
 }
 
+// SQL-backed mutator (Phase 1). Same external contract as the legacy
+// JSON path: mutator receives a `{ pending, active, completed, review }`
+// object, mutates in place (or returns a replacement), and the changes
+// land transactionally in the dispatches table. The diff-then-apply trick
+// in dispatch-store preserves every field via the `data` JSON column, so
+// new dispatch fields the engine adds in future commits don't need any
+// migration — they round-trip automatically.
+//
+// Fallback path: if SQLite is unavailable (Node < 22.5, the DB init failed,
+// the table doesn't exist yet because the migration hasn't run) we fall
+// back to the legacy mutateJsonFileLocked flow against dispatch.json.
+// The migration leaves dispatch.json.pre-sql-<ts> on disk, so an operator
+// pinning to an older release can rename it back.
 function mutateDispatch(mutator) {
+  // Try the SQL path first.
+  try {
+    const store = require('./dispatch-store');
+    const { wrote, result } = store.applyDispatchMutation((dispatch) => {
+      dispatch.pending = Array.isArray(dispatch.pending) ? dispatch.pending : [];
+      dispatch.active = Array.isArray(dispatch.active) ? dispatch.active : [];
+      dispatch.completed = Array.isArray(dispatch.completed) ? dispatch.completed : [];
+      dispatch.review = Array.isArray(dispatch.review) ? dispatch.review : [];
+      const prevSnap = _snapshotPrompts(dispatch);
+      const next = mutator(dispatch) ?? dispatch;
+      // Prompt-size guard: only scan items whose prompt changed (or new items),
+      // so a 100-item status-flip doesn't re-byte-count every prompt.
+      _sidecarChangedPrompts(next, prevSnap);
+      return next;
+    });
+    if (wrote) {
+      try { require('./queries').invalidateDispatchCache(); } catch {}
+      try { require('./db-events').emitStateEvent('dispatch'); } catch { /* optional */ }
+      // Phase 1 dual-write: mirror the canonical SQL state to dispatch.json
+      // so legacy direct-readers (engine/queries.js fallback, engine/routing.js,
+      // test infrastructure that fs.readFileSync's the file) stay in sync.
+      // SQL is source of truth; the JSON file is regenerated from SQL on
+      // every successful mutation, never independently mutated.
+      try { store._mirrorJsonFromSql(); } catch { /* best-effort */ }
+    }
+    return result;
+  } catch (e) {
+    // Only fall back if the failure looks like a "DB not available" case,
+    // not a programming error. Surface mutator exceptions to the caller.
+    if (e && /SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
+      // fall through to legacy JSON path below
+    } else {
+      throw e;
+    }
+  }
+
+  // Legacy JSON path (pre-Phase 1 fallback). Same code as before.
   const defaultDispatch = { pending: [], active: [], completed: [] };
   const result = mutateJsonFileLocked(DISPATCH_PATH, (dispatch) => {
     dispatch.pending = Array.isArray(dispatch.pending) ? dispatch.pending : [];
@@ -66,8 +116,6 @@ function mutateDispatch(mutator) {
     dispatch.completed = Array.isArray(dispatch.completed) ? dispatch.completed : [];
     const prevSnap = _snapshotPrompts(dispatch);
     const next = mutator(dispatch) ?? dispatch;
-    // Prompt-size guard: only scan items whose prompt changed (or new items),
-    // so a 100-item status-flip doesn't re-byte-count every prompt.
     _sidecarChangedPrompts(next, prevSnap);
     return next;
   }, {
@@ -76,7 +124,6 @@ function mutateDispatch(mutator) {
       try { require('./db-events').emitStateEvent('dispatch'); } catch { /* optional */ }
     },
   });
-  // Invalidate the read cache so next getDispatch() sees fresh data
   try { require('./queries').invalidateDispatchCache(); } catch {}
   return result;
 }

package/engine/llm.js

@@ -224,6 +224,7 @@ function _missingRuntimeMessage(runtimeName, runtime, reason) {
       '',
       'Choose a supported runtime in Settings -> Engine -> defaultCli/ccCli, then install its CLI:',
       '- Claude Code: npm install -g @anthropic-ai/claude-code or download from https://claude.ai/download',
+      '- OpenAI Codex CLI: npm install -g @openai/codex or brew install --cask codex',
       '- GitHub Copilot CLI: install via GitHub CLI/gh-copilot or download from https://github.com/github/copilot-cli/releases',
       '',
       'After installing, restart Minions so the dashboard and engine inherit the updated PATH.',
@@ -238,10 +239,10 @@ function _missingRuntimeMessage(runtimeName, runtime, reason) {
     '',
     'After installing, restart Minions so the dashboard and engine inherit the updated PATH.',
   ];
-  if (selected !== 'claude') {
-    lines.push('Or switch Settings -> Engine -> defaultCli/ccCli back to "claude" after installing Claude Code.');
+  if (selected === 'claude') {
+    lines.push('Or switch Settings -> Engine -> defaultCli/ccCli to "copilot" or "codex" after installing another supported runtime.');
   } else {
-    lines.push('Or switch Settings -> Engine -> defaultCli/ccCli to "copilot" after installing GitHub Copilot CLI.');
+    lines.push('Or switch Settings -> Engine -> defaultCli/ccCli back to "claude" after installing Claude Code.');
   }
   return lines.join('\n');
 }

package/engine/queries.js

@@ -174,10 +174,24 @@ function getControl() {
 let _dispatchCache = null;
 let _dispatchCacheAt = 0;
 function getDispatch() {
-  // Short-lived cache — dispatch.json is read 10+ times per tick but only changes on mutateDispatch
+  // Short-lived cache — getDispatch is called 10+ times per tick (per-agent
+  // status derivation, work-item PR enrichment, etc.) but only changes on
+  // mutateDispatch. SQL reads are sub-millisecond, so the cache is now
+  // mostly a per-request memo rather than the disk-read amortizer it used
+  // to be when dispatch.json could be 1+ MB.
   const now = Date.now();
   if (_dispatchCache && (now - _dispatchCacheAt) < 2000) return _dispatchCache;
-  _dispatchCache = readJsonNoRestore(DISPATCH_PATH) || { pending: [], active: [], completed: [] };
+  // Try SQL first (post-Phase 1). If the table doesn't exist (greenfield
+  // install, Node < 22.5, or DB init failed) fall back to the legacy JSON
+  // reader. The migration leaves dispatch.json.pre-sql-<ts> on disk so
+  // operators rolling back to a pre-migration release can rename it back.
+  try {
+    const store = require('./dispatch-store');
+    const sectioned = store.readDispatchSectioned();
+    _dispatchCache = sectioned;
+  } catch {
+    _dispatchCache = readJsonNoRestore(DISPATCH_PATH) || { pending: [], active: [], completed: [] };
+  }
   _dispatchCacheAt = now;
   return _dispatchCache;
 }
@@ -1247,30 +1261,44 @@ function getWorkItems(config) {
     if (!_workItemsMtimesDiffer(_workItemsCacheMtimes, currMtimes)) return _workItemsCache;
   }
   const projects = getProjects(config);
-  const allItems = [];
+  let allItems = [];
 
-  // Central work items
-  const centralData = safeRead(path.join(MINIONS_DIR, 'work-items.json'));
-  if (centralData) {
-    try {
-      for (const item of JSON.parse(centralData)) {
-        item._source = 'central';
-        allItems.push(item);
-      }
-    } catch {}
-  }
+  // Phase 2: prefer the SQL store. Returns every scope's items tagged
+  // with `_source = <scope>`, matching the legacy decoration. Empty SQL
+  // table OR a SQLite failure falls through to the JSON path below so a
+  // fresh install + a node:sqlite-broken environment both keep working.
+  try {
+    const store = require('./work-items-store');
+    const sqlItems = store.readAllWorkItems();
+    if (Array.isArray(sqlItems) && sqlItems.length > 0) {
+      allItems = sqlItems;
+    }
+  } catch { /* fall through to JSON */ }
 
-  // Per-project work items
-  for (const project of projects) {
-    const data = safeRead(projectWorkItemsPath(project));
-    if (data) {
+  if (allItems.length === 0) {
+    // Central work items
+    const centralData = safeRead(path.join(MINIONS_DIR, 'work-items.json'));
+    if (centralData) {
       try {
-        for (const item of JSON.parse(data)) {
-          item._source = project.name || 'project';
+        for (const item of JSON.parse(centralData)) {
+          item._source = 'central';
           allItems.push(item);
         }
       } catch {}
     }
+
+    // Per-project work items
+    for (const project of projects) {
+      const data = safeRead(projectWorkItemsPath(project));
+      if (data) {
+        try {
+          for (const item of JSON.parse(data)) {
+            item._source = project.name || 'project';
+            allItems.push(item);
+          }
+        } catch {}
+      }
+    }
   }
 
   // Cross-reference with dispatch (fill in agent from active dispatch if missing on work item)

package/engine/restart-health.js

@@ -55,10 +55,16 @@ function isPortListening(port) {
       const re6 = new RegExp(`\\s\\[::1?\\]:${n}\\s+\\S+\\s+LISTENING`, 'i');
       return re.test(out) || re6.test(out);
     }
-    const out = execSync(`ss -ltn 'sport = :${n}' 2>/dev/null || netstat -ltn 2>/dev/null`, {
+    try {
+      const out = execSync(`lsof -nP -iTCP:${n} -sTCP:LISTEN`, {
+        encoding: 'utf8', timeout: 3000,
+      });
+      if (/\bLISTEN\b/i.test(out)) return true;
+    } catch {}
+    const out = execSync(`ss -ltn 'sport = :${n}' 2>/dev/null || netstat -ltn 2>/dev/null || netstat -an -p tcp 2>/dev/null`, {
       encoding: 'utf8', timeout: 3000, shell: true,
     });
-    return new RegExp(`[:.]${n}\\b`).test(out);
+    return new RegExp(`(?:[:.])${n}\\b[^\\n]*\\b(?:LISTEN|LISTENING)\\b`, 'i').test(out);
   } catch { return false; }
 }