@yemi33/minions 0.1.2066 → 0.1.2068

package/engine/runtimes/index.js

@@ -17,6 +17,7 @@
 
 const registry = new Map();
 registry.set('claude', require('./claude'));
+registry.set('codex', require('./codex'));
 registry.set('copilot', require('./copilot'));
 
 /**

package/engine/shared.js

@@ -4572,24 +4572,53 @@ function listProcessReachable(rootPids, allProcesses = null) {
  * @param {Function} mutator - Receives the array, mutates in place or returns new value
  */
 function mutateWorkItems(filePath, mutator) {
+  // Phase 2 SQL path. Route through work-items-store so SQL is the source
+  // of truth; mirror back to the JSON file for legacy direct-readers.
+  //
+  // The SQL store identifies records by `scope` (central or project name)
+  // derived from the file path's last two segments. Ad-hoc file paths
+  // outside the MINIONS_DIR layout (e.g. tests using createTmpDir()) can't
+  // be mapped to a stable scope, so we short-circuit to the legacy
+  // JSON path for those. Production callers always use
+  // shared.projectWorkItemsPath(p) / MINIONS_DIR/work-items.json.
+  //
+  // SQLite failures fall through to the legacy JSON path below — keeps a
+  // node:sqlite-broken install fully functional.
+  const fpNorm = String(filePath).replace(/\\/g, '/');
+  const minionsNorm = String(MINIONS_DIR).replace(/\\/g, '/');
+  const insideMinionsDir = fpNorm.startsWith(minionsNorm + '/') || fpNorm === minionsNorm + '/work-items.json';
+  if (insideMinionsDir) {
+    try {
+      const store = require('./work-items-store');
+      const scope = store.scopeForFilePath(filePath);
+      const { wrote, result } = store.applyWorkItemsMutation(scope, (items) => {
+        if (!Array.isArray(items)) items = [];
+        return mutator(items) || items;
+      });
+      if (wrote) {
+        try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror is best-effort */ }
+        try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
+      }
+      try { require('./queries').invalidateWorkItemsCache(); } catch { /* queries not loaded */ }
+      return result;
+    } catch (e) {
+      if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
+        throw e;
+      }
+      // Fall through to the legacy JSON path on SQLite errors only.
+    }
+  }
+
   const result = mutateJsonFileLocked(filePath, (data) => {
     if (!Array.isArray(data)) data = [];
     return mutator(data) || data;
   }, {
     defaultValue: [],
     skipWriteIfUnchanged: true,
-    // Emit only when an actual write happened. The dedup path through
-    // createWorkItemWithDedup returns the array unchanged → skipWriteIfUnchanged
-    // suppresses the write, and we want to suppress the event too (otherwise
-    // duplicate POSTs bump the cache version for no observable change).
     onWrote: () => {
       try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
     },
   });
-  // Invalidate the read cache so the next getWorkItems() sees fresh data
-  // (W-mpodww9h000b460a). Mirrors dispatch.js's invalidateDispatchCache
-  // call after mutateDispatch. Lazy-required to avoid the queries→shared
-  // require cycle.
   try { require('./queries').invalidateWorkItemsCache(); } catch { /* queries not loaded */ }
   return result;
 }

package/engine/work-items-store.js

@@ -0,0 +1,351 @@
+// engine/work-items-store.js — SQL-backed implementation of the
+// per-file work-items array. Same external contract as the legacy
+// JSON-file reader:
+//
+//   readWorkItemsForScope(scope)        -> [wi, wi, ...]
+//   applyWorkItemsMutation(scope, fn)   -> runs fn against the array,
+//                                          diffs by id, applies in one
+//                                          transaction, returns
+//                                          { wrote, result }.
+//
+// `scope` is 'central' for the top-level work-items.json or a project
+// name for projects/<name>/work-items.json. Mirrors the file convention
+// established by migration 003.
+//
+// All callers go through shared.mutateWorkItems and
+// queries.getWorkItems — those are thin wrappers over the helpers here.
+
+const path = require('path');
+const fs = require('fs');
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+// Map an absolute filePath → scope key. The store doesn't know about the
+// caller's file layout; we derive scope from the path's last two segments
+// because every work-items file in this codebase sits at either
+// <root>/work-items.json (central) or <root>/projects/<name>/work-items.json.
+function scopeForFilePath(filePath) {
+  const norm = String(filePath).replace(/\\/g, '/');
+  const m = norm.match(/projects\/([^/]+)\/work-items\.json$/);
+  if (m) return m[1];
+  // Anything else — including the central work-items.json under MINIONS_DIR
+  // and any one-off test fixture file — lands in 'central'. Tests that
+  // need to keep multiple fixture files distinct should put them under a
+  // synthetic projects/<name>/ subtree.
+  return 'central';
+}
+
+// JSON fallback path resolution. Used both for the empty-SQL fallback
+// (test seeding pattern from Phase 1) and for the post-write JSON mirror.
+function _filePathForScope(scope) {
+  const shared = require('./shared');
+  if (scope === 'central') {
+    return path.join(shared.MINIONS_DIR, 'work-items.json');
+  }
+  return path.join(shared.MINIONS_DIR, 'projects', scope, 'work-items.json');
+}
+
+function _parseRow(row) {
+  if (!row || !row.data) return null;
+  try { return JSON.parse(row.data); }
+  catch { return null; }
+}
+
+function _readJsonArrayFallback(scope) {
+  const fp = _filePathForScope(scope);
+  let raw;
+  try { raw = fs.readFileSync(fp, 'utf8'); }
+  catch { return []; }
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch (e) {
+    try {
+      // eslint-disable-next-line no-console
+      console.warn(`[work-items-store] corrupt JSON in ${fp}: ${e.message}`);
+    } catch { /* console may be wrapped in tests */ }
+    return [];
+  }
+}
+
+function readWorkItemsForScope(scope) {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return _readJsonArrayFallback(scope); }
+
+  _resyncScopeIfJsonDiverged(db, scope);
+
+  const rows = db.prepare(`
+    SELECT data FROM work_items
+    WHERE scope = ?
+    ORDER BY rowid
+  `).all(scope);
+
+  if (rows.length === 0) {
+    // Same fallback shape as dispatch-store: SQL empty AND JSON has
+    // content means a test seeded via fs.writeFileSync (legacy helper)
+    // or a fresh install pre-migration. Returning the JSON keeps those
+    // call sites working without touching every helper.
+    const fallback = _readJsonArrayFallback(scope);
+    if (fallback.length > 0) return fallback;
+    return [];
+  }
+
+  const out = [];
+  for (const row of rows) {
+    const wi = _parseRow(row);
+    if (wi) out.push(wi);
+  }
+  return out;
+}
+
+// If the JSON mirror's mtime advanced since our last mirror write, the
+// file was rewritten outside the store (test cleanup + safeWrite, manual
+// operator edit, external tool). Treat the JSON as canonical and rebuild
+// SQL for that scope from it. Called from BOTH read and write paths so
+// readers don't see stale SQL state and writers don't compute diffs
+// against a SQL snapshot that no longer matches what callers expect.
+//
+// Safe to call inside `withTransaction` — DELETE + INSERT against a
+// scope is bounded by the JSON's row count.
+function _resyncScopeIfJsonDiverged(db, scope) {
+  const jsonPath = _filePathForScope(scope);
+  const currentMtime = _statMtimeMs(jsonPath);
+  const lastMirrorMtime = _lastMirrorMtimeByScope.get(scope);
+  if (currentMtime == null) return;  // JSON absent → nothing to resync
+  if (lastMirrorMtime != null && Math.abs(currentMtime - lastMirrorMtime) <= 1) return;  // in sync
+  // No mirror record but JSON exists: first-time hydrate iff SQL is empty
+  // for this scope (avoid trampling a freshly-backfilled migration state
+  // on the very first read).
+  if (lastMirrorMtime == null) {
+    const sqlHas = db.prepare('SELECT 1 FROM work_items WHERE scope = ? LIMIT 1').get(scope);
+    if (sqlHas) {
+      // Record current mtime as the baseline so the next divergence is
+      // detected, but don't rebuild — migration already populated SQL.
+      _lastMirrorMtimeByScope.set(scope, currentMtime);
+      return;
+    }
+  }
+  _hydrateScopeFromJson(db, scope);
+  _lastMirrorMtimeByScope.set(scope, currentMtime);
+}
+
+// Read all rows across all scopes — used by queries.getWorkItems which
+// needs to surface central + every project's items in a single shot,
+// tagged with their source scope.
+function readAllWorkItems() {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return null; }  // signal: SQL unavailable, caller must fall back
+
+  // Pick up any external JSON edits for every scope SQL knows about.
+  // Also resync 'central' explicitly so first-time reads on a JSON-only
+  // install hydrate the central scope. Per-project scopes get checked
+  // lazily on next read or mutation of that scope.
+  try { _resyncScopeIfJsonDiverged(db, 'central'); } catch {}
+  const knownScopes = db.prepare('SELECT DISTINCT scope FROM work_items').all().map(r => r.scope);
+  for (const s of knownScopes) {
+    if (s === 'central') continue;
+    try { _resyncScopeIfJsonDiverged(db, s); } catch {}
+  }
+
+  const rows = db.prepare('SELECT data, scope FROM work_items ORDER BY rowid').all();
+  const out = [];
+  for (const row of rows) {
+    const wi = _parseRow(row);
+    if (!wi) continue;
+    // Preserve the legacy `_source` convention from queries.getWorkItems.
+    wi._source = row.scope;
+    out.push(wi);
+  }
+  return out;
+}
+
+function _indexById(arr) {
+  const out = new Map();
+  for (const wi of arr) {
+    if (!wi || !wi.id) continue;
+    out.set(String(wi.id), wi);
+  }
+  return out;
+}
+
+function _computeWorkItemsDiff(scope, before, after) {
+  const beforeMap = _indexById(before);
+  const afterMap = _indexById(after);
+  const toUpsert = [];
+  const toDelete = [];
+  for (const [id, wi] of afterMap) {
+    const prev = beforeMap.get(id);
+    if (!prev) { toUpsert.push(wi); continue; }
+    if (JSON.stringify(prev) !== JSON.stringify(wi)) toUpsert.push(wi);
+  }
+  for (const [id] of beforeMap) {
+    if (!afterMap.has(id)) toDelete.push(id);
+  }
+  return { scope, toUpsert, toDelete };
+}
+
+function _applyWorkItemsDiff(db, diff) {
+  if (diff.toUpsert.length === 0 && diff.toDelete.length === 0) return false;
+  const now = Date.now();
+  const upsertStmt = db.prepare(`
+    INSERT INTO work_items (id, scope, status, type, agent, parent_id, created_at, completed_at, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(scope, id) DO UPDATE SET
+      status        = excluded.status,
+      type          = excluded.type,
+      agent         = excluded.agent,
+      parent_id     = excluded.parent_id,
+      created_at    = excluded.created_at,
+      completed_at  = excluded.completed_at,
+      data          = excluded.data,
+      updated_at    = excluded.updated_at
+  `);
+  const deleteStmt = db.prepare('DELETE FROM work_items WHERE scope = ? AND id = ?');
+
+  for (const wi of diff.toUpsert) {
+    upsertStmt.run(
+      String(wi.id),
+      diff.scope,
+      String(wi.status || 'pending'),
+      wi.type || null,
+      wi.dispatched_to || wi.agent || null,
+      wi.parent_id || null,
+      _toMs(wi.created_at),
+      _toMs(wi.completed_at || wi.completedAt),
+      JSON.stringify(wi),
+      now,
+    );
+  }
+  for (const id of diff.toDelete) {
+    deleteStmt.run(diff.scope, id);
+  }
+  return true;
+}
+
+// Apply a mutation to the work-items array for a given scope. Wrapped in
+// a single transaction so the diff and apply can't race against a
+// concurrent reader/writer.
+//
+// Returns { wrote, result }:
+//   wrote  — true iff at least one INSERT/UPDATE/DELETE landed
+//   result — the post-mutation array (legacy return shape of
+//            mutateJsonFileLocked)
+// In-process record of the JSON mirror mtime we last wrote, keyed by
+// scope. Used to detect external JSON edits between mutations — both
+// (a) tests that clean up + re-seed via fs.writeFileSync, and
+// (b) operators that hand-edit work-items.json in production. When the
+// current JSON mtime differs from what we recorded, the JSON has been
+// rewritten outside the store; we treat the JSON as canonical and
+// re-hydrate SQL from it before computing the diff.
+const _lastMirrorMtimeByScope = new Map();
+
+function _statMtimeMs(filePath) {
+  try { return fs.statSync(filePath).mtimeMs; }
+  catch { return null; }
+}
+
+function _hydrateScopeFromJson(db, scope) {
+  const jsonItems = _readJsonArrayFallback(scope);
+  // DELETE before re-insert: callers that wrote a smaller JSON file
+  // (test cleanup() removing items) must end up with a smaller SQL state.
+  db.prepare('DELETE FROM work_items WHERE scope = ?').run(scope);
+  if (jsonItems.length === 0) return;
+  const now = Date.now();
+  const ins = db.prepare(`
+    INSERT INTO work_items (id, scope, status, type, agent, parent_id, created_at, completed_at, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(scope, id) DO NOTHING
+  `);
+  for (const wi of jsonItems) {
+    if (!wi || !wi.id) continue;
+    ins.run(
+      String(wi.id),
+      scope,
+      String(wi.status || 'pending'),
+      wi.type || null,
+      wi.dispatched_to || wi.agent || null,
+      wi.parent_id || null,
+      _toMs(wi.created_at),
+      _toMs(wi.completed_at || wi.completedAt),
+      JSON.stringify(wi),
+      now,
+    );
+  }
+}
+
+function applyWorkItemsMutation(scope, mutator) {
+  const { getDb, withTransaction } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch (e) {
+    throw new Error(`engine/work-items-store: SQLite unavailable (${e.message}); cannot mutate work_items`);
+  }
+
+  return withTransaction(db, () => {
+    // Re-hydrate SQL from JSON if the file was touched outside the
+    // store (external edit, test re-seed, first-time hydrate after a
+    // fresh install). Without this, the diff would compare against a
+    // stale SQL snapshot and the mirror write would overwrite the JSON
+    // with that stale state.
+    _resyncScopeIfJsonDiverged(db, scope);
+
+    const before = readWorkItemsForScope(scope);
+    // Strip the _source decoration before handing to the mutator — it's
+    // added by readAllWorkItems only and would round-trip as garbage into
+    // the data column otherwise.
+    for (const wi of before) { if (wi && wi._source) delete wi._source; }
+    const beforeSnapshot = JSON.parse(JSON.stringify(before));
+    const next = mutator(before);
+    const after = (next === undefined || next === null)
+      ? before
+      : (Array.isArray(next) ? next : before);
+    const diff = _computeWorkItemsDiff(scope, beforeSnapshot, after);
+    const wrote = _applyWorkItemsDiff(db, diff);
+    return { wrote, result: after };
+  });
+}
+
+// Mirror SQL → JSON file. Phase 2 keeps the file as a read-only derivative
+// so legacy direct-readers keep working; SQL is the source of truth.
+function _mirrorJsonFromSql(scope, filePath) {
+  try {
+    const shared = require('./shared');
+    const items = readWorkItemsForScope(scope);
+    // Strip _source if present — only added by readAllWorkItems, but
+    // belt-and-suspenders since the mirror is observable.
+    for (const wi of items) { if (wi && wi._source) delete wi._source; }
+    const target = filePath || _filePathForScope(scope);
+    shared.safeWrite(target, items);
+    // Record the mtime we just wrote so the next mutation can detect
+    // an external edit (mtime advanced while we weren't looking).
+    const m = _statMtimeMs(target);
+    if (m != null) _lastMirrorMtimeByScope.set(scope, m);
+  } catch {
+    // Mirror failures are non-fatal: SQL has already committed.
+  }
+}
+
+// Test seam.
+function _resetWorkItemsTableForTest() {
+  const { getDb } = require('./db');
+  try { getDb().exec('DELETE FROM work_items'); } catch { /* not initialized */ }
+}
+
+module.exports = {
+  scopeForFilePath,
+  readWorkItemsForScope,
+  readAllWorkItems,
+  applyWorkItemsMutation,
+  _filePathForScope,
+  _mirrorJsonFromSql,
+  _resetWorkItemsTableForTest,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2066",
+  "version": "0.1.2068",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"