@yemi33/minions 0.1.2065 → 0.1.2067

package/README.md

@@ -11,11 +11,11 @@ Inspired by and initially scaffolded from [Brady Gaster's Squad](https://bradyga
 ## Prerequisites
 
 - **Node.js** 18+ (LTS recommended)
-- **A supported runtime CLI** — Minions defaults to GitHub Copilot CLI (`npm install -g @github/copilot`). Claude Code CLI (`npm install -g @anthropic-ai/claude-code`) is also supported; switch with `minions config set-cli claude` or per-agent `cli` overrides.
-- **Auth for your runtime** — GitHub Copilot subscription (Copilot CLI handles its own auth) or an Anthropic API key / Claude Max subscription
+- **A supported runtime CLI** — Minions defaults to GitHub Copilot CLI (`npm install -g @github/copilot`). Claude Code CLI (`npm install -g @anthropic-ai/claude-code`) and OpenAI Codex CLI (`npm install -g @openai/codex` or `brew install --cask codex`) are also supported; switch with `minions config set-cli claude`, `minions config set-cli codex`, or per-agent `cli` overrides.
+- **Auth for your runtime** — GitHub Copilot subscription (Copilot CLI handles its own auth), OpenAI Codex login / API key, or an Anthropic API key / Claude Max subscription
 - **Git** — agents create worktrees for all code changes
 
-> **Note:** you do **not** need to configure your CLI for "autopilot" / "bypass permissions" / "dangerous mode". Minions passes the right bypass flag per spawn (`--dangerously-skip-permissions` for Claude; `--autopilot --allow-all --no-ask-user` for Copilot), independent of your global CLI config. Run `minions doctor` to verify your installed CLI accepts those flags.
+> **Note:** you do **not** need to configure your CLI for "autopilot" / "bypass permissions" / "dangerous mode". Minions passes the right runtime-owned flag per spawn (`--dangerously-skip-permissions` for Claude; `--autopilot --allow-all --no-ask-user` for Copilot; `codex exec --sandbox workspace-write --json -` for Codex), independent of your global CLI config. Run `minions doctor` to verify your installed CLI accepts those flags.
 
 ## Installation
 

package/dashboard.js

@@ -1637,6 +1637,23 @@ function _mtimeTrackedFiles() {
 }
 let _lastMtimes = {}; // { filePath: mtimeMs } — baseline since last build
 
+// MAX(events.id) at last rebuild. Phase 0 cache-version source running
+// alongside the mtime tracker — either signal busts the cache. Once the
+// engine state files migrate into SQL tables (Phase 1+) the mtime tracker
+// goes away and this becomes the sole invalidator. Reads are O(index seek).
+let _lastEventVersion = -1;
+function _getCurrentEventVersion() {
+  try {
+    const { getDb } = require('./engine/db');
+    const row = getDb().prepare('SELECT COALESCE(MAX(id), 0) AS v FROM events').get();
+    return Number(row.v) || 0;
+  } catch {
+    // DB unavailable (Node < 22.5, missing file, etc.) — return a sentinel that
+    // disables event-based cache busting. The mtime tracker still works.
+    return -1;
+  }
+}
+
 // Stat a tracked path with transient-error tolerance. ENOENT (file/dir doesn't
 // exist) is normal — fresh installs, deleted projects, empty PRD dirs all hit
 // this — and maps to 0 so the entry just doesn't bust the cache. EBUSY /
@@ -1693,6 +1710,11 @@ function invalidateStatusCache(_opts) {
   _statusCacheJson = null;
   _statusCacheGzip = null;
   _lastMtimes = {};
+  _lastEventVersion = -1;
+  // Emit a 'cache-invalidate' event row so any peer dashboard process
+  // sharing the SQLite file also sees the bust on its next poll.
+  // Best-effort — db-events swallows all errors.
+  try { require('./engine/db-events').emitStateEvent('cache-invalidate'); } catch { /* optional */ }
   // Tell any in-flight refreshStatusAsync() that its result is stale and must
   // not be published. Bumping the generation also forces the next ETag to
   // differ from anything a client already has cached.
@@ -2098,12 +2120,23 @@ function _markStatusCacheBuilt() {
 }
 
 function getStatus() {
-  // Steady-state fast path: cache present + no tracked file changed → return
-  // cached snapshot. Single mtime check covers both fast + slow tracker
-  // contributions (see _mtimeTrackedFiles).
+  // Steady-state fast path: cache present + neither signal moved → return
+  // cached snapshot. Two cache-version sources running side by side during
+  // Phase 0 migration:
+  //   1. _lastMtimes  — legacy file mtime tracker. Single check covers
+  //                     both former fast + slow registries (see _mtimeTrackedFiles).
+  //   2. _lastEventVersion — MAX(events.id) from engine/state.db.
+  //                     Bumped by emitStateEvent() in every mutator wrapper
+  //                     and by invalidateStatusCache() itself.
+  // Either signal advancing busts the cache. Once engine state files
+  // migrate into SQL tables (Phase 1+) the mtime tracker goes away and
+  // event-version becomes the sole invalidator.
+  const preBuildEventVersion = _getCurrentEventVersion();
   if (_statusCache) {
     const currMtimes = _getMtimes();
-    if (!_mtimesChanged(_lastMtimes, currMtimes)) return _statusCache;
+    if (preBuildEventVersion === _lastEventVersion && !_mtimesChanged(_lastMtimes, currMtimes)) {
+      return _statusCache;
+    }
   }
   // Stale or first-call: rebuild everything. Reload config first so newly-
   // added projects / agents land before the slice builders read them.
@@ -2118,6 +2151,7 @@ function getStatus() {
   const slow = _buildStatusSlowState();
   _statusCache = { ...fast, ...slow, timestamp: new Date().toISOString() };
   _lastMtimes = preBuildMtimes;
+  _lastEventVersion = preBuildEventVersion;
   _markStatusCacheBuilt();
   return _statusCache;
 }
@@ -2185,10 +2219,11 @@ function refreshStatusAsync() {
     try {
       const startGeneration = _statusInvalidationGeneration;
 
-      // Steady-state fast path — same single mtime check as the sync getStatus.
+      // Steady-state fast path — same dual-signal check as the sync getStatus.
+      const preBuildEventVersion = _getCurrentEventVersion();
       if (_statusCache) {
         const currMtimes = _getMtimes();
-        if (!_mtimesChanged(_lastMtimes, currMtimes)) {
+        if (preBuildEventVersion === _lastEventVersion && !_mtimesChanged(_lastMtimes, currMtimes)) {
           if (profile) {
             _emitStatusTiming({
               phase: 'cache-hit',
@@ -2241,6 +2276,7 @@ function refreshStatusAsync() {
 
       _statusCache = { ...fast, ...slow, timestamp: new Date().toISOString() };
       _lastMtimes = preBuildMtimes;
+      _lastEventVersion = preBuildEventVersion;
       _markStatusCacheBuilt();
       if (profile) {
         _emitStatusTiming({
@@ -2277,6 +2313,7 @@ function _resetStatusCacheForTesting() {
   _statusInvalidationGeneration = 0;
   _statusRefreshHook = null;
   _lastMtimes = {};
+  _lastEventVersion = -1;
 }
 
 /** Return cached JSON string of status — single stringify, reused by SSE and /api/status */

package/docs/runtime-adapters.md

@@ -11,6 +11,7 @@ behavior is hidden behind an adapter object resolved through `resolveRuntime()`.
 | `engine/runtimes/index.js` | Adapter registry. `resolveRuntime(name)`, `listRuntimes()`, `registerRuntime()`. Engine code MUST go through these. |
 | `engine/runtimes/claude.js` | Claude Code adapter. Owns binary probe, `--system-prompt-file`, JSONL parser, model shorthands, budget cap, bare mode. |
 | `engine/runtimes/copilot.js` | GitHub Copilot CLI adapter. Owns standalone-vs-`gh-copilot` resolution, stdin-only prompt delivery, `https://api.githubcopilot.com/models` discovery, effort `'max' → 'xhigh'` mapping. |
+| `engine/runtimes/codex.js` | OpenAI Codex CLI adapter. Owns `@openai/codex`/native binary resolution, `codex exec --json -` prompt delivery, `codex debug models --bundled` discovery, `.agents/skills` roots, and Codex JSONL parsing. |
 
 `resolveRuntime(name)` throws when `name` is unknown so misconfigurations surface
 at dispatch time instead of producing silent fallbacks deep inside spawn logic.
@@ -62,24 +63,24 @@ this table is the human-readable mirror.
 Engine code branches on flags, never on runtime names. Add a flag whenever a
 real behavioral split appears between adapters.
 
-| Flag | Claude | Copilot | Gates |
-|------|--------|---------|-------|
-| `streaming` | ✓ | ✓ | JSONL events on stdout. |
-| `sessionResume` | ✓ | ✓ | `--resume <id>`. |
-| `midRunSessionId` | ✓ | ✗ | Resumable session ID emitted before the terminal `result` event; when false, steering waits for a checkpoint. |
-| `systemPromptFile` | ✓ | ✗ | sysprompt via `--system-prompt-file` (else inlined into stdin by `buildPrompt`). |
-| `effortLevels` | ✓ | ✓ | `--effort low\|medium\|high\|xhigh`. Copilot maps `'max' → 'xhigh'` inside `resolveModel`/`buildArgs`; Claude leaves `'max'` alone. |
-| `costTracking` | ✓ | ✗ | USD + token counts in the result event. Copilot only emits `premiumRequests`. |
-| `modelShorthands` | ✓ | ✗ | Bare `sonnet`/`opus`/`haiku` accepted. Copilot expects full model IDs (`claude-sonnet-4.5`, `gpt-5.4`). |
-| `modelDiscovery` | ✗ | ✓ | `listModels()` returns a real catalog (Copilot reads `https://api.githubcopilot.com/models`). |
-| `promptViaArg` | ✗ | ✗ | If true, prompt goes via `--prompt <text>` instead of stdin. False on both because Windows ARG_MAX (~32 KB) breaks `-p "<long-prompt>"` outright. |
-| `budgetCap` | ✓ | ✗ | `--max-budget-usd <n>`. |
-| `bareMode` | ✓ | ✗ | `--bare` (suppresses CLAUDE.md auto-discovery). Closest Copilot equivalent is `--no-custom-instructions`, gated separately. |
-| `fallbackModel` | ✓ | ✗ | `--fallback-model <id>` on rate-limit. |
-| `sessionPersistenceControl` | ✓ | ✗ | Engine writes `session.json`. Copilot manages session state internally in `~/.copilot/session-state/`. |
-| `resumePromptCarryover` | ✗ | ✓ | CC resume turns prepend recent visible Q&A in stdin because Copilot's session store is opaque to Minions. |
-| `resumeBookkeepingTurn` | ✓ | — | Claude CLI injects a synthetic "Continue from where you left off." meta turn on `--resume`; CC prompts must tell the model not to treat it as user intent. |
-| `streamConsumer` | ✓ | ✓ | Adapter implements `createStreamConsumer(ctx)` — required by `engine/llm.js` accumulator. |
+| Flag | Claude | Copilot | Codex | Gates |
+|------|--------|---------|-------|-------|
+| `streaming` | ✓ | ✓ | ✓ | JSONL events on stdout. |
+| `sessionResume` | ✓ | ✓ | ✓ | Runtime-specific resume args. |
+| `midRunSessionId` | ✓ | ✗ | ✗ | Resumable session ID emitted before the terminal `result` event; when false, steering waits for a checkpoint. |
+| `systemPromptFile` | ✓ | ✗ | ✗ | sysprompt via `--system-prompt-file` (else inlined into stdin by `buildPrompt`). |
+| `effortLevels` | ✓ | ✓ | ✓ | Runtime reasoning-effort controls. Codex maps through `--config model_reasoning_effort=...`; Copilot maps `'max'` to `'xhigh'`; Claude leaves `'max'` alone. |
+| `costTracking` | ✓ | ✗ | ✗ | USD + token counts in the result event. Copilot only emits `premiumRequests`; Codex accounting is not assumed. |
+| `modelShorthands` | ✓ | ✗ | ✗ | Bare `sonnet`/`opus`/`haiku` accepted by Claude only. |
+| `modelDiscovery` | ✗ | ✓ | ✓ | `listModels()` returns a real catalog when supported (Copilot API, Codex `debug models --bundled`). |
+| `promptViaArg` | ✗ | ✗ | ✗ | If true, prompt goes via `--prompt <text>` instead of stdin. False for all current runtimes. |
+| `budgetCap` | ✓ | ✗ | ✗ | `--max-budget-usd <n>`. |
+| `bareMode` | ✓ | ✗ | ✗ | `--bare` (suppresses CLAUDE.md auto-discovery). Other runtimes use their own config/suppression knobs. |
+| `fallbackModel` | ✓ | ✗ | ✗ | `--fallback-model <id>` on rate-limit. |
+| `sessionPersistenceControl` | ✓ | ✗ | ✗ | Engine writes `session.json`; Copilot and Codex own their runtime stores, while Minions records IDs only for safe resume. |
+| `resumePromptCarryover` | ✗ | ✓ | ✓ | CC resume turns prepend recent visible Q&A in stdin when the runtime session store is opaque to Minions. |
+| `resumeBookkeepingTurn` | ✓ | — | — | Claude CLI injects a synthetic "Continue from where you left off." meta turn on `--resume`; CC prompts must tell the model not to treat it as user intent. |
+| `streamConsumer` | ✓ | ✓ | ✓ | Adapter implements `createStreamConsumer(ctx)` — required by `engine/llm.js` accumulator. |
 
 When a behavior is genuinely uniform across all current adapters, it still gets
 a flag if a future runtime might disagree — flags are cheap.
@@ -165,6 +166,7 @@ contracts.
 |---------|---------------------|---------------------|-------------------------------|
 | Claude  | `~/.claude/skills`  | `<repo>/.claude/skills` | personal: `~/.claude/skills`; project: `<repo>/.claude/skills` |
 | Copilot | `~/.copilot/skills` (+ `~/.agents/skills`) | `<repo>/.github/skills`, `<repo>/.agents/skills` | personal: `~/.copilot/skills`; project: `<repo>/.github/skills` |
+| Codex | `~/.agents/skills` (+ `/etc/codex/skills`) | `<repo>/.agents/skills` | personal: `~/.agents/skills`; project: `<repo>/.agents/skills` |
 
 `getUserAssetDirs()` returns the union of runtime-native global roots and is
 passed to spawn as `--add-dir` so spawned agents read the same global assets the

package/engine/db/index.js

@@ -0,0 +1,141 @@
+// engine/db/index.js — Local SQLite state store, zero deps.
+//
+// Backed by Node's built-in `node:sqlite` module (stable in Node 22.5+,
+// matures further in 24.x). The whole project's "zero deps beyond Node
+// built-ins" contract stays intact.
+//
+// This file is intentionally tiny: open the DB, set sane pragmas, run
+// migrations, hand back a singleton. Everything else lives in callers.
+// Singleton (not per-require) so the WAL writer + dashboard reader share
+// one connection per process — better-sqlite3-style.
+//
+// Failure mode: if SQLite is unavailable (e.g. user on Node < 22.5 despite
+// our engines field bumping the minimum), `getDb()` throws with a clear
+// install-version error. Callers that want best-effort behaviour
+// (emitStateEvent, the dashboard's cache-version bump) should try/catch.
+
+const path = require('path');
+const fs = require('fs');
+
+let _db = null;
+let _dbPath = null;
+let _dbInitError = null;
+
+function _resolveDbPath() {
+  // Lazy-require shared/queries so this module can be safely required
+  // before MINIONS_DIR is computed (e.g. in tests). Falls back to
+  // process.env.MINIONS_HOME when available.
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  let minionsDir = envHome;
+  if (!minionsDir) {
+    try { minionsDir = require('../shared').MINIONS_DIR; } catch { /* shared not loaded */ }
+  }
+  if (!minionsDir) throw new Error('engine/db: MINIONS_DIR not resolvable');
+  const engineDir = path.join(minionsDir, 'engine');
+  try { fs.mkdirSync(engineDir, { recursive: true }); } catch { /* exists */ }
+  return path.join(engineDir, 'state.db');
+}
+
+// Suppress Node's ExperimentalWarning for `node:sqlite`. It fires once per
+// process the first time the module is required; without filtering it
+// leaks into every engine/dashboard log and CI run. Re-fires only for
+// non-sqlite experimental warnings.
+let _warningFilterInstalled = false;
+function _installExperimentalWarningFilter() {
+  if (_warningFilterInstalled) return;
+  _warningFilterInstalled = true;
+  const origEmit = process.emit;
+  process.emit = function (name, data, ...rest) {
+    if (name === 'warning' && data && data.name === 'ExperimentalWarning' && /SQLite/.test(String(data.message))) {
+      return false;
+    }
+    return origEmit.call(process, name, data, ...rest);
+  };
+}
+
+function getDb() {
+  // Re-resolve the DB path on every call so tests that swap MINIONS_TEST_DIR
+  // (or production-side configuration that changes MINIONS_HOME between
+  // operations) get a fresh connection to the new location instead of
+  // a stale handle pointing at a now-deleted tmpdir. The resolve itself
+  // is just env-var reads + path.join — sub-microsecond.
+  let dbPath;
+  try { dbPath = _resolveDbPath(); }
+  catch (e) {
+    _dbInitError = new Error(`engine/db: cannot resolve DB path — ${e.message}`);
+    throw _dbInitError;
+  }
+
+  if (_db) {
+    if (_dbPath === dbPath) return _db;
+    // Path changed (test isolation swap, or operator reconfigured
+    // MINIONS_HOME). Close the old handle and reopen against the new path.
+    try { _db.close(); } catch { /* already closed */ }
+    _db = null;
+    _dbPath = null;
+    _dbInitError = null;
+  }
+
+  if (_dbInitError) throw _dbInitError;
+  try {
+    _installExperimentalWarningFilter();
+    const { DatabaseSync } = require('node:sqlite');
+    _db = new DatabaseSync(dbPath);
+    _dbPath = dbPath;
+    // WAL mode lets the engine writer and dashboard reader hit the same
+    // file from two processes without lock contention. NORMAL synchronous
+    // is the recommended WAL pairing (durable enough; one OS-level fsync
+    // per checkpoint instead of per write).
+    _db.exec('PRAGMA journal_mode = WAL');
+    _db.exec('PRAGMA foreign_keys = ON');
+    _db.exec('PRAGMA synchronous = NORMAL');
+    _db.exec('PRAGMA busy_timeout = 5000');
+    const { runMigrations } = require('./migrate');
+    runMigrations(_db);
+    return _db;
+  } catch (e) {
+    _dbInitError = new Error(
+      `engine/db: failed to open SQLite — ${e.message}. ` +
+      `Node 22.5+ required for built-in 'node:sqlite' support.`
+    );
+    throw _dbInitError;
+  }
+}
+
+// Force-close — used by tests + graceful shutdown. Safe to call when not open.
+function closeDb() {
+  if (_db) { try { _db.close(); } catch { /* already closed */ } _db = null; }
+  _dbPath = null;
+  _dbInitError = null;
+}
+
+// Run `fn(db)` inside a transaction. Nestable: only the OUTERMOST call
+// issues BEGIN/COMMIT; inner calls run inside the enclosing tx so multiple
+// stores (dispatches + work_items + ...) can compose into one atomic
+// mutation. Inner rollback propagates outward, which is the correct
+// semantics for cross-store invariants — if a nested mutation fails, the
+// enclosing one must also fail.
+//
+// We track depth on the db instance itself rather than module-level so
+// reopening the singleton (test isolation, path changes) doesn't leak
+// stale depth into a fresh connection.
+function withTransaction(db, fn) {
+  if (!db) throw new Error('engine/db.withTransaction: db is required');
+  const isOuter = !db._minionsTxDepth;
+  db._minionsTxDepth = (db._minionsTxDepth || 0) + 1;
+  if (isOuter) db.exec('BEGIN IMMEDIATE');
+  try {
+    const result = fn(db);
+    db._minionsTxDepth -= 1;
+    if (db._minionsTxDepth === 0) db.exec('COMMIT');
+    return result;
+  } catch (e) {
+    db._minionsTxDepth -= 1;
+    if (db._minionsTxDepth === 0) {
+      try { db.exec('ROLLBACK'); } catch { /* best-effort */ }
+    }
+    throw e;
+  }
+}
+
+module.exports = { getDb, closeDb, withTransaction };

package/engine/db/migrate.js

@@ -0,0 +1,65 @@
+// engine/db/migrate.js — minimal versioned migration runner.
+//
+// Convention:
+//   migrations/<NNN>-<slug>.js exports { version, description, up(db, ctx) }
+//   - version is an integer matching the file's leading NNN
+//   - description is shown in the [db-migrate] log line
+//   - up(db, ctx) runs any DDL + data backfill; throwing rolls back the
+//     enclosing transaction (the rest of `up` never runs, schema_version
+//     is NOT bumped, the next startup retries).
+//
+// All migrations are wrapped in a single SQLite transaction. Either every
+// statement in `up` lands AND the schema_version row is written, or
+// nothing does. No partial-migration state can stick on disk.
+
+const path = require('path');
+const fs = require('fs');
+
+function _loadMigrations() {
+  const dir = path.join(__dirname, 'migrations');
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter(f => /^\d{3,}-[\w-]+\.js$/.test(f))
+    .sort()  // lexical sort works because of the fixed-width NNN prefix
+    .map(f => require(path.join(dir, f)));
+}
+
+function runMigrations(db) {
+  // schema_version is itself created via raw exec (chicken-and-egg with
+  // the migrations themselves).
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS schema_version (
+      version INTEGER PRIMARY KEY,
+      description TEXT,
+      applied_at INTEGER NOT NULL
+    )
+  `);
+
+  const currentRow = db.prepare('SELECT COALESCE(MAX(version), 0) AS v FROM schema_version').get();
+  const current = Number(currentRow.v) || 0;
+  const migrations = _loadMigrations();
+
+  for (const m of migrations) {
+    if (typeof m.version !== 'number' || typeof m.up !== 'function') {
+      throw new Error(`engine/db/migrate: migration missing { version, up }: ${JSON.stringify(Object.keys(m))}`);
+    }
+    if (m.version <= current) continue;
+    db.exec('BEGIN');
+    try {
+      // eslint-disable-next-line no-console
+      console.log(`[db-migrate] Applying v${m.version}: ${m.description || '(no description)'}`);
+      m.up(db, { fs, path });
+      db.prepare(
+        'INSERT INTO schema_version (version, description, applied_at) VALUES (?, ?, ?)'
+      ).run(m.version, m.description || '', Date.now());
+      db.exec('COMMIT');
+    } catch (e) {
+      db.exec('ROLLBACK');
+      // eslint-disable-next-line no-console
+      console.error(`[db-migrate] v${m.version} FAILED, rolled back: ${e.message}`);
+      throw e;
+    }
+  }
+}
+
+module.exports = { runMigrations };

package/engine/db/migrations/001-init.js

@@ -0,0 +1,37 @@
+// engine/db/migrations/001-init.js — first migration.
+//
+// Creates the `events` table: a single append-only event stream that
+// engine writers emit into and the dashboard's status cache reads
+// `MAX(id)` from for cross-process cache invalidation. Replaces the
+// mtime-tracker registry over the next few releases.
+//
+// Topic conventions (free-form text, but document the canonical list
+// here so future contributors don't fragment):
+//   - 'cache-invalidate'  — explicit dashboard.invalidateStatusCache() call
+//   - 'dispatch'          — engine/dispatch.json mutation (mutateDispatch)
+//   - 'work_items'        — engine/work-items.json or per-project file mutation
+//   - 'pull_requests'     — engine/pull-requests.json or per-project file mutation
+//   - 'prd'               — prd/*.json mutation
+//   - 'watches'           — watches.json mutation
+//   - 'config'            — config.json mutation
+//
+// Payload is optional JSON sidecar — currently unused by the dashboard
+// cache check (single MAX(id) is sufficient) but useful for future
+// debug tooling and topic-scoped subscribers.
+
+module.exports = {
+  version: 1,
+  description: 'init: events table + topic/ts indices',
+  up(db) {
+    db.exec(`
+      CREATE TABLE events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)),
+        topic TEXT NOT NULL,
+        payload TEXT
+      );
+      CREATE INDEX idx_events_topic_id ON events(topic, id DESC);
+      CREATE INDEX idx_events_ts ON events(ts DESC);
+    `);
+  },
+};

package/engine/db/migrations/002-dispatches.js

@@ -0,0 +1,113 @@
+// engine/db/migrations/002-dispatches.js
+//
+// Phase 1: move engine/dispatch.json into a `dispatches` table.
+//
+// Schema is intentionally hybrid: a small set of indexed projection columns
+// (status, agent, type, timestamps) for hot filters/sorts, plus a single
+// `data` TEXT column holding the full record JSON. This way the
+// `mutateDispatch(fn)` API contract — "mutator receives the section-shaped
+// object, mutates in place, returns" — round-trips every field cleanly
+// without us needing to enumerate every possible column ahead of time.
+// New fields the engine adds in a future commit land in `data`
+// automatically.
+//
+// Backfill: read engine/dispatch.json's four sections (pending / active /
+// completed / review), INSERT one row per dispatch with `status` derived
+// from the section name. Timestamps are parsed from ISO strings to
+// milliseconds since epoch (matches the rest of the codebase). The JSON
+// file is renamed to engine/dispatch.json.pre-sql-<ts> for rollback —
+// NOT deleted. A separate `minions db rollback-to-json` command (next
+// phase) will read the table and reconstruct the JSON if the user pins
+// to a pre-migration version.
+
+const path = require('path');
+const fs = require('fs');
+
+function _resolveMinionsDir() {
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  if (envHome) return envHome;
+  try { return require('../../shared').MINIONS_DIR; } catch { return null; }
+}
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+module.exports = {
+  version: 2,
+  description: 'dispatches: schema + dispatch.json backfill',
+  up(db) {
+    db.exec(`
+      CREATE TABLE dispatches (
+        id            TEXT PRIMARY KEY,
+        status        TEXT NOT NULL,
+        agent         TEXT,
+        type          TEXT,
+        created_at    INTEGER,
+        started_at    INTEGER,
+        completed_at  INTEGER,
+        data          TEXT NOT NULL,
+        updated_at    INTEGER NOT NULL
+      );
+      CREATE INDEX idx_dispatches_status            ON dispatches(status);
+      CREATE INDEX idx_dispatches_agent             ON dispatches(agent);
+      CREATE INDEX idx_dispatches_status_completed  ON dispatches(status, completed_at DESC);
+      CREATE INDEX idx_dispatches_status_created    ON dispatches(status, created_at DESC);
+    `);
+
+    const minionsDir = _resolveMinionsDir();
+    if (!minionsDir) return;
+    const dispatchPath = path.join(minionsDir, 'engine', 'dispatch.json');
+    if (!fs.existsSync(dispatchPath)) return;
+
+    let raw;
+    try { raw = JSON.parse(fs.readFileSync(dispatchPath, 'utf8')); }
+    catch (e) {
+      // Corrupt JSON — abort the migration, surfacing the error. The user
+      // can fix the file (or restore from the .backup sidecar safeWrite
+      // keeps) and re-run the engine; the migration will retry.
+      throw new Error(`engine/db/002-dispatches: cannot parse dispatch.json: ${e.message}`);
+    }
+
+    const now = Date.now();
+    const insert = db.prepare(`
+      INSERT INTO dispatches (id, status, agent, type, created_at, started_at, completed_at, data, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    let inserted = 0;
+    for (const status of ['pending', 'active', 'completed', 'review']) {
+      const rows = raw[status];
+      if (!Array.isArray(rows)) continue;
+      for (const d of rows) {
+        if (!d || typeof d !== 'object' || !d.id) continue;
+        insert.run(
+          String(d.id),
+          status,
+          d.agent || null,
+          d.type || null,
+          _toMs(d.created_at),
+          _toMs(d.started_at),
+          _toMs(d.completed_at),
+          JSON.stringify(d),
+          now,
+        );
+        inserted += 1;
+      }
+    }
+
+    // dispatch.json stays on disk as a dual-write mirror — every successful
+    // mutateDispatch refreshes it from SQL via dispatch-store._mirrorJsonFromSql.
+    // SQL is the source of truth; the file is a read-only derivative kept
+    // for backward compatibility with legacy direct-readers (engine/routing.js,
+    // engine/queries.js fallback, test infrastructure). Rollback path: if a
+    // user pins back to a pre-Phase-1 release, dispatch.json already contains
+    // the latest committed state (last mirror write), and the SQL table can
+    // be discarded.
+    // eslint-disable-next-line no-console
+    console.log(`[db-migrate] v2: backfilled ${inserted} dispatches; dispatch.json kept as dual-write mirror`);
+  },
+};

package/engine/db/migrations/003-work-items.js

@@ -0,0 +1,128 @@
+// engine/db/migrations/003-work-items.js
+//
+// Phase 2: move work-items.json (central + per-project) into a `work_items`
+// table.
+//
+// Same hybrid schema as `dispatches`: typed projection columns for hot
+// filters (scope/status/agent/parent), plus a `data` TEXT column holding
+// the full record JSON so future fields land transparently.
+//
+// `scope` keys the file the record came from: 'central' for
+// <MINIONS_DIR>/work-items.json, otherwise the project name (matches the
+// directory under projects/<name>/). Per-scope writes diff against the
+// scope's rows so a mutation on one project's file never touches another's.
+//
+// Backfill: walk central + every projects/<name>/work-items.json. The
+// JSON files stay on disk as a dual-write mirror so legacy direct-readers
+// continue to function while we migrate them in Phase 2.5.
+
+const path = require('path');
+const fs = require('fs');
+
+function _resolveMinionsDir() {
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  if (envHome) return envHome;
+  try { return require('../../shared').MINIONS_DIR; } catch { return null; }
+}
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function _readJsonArray(filePath) {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+function _listProjectDirs(minionsDir) {
+  const projectsRoot = path.join(minionsDir, 'projects');
+  let entries;
+  try { entries = fs.readdirSync(projectsRoot, { withFileTypes: true }); }
+  catch { return []; }
+  const out = [];
+  for (const e of entries) {
+    if (!e.isDirectory()) continue;
+    if (e.name === '.archived') continue;
+    out.push(e.name);
+  }
+  return out;
+}
+
+module.exports = {
+  version: 3,
+  description: 'work_items: schema + work-items.json backfill (central + per-project)',
+  up(db) {
+    db.exec(`
+      CREATE TABLE work_items (
+        id            TEXT NOT NULL,
+        scope         TEXT NOT NULL,
+        status        TEXT NOT NULL,
+        type          TEXT,
+        agent         TEXT,
+        parent_id     TEXT,
+        created_at    INTEGER,
+        completed_at  INTEGER,
+        data          TEXT NOT NULL,
+        updated_at    INTEGER NOT NULL,
+        PRIMARY KEY (scope, id)
+      );
+      CREATE INDEX idx_wi_status        ON work_items(status);
+      CREATE INDEX idx_wi_scope_status  ON work_items(scope, status);
+      CREATE INDEX idx_wi_agent         ON work_items(agent);
+      CREATE INDEX idx_wi_parent        ON work_items(parent_id);
+    `);
+
+    const minionsDir = _resolveMinionsDir();
+    if (!minionsDir) return;
+
+    const insert = db.prepare(`
+      INSERT INTO work_items (id, scope, status, type, agent, parent_id, created_at, completed_at, data, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const now = Date.now();
+    let inserted = 0;
+
+    const backfillFile = (scope, filePath) => {
+      const rows = _readJsonArray(filePath);
+      for (const wi of rows) {
+        if (!wi || typeof wi !== 'object' || !wi.id) continue;
+        try {
+          insert.run(
+            String(wi.id),
+            scope,
+            String(wi.status || 'pending'),
+            wi.type || null,
+            wi.dispatched_to || wi.agent || null,
+            wi.parent_id || null,
+            _toMs(wi.created_at),
+            _toMs(wi.completed_at || wi.completedAt),
+            JSON.stringify(wi),
+            now,
+          );
+          inserted += 1;
+        } catch {
+          // Duplicate (scope, id) within the same file → skip. Real-world
+          // work-items.json doesn't carry dupes, but defensive: a corrupted
+          // file shouldn't abort the whole migration.
+        }
+      }
+    };
+
+    backfillFile('central', path.join(minionsDir, 'work-items.json'));
+    for (const projectName of _listProjectDirs(minionsDir)) {
+      backfillFile(projectName, path.join(minionsDir, 'projects', projectName, 'work-items.json'));
+    }
+
+    // eslint-disable-next-line no-console
+    console.log(`[db-migrate] v3: backfilled ${inserted} work items; work-items.json files kept as dual-write mirrors`);
+  },
+};