@yemi33/minions 0.1.2065 → 0.1.2066

package/dashboard.js

@@ -1637,6 +1637,23 @@ function _mtimeTrackedFiles() {
 }
 let _lastMtimes = {}; // { filePath: mtimeMs } — baseline since last build
 
+// MAX(events.id) at last rebuild. Phase 0 cache-version source running
+// alongside the mtime tracker — either signal busts the cache. Once the
+// engine state files migrate into SQL tables (Phase 1+) the mtime tracker
+// goes away and this becomes the sole invalidator. Reads are O(index seek).
+let _lastEventVersion = -1;
+function _getCurrentEventVersion() {
+  try {
+    const { getDb } = require('./engine/db');
+    const row = getDb().prepare('SELECT COALESCE(MAX(id), 0) AS v FROM events').get();
+    return Number(row.v) || 0;
+  } catch {
+    // DB unavailable (Node < 22.5, missing file, etc.) — return a sentinel that
+    // disables event-based cache busting. The mtime tracker still works.
+    return -1;
+  }
+}
+
 // Stat a tracked path with transient-error tolerance. ENOENT (file/dir doesn't
 // exist) is normal — fresh installs, deleted projects, empty PRD dirs all hit
 // this — and maps to 0 so the entry just doesn't bust the cache. EBUSY /
@@ -1693,6 +1710,11 @@ function invalidateStatusCache(_opts) {
   _statusCacheJson = null;
   _statusCacheGzip = null;
   _lastMtimes = {};
+  _lastEventVersion = -1;
+  // Emit a 'cache-invalidate' event row so any peer dashboard process
+  // sharing the SQLite file also sees the bust on its next poll.
+  // Best-effort — db-events swallows all errors.
+  try { require('./engine/db-events').emitStateEvent('cache-invalidate'); } catch { /* optional */ }
   // Tell any in-flight refreshStatusAsync() that its result is stale and must
   // not be published. Bumping the generation also forces the next ETag to
   // differ from anything a client already has cached.
@@ -2098,12 +2120,23 @@ function _markStatusCacheBuilt() {
 }
 
 function getStatus() {
-  // Steady-state fast path: cache present + no tracked file changed → return
-  // cached snapshot. Single mtime check covers both fast + slow tracker
-  // contributions (see _mtimeTrackedFiles).
+  // Steady-state fast path: cache present + neither signal moved → return
+  // cached snapshot. Two cache-version sources running side by side during
+  // Phase 0 migration:
+  //   1. _lastMtimes  — legacy file mtime tracker. Single check covers
+  //                     both former fast + slow registries (see _mtimeTrackedFiles).
+  //   2. _lastEventVersion — MAX(events.id) from engine/state.db.
+  //                     Bumped by emitStateEvent() in every mutator wrapper
+  //                     and by invalidateStatusCache() itself.
+  // Either signal advancing busts the cache. Once engine state files
+  // migrate into SQL tables (Phase 1+) the mtime tracker goes away and
+  // event-version becomes the sole invalidator.
+  const preBuildEventVersion = _getCurrentEventVersion();
   if (_statusCache) {
     const currMtimes = _getMtimes();
-    if (!_mtimesChanged(_lastMtimes, currMtimes)) return _statusCache;
+    if (preBuildEventVersion === _lastEventVersion && !_mtimesChanged(_lastMtimes, currMtimes)) {
+      return _statusCache;
+    }
   }
   // Stale or first-call: rebuild everything. Reload config first so newly-
   // added projects / agents land before the slice builders read them.
@@ -2118,6 +2151,7 @@ function getStatus() {
   const slow = _buildStatusSlowState();
   _statusCache = { ...fast, ...slow, timestamp: new Date().toISOString() };
   _lastMtimes = preBuildMtimes;
+  _lastEventVersion = preBuildEventVersion;
   _markStatusCacheBuilt();
   return _statusCache;
 }
@@ -2185,10 +2219,11 @@ function refreshStatusAsync() {
     try {
       const startGeneration = _statusInvalidationGeneration;
 
-      // Steady-state fast path — same single mtime check as the sync getStatus.
+      // Steady-state fast path — same dual-signal check as the sync getStatus.
+      const preBuildEventVersion = _getCurrentEventVersion();
       if (_statusCache) {
         const currMtimes = _getMtimes();
-        if (!_mtimesChanged(_lastMtimes, currMtimes)) {
+        if (preBuildEventVersion === _lastEventVersion && !_mtimesChanged(_lastMtimes, currMtimes)) {
           if (profile) {
             _emitStatusTiming({
               phase: 'cache-hit',
@@ -2241,6 +2276,7 @@ function refreshStatusAsync() {
 
       _statusCache = { ...fast, ...slow, timestamp: new Date().toISOString() };
       _lastMtimes = preBuildMtimes;
+      _lastEventVersion = preBuildEventVersion;
       _markStatusCacheBuilt();
       if (profile) {
         _emitStatusTiming({
@@ -2277,6 +2313,7 @@ function _resetStatusCacheForTesting() {
   _statusInvalidationGeneration = 0;
   _statusRefreshHook = null;
   _lastMtimes = {};
+  _lastEventVersion = -1;
 }
 
 /** Return cached JSON string of status — single stringify, reused by SSE and /api/status */

package/engine/db/index.js

@@ -0,0 +1,94 @@
+// engine/db/index.js — Local SQLite state store, zero deps.
+//
+// Backed by Node's built-in `node:sqlite` module (stable in Node 22.5+,
+// matures further in 24.x). The whole project's "zero deps beyond Node
+// built-ins" contract stays intact.
+//
+// This file is intentionally tiny: open the DB, set sane pragmas, run
+// migrations, hand back a singleton. Everything else lives in callers.
+// Singleton (not per-require) so the WAL writer + dashboard reader share
+// one connection per process — better-sqlite3-style.
+//
+// Failure mode: if SQLite is unavailable (e.g. user on Node < 22.5 despite
+// our engines field bumping the minimum), `getDb()` throws with a clear
+// install-version error. Callers that want best-effort behaviour
+// (emitStateEvent, the dashboard's cache-version bump) should try/catch.
+
+const path = require('path');
+const fs = require('fs');
+
+let _db = null;
+let _dbInitError = null;
+
+function _resolveDbPath() {
+  // Lazy-require shared/queries so this module can be safely required
+  // before MINIONS_DIR is computed (e.g. in tests). Falls back to
+  // process.env.MINIONS_HOME when available.
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  let minionsDir = envHome;
+  if (!minionsDir) {
+    try { minionsDir = require('../shared').MINIONS_DIR; } catch { /* shared not loaded */ }
+  }
+  if (!minionsDir) throw new Error('engine/db: MINIONS_DIR not resolvable');
+  const engineDir = path.join(minionsDir, 'engine');
+  try { fs.mkdirSync(engineDir, { recursive: true }); } catch { /* exists */ }
+  return path.join(engineDir, 'state.db');
+}
+
+// Suppress Node's ExperimentalWarning for `node:sqlite`. It fires once per
+// process the first time the module is required; without filtering it
+// leaks into every engine/dashboard log and CI run. Re-fires only for
+// non-sqlite experimental warnings.
+let _warningFilterInstalled = false;
+function _installExperimentalWarningFilter() {
+  if (_warningFilterInstalled) return;
+  _warningFilterInstalled = true;
+  const origEmit = process.emit;
+  process.emit = function (name, data, ...rest) {
+    if (name === 'warning' && data && data.name === 'ExperimentalWarning' && /SQLite/.test(String(data.message))) {
+      return false;
+    }
+    return origEmit.call(process, name, data, ...rest);
+  };
+}
+
+function getDb() {
+  if (_db) return _db;
+  if (_dbInitError) throw _dbInitError;
+  try {
+    _installExperimentalWarningFilter();
+    const { DatabaseSync } = require('node:sqlite');
+    const dbPath = _resolveDbPath();
+    _db = new DatabaseSync(dbPath);
+    // WAL mode lets the engine writer and dashboard reader hit the same
+    // file from two processes without lock contention. NORMAL synchronous
+    // is the recommended WAL pairing (durable enough; one OS-level fsync
+    // per checkpoint instead of per write).
+    _db.exec('PRAGMA journal_mode = WAL');
+    _db.exec('PRAGMA foreign_keys = ON');
+    _db.exec('PRAGMA synchronous = NORMAL');
+    _db.exec('PRAGMA busy_timeout = 5000');
+    const { runMigrations } = require('./migrate');
+    runMigrations(_db);
+    return _db;
+  } catch (e) {
+    // Cache the failure so we don't repeatedly retry a broken setup. Most
+    // callers wrap getDb() in try/catch and treat the cache-version probe
+    // as best-effort, so a missing-Node-module install still boots the
+    // dashboard fine — it just won't benefit from cross-process cache
+    // invalidation until the install is fixed.
+    _dbInitError = new Error(
+      `engine/db: failed to open SQLite — ${e.message}. ` +
+      `Node 22.5+ required for built-in 'node:sqlite' support.`
+    );
+    throw _dbInitError;
+  }
+}
+
+// Force-close — used by tests + graceful shutdown. Safe to call when not open.
+function closeDb() {
+  if (_db) { try { _db.close(); } catch { /* already closed */ } _db = null; }
+  _dbInitError = null;
+}
+
+module.exports = { getDb, closeDb };

package/engine/db/migrate.js

@@ -0,0 +1,65 @@
+// engine/db/migrate.js — minimal versioned migration runner.
+//
+// Convention:
+//   migrations/<NNN>-<slug>.js exports { version, description, up(db, ctx) }
+//   - version is an integer matching the file's leading NNN
+//   - description is shown in the [db-migrate] log line
+//   - up(db, ctx) runs any DDL + data backfill; throwing rolls back the
+//     enclosing transaction (the rest of `up` never runs, schema_version
+//     is NOT bumped, the next startup retries).
+//
+// All migrations are wrapped in a single SQLite transaction. Either every
+// statement in `up` lands AND the schema_version row is written, or
+// nothing does. No partial-migration state can stick on disk.
+
+const path = require('path');
+const fs = require('fs');
+
+function _loadMigrations() {
+  const dir = path.join(__dirname, 'migrations');
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter(f => /^\d{3,}-[\w-]+\.js$/.test(f))
+    .sort()  // lexical sort works because of the fixed-width NNN prefix
+    .map(f => require(path.join(dir, f)));
+}
+
+function runMigrations(db) {
+  // schema_version is itself created via raw exec (chicken-and-egg with
+  // the migrations themselves).
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS schema_version (
+      version INTEGER PRIMARY KEY,
+      description TEXT,
+      applied_at INTEGER NOT NULL
+    )
+  `);
+
+  const currentRow = db.prepare('SELECT COALESCE(MAX(version), 0) AS v FROM schema_version').get();
+  const current = Number(currentRow.v) || 0;
+  const migrations = _loadMigrations();
+
+  for (const m of migrations) {
+    if (typeof m.version !== 'number' || typeof m.up !== 'function') {
+      throw new Error(`engine/db/migrate: migration missing { version, up }: ${JSON.stringify(Object.keys(m))}`);
+    }
+    if (m.version <= current) continue;
+    db.exec('BEGIN');
+    try {
+      // eslint-disable-next-line no-console
+      console.log(`[db-migrate] Applying v${m.version}: ${m.description || '(no description)'}`);
+      m.up(db, { fs, path });
+      db.prepare(
+        'INSERT INTO schema_version (version, description, applied_at) VALUES (?, ?, ?)'
+      ).run(m.version, m.description || '', Date.now());
+      db.exec('COMMIT');
+    } catch (e) {
+      db.exec('ROLLBACK');
+      // eslint-disable-next-line no-console
+      console.error(`[db-migrate] v${m.version} FAILED, rolled back: ${e.message}`);
+      throw e;
+    }
+  }
+}
+
+module.exports = { runMigrations };

package/engine/db/migrations/001-init.js

@@ -0,0 +1,37 @@
+// engine/db/migrations/001-init.js — first migration.
+//
+// Creates the `events` table: a single append-only event stream that
+// engine writers emit into and the dashboard's status cache reads
+// `MAX(id)` from for cross-process cache invalidation. Replaces the
+// mtime-tracker registry over the next few releases.
+//
+// Topic conventions (free-form text, but document the canonical list
+// here so future contributors don't fragment):
+//   - 'cache-invalidate'  — explicit dashboard.invalidateStatusCache() call
+//   - 'dispatch'          — engine/dispatch.json mutation (mutateDispatch)
+//   - 'work_items'        — engine/work-items.json or per-project file mutation
+//   - 'pull_requests'     — engine/pull-requests.json or per-project file mutation
+//   - 'prd'               — prd/*.json mutation
+//   - 'watches'           — watches.json mutation
+//   - 'config'            — config.json mutation
+//
+// Payload is optional JSON sidecar — currently unused by the dashboard
+// cache check (single MAX(id) is sufficient) but useful for future
+// debug tooling and topic-scoped subscribers.
+
+module.exports = {
+  version: 1,
+  description: 'init: events table + topic/ts indices',
+  up(db) {
+    db.exec(`
+      CREATE TABLE events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)),
+        topic TEXT NOT NULL,
+        payload TEXT
+      );
+      CREATE INDEX idx_events_topic_id ON events(topic, id DESC);
+      CREATE INDEX idx_events_ts ON events(ts DESC);
+    `);
+  },
+};

package/engine/db-events.js

@@ -0,0 +1,40 @@
+// engine/db-events.js — `emitStateEvent(topic, payload?)` helper.
+//
+// Append a row to the `events` table so the dashboard's status cache (and
+// any future topic-scoped subscriber) sees the change on its next poll.
+// Designed for the engine-side mutator wrappers (mutateDispatch,
+// mutateWorkItems, mutatePullRequests, …) to call after a successful
+// write — the dashboard's getStatus polls MAX(events.id) as its single
+// cache-version source of truth.
+//
+// Hard contract: this MUST be best-effort. Engine writers throwing from
+// a cache-invalidation side effect would be a serious regression. Any
+// SQLite failure (Node too old, DB locked, disk full) is swallowed.
+// The mtime tracker remains as a fallback while we transition.
+
+let _logged = false;
+
+function emitStateEvent(topic, payload) {
+  try {
+    const { getDb } = require('./db');
+    const db = getDb();
+    if (payload !== undefined && payload !== null) {
+      db.prepare('INSERT INTO events (topic, payload) VALUES (?, ?)')
+        .run(String(topic), JSON.stringify(payload));
+    } else {
+      db.prepare('INSERT INTO events (topic) VALUES (?)').run(String(topic));
+    }
+  } catch (e) {
+    // Log once per process so the operator notices a broken install
+    // without spamming on every write.
+    if (!_logged) {
+      _logged = true;
+      try {
+        // eslint-disable-next-line no-console
+        console.warn(`[db-events] state event emission disabled: ${e.message}`);
+      } catch { /* console unavailable in some test seams */ }
+    }
+  }
+}
+
+module.exports = { emitStateEvent };

package/engine/dispatch.js

@@ -70,7 +70,12 @@ function mutateDispatch(mutator) {
     // so a 100-item status-flip doesn't re-byte-count every prompt.
     _sidecarChangedPrompts(next, prevSnap);
     return next;
-  }, { defaultValue: defaultDispatch });
+  }, {
+    defaultValue: defaultDispatch,
+    onWrote: () => {
+      try { require('./db-events').emitStateEvent('dispatch'); } catch { /* optional */ }
+    },
+  });
   // Invalidate the read cache so next getDispatch() sees fresh data
   try { require('./queries').invalidateDispatchCache(); } catch {}
   return result;

package/engine/shared.js

@@ -1088,7 +1088,8 @@ function mutateJsonFileLocked(filePath, mutateFn, {
   defaultValue = {},
   lockRetries,
   lockRetryBackoffMs,
-  skipWriteIfUnchanged = false
+  skipWriteIfUnchanged = false,
+  onWrote = null,
 } = {}) {
   const lockPath = `${filePath}.lock`;
   const retries = lockRetries ?? ENGINE_DEFAULTS.lockRetries;
@@ -1110,6 +1111,13 @@ function mutateJsonFileLocked(filePath, mutateFn, {
       const backupPath = filePath + '.backup';
       try { if (fileExists) fs.copyFileSync(filePath, backupPath); } catch { /* backup is best-effort */ }
       safeWrite(filePath, finalData);
+      // Side-effect hook fired only when an actual write happened. Callers
+      // use this to emit cache-invalidation signals (events table row) so
+      // skip-write-unchanged paths (dedup, idempotent no-op POSTs) don't
+      // produce spurious cache busts.
+      if (typeof onWrote === 'function') {
+        try { onWrote(finalData); } catch { /* hook errors must not break the write */ }
+      }
     }
     return finalData;
   }, { retries, retryBackoffMs });
@@ -4567,7 +4575,17 @@ function mutateWorkItems(filePath, mutator) {
   const result = mutateJsonFileLocked(filePath, (data) => {
     if (!Array.isArray(data)) data = [];
     return mutator(data) || data;
-  }, { defaultValue: [], skipWriteIfUnchanged: true });
+  }, {
+    defaultValue: [],
+    skipWriteIfUnchanged: true,
+    // Emit only when an actual write happened. The dedup path through
+    // createWorkItemWithDedup returns the array unchanged → skipWriteIfUnchanged
+    // suppresses the write, and we want to suppress the event too (otherwise
+    // duplicate POSTs bump the cache version for no observable change).
+    onWrote: () => {
+      try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
+    },
+  });
   // Invalidate the read cache so the next getWorkItems() sees fresh data
   // (W-mpodww9h000b460a). Mirrors dispatch.js's invalidateDispatchCache
   // call after mutateDispatch. Lazy-required to avoid the queries→shared
@@ -4599,7 +4617,12 @@ function mutatePullRequests(filePath, mutator) {
   return mutateJsonFileLocked(filePath, (data) => {
     if (!Array.isArray(data)) data = [];
     return mutator(data) || data;
-  }, { defaultValue: [] });
+  }, {
+    defaultValue: [],
+    onWrote: () => {
+      try { require('./db-events').emitStateEvent('pull_requests'); } catch { /* optional */ }
+    },
+  });
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2065",
+  "version": "0.1.2066",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"
@@ -60,7 +60,7 @@
   },
   "homepage": "https://yemi33.github.io/minions/",
   "engines": {
-    "node": ">=18"
+    "node": ">=22.5"
   },
   "dependencies": {
     "@azure-devops/mcp": "2.7.0"