@yemi33/minions 0.1.2067 → 0.1.2069

package/engine/db/migrations/004-pull-requests.js

@@ -0,0 +1,129 @@
+// engine/db/migrations/004-pull-requests.js
+//
+// Phase 3: move pull-requests.json (central + per-project) into a
+// `pull_requests` table.
+//
+// Same hybrid schema as `work_items`: typed projection columns for hot
+// filters (scope/status/agent/branch/timestamps), plus a `data` TEXT
+// column holding the full record JSON so future fields land transparently.
+//
+// `scope` keys the file the record came from: 'central' for
+// <MINIONS_DIR>/pull-requests.json, otherwise the project name (matches
+// the directory under projects/<name>/). Per-scope writes diff against
+// that scope's rows so a mutation on one project's file never touches
+// another's.
+//
+// Backfill: walk central + every projects/<name>/pull-requests.json. The
+// JSON files stay on disk as a dual-write mirror so legacy direct-readers
+// continue to function while we migrate them in Phase 3.5.
+
+const path = require('path');
+const fs = require('fs');
+
+function _resolveMinionsDir() {
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  if (envHome) return envHome;
+  try { return require('../../shared').MINIONS_DIR; } catch { return null; }
+}
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function _readJsonArray(filePath) {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+function _listProjectDirs(minionsDir) {
+  const projectsRoot = path.join(minionsDir, 'projects');
+  let entries;
+  try { entries = fs.readdirSync(projectsRoot, { withFileTypes: true }); }
+  catch { return []; }
+  const out = [];
+  for (const e of entries) {
+    if (!e.isDirectory()) continue;
+    if (e.name.startsWith('.')) continue;  // .archived + any other hidden sidecars
+    out.push(e.name);
+  }
+  return out;
+}
+
+module.exports = {
+  version: 4,
+  description: 'pull_requests: schema + pull-requests.json backfill (central + per-project)',
+  up(db) {
+    db.exec(`
+      CREATE TABLE pull_requests (
+        id              TEXT NOT NULL,
+        scope           TEXT NOT NULL,
+        status          TEXT NOT NULL,
+        review_status   TEXT,
+        agent           TEXT,
+        branch          TEXT,
+        pr_number       INTEGER,
+        created_at      INTEGER,
+        last_polled_at  INTEGER,
+        data            TEXT NOT NULL,
+        updated_at      INTEGER NOT NULL,
+        PRIMARY KEY (scope, id)
+      );
+      CREATE INDEX idx_pr_status            ON pull_requests(status);
+      CREATE INDEX idx_pr_scope_status      ON pull_requests(scope, status);
+      CREATE INDEX idx_pr_agent             ON pull_requests(agent);
+      CREATE INDEX idx_pr_branch            ON pull_requests(branch);
+    `);
+
+    const minionsDir = _resolveMinionsDir();
+    if (!minionsDir) return;
+
+    const insert = db.prepare(`
+      INSERT INTO pull_requests (id, scope, status, review_status, agent, branch, pr_number, created_at, last_polled_at, data, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const now = Date.now();
+    let inserted = 0;
+
+    const backfillFile = (scope, filePath) => {
+      const rows = _readJsonArray(filePath);
+      for (const pr of rows) {
+        if (!pr || typeof pr !== 'object' || !pr.id) continue;
+        try {
+          insert.run(
+            String(pr.id),
+            scope,
+            String(pr.status || 'active'),
+            pr.reviewStatus || null,
+            pr.agent || null,
+            pr.branch || null,
+            Number.isFinite(pr.prNumber) ? pr.prNumber : null,
+            _toMs(pr.created),
+            _toMs(pr.lastPolledAt || pr.lastPolled),
+            JSON.stringify(pr),
+            now,
+          );
+          inserted += 1;
+        } catch {
+          // Duplicate (scope, id) — skip. Defensive against malformed files.
+        }
+      }
+    };
+
+    backfillFile('central', path.join(minionsDir, 'pull-requests.json'));
+    for (const projectName of _listProjectDirs(minionsDir)) {
+      backfillFile(projectName, path.join(minionsDir, 'projects', projectName, 'pull-requests.json'));
+    }
+
+    // eslint-disable-next-line no-console
+    console.log(`[db-migrate] v4: backfilled ${inserted} pull requests; pull-requests.json files kept as dual-write mirrors`);
+  },
+};

package/engine/db/migrations/005-logs.js

@@ -0,0 +1,39 @@
+// engine/db/migrations/005-logs.js
+//
+// Phase 4: replace engine/log.json with a `logs` table.
+//
+// Logs are append-only with a 2500-entry ring buffer (the legacy JSON
+// path trimmed to 2000 when length hit 2500). The table preserves that
+// behaviour:
+//   - Each log() emits one row via INSERT.
+//   - After insert, a tail-trim DELETEs everything older than the most
+//     recent N (default 2500, configurable via ENGINE_DEFAULTS.logCapEntries).
+//
+// Schema is intentionally narrow: an auto-incrementing id (preserves
+// insertion order for the ring-buffer trim), the timestamp in ms (so the
+// dashboard can do range queries without a TEXT parse), a level/message
+// pair the dashboard already keys on, and a `meta` JSON blob for any
+// extra fields the caller passed.
+//
+// No backfill — engine/log.json is ephemeral and the next tick will
+// start populating SQL fresh. The legacy file stays on disk as a
+// dual-write mirror (test fixtures + getEngineLog fallback) until
+// Phase 4.5 migrates the readers.
+
+module.exports = {
+  version: 5,
+  description: 'logs: append-only table for engine log entries',
+  up(db) {
+    db.exec(`
+      CREATE TABLE logs (
+        id        INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts_ms     INTEGER NOT NULL,
+        level     TEXT NOT NULL,
+        message   TEXT NOT NULL,
+        meta      TEXT
+      );
+      CREATE INDEX idx_logs_ts        ON logs(ts_ms DESC);
+      CREATE INDEX idx_logs_level_ts  ON logs(level, ts_ms DESC);
+    `);
+  },
+};

package/engine/logs-store.js

@@ -0,0 +1,190 @@
+// engine/logs-store.js — SQL-backed implementation of engine/log.json.
+//
+// Logs differ from work_items / pull_requests / dispatches: they are
+// pure append-only with a tail-cap, so the store exposes a different
+// shape:
+//
+//   appendLogsToDbFile(dbFilePath, entries)
+//     → bulk INSERT into the `logs` table at the given state.db path,
+//       then trim the head if the table exceeded the cap. Routes per
+//       dbFilePath so test isolation (each MINIONS_TEST_DIR has its own
+//       state.db) keeps working — the legacy `_logPath` capture-at-write
+//       pattern was the only correct way to keep test logs out of
+//       production storage, and we preserve the same idea here.
+//
+//   readRecentLogs(n, { level? })
+//     → SELECT ... ORDER BY id DESC LIMIT n against the singleton DB.
+//       Honors MINIONS_TEST_DIR / MINIONS_HOME automatically through
+//       engine/db.getDb().
+//
+// All writes happen through the per-path appender; the singleton is only
+// used for reads. This way a deferred buffer flush after a test ends
+// can't accidentally drop entries into the production state.db.
+
+const fs = require('fs');
+const path = require('path');
+
+// Per-dbPath connection cache. Avoids close+reopen thrash on the
+// singleton when buffered entries are routed to multiple test state.db
+// files in quick succession. Connections are closed via
+// closeAllLogWriterConnections() at process shutdown.
+const _connCache = new Map();
+
+function _ensureDir(filePath) {
+  try { fs.mkdirSync(path.dirname(filePath), { recursive: true }); } catch { /* exists */ }
+}
+
+function _openWriter(dbFilePath) {
+  let conn = _connCache.get(dbFilePath);
+  if (conn) return conn;
+  _ensureDir(dbFilePath);
+  // Use the same migration runner / pragma setup as the singleton, but
+  // against an explicit path. Re-running runMigrations against an
+  // already-migrated DB is a no-op (the version row gates it), so this
+  // is safe regardless of whether the singleton has touched the file.
+  const { DatabaseSync } = require('node:sqlite');
+  const db = new DatabaseSync(dbFilePath);
+  db.exec('PRAGMA journal_mode = WAL');
+  db.exec('PRAGMA foreign_keys = ON');
+  db.exec('PRAGMA synchronous = NORMAL');
+  db.exec('PRAGMA busy_timeout = 5000');
+  const { runMigrations } = require('./db/migrate');
+  runMigrations(db);
+  conn = {
+    db,
+    insertStmt: db.prepare('INSERT INTO logs (ts_ms, level, message, meta) VALUES (?, ?, ?, ?)'),
+    countStmt: db.prepare('SELECT COUNT(*) AS n FROM logs'),
+  };
+  _connCache.set(dbFilePath, conn);
+  return conn;
+}
+
+function _toMs(v) {
+  if (v == null) return Date.now();
+  if (typeof v === 'number') return Number.isFinite(v) ? v : Date.now();
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : Date.now();
+}
+
+// Strip the routing-only fields (timestamp, level, message, _logPath,
+// _dbPath) so `meta` only contains caller-provided extras.
+const _BUILTIN_KEYS = new Set(['timestamp', 'level', 'message', '_logPath', '_dbPath']);
+function _extractMeta(entry) {
+  const meta = {};
+  for (const k of Object.keys(entry)) {
+    if (_BUILTIN_KEYS.has(k)) continue;
+    meta[k] = entry[k];
+  }
+  return Object.keys(meta).length > 0 ? JSON.stringify(meta) : null;
+}
+
+// Append a batch of log entries to a specific state.db file.
+//
+// Bulk-wrapped in a transaction so a partial failure rolls back cleanly
+// and disk fsync only fires once per batch (matters at the 50-entry
+// buffer threshold under high load).
+//
+// After insert, if the table exceeds the ring-buffer cap, trim the
+// oldest entries down to logCapTrimTo. Trim is bounded — only runs when
+// we've actually overflowed, not on every batch.
+function appendLogsToDbFile(dbFilePath, entries) {
+  if (!entries || entries.length === 0) return 0;
+  let conn;
+  try { conn = _openWriter(dbFilePath); }
+  catch (e) {
+    // Logging must never crash the caller — swallow and bail.
+    return 0;
+  }
+
+  const shared = require('./shared');
+  const cap = (shared.ENGINE_DEFAULTS && shared.ENGINE_DEFAULTS.logCapEntries) || 2500;
+  const trimTo = (shared.ENGINE_DEFAULTS && shared.ENGINE_DEFAULTS.logCapTrimTo) || 2000;
+
+  let inserted = 0;
+  try {
+    conn.db.exec('BEGIN IMMEDIATE');
+    for (const entry of entries) {
+      const tsMs = _toMs(entry.timestamp);
+      const level = String(entry.level || 'info');
+      const message = String(entry.message != null ? entry.message : '');
+      const meta = _extractMeta(entry);
+      conn.insertStmt.run(tsMs, level, message, meta);
+      inserted += 1;
+    }
+    const total = conn.countStmt.get().n;
+    if (total > cap) {
+      // Trim the oldest rows. `id` is monotonic via AUTOINCREMENT, so
+      // ORDER BY id ASC + LIMIT (total - trimTo) deletes exactly the
+      // overflow. Single DELETE; no rowid scan required.
+      const toRemove = total - trimTo;
+      conn.db.prepare(`
+        DELETE FROM logs
+        WHERE id IN (SELECT id FROM logs ORDER BY id ASC LIMIT ?)
+      `).run(toRemove);
+    }
+    conn.db.exec('COMMIT');
+  } catch {
+    try { conn.db.exec('ROLLBACK'); } catch { /* best-effort */ }
+    return 0;
+  }
+
+  // Emit a single events row so dashboards / cache invalidators wake up
+  // once per batch instead of per entry. Routed against the singleton
+  // (not per-dbPath) because events are a cross-cutting concern keyed
+  // off the engine's active state.db.
+  try { require('./db-events').emitStateEvent('logs', { batch: inserted }); } catch { /* optional */ }
+  return inserted;
+}
+
+// Read the most recent N entries from the singleton DB (test-aware
+// because getDb() re-resolves MINIONS_TEST_DIR on every call).
+//
+// Returns entries in NEW-EST-FIRST order — matches what the dashboard
+// expects when slicing the tail of the array. Optional `level` filter
+// uses the level index.
+function readRecentLogs(n, opts) {
+  const limit = Math.max(1, Math.min(10000, Number(n) || 50));
+  const level = opts && opts.level ? String(opts.level) : null;
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return []; }
+  const rows = level
+    ? db.prepare('SELECT ts_ms, level, message, meta FROM logs WHERE level = ? ORDER BY id DESC LIMIT ?').all(level, limit)
+    : db.prepare('SELECT ts_ms, level, message, meta FROM logs ORDER BY id DESC LIMIT ?').all(limit);
+
+  const out = [];
+  for (const row of rows) {
+    const entry = {
+      timestamp: new Date(row.ts_ms).toISOString(),
+      level: row.level,
+      message: row.message,
+    };
+    if (row.meta) {
+      try { Object.assign(entry, JSON.parse(row.meta)); } catch { /* skip malformed meta */ }
+    }
+    out.push(entry);
+  }
+  return out;
+}
+
+// Convenience: same shape as the legacy `JSON.parse(log.json).slice(-N)`
+// — chronological order (oldest first). Inverts the readRecentLogs DESC
+// output so existing callers can drop in without re-sorting.
+function readRecentLogsChronological(n, opts) {
+  return readRecentLogs(n, opts).reverse();
+}
+
+function closeAllLogWriterConnections() {
+  for (const [, conn] of _connCache) {
+    try { conn.db.close(); } catch { /* already closed */ }
+  }
+  _connCache.clear();
+}
+
+module.exports = {
+  appendLogsToDbFile,
+  readRecentLogs,
+  readRecentLogsChronological,
+  closeAllLogWriterConnections,
+};

package/engine/pull-requests-store.js

@@ -0,0 +1,286 @@
+// engine/pull-requests-store.js — SQL-backed implementation of the
+// per-file pull-requests array. Same external contract as the legacy
+// JSON-file reader, modeled directly on work-items-store.js.
+//
+//   readPullRequestsForScope(scope)        -> [pr, pr, ...]
+//   applyPullRequestsMutation(scope, fn)   -> diff-then-apply,
+//                                             returns { wrote, result }
+//
+// `scope` is 'central' for the top-level pull-requests.json or a project
+// name for projects/<name>/pull-requests.json.
+//
+// External-edit detection mirrors work-items-store: per-scope last-mirror
+// mtime is recorded after every write; if the JSON file's mtime advances
+// outside our knowledge (test re-seed, operator hand-edit), we treat the
+// JSON as canonical and rebuild SQL before the next read or mutation.
+
+const path = require('path');
+const fs = require('fs');
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function scopeForFilePath(filePath) {
+  const norm = String(filePath).replace(/\\/g, '/');
+  const m = norm.match(/projects\/([^/]+)\/pull-requests\.json$/);
+  if (m) return m[1];
+  return 'central';
+}
+
+function _filePathForScope(scope) {
+  const shared = require('./shared');
+  if (scope === 'central') {
+    return path.join(shared.MINIONS_DIR, 'pull-requests.json');
+  }
+  return path.join(shared.MINIONS_DIR, 'projects', scope, 'pull-requests.json');
+}
+
+function _parseRow(row) {
+  if (!row || !row.data) return null;
+  try { return JSON.parse(row.data); }
+  catch { return null; }
+}
+
+function _readJsonArrayFallback(scope) {
+  const fp = _filePathForScope(scope);
+  let raw;
+  try { raw = fs.readFileSync(fp, 'utf8'); }
+  catch { return []; }
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch (e) {
+    try {
+      // eslint-disable-next-line no-console
+      console.warn(`[pull-requests-store] corrupt JSON in ${fp}: ${e.message}`);
+    } catch { /* console may be wrapped in tests */ }
+    return [];
+  }
+}
+
+const _lastMirrorMtimeByScope = new Map();
+
+function _statMtimeMs(filePath) {
+  try { return fs.statSync(filePath).mtimeMs; }
+  catch { return null; }
+}
+
+function _hydrateScopeFromJson(db, scope) {
+  const jsonItems = _readJsonArrayFallback(scope);
+  db.prepare('DELETE FROM pull_requests WHERE scope = ?').run(scope);
+  if (jsonItems.length === 0) return;
+  const now = Date.now();
+  const ins = db.prepare(`
+    INSERT INTO pull_requests (id, scope, status, review_status, agent, branch, pr_number, created_at, last_polled_at, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(scope, id) DO NOTHING
+  `);
+  for (const pr of jsonItems) {
+    if (!pr || !pr.id) continue;
+    ins.run(
+      String(pr.id),
+      scope,
+      String(pr.status || 'active'),
+      pr.reviewStatus || null,
+      pr.agent || null,
+      pr.branch || null,
+      Number.isFinite(pr.prNumber) ? pr.prNumber : null,
+      _toMs(pr.created),
+      _toMs(pr.lastPolledAt || pr.lastPolled),
+      JSON.stringify(pr),
+      now,
+    );
+  }
+}
+
+function _resyncScopeIfJsonDiverged(db, scope) {
+  const jsonPath = _filePathForScope(scope);
+  const currentMtime = _statMtimeMs(jsonPath);
+  const lastMirrorMtime = _lastMirrorMtimeByScope.get(scope);
+  if (currentMtime == null) return;
+  if (lastMirrorMtime != null && Math.abs(currentMtime - lastMirrorMtime) <= 1) return;
+  if (lastMirrorMtime == null) {
+    const sqlHas = db.prepare('SELECT 1 FROM pull_requests WHERE scope = ? LIMIT 1').get(scope);
+    if (sqlHas) {
+      _lastMirrorMtimeByScope.set(scope, currentMtime);
+      return;
+    }
+  }
+  _hydrateScopeFromJson(db, scope);
+  _lastMirrorMtimeByScope.set(scope, currentMtime);
+}
+
+function readPullRequestsForScope(scope) {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return _readJsonArrayFallback(scope); }
+
+  _resyncScopeIfJsonDiverged(db, scope);
+
+  const rows = db.prepare(`
+    SELECT data FROM pull_requests
+    WHERE scope = ?
+    ORDER BY rowid
+  `).all(scope);
+
+  if (rows.length === 0) {
+    const fallback = _readJsonArrayFallback(scope);
+    if (fallback.length > 0) return fallback;
+    return [];
+  }
+
+  const out = [];
+  for (const row of rows) {
+    const pr = _parseRow(row);
+    if (pr) out.push(pr);
+  }
+  return out;
+}
+
+function readAllPullRequests() {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return null; }
+
+  try { _resyncScopeIfJsonDiverged(db, 'central'); } catch {}
+  const knownScopes = db.prepare('SELECT DISTINCT scope FROM pull_requests').all().map(r => r.scope);
+  for (const s of knownScopes) {
+    if (s === 'central') continue;
+    try { _resyncScopeIfJsonDiverged(db, s); } catch {}
+  }
+
+  const rows = db.prepare('SELECT data, scope FROM pull_requests ORDER BY rowid').all();
+  const out = [];
+  for (const row of rows) {
+    const pr = _parseRow(row);
+    if (!pr) continue;
+    pr._scope = row.scope;
+    out.push(pr);
+  }
+  return out;
+}
+
+function _indexById(arr) {
+  const out = new Map();
+  for (const pr of arr) {
+    if (!pr || !pr.id) continue;
+    out.set(String(pr.id), pr);
+  }
+  return out;
+}
+
+function _computePullRequestsDiff(scope, before, after) {
+  const beforeMap = _indexById(before);
+  const afterMap = _indexById(after);
+  const toUpsert = [];
+  const toDelete = [];
+  for (const [id, pr] of afterMap) {
+    const prev = beforeMap.get(id);
+    if (!prev) { toUpsert.push(pr); continue; }
+    if (JSON.stringify(prev) !== JSON.stringify(pr)) toUpsert.push(pr);
+  }
+  for (const [id] of beforeMap) {
+    if (!afterMap.has(id)) toDelete.push(id);
+  }
+  return { scope, toUpsert, toDelete };
+}
+
+function _applyPullRequestsDiff(db, diff) {
+  if (diff.toUpsert.length === 0 && diff.toDelete.length === 0) return false;
+  const now = Date.now();
+  const upsertStmt = db.prepare(`
+    INSERT INTO pull_requests (id, scope, status, review_status, agent, branch, pr_number, created_at, last_polled_at, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(scope, id) DO UPDATE SET
+      status         = excluded.status,
+      review_status  = excluded.review_status,
+      agent          = excluded.agent,
+      branch         = excluded.branch,
+      pr_number      = excluded.pr_number,
+      created_at     = excluded.created_at,
+      last_polled_at = excluded.last_polled_at,
+      data           = excluded.data,
+      updated_at     = excluded.updated_at
+  `);
+  const deleteStmt = db.prepare('DELETE FROM pull_requests WHERE scope = ? AND id = ?');
+
+  for (const pr of diff.toUpsert) {
+    upsertStmt.run(
+      String(pr.id),
+      diff.scope,
+      String(pr.status || 'active'),
+      pr.reviewStatus || null,
+      pr.agent || null,
+      pr.branch || null,
+      Number.isFinite(pr.prNumber) ? pr.prNumber : null,
+      _toMs(pr.created),
+      _toMs(pr.lastPolledAt || pr.lastPolled),
+      JSON.stringify(pr),
+      now,
+    );
+  }
+  for (const id of diff.toDelete) {
+    deleteStmt.run(diff.scope, id);
+  }
+  return true;
+}
+
+function applyPullRequestsMutation(scope, mutator) {
+  const { getDb, withTransaction } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch (e) {
+    throw new Error(`engine/pull-requests-store: SQLite unavailable (${e.message}); cannot mutate pull_requests`);
+  }
+
+  return withTransaction(db, () => {
+    _resyncScopeIfJsonDiverged(db, scope);
+    const before = readPullRequestsForScope(scope);
+    // Strip the _scope decoration that readAllPullRequests adds, in case
+    // a caller round-trips records from there back through here.
+    for (const pr of before) { if (pr && pr._scope) delete pr._scope; }
+    const beforeSnapshot = JSON.parse(JSON.stringify(before));
+    const next = mutator(before);
+    const after = (next === undefined || next === null)
+      ? before
+      : (Array.isArray(next) ? next : before);
+    const diff = _computePullRequestsDiff(scope, beforeSnapshot, after);
+    const wrote = _applyPullRequestsDiff(db, diff);
+    return { wrote, result: after };
+  });
+}
+
+function _mirrorJsonFromSql(scope, filePath) {
+  try {
+    const shared = require('./shared');
+    const items = readPullRequestsForScope(scope);
+    for (const pr of items) { if (pr && pr._scope) delete pr._scope; }
+    const target = filePath || _filePathForScope(scope);
+    shared.safeWrite(target, items);
+    const m = _statMtimeMs(target);
+    if (m != null) _lastMirrorMtimeByScope.set(scope, m);
+  } catch {
+    // Mirror failures are non-fatal — SQL has already committed.
+  }
+}
+
+function _resetPullRequestsTableForTest() {
+  const { getDb } = require('./db');
+  try { getDb().exec('DELETE FROM pull_requests'); } catch { /* not initialized */ }
+}
+
+module.exports = {
+  scopeForFilePath,
+  readPullRequestsForScope,
+  readAllPullRequests,
+  applyPullRequestsMutation,
+  _filePathForScope,
+  _mirrorJsonFromSql,
+  _resetPullRequestsTableForTest,
+};

package/engine/queries.js

@@ -327,8 +327,17 @@ function getNotesWithMeta() {
 }
 
 function getEngineLog() {
-  // Use the lazy log-path resolver so test isolation (MINIONS_TEST_DIR) is
-  // honored even when this module's require cache wasn't busted.
+  // Phase 4: prefer the SQL store. readRecentLogsChronological returns
+  // entries oldest-first (same shape as the legacy `.slice(-50)`), and is
+  // test-aware via engine/db.getDb() which re-resolves MINIONS_TEST_DIR.
+  // Falls back to the JSON mirror on SQL failure or when the table is
+  // empty (fresh install / pre-migration).
+  try {
+    const store = require('./logs-store');
+    const sqlEntries = store.readRecentLogsChronological(50);
+    if (Array.isArray(sqlEntries) && sqlEntries.length > 0) return sqlEntries;
+  } catch { /* fall through to JSON */ }
+
   const logJson = safeRead(shared.currentLogPath());
   if (!logJson) return [];
   try {
@@ -651,45 +660,74 @@ function getPullRequests(config) {
   const projectByName = new Map(projects.map(p => [p.name, p]));
   const allPrs = [];
   const seenIds = new Set();
-  // Single pass over projects/* intersected with the configured project list.
-  // Filesystem dirs not in CONFIG.projects (e.g. a leftover .minions/projects/
-  // <removedName>/ recreated by a stale code path between archive and reload)
-  // are skipped so removed projects can't resurrect themselves through the
-  // status payload. Hidden dirs (.archived sidecar) are also skipped defensively
-  // — they live as siblings under projects/ and would otherwise be enumerated.
-  let projectDirs = [];
+
+  // Phase 3: prefer the SQL store. Returns every scope's PRs tagged with
+  // `_scope`; we re-map to `_project` for backward compatibility with
+  // downstream consumers, intersecting with the configured project list
+  // to suppress orphan/removed projects (matches legacy behaviour).
+  let sqlPrs = null;
   try {
-    projectDirs = fs.readdirSync(path.join(MINIONS_DIR, 'projects'), { withFileTypes: true })
-      .filter(d => d.isDirectory() && !d.name.startsWith('.')).map(d => d.name);
-  } catch { /* projects dir missing */ }
-  for (const dirName of projectDirs) {
-    const project = projectByName.get(dirName);
-    if (!project) continue; // unconfigured/removed — don't surface
-    const prPath = projectPrPath(project);
-    const prs = readJsonNoRestore(prPath);
-    if (!Array.isArray(prs)) continue;
-    shared.normalizePrRecords(prs, project);
-    const base = project.prUrlBase || '';
-    for (const pr of prs) {
+    const store = require('./pull-requests-store');
+    sqlPrs = store.readAllPullRequests();
+  } catch { /* fall through to JSON */ }
+
+  if (Array.isArray(sqlPrs) && sqlPrs.length > 0) {
+    for (const pr of sqlPrs) {
       if (!pr?.id || seenIds.has(pr.id)) continue;
-      if (!pr.url && base) {
-        const prNumber = shared.getPrNumber(pr);
-        if (prNumber != null) pr.url = base + prNumber;
+      const scope = pr._scope;
+      delete pr._scope;
+      if (scope === 'central') {
+        pr._project = 'central';
+      } else {
+        const project = projectByName.get(scope);
+        if (!project) continue;  // orphan/removed project — don't surface
+        const base = project.prUrlBase || '';
+        if (!pr.url && base) {
+          const prNumber = shared.getPrNumber(pr);
+          if (prNumber != null) pr.url = base + prNumber;
+        }
+        shared.normalizePrRecords([pr], project);
+        pr._project = project.name || 'Project';
       }
-      pr._project = project.name || 'Project';
       allPrs.push(pr);
       seenIds.add(pr.id);
     }
-  }
-  // Central pull-requests.json — manually linked PRs without a project
-  const centralPrs = readJsonNoRestore(path.join(MINIONS_DIR, 'pull-requests.json'));
-  if (centralPrs) {
-    shared.normalizePrRecords(centralPrs, null);
-    for (const pr of centralPrs) {
-      if (!pr?.id || seenIds.has(pr.id)) continue;
-      pr._project = 'central';
-      allPrs.push(pr);
-      seenIds.add(pr.id);
+  } else {
+    // SQL empty or unavailable — fall back to per-file JSON read. Matches
+    // the original layout: per-project dirs first, central last.
+    let projectDirs = [];
+    try {
+      projectDirs = fs.readdirSync(path.join(MINIONS_DIR, 'projects'), { withFileTypes: true })
+        .filter(d => d.isDirectory() && !d.name.startsWith('.')).map(d => d.name);
+    } catch { /* projects dir missing */ }
+    for (const dirName of projectDirs) {
+      const project = projectByName.get(dirName);
+      if (!project) continue;
+      const prPath = projectPrPath(project);
+      const prs = readJsonNoRestore(prPath);
+      if (!Array.isArray(prs)) continue;
+      shared.normalizePrRecords(prs, project);
+      const base = project.prUrlBase || '';
+      for (const pr of prs) {
+        if (!pr?.id || seenIds.has(pr.id)) continue;
+        if (!pr.url && base) {
+          const prNumber = shared.getPrNumber(pr);
+          if (prNumber != null) pr.url = base + prNumber;
+        }
+        pr._project = project.name || 'Project';
+        allPrs.push(pr);
+        seenIds.add(pr.id);
+      }
+    }
+    const centralPrs = readJsonNoRestore(path.join(MINIONS_DIR, 'pull-requests.json'));
+    if (centralPrs) {
+      shared.normalizePrRecords(centralPrs, null);
+      for (const pr of centralPrs) {
+        if (!pr?.id || seenIds.has(pr.id)) continue;
+        pr._project = 'central';
+        allPrs.push(pr);
+        seenIds.add(pr.id);
+      }
     }
   }
   allPrs.sort((a, b) => {

package/engine/shared.js

@@ -285,6 +285,17 @@ function log(level, msg, meta = {}) {
     writable: true,
     configurable: true,
   });
+  // Capture the SQL state.db path too — same write-time semantics so a
+  // deferred buffer flush after the test ends still routes the entry to
+  // the test's state.db (not the production one). Path resolution mirrors
+  // engine/db.js but inlined to avoid the require cycle that loading the
+  // db module here would create on module init.
+  Object.defineProperty(entry, '_dbPath', {
+    value: _currentLogDbPath(),
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  });
 
   _logBuffer.push(entry);
 
@@ -337,6 +348,19 @@ function _currentLogPath() {
   return path.join(root, 'engine', 'log.json');
 }
 
+// Resolve the SQL state.db path at write time — same isolation contract
+// as _currentLogPath. MINIONS_LOG_PATH only redirects the JSON sidecar;
+// the SQL store stays attached to MINIONS_TEST_DIR / MINIONS_HOME because
+// it shares the file with the rest of the engine state (dispatches,
+// work_items, pull_requests).
+function _currentLogDbPath() {
+  if (process.env.MINIONS_TEST_DIR) {
+    return path.join(path.resolve(process.env.MINIONS_TEST_DIR), 'engine', 'state.db');
+  }
+  const root = process.env.MINIONS_HOME ? path.resolve(process.env.MINIONS_HOME) : path.resolve(__dirname, '..');
+  return path.join(root, 'engine', 'state.db');
+}
+
 // MINIONS_TEST_DIR-aware engine dir for cache files (runtime caps, models,
 // spawn-debug.log). Returns `<MINIONS_TEST_DIR>/engine` under tests; otherwise
 // the caller's source-adjacent fallback (typically `__dirname`-derived). Never
@@ -366,23 +390,44 @@ function openUrlInBrowser(url) {
 function _flushLogBuffer() {
   if (_logBuffer.length === 0) return;
   const drained = _logBuffer.splice(0);
-  // Group entries by their captured _logPath so test-originated entries always
-  // land in the test dir's log.json even if MINIONS_TEST_DIR has been cleared
-  // by the time we flush. Entries without _logPath fall back to current path
-  // (eg. direct _logBuffer.push() from tests).
-  const fallbackPath = _currentLogPath();
-  const byPath = new Map();
+  // Group entries by their captured (_logPath, _dbPath) so test-originated
+  // entries always land in the test dir's storage even if MINIONS_TEST_DIR
+  // has been cleared by the time we flush. Entries without captured paths
+  // fall back to the current resolved values (eg. direct _logBuffer.push()
+  // from tests).
+  const fallbackJsonPath = _currentLogPath();
+  const fallbackDbPath = _currentLogDbPath();
+  const byJsonPath = new Map();
+  const byDbPath = new Map();
   for (const raw of drained) {
-    const target = raw._logPath || fallbackPath;
+    const jsonTarget = raw._logPath || fallbackJsonPath;
+    const dbTarget = raw._dbPath || fallbackDbPath;
     // SEC-09 defense-in-depth: redact again at flush time so any direct
     // `_logBuffer.push(entry)` callers (tests, future paths) can't leak secrets.
     const entry = redactSecrets(raw);
     // Strip the routing-only metadata before persisting.
     delete entry._logPath;
-    if (!byPath.has(target)) byPath.set(target, []);
-    byPath.get(target).push(entry);
+    delete entry._dbPath;
+    if (!byJsonPath.has(jsonTarget)) byJsonPath.set(jsonTarget, []);
+    byJsonPath.get(jsonTarget).push(entry);
+    if (!byDbPath.has(dbTarget)) byDbPath.set(dbTarget, []);
+    byDbPath.get(dbTarget).push(entry);
   }
-  for (const [target, entries] of byPath) {
+
+  // SQL append (Phase 4 source of truth). Per-path append so test
+  // state.db files stay isolated; logging must never crash the caller, so
+  // appendLogsToDbFile already swallows its own errors.
+  try {
+    const store = require('./logs-store');
+    for (const [dbPath, entries] of byDbPath) {
+      store.appendLogsToDbFile(dbPath, entries);
+    }
+  } catch { /* logs-store unavailable (node<22.5) — JSON mirror still works */ }
+
+  // JSON mirror — kept for backward compat with test fixtures that
+  // safeJson(log.json) directly, and the legacy getEngineLog fallback.
+  // Phase 4.5 will retire this once readers move to readRecentLogs().
+  for (const [target, entries] of byJsonPath) {
     try {
       mutateJsonFileLocked(target, (logData) => {
         if (!Array.isArray(logData)) logData = logData?.entries || [];
@@ -1806,6 +1851,8 @@ const ENGINE_DEFAULTS = {
   versionCheckInterval: 3600000, // 1 hour — how often to check npm for updates (ms)
   logFlushInterval: 5000, // 5s — how often to flush buffered log entries to disk
   logBufferSize: 50, // flush immediately when buffer exceeds this many entries
+  logCapEntries: 2500, // ring-buffer cap on `logs` SQL table; trims to logCapEntries-logCapTrimTo on overflow
+  logCapTrimTo: 2000, // size to trim back to when cap is hit (matches legacy JSON splice 2500→2000)
   lockRetries: 0, // no retries — single 5s timeout window with 25ms polling (200 attempts) is sufficient; stale lock recovery at 60s handles crashes
   lockRetryBackoffMs: 500, // base backoff between lock retries (doubles each attempt: 500ms, 1s, 2s, ...)
   buildFixGracePeriod: 600000, // 10min — wait for CI to run after a verified build-fix push before re-dispatching
@@ -4643,6 +4690,35 @@ function reopenWorkItem(wi) {
  * @param {Function} mutator - Receives the array, mutates in place or returns new value
  */
 function mutatePullRequests(filePath, mutator) {
+  // Phase 3 SQL path. Same shape as Phase 2's mutateWorkItems: route
+  // through the pull-requests-store when filePath sits under MINIONS_DIR,
+  // mirror back to JSON for legacy direct-readers, emit a pull_requests
+  // event only on real writes. Ad-hoc tmp paths (legacy tests using
+  // createTmpDir) and SQLite failures fall through to the JSON path.
+  const fpNorm = String(filePath).replace(/\\/g, '/');
+  const minionsNorm = String(MINIONS_DIR).replace(/\\/g, '/');
+  const insideMinionsDir = fpNorm.startsWith(minionsNorm + '/') || fpNorm === minionsNorm + '/pull-requests.json';
+  if (insideMinionsDir) {
+    try {
+      const store = require('./pull-requests-store');
+      const scope = store.scopeForFilePath(filePath);
+      const { wrote, result } = store.applyPullRequestsMutation(scope, (prs) => {
+        if (!Array.isArray(prs)) prs = [];
+        return mutator(prs) || prs;
+      });
+      if (wrote) {
+        try { store._mirrorJsonFromSql(scope, filePath); } catch { /* mirror is best-effort */ }
+        try { require('./db-events').emitStateEvent('pull_requests'); } catch { /* optional */ }
+      }
+      return result;
+    } catch (e) {
+      if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
+        throw e;
+      }
+      // Fall through to legacy JSON path on SQLite errors only.
+    }
+  }
+
   return mutateJsonFileLocked(filePath, (data) => {
     if (!Array.isArray(data)) data = [];
     return mutator(data) || data;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2067",
+  "version": "0.1.2069",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"