@yemi33/minions 0.1.2069 → 0.1.2070

package/engine/db/migrations/006-metrics.js

@@ -0,0 +1,106 @@
+// engine/db/migrations/006-metrics.js
+//
+// Phase 5: move engine/metrics.json into a `metrics` table.
+//
+// metrics.json's shape is heterogeneous: a flat object whose top-level
+// keys mix per-agent counters (regular agent ids) with reserved keys
+// (`_engine`, `_daily`, `_contextPressure`). The SQL projection uses a
+// single table with a (kind, key) primary key so each top-level entry
+// becomes one row:
+//
+//   kind                | key                | data (JSON blob)
+//   --------------------+--------------------+-----------------------------
+//   agent               | dallas             | { tasksCompleted, ... }
+//   agent               | ripley             | { ...                 }
+//   engine_category     | command-center     | { calls, costUsd, ... }
+//   engine_category     | doc-chat           | { ...                 }
+//   daily               | 2026-05-28         | { costUsd, tasks, ... }
+//   context_pressure    | current            | { ...                 }
+//
+// Engineless extensions land transparently in `data` — no schema bump
+// required to add a new counter field per category. New top-level kinds
+// (e.g. `pipeline`) only need a new sentinel in the reserved-key list
+// inside the store; the table schema is forward-compatible.
+//
+// Backfill: read metrics.json once, split by reserved-key sentinel,
+// INSERT one row per entry. The JSON file stays on disk as a dual-write
+// mirror so legacy direct-readers (none in engine, a few in tests) keep
+// working.
+
+const path = require('path');
+const fs = require('fs');
+
+function _resolveMinionsDir() {
+  const envHome = process.env.MINIONS_HOME || process.env.MINIONS_TEST_DIR;
+  if (envHome) return envHome;
+  try { return require('../../shared').MINIONS_DIR; } catch { return null; }
+}
+
+const RESERVED_KEYS = new Map([
+  ['_engine', 'engine_category'],
+  ['_daily', 'daily'],
+  ['_contextPressure', 'context_pressure'],
+]);
+
+module.exports = {
+  version: 6,
+  description: 'metrics: schema + metrics.json backfill',
+  up(db) {
+    db.exec(`
+      CREATE TABLE metrics (
+        kind        TEXT NOT NULL,
+        key         TEXT NOT NULL,
+        data        TEXT NOT NULL,
+        updated_at  INTEGER NOT NULL,
+        PRIMARY KEY (kind, key)
+      );
+      CREATE INDEX idx_metrics_kind ON metrics(kind);
+    `);
+
+    const minionsDir = _resolveMinionsDir();
+    if (!minionsDir) return;
+    const metricsPath = path.join(minionsDir, 'engine', 'metrics.json');
+    if (!fs.existsSync(metricsPath)) return;
+
+    let raw;
+    try { raw = JSON.parse(fs.readFileSync(metricsPath, 'utf8')); }
+    catch (e) {
+      throw new Error(`engine/db/006-metrics: cannot parse metrics.json: ${e.message}`);
+    }
+    if (!raw || typeof raw !== 'object') return;
+
+    const insert = db.prepare(`
+      INSERT INTO metrics (kind, key, data, updated_at)
+      VALUES (?, ?, ?, ?)
+    `);
+
+    const now = Date.now();
+    let inserted = 0;
+
+    for (const [topKey, topValue] of Object.entries(raw)) {
+      const reservedKind = RESERVED_KEYS.get(topKey);
+      if (reservedKind) {
+        if (reservedKind === 'context_pressure') {
+          // Single-blob reserved key — store as one row keyed 'current'.
+          insert.run('context_pressure', 'current', JSON.stringify(topValue), now);
+          inserted += 1;
+          continue;
+        }
+        // Nested reserved key (_engine / _daily) — one row per inner key.
+        if (topValue && typeof topValue === 'object') {
+          for (const [innerKey, innerValue] of Object.entries(topValue)) {
+            insert.run(reservedKind, innerKey, JSON.stringify(innerValue), now);
+            inserted += 1;
+          }
+        }
+        continue;
+      }
+      // Regular agent counter row.
+      insert.run('agent', topKey, JSON.stringify(topValue), now);
+      inserted += 1;
+    }
+
+    // eslint-disable-next-line no-console
+    console.log(`[db-migrate] v6: backfilled ${inserted} metrics rows; metrics.json kept as dual-write mirror`);
+  },
+};

package/engine/lifecycle.js

@@ -1788,12 +1788,12 @@ async function updatePrAfterReview(agentId, pr, project, config, resultSummary,
   // Track reviewer for metrics purposes (separate file, separate lock)
   const authorAgentId = (reviewPr.agent || '').toLowerCase();
   if (authorAgentId && config.agents?.[authorAgentId]) {
-    shared.mutateJsonFileLocked(path.join(ENGINE_DIR, 'metrics.json'), (metrics) => {
+    shared.mutateMetrics((metrics) => {
       if (!metrics[authorAgentId]) metrics[authorAgentId] = { ...DEFAULT_AGENT_METRICS };
       if (!metrics[agentId]) metrics[agentId] = { ...DEFAULT_AGENT_METRICS };
       metrics[agentId].reviewsDone = (metrics[agentId].reviewsDone || 0) + 1;
       return metrics;
-    }, { defaultValue: {} });
+    });
   }
 
   log('info', `Updated ${reviewPr.id} → minions review: ${postReviewStatus || 'waiting'} by ${reviewerName}`);
@@ -2559,8 +2559,7 @@ async function handlePostMerge(pr, project, config, newStatus) {
 
   const agentId = (pr.agent || '').toLowerCase();
   if (agentId && config.agents?.[agentId]) {
-    const metricsPath = path.join(ENGINE_DIR, 'metrics.json');
-    mutateJsonFileLocked(metricsPath, (metrics) => {
+    shared.mutateMetrics((metrics) => {
       if (!metrics[agentId]) metrics[agentId] = { ...DEFAULT_AGENT_METRICS };
       metrics[agentId].prsMerged = (metrics[agentId].prsMerged || 0) + 1;
       return metrics;

package/engine/llm.js

@@ -103,8 +103,8 @@ function flushMetricsBuffer() {
   _pendingMetrics = { engine: Object.create(null), daily: Object.create(null) };
 
   try {
-    const metricsPath = path.join(ENGINE_DIR, 'metrics.json');
-    mutateJsonFileLocked(metricsPath, (metrics) => {
+    const { mutateMetrics } = require('./shared');
+    mutateMetrics((metrics) => {
       if (!metrics._engine) metrics._engine = {};
       for (const [category, delta] of Object.entries(pending.engine)) {
         if (!metrics._engine[category]) {

package/engine/pull-requests-store.js

@@ -62,10 +62,16 @@ function _readJsonArrayFallback(scope) {
   }
 }
 
-const _lastMirrorMtimeByScope = new Map();
+// Track (mtime, size) per scope so back-to-back writes inside the same
+// ms tick still get detected as external edits (mtime-only checks miss
+// them on Windows because NTFS reports ms-rounded mtimeMs).
+const _lastMirrorByScope = new Map();
 
-function _statMtimeMs(filePath) {
-  try { return fs.statSync(filePath).mtimeMs; }
+function _statFingerprint(filePath) {
+  try {
+    const st = fs.statSync(filePath);
+    return { mtime: st.mtimeMs, size: st.size };
+  }
   catch { return null; }
 }
 
@@ -99,19 +105,23 @@ function _hydrateScopeFromJson(db, scope) {
 
 function _resyncScopeIfJsonDiverged(db, scope) {
   const jsonPath = _filePathForScope(scope);
-  const currentMtime = _statMtimeMs(jsonPath);
-  const lastMirrorMtime = _lastMirrorMtimeByScope.get(scope);
-  if (currentMtime == null) return;
-  if (lastMirrorMtime != null && Math.abs(currentMtime - lastMirrorMtime) <= 1) return;
-  if (lastMirrorMtime == null) {
+  const current = _statFingerprint(jsonPath);
+  const lastMirror = _lastMirrorByScope.get(scope);
+  if (current == null) return;
+  // In-sync iff BOTH mtime AND size match. Size catches same-ms-tick
+  // external writes that mtime alone misses. Resync is idempotent.
+  if (lastMirror != null
+    && current.mtime === lastMirror.mtime
+    && current.size === lastMirror.size) return;
+  if (lastMirror == null) {
     const sqlHas = db.prepare('SELECT 1 FROM pull_requests WHERE scope = ? LIMIT 1').get(scope);
     if (sqlHas) {
-      _lastMirrorMtimeByScope.set(scope, currentMtime);
+      _lastMirrorByScope.set(scope, current);
       return;
     }
   }
   _hydrateScopeFromJson(db, scope);
-  _lastMirrorMtimeByScope.set(scope, currentMtime);
+  _lastMirrorByScope.set(scope, current);
 }
 
 function readPullRequestsForScope(scope) {
@@ -263,8 +273,8 @@ function _mirrorJsonFromSql(scope, filePath) {
     for (const pr of items) { if (pr && pr._scope) delete pr._scope; }
     const target = filePath || _filePathForScope(scope);
     shared.safeWrite(target, items);
-    const m = _statMtimeMs(target);
-    if (m != null) _lastMirrorMtimeByScope.set(scope, m);
+    const fp = _statFingerprint(target);
+    if (fp != null) _lastMirrorByScope.set(scope, fp);
   } catch {
     // Mirror failures are non-fatal — SQL has already committed.
   }

package/engine/queries.js

@@ -348,7 +348,16 @@ function getEngineLog() {
 }
 
 function getMetrics() {
-  const metrics = readJsonNoRestore(path.join(ENGINE_DIR, 'metrics.json')) || {};
+  // Phase 5: prefer the SQL store. Falls back to the JSON mirror on
+  // SQLite failure or when the table is empty (fresh install /
+  // pre-migration). Returns the legacy flat object shape.
+  let metrics;
+  try {
+    const store = require('./metrics-store');
+    metrics = store.readMetrics() || {};
+  } catch {
+    metrics = readJsonNoRestore(path.join(ENGINE_DIR, 'metrics.json')) || {};
+  }
 
   for (const [agentId, m] of Object.entries(metrics)) {
     if (agentId.startsWith('_')) continue;

package/engine/shared.js

@@ -2690,13 +2690,52 @@ const WATCH_ACTION_TYPE = {
   RESUME_PLAN: 'resume-plan',
 };
 
+/**
+ * Route a metrics mutation through the SQL store with a JSON dual-write
+ * mirror. Same shape as mutateWorkItems / mutatePullRequests: mutator
+ * receives the full legacy-shape metrics object, mutates in place or
+ * returns a replacement, and the store diffs by (kind, key) row.
+ *
+ * Falls back to the legacy mutateJsonFileLocked path on SQLite failure
+ * so a node:sqlite-broken install keeps recording metrics.
+ */
+function mutateMetrics(mutator) {
+  const metricsPath = path.join(MINIONS_DIR, 'engine', 'metrics.json');
+  try {
+    const store = require('./metrics-store');
+    const { wrote, result } = store.applyMetricsMutation((m) => {
+      if (!m || typeof m !== 'object') m = {};
+      return mutator(m) || m;
+    });
+    if (wrote) {
+      try { store._mirrorJsonFromSql(metricsPath); } catch { /* mirror best-effort */ }
+      try { require('./db-events').emitStateEvent('metrics'); } catch { /* optional */ }
+    }
+    return result;
+  } catch (e) {
+    if (!e || !/SQLite unavailable|no such table|node:sqlite/.test(String(e.message))) {
+      throw e;
+    }
+    // SQLite unavailable — fall through to legacy JSON path.
+  }
+  return mutateJsonFileLocked(metricsPath, (metrics) => {
+    if (!metrics || typeof metrics !== 'object') metrics = {};
+    return mutator(metrics) || metrics;
+  }, {
+    defaultValue: {},
+    onWrote: () => {
+      try { require('./db-events').emitStateEvent('metrics'); } catch { /* optional */ }
+    },
+  });
+}
+
 /** Update per-agent review metrics (prsApproved/prsRejected). Only writes for configured agents. */
 function trackReviewMetric(pr, newReviewStatus, config) {
   if (newReviewStatus !== 'approved' && newReviewStatus !== 'changes-requested') return;
   const authorId = (pr.agent || '').toLowerCase();
   if (!authorId || !config?.agents?.[authorId]) return;
   try {
-    mutateJsonFileLocked(path.join(MINIONS_DIR, 'engine', 'metrics.json'), (metrics) => {
+    mutateMetrics((metrics) => {
       if (!metrics[authorId]) metrics[authorId] = { ...DEFAULT_AGENT_METRICS };
       if (newReviewStatus === 'approved') metrics[authorId].prsApproved = (metrics[authorId].prsApproved || 0) + 1;
       else metrics[authorId].prsRejected = (metrics[authorId].prsRejected || 0) + 1;
@@ -5127,7 +5166,7 @@ module.exports = {
   runtimeConfigWarnings,
   projectWorkSourceWarnings,
   backfillProjectWorkSourceDefaults,
-  WI_STATUS, DONE_STATUSES, PLAN_TERMINAL_STATUSES, WORK_TYPE, WORKTREE_REQUIRING_TYPES, PLAN_STATUS, PRD_ITEM_STATUS, PRD_MATERIALIZABLE, PR_STATUS, PR_POLLABLE_STATUSES, PR_PENDING_REASON, DISPATCH_RESULT, trackReviewMetric, queuePlanToPrd, extractPlanDeclaredProject,
+  WI_STATUS, DONE_STATUSES, PLAN_TERMINAL_STATUSES, WORK_TYPE, WORKTREE_REQUIRING_TYPES, PLAN_STATUS, PRD_ITEM_STATUS, PRD_MATERIALIZABLE, PR_STATUS, PR_POLLABLE_STATUSES, PR_PENDING_REASON, DISPATCH_RESULT, mutateMetrics, trackReviewMetric, queuePlanToPrd, extractPlanDeclaredProject,
   WATCH_STATUS, WATCH_TARGET_TYPE, WATCH_CONDITION, WATCH_ABSOLUTE_CONDITIONS, WATCH_ACTION_TYPE,
   WATCH_STALLED_DEFAULT_TICKS, WATCH_STUCK_STAGE_DEFAULT_TICKS,
   PIPELINE_STATUS, STAGE_TYPE, MEETING_STATUS, AGENT_STATUS,

package/engine/work-items-store.js

@@ -116,24 +116,28 @@ function readWorkItemsForScope(scope) {
 // scope is bounded by the JSON's row count.
 function _resyncScopeIfJsonDiverged(db, scope) {
   const jsonPath = _filePathForScope(scope);
-  const currentMtime = _statMtimeMs(jsonPath);
-  const lastMirrorMtime = _lastMirrorMtimeByScope.get(scope);
-  if (currentMtime == null) return;  // JSON absent → nothing to resync
-  if (lastMirrorMtime != null && Math.abs(currentMtime - lastMirrorMtime) <= 1) return;  // in sync
+  const current = _statFingerprint(jsonPath);
+  const lastMirror = _lastMirrorByScope.get(scope);
+  if (current == null) return;  // JSON absent → nothing to resync
+  // In-sync iff BOTH mtime AND size match. Either differing signals an
+  // external write — including the same-ms-tick safeWrite pattern that
+  // mtime-only checks miss on Windows (ms-rounded mtimeMs collides).
+  // Resync is idempotent (DELETE + bulk re-INSERT from JSON).
+  if (lastMirror != null
+    && current.mtime === lastMirror.mtime
+    && current.size === lastMirror.size) return;
   // No mirror record but JSON exists: first-time hydrate iff SQL is empty
   // for this scope (avoid trampling a freshly-backfilled migration state
   // on the very first read).
-  if (lastMirrorMtime == null) {
+  if (lastMirror == null) {
     const sqlHas = db.prepare('SELECT 1 FROM work_items WHERE scope = ? LIMIT 1').get(scope);
     if (sqlHas) {
-      // Record current mtime as the baseline so the next divergence is
-      // detected, but don't rebuild — migration already populated SQL.
-      _lastMirrorMtimeByScope.set(scope, currentMtime);
+      _lastMirrorByScope.set(scope, current);
       return;
     }
   }
   _hydrateScopeFromJson(db, scope);
-  _lastMirrorMtimeByScope.set(scope, currentMtime);
+  _lastMirrorByScope.set(scope, current);
 }
 
 // Read all rows across all scopes — used by queries.getWorkItems which
@@ -239,17 +243,19 @@ function _applyWorkItemsDiff(db, diff) {
 //   wrote  — true iff at least one INSERT/UPDATE/DELETE landed
 //   result — the post-mutation array (legacy return shape of
 //            mutateJsonFileLocked)
-// In-process record of the JSON mirror mtime we last wrote, keyed by
-// scope. Used to detect external JSON edits between mutations — both
-// (a) tests that clean up + re-seed via fs.writeFileSync, and
-// (b) operators that hand-edit work-items.json in production. When the
-// current JSON mtime differs from what we recorded, the JSON has been
-// rewritten outside the store; we treat the JSON as canonical and
-// re-hydrate SQL from it before computing the diff.
-const _lastMirrorMtimeByScope = new Map();
+// In-process record of (mtime, size) of the JSON mirror we last wrote,
+// keyed by scope. Two-field fingerprint catches the same-ms-tick write
+// pattern that mtime-only checks miss on Windows: NTFS reports
+// ms-rounded mtimeMs, so back-to-back writes inside the same tick look
+// identical to "no change" by mtime alone. File size catches the rest
+// — content swaps almost always change byte length.
+const _lastMirrorByScope = new Map();
 
-function _statMtimeMs(filePath) {
-  try { return fs.statSync(filePath).mtimeMs; }
+function _statFingerprint(filePath) {
+  try {
+    const st = fs.statSync(filePath);
+    return { mtime: st.mtimeMs, size: st.size };
+  }
   catch { return null; }
 }
 
@@ -325,10 +331,11 @@ function _mirrorJsonFromSql(scope, filePath) {
     for (const wi of items) { if (wi && wi._source) delete wi._source; }
     const target = filePath || _filePathForScope(scope);
     shared.safeWrite(target, items);
-    // Record the mtime we just wrote so the next mutation can detect
-    // an external edit (mtime advanced while we weren't looking).
-    const m = _statMtimeMs(target);
-    if (m != null) _lastMirrorMtimeByScope.set(scope, m);
+    // Record (mtime, size) we just wrote so the next mutation can detect
+    // an external edit. mtime alone misses same-ms-tick writes; size
+    // catches them.
+    const fp = _statFingerprint(target);
+    if (fp != null) _lastMirrorByScope.set(scope, fp);
   } catch {
     // Mirror failures are non-fatal: SQL has already committed.
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2069",
+  "version": "0.1.2070",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"