@yemi33/minions 0.1.2072 → 0.1.2074

package/engine/watches-store.js

@@ -0,0 +1,259 @@
+// engine/watches-store.js — SQL-backed implementation of engine/watches.json.
+//
+// Mirrors the Phase 2/3 work_items / pull_requests stores: a single
+// table (no per-scope split — watches.json is a flat array, not
+// per-project), diff-then-apply mutator, JSON dual-write mirror with
+// (mtime, size) fingerprint for external-edit detection.
+//
+//   readWatches()           -> [watch, watch, ...]
+//   applyWatchesMutation(fn) -> diff-then-apply, returns { wrote, result }
+
+const fs = require('fs');
+const path = require('path');
+
+function _toMs(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+  const parsed = Date.parse(v);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function _watchesFilePath() {
+  const shared = require('./shared');
+  return path.join(shared.MINIONS_DIR, 'engine', 'watches.json');
+}
+
+function _parseRow(row) {
+  if (!row || !row.data) return null;
+  try { return JSON.parse(row.data); }
+  catch { return null; }
+}
+
+function _readJsonArrayFallback() {
+  const fp = _watchesFilePath();
+  let raw;
+  try { raw = fs.readFileSync(fp, 'utf8'); }
+  catch { return []; }
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch (e) {
+    try {
+      // eslint-disable-next-line no-console
+      console.warn(`[watches-store] corrupt JSON in ${fp}: ${e.message}`);
+    } catch { /* console may be wrapped */ }
+    return [];
+  }
+}
+
+// Content-hash fingerprint — (mtime, size) is too coarse: same-size
+// content swaps (e.g. last_checked timestamps that always have the same
+// byte length) inside the same ms tick collide on both axes. Hashing the
+// bytes is the unambiguous signal. SHA-1 on a few-KB file is sub-ms.
+let _lastMirrorHash = null;
+
+const crypto = require('crypto');
+
+function _fileContentHash(filePath) {
+  try {
+    const buf = fs.readFileSync(filePath);
+    return crypto.createHash('sha1').update(buf).digest('hex');
+  }
+  catch { return null; }
+}
+
+function _hydrateFromJson(db) {
+  const items = _readJsonArrayFallback();
+  db.prepare('DELETE FROM watches').run();
+  if (items.length === 0) return;
+  const now = Date.now();
+  const ins = db.prepare(`
+    INSERT INTO watches (id, target, target_type, condition, status, owner, created_at, last_checked, last_triggered, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO NOTHING
+  `);
+  for (const w of items) {
+    if (!w || !w.id) continue;
+    ins.run(
+      String(w.id),
+      typeof w.target === 'string' ? w.target : (w.target == null ? null : JSON.stringify(w.target)),
+      w.targetType || null,
+      w.condition || null,
+      String(w.status || 'active'),
+      w.owner || null,
+      _toMs(w.created_at),
+      _toMs(w.last_checked),
+      _toMs(w.last_triggered),
+      JSON.stringify(w),
+      now,
+    );
+  }
+}
+
+function _resyncIfJsonDiverged(db) {
+  const jsonPath = _watchesFilePath();
+  const currentHash = _fileContentHash(jsonPath);
+  if (currentHash == null) return;
+  if (_lastMirrorHash != null && currentHash === _lastMirrorHash) return;
+  if (_lastMirrorHash == null) {
+    const sqlHas = db.prepare('SELECT 1 FROM watches LIMIT 1').get();
+    if (sqlHas) {
+      _lastMirrorHash = currentHash;
+      return;
+    }
+  }
+  _hydrateFromJson(db);
+  _lastMirrorHash = currentHash;
+}
+
+// Read all watch rows from SQL only. No JSON fallback — used by the
+// mirror writer to avoid resurrecting deleted rows: after the mutation
+// deletes a watch from SQL but BEFORE the mirror lands, the JSON still
+// holds the pre-mutation content, so a fallback read would round-trip
+// the deleted row right back into JSON.
+function _readWatchesFromSqlOnly(db) {
+  const rows = db.prepare('SELECT data FROM watches ORDER BY rowid').all();
+  const out = [];
+  for (const row of rows) {
+    const w = _parseRow(row);
+    if (w) out.push(w);
+  }
+  return out;
+}
+
+function readWatches() {
+  const { getDb } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch { return _readJsonArrayFallback(); }
+
+  _resyncIfJsonDiverged(db);
+
+  const out = _readWatchesFromSqlOnly(db);
+  if (out.length === 0) {
+    // SQL is empty for first-time hydrate: trust the JSON if it has
+    // content. After hydrate ran, SQL is empty iff the user really has
+    // no watches.
+    const fallback = _readJsonArrayFallback();
+    if (fallback.length > 0) return fallback;
+    return [];
+  }
+  return out;
+}
+
+function _indexById(arr) {
+  const out = new Map();
+  for (const w of arr) {
+    if (!w || !w.id) continue;
+    out.set(String(w.id), w);
+  }
+  return out;
+}
+
+function _computeWatchesDiff(before, after) {
+  const beforeMap = _indexById(before);
+  const afterMap = _indexById(after);
+  const toUpsert = [];
+  const toDelete = [];
+  for (const [id, w] of afterMap) {
+    const prev = beforeMap.get(id);
+    if (!prev) { toUpsert.push(w); continue; }
+    if (JSON.stringify(prev) !== JSON.stringify(w)) toUpsert.push(w);
+  }
+  for (const [id] of beforeMap) {
+    if (!afterMap.has(id)) toDelete.push(id);
+  }
+  return { toUpsert, toDelete };
+}
+
+function _applyWatchesDiff(db, diff) {
+  if (diff.toUpsert.length === 0 && diff.toDelete.length === 0) return false;
+  const now = Date.now();
+  const upsertStmt = db.prepare(`
+    INSERT INTO watches (id, target, target_type, condition, status, owner, created_at, last_checked, last_triggered, data, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      target         = excluded.target,
+      target_type    = excluded.target_type,
+      condition      = excluded.condition,
+      status         = excluded.status,
+      owner          = excluded.owner,
+      created_at     = excluded.created_at,
+      last_checked   = excluded.last_checked,
+      last_triggered = excluded.last_triggered,
+      data           = excluded.data,
+      updated_at     = excluded.updated_at
+  `);
+  const deleteStmt = db.prepare('DELETE FROM watches WHERE id = ?');
+
+  for (const w of diff.toUpsert) {
+    upsertStmt.run(
+      String(w.id),
+      typeof w.target === 'string' ? w.target : (w.target == null ? null : JSON.stringify(w.target)),
+      w.targetType || null,
+      w.condition || null,
+      String(w.status || 'active'),
+      w.owner || null,
+      _toMs(w.created_at),
+      _toMs(w.last_checked),
+      _toMs(w.last_triggered),
+      JSON.stringify(w),
+      now,
+    );
+  }
+  for (const id of diff.toDelete) {
+    deleteStmt.run(id);
+  }
+  return true;
+}
+
+function applyWatchesMutation(mutator) {
+  const { getDb, withTransaction } = require('./db');
+  let db;
+  try { db = getDb(); }
+  catch (e) {
+    throw new Error(`engine/watches-store: SQLite unavailable (${e.message}); cannot mutate watches`);
+  }
+
+  return withTransaction(db, () => {
+    _resyncIfJsonDiverged(db);
+    const before = readWatches();
+    const beforeSnapshot = JSON.parse(JSON.stringify(before));
+    const next = mutator(before);
+    const after = (next === undefined || next === null)
+      ? before
+      : (Array.isArray(next) ? next : before);
+    const diff = _computeWatchesDiff(beforeSnapshot, after);
+    const wrote = _applyWatchesDiff(db, diff);
+    return { wrote, result: after };
+  });
+}
+
+function _mirrorJsonFromSql(filePath) {
+  try {
+    const shared = require('./shared');
+    const { getDb } = require('./db');
+    // Read SQL directly — bypass the JSON fallback so a deletion that
+    // leaves SQL empty doesn't resurrect from the stale JSON content.
+    const items = _readWatchesFromSqlOnly(getDb());
+    const target = filePath || _watchesFilePath();
+    shared.safeWrite(target, items);
+    const h = _fileContentHash(target);
+    if (h != null) _lastMirrorHash = h;
+  } catch {
+    // Mirror failures are non-fatal — SQL has already committed.
+  }
+}
+
+function _resetWatchesTableForTest() {
+  const { getDb } = require('./db');
+  try { getDb().exec('DELETE FROM watches'); } catch { /* not initialized */ }
+  _lastMirrorHash = null;
+}
+
+module.exports = {
+  readWatches,
+  applyWatchesMutation,
+  _mirrorJsonFromSql,
+  _resetWatchesTableForTest,
+};

package/engine/watches.js

@@ -67,7 +67,7 @@
 const fs = require('fs');
 const path = require('path');
 const shared = require('./shared');
-const { safeJsonArr, mutateJsonFileLocked, ts, uid, log, writeToInbox,
+const { safeJsonArr, mutateJsonFileLocked, mutateWatches, ts, uid, log, writeToInbox,
   WATCH_STATUS, WATCH_TARGET_TYPE, WATCH_CONDITION, WATCH_ABSOLUTE_CONDITIONS } = shared;
 const watchActions = require('./watch-actions');
 
@@ -230,11 +230,10 @@ function createWatch({ target, targetType, condition, interval, owner, descripti
     _history: [],
   };
 
-  mutateJsonFileLocked(_watchesPath(), (watches) => {
-    if (!Array.isArray(watches)) watches = [];
+  mutateWatches((watches) => {
     watches.push(watch);
     return watches;
-  }, { defaultValue: [] });
+  });
 
   log('info', `Watch created: ${watch.id} → ${watch.target} (${watch.condition})`);
   return watch;
@@ -264,8 +263,7 @@ function updateWatch(id, updates) {
     if (reqErr) throw new Error(reqErr);
   }
   let found = null;
-  mutateJsonFileLocked(_watchesPath(), (watches) => {
-    if (!Array.isArray(watches)) return watches;
+  mutateWatches((watches) => {
     const watch = watches.find(w => w.id === id);
     if (!watch) return watches;
     // Only allow safe field updates
@@ -275,7 +273,7 @@ function updateWatch(id, updates) {
     }
     found = { ...watch };
     return watches;
-  }, { defaultValue: [] });
+  });
   return found;
 }
 
@@ -287,15 +285,14 @@ function updateWatch(id, updates) {
 function deleteWatch(id) {
   if (!id) return false;
   let deleted = false;
-  mutateJsonFileLocked(_watchesPath(), (watches) => {
-    if (!Array.isArray(watches)) return watches;
+  mutateWatches((watches) => {
     const idx = watches.findIndex(w => w.id === id);
     if (idx >= 0) {
       watches.splice(idx, 1);
       deleted = true;
     }
     return watches;
-  }, { defaultValue: [] });
+  });
   if (deleted) log('info', `Watch deleted: ${id}`);
   return deleted;
 }
@@ -413,8 +410,8 @@ function checkWatches(config, state) {
   const notifications = [];
   const actionsToRun = [];
 
-  mutateJsonFileLocked(_watchesPath(), (watches) => {
-    if (!Array.isArray(watches) || watches.length === 0) return watches;
+  mutateWatches((watches) => {
+    if (watches.length === 0) return watches;
 
     for (const watch of watches) {
       try {
@@ -539,7 +536,7 @@ function checkWatches(config, state) {
     }
 
     return watches;
-  }, { defaultValue: [] });
+  });
 
   // Fire notifications outside the lock — writeToInbox does disk I/O
   for (const n of notifications) {
@@ -591,8 +588,7 @@ async function _runActionTask(task) {
   // can inspect each step's result. Single-action watches keep the legacy
   // `dispatchedItemId` shortcut for back-compat.
   try {
-    mutateJsonFileLocked(_watchesPath(), (watches) => {
-      if (!Array.isArray(watches)) return watches;
+    mutateWatches((watches) => {
       const w = watches.find(x => x.id === task.watchId);
       if (w) {
         if (isChain) {
@@ -639,7 +635,7 @@ async function _runActionTask(task) {
         }
       }
       return watches;
-    }, { defaultValue: [] });
+    });
   } catch (err) {
     log('warn', `Watch ${task.watchId} action result persist failed: ${err.message}`);
   }

package/engine/work-items-store.js

@@ -116,28 +116,19 @@ function readWorkItemsForScope(scope) {
 // scope is bounded by the JSON's row count.
 function _resyncScopeIfJsonDiverged(db, scope) {
   const jsonPath = _filePathForScope(scope);
-  const current = _statFingerprint(jsonPath);
-  const lastMirror = _lastMirrorByScope.get(scope);
-  if (current == null) return;  // JSON absent → nothing to resync
-  // In-sync iff BOTH mtime AND size match. Either differing signals an
-  // external write — including the same-ms-tick safeWrite pattern that
-  // mtime-only checks miss on Windows (ms-rounded mtimeMs collides).
-  // Resync is idempotent (DELETE + bulk re-INSERT from JSON).
-  if (lastMirror != null
-    && current.mtime === lastMirror.mtime
-    && current.size === lastMirror.size) return;
-  // No mirror record but JSON exists: first-time hydrate iff SQL is empty
-  // for this scope (avoid trampling a freshly-backfilled migration state
-  // on the very first read).
-  if (lastMirror == null) {
+  const currentHash = _fileContentHash(jsonPath);
+  const lastHash = _lastMirrorHashByScope.get(scope);
+  if (currentHash == null) return;
+  if (lastHash != null && currentHash === lastHash) return;
+  if (lastHash == null) {
     const sqlHas = db.prepare('SELECT 1 FROM work_items WHERE scope = ? LIMIT 1').get(scope);
     if (sqlHas) {
-      _lastMirrorByScope.set(scope, current);
+      _lastMirrorHashByScope.set(scope, currentHash);
       return;
     }
   }
   _hydrateScopeFromJson(db, scope);
-  _lastMirrorByScope.set(scope, current);
+  _lastMirrorHashByScope.set(scope, currentHash);
 }
 
 // Read all rows across all scopes — used by queries.getWorkItems which
@@ -243,18 +234,18 @@ function _applyWorkItemsDiff(db, diff) {
 //   wrote  — true iff at least one INSERT/UPDATE/DELETE landed
 //   result — the post-mutation array (legacy return shape of
 //            mutateJsonFileLocked)
-// In-process record of (mtime, size) of the JSON mirror we last wrote,
-// keyed by scope. Two-field fingerprint catches the same-ms-tick write
-// pattern that mtime-only checks miss on Windows: NTFS reports
-// ms-rounded mtimeMs, so back-to-back writes inside the same tick look
-// identical to "no change" by mtime alone. File size catches the rest
-// — content swaps almost always change byte length.
-const _lastMirrorByScope = new Map();
+// Content-hash fingerprint of the JSON mirror per scope, so external
+// edits are detected unambiguously. (mtime, size) is too coarse: same-
+// length timestamp swaps (last_checked / last_triggered / completed_at)
+// preserve both axes. SHA-1 is sub-ms on a few-hundred-KB file.
+const _lastMirrorHashByScope = new Map();
 
-function _statFingerprint(filePath) {
+const crypto = require('crypto');
+
+function _fileContentHash(filePath) {
   try {
-    const st = fs.statSync(filePath);
-    return { mtime: st.mtimeMs, size: st.size };
+    const buf = fs.readFileSync(filePath);
+    return crypto.createHash('sha1').update(buf).digest('hex');
   }
   catch { return null; }
 }
@@ -325,17 +316,24 @@ function applyWorkItemsMutation(scope, mutator) {
 function _mirrorJsonFromSql(scope, filePath) {
   try {
     const shared = require('./shared');
-    const items = readWorkItemsForScope(scope);
-    // Strip _source if present — only added by readAllWorkItems, but
-    // belt-and-suspenders since the mirror is observable.
-    for (const wi of items) { if (wi && wi._source) delete wi._source; }
+    const { getDb } = require('./db');
+    // Read SQL directly for this scope — bypass the JSON fallback so a
+    // mutation that empties SQL for the scope doesn't resurrect stale
+    // JSON content. (See pull-requests-store and watches-store for the
+    // same defense.)
+    const db = getDb();
+    const rows = db.prepare('SELECT data FROM work_items WHERE scope = ? ORDER BY rowid').all(scope);
+    const items = [];
+    for (const row of rows) {
+      const wi = _parseRow(row);
+      if (!wi) continue;
+      if (wi._source) delete wi._source;
+      items.push(wi);
+    }
     const target = filePath || _filePathForScope(scope);
     shared.safeWrite(target, items);
-    // Record (mtime, size) we just wrote so the next mutation can detect
-    // an external edit. mtime alone misses same-ms-tick writes; size
-    // catches them.
-    const fp = _statFingerprint(target);
-    if (fp != null) _lastMirrorByScope.set(scope, fp);
+    const h = _fileContentHash(target);
+    if (h != null) _lastMirrorHashByScope.set(scope, h);
   } catch {
     // Mirror failures are non-fatal: SQL has already committed.
   }

package/engine/worktree-pool.js

@@ -61,12 +61,16 @@ function readPool() {
 }
 
 function mutateWorktreePool(mutator) {
-  return mutateJsonFileLocked(getPoolPath(), (data) => {
+  // Phase 7: route through shared.mutateWorktreePool (SQL-backed store +
+  // JSON mirror). Defensive wrappers around `data.entries` kept for the
+  // legacy fallback path inside the helper (when SQLite is unavailable
+  // and we fall through to mutateJsonFileLocked).
+  return shared.mutateWorktreePool((data) => {
     if (!data || typeof data !== 'object' || Array.isArray(data)) data = { entries: [] };
     if (!Array.isArray(data.entries)) data.entries = [];
     const next = mutator(data);
     return next === undefined ? data : next;
-  }, { defaultValue: { entries: [] }, skipWriteIfUnchanged: true });
+  });
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2072",
+  "version": "0.1.2074",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"