@yemi33/minions 0.1.2369 → 0.1.2371

package/engine/work-items-store.js

@@ -1,6 +1,4 @@
-// engine/work-items-store.js — SQL-backed implementation of the
-// per-file work-items array. Same external contract as the legacy
-// JSON-file reader:
+// engine/work-items-store.js — SQL-backed scoped work-item storage.
 //
 //   readWorkItemsForScope(scope)        -> [wi, wi, ...]
 //   applyWorkItemsMutation(scope, fn)   -> runs fn against the array,
@@ -8,16 +6,11 @@
 //                                          transaction, returns
 //                                          { wrote, result }.
 //
-// `scope` is 'central' for the top-level work-items.json or a project
-// name for projects/<name>/work-items.json. Mirrors the file convention
-// established by migration 003.
+// `scope` is 'central' or a project name.
 //
 // All callers go through shared.mutateWorkItems and
 // queries.getWorkItems — those are thin wrappers over the helpers here.
 
-const path = require('path');
-const fs = require('fs');
-
 function _toMs(v) {
   if (v == null) return null;
   if (typeof v === 'number') return Number.isFinite(v) ? v : null;
@@ -40,60 +33,22 @@ function scopeForFilePath(filePath) {
   return 'central';
 }
 
-// JSON fallback path resolution. Used both for the empty-SQL fallback
-// (test seeding pattern from Phase 1) and for the post-write JSON mirror.
-function _filePathForScope(scope) {
-  const shared = require('./shared');
-  if (scope === 'central') {
-    return path.join(shared.MINIONS_DIR, 'work-items.json');
-  }
-  return path.join(shared.MINIONS_DIR, 'projects', scope, 'work-items.json');
-}
-
 function _parseRow(row) {
   if (!row || !row.data) return null;
   try { return JSON.parse(row.data); }
   catch { return null; }
 }
 
-function _readJsonArrayFallback(scope) {
-  const fp = _filePathForScope(scope);
-  let raw;
-  try { raw = fs.readFileSync(fp, 'utf8'); }
-  catch { return []; }
-  try {
-    const parsed = JSON.parse(raw);
-    return Array.isArray(parsed) ? parsed : [];
-  } catch (e) {
-    try {
-      // eslint-disable-next-line no-console
-      console.warn(`[work-items-store] corrupt JSON in ${fp}: ${e.message}`);
-    } catch { /* console may be wrapped in tests */ }
-    return [];
-  }
-}
-
 function readWorkItemsForScope(scope) {
   const { getDb } = require('./db');
   const db = getDb();
 
-  _resyncScopeIfJsonDiverged(db, scope);
-
   const rows = db.prepare(`
     SELECT data FROM work_items
     WHERE scope = ?
     ORDER BY rowid
   `).all(scope);
 
-  if (rows.length === 0) {
-    // SQL empty AND JSON has content means a test seeded via fs.writeFileSync
-    // (legacy helper) or a fresh install pre-migration. Returning the JSON
-    // keeps those call sites working without touching every helper.
-    const fallback = _readJsonArrayFallback(scope);
-    if (fallback.length > 0) return fallback;
-    return [];
-  }
-
   const out = [];
   for (const row of rows) {
     const wi = _parseRow(row);
@@ -102,58 +57,6 @@ function readWorkItemsForScope(scope) {
   return out;
 }
 
-// If the JSON mirror's mtime advanced since our last mirror write, the
-// file was rewritten outside the store (test cleanup + safeWrite, manual
-// operator edit, external tool). Treat the JSON as canonical and rebuild
-// SQL for that scope from it. Called from BOTH read and write paths so
-// readers don't see stale SQL state and writers don't compute diffs
-// against a SQL snapshot that no longer matches what callers expect.
-//
-// Safe to call inside `withTransaction` — DELETE + INSERT against a
-// scope is bounded by the JSON's row count.
-function _resyncScopeIfJsonDiverged(db, scope) {
-  const jsonPath = _filePathForScope(scope);
-  const currentHash = _fileContentHash(jsonPath);
-  const lastHash = _lastMirrorHashByScope.get(scope);
-  if (currentHash == null) return;
-  if (lastHash != null && currentHash === lastHash) return;
-  // First touch in this process (lastHash == null): if SQL already holds rows
-  // for the scope, SQL is the established source of truth (post-Phase-9.4) —
-  // adopt the current JSON hash and DO NOT hydrate. Hydrating here would let a
-  // crash-stale / .backup-restored / git-checked-out older JSON mirror DELETE +
-  // overwrite newer SQL. Only an empty-in-SQL scope (genuine first-install
-  // migration) falls through to hydrate-from-JSON, which is purely additive.
-  if (lastHash == null) {
-    const sqlHas = db.prepare('SELECT 1 FROM work_items WHERE scope = ? LIMIT 1').get(scope);
-    if (sqlHas) {
-      _lastMirrorHashByScope.set(scope, currentHash);
-      return;
-    }
-  }
-  _hydrateScopeFromJson(db, scope);
-  _lastMirrorHashByScope.set(scope, currentHash);
-}
-
-// Enumerate scopes that have a JSON file on disk — used as the SQL-
-// unavailable fallback for `readAllWorkItems`. Walks `<MINIONS_DIR>/
-// projects/*/work-items.json` plus the central `<MINIONS_DIR>/work-
-// items.json`. Silently swallows missing dirs (fresh installs).
-function _enumerateJsonScopes() {
-  const shared = require('./shared');
-  const scopes = [];
-  const centralPath = path.join(shared.MINIONS_DIR, 'work-items.json');
-  if (fs.existsSync(centralPath)) scopes.push('central');
-  const projectsDir = path.join(shared.MINIONS_DIR, 'projects');
-  try {
-    for (const d of fs.readdirSync(projectsDir, { withFileTypes: true })) {
-      if (!d.isDirectory()) continue;
-      const fp = path.join(projectsDir, d.name, 'work-items.json');
-      if (fs.existsSync(fp)) scopes.push(d.name);
-    }
-  } catch { /* projects dir missing on fresh install */ }
-  return scopes;
-}
-
 // Read all rows across all scopes — used by queries.getWorkItems which
 // needs to surface central + every project's items in a single shot,
 // tagged with their source scope.
@@ -161,17 +64,6 @@ function readAllWorkItems() {
   const { getDb } = require('./db');
   const db = getDb();
 
-  // Pick up any external JSON edits for every scope SQL knows about.
-  // Also resync 'central' explicitly so first-time reads on a JSON-only
-  // install hydrate the central scope. Per-project scopes get checked
-  // lazily on next read or mutation of that scope.
-  try { _resyncScopeIfJsonDiverged(db, 'central'); } catch {}
-  const knownScopes = db.prepare('SELECT DISTINCT scope FROM work_items').all().map(r => r.scope);
-  for (const s of knownScopes) {
-    if (s === 'central') continue;
-    try { _resyncScopeIfJsonDiverged(db, s); } catch {}
-  }
-
   const rows = db.prepare('SELECT data, scope FROM work_items ORDER BY rowid').all();
   const out = [];
   for (const row of rows) {
@@ -255,83 +147,11 @@ function _applyWorkItemsDiff(db, diff) {
 //   wrote  — true iff at least one INSERT/UPDATE/DELETE landed
 //   result — the post-mutation array (legacy return shape of
 //            mutateJsonFileLocked)
-// Content-hash fingerprint of the JSON mirror per scope, so external
-// edits are detected unambiguously. (mtime, size) is too coarse: same-
-// length timestamp swaps (last_checked / last_triggered / completed_at)
-// preserve both axes. SHA-1 is sub-ms on a few-hundred-KB file.
-const _lastMirrorHashByScope = new Map();
-
-const crypto = require('crypto');
-
-function _fileContentHash(filePath) {
-  try {
-    const buf = fs.readFileSync(filePath);
-    return crypto.createHash('sha1').update(buf).digest('hex');
-  }
-  catch { return null; }
-}
-
-function _hydrateScopeFromJson(db, scope) {
-  const jsonItems = _readJsonArrayFallback(scope);
-  // Preserve the SQL-only prd_item_id FK across the delete+reinsert. It is owned
-  // by the PRD dual-write (prd-store.js#_upsertPrd), NOT represented in the JSON
-  // mirror, so a naive rebuild-from-JSON silently zeroes it for the whole scope —
-  // regressing the WI↔PRD link every time the JSON diverges (the count would
-  // oscillate as the PRD mirror re-stamps and the next hydrate wipes it again).
-  // Snapshot before the DELETE, restore onto the rows that survive the rebuild.
-  const fkById = new Map();
-  try {
-    for (const r of db.prepare('SELECT id, prd_item_id FROM work_items WHERE scope = ? AND prd_item_id IS NOT NULL').all(scope)) {
-      fkById.set(r.id, r.prd_item_id);
-    }
-  } catch { /* column absent on a pre-v15 schema — nothing to preserve */ }
-  // DELETE before re-insert: callers that wrote a smaller JSON file
-  // (test cleanup() removing items) must end up with a smaller SQL state.
-  db.prepare('DELETE FROM work_items WHERE scope = ?').run(scope);
-  if (jsonItems.length === 0) return;
-  const now = Date.now();
-  const ins = db.prepare(`
-    INSERT INTO work_items (id, scope, status, type, agent, parent_id, created_at, completed_at, data, updated_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(scope, id) DO NOTHING
-  `);
-  for (const wi of jsonItems) {
-    if (!wi || !wi.id) continue;
-    ins.run(
-      String(wi.id),
-      scope,
-      String(wi.status || 'pending'),
-      wi.type || null,
-      wi.dispatched_to || wi.agent || null,
-      wi.parent_id || null,
-      _toMs(wi.created_at),
-      _toMs(wi.completed_at || wi.completedAt),
-      JSON.stringify(wi),
-      now,
-    );
-  }
-  // Restore the FK for rows that still exist after the rebuild (a WI dropped
-  // from the JSON is gone, and its link goes with it — correct).
-  if (fkById.size) {
-    const restore = db.prepare('UPDATE work_items SET prd_item_id = ? WHERE scope = ? AND id = ?');
-    for (const [id, fk] of fkById) {
-      try { restore.run(fk, scope, id); } catch { /* best effort */ }
-    }
-  }
-}
-
-function applyWorkItemsMutation(scope, mutator, options = {}) {
+function applyWorkItemsMutation(scope, mutator) {
   const { getDb, withTransaction } = require('./db');
   const db = getDb();
 
   return withTransaction(db, () => {
-    // Re-hydrate SQL from JSON if the file was touched outside the
-    // store (external edit, test re-seed, first-time hydrate after a
-    // fresh install). Without this, the diff would compare against a
-    // stale SQL snapshot and the mirror write would overwrite the JSON
-    // with that stale state.
-    _resyncScopeIfJsonDiverged(db, scope);
-
     const before = readWorkItemsForScope(scope);
     // Strip the _source decoration before handing to the mutator — it's
     // added by readAllWorkItems only and would round-trip as garbage into
@@ -344,63 +164,10 @@ function applyWorkItemsMutation(scope, mutator, options = {}) {
       : (Array.isArray(next) ? next : before);
     const diff = _computeWorkItemsDiff(scope, beforeSnapshot, after);
     const wrote = _applyWorkItemsDiff(db, diff);
-    // W-mr9vcxvy000cdb4e — phantom-WI race fix: mirror the JSON sidecar
-    // INSIDE this same SQL write transaction, BEFORE it commits, instead
-    // of after (as the caller used to do post-return). SQLite only allows
-    // one writer transaction at a time (BEGIN IMMEDIATE holds an
-    // exclusive writer lock), so mirroring here guarantees the on-disk
-    // JSON file is fully up to date by the time this commit becomes
-    // visible to any other reader/writer — dashboard and engine are
-    // SEPARATE Node processes, each with its own in-memory
-    // `_lastMirrorHashByScope` cache, so without this ordering a sibling
-    // process could observe a freshly committed SQL row before the mirror
-    // file caught up, decide (via its own stale hash) that the JSON file
-    // was "externally edited", and call `_hydrateScopeFromJson` — which
-    // DELETEs + reinserts the whole scope from that stale file, silently
-    // dropping the row a POST /api/work-items caller had just been told
-    // was created successfully (visible to the engine's dispatch loop
-    // one moment, gone from work_items the next). Mirroring pre-commit
-    // means "SQL says the row exists" and "JSON mirror has the row" can
-    // never observably disagree across processes.
-    if (wrote) {
-      try { _mirrorJsonFromSql(scope, options.mirrorPath); } catch { /* best-effort — mirror failure must not block the SQL write */ }
-    }
     return { wrote, result: after };
   });
 }
 
-// Mirror SQL → JSON file. Phase 2 keeps the file as a read-only derivative
-// so legacy direct-readers keep working; SQL is the source of truth.
-function _mirrorJsonFromSql(scope, filePath) {
-  try {
-    const shared = require('./shared');
-    const { getDb } = require('./db');
-    // Read SQL directly for this scope — bypass the JSON fallback so a
-    // mutation that empties SQL for the scope doesn't resurrect stale
-    // JSON content. (See pull-requests-store and watches-store for the
-    // same defense.)
-    const db = getDb();
-    const rows = db.prepare('SELECT data FROM work_items WHERE scope = ? ORDER BY rowid').all(scope);
-    const items = [];
-    for (const row of rows) {
-      const wi = _parseRow(row);
-      if (!wi) continue;
-      if (wi._source) delete wi._source;
-      items.push(wi);
-    }
-    const target = filePath || _filePathForScope(scope);
-    shared.safeWrite(target, items);
-    const h = _fileContentHash(target);
-    if (h != null) _lastMirrorHashByScope.set(scope, h);
-  } catch {
-    // Mirror failures are non-fatal: SQL has already committed.
-  }
-}
-
-// Drop every SQL row for `scope` and forget its mirror-hash. Called by
-// removeProject after the JSON file is archived so safeJson shim reads no
-// longer surface the removed project's records.
-//
 // Per CLAUDE.md's mutator contract, every mutator wrapper must call
 // emitStateEvent() so the dashboard's MAX(events.id) cache-version
 // advances — otherwise the dashboard keeps serving stale cached
@@ -411,21 +178,6 @@ function dropScope(scope) {
   try {
     const { getDb } = require('./db');
     const result = getDb().prepare('DELETE FROM work_items WHERE scope = ?').run(scope);
-    // Do NOT simply forget the mirror-hash here. _resyncScopeIfJsonDiverged
-    // treats "no recorded hash" + "zero SQL rows for the scope" as a genuine
-    // first-install hydrate and rebuilds SQL from whatever JSON is on disk.
-    // When the caller is removeProject with dataMode:'keep', the JSON file for
-    // this scope is deliberately left untouched — so the very next read/write
-    // against this scope would silently resurrect the just-dropped project's
-    // stale rows straight back into SQL. Instead, adopt the JSON file's
-    // *current* hash (if it still exists) so the next resync correctly reads
-    // that as "unchanged" and skips the rehydrate. If the file is genuinely
-    // gone (dataMode: 'purge'/'archive'), forgetting the hash is safe since
-    // _resyncScopeIfJsonDiverged no-ops when the file can't be hashed.
-    const jsonPath = _filePathForScope(scope);
-    const currentHash = _fileContentHash(jsonPath);
-    if (currentHash == null) _lastMirrorHashByScope.delete(scope);
-    else _lastMirrorHashByScope.set(scope, currentHash);
     if (result && result.changes > 0) {
       try { require('./db-events').emitStateEvent('work_items'); } catch { /* optional */ }
       try { require('./queries').invalidateWorkItemsCache(); } catch { /* queries not loaded */ }
@@ -439,7 +191,4 @@ module.exports = {
   readAllWorkItems,
   applyWorkItemsMutation,
   dropScope,
-  _filePathForScope,
-  _mirrorJsonFromSql,
-  _hydrateScopeFromJson, // exported for testing (prd_item_id-preservation)
 };