@yemi33/minions 0.1.2369 → 0.1.2371

package/engine/db/migrations/018-sql-only-cutover.js

@@ -0,0 +1,56 @@
+// Retire live JSON mirror metadata after all runtime stores become SQL-only.
+module.exports = {
+  version: 18,
+  description: 'SQL-only runtime-state cutover; drop obsolete PR mirror hashes',
+  up(db, { fs, path }) {
+    let legacyPrdCount = 0;
+    let importedPrdCount = 0;
+    const minionsDir = process.env.MINIONS_TEST_DIR || process.env.MINIONS_HOME
+      || (() => { try { return require('../../shared').MINIONS_DIR; } catch { return null; } })();
+    if (minionsDir) {
+      const importDir = (dir, archived) => {
+        let files = [];
+        try { files = fs.readdirSync(dir).filter(f => f.endsWith('.json')); } catch { return; }
+        const { _upsertPrd } = require('../../prd-store');
+        for (const filename of files) {
+          legacyPrdCount++;
+          const filePath = path.join(dir, filename);
+          let prd;
+          try { prd = JSON.parse(fs.readFileSync(filePath, 'utf8')); }
+          catch (e) { throw new Error(`cannot import legacy PRD ${filePath}: ${e.message}`); }
+          if (!prd || typeof prd !== 'object' || Array.isArray(prd)) {
+            throw new Error(`cannot import legacy PRD ${filePath}: expected an object`);
+          }
+          _upsertPrd(db, filename, archived, prd, Date.now());
+          importedPrdCount++;
+        }
+      };
+      importDir(path.join(minionsDir, 'prd'), 0);
+      importDir(path.join(minionsDir, 'prd', 'archive'), 1);
+    }
+    if (importedPrdCount !== legacyPrdCount) {
+      throw new Error(`legacy PRD import count mismatch: found ${legacyPrdCount}, imported ${importedPrdCount}`);
+    }
+    db.exec(`
+      DROP TABLE IF EXISTS pr_mirror_hashes;
+      CREATE TABLE IF NOT EXISTS state_markers (
+        key        TEXT PRIMARY KEY,
+        value      TEXT NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+    `);
+    const quickCheck = db.prepare('PRAGMA quick_check').all();
+    if (quickCheck.length !== 1 || quickCheck[0].quick_check !== 'ok') {
+      throw new Error(`SQLite quick_check failed: ${JSON.stringify(quickCheck)}`);
+    }
+    const foreignKeyErrors = db.prepare('PRAGMA foreign_key_check').all();
+    if (foreignKeyErrors.length) {
+      throw new Error(`SQLite foreign_key_check failed: ${JSON.stringify(foreignKeyErrors)}`);
+    }
+    db.prepare(`
+      INSERT INTO state_markers (key, value, updated_at)
+      VALUES ('sql_only_cutover', '1', ?)
+      ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at
+    `).run(Date.now());
+  },
+};

package/engine/dispatch-store.js

@@ -17,18 +17,6 @@
 
 const SECTIONS = ['pending', 'active', 'completed', 'review'];
 
-// First-touch guard (P-5f0e8c27). Tracks whether a dispatch mutation has
-// written SQL in THIS process. Once SQL has been written, an empty read
-// reflects a genuine prune-to-zero — NOT a fresh install — so we must not
-// resurrect a stale dispatch.json (e.g. a crash between the SQL COMMIT and
-// the post-commit mirror write leaves dispatch.json holding the pre-prune
-// rows). This mirrors the `_resyncIfJsonDiverged` first-touch discipline the
-// work-items / pull-requests / metrics / watches stores already use: once SQL
-// is the established source of truth this process, JSON never overrides it.
-// Reset only by process restart (module reload); test isolation re-requires
-// the module via withFreshDb's cache-bust, so each test starts fresh.
-let _sqlWrittenThisProcess = false;
-
 function _emptySectioned() {
   return { pending: [], active: [], completed: [], review: [] };
 }
@@ -60,33 +48,6 @@ function readDispatchSectioned() {
     ORDER BY status, created_at
   `).all();
 
-  if (rows.length === 0 && !_sqlWrittenThisProcess) {
-    // SQL has no live dispatches AND no mutation has written SQL this process.
-    // Only then is the JSON file a trustworthy fallback. Two scenarios:
-    //   (a) Production fresh install — JSON file is empty/missing, fallback
-    //       returns the same empty sectioned object. No harm.
-    //   (b) Test that wrote dispatch.json directly (legacy test helpers in
-    //       timeout-behavioral / orphan-* / etc. — they bypass the proper
-    //       mutateDispatch API and seed via fs.writeFileSync). Picking up
-    //       the JSON keeps those tests working through the transition without
-    //       touching every helper. Phase 1.5 migrates them to the proper API.
-    //
-    // The `!_sqlWrittenThisProcess` gate is the first-touch guard: once a
-    // mutation has written SQL this process (e.g. the engine pruned the last
-    // dispatch to zero), an empty SQL read is the real state. Adopting a stale
-    // non-empty dispatch.json here would resurrect pruned records and could
-    // trigger a duplicate dispatch before the next mutation self-heals the
-    // mirror — the exact crash-window bug this guard closes.
-    const fallback = _readDispatchJsonFallback();
-    const hasContent = (
-      (fallback.pending && fallback.pending.length > 0) ||
-      (fallback.active && fallback.active.length > 0) ||
-      (fallback.completed && fallback.completed.length > 0) ||
-      (fallback.review && fallback.review.length > 0)
-    );
-    if (hasContent) return fallback;
-  }
-
   const out = _emptySectioned();
   for (const row of rows) {
     const rec = _parseRow(row);
@@ -96,37 +57,6 @@ function readDispatchSectioned() {
   return out;
 }
 
-function _readDispatchJsonFallback() {
-  const path = require('path');
-  const fs = require('fs');
-  const shared = require('./shared');
-  const dispatchPath = path.join(shared.MINIONS_DIR, 'engine', 'dispatch.json');
-  let raw;
-  try {
-    raw = fs.readFileSync(dispatchPath, 'utf8');
-  } catch {
-    // ENOENT / permission / etc. — file genuinely absent. Silently
-    // return an empty sectioned object; that's the legacy semantics.
-    return _emptySectioned();
-  }
-  try {
-    const parsed = JSON.parse(raw);
-    const out = _emptySectioned();
-    for (const s of SECTIONS) {
-      if (Array.isArray(parsed[s])) out[s] = parsed[s];
-    }
-    return out;
-  } catch (e) {
-    // File exists but parse failed — that's user-visible corruption.
-    // Surface to logs the same way the legacy queries.getDispatch path
-    // did via shared.readJsonNoRestore. (Phase 1 contract test
-    // "getDispatch warns on corrupt dispatch.json".)
-    // eslint-disable-next-line no-console
-    console.warn(`[dispatch-store] corrupt JSON in ${dispatchPath}: ${e.message}`);
-    return _emptySectioned();
-  }
-}
-
 // Internal: read the full (unfiltered) dispatch state from SQL.
 // Used by applyDispatchMutation so the diff sees ALL completed rows,
 // not just the recent window. Mutators that prune old completed
@@ -272,57 +202,13 @@ function applyDispatchMutation(mutator) {
     const wrote = _applyDispatchDiff(db, diff);
     // First-touch guard: record that SQL was written this process so a later
     // empty read won't resurrect a stale dispatch.json (see readDispatchSectioned).
-    if (wrote) _sqlWrittenThisProcess = true;
     return { wrote, result: after };
   });
 }
 
-// Mirror the canonical SQL state back to dispatch.json after a successful
-// mutation. Phase 1 keeps the JSON file as a *read-only derivative* of SQL
-// — never an independent source of truth — so legacy callers that
-// readFileSync the file directly (a couple of internal paths + the broader
-// test suite) keep seeing consistent data. There's no drift opportunity
-// because:
-//
-//   - The mirror is always regenerated from the SQL table, never from
-//     in-memory state, so a write that lands transactionally in SQL but
-//     then crashes before the mirror is also written produces a stale
-//     mirror — but the SQL state is correct, and the next successful
-//     mutation overwrites the mirror.
-//   - No code path writes to dispatch.json independently anymore;
-//     mutateDispatch is the only writer.
-//
-// Phase 1.5 will remove the mirror once we've migrated the remaining
-// direct-readers (engine/queries.js fallback path, engine/routing.js,
-// the test infrastructure) to use readDispatchSectioned() / getDispatch().
-function _mirrorJsonFromSql() {
-  try {
-    const path = require('path');
-    const fs = require('fs');
-    const shared = require('./shared');
-    // The full sectioned view (not the 24h-windowed one) so the JSON
-    // mirror matches what a legacy reader would have seen before Phase 1.
-    const full = _readDispatchSectionedFull();
-    const dispatchPath = path.join(shared.MINIONS_DIR, 'engine', 'dispatch.json');
-    // safeWrite handles the atomic rename + .backup sidecar. Best-effort —
-    // failure here doesn't propagate (SQL is source of truth, mirror is
-    // optional refresh material).
-    shared.safeWrite(dispatchPath, full);
-    // Re-derive what the parent shared.safeWrite would have done so an
-    // operator copying the file out gets matched fs.statSync mtimes
-    // (the dashboard's slow-state tracker watches this file as part of
-    // the dispatch path list). No-op if safeWrite already advanced mtime.
-    try { fs.utimesSync(dispatchPath, new Date(), new Date()); } catch { /* best-effort */ }
-  } catch {
-    // Mirror failures are non-fatal: SQL has already committed. Tests +
-    // legacy readers will see the previous mirror until the next write.
-  }
-}
-
 module.exports = {
   readDispatchSectioned,
   applyDispatchMutation,
   _readDispatchSectionedFull,
   _computeDispatchDiff,
-  _mirrorJsonFromSql,
 };

package/engine/dispatch.js

@@ -122,11 +122,6 @@ function mutateDispatch(mutator) {
   if (wrote) {
     try { require('./queries').invalidateDispatchCache(); } catch {}
     try { require('./db-events').emitStateEvent('dispatch'); } catch { /* optional */ }
-    // Mirror back to dispatch.json for tests + tools that fs.readFileSync
-    // the file directly. SQL is the source of truth; the JSON file is
-    // regenerated from SQL on every successful mutation, never independently
-    // mutated. Cheap to delete once those direct-JSON readers are gone.
-    try { store._mirrorJsonFromSql(); } catch { /* best-effort */ }
   }
   return result;
 }
@@ -784,6 +779,7 @@ function isRetryableFailureReason(reason = '', failureClass = '') {
       FAILURE_CLASS.LIVE_CHECKOUT_STALE_BASE, // W-mr98op8w000ma4ad — live-checkout STALE-BASE GUARD refused to fork a new branch off a local mainRef that has diverged from origin/<mainRef> with unpushed commits; contaminated local base does not change on retry (operator must reconcile the local base branch — the engine never resets it — then re-dispatch)
       FAILURE_CLASS.LIVE_CHECKOUT_WRONG_BASE, // W-mr3lunnq000o9f41 — live-checkout refused to fork a new branch because the operator HEAD is on an unrelated branch (not the project base) and no local base ref was usable; forking off it would bake that branch's commits into the new PR (scope contamination). Deterministic — mechanical retry reproduces it (operator must `git checkout <mainRef>`, then re-dispatch)
       FAILURE_CLASS.OUTPUT_TRUNCATED, // P-8e4c2a17 — agent stdout exceeded the hard capture cap before the terminal result event; mechanical retry just reproduces the overflow (agent must reduce output volume or the task must be split)
+      FAILURE_CLASS.PRE_DISPATCH_EVAL_STUCK, // W-mrcrh5hj0017d22f — pre-dispatch-eval rejected the SAME unchanged description ENGINE_DEFAULTS.preDispatchEvalMaxRejections times in a row; non-retryable until the description changes (engine/shared.js:4986)
     ]);
     if (neverRetry.has(failureClass)) return false;
   }
@@ -1454,7 +1450,6 @@ function cleanDispatchEntries(matchFn) {
  * Sets status=CANCELLED + _cancelledBy=reason. Returns count cancelled.
  */
 function cancelPendingWorkItems(wiPath, matchFn, reason) {
-  if (!fs.existsSync(wiPath)) return 0;
   let cancelled = 0;
   try {
     mutateWorkItems(wiPath, items => {

package/engine/features.js

@@ -95,6 +95,12 @@ const FEATURES = {
     addedIn: '0.1.2232',
     expires: '2026-12-01',
   },
+  'memoryRetrieval': {
+    description: 'Select bounded task-relevant long-term memory with SQLite FTS5 instead of injecting only recent team and personal notes. Starts in shadow mode by default.',
+    default: false,
+    addedIn: '0.1.2233',
+    expires: '2027-01-31',
+  },
 };
 
 const ENV_TRUTHY = new Set(['1', 'true', 'on', 'yes']);

package/engine/gh-comment.js

@@ -42,8 +42,6 @@ const os = require('os');
 const { execFileSync: _execFileSync } = require('child_process');
 const { resolveTokenForSlug: _defaultResolveTokenForSlug } = require('./gh-token');
 const {
-  buildHarnessUsedSection,
-  buildSkippedSkillSection,
   linkifyBrandTrailer,
   MINIONS_BRAND_URL,
   // Neutral marker + body builder now lives in comment-format.js so the GitHub
@@ -155,15 +153,13 @@ function postPrComment({
   agentId,
   kind,
   workItemId,
-  harnessUsed,
-  skillSkipped,
   timeoutMs = 30000,
   execFileSync = _execFileSync,
   resolveTokenForSlug,
 } = {}) {
   _validateRepo(repo);
   _validatePrNumber(prNumber);
-  const finalBody = buildMinionsCommentBody({ agentId, kind, workItemId, body, harnessUsed, skillSkipped });
+  const finalBody = buildMinionsCommentBody({ agentId, kind, workItemId, body });
   const file = _writeTempBodyFile(finalBody);
   const env = _resolveTokenEnvForRepo(repo, resolveTokenForSlug);
   try {
@@ -186,15 +182,13 @@ function postPrReviewComment({
   agentId,
   kind,
   workItemId,
-  harnessUsed,
-  skillSkipped,
   timeoutMs = 30000,
   execFileSync = _execFileSync,
   resolveTokenForSlug,
 } = {}) {
   _validateRepo(repo);
   _validatePrNumber(prNumber);
-  const finalBody = buildMinionsCommentBody({ agentId, kind, workItemId, body, harnessUsed, skillSkipped });
+  const finalBody = buildMinionsCommentBody({ agentId, kind, workItemId, body });
   const file = _writeTempBodyFile(finalBody);
   const env = _resolveTokenEnvForRepo(repo, resolveTokenForSlug);
   try {
@@ -213,8 +207,6 @@ function postPrReviewComment({
 module.exports = {
   // Builders / parsers (pure functions — usable from anywhere)
   buildMinionsCommentBody,
-  buildHarnessUsedSection, // re-export of engine/comment-format.js for comment callers
-  buildSkippedSkillSection, // re-export of engine/comment-format.js for comment callers
   linkifyBrandTrailer, // re-export of engine/comment-format.js for comment callers
   MINIONS_BRAND_URL, // re-export of engine/comment-format.js for comment callers
   parseMinionsMarker,

package/engine/github.js

@@ -840,12 +840,14 @@ async function pollPrStatus(config) {
     else if (prData.state === 'closed') newStatus = PR_STATUS.ABANDONED;
     else if (prData.state === 'open') newStatus = PR_STATUS.ACTIVE;
 
-    // W-mpej044m00076d63: backfill `pr.created` from the platform's real
-    // creation timestamp. lifecycle.js attach paths deliberately omit
-    // `created` so this poll is the single source of truth — and historical
-    // records that pre-date the fix get backfilled the next time they're
-    // polled. Idempotent: only writes when missing.
-    if (!pr.created && prData.created_at) {
+    // W-mpej044m00076d63 / W-mrezh0yb0007f733: keep `pr.created` aligned with
+    // GitHub's real `created_at`. The platform is authoritative for when a PR
+    // was opened, so adopt its value whenever the stored one is missing, a
+    // legacy date-only stamp, or a link/attach-time `now()` default (the bug
+    // where manually-linked PRs showed a "created today" date).
+    // `shouldAdoptPlatformCreated` compares instants so equivalent
+    // representations don't churn. Idempotent.
+    if (shared.shouldAdoptPlatformCreated(pr.created, prData.created_at)) {
       pr.created = prData.created_at;
       updated = true;
     }