@yemi33/minions 0.1.2075 → 0.1.2076

package/docs/deprecated.json

@@ -56,6 +56,23 @@
     "targetRemovalDate": null,
     "notes": "Do NOT set targetRemovalDate — gating is signal-based. The function is silent on no-op (returns false without logging), so the meaningful telemetry signal is the absence of the promotion log line over the sweep window, NOT the absence of function invocations (cli.js calls it every boot regardless)."
   },
+  {
+    "id": "sql-state-json-mirrors",
+    "description": "Phase X.5 follow-up to the SQL state migration (commits 62bd6a2c..1111cf54, phases 0–7). Every engine state file that previously used mutateJsonFileLocked now routes through a SQL store, but each store still writes a JSON dual-write mirror after every mutation because a handful of direct-readers (a few unit tests + a couple of inline safeJson calls) have not been migrated to the SQL read path. Once those readers are confirmed routed through the SQL store (or rewritten to use the store's read helper), the mirror writers can be deleted and the JSON files retired.",
+    "code": [
+      { "file": "engine/dispatch-store.js", "note": "_mirrorJsonFromSql + _readDispatchJsonFallback — used when SQL is empty AND JSON has content (test seeding + first-time hydrate)" },
+      { "file": "engine/work-items-store.js", "note": "_mirrorJsonFromSql + _readJsonArrayFallback (per scope) — same fallback contract as dispatch-store" },
+      { "file": "engine/pull-requests-store.js", "note": "_mirrorJsonFromSql + _readJsonArrayFallback (per scope)" },
+      { "file": "engine/logs-store.js", "note": "engine/log.json mirror written by shared._flushLogBuffer's byJsonPath loop — Phase 4.5 will retire" },
+      { "file": "engine/metrics-store.js", "note": "_mirrorJsonFromSql + _readJsonObjectFallback" },
+      { "file": "engine/watches-store.js", "note": "_mirrorJsonFromSql + _readJsonArrayFallback" },
+      { "file": "engine/small-state-store.js", "note": "_mirrorScheduleRunsJson, _mirrorPipelineRunsJson, _mirrorManagedProcessesJson, _mirrorWorktreePoolJson + each store's _readJson fallback path" },
+      { "file": "CLAUDE.md", "lines": "47-66, 240-265", "note": "State Files + Concurrency sections still describe JSON files as the source of truth; they describe a layered SQLite-then-mirror reality in places but the headline contract still reads as JSON-primary. Rewrite these sections to make SQL-as-source-of-truth the headline and the JSON mirrors a transitional compatibility detail." }
+    ],
+    "removalGate": "All direct-readers of the mirror JSON files must be confirmed routed through their respective SQL store's read helper. Specifically: (a) grep the codebase for `safeJson`, `safeJsonArr`, `safeJsonObj`, `readFileSync(...work-items.json|pull-requests.json|metrics.json|watches.json|schedule-runs.json|pipeline-runs.json|managed-processes.json|worktree-pool.json|log.json|dispatch.json...)` and confirm every hit is either (i) a test fixture that can move to the SQL helper, or (ii) intentionally documented as bypassing SQL. (b) Run the full test suite with each store's _mirrorJsonFromSql temporarily neutered (returning early before safeWrite) and confirm 0 failures — that proves no production code path depends on the mirror. Once both conditions hold, removal deletes each store's _mirrorJsonFromSql call site in shared.js (mutateWorkItems/mutatePullRequests/etc.), the corresponding _readJsonArrayFallback paths, and the JSON file gitignore entries. CLAUDE.md update can ship independently as soon as someone has bandwidth.",
+    "targetRemovalDate": null,
+    "notes": "Do NOT set targetRemovalDate — gating is signal-based, not calendar-based. The mirror writes are cheap (a few KB per write, sub-ms) so there is no production cost to keeping them indefinitely; the only reason to remove them is to simplify the codebase and lock in SQL-as-the-single-source-of-truth. Order matters: when retiring a specific store's mirror, retire the corresponding CLAUDE.md mention in the same PR so the docs never claim SQL-only while a mirror still writes."
+  },
   {
     "id": "prune-default-claude-config",
     "description": "pruneDefaultClaudeConfig: active sanitizer that strips generated `config.claude.{binary,outputFormat,allowedTools,permissionMode}` defaults from persisted config.json so the `deprecated-config-claude` warning stops tripping on stale defaults left by older `minions init` versions. Sub-cluster of `config-claude-binary-override` — the prune deliberately preserves non-default user overrides (binary/allowedTools), which is what keeps the override branch in engine/runtimes/claude.js load-bearing.",

package/engine/pipeline.js

@@ -357,7 +357,7 @@ async function executeStage(stage, run, pipeline, config) {
     case STAGE_TYPE.PLAN:
       return executePlanStage(resolved, stageState, run, config, pipeline);
     case STAGE_TYPE.API:
-      return executeApiStage(resolved, stageState, run);
+      return await executeApiStage(resolved, stageState, run);
     case STAGE_TYPE.MERGE_PRS:
       return executeMergePrsStage(resolved, stageState, run, config);
     case STAGE_TYPE.SCHEDULE:
@@ -708,14 +708,27 @@ async function executePlanStage(stage, stageState, run, config, pipeline = {}) {
   };
 }
 
-function executeApiStage(stage, stageState, run) {
+// P-bfa1e-pipeline-state-machine-b — async + timeout-aware.
+// Each call is awaited end-to-end so the stage result reflects the real
+// outcome (previously the stage returned COMPLETED while requests were
+// still in flight, silently hiding API failures). On per-attempt timeout
+// the request is destroyed; the timeout flows through the existing retry
+// path. After `pipelineApiRetries` attempts exhaust on any call — or any
+// attempt times out past the retry budget — the stage returns FAILED with
+// `<endpoint>: <reason>` so updateRunStage records a terminal failure.
+async function executeApiStage(stage, stageState, run) {
   const calls = stage.calls || [{ endpoint: stage.endpoint, method: stage.method || 'POST', body: stage.body }];
+  const maxAttempts = ENGINE_DEFAULTS.pipelineApiRetries;
+  const retryDelay = ENGINE_DEFAULTS.pipelineApiRetryDelay;
+  const timeoutMs = ENGINE_DEFAULTS.pipelineApiTimeoutMs;
+
   for (const call of calls) {
     const url = `http://localhost:${process.env.MINIONS_PORT || 7331}${call.endpoint}`;
     const body = typeof call.body === 'string' ? call.body : JSON.stringify(call.body || {});
-    const maxAttempts = ENGINE_DEFAULTS.pipelineApiRetries;
-    const retryDelay = ENGINE_DEFAULTS.pipelineApiRetryDelay;
-    const makeRequest = (attempt) => {
+
+    const attemptOnce = (attempt) => new Promise((resolve) => {
+      let settled = false;
+      const finish = (result) => { if (!settled) { settled = true; resolve(result); } };
       try {
         const parsed = new URL(url);
         const req = http.request({
@@ -724,23 +737,45 @@ function executeApiStage(stage, stageState, run) {
           headers: { 'Content-Type': 'application/json' },
         }, (res) => {
           res.resume(); // drain body to free socket
-          if (res.statusCode >= 400) {
-            log('warn', `Pipeline API call to ${call.endpoint} returned ${res.statusCode} (attempt ${attempt})`);
-            if (attempt < maxAttempts) setTimeout(() => makeRequest(attempt + 1), retryDelay);
-          }
+          res.on('end', () => {
+            if (res.statusCode >= 400) {
+              log('warn', `Pipeline API call to ${call.endpoint} returned ${res.statusCode} (attempt ${attempt})`);
+              finish({ ok: false, reason: `HTTP ${res.statusCode}` });
+            } else {
+              finish({ ok: true });
+            }
+          });
+          res.on('error', (err) => finish({ ok: false, reason: err.message }));
+        });
+        req.setTimeout(timeoutMs, () => {
+          req.destroy(new Error('timeout'));
         });
         req.on('error', (err) => {
-          log('warn', `Pipeline API call to ${call.endpoint} failed: ${err.message} (attempt ${attempt})`);
-          if (attempt < maxAttempts) setTimeout(() => makeRequest(attempt + 1), retryDelay);
+          const reason = err && err.message ? err.message : (err && err.code) || 'request error';
+          log('warn', `Pipeline API call to ${call.endpoint} failed: ${reason} (attempt ${attempt})`);
+          finish({ ok: false, reason });
         });
         req.write(body);
         req.end();
       } catch (e) {
         log('warn', `Pipeline API call to ${call.endpoint} threw: ${e.message} (attempt ${attempt})`);
-        if (attempt < maxAttempts) setTimeout(() => makeRequest(attempt + 1), retryDelay);
+        finish({ ok: false, reason: e.message });
+      }
+    });
+
+    let lastReason = 'unknown';
+    let succeeded = false;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      const result = await attemptOnce(attempt);
+      if (result.ok) { succeeded = true; break; }
+      lastReason = result.reason;
+      if (attempt < maxAttempts && retryDelay > 0) {
+        await new Promise((r) => setTimeout(r, retryDelay));
       }
-    };
-    makeRequest(1);
+    }
+    if (!succeeded) {
+      return { status: PIPELINE_STATUS.FAILED, error: `${call.endpoint}: ${lastReason}`, completedAt: ts() };
+    }
   }
   return { status: PIPELINE_STATUS.COMPLETED, completedAt: ts() };
 }
@@ -1165,7 +1200,7 @@ module.exports = {
   getPipelineRuns, getActiveRun, startRun, updateRunStage, completeRun,
   discoverPipelineWork,
   evaluateCondition, // exported for testing
-  executeTaskStage, executePlanStage, executeScheduleStage, isStageComplete, resolveTemplate, // exported for testing
+  executeTaskStage, executePlanStage, executeScheduleStage, executeApiStage, isStageComplete, resolveTemplate, // exported for testing
   _resolvePipelineProjects, // exported for testing
   _findMeetingsInRun, _findExistingPlanForMeeting, _findExistingPrdForPlan, // exported for testing
 };

package/engine/shared.js

@@ -1923,6 +1923,7 @@ const ENGINE_DEFAULTS = {
   minRetryGapMs: 120000, // 2min — minimum gap between retry dispatches for the same work item; prevents tight retry loops when an idempotent agent (e.g. review bailing out on a duplicate) cannot produce the expected output (#1770)
   pipelineApiRetries: 2, // max attempts for pipeline API calls
   pipelineApiRetryDelay: 2000, // ms delay between pipeline API retries
+  pipelineApiTimeoutMs: 30000, // P-bfa1e-pipeline-state-machine-b — per-attempt request timeout for pipeline API calls; on timeout the request is destroyed and the attempt fails through the normal retry path. After all retries exhaust, executeApiStage returns FAILED instead of COMPLETED.
   prAutoLinkRetries: 3, // max attempts for gh pr list lookup when auto-linking PR after merge (3s backoff between attempts)
   rebaseQueueRetries: 3, // max rebase attempts per queued PR before giving up
   versionCheckInterval: 3600000, // 1 hour — how often to check npm for updates (ms)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2075",
+  "version": "0.1.2076",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"