@groundnuty/macf-core 0.2.43 → 0.2.45

package/dist/fleet-reconcile.js

@@ -0,0 +1,249 @@
+/** The zero state for an agent with no persisted history. */
+export const EMPTY_RECONCILE_STATE = {
+    deafSweeps: 0,
+    restartAttempts: 0,
+    backoffUntil: 0,
+    alertOpen: false,
+    lastExit: null,
+    paused: false,
+};
+/** Sensible defaults; a caller overrides selectively via `resolveConfig`. */
+export const DEFAULT_RECONCILE_CONFIG = {
+    execute: false,
+    allowRestart: false,
+    escalateAfterSweeps: 1,
+    backoffBaseMs: 60_000,
+    backoffMaxMs: 3_600_000,
+    stuckMax: 3,
+    launchStaggerMs: 15_000,
+    injectMessage: '⚠️ You appear OFF-CHANNELS (watchdog): your channel is down / not accepting. ' +
+        'Investigate, clear any stale registry entry, and request a relaunch.',
+};
+/** Merge partial overrides onto the defaults. */
+export function resolveReconcileConfig(overrides) {
+    return { ...DEFAULT_RECONCILE_CONFIG, ...(overrides ?? {}) };
+}
+// --- pure helpers -----------------------------------------------------------
+/**
+ * How the probe sees a desired agent (three-valued, mirroring the reference's
+ * empty/`true`/`false` `reachable` field):
+ *   - `absent`    — not in the registry roster at all (not running / never registered)
+ *   - `deaf`      — in the roster but `/health` did NOT answer (registered, channel down)
+ *   - `reachable` — in the roster + `/health` answered (mTLS 2xx)
+ */
+export function classifyReachability(state, agent) {
+    const found = state.agents.find((a) => a.name === agent);
+    if (!found)
+        return 'absent';
+    return found.online ? 'reachable' : 'deaf';
+}
+/**
+ * The EXPONENTIAL restart-backoff window after `attempts` consecutive restarts:
+ * `base * 2^(attempts-1)`, clamped to `maxMs`. `attempts <= 0` → no window (0).
+ */
+export function nextBackoffMs(attempts, config) {
+    if (attempts <= 0)
+        return 0;
+    const raw = config.backoffBaseMs * 2 ** (attempts - 1);
+    return Math.min(raw, config.backoffMaxMs);
+}
+/**
+ * Minimal parser for the fixed `desired-agents.yaml` shape (no yaml dep — the
+ * cron-light awk parse's TS twin). Pairs each `- agent: <x>` with the FOLLOWING
+ * `workspace: <y>`; strips inline `# comments` + surrounding quotes; ignores the
+ * `agents:` header, blank lines, and standalone comments.
+ */
+export function parseDesiredAgents(text) {
+    const out = [];
+    let pendingAgent = null;
+    for (const line of text.split(/\r?\n/)) {
+        const agentMatch = line.match(/^\s*-\s*agent:\s*(.+)$/);
+        if (agentMatch) {
+            pendingAgent = cleanScalar(agentMatch[1]);
+            continue;
+        }
+        const wsMatch = line.match(/^\s*workspace:\s*(.+)$/);
+        if (wsMatch && pendingAgent) {
+            out.push({ agent: pendingAgent, workspace: cleanScalar(wsMatch[1]) });
+            pendingAgent = null;
+        }
+    }
+    return out;
+}
+/** Strip a trailing `# comment`, trim, and drop one pair of surrounding quotes. */
+function cleanScalar(raw) {
+    const noComment = raw.replace(/\s+#.*$/, '').trim();
+    return noComment.replace(/^["']|["']$/g, '');
+}
+// --- report formatting ------------------------------------------------------
+function headerRows() {
+    return [
+        `${'AGENT'.padEnd(16)} ${'DECISION'.padEnd(10)} DETAIL`,
+        `${'-----'.padEnd(16)} ${'--------'.padEnd(10)} ------`,
+    ];
+}
+function reportRow(row) {
+    return `${row.agent.padEnd(16)} ${row.decision.padEnd(10)} ${row.detail}`;
+}
+// --- the orchestrator -------------------------------------------------------
+/**
+ * Run one reconcile sweep over the `desired` set. Probes ACTUAL state, decides +
+ * (on `execute`) acts per agent through the injected driver + seams, and returns
+ * the report + exit code. Dry-run (the default) prints the report and touches
+ * nothing (no driver mutations, no state writes, no heartbeat).
+ */
+export async function reconcileFleet(desired, deps, config) {
+    let fleet;
+    try {
+        fleet = await deps.driver.probe();
+    }
+    catch (err) {
+        deps.log(`FATAL: fleet probe failed — ${err instanceof Error ? err.message : String(err)}`);
+        return { rc: 2, rows: [] };
+    }
+    for (const h of headerRows())
+        deps.log(h);
+    const rows = [];
+    let rc = 0;
+    let launchCount = 0;
+    for (const d of desired) {
+        const state = deps.store.read(d.agent);
+        const row = await reconcileAgent(d, state, fleet, deps, config, launchCount);
+        deps.log(reportRow(row.row));
+        rows.push(row.row);
+        if (row.row.decision === 'LAUNCH')
+            launchCount += 1;
+        if (row.acted)
+            rc = 1;
+    }
+    deps.log('');
+    const mode = config.execute ? 'EXECUTE' : 'dry-run';
+    const restart = config.allowRestart ? 'on' : 'off';
+    if (config.execute) {
+        await deps.heartbeat({ rc, at: deps.now(), restartEnabled: config.allowRestart });
+    }
+    deps.log(rc === 0
+        ? `reconcile [${mode}, restart:${restart}]: all desired agents OK / down — no action.`
+        : `reconcile [${mode}, restart:${restart}]: action(s) taken or needed above.`);
+    return { rc, rows };
+}
+/** Per-agent reconcile: classify → decide → (execute) act. Returns the row + whether it counts as an action. */
+async function reconcileAgent(d, state, fleet, deps, config, launchCount) {
+    const agent = d.agent;
+    // desired-down (SKIP): explicit paused sentinel.
+    if (state.paused) {
+        return { row: mkRow(agent, 'SKIP', 'paused (desired-down; explicit) — not resurrected'), acted: false };
+    }
+    const actual = classifyReachability(fleet, agent);
+    if (actual === 'absent') {
+        // exit-code intent: not running + last-exit==0 (operator typed /exit) → desired-down.
+        if (state.lastExit === 0) {
+            return {
+                row: mkRow(agent, 'SKIP', 'not running + last-exit==0 (operator /exit) — desired-down, not restarted'),
+                acted: false,
+            };
+        }
+        await doLaunch(d, deps, config, launchCount);
+        return {
+            row: mkRow(agent, 'LAUNCH', `not running, last-exit!=0/absent → cold-start (ws: ${d.workspace})`),
+            acted: true,
+        };
+    }
+    if (actual === 'reachable') {
+        if (config.execute)
+            deps.store.reset(agent);
+        return { row: mkRow(agent, 'OK', 'reachable + accepting'), acted: false };
+    }
+    // deaf → HEAL ladder.
+    await healAgent(d, state, deps, config);
+    return { row: mkRow(agent, 'HEAL', 'registered but channel DOWN (deaf) → ladder'), acted: true };
+}
+/** Cold-start a desired-but-down agent, STAGGERED so N dead agents don't thundering-herd. */
+async function doLaunch(d, deps, config, launchCount) {
+    if (!config.execute)
+        return;
+    // Space every launch after the first by the stagger window.
+    if (launchCount > 0)
+        await deps.sleep(config.launchStaggerMs);
+    await deps.driver.launch(d.agent);
+}
+/**
+ * The tiered HEAL ladder for one deaf agent (cross-sweep). Sweep 1 → Tier-1
+ * inject; sweep 2+ → escalate under the aliveness-gate + restart-backoff before
+ * any Tier-2 restart, with Tier-3 alert on every escalation path.
+ */
+async function healAgent(d, state, deps, config) {
+    const agent = d.agent;
+    const n = state.deafSweeps + 1;
+    if (config.execute)
+        deps.store.write(agent, { ...state, deafSweeps: n });
+    if (n <= config.escalateAfterSweeps) {
+        deps.log(`    [sweep ${n}] first deaf sweep → Tier-1 (gated inject)`);
+        if (config.execute)
+            await deps.driver.inject(agent, config.injectMessage);
+        return;
+    }
+    deps.log(`    [sweep ${n}] still deaf after ${n - 1} prior sweep(s) → escalate (Tier-1 did not recover)`);
+    // Aliveness-gate — NEVER restart a busy agent (the #642 channel-dead/TUI-alive mode).
+    // Only probed on execute (dry-run constructs the plan without touching the driver).
+    if (config.execute && (await deps.driver.isBusy(agent))) {
+        deps.log(`    [ABORT] ${agent} pane ACTIVE — agent ALIVE despite channel-down (#642). NOT restarting a working agent.`);
+        await raiseAlert(agent, `deaf ${n} sweeps but pane active (#642) — human check`, deps, config);
+        return;
+    }
+    // Restart-backoff gate — NEVER restart-storm a repeatedly-failing agent.
+    if (deps.now() < state.backoffUntil) {
+        deps.log(`    [BACKOFF] ${agent} restart suppressed (attempt ${state.restartAttempts}, window not elapsed)`);
+        if (state.restartAttempts >= config.stuckMax) {
+            deps.log(`    [STUCK] ${agent} stuck in restart-backoff after ${state.restartAttempts} attempts → Tier-3`);
+            await raiseAlert(agent, `stuck in restart-backoff after ${state.restartAttempts} restarts`, deps, config);
+        }
+        return;
+    }
+    await escalateRestart(d, state, n, deps, config);
+}
+/** The Tier-2/Tier-3 tail once the aliveness + backoff gates pass. */
+async function escalateRestart(d, state, n, deps, config) {
+    const agent = d.agent;
+    // Restarts exhausted (>= stuckMax) but still deaf → escalate, do NOT restart again.
+    if (state.restartAttempts >= config.stuckMax) {
+        deps.log(`    [STUCK] ${agent} ${state.restartAttempts} restarts did not recover → Tier-3 (not restarting again)`);
+        await raiseAlert(agent, `stuck: ${state.restartAttempts} restarts did not recover`, deps, config);
+        return;
+    }
+    // Tier-2 graceful-restart — held behind operator sign-off.
+    if (!config.allowRestart) {
+        deps.log(`    [held] Tier-2 graceful-restart of ${agent} SUPPRESSED (no --allow-restart; operator sign-off required)`);
+        await raiseAlert(agent, `deaf ${n} sweeps; Tier-2 held (no --allow-restart)`, deps, config);
+        return;
+    }
+    deps.log(`    Tier-2 graceful-restart ${agent} (attempt ${state.restartAttempts + 1})`);
+    if (config.execute) {
+        await deps.driver.restart(agent);
+        const attempts = state.restartAttempts + 1;
+        const fresh = deps.store.read(agent);
+        deps.store.write(agent, {
+            ...fresh,
+            restartAttempts: attempts,
+            backoffUntil: deps.now() + nextBackoffMs(attempts, config),
+        });
+    }
+    await raiseAlert(agent, `restarted after ${n} deaf sweeps`, deps, config);
+}
+/** Tier-3 alert (a `gh issue create`), dedup'd via the persisted `alertOpen` sentinel. */
+async function raiseAlert(agent, reason, deps, config) {
+    if (deps.store.read(agent).alertOpen) {
+        deps.log(`    [skip] Tier-3 alert for ${agent} already open (dedup)`);
+        return;
+    }
+    deps.log(`    Tier-3 alert for ${agent} (${reason})`);
+    if (!config.execute)
+        return;
+    await deps.alert(agent, reason);
+    const fresh = deps.store.read(agent);
+    deps.store.write(agent, { ...fresh, alertOpen: true });
+}
+function mkRow(agent, decision, detail) {
+    return { agent, decision, detail };
+}
+//# sourceMappingURL=fleet-reconcile.js.map

package/dist/fleet-reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fleet-reconcile.js","sourceRoot":"","sources":["../src/fleet-reconcile.ts"],"names":[],"mappings":"AAmEA,6DAA6D;AAC7D,MAAM,CAAC,MAAM,qBAAqB,GAAwB;IACxD,UAAU,EAAE,CAAC;IACb,eAAe,EAAE,CAAC;IAClB,YAAY,EAAE,CAAC;IACf,SAAS,EAAE,KAAK;IAChB,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,KAAK;CACd,CAAC;AAwCF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,OAAO,EAAE,KAAK;IACd,YAAY,EAAE,KAAK;IACnB,mBAAmB,EAAE,CAAC;IACtB,aAAa,EAAE,MAAM;IACrB,YAAY,EAAE,SAAS;IACvB,QAAQ,EAAE,CAAC;IACX,eAAe,EAAE,MAAM;IACvB,aAAa,EACX,+EAA+E;QAC/E,sEAAsE;CACzE,CAAC;AAEF,iDAAiD;AACjD,MAAM,UAAU,sBAAsB,CAAC,SAAoC;IACzE,OAAO,EAAE,GAAG,wBAAwB,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;AAC/D,CAAC;AAqDD,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAiB,EACjB,KAAa;IAEb,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,MAAuB;IACrE,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxD,IAAI,UAAU,EAAE,CAAC;YACf,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,CAAC;YAC3C,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACrD,IAAI,OAAO,IAAI,YAAY,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC,CAAC;YACvE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,mFAAmF;AACnF,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,OAAO,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,+EAA+E;AAE/E,SAAS,UAAU;IACjB,OAAO;QACL,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS;QACvD,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,GAAiB;IAClC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;AAC5E,CAAC;AAED,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAgC,EAChC,IAAmB,EACnB,MAAuB;IAEvB,IAAI,KAAiB,CAAC;IACtB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,OAAO,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE;QAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAmB,EAAE,CAAC;IAChC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAC7E,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,WAAW,IAAI,CAAC,CAAC;QACpD,IAAI,GAAG,CAAC,KAAK;YAAE,EAAE,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IACnD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,CAAC,GAAG,CACN,EAAE,KAAK,CAAC;QACN,CAAC,CAAC,cAAc,IAAI,aAAa,OAAO,8CAA8C;QACtF,CAAC,CAAC,cAAc,IAAI,aAAa,OAAO,qCAAqC,CAChF,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,gHAAgH;AAChH,KAAK,UAAU,cAAc,CAC3B,CAAe,EACf,KAA0B,EAC1B,KAAiB,EACjB,IAAmB,EACnB,MAAuB,EACvB,WAAmB;IAEnB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAEtB,iDAAiD;IACjD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,mDAAmD,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1G,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAElD,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,sFAAsF;QACtF,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,2EAA2E,CAAC;gBACtG,KAAK,EAAE,KAAK;aACb,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAC7C,OAAO;YACL,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,sDAAsD,CAAC,CAAC,SAAS,GAAG,CAAC;YACjG,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,IAAI,MAAM,CAAC,OAAO;YAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC5E,CAAC;IAED,sBAAsB;IACtB,MAAM,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,6CAA6C,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACnG,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,QAAQ,CACrB,CAAe,EACf,IAAmB,EACnB,MAAuB,EACvB,WAAmB;IAEnB,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO;IAC5B,4DAA4D;IAC5D,IAAI,WAAW,GAAG,CAAC;QAAE,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC9D,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,SAAS,CACtB,CAAe,EACf,KAA0B,EAC1B,IAAmB,EACnB,MAAuB;IAEvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACtB,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,OAAO;QAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzE,IAAI,CAAC,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,4CAA4C,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QAC1E,OAAO;IACT,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,sBAAsB,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IAE1G,sFAAsF;IACtF,oFAAoF;IACpF,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,GAAG,CAAC,eAAe,KAAK,yFAAyF,CAAC,CAAC;QACxH,MAAM,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,8CAA8C,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/F,OAAO;IACT,CAAC;IAED,yEAAyE;IACzE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,iBAAiB,KAAK,gCAAgC,KAAK,CAAC,eAAe,uBAAuB,CAAC,CAAC;QAC7G,IAAI,KAAK,CAAC,eAAe,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,eAAe,KAAK,mCAAmC,KAAK,CAAC,eAAe,oBAAoB,CAAC,CAAC;YAC3G,MAAM,UAAU,CAAC,KAAK,EAAE,kCAAkC,KAAK,CAAC,eAAe,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5G,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,sEAAsE;AACtE,KAAK,UAAU,eAAe,CAC5B,CAAe,EACf,KAA0B,EAC1B,CAAS,EACT,IAAmB,EACnB,MAAuB;IAEvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAEtB,oFAAoF;IACpF,IAAI,KAAK,CAAC,eAAe,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG,CAAC,eAAe,KAAK,IAAI,KAAK,CAAC,eAAe,2DAA2D,CAAC,CAAC;QACnH,MAAM,UAAU,CAAC,KAAK,EAAE,UAAU,KAAK,CAAC,eAAe,2BAA2B,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAClG,OAAO;IACT,CAAC;IAED,2DAA2D;IAC3D,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,yCAAyC,KAAK,8DAA8D,CAAC,CAAC;QACvH,MAAM,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,2CAA2C,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5F,OAAO;IACT,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,+BAA+B,KAAK,aAAa,KAAK,CAAC,eAAe,GAAG,CAAC,GAAG,CAAC,CAAC;IACxF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE;YACtB,GAAG,KAAK;YACR,eAAe,EAAE,QAAQ;YACzB,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IACD,MAAM,UAAU,CAAC,KAAK,EAAE,mBAAmB,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAC5E,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,UAAU,CACvB,KAAa,EACb,MAAc,EACd,IAAmB,EACnB,MAAuB;IAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,CAAC,GAAG,CAAC,+BAA+B,KAAK,uBAAuB,CAAC,CAAC;QACtE,OAAO;IACT,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,wBAAwB,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO;IAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,QAA2B,EAAE,MAAc;IACvE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC"}

package/dist/fleet-upgrade.d.ts

@@ -0,0 +1,171 @@
+import type { FleetDriver, FleetState } from './fleet-driver.js';
+import type { WorkspaceRecord } from './discovery.js';
+import type { VerifyGreenOptions, VerifyGreenResult } from './verify-green.js';
+/**
+ * How a discovered fleet member classifies against the target version. Pure,
+ * version-only (the busy-gate is an EXECUTE-time driver decision, not a plan
+ * disposition):
+ * - `behind`    — reachable + running a version `< target` ⇒ a roll candidate.
+ * - `at-target` — reachable + running `>= target` ⇒ skip (already current).
+ * - `offline`   — no comparable running version (unreachable OR a `/health` body
+ *                 with no version) ⇒ skip + report (let reconcile heal it first;
+ *                 a bad-release HALT must never be attributed to a down agent).
+ */
+export type UpgradeDisposition = 'behind' | 'at-target' | 'offline';
+/** One member's version-classification against the target (pure planner output). */
+export interface AgentUpgradePlan {
+    /** The agent's routing label (registry key). */
+    readonly agent: string;
+    /** The fleet (registry identifier) this member belongs to. */
+    readonly fleet: string;
+    /** The agent's live self-reported `/health.version`, or `null` when offline/version-less. */
+    readonly runningVersion: string | null;
+    /** The workspace's on-disk version pin (legible even when the agent is dead). */
+    readonly pinnedVersion: string | null;
+    /** The version-only disposition (see `UpgradeDisposition`). */
+    readonly disposition: UpgradeDisposition;
+}
+/**
+ * The EXECUTE-path outcome of a single agent in the roll:
+ * - `upgraded`     — rolled + came up green on the target.
+ * - `busy-skipped` — busy-gated (or `--wait` timed out still busy); NOT a
+ *                    failure, so the roll CONTINUES to the next agent.
+ * - `halted`       — did not verify-green within the budget; the roll STOPS here.
+ */
+export type RollOutcome = 'upgraded' | 'busy-skipped' | 'halted';
+/** The per-agent EXECUTE result. `detail` carries the green version or halt reason. */
+export interface AgentRollResult {
+    readonly agent: string;
+    readonly outcome: RollOutcome;
+    readonly detail?: string;
+}
+/** The result of rolling ONE fleet (the EXECUTE path). */
+export interface FleetRollResult {
+    readonly results: readonly AgentRollResult[];
+    /** True when a member failed verify-green and the roll stopped early. */
+    readonly halted: boolean;
+    /** Count of agents rolled + verified green. */
+    readonly upgraded: number;
+    /** Count of agents skipped because they were busy. */
+    readonly busySkipped: number;
+}
+/** Progress events for CLI rendering (the RESULT objects are what tests assert on). */
+export type UpgradeEvent = {
+    readonly kind: 'fleet-start';
+    readonly fleet: string;
+    readonly behind: number;
+    readonly total: number;
+} | {
+    readonly kind: 'fleet-skipped';
+    readonly fleet: string;
+    readonly reason: string;
+} | {
+    readonly kind: 'roll-start';
+    readonly agent: string;
+    readonly from: string | null;
+    readonly to: string;
+} | {
+    readonly kind: 'busy-skip';
+    readonly agent: string;
+    readonly waited: boolean;
+} | {
+    readonly kind: 'upgraded';
+    readonly agent: string;
+    readonly version: string;
+} | {
+    readonly kind: 'halt';
+    readonly agent: string;
+    readonly reason: string;
+    readonly lastVersion: string | null;
+};
+/**
+ * Classify each discovered fleet member against the target version. PURE — no
+ * driver, no clock, no network. The single source of the behind/at-target/offline
+ * decision, unit-tested in isolation.
+ */
+export declare function planFleetUpgrade(members: readonly WorkspaceRecord[], state: FleetState, targetVersion: string): readonly AgentUpgradePlan[];
+/** Tunables for a single-fleet roll. */
+export interface RollFleetOptions {
+    /** The version each rolled agent must self-report to be considered green. */
+    readonly targetVersion: string;
+    /** Per-agent verify-green wall-time budget (ms). */
+    readonly verifyTimeoutMs: number;
+    /** `--wait`: on a busy agent, poll for idle instead of skipping. */
+    readonly wait?: boolean;
+    /** Total budget to wait for a busy agent to go idle when `wait` is set (ms). */
+    readonly waitTimeoutMs?: number;
+    /** Re-poll interval while waiting for idle (ms). */
+    readonly waitPollMs?: number;
+}
+/** Injected side-effect seams for the roll (all fakeable in tests). */
+export interface RollFleetDeps {
+    /** The fleet driver whose verbs the sequencer calls. */
+    readonly driver: FleetDriver;
+    /**
+     * The post-restart verify-green runner. Injected whole (not built here) so the
+     * command wires its RE-RESOLVING probe (DR-037 Decision 5 — re-read the registry
+     * each poll for the fresh restart-self port) and tests supply a fake outcome.
+     */
+    readonly verifyGreen: (opts: VerifyGreenOptions) => Promise<VerifyGreenResult>;
+    /** Sleep `ms` (used only by the `--wait` busy re-poll; injected so tests don't wait). */
+    readonly sleep: (ms: number) => Promise<void>;
+    /** Monotonic clock in ms (drives the `--wait` budget; injected in tests). */
+    readonly now: () => number;
+    /** Optional progress sink for CLI rendering. */
+    readonly onEvent?: (ev: UpgradeEvent) => void;
+}
+/** Default total `--wait` budget for a busy agent to go idle (ms). */
+export declare const DEFAULT_WAIT_TIMEOUT_MS = 300000;
+/** Default `--wait` re-poll interval (ms). */
+export declare const DEFAULT_WAIT_POLL_MS = 5000;
+/**
+ * Roll ONE fleet: for every `behind` plan (in order) run the busy-gate → upgrade
+ * → restart → verify-green cycle, HALTING the roll on the first agent that fails
+ * to come up green. Non-`behind` plans are ignored (planned skips). Busy agents
+ * are skipped + reported (never a halt); with `wait`, they are re-polled for idle
+ * first. The verb order + HALT semantics mirror `fleet/upgrade.sh`'s `roll_one`.
+ */
+export declare function rollFleet(plans: readonly AgentUpgradePlan[], opts: RollFleetOptions, deps: RollFleetDeps): Promise<FleetRollResult>;
+/** Options for the multi-fleet orchestrator. */
+export interface UpgradeFleetsOptions extends RollFleetOptions {
+    /** `false` = dry-run: probe + plan + report, NO driver mutations (the default). */
+    readonly execute: boolean;
+}
+/** Injected seams for the multi-fleet orchestrator. */
+export interface UpgradeFleetsDeps {
+    /**
+     * Resolve the driver for a fleet (registry identifier). Production binds
+     * `createVmDriverFromConfig` at a representative workspace; tests return a fake.
+     * `null` ⇒ the fleet cannot be reached (skip + report, NOT a halt).
+     */
+    readonly resolveDriver: (fleet: string) => Promise<FleetDriver | null>;
+    readonly verifyGreen: (opts: VerifyGreenOptions) => Promise<VerifyGreenResult>;
+    readonly sleep: (ms: number) => Promise<void>;
+    readonly now: () => number;
+    readonly onEvent?: (ev: UpgradeEvent) => void;
+}
+/** One fleet's slice of the run report. */
+export interface FleetPlanReport {
+    readonly fleet: string;
+    readonly plans: readonly AgentUpgradePlan[];
+    /** Present only in EXECUTE mode. */
+    readonly rolled?: FleetRollResult;
+    /** Present when the fleet was skipped whole (e.g. driver unresolved). */
+    readonly skipped?: string;
+}
+/** The full run report across all selected fleets. */
+export interface FleetUpgradeReport {
+    readonly target: string;
+    readonly fleets: readonly FleetPlanReport[];
+    /** True when a fleet HALTED (and later fleets were consequently not started). */
+    readonly halted: boolean;
+}
+/**
+ * Orchestrate a rolling upgrade across the ORDERED `fleets` (registry
+ * identifiers), fleet-by-fleet. Each fleet is probed + planned; in EXECUTE mode
+ * it is then rolled, and a HALT in one fleet STOPS the whole run — later fleets
+ * are never started (a bad release cannot cascade across fleets). An unresolvable
+ * fleet is skipped + reported and does NOT halt the run.
+ */
+export declare function upgradeFleets(fleets: readonly string[], opts: UpgradeFleetsOptions, deps: UpgradeFleetsDeps): Promise<FleetUpgradeReport>;
+//# sourceMappingURL=fleet-upgrade.d.ts.map

package/dist/fleet-upgrade.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fleet-upgrade.d.ts","sourceRoot":"","sources":["../src/fleet-upgrade.ts"],"names":[],"mappings":"AAiCA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;GASG;AACH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;AAEpE,oFAAoF;AACpF,MAAM,WAAW,gBAAgB;IAC/B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8DAA8D;IAC9D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,6FAA6F;IAC7F,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,iFAAiF;IACjF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,+DAA+D;IAC/D,QAAQ,CAAC,WAAW,EAAE,kBAAkB,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,QAAQ,CAAC;AAEjE,uFAAuF;AACvF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,+CAA+C;IAC/C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,uFAAuF;AACvF,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzG;IAAE,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnF;IAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC1G;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GAChF;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAEpH;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,KAAK,EAAE,UAAU,EACjB,aAAa,EAAE,MAAM,GACpB,SAAS,gBAAgB,EAAE,CAqB7B;AAED,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,6EAA6E;IAC7E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,oDAAoD;IACpD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,oEAAoE;IACpE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,gFAAgF;IAChF,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,oDAAoD;IACpD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,kBAAkB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC/E,yFAAyF;IACzF,QAAQ,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,sEAAsE;AACtE,eAAO,MAAM,uBAAuB,SAAU,CAAC;AAC/C,8CAA8C;AAC9C,eAAO,MAAM,oBAAoB,OAAQ,CAAC;AAmB1C;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,KAAK,EAAE,SAAS,gBAAgB,EAAE,EAClC,IAAI,EAAE,gBAAgB,EACtB,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,eAAe,CAAC,CAiD1B;AAED,gDAAgD;AAChD,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC5D,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IACvE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,kBAAkB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC/E,QAAQ,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,2CAA2C;AAC3C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC5C,oCAAoC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC;IAClC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAC5C,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,IAAI,EAAE,oBAAoB,EAC1B,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC,kBAAkB,CAAC,CAsC7B"}

package/dist/fleet-upgrade.js

@@ -0,0 +1,177 @@
+/**
+ * The rolling fleet-upgrade DECISION layer (DR-037 / macf#682 Phase 2) — the
+ * runtime-agnostic sequencer that promotes the devops `fleet/upgrade.sh`
+ * reference impl onto the framework substrate.
+ *
+ * It is the pure decision half of the DR-037 decision/driver split (OQ2 — pure
+ * logic lives in macf-core; the per-runtime driver bodies live in `packages/macf`
+ * or a future K8s package). It NEVER touches `/proc`, tmux, `capture-pane`, or
+ * kubectl — it drives entirely off the injected `FleetDriver` verbs
+ * (`probe`/`discoverWorkspaces`/`isBusy`/`upgrade`/`restart`) plus an injected
+ * `verifyGreen` runner, so the SAME sequencer rolls a VM tmux fleet, a macOS
+ * host, and a far-future K8s deployment by swapping only the driver.
+ *
+ * The state machine (per fleet, one agent at a time):
+ *
+ *   probe() + discoverWorkspaces()
+ *     → for each member whose RUNNING version < target (compareSemver):
+ *         busy-gate (driver.isBusy → busy ⇒ SKIP+REPORT, or `--wait` for idle)
+ *           → driver.upgrade(agent)
+ *           → driver.restart(agent)
+ *           → verifyGreen({agent, target})   // re-resolves the fresh endpoint
+ *               → green ⇒ next agent
+ *               → NOT green within the budget ⇒ HALT the roll (a bad release
+ *                 stalls at agent 1 and CANNOT brick the fleet)
+ *
+ * Multi-fleet (`--fleet a,b,c`) rolls fleet-by-fleet: each fleet's roll must
+ * finish without a HALT before the next fleet starts (a bad release in fleet-1
+ * stops fleet-2 from ever being touched).
+ *
+ * DRY-RUN is the default at the command boundary: `execute: false` computes +
+ * reports the PLAN (probe + classify) and performs NO driver mutations.
+ */
+import { compareSemver } from './semver.js';
+/**
+ * Classify each discovered fleet member against the target version. PURE — no
+ * driver, no clock, no network. The single source of the behind/at-target/offline
+ * decision, unit-tested in isolation.
+ */
+export function planFleetUpgrade(members, state, targetVersion) {
+    const byName = new Map(state.agents.map((a) => [a.name, a]));
+    return members.map((m) => {
+        const live = byName.get(m.agent);
+        const runningVersion = live?.online ? (live.version ?? null) : null;
+        let disposition;
+        if (runningVersion === null) {
+            disposition = 'offline';
+        }
+        else if (compareSemver(runningVersion, targetVersion) >= 0) {
+            disposition = 'at-target';
+        }
+        else {
+            disposition = 'behind';
+        }
+        return {
+            agent: m.agent,
+            fleet: m.registry,
+            runningVersion,
+            pinnedVersion: m.versionPin,
+            disposition,
+        };
+    });
+}
+/** Default total `--wait` budget for a busy agent to go idle (ms). */
+export const DEFAULT_WAIT_TIMEOUT_MS = 300_000;
+/** Default `--wait` re-poll interval (ms). */
+export const DEFAULT_WAIT_POLL_MS = 5_000;
+/**
+ * Poll `driver.isBusy(agent)` until it reports idle or the budget is exhausted.
+ * Returns `true` if the agent became idle (safe to roll), `false` if it stayed
+ * busy for the whole budget. Called only after an initial busy reading, so it
+ * sleeps-then-checks and always terminates on the injected clock.
+ */
+async function waitForIdle(agent, opts, deps) {
+    const budget = opts.waitTimeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
+    const poll = opts.waitPollMs ?? DEFAULT_WAIT_POLL_MS;
+    const start = deps.now();
+    while (deps.now() - start < budget) {
+        await deps.sleep(poll);
+        if (!(await deps.driver.isBusy(agent)))
+            return true;
+    }
+    return false;
+}
+/**
+ * Roll ONE fleet: for every `behind` plan (in order) run the busy-gate → upgrade
+ * → restart → verify-green cycle, HALTING the roll on the first agent that fails
+ * to come up green. Non-`behind` plans are ignored (planned skips). Busy agents
+ * are skipped + reported (never a halt); with `wait`, they are re-polled for idle
+ * first. The verb order + HALT semantics mirror `fleet/upgrade.sh`'s `roll_one`.
+ */
+export async function rollFleet(plans, opts, deps) {
+    const results = [];
+    let upgraded = 0;
+    let busySkipped = 0;
+    for (const plan of plans) {
+        if (plan.disposition !== 'behind')
+            continue;
+        const agent = plan.agent;
+        let busy = await deps.driver.isBusy(agent);
+        let waited = false;
+        if (busy && opts.wait) {
+            waited = true;
+            busy = !(await waitForIdle(agent, opts, deps));
+        }
+        if (busy) {
+            busySkipped += 1;
+            results.push({ agent, outcome: 'busy-skipped', detail: waited ? 'still busy after --wait' : 'busy' });
+            deps.onEvent?.({ kind: 'busy-skip', agent, waited });
+            continue;
+        }
+        deps.onEvent?.({ kind: 'roll-start', agent, from: plan.runningVersion, to: opts.targetVersion });
+        await deps.driver.upgrade(agent);
+        await deps.driver.restart(agent);
+        const green = await deps.verifyGreen({
+            agent,
+            targetVersion: opts.targetVersion,
+            timeoutMs: opts.verifyTimeoutMs,
+        });
+        if (green.ok) {
+            upgraded += 1;
+            results.push({ agent, outcome: 'upgraded', detail: green.version });
+            deps.onEvent?.({ kind: 'upgraded', agent, version: green.version });
+            continue;
+        }
+        const reason = green.reason;
+        results.push({
+            agent,
+            outcome: 'halted',
+            detail: `verify-green ${reason} (last=${green.lastVersion ?? 'down'})`,
+        });
+        deps.onEvent?.({ kind: 'halt', agent, reason, lastVersion: green.lastVersion });
+        return { results, halted: true, upgraded, busySkipped };
+    }
+    return { results, halted: false, upgraded, busySkipped };
+}
+/**
+ * Orchestrate a rolling upgrade across the ORDERED `fleets` (registry
+ * identifiers), fleet-by-fleet. Each fleet is probed + planned; in EXECUTE mode
+ * it is then rolled, and a HALT in one fleet STOPS the whole run — later fleets
+ * are never started (a bad release cannot cascade across fleets). An unresolvable
+ * fleet is skipped + reported and does NOT halt the run.
+ */
+export async function upgradeFleets(fleets, opts, deps) {
+    const reports = [];
+    let halted = false;
+    for (const fleet of fleets) {
+        const driver = await deps.resolveDriver(fleet);
+        if (!driver) {
+            reports.push({ fleet, plans: [], skipped: 'driver-unresolved' });
+            deps.onEvent?.({ kind: 'fleet-skipped', fleet, reason: 'driver-unresolved' });
+            continue;
+        }
+        const members = driver.discoverWorkspaces().filter((r) => r.registry === fleet);
+        const state = await driver.probe();
+        const plans = planFleetUpgrade(members, state, opts.targetVersion);
+        const behind = plans.filter((p) => p.disposition === 'behind').length;
+        deps.onEvent?.({ kind: 'fleet-start', fleet, behind, total: plans.length });
+        if (!opts.execute) {
+            reports.push({ fleet, plans });
+            continue;
+        }
+        const rolled = await rollFleet(plans, opts, {
+            driver,
+            verifyGreen: deps.verifyGreen,
+            sleep: deps.sleep,
+            now: deps.now,
+            onEvent: deps.onEvent,
+        });
+        reports.push({ fleet, plans, rolled });
+        if (rolled.halted) {
+            halted = true;
+            break; // do NOT start the next fleet — a bad release stops the whole run
+        }
+    }
+    return { target: opts.targetVersion, fleets: reports, halted };
+}
+//# sourceMappingURL=fleet-upgrade.js.map

package/dist/fleet-upgrade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fleet-upgrade.js","sourceRoot":"","sources":["../src/fleet-upgrade.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAmE5C;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAmC,EACnC,KAAiB,EACjB,aAAqB;IAErB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,cAAc,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpE,IAAI,WAA+B,CAAC;QACpC,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAC5B,WAAW,GAAG,SAAS,CAAC;QAC1B,CAAC;aAAM,IAAI,aAAa,CAAC,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7D,WAAW,GAAG,WAAW,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,QAAQ,CAAC;QACzB,CAAC;QACD,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAE,CAAC,CAAC,QAAQ;YACjB,cAAc;YACd,aAAa,EAAE,CAAC,CAAC,UAAU;YAC3B,WAAW;SACZ,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAkCD,sEAAsE;AACtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAC/C,8CAA8C;AAC9C,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAE1C;;;;;GAKG;AACH,KAAK,UAAU,WAAW,CAAC,KAAa,EAAE,IAAsB,EAAE,IAAmB;IACnF,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAuB,CAAC;IAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,IAAI,oBAAoB,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,MAAM,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAkC,EAClC,IAAsB,EACtB,IAAmB;IAEnB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ;YAAE,SAAS;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAEzB,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACtB,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,GAAG,CAAC,CAAC,MAAM,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,WAAW,IAAI,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACtG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;YACnC,KAAK;YACL,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,SAAS,EAAE,IAAI,CAAC,eAAe;SAChC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC;YACb,QAAQ,IAAI,CAAC,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC;YACX,KAAK;YACL,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,gBAAgB,MAAM,UAAU,KAAK,CAAC,WAAW,IAAI,MAAM,GAAG;SACvE,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAChF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAC3D,CAAC;AAwCD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAyB,EACzB,IAA0B,EAC1B,IAAuB;IAEvB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE;YAC1C,MAAM;YACN,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,GAAG,IAAI,CAAC;YACd,MAAM,CAAC,kEAAkE;QAC3E,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACjE,CAAC"}