@hivemagnus/magnus-server 1.1.2 → 1.1.4

package/dist/services/agent-upgrade-service.js

@@ -2,29 +2,54 @@
  * Server-driven agent upgrades (MAGNUS_AGENT_UPGRADES). The server pins one
  * exact version and its npm integrity, asks one canary agent to stage it,
  * waits until that agent reports the version on trial as confirmed, then
- * upgrades the rest a few at a time. The first revert or failure stops the
- * run. Agents decide for themselves whether they can upgrade
- * (MAGNUS_AGENT_SELF_UPGRADE, launcher layout) and report it with every
- * heartbeat, so the console can grey out the ones that cannot.
+ * upgrades the rest a few at a time. A revert, a release problem or a node
+ * that goes silent after the request stops the run; a node that refuses
+ * before anything changed (busy, enrollment, unreachable) is recorded as
+ * `refused` and the wave goes on. Agents decide for themselves whether they
+ * can upgrade (MAGNUS_AGENT_SELF_UPGRADE, launcher layout) and report it with
+ * every heartbeat, so the console can grey out the ones that cannot.
+ *
+ * Eligibility uses the same read-only predicates as resource admission
+ * (admission fence, enrollment, held work and leases, fresh health), so a node
+ * the console offers is one that dispatch will not refuse for those reasons.
  */
 import { randomUUID } from 'node:crypto';
-import { AGENT_SELF_UPGRADE_PROTOCOL, agentSelfUpgradeStatusSchema, checkUpgradeAllowed, compareVersions, isVersion, } from '@hivemagnus/magnus-contracts';
+import { AGENT_SELF_UPGRADE_PROTOCOL, agentSelfUpgradeStatusSchema, checkUpgradeAllowed, compareVersions, isVersion, parseVersion, } from '@hivemagnus/magnus-contracts';
 import { upgradeSettings } from '../config/features.js';
 import { AGENT_VERSION } from '../config/version.js';
 import { getDb } from '../db/client.js';
 import { HttpError } from '../middleware/error-handler.js';
-import { NodeApiClient } from '../sdk/node-api-client.js';
+import { AgentResponseError, DispatchOutcomeUnknownError, NodeApiClient, NotDispatchedError } from '../sdk/node-api-client.js';
 import { getNodeManagerSecret } from '../utils/env.js';
 import { getNodeBaseUrl } from '../utils/node-url.js';
 import { SENTINEL_NODE_IDS } from '../utils/sentinel-nodes.js';
 import { logAuditEvent } from './audit-service.js';
+import { HELD_RESOURCE_STATES, hasFreshOnlineHealth, legacyAdmissionPossible, resourceLeasesRequested } from './command-resource-leases.js';
+import { enrollmentIsComplete } from './resource-enrollment.js';
+import { stickyEnrollmentCause } from './resource-ownership.js';
 export const AGENT_PACKAGE = '@hivemagnus/magnus-agent';
-/** Heartbeats are 30 s apart; older than this and the node counts as offline for upgrades. */
-const FRESH_HEARTBEAT_MS = 2 * 60_000;
-/** Install time plus the agent's confirmation window (default 180 s), with margin. */
-const CONFIRM_TIMEOUT_MS = 20 * 60_000;
+const HEARTBEAT_MS = 30_000;
+/** MAGNUS_AGENT_UPGRADE_CONFIRM_SECONDS default, for agents that do not report theirs. */
+const DEFAULT_CONFIRM_SECONDS = 180;
+/** Restart and first heartbeat of the new version, on top of the trial window. */
+const CONFIRM_MARGIN_MS = 2 * 60_000;
+/** A request whose reply was lost may still be installing. */
+const UNKNOWN_DISPATCH_MARGIN_MS = 10 * 60_000;
 const POLL_MS = 5_000;
 const RUN_TTL_SECONDS = 7 * 24 * 60 * 60;
+/** The owner renews its lease every LEASE_MS / 3; a lease this old means the owner is gone. */
+const LEASE_MS = 90_000;
+/** Runs written by servers without leases count as abandoned after this long. */
+const UNLEASED_RUN_MS = 2 * 60 * 60_000;
+const MAX_LOCAL_RUNS = 20;
+/** This process, as the owner of the runs it executes. */
+export const INSTANCE_ID = randomUUID();
+/** Agent error codes that mean the node was not touched. */
+export const UNTOUCHED_CODES = new Set(['SELF_UPGRADE_BUSY', 'SELF_UPGRADE_CURRENT', 'SELF_UPGRADE_UNAVAILABLE', 'SELF_UPGRADE_PACKAGE',
+    'SELF_UPGRADE_REFUSED', 'SELF_UPGRADE_LAUNCHER', 'SELF_UPGRADE_ENGINE', 'SELF_UPGRADE_TIMEOUT',
+    'RESOURCE_BUSY', 'UNMANAGED_MUTATION', 'ENROLLMENT_AUTHORITY_UNAVAILABLE', 'RESOURCE_CLAIM_INVALID']);
+/** Agent error codes that point at the release itself: the wave stops. */
+export const RELEASE_PROBLEM_CODES = new Set(['SELF_UPGRADE_INTEGRITY', 'SELF_UPGRADE_FAILED', 'SELF_UPGRADE_SOURCE']);
 function parseMetadata(raw) {
     try {
         const parsed = JSON.parse(raw ?? '{}');
@@ -34,50 +59,121 @@ function parseMetadata(raw) {
         return {};
     }
 }
+const parseWarnings = new Map();
+/**
+ * Parse a self-upgrade report leniently: a malformed `upgrade` entry is
+ * dropped instead of the whole report. Failures are logged (once an hour per source).
+ */
+export function parseSelfUpgradeStatus(value, source) {
+    if (value === undefined || value === null)
+        return null;
+    const parsed = agentSelfUpgradeStatusSchema.safeParse(value);
+    if (parsed.success)
+        return parsed.data;
+    const partial = typeof value === 'object' && !Array.isArray(value)
+        ? agentSelfUpgradeStatusSchema.safeParse({ ...value, upgrade: null }) : null;
+    if (source && Date.now() - (parseWarnings.get(source) ?? 0) > 60 * 60_000) {
+        parseWarnings.set(source, Date.now());
+        console.warn(`[agent-upgrade] ${source} sent a self-upgrade report that does not parse (${parsed.error.issues.slice(0, 3)
+            .map(issue => `${issue.path.join('.') || '<root>'}: ${issue.message}`).join('; ')})${partial?.success ? '; kept it without its upgrade entry' : '; ignored it'}`);
+    }
+    return partial?.success ? partial.data : null;
+}
 export function storedSelfUpgrade(metadataRaw) {
     const value = parseMetadata(metadataRaw)['agentSelfUpgrade'];
-    const parsed = agentSelfUpgradeStatusSchema.safeParse(value);
-    if (!parsed.success)
+    const parsed = parseSelfUpgradeStatus(value);
+    if (!parsed)
         return null;
     const extra = value;
-    return { ...parsed.data, nodeVersion: typeof extra.nodeVersion === 'string' ? extra.nodeVersion : null,
+    return { ...parsed, nodeVersion: typeof extra.nodeVersion === 'string' ? extra.nodeVersion : null,
         observedAt: typeof extra.observedAt === 'string' ? extra.observedAt : new Date(0).toISOString() };
 }
+const AGENT_STATUSES = new Set(['RUNNING', 'STOPPED', 'ERROR']);
+const priorStatus = (metadata) => {
+    const value = metadata['agentStatusBeforeUpgrade'];
+    return typeof value === 'string' && AGENT_STATUSES.has(value) ? value : 'RUNNING';
+};
+/** End UPDATING (node went offline, or the agent stopped reporting an upgrade) and restore the status it had before. */
+export async function clearAgentUpdating(nodeId) {
+    const db = getDb();
+    const agent = await db.agent.findUnique({ where: { nodeId }, select: { status: true } });
+    if (agent?.status !== 'UPDATING')
+        return;
+    const row = await db.node.findUnique({ where: { id: nodeId }, select: { metadata: true } });
+    const metadata = parseMetadata(row?.metadata);
+    await db.agent.updateMany({ where: { nodeId, status: 'UPDATING' }, data: { status: priorStatus(metadata) } });
+    if ('agentStatusBeforeUpgrade' in metadata) {
+        delete metadata['agentStatusBeforeUpgrade'];
+        await db.node.update({ where: { id: nodeId }, data: { metadata: JSON.stringify(metadata) } });
+    }
+}
 /**
  * Store the self-upgrade report from a heartbeat, keep Agent.status in step
- * (UPDATING while a version is staged or on trial) and audit each revert once.
+ * (UPDATING while a version is staged or on trial, then the status it had
+ * before), audit each confirmation and revert once, and record outcomes that
+ * arrive after their run stopped watching.
  */
-export async function recordSelfUpgradeReport(node, body) {
-    const parsed = agentSelfUpgradeStatusSchema.safeParse(body.selfUpgrade);
-    if (!parsed.success)
+export async function recordSelfUpgradeReport(node, body, store = defaultRunStore()) {
+    const parsed = parseSelfUpgradeStatus(body.selfUpgrade, node.name);
+    if (!parsed) {
+        await clearAgentUpdating(node.id);
         return;
+    }
     const db = getDb();
     const row = await db.node.findUnique({ where: { id: node.id }, select: { metadata: true } });
     const metadata = parseMetadata(row?.metadata);
     const previous = storedSelfUpgrade(row?.metadata);
     const nodeVersion = typeof body.nodeVersion === 'string' ? body.nodeVersion.slice(0, 32) : null;
-    const next = { ...parsed.data, nodeVersion, observedAt: new Date().toISOString() };
-    const unchanged = previous && JSON.stringify({ ...previous, observedAt: '' }) === JSON.stringify({ ...next, observedAt: '' })
-        && Date.now() - Date.parse(previous.observedAt) < 10 * 60_000;
-    if (!unchanged) {
-        await db.node.update({ where: { id: node.id }, data: { metadata: JSON.stringify({ ...metadata, agentSelfUpgrade: next, nodeVersion }) } });
-    }
-    const state = parsed.data.upgrade?.state;
+    const next = { ...parsed, nodeVersion, observedAt: new Date().toISOString() };
+    const nextMetadata = { ...metadata, agentSelfUpgrade: next, nodeVersion };
+    let changed = !(previous && JSON.stringify({ ...previous, observedAt: '' }) === JSON.stringify({ ...next, observedAt: '' })
+        && Date.now() - Date.parse(previous.observedAt) < 10 * 60_000);
+    const state = parsed.upgrade?.state;
     const updating = state === 'staged' || state === 'trial';
-    await db.agent.updateMany({
-        where: { nodeId: node.id, status: updating ? { not: 'UPDATING' } : 'UPDATING' },
-        data: { status: updating ? 'UPDATING' : 'RUNNING' },
-    });
-    const upgrade = parsed.data.upgrade;
-    if (upgrade && (upgrade.state === 'reverted' || upgrade.state === 'confirmed')
-        && upgrade.requestId && previous?.upgrade?.requestId === upgrade.requestId && previous.upgrade.state !== upgrade.state) {
-        await logAuditEvent({
-            action: upgrade.state === 'reverted' ? 'agent.upgrade_reverted' : 'agent.upgrade',
-            actorId: 'system:agent-upgrade', actorType: 'system', targetType: 'node', targetId: node.id,
-            details: { node: node.name, from: upgrade.from, to: upgrade.to, requestId: upgrade.requestId, result: upgrade.state, reason: upgrade.reason ?? null },
-        }).catch(() => undefined);
+    const agent = await db.agent.findUnique({ where: { nodeId: node.id }, select: { status: true } });
+    if (agent && updating && agent.status !== 'UPDATING') {
+        nextMetadata['agentStatusBeforeUpgrade'] = agent.status;
+        changed = true;
+        await db.agent.updateMany({ where: { nodeId: node.id, status: { not: 'UPDATING' } }, data: { status: 'UPDATING' } });
+    }
+    else if (agent && !updating && agent.status === 'UPDATING') {
+        delete nextMetadata['agentStatusBeforeUpgrade'];
+        changed = true;
+        await db.agent.updateMany({ where: { nodeId: node.id, status: 'UPDATING' }, data: { status: priorStatus(metadata) } });
     }
+    if (changed)
+        await db.node.update({ where: { id: node.id }, data: { metadata: JSON.stringify(nextMetadata) } });
+    const upgrade = parsed.upgrade;
+    if (upgrade && (upgrade.state === 'reverted' || upgrade.state === 'confirmed' || upgrade.state === 'failed') && upgrade.requestId
+        && !(previous?.upgrade?.requestId === upgrade.requestId && previous.upgrade.state === upgrade.state)) {
+        const runId = await store.runForRequest(upgrade.requestId).catch(() => null);
+        if (upgrade.state !== 'failed') {
+            await logAuditEvent({
+                action: upgrade.state === 'reverted' ? 'agent.upgrade.node_reverted' : 'agent.upgrade.node_confirmed',
+                actorId: 'system:agent-upgrade', actorType: 'system', targetType: 'node', targetId: node.id,
+                details: { node: node.name, from: upgrade.from, to: upgrade.to, requestId: upgrade.requestId, runId, result: upgrade.state, reason: upgrade.reason ?? null },
+            }).catch(() => undefined);
+        }
+        if (runId)
+            await recordLateOutcome(store, runId, upgrade.requestId, upgrade.state, upgrade.reason ?? null).catch(() => undefined);
+    }
+}
+/** A report for a node whose run already ended without knowing its outcome. */
+async function recordLateOutcome(store, runId, requestId, state, reason) {
+    const run = await store.get(runId);
+    if (!run || run.status === 'running')
+        return; // the owner is watching and records it itself
+    const entry = run.nodes.find(node => node.requestId === requestId);
+    if (!entry || !(entry.awaitingReport || entry.state === 'requested'))
+        return;
+    entry.state = state;
+    entry.code = state === 'confirmed' ? null : entry.code;
+    entry.detail = `${state === 'confirmed' ? 'confirmed' : state === 'reverted' ? `reverted: ${reason ?? 'the agent switched back'}` : `failed: ${reason ?? 'the agent reported a failure'}`} (reported after the run ended)`;
+    entry.awaitingReport = false;
+    run.summary = summarizeRun(run);
+    await store.save(run);
 }
+export const isPrerelease = (version) => (parseVersion(version)?.prerelease.length ?? 0) > 0;
 /** The exact version (default: the one this server ships) and its npm integrity. */
 export async function resolveAgentRelease(options = {}) {
     const version = options.version ?? AGENT_VERSION;
@@ -92,7 +188,7 @@ export async function resolveAgentRelease(options = {}) {
     const url = `${registry}/${AGENT_PACKAGE.replace('/', '%2f')}/${version}`;
     let response;
     try {
-        response = await (options.fetchImpl ?? fetch)(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(15_000) });
+        response = await (options.fetchImpl ?? fetch)(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(options.timeoutMs ?? 15_000) });
     }
     catch (error) {
         throw new HttpError(502, 'REGISTRY_UNREACHABLE', `Cannot read ${AGENT_PACKAGE}@${version} from ${registry}: ${error instanceof Error ? error.message : String(error)}. Pass the integrity explicitly for offline installs.`);
@@ -107,14 +203,34 @@ export async function resolveAgentRelease(options = {}) {
         throw new HttpError(502, 'REGISTRY_ERROR', `${AGENT_PACKAGE}@${version} has no sha512 integrity`);
     return { version, integrity };
 }
-/** Pure, for tests: why one node can or cannot take `target` now. */
-export function classifyCandidate(row, target, now = Date.now()) {
+const verifications = new Map();
+/** Whether the registry has `version` (cached ten minutes); eligibility shows unpublished targets as unverified. */
+export async function verifyAgentRelease(version, fetchImpl) {
+    const cached = verifications.get(version);
+    if (cached && Date.now() - cached.at < 10 * 60_000)
+        return cached.value;
+    let value;
+    try {
+        const release = await resolveAgentRelease({ version, fetchImpl, timeoutMs: 5_000 });
+        value = { verified: true, integrity: release.integrity, reason: null };
+    }
+    catch (error) {
+        value = { verified: false, integrity: null, reason: error instanceof Error ? error.message : String(error) };
+    }
+    verifications.set(version, { at: Date.now(), value });
+    return value;
+}
+const day = (value) => value ? new Date(value).toISOString().slice(0, 10) : 'an unknown date';
+const ENROLLMENT_DOC = 'docs/COMMAND_RESOURCE_LEASES.md';
+/** Pure, for tests: why one node can or cannot take `target` now. Mirrors the dispatch preconditions. */
+export function classifyCandidate(row, target, options = {}) {
+    const now = options.now ?? Date.now();
     const version = row.agent?.version ?? 'unknown';
     const report = storedSelfUpgrade(row.metadata);
     const base = { nodeId: row.id, name: row.name, version, upgrade: report?.upgrade ?? null, layout: report?.layout ?? null };
     const no = (reason) => ({ ...base, eligible: false, reason });
-    if (row.status !== 'ONLINE' || !row.lastHeartbeatAt || now - row.lastHeartbeatAt.getTime() > FRESH_HEARTBEAT_MS)
-        return no('offline or no recent heartbeat');
+    if (!hasFreshOnlineHealth(row, now))
+        return no(`offline or no recent heartbeat${row.lastHeartbeatAt ? ` (last ${row.lastHeartbeatAt.toISOString()})` : ''}`);
     if (!isVersion(version))
         return no('agent version unknown');
     if (compareVersions(version, target) === 0)
@@ -128,166 +244,510 @@ export function classifyCandidate(row, target, now = Date.now()) {
         return no('agent self-upgrade protocol too old');
     if (!report.enabled)
         return no(report.reason ?? 'self-upgrade disabled on this node');
-    if (report.upgrade && (report.upgrade.state === 'staged' || report.upgrade.state === 'trial'))
-        return no(`an upgrade to ${report.upgrade.to} is in progress`);
+    const upgrade = report.upgrade;
+    if (upgrade && (upgrade.state === 'staged' || upgrade.state === 'trial'))
+        return no(`an upgrade to ${upgrade.to} is in progress`);
+    if (upgrade?.state === 'reverted' && upgrade.to === target && !options.allowRetryReverted)
+        return no(`reverted ${target} at ${upgrade.updatedAt ?? report.observedAt}: ${upgrade.reason ?? 'no reason given'}; try again only with allowRetryReverted`);
+    if (row.admissionFence)
+        return no(`release ${row.admissionFence.releaseId} closed admission to this node at ${row.admissionFence.closedAt.toISOString()}; release the fence first`);
+    const enrollment = row.resourceEnrollment ?? null;
+    const hostname = row.hostname ?? '';
+    const marker = report.managedEnrollment;
+    if (marker && (!enrollmentIsComplete(enrollment, marker.nodeFqdn) || marker.nodeId !== row.id || marker.nodeFqdn !== hostname)) {
+        const server = enrollment ? `a ${enrollment.state} enrollment for ${enrollment.nodeFqdn}` : 'no enrollment row';
+        const cause = stickyEnrollmentCause(enrollment, { lease: Boolean(row.leaseHistory), managedCron: Boolean(row.managedCronHistory),
+            participantWork: Boolean(row.participantHistory) });
+        return no(`agent and server disagree about resource enrollment: the agent holds a managed-enrollment marker for ${marker.nodeFqdn} but the server has ${server}`
+            + `${cause ? ` (the server answers this agent as enrolled because of ${cause})` : ''}; `
+            + `complete the node's resource enrollment on the server (${ENROLLMENT_DOC}), or upgrade this agent by hand on the node`);
+    }
+    // Same rule as admission: off the legacy path once leases are on, enrollment started, or any lease exists.
+    const managed = !legacyAdmissionPossible(enrollment, options.leasesRequested ?? resourceLeasesRequested()) || Boolean(row.leaseHistory);
+    if (managed) {
+        if (!enrollmentIsComplete(enrollment, hostname)) {
+            if (!enrollment)
+                return no(`resource leases apply to this node (${row.leaseHistory ? 'it has lease history' : 'MAGNUS_RESOURCE_LEASES_ENABLED=true'}) but it is not enrolled; enroll it first (${ENROLLMENT_DOC})`);
+            if (enrollment.state === 'draining')
+                return no(`resource enrollment stuck in draining since ${day(enrollment.startedAt ?? enrollment.updatedAt)}; finish the drain (POST /api/v1/resource-leases/enrollment/${row.id}/complete) first`);
+            if (enrollment.state === 'enrolled')
+                return no(`resource enrollment is for ${enrollment.nodeFqdn} but the inventory host name is ${hostname || 'unknown'}; fix the inventory host name first`);
+            return no(`resource enrollment is ${enrollment.state} while resource leases apply to this node; enroll it first (${ENROLLMENT_DOC})`);
+        }
+        if (row.heldWork)
+            return no(`${row.heldWork} participant work item(s) still held on this node; reconcile them first`);
+        if (row.heldLeases)
+            return no(`${row.heldLeases} command lease(s) unsettled on this node (reserved, running or unknown); reconcile them first`);
+    }
     return { ...base, eligible: true, reason: null };
 }
-export async function listUpgradeCandidates(target, nodeIds) {
-    const rows = await getDb().node.findMany({
-        where: { id: { notIn: SENTINEL_NODE_IDS, ...(nodeIds?.length ? { in: [...nodeIds] } : {}) }, status: { not: 'DECOMMISSIONED' } },
-        select: { id: true, name: true, status: true, lastHeartbeatAt: true, metadata: true, agent: { select: { version: true } } },
+/** Candidates for `target`; `nodeIds: []` selects nothing (never the whole fleet). */
+export async function listUpgradeCandidates(target, nodeIds, options = {}) {
+    if (nodeIds && nodeIds.length === 0)
+        return [];
+    const db = getDb();
+    const rows = await db.node.findMany({
+        where: { id: { notIn: SENTINEL_NODE_IDS, ...(nodeIds ? { in: [...nodeIds] } : {}) }, status: { not: 'DECOMMISSIONED' } },
+        select: { id: true, name: true, hostname: true, status: true, lastHeartbeatAt: true, metadata: true, agent: { select: { version: true } },
+            resourceEnrollment: { select: { state: true, nodeFqdn: true, startedAt: true, updatedAt: true } },
+            admissionFence: { select: { releaseId: true, closedAt: true } } },
         orderBy: { name: 'asc' },
     });
-    return rows.map(row => classifyCandidate(row, target));
+    const ids = rows.map(row => row.id);
+    const [leases, work, crons] = ids.length ? await Promise.all([
+        db.commandResourceLease.groupBy({ by: ['nodeId', 'state'], where: { nodeId: { in: ids } }, _count: { _all: true } }),
+        db.resourceParticipantWork.groupBy({ by: ['nodeId', 'state'], where: { nodeId: { in: ids } }, _count: { _all: true } }),
+        db.managedCronDefinition.groupBy({ by: ['nodeId'], where: { nodeId: { in: ids } }, _count: { _all: true } }),
+    ]) : [[], [], []];
+    const held = (items, nodeId) => items.filter(item => item.nodeId === nodeId && HELD_RESOURCE_STATES.includes(item.state)).reduce((sum, item) => sum + item._count._all, 0);
+    return rows.map(row => classifyCandidate({ ...row,
+        leaseHistory: leases.some(lease => lease.nodeId === row.id),
+        heldLeases: held(leases, row.id),
+        heldWork: held(work, row.id),
+        participantHistory: work.some(item => item.nodeId === row.id),
+        managedCronHistory: crons.some(item => item.nodeId === row.id),
+    }, target, options));
 }
-const runs = new Map();
-async function saveRun(run) {
-    runs.set(run.id, run);
-    try {
-        const { getRedis } = await import('../queue/redis.js');
-        await getRedis().set(`agent-upgrade-run:${run.id}`, JSON.stringify(run), 'EX', RUN_TTL_SECONDS);
-        await getRedis().set('agent-upgrade-run:latest', run.id, 'EX', RUN_TTL_SECONDS);
-    }
-    catch {
-        // in-memory only
-    }
-}
-export async function getUpgradeRun(id) {
-    const local = runs.get(id);
-    if (local)
-        return local;
-    try {
-        const { getRedis } = await import('../queue/redis.js');
-        const raw = await getRedis().get(`agent-upgrade-run:${id}`);
-        return raw ? JSON.parse(raw) : null;
-    }
-    catch {
-        return null;
-    }
+/** In-process store: tests, and the fallback without Redis. Keeps the newest MAX_LOCAL_RUNS runs. */
+export function memoryRunStore() {
+    const runs = new Map();
+    const stops = new Map();
+    const requests = new Map();
+    let held = null;
+    let latestId = null;
+    return {
+        async save(run) {
+            runs.delete(run.id);
+            runs.set(run.id, run);
+            latestId = run.id;
+            for (const [id, stored] of runs) {
+                if (runs.size <= MAX_LOCAL_RUNS)
+                    break;
+                if (stored.status !== 'running') {
+                    runs.delete(id);
+                    stops.delete(id);
+                    for (const node of stored.nodes)
+                        requests.delete(node.requestId);
+                }
+            }
+        },
+        async get(id) { return runs.get(id) ?? null; },
+        async latest() { return latestId ? runs.get(latestId) ?? null : null; },
+        async lock(token) {
+            if (held && held.until > Date.now() && held.token !== token)
+                return false;
+            held = { token, until: Date.now() + LEASE_MS };
+            return true;
+        },
+        async renew(token) {
+            if (held?.token !== token)
+                return false;
+            held.until = Date.now() + LEASE_MS;
+            return true;
+        },
+        async unlock(token) { if (held?.token === token)
+            held = null; },
+        async requestStop(id, actorId) { stops.set(id, actorId); },
+        async stopRequested(id) { return stops.get(id) ?? null; },
+        async index(run) { for (const node of run.nodes)
+            requests.set(node.requestId, run.id); },
+        async runForRequest(requestId) { return requests.get(requestId) ?? null; },
+    };
 }
-export async function latestUpgradeRun() {
-    try {
-        const { getRedis } = await import('../queue/redis.js');
-        const id = await getRedis().get('agent-upgrade-run:latest');
-        return id ? getUpgradeRun(id) : null;
-    }
-    catch {
-        return [...runs.values()].at(-1) ?? null;
-    }
+const LOCK_KEY = 'agent-upgrade-run:lock';
+const RELEASE_IF_OWNED = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end";
+const RENEW_IF_OWNED = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end";
+let sharedStore = null;
+/** Redis-backed store; every call falls back to this process's memory when Redis is unavailable. */
+export function defaultRunStore() {
+    if (sharedStore)
+        return sharedStore;
+    const memory = memoryRunStore();
+    const redis = async () => (await import('../queue/redis.js')).getRedis();
+    const either = async (remote, local) => {
+        try {
+            return await remote();
+        }
+        catch {
+            return local();
+        }
+    };
+    sharedStore = {
+        async save(run) {
+            await memory.save(run);
+            await either(async () => {
+                const client = await redis();
+                await client.set(`agent-upgrade-run:${run.id}`, JSON.stringify(run), 'EX', RUN_TTL_SECONDS);
+                await client.set('agent-upgrade-run:latest', run.id, 'EX', RUN_TTL_SECONDS);
+            }, async () => undefined);
+        },
+        get: id => either(async () => {
+            const raw = await (await redis()).get(`agent-upgrade-run:${id}`);
+            return raw ? JSON.parse(raw) : memory.get(id);
+        }, () => memory.get(id)),
+        latest: () => either(async () => {
+            const id = await (await redis()).get('agent-upgrade-run:latest');
+            return id ? sharedStore.get(id) : memory.latest();
+        }, () => memory.latest()),
+        lock: token => either(async () => await (await redis()).set(LOCK_KEY, token, 'PX', LEASE_MS, 'NX') === 'OK', () => memory.lock(token)),
+        renew: token => either(async () => Number(await (await redis()).eval(RENEW_IF_OWNED, 1, LOCK_KEY, token, String(LEASE_MS))) === 1, () => memory.renew(token)),
+        unlock: token => either(async () => { await (await redis()).eval(RELEASE_IF_OWNED, 1, LOCK_KEY, token); }, () => memory.unlock(token)),
+        async requestStop(id, actorId) {
+            await memory.requestStop(id, actorId);
+            await either(async () => { await (await redis()).set(`agent-upgrade-run:${id}:stop`, actorId, 'EX', RUN_TTL_SECONDS); }, async () => undefined);
+        },
+        stopRequested: id => either(async () => await (await redis()).get(`agent-upgrade-run:${id}:stop`) ?? memory.stopRequested(id), () => memory.stopRequested(id)),
+        async index(run) {
+            await memory.index(run);
+            await either(async () => {
+                const pipeline = (await redis()).pipeline();
+                for (const node of run.nodes)
+                    pipeline.set(`agent-upgrade-request:${node.requestId}`, run.id, 'EX', RUN_TTL_SECONDS);
+                await pipeline.exec();
+            }, async () => undefined);
+        },
+        runForRequest: requestId => either(async () => await (await redis()).get(`agent-upgrade-request:${requestId}`) ?? memory.runForRequest(requestId), () => memory.runForRequest(requestId)),
+    };
+    return sharedStore;
 }
+/** Runs this process is executing right now. */
+const executing = new Map();
 function clientFor(node) {
     return new NodeApiClient({ baseUrl: getNodeBaseUrl(node.ipAddress, node.port), resourceNodeId: node.id, nodeToken: getNodeManagerSecret(),
-        timeout: 15 * 60_000 });
+        resourceActorId: 'system:agent-upgrade', timeout: 15 * 60_000 });
+}
+/** Trial window the agent reported (or the default), two heartbeats, and the restart margin. */
+export function confirmWindowMs(report, fallbackMs = DEFAULT_CONFIRM_SECONDS * 1000 + 2 * HEARTBEAT_MS + CONFIRM_MARGIN_MS) {
+    return report?.confirmSeconds ? report.confirmSeconds * 1000 + 2 * HEARTBEAT_MS + CONFIRM_MARGIN_MS : fallbackMs;
 }
-export function defaultDeps() {
+export function defaultDeps(store = defaultRunStore()) {
     return {
         request: async (nodeId, body) => {
-            const node = await getDb().node.findUniqueOrThrow({ where: { id: nodeId }, select: { id: true, ipAddress: true, port: true } });
+            const node = await getDb().node.findUnique({ where: { id: nodeId }, select: { id: true, ipAddress: true, port: true } });
+            if (!node)
+                throw new NotDispatchedError('NODE_NOT_FOUND', 'node is no longer in the inventory');
             await clientFor(node).upgradeAgent(body);
         },
         observe: async (nodeId) => {
-            const row = await getDb().node.findUnique({ where: { id: nodeId }, select: { metadata: true, agent: { select: { version: true } } } });
-            return { version: row?.agent?.version ?? null, report: storedSelfUpgrade(row?.metadata) };
+            const row = await getDb().node.findUnique({ where: { id: nodeId }, select: { metadata: true, lastHeartbeatAt: true, agent: { select: { version: true } } } });
+            return { version: row?.agent?.version ?? null, report: storedSelfUpgrade(row?.metadata), lastHeartbeatAt: row?.lastHeartbeatAt ?? null };
+        },
+        check: async (nodeId, run) => {
+            const [candidate] = await listUpgradeCandidates(run.version, [nodeId], { allowRetryReverted: run.allowRetryReverted });
+            return candidate ? candidate.eligible ? null : candidate.reason : 'node is no longer in the inventory';
         },
         sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
         now: () => Date.now(),
-        save: saveRun,
-        confirmTimeoutMs: CONFIRM_TIMEOUT_MS,
+        save: run => store.save(run),
+        stopRequested: async (runId) => await store.stopRequested(runId).catch(() => null),
+        confirmTimeoutMs: confirmWindowMs(null),
         pollMs: POLL_MS,
     };
 }
+/** Whether a failed request left the node untouched (refused), hit a release problem (failed), or may have run (unknown). */
+export function classifyDispatchFailure(error) {
+    const message = (error instanceof AgentResponseError ? error.agentMessage ?? error.message : error instanceof Error ? error.message : String(error)).slice(0, 1000);
+    const code = error?.code;
+    if (error instanceof NotDispatchedError || error?.dispatched === false)
+        return { kind: 'refused', code: typeof code === 'string' ? code : 'NOT_DISPATCHED', message };
+    if (error instanceof AgentResponseError) {
+        const agentCode = error.code ?? `HTTP_${error.agentStatus}`;
+        if (agentCode === 'MUTATION_OUTCOME_UNKNOWN')
+            return { kind: 'unknown', code: agentCode, message };
+        if (UNTOUCHED_CODES.has(agentCode))
+            return { kind: 'refused', code: agentCode, message };
+        return { kind: 'failed', code: agentCode, message: RELEASE_PROBLEM_CODES.has(agentCode) ? message : `agent answered HTTP ${error.agentStatus}: ${message}` };
+    }
+    if (error instanceof DispatchOutcomeUnknownError)
+        return { kind: 'unknown', code: error.code, message };
+    return { kind: 'unknown', code: typeof code === 'string' ? code : 'MUTATION_OUTCOME_UNKNOWN', message };
+}
+const audit = (action, run, details, targetType = 'fleet', targetId = run.id) => logAuditEvent({ action, actorId: run.actorId, targetType, targetId, details: { runId: run.id, version: run.version, ...details } }).catch(() => undefined);
 /** Ask one node to upgrade and wait for its confirmed (or reverted) report. */
 async function upgradeOne(run, entry, deps) {
+    const iso = () => new Date(deps.now()).toISOString();
+    const end = (state, detail, code = null, awaitingReport = false) => {
+        Object.assign(entry, { state, detail: detail?.slice(0, 1000) ?? null, code, finishedAt: iso(), awaitingReport });
+        if (state === 'refused' || state === 'failed' || (state === 'confirmed' && detail?.startsWith('confirmed by version')))
+            void audit(`agent.upgrade.node_${state}`, run, { node: entry.name, requestId: entry.requestId, code, detail }, 'node', entry.nodeId);
+    };
+    let reason;
+    try {
+        reason = await deps.check(entry.nodeId, run);
+    }
+    catch (error) {
+        reason = `eligibility check failed: ${error instanceof Error ? error.message : String(error)}`;
+    }
+    if (reason)
+        return void Object.assign(entry, { state: 'skipped', detail: reason, finishedAt: iso() });
     entry.state = 'requested';
     await deps.save(run);
+    const startedAt = deps.now();
+    let unknown = null;
     try {
         await deps.request(entry.nodeId, { version: run.version, integrity: run.integrity, requestId: entry.requestId, packageName: AGENT_PACKAGE });
     }
     catch (error) {
-        entry.state = 'failed';
-        entry.detail = (error instanceof Error ? error.message : String(error)).slice(0, 500);
-        return;
+        const failure = classifyDispatchFailure(error);
+        if (failure.kind === 'refused')
+            return end('refused', failure.message, failure.code);
+        if (failure.kind === 'failed')
+            return end('failed', failure.message, failure.code);
+        unknown = failure;
     }
-    const deadline = deps.now() + deps.confirmTimeoutMs;
-    while (deps.now() < deadline) {
+    entry.dispatchedAt = iso();
+    let windowMs = deps.confirmTimeoutMs;
+    let versionSince = null;
+    let lastHeartbeat = null;
+    const deadline = () => Math.max(startedAt + windowMs + (unknown ? UNKNOWN_DISPATCH_MARGIN_MS : 0), versionSince === null ? 0 : versionSince + windowMs + deps.pollMs);
+    while (deps.now() < deadline()) {
         await deps.sleep(deps.pollMs);
-        const { version, report } = await deps.observe(entry.nodeId);
+        let observation;
+        try {
+            observation = await deps.observe(entry.nodeId);
+        }
+        catch (error) {
+            console.warn(`[agent-upgrade] could not read the state of ${entry.name}: ${error instanceof Error ? error.message : String(error)}`);
+            continue;
+        }
+        const { version, report } = observation;
+        lastHeartbeat = observation.lastHeartbeatAt ?? lastHeartbeat;
+        if (report?.confirmSeconds)
+            windowMs = confirmWindowMs(report);
         const upgrade = report?.upgrade;
-        if (upgrade?.requestId === entry.requestId && upgrade.state === 'reverted') {
-            entry.state = 'reverted';
-            entry.detail = upgrade.reason ?? 'the agent switched back to its previous version';
-            return;
+        if (upgrade?.requestId === entry.requestId) {
+            if (upgrade.state === 'confirmed')
+                return end('confirmed', unknown ? 'confirmed (the reply to the request was lost)' : null);
+            if (upgrade.state === 'reverted')
+                return end('reverted', upgrade.reason ?? 'the agent switched back to its previous version', 'SELF_UPGRADE_REVERTED');
+            if (upgrade.state === 'failed')
+                return end('failed', upgrade.reason ?? 'the agent reported the upgrade as failed', 'SELF_UPGRADE_FAILED');
+        }
+        else if (!upgrade?.requestId && version === run.version) {
+            // Agents that do not report request ids: the version has to hold for one full trial window.
+            versionSince ??= deps.now();
+            if (deps.now() - versionSince >= windowMs)
+                return end('confirmed', 'confirmed by version (the agent does not report upgrade requests)');
         }
-        if (version === run.version && (upgrade?.requestId !== entry.requestId || upgrade.state === 'confirmed')) {
-            entry.state = 'confirmed';
-            entry.detail = null;
-            return;
+        else {
+            versionSince = null;
         }
+        // Silent for longer than the agent's own trial window: it will not confirm, even if the extended deadline is not up.
+        if (lastHeartbeat && deps.now() - lastHeartbeat.getTime() > windowMs)
+            return end('failed', `no heartbeat since ${lastHeartbeat.toISOString()}`, 'AGENT_SILENT', true);
     }
-    entry.state = 'failed';
-    entry.detail = `no confirmation within ${Math.round(deps.confirmTimeoutMs / 60_000)} min`;
+    const minutes = Math.round((deadline() - startedAt) / 60_000);
+    if (unknown)
+        return end('failed', `outcome unknown (${unknown.message}); no report for this request within ${minutes} min`, unknown.code, true);
+    end('failed', `no confirmation within ${minutes} min`, 'CONFIRM_TIMEOUT', true);
+}
+/** "N confirmed, N refused, N failed, N skipped" and each refusal reason. */
+export function summarizeRun(run) {
+    const count = (state) => run.nodes.filter(node => node.state === state).length;
+    const reverted = count('reverted');
+    const refused = run.nodes.filter(node => node.state === 'refused');
+    return `${count('confirmed')} confirmed, ${refused.length} refused, ${count('failed')} failed, ${reverted ? `${reverted} reverted, ` : ''}${count('skipped')} skipped`
+        + (refused.length ? `; refused: ${refused.map(node => `${node.name} (${node.code ? `${node.code}: ` : ''}${node.detail ?? 'no reason given'})`).join('; ')}` : '');
 }
-/** Canary first, then batches of `concurrency`. Stops at the first node that does not confirm. */
+const halts = (node) => node.state === 'failed' || node.state === 'reverted';
+/**
+ * Canary first, then batches of `concurrency`. A refused canary hands the
+ * role to the next node. Stops at the first revert or failure, or between
+ * batches when someone asked to stop.
+ */
 export async function executeUpgradeRun(run, deps) {
-    const queue = run.nodes.filter(node => node.state === 'pending');
-    const [canary, ...rest] = queue;
     const finish = async (status, message) => {
         for (const node of run.nodes)
             if (node.state === 'pending')
-                node.state = 'skipped';
+                Object.assign(node, { state: 'skipped', detail: status === 'stopped' ? 'run stopped' : 'not reached' });
         run.status = status;
-        run.message = message;
+        run.summary = summarizeRun(run);
+        run.message = `${message} (${run.summary})`;
         run.finishedAt = new Date(deps.now()).toISOString();
         await deps.save(run);
+        await audit(status === 'stopped' ? 'agent.upgrade_run.stopped' : 'agent.upgrade_run.finished', run, { status, summary: run.summary,
+            stoppedBy: run.stopRequestedBy ?? null });
         return run;
     };
-    if (!canary)
-        return finish('succeeded', 'no eligible nodes');
-    await upgradeOne(run, canary, deps);
-    await deps.save(run);
-    if (canary.state !== 'confirmed')
-        return finish('failed', `canary ${canary.name} did not confirm (${canary.state}: ${canary.detail ?? ''}); the other nodes were not touched`);
-    for (let index = 0; index < rest.length; index += run.concurrency) {
-        const batch = rest.slice(index, index + run.concurrency);
-        await Promise.all(batch.map(node => upgradeOne(run, node, deps)));
-        await deps.save(run);
-        const bad = batch.find(node => node.state !== 'confirmed');
-        if (bad)
-            return finish('failed', `${bad.name} did not confirm (${bad.state}: ${bad.detail ?? ''}); remaining nodes were skipped`);
+    const pending = () => run.nodes.filter(node => node.state === 'pending');
+    const stopped = async () => {
+        const actor = await deps.stopRequested(run.id).catch(() => null);
+        if (actor)
+            run.stopRequestedBy = actor;
+        return actor;
+    };
+    try {
+        let canary;
+        while ((canary = pending()[0])) {
+            if (await stopped())
+                return finish('stopped', `stopped by ${run.stopRequestedBy}`);
+            run.canaryNodeId = canary.nodeId;
+            await upgradeOne(run, canary, deps);
+            await deps.save(run);
+            if (canary.state === 'confirmed')
+                break;
+            if (halts(canary))
+                return finish('failed', `canary ${canary.name} did not confirm (${canary.state}: ${canary.detail ?? ''}); the other nodes were not touched`);
+        }
+        for (let batch = pending().slice(0, run.concurrency); batch.length; batch = pending().slice(0, run.concurrency)) {
+            if (await stopped())
+                return finish('stopped', `stopped by ${run.stopRequestedBy}`);
+            await Promise.all(batch.map(node => upgradeOne(run, node, deps)));
+            await deps.save(run);
+            const bad = batch.find(halts);
+            if (bad)
+                return finish('failed', `${bad.name} did not confirm (${bad.state}: ${bad.detail ?? ''}); remaining nodes were skipped`);
+        }
+        const confirmed = run.nodes.filter(node => node.state === 'confirmed').length;
+        if (!run.nodes.length)
+            return finish('succeeded', 'no eligible nodes');
+        return confirmed ? finish('succeeded', `${confirmed} agent(s) upgraded to ${run.version}`) : finish('failed', 'no agent was upgraded');
+    }
+    catch (error) {
+        return finish('failed', `run aborted: ${error instanceof Error ? error.message : String(error)}`);
     }
-    return finish('succeeded', `${run.nodes.filter(node => node.state === 'confirmed').length} agent(s) upgraded to ${run.version}`);
 }
-/** Validate, create the run and start it in the background. */
+// ── Interrupted runs ──
+const leaseExpired = (run, now = Date.now()) => run.status === 'running' && !executing.has(run.id)
+    && (run.leaseExpiresAt ? Date.parse(run.leaseExpiresAt) : Date.parse(run.startedAt) + UNLEASED_RUN_MS) < now;
+/**
+ * The owner stopped renewing (server restart, CLI interrupted): settle the
+ * requested nodes from their stored heartbeat reports, skip the pending ones
+ * and mark the run `interrupted`. Unresolved nodes keep `awaitingReport`.
+ */
+export async function reconcileInterruptedRun(run, observe, save, now = Date.now()) {
+    if (!leaseExpired(run, now))
+        return run;
+    const at = new Date(now).toISOString();
+    for (const node of run.nodes) {
+        if (node.state === 'pending')
+            Object.assign(node, { state: 'skipped', detail: 'run interrupted before this node', finishedAt: at });
+        if (node.state !== 'requested')
+            continue;
+        const upgrade = await observe(node.nodeId).then(observation => observation.report?.upgrade, () => null);
+        const mine = upgrade?.requestId === node.requestId ? upgrade : null;
+        if (mine?.state === 'confirmed')
+            Object.assign(node, { state: 'confirmed', detail: 'confirmed (recorded after the run was interrupted)', code: null });
+        else if (mine?.state === 'reverted')
+            Object.assign(node, { state: 'reverted', detail: mine.reason ?? 'the agent switched back', code: 'SELF_UPGRADE_REVERTED' });
+        else if (mine?.state === 'failed')
+            Object.assign(node, { state: 'failed', detail: mine.reason ?? 'the agent reported a failure', code: 'SELF_UPGRADE_FAILED' });
+        else
+            Object.assign(node, { state: 'failed', code: 'RUN_INTERRUPTED', awaitingReport: true,
+                detail: `the server stopped watching${mine ? ` while the agent reported ${mine.state}` : ''}; a later report for this request is recorded here` });
+        node.finishedAt = at;
+    }
+    run.status = 'interrupted';
+    run.summary = summarizeRun(run);
+    run.message = `interrupted: the server that ran it stopped renewing its lease (restart or interrupted CLI) (${run.summary})`;
+    run.finishedAt = at;
+    await save(run);
+    await audit('agent.upgrade_run.finished', run, { status: 'interrupted', summary: run.summary });
+    return run;
+}
+export async function getUpgradeRun(id, store = defaultRunStore()) {
+    const run = executing.get(id)?.run ?? await store.get(id).catch(() => null);
+    return run ? reconcileInterruptedRun(run, defaultDeps(store).observe, saved => store.save(saved)) : null;
+}
+export async function latestUpgradeRun(store = defaultRunStore()) {
+    const run = await store.latest().catch(() => null);
+    return run ? getUpgradeRun(run.id, store) : null;
+}
+/** Validate, take the fleet-wide run lock, create the run and start it in the background. */
 export async function startUpgradeRun(options) {
     const settings = upgradeSettings();
     if (!settings.agentUpgrades)
         throw new HttpError(403, 'AGENT_UPGRADES_DISABLED', 'Server-driven agent upgrades are off (MAGNUS_AGENT_UPGRADES=false)');
-    const release = await resolveAgentRelease({ version: options.version, integrity: options.integrity });
-    const candidates = await listUpgradeCandidates(release.version, options.nodeIds);
-    const eligible = candidates.filter(candidate => candidate.eligible);
-    if (eligible.length === 0) {
-        const reasons = candidates.slice(0, 5).map(candidate => `${candidate.name}: ${candidate.reason}`).join('; ');
-        throw new HttpError(409, 'NO_ELIGIBLE_AGENTS', `No selected agent can be upgraded now${reasons ? ` (${reasons})` : ''}`);
-    }
-    const running = await latestUpgradeRun();
-    if (running?.status === 'running')
-        throw new HttpError(409, 'AGENT_UPGRADE_RUNNING', `Upgrade run ${running.id} is still in progress`);
-    const run = {
-        id: randomUUID(), version: release.version, integrity: release.integrity, actorId: options.actorId, status: 'running',
-        startedAt: new Date().toISOString(), finishedAt: null, concurrency: settings.agentUpgradeConcurrency, message: null,
-        nodes: eligible.map(candidate => ({ nodeId: candidate.nodeId, name: candidate.name, from: candidate.version, state: 'pending', requestId: randomUUID(), detail: null })),
-    };
-    const deps = options.deps ?? defaultDeps();
-    await deps.save(run);
-    await logAuditEvent({ action: 'agent.upgrade', actorId: options.actorId, targetType: 'fleet', targetId: run.id,
-        details: { version: run.version, nodes: run.nodes.map(node => node.name) } }).catch(() => undefined);
-    const done = executeUpgradeRun(run, deps).catch(async (error) => {
-        run.status = 'failed';
-        run.message = error instanceof Error ? error.message : String(error);
-        run.finishedAt = new Date().toISOString();
+    const version = options.version ?? AGENT_VERSION;
+    if (isVersion(version) && isPrerelease(version) && !options.allowPrerelease)
+        throw new HttpError(400, 'PRERELEASE_NOT_ALLOWED', `${version} is a prerelease; pass allowPrerelease to install it`);
+    if (options.nodeIds && options.nodeIds.length === 0)
+        throw new HttpError(400, 'NO_NODES_SELECTED', 'nodeIds is empty; leave it out to upgrade every eligible agent');
+    const store = options.store ?? defaultRunStore();
+    const id = randomUUID();
+    const token = `${INSTANCE_ID}:${id}`;
+    if (!await store.lock(token))
+        throw new HttpError(409, 'AGENT_UPGRADE_RUNNING', 'Another agent upgrade run is starting or in progress');
+    try {
+        const running = await latestUpgradeRun(store);
+        if (running?.status === 'running')
+            throw new HttpError(409, 'AGENT_UPGRADE_RUNNING', `Upgrade run ${running.id} is still in progress`);
+        const release = await resolveAgentRelease({ version, integrity: options.integrity });
+        const candidates = await listUpgradeCandidates(release.version, options.nodeIds, { allowRetryReverted: options.allowRetryReverted });
+        const unknown = (options.nodeIds ?? []).filter(nodeId => !candidates.some(candidate => candidate.nodeId === nodeId));
+        if (unknown.length)
+            throw new HttpError(400, 'UNKNOWN_NODES', `Unknown or decommissioned node(s): ${unknown.slice(0, 20).join(', ')}`);
+        const eligible = candidates.filter(candidate => candidate.eligible);
+        if (options.canaryNodeId && !eligible.some(candidate => candidate.nodeId === options.canaryNodeId)) {
+            const canary = candidates.find(candidate => candidate.nodeId === options.canaryNodeId);
+            throw new HttpError(400, 'INVALID_CANARY', canary ? `Canary ${canary.name} cannot be upgraded now: ${canary.reason}` : 'canaryNodeId is not among the selected nodes');
+        }
+        if (eligible.length === 0) {
+            const reasons = candidates.slice(0, 5).map(candidate => `${candidate.name}: ${candidate.reason}`).join('; ');
+            throw new HttpError(409, 'NO_ELIGIBLE_AGENTS', `No selected agent can be upgraded now${reasons ? ` (${reasons})` : ''}`);
+        }
+        if (options.canaryNodeId)
+            eligible.unshift(...eligible.splice(eligible.findIndex(candidate => candidate.nodeId === options.canaryNodeId), 1));
+        const deps = options.deps ?? defaultDeps(store);
+        const run = {
+            id, version: release.version, integrity: release.integrity, actorId: options.actorId, status: 'running',
+            startedAt: new Date(deps.now()).toISOString(), finishedAt: null, concurrency: settings.agentUpgradeConcurrency, message: null, summary: null,
+            canaryNodeId: eligible[0].nodeId, owner: INSTANCE_ID, leaseExpiresAt: new Date(deps.now() + LEASE_MS).toISOString(), stopRequestedBy: null,
+            retryOf: options.retryOf ?? null, allowRetryReverted: options.allowRetryReverted ?? false,
+            excluded: candidates.filter(candidate => !candidate.eligible).map(candidate => ({ nodeId: candidate.nodeId, name: candidate.name, reason: candidate.reason ?? 'not eligible' })),
+            nodes: eligible.map(candidate => ({ nodeId: candidate.nodeId, name: candidate.name, from: candidate.version, state: 'pending', requestId: randomUUID(), detail: null })),
+        };
+        await store.index(run);
         await deps.save(run);
-        return run;
-    });
-    return { run, candidates, done };
+        await audit('agent.upgrade_run.started', run, { nodes: run.nodes.map(node => node.name), canary: run.nodes[0].name, retryOf: run.retryOf,
+            excluded: run.excluded.map(node => `${node.name}: ${node.reason}`).slice(0, 50) });
+        // Renew the lease while this process runs it; a reader that finds it expired marks the run interrupted.
+        const renew = async () => {
+            run.leaseExpiresAt = new Date(Date.now() + LEASE_MS).toISOString();
+            if (!await store.renew(token).catch(() => false))
+                console.warn(`[agent-upgrade] run ${run.id}: could not renew the run lock`);
+            await deps.save(run).catch(() => undefined);
+        };
+        const timer = setInterval(() => void renew(), LEASE_MS / 3);
+        timer.unref?.();
+        executing.set(run.id, { run, timer });
+        const done = executeUpgradeRun(run, deps).finally(async () => {
+            clearInterval(timer);
+            executing.delete(run.id);
+            await store.unlock(token).catch(() => undefined);
+        });
+        return { run, candidates, done };
+    }
+    catch (error) {
+        await store.unlock(token).catch(() => undefined);
+        throw error;
+    }
+}
+/** Ask a run to stop; the owner honours it before the next batch. A run nobody owns any more is closed as interrupted. */
+export async function stopUpgradeRun(id, actorId, store = defaultRunStore()) {
+    const run = await getUpgradeRun(id, store);
+    if (!run)
+        throw new HttpError(404, 'NOT_FOUND', 'Upgrade run not found');
+    if (run.status !== 'running')
+        throw new HttpError(409, 'RUN_NOT_RUNNING', `Upgrade run ${id} already ended (${run.status})`);
+    await store.requestStop(id, actorId);
+    run.stopRequestedBy = actorId;
+    await logAuditEvent({ action: 'agent.upgrade_run.stop_requested', actorId, targetType: 'fleet', targetId: id, details: { runId: id } }).catch(() => undefined);
+    return run;
+}
+/** Retry the nodes a finished run did not upgrade, with the same version and integrity. */
+export async function retryUpgradeRun(id, options) {
+    const store = options.store ?? defaultRunStore();
+    const previous = await getUpgradeRun(id, store);
+    if (!previous)
+        throw new HttpError(404, 'NOT_FOUND', 'Upgrade run not found');
+    if (previous.status === 'running')
+        throw new HttpError(409, 'AGENT_UPGRADE_RUNNING', `Upgrade run ${id} is still in progress`);
+    const retry = new Set(['failed', 'refused', 'skipped', 'requested', ...options.allowRetryReverted ? ['reverted'] : []]);
+    const nodeIds = previous.nodes.filter(node => retry.has(node.state)).map(node => node.nodeId);
+    if (!nodeIds.length)
+        throw new HttpError(409, 'NOTHING_TO_RETRY', `Run ${id} has no failed, refused, skipped or interrupted nodes${options.allowRetryReverted ? '' : ' (reverted nodes need allowRetryReverted)'}`);
+    return startUpgradeRun({ ...options, store, version: previous.version, integrity: previous.integrity, nodeIds, retryOf: previous.id,
+        allowPrerelease: options.allowPrerelease ?? isPrerelease(previous.version) });
 }
 //# sourceMappingURL=agent-upgrade-service.js.map