@agent-link/agent 0.1.299 → 0.1.301

package/worker/worker.cjs

@@ -9,13 +9,20 @@ const path = require('path');
 
 const PROTOCOL_VERSION = 1;
 const PHASES = {
+  created: ['worker-starting', 'failed'],
   'worker-starting': ['worker-ready', 'failed'],
   'worker-ready': ['shutdown-requested', 'failed'],
   'shutdown-requested': ['old-agent-stopped', 'failed'],
-  'old-agent-stopped': ['switching', 'failed'],
-  switching: ['target-starting', 'failed'],
-  'target-starting': ['target-started', 'failed'],
+  'old-agent-stopped': ['switching', 'rolling-back', 'repair-required'],
+  switching: ['target-starting', 'rolling-back', 'repair-required'],
+  'target-starting': ['target-local-health', 'rolling-back', 'repair-required'],
+  'target-local-health': ['relay-pending', 'completed', 'rolling-back', 'repair-required'],
+  'relay-pending': ['completed', 'rolling-back', 'repair-required'],
+  'rolling-back': ['rollback-starting', 'repair-required'],
+  'rollback-starting': ['rollback-local-health', 'repair-required'],
+  'rollback-local-health': ['rolled-back', 'repair-required'],
 };
+const TERMINAL_PHASES = new Set(['completed', 'rolled-back', 'repair-required', 'failed']);
 
 function fail(message) {
   process.stderr.write(`[AgentLink Worker] ${message}\n`);
@@ -163,7 +170,7 @@ function processIdentity(pid) {
       ], {
         encoding: 'utf8',
         windowsHide: true,
-        timeout: 3000,
+        timeout: 10000,
       }).trim();
       return started
         ? { status: 'found', identity: `win32:${started}` }
@@ -195,6 +202,92 @@ function assignLockWorker(job, worker) {
     throw new Error('active upgrade lock does not belong to this job');
   }
   const claimPath = path.join(lockDirectory, `handoff-${job.lockNonce}.claim`);
+  let reclaimFence = null;
+  if (fs.existsSync(claimPath)) {
+    const existingClaim = loadJson(claimPath);
+    if (
+      existingClaim.role !== 'worker'
+      || existingClaim.jobId !== job.jobId
+      || existingClaim.nonce !== job.lockNonce
+      || !existingClaim.worker
+    ) {
+      throw new Error('active upgrade lock has an incompatible handoff claim');
+    }
+    const existing = matchProcessIdentity(
+      existingClaim.worker.pid,
+      existingClaim.worker.processStartIdentity,
+    );
+    if (existing.status === 'match') {
+      throw new Error('the recorded updater worker is still running');
+    }
+    if (existing.status === 'error') {
+      throw new Error(`cannot safely reclaim updater worker claim: ${existing.error}`);
+    }
+    const claimGeneration = crypto.createHash('sha256')
+      .update(`${existingClaim.jobId}\0${existingClaim.nonce}\0${existingClaim.worker.pid}\0${existingClaim.worker.processStartIdentity}`)
+      .digest('hex')
+      .slice(0, 16);
+    reclaimFence = `${claimPath}.reclaim-${claimGeneration}`;
+    let fenceHandle;
+    while (fenceHandle === undefined) {
+      try {
+        fenceHandle = fs.openSync(reclaimFence, 'wx', 0o600);
+      } catch (error) {
+        if (error.code !== 'EEXIST') throw error;
+        let fenceOwner;
+        try {
+          fenceOwner = loadJson(reclaimFence);
+        } catch (fenceError) {
+          const age = Date.now() - fs.statSync(reclaimFence).mtimeMs;
+          if (age < 5000) {
+            throw new Error('another recovery worker is claiming the stale handoff');
+          }
+          fs.rmSync(reclaimFence, { force: true });
+          continue;
+        }
+        const fenceState = matchProcessIdentity(
+          fenceOwner.pid,
+          fenceOwner.processStartIdentity,
+        );
+        if (fenceState.status === 'match') {
+          throw new Error('another recovery worker is claiming the stale handoff');
+        }
+        if (fenceState.status === 'error') {
+          throw new Error(`cannot safely reclaim recovery fence: ${fenceState.error}`);
+        }
+        fs.rmSync(reclaimFence, { force: true });
+      }
+    }
+    try {
+      fs.writeFileSync(fenceHandle, `${JSON.stringify(worker, null, 2)}\n`, 'utf8');
+      fs.fsyncSync(fenceHandle);
+    } finally {
+      fs.closeSync(fenceHandle);
+    }
+    const confirmedFenceOwner = loadJson(reclaimFence);
+    if (
+      confirmedFenceOwner.pid !== worker.pid
+      || confirmedFenceOwner.processStartIdentity !== worker.processStartIdentity
+    ) {
+      throw new Error('recovery fence ownership changed before stale claim removal');
+    }
+    const currentClaim = loadJson(claimPath);
+    if (
+      currentClaim.role !== 'worker'
+      || currentClaim.jobId !== existingClaim.jobId
+      || currentClaim.nonce !== existingClaim.nonce
+      || currentClaim.worker?.pid !== existingClaim.worker.pid
+      || currentClaim.worker?.processStartIdentity !== existingClaim.worker.processStartIdentity
+    ) {
+      throw new Error('stale handoff claim changed during recovery');
+    }
+    fs.rmSync(claimPath);
+    appendEvent(
+      path.join(job.dataRoot, 'upgrades', job.jobId, 'job.json'),
+      job,
+      'stale-worker-reclaimed',
+    );
+  }
   const claimFd = fs.openSync(claimPath, 'wx', 0o600);
   try {
     fs.writeFileSync(claimFd, `${JSON.stringify({
@@ -208,6 +301,7 @@ function assignLockWorker(job, worker) {
   } finally {
     fs.closeSync(claimFd);
   }
+  if (reclaimFence) fs.rmSync(reclaimFence, { force: true });
   owner = loadJson(lockPath);
   if (
     owner.jobId !== job.jobId
@@ -297,8 +391,20 @@ function isInside(root, candidate) {
   return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
 }
 
-function validateSelection(job) {
-  const pointer = job.targetSelection;
+function validateLaunchSpecPath(job) {
+  const canonicalRoot = fs.realpathSync(job.dataRoot);
+  const canonicalSpec = fs.realpathSync(job.launchSpecPath);
+  if (
+    path.resolve(job.dataRoot) !== canonicalRoot
+    || path.resolve(job.launchSpecPath) !== canonicalSpec
+    || !isInside(canonicalRoot, canonicalSpec)
+    || !fs.statSync(canonicalSpec).isFile()
+  ) {
+    throw new Error('LaunchSpec is relocated or escapes the trusted data root');
+  }
+}
+
+function validateSelection(job, pointer) {
   const encoded = pointer.integrity.startsWith('sha512-')
     ? pointer.integrity.slice('sha512-'.length)
     : '';
@@ -341,8 +447,16 @@ function isStrictlyNewerVersion(current, target) {
   return false;
 }
 
+function sameSelection(left, right) {
+  return left
+    && left.version === right.version
+    && left.integrity === right.integrity
+    && path.resolve(left.entry) === path.resolve(right.entry);
+}
+
 function selectTarget(job) {
   const current = loadJson(path.join(job.dataRoot, 'current-version.json'));
+  if (sameSelection(current, job.targetSelection)) return;
   if (
     current.version !== job.sourceSelection.version
     || current.integrity !== job.sourceSelection.integrity
@@ -355,7 +469,7 @@ function selectTarget(job) {
       `target ${job.targetSelection.version} is not newer than current ${current.version}`,
     );
   }
-  validateSelection(job);
+  validateSelection(job, job.targetSelection);
   atomicJson(path.join(job.dataRoot, 'current-version.json'), job.targetSelection);
 }
 
@@ -370,7 +484,7 @@ function waitForSpawn(child) {
   });
 }
 
-async function waitForTargetRuntime(job, child, timeoutMs = 15000) {
+async function waitForTargetRuntime(job, child, timeoutMs = Number(process.env.AGENTLINK_UPGRADE_HEALTH_TIMEOUT_MS) || 15000) {
   const runtimePath = path.join(job.dataRoot, 'agent.json');
   const deadline = Date.now() + timeoutMs;
   let earlyExit = null;
@@ -392,10 +506,18 @@ async function waitForTargetRuntime(job, child, timeoutMs = 15000) {
         runtime.upgradeJobId === job.jobId
         && Number.isSafeInteger(runtime.pid)
         && runtime.pid > 0
-        && runtime.pid !== job.oldRuntime.pid
       ) {
         const lookup = processIdentity(runtime.pid);
-        if (lookup.status === 'found') return;
+        if (lookup.status === 'found') {
+          return {
+            runtime,
+            identity: {
+              pid: runtime.pid,
+              processStartIdentity: lookup.identity,
+              processInstanceId: runtime.processInstanceId,
+            },
+          };
+        }
         if (lookup.status === 'error') {
           fail(`target acknowledgement identity lookup delayed: ${lookup.error}`);
         }
@@ -413,6 +535,35 @@ async function waitForTargetRuntime(job, child, timeoutMs = 15000) {
 }
 
 async function startTarget(job) {
+  validateLaunchSpecPath(job);
+  const isRollback = job.phase === 'rollback-starting';
+  const expectedRuntime = isRollback
+    ? job.rollbackRuntime
+    : job.targetRuntime;
+  try {
+    const existing = loadJson(path.join(job.dataRoot, 'agent.json'));
+    if (
+      expectedRuntime
+      && existing.upgradeJobId === job.jobId
+      && existing.pid === expectedRuntime.pid
+      && existing.processInstanceId === expectedRuntime.processInstanceId
+    ) {
+      const identity = matchProcessIdentity(
+        existing.pid,
+        expectedRuntime.processStartIdentity,
+      );
+      if (identity.status === 'match') {
+        return {
+          runtime: existing,
+          identity: expectedRuntime,
+        };
+      }
+    }
+  } catch {
+    // A replacement has not published its runtime yet.
+  }
+  const previousLauncher = isRollback ? job.rollbackLauncher : job.targetLauncher;
+  if (previousLauncher) await terminateExactRuntime(previousLauncher);
   const launcher = path.join(job.dataRoot, 'launcher', 'launcher.cjs');
   if (!fs.statSync(launcher).isFile()) throw new Error('stable launcher is not installed');
   const hostType = loadJson(job.launchSpecPath).host.type;
@@ -420,12 +571,14 @@ async function startTarget(job) {
     ...process.env,
     AGENTLINK_HOME: job.dataRoot,
     AGENTLINK_UPGRADE_JOB: job.jobId,
+    AGENTLINK_RECOVERY_OWNER: 'worker',
   };
   if (hostType === 'systemd') {
     childProcess.execFileSync('systemctl', [
       '--user',
       'set-environment',
       `AGENTLINK_UPGRADE_JOB=${job.jobId}`,
+      'AGENTLINK_RECOVERY_OWNER=worker',
     ], { stdio: 'ignore', timeout: 5000 });
     try {
       childProcess.execFileSync('systemctl', ['--user', 'restart', 'agentlink.service'], {
@@ -438,13 +591,13 @@ async function startTarget(job) {
           '--user',
           'unset-environment',
           'AGENTLINK_UPGRADE_JOB',
+          'AGENTLINK_RECOVERY_OWNER',
         ], { stdio: 'ignore', timeout: 5000 });
       } catch (error) {
         fail(`could not clear systemd upgrade environment: ${error.message}`);
       }
     }
-    await waitForTargetRuntime(job, null);
-    return;
+    return waitForTargetRuntime(job, null);
   }
   if (hostType === 'launchd') {
     const uid = process.getuid();
@@ -452,6 +605,10 @@ async function startTarget(job) {
       stdio: 'ignore',
       timeout: 5000,
     });
+    childProcess.execFileSync('launchctl', ['setenv', 'AGENTLINK_RECOVERY_OWNER', 'worker'], {
+      stdio: 'ignore',
+      timeout: 5000,
+    });
     try {
       childProcess.execFileSync('launchctl', [
         'kickstart',
@@ -464,12 +621,15 @@ async function startTarget(job) {
           stdio: 'ignore',
           timeout: 5000,
         });
+        childProcess.execFileSync('launchctl', ['unsetenv', 'AGENTLINK_RECOVERY_OWNER'], {
+          stdio: 'ignore',
+          timeout: 5000,
+        });
       } catch (error) {
         fail(`could not clear launchd upgrade environment: ${error.message}`);
       }
     }
-    await waitForTargetRuntime(job, null);
-    return;
+    return waitForTargetRuntime(job, null);
   }
   const targetLog = path.join(path.dirname(job.launchSpecPath), '..', 'logs', 'upgrade-target.log');
   fs.mkdirSync(path.dirname(targetLog), { recursive: true, mode: 0o700 });
@@ -490,12 +650,434 @@ async function startTarget(job) {
       stdio: ['ignore', output, output],
       windowsHide: true,
     });
-    await waitForSpawn(child);
+    try {
+      await waitForSpawn(child);
+      const launcherIdentity = processIdentity(child.pid);
+      if (launcherIdentity.status !== 'found') {
+        throw new Error(
+          `cannot determine exact target launcher identity: ${launcherIdentity.error || launcherIdentity.status}`,
+        );
+      }
+      job = transition(
+        path.join(job.dataRoot, 'upgrades', job.jobId, 'job.json'),
+        job.jobId,
+        job.phase,
+        {
+          [isRollback ? 'rollbackLauncher' : 'targetLauncher']: {
+            pid: child.pid,
+            processStartIdentity: launcherIdentity.identity,
+          },
+        },
+      );
+    } catch (error) {
+      try {
+        await terminateSpawnedLauncherTree(child);
+      } catch (cleanupError) {
+        const fatal = new Error(
+          `${error.message || error}; spawned launcher cleanup failed: `
+          + `${cleanupError.message || cleanupError}`,
+        );
+        fatal.preventRollback = true;
+        throw fatal;
+      }
+      throw error;
+    }
   } finally {
     fs.closeSync(output);
   }
-  await waitForTargetRuntime(job, child);
+  const result = await waitForTargetRuntime(job, child);
   child.unref();
+  return result;
+}
+
+function callStatus(runtime, timeoutMs = 1500) {
+  const discovery = runtime.controlPlane;
+  if (
+    !discovery
+    || discovery.apiVersion !== 1
+    || discovery.host !== '127.0.0.1'
+    || !Number.isSafeInteger(discovery.port)
+    || typeof discovery.token !== 'string'
+  ) {
+    return Promise.reject(new Error('target did not publish valid Local Control discovery'));
+  }
+  return new Promise((resolve, reject) => {
+    const request = http.request({
+      host: '127.0.0.1',
+      port: discovery.port,
+      path: '/v1/status',
+      method: 'GET',
+      headers: {
+        Host: `127.0.0.1:${discovery.port}`,
+        Authorization: `Bearer ${discovery.token}`,
+        'X-Request-Id': crypto.randomUUID(),
+      },
+    }, response => {
+      const chunks = [];
+      let size = 0;
+      response.on('data', chunk => {
+        size += chunk.length;
+        if (size > 64 * 1024) request.destroy(new Error('Local Control response exceeded 64 KiB'));
+        else chunks.push(Buffer.from(chunk));
+      });
+      response.on('end', () => {
+        try {
+          const envelope = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+          if (response.statusCode !== 200 || !envelope.ok || !envelope.data) {
+            reject(new Error(`Local Control status failed (${response.statusCode || 0})`));
+          } else {
+            resolve(envelope.data);
+          }
+        } catch {
+          reject(new Error('Local Control returned invalid status JSON'));
+        }
+      });
+    });
+    request.setTimeout(timeoutMs, () => request.destroy(new Error('Local Control status timed out')));
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+function launchSpecDigest(job) {
+  validateLaunchSpecPath(job);
+  return crypto.createHash('sha256')
+    .update(JSON.stringify(loadJson(job.launchSpecPath)))
+    .digest('hex');
+}
+
+function sessionUrlHash(runtime) {
+  return crypto.createHash('sha256').update(runtime.sessionUrl || '').digest('hex');
+}
+
+function sessionIdentityMatches(job, runtime, status) {
+  const preserved = runtime.sessionId === job.oldRuntime.sessionId
+    && sessionUrlHash(runtime) === job.oldRuntime.sessionUrlHash
+    && status.connection?.sessionId === job.oldRuntime.sessionId;
+  if (preserved) return true;
+  const registration = runtime.sessionRegistration;
+  return registration
+    && registration.previousSessionId === job.oldRuntime.sessionId
+    && registration.sessionId === runtime.sessionId
+    && registration.sessionUrlHash === sessionUrlHash(runtime)
+    && status.connection?.sessionId === runtime.sessionId
+    && Number.isFinite(Date.parse(registration.registeredAt));
+}
+
+async function waitForLocalHealth(job, expectedSelection, expectedIdentity, timeoutMs = Number(process.env.AGENTLINK_UPGRADE_HEALTH_TIMEOUT_MS) || 15000) {
+  validateSelection(job, expectedSelection);
+  if (
+    process.env.AGENTLINK_WORKER_TEST_FAIL_LOCAL_HEALTH_VERSION
+    === expectedSelection.version
+  ) {
+    throw new Error(`injected local health failure for ${expectedSelection.version}`);
+  }
+  if (launchSpecDigest(job) !== job.launchSpecDigest) {
+    throw new Error('target LaunchSpec digest does not match the upgrade job');
+  }
+  const deadline = Date.now() + timeoutMs;
+  let lastError = 'target runtime was not published';
+  while (Date.now() < deadline) {
+    try {
+      const runtime = loadJson(path.join(job.dataRoot, 'agent.json'));
+      const actualIdentity = matchProcessIdentity(
+        runtime.pid,
+        expectedIdentity.processStartIdentity,
+      );
+      if (
+        runtime.pid !== expectedIdentity.pid
+        || runtime.processInstanceId !== expectedIdentity.processInstanceId
+        || actualIdentity.status !== 'match'
+      ) {
+        throw new Error('target exact process identity changed');
+      }
+      if (
+        runtime.versionStore?.active !== true
+        || runtime.versionStore.selectedVersion !== expectedSelection.version
+        || runtime.versionStore.selectedIntegrity !== expectedSelection.integrity
+        || runtime.upgradeJobId !== job.jobId
+        || path.resolve(runtime.launchSpecPath || '') !== path.resolve(job.launchSpecPath)
+        || runtime.launchSpecDigest !== job.launchSpecDigest
+      ) {
+        throw new Error('target runtime identity contract does not match the upgrade job');
+      }
+      const status = await callStatus(runtime);
+      if (
+        status.pid !== runtime.pid
+        || status.version !== expectedSelection.version
+        || status.shutdown?.phase !== 'idle'
+        || status.connection?.acceptingWork !== true
+        || status.connection?.backendReady !== true
+        || !sessionIdentityMatches(job, runtime, status)
+      ) {
+        throw new Error('target Local Control is not idle, accepting work, and backend-ready');
+      }
+      return { runtime, status, identity: expectedIdentity };
+    } catch (error) {
+      lastError = String(error.message || error);
+    }
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+  throw new Error(`mandatory local health failed: ${lastError}`);
+}
+
+function relayHealthy(health) {
+  return health.status.connection.websocketConnected === true
+    && health.status.connection.relayReady === true
+    && health.status.connection.entraReady === true
+    && health.status.connection.sessionRestored === true;
+}
+
+async function waitForRelay(job, health) {
+  let current = health;
+  while (true) {
+    if (relayHealthy(current)) return current;
+    await new Promise(resolve => setTimeout(resolve, 1000));
+    current = await waitForLocalHealth(job, job.targetSelection, current.identity, 2000);
+  }
+}
+
+async function terminateExactRuntime(identity) {
+  const match = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+  if (match.status === 'absent' || match.status === 'mismatch') {
+    if (process.platform !== 'win32') {
+      try {
+        process.kill(-identity.pid, 0);
+        throw new Error(
+          `exact launcher PID ${identity.pid} disappeared while its process group is still running`,
+        );
+      } catch (error) {
+        if (error.code !== 'ESRCH') throw error;
+      }
+    }
+    return;
+  }
+  if (match.status !== 'match') {
+    throw new Error(`cannot safely terminate target: ${match.error || match.status}`);
+  }
+
+  let unixSignalTarget = null;
+  if (process.platform === 'win32') {
+    try {
+      childProcess.execFileSync('taskkill.exe', ['/pid', String(identity.pid), '/t', '/f'], {
+        stdio: 'ignore',
+        windowsHide: true,
+        timeout: 10000,
+      });
+    } catch (error) {
+      const after = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+      if (after.status === 'match' || after.status === 'error') throw error;
+      return;
+    }
+  } else {
+    let signalTarget = identity.pid;
+    try {
+      process.kill(-identity.pid, 0);
+      signalTarget = -identity.pid;
+    } catch (error) {
+      if (error.code !== 'ESRCH') throw error;
+    }
+    unixSignalTarget = signalTarget;
+    try { process.kill(signalTarget, 'SIGTERM'); } catch (error) {
+      if (error.code !== 'ESRCH') throw error;
+    }
+    const deadline = Date.now() + 5000;
+    while (Date.now() < deadline) {
+      if (signalTarget < 0) {
+        try {
+          process.kill(signalTarget, 0);
+        } catch (error) {
+          if (error.code === 'ESRCH') return;
+          throw error;
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+        continue;
+      }
+      const state = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+      if (state.status === 'absent' || state.status === 'mismatch') return;
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    if (signalTarget < 0) {
+      try { process.kill(signalTarget, 'SIGKILL'); } catch (error) {
+        if (error.code !== 'ESRCH') throw error;
+      }
+    } else {
+      const final = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+      if (final.status === 'match') process.kill(identity.pid, 'SIGKILL');
+    }
+  }
+  const deadline = Date.now() + 5000;
+  while (Date.now() < deadline) {
+    if (unixSignalTarget && unixSignalTarget < 0) {
+      try {
+        process.kill(unixSignalTarget, 0);
+      } catch (error) {
+        if (error.code === 'ESRCH') return;
+        throw error;
+      }
+      await new Promise(resolve => setTimeout(resolve, 50));
+      continue;
+    }
+    const state = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+    if (state.status === 'absent' || state.status === 'mismatch') return;
+    if (state.status === 'error') {
+      throw new Error(`cannot confirm terminated target identity: ${state.error}`);
+    }
+    await new Promise(resolve => setTimeout(resolve, 50));
+  }
+  if (unixSignalTarget && unixSignalTarget < 0) {
+    try {
+      process.kill(unixSignalTarget, 0);
+      throw new Error(`launcher process group ${-unixSignalTarget} did not exit after termination`);
+    } catch (error) {
+      if (error.code === 'ESRCH') return;
+      throw error;
+    }
+  }
+  const final = matchProcessIdentity(identity.pid, identity.processStartIdentity);
+  if (final.status === 'match') {
+    throw new Error(`target PID ${identity.pid} did not exit after termination`);
+  }
+  if (final.status === 'error') {
+    throw new Error(`cannot confirm terminated target identity: ${final.error}`);
+  }
+}
+
+async function terminateSpawnedLauncherTree(child) {
+  if (!child?.pid || child.exitCode !== null || child.signalCode) return;
+  const exited = new Promise(resolve => child.once('exit', resolve));
+  if (process.platform === 'win32') {
+    try {
+      childProcess.execFileSync('taskkill.exe', ['/pid', String(child.pid), '/t', '/f'], {
+        stdio: 'ignore',
+        windowsHide: true,
+        timeout: 10000,
+      });
+    } catch (error) {
+      if (child.exitCode === null && !child.signalCode) throw error;
+    }
+  } else {
+    try { process.kill(-child.pid, 'SIGTERM'); } catch (error) {
+      if (error.code !== 'ESRCH') throw error;
+    }
+  }
+  const exitedPromptly = await Promise.race([
+    exited.then(() => true),
+    new Promise(resolve => setTimeout(() => resolve(false), 5000)),
+  ]);
+  if (!exitedPromptly && process.platform !== 'win32') {
+    try { process.kill(-child.pid, 'SIGKILL'); } catch (error) {
+      if (error.code !== 'ESRCH') throw error;
+    }
+    await exited;
+  } else if (!exitedPromptly) {
+    throw new Error('spawned launcher tree did not exit after forced termination');
+  }
+}
+
+function restoreLastKnownGood(job) {
+  validateSelection(job, job.lastKnownGoodSelection);
+  atomicJson(path.join(job.dataRoot, 'last-known-good.json'), job.lastKnownGoodSelection);
+  atomicJson(path.join(job.dataRoot, 'current-version.json'), job.lastKnownGoodSelection);
+}
+
+async function rollback(jobPath, job, worker, reason) {
+  if (job.phase !== 'rolling-back' && job.phase !== 'rollback-starting' && job.phase !== 'rollback-local-health') {
+    job = transition(jobPath, job.jobId, 'rolling-back', {
+      error: String(reason).slice(0, 512),
+    });
+  }
+  try {
+    if (job.phase === 'rolling-back') {
+      const targetIdentity = job.targetRuntime;
+      if (targetIdentity) await terminateExactRuntime(targetIdentity);
+      if (job.targetLauncher) await terminateExactRuntime(job.targetLauncher);
+      restoreLastKnownGood(job);
+      job = transition(jobPath, job.jobId, 'rollback-starting');
+    }
+    let acknowledged;
+    let health;
+    if (job.phase === 'rollback-starting') {
+      acknowledged = await startTarget(job);
+      health = await waitForLocalHealth(
+        job,
+        job.lastKnownGoodSelection,
+        acknowledged.identity,
+      );
+      job = transition(jobPath, job.jobId, 'rollback-local-health', {
+        rollbackRuntime: acknowledged.identity,
+      });
+    } else {
+      const runtime = loadJson(path.join(job.dataRoot, 'agent.json'));
+      if (!job.rollbackRuntime) {
+        throw new Error('rollback phase is missing its durable exact runtime identity');
+      }
+      const lookup = processIdentity(runtime.pid);
+      if (lookup.status !== 'found') throw new Error('rollback runtime identity is unavailable');
+      acknowledged = {
+        runtime,
+        identity: job.rollbackRuntime,
+      };
+    }
+    if (!health) {
+      health = await waitForLocalHealth(
+        job,
+        job.lastKnownGoodSelection,
+        acknowledged.identity,
+      );
+    }
+    job = transition(jobPath, job.jobId, 'rolled-back', {
+      error: String(reason).slice(0, 512),
+    });
+  } catch (rollbackError) {
+    job = loadJob(jobPath);
+    const cleanupErrors = [];
+    for (const identity of [job.rollbackRuntime, job.rollbackLauncher]) {
+      if (!identity) continue;
+      try {
+        await terminateExactRuntime(identity);
+      } catch (cleanupError) {
+        cleanupErrors.push(String(cleanupError.message || cleanupError));
+      }
+    }
+    if (cleanupErrors.length) {
+      transition(jobPath, job.jobId, job.phase, {
+        error: `rollback failure: ${String(rollbackError.message || rollbackError).slice(0, 220)}; rollback cleanup failure: ${cleanupErrors.join('; ').slice(0, 220)}`,
+      });
+      throw new Error(`rollback cleanup failed before lock release: ${cleanupErrors.join('; ')}`);
+    }
+    job = transition(jobPath, job.jobId, 'repair-required', {
+      error: `target failure: ${String(reason).slice(0, 220)}; rollback failure: ${String(rollbackError.message || rollbackError).slice(0, 220)}`,
+    });
+    releaseLock(job, worker);
+    cleanupLaunchdWorker(job);
+    throw new Error(`rollback failed; local repair is required: ${rollbackError.message || rollbackError}`);
+  }
+  finalizeRolledBackJob(jobPath, job, worker);
+  return job;
+}
+
+function recordRolledBackFinalizationIssue(jobPath, job, step, error) {
+  const detail = `${step} failed after rollback completed: ${identityError(error)}`;
+  fail(detail);
+  try {
+    appendEvent(jobPath, job, 'rollback-finalization-failed', detail);
+  } catch (recordError) {
+    fail(`could not record rollback finalization failure: ${identityError(recordError)}`);
+  }
+}
+
+function finalizeRolledBackJob(jobPath, job, worker) {
+  try {
+    releaseLock(job, worker);
+  } catch (error) {
+    recordRolledBackFinalizationIssue(jobPath, job, 'upgrade lock release', error);
+  }
+  try {
+    cleanupLaunchdWorker(job);
+  } catch (error) {
+    recordRolledBackFinalizationIssue(jobPath, job, 'worker host cleanup', error);
+  }
 }
 
 function releaseLock(job, worker) {
@@ -542,12 +1124,14 @@ function cleanupLaunchdWorker(job) {
   }
 }
 
+let acquiredOwnership = false;
+
 async function main() {
   const jobIndex = process.argv.indexOf('--job');
   if (jobIndex < 0 || !process.argv[jobIndex + 1]) throw new Error('missing --job path');
   const jobPath = path.resolve(process.argv[jobIndex + 1]);
   let job = loadJob(jobPath);
-  if (job.phase !== 'worker-starting') throw new Error(`job is not worker-starting: ${job.phase}`);
+  if (TERMINAL_PHASES.has(job.phase)) return;
   const identityLookup = processIdentity(process.pid);
   if (identityLookup.status !== 'found') {
     throw new Error(
@@ -561,46 +1145,153 @@ async function main() {
     protocolVersion: PROTOCOL_VERSION,
   };
   assignLockWorker(job, worker);
-  job = transition(jobPath, job.jobId, 'worker-ready', { worker, error: null });
-  const runtimePath = path.join(job.dataRoot, 'agent.json');
-  const runtime = loadJson(runtimePath);
+  acquiredOwnership = true;
+  if (job.phase === 'created') {
+    job = transition(jobPath, job.jobId, 'worker-starting', { worker, error: null });
+  }
+  if (job.phase === 'worker-starting') {
+    job = transition(jobPath, job.jobId, 'worker-ready', { worker, error: null });
+  } else {
+    job = transition(jobPath, job.jobId, job.phase, { worker });
+  }
+
+  if (job.phase === 'worker-ready') {
+    const runtime = loadJson(path.join(job.dataRoot, 'agent.json'));
+    if (
+      runtime.pid !== job.oldRuntime.pid
+      || runtime.processInstanceId !== job.oldRuntime.processInstanceId
+    ) {
+      const old = matchProcessIdentity(job.oldRuntime.pid, job.oldRuntime.processStartIdentity);
+      if (old.status === 'absent' || old.status === 'mismatch') {
+        job = transition(jobPath, job.jobId, 'shutdown-requested');
+      } else {
+        throw new Error('runtime state no longer identifies the expected old Agent');
+      }
+    } else {
+      await requestShutdown(runtime, job);
+      job = transition(jobPath, job.jobId, 'shutdown-requested');
+    }
+  }
+  if (job.phase === 'shutdown-requested') {
+    await waitForOldRuntime(job);
+    job = transition(jobPath, job.jobId, 'old-agent-stopped');
+  }
+  if (job.phase === 'old-agent-stopped') {
+    job = transition(jobPath, job.jobId, 'switching');
+  }
+  if (job.phase === 'switching') {
+    selectTarget(job);
+    job = transition(jobPath, job.jobId, 'target-starting');
+  }
+
   if (
-    runtime.pid !== job.oldRuntime.pid
-    || runtime.processInstanceId !== job.oldRuntime.processInstanceId
+    job.phase === 'rolling-back'
+    || job.phase === 'rollback-starting'
+    || job.phase === 'rollback-local-health'
   ) {
-    throw new Error('runtime state no longer identifies the expected old Agent');
-  }
-  await requestShutdown(runtime, job);
-  job = transition(jobPath, job.jobId, 'shutdown-requested');
-  await waitForOldRuntime(job);
-  job = transition(jobPath, job.jobId, 'old-agent-stopped');
-  job = transition(jobPath, job.jobId, 'switching');
-  selectTarget(job);
-  job = transition(jobPath, job.jobId, 'target-starting');
-  await startTarget(job);
-  job = transition(jobPath, job.jobId, 'target-started');
+    await rollback(jobPath, job, worker, job.error || 'resuming interrupted rollback');
+    return;
+  }
+
+  let acknowledged;
+  let health;
+  if (job.phase === 'target-starting') {
+    acknowledged = await startTarget(job);
+    try {
+      health = await waitForLocalHealth(job, job.targetSelection, acknowledged.identity);
+    } catch (error) {
+      await rollback(jobPath, loadJob(jobPath), worker, error.message || error);
+      return;
+    }
+    job = transition(jobPath, job.jobId, 'target-local-health', {
+      targetRuntime: acknowledged.identity,
+    });
+  } else if (job.phase === 'target-local-health' || job.phase === 'relay-pending') {
+    if (!job.targetRuntime) {
+      throw new Error('target health phase is missing its durable exact runtime identity');
+    }
+    const runtime = loadJson(path.join(job.dataRoot, 'agent.json'));
+    const lookup = processIdentity(runtime.pid);
+    if (lookup.status !== 'found') throw new Error('target process identity is unavailable');
+    acknowledged = {
+      runtime,
+      identity: job.targetRuntime,
+    };
+  } else {
+    throw new Error(`cannot recover external upgrade phase: ${job.phase}`);
+  }
+
+  if (!health) {
+    try {
+      health = await waitForLocalHealth(job, job.targetSelection, acknowledged.identity);
+    } catch (error) {
+      await rollback(jobPath, loadJob(jobPath), worker, error.message || error);
+      return;
+    }
+  }
+  job = loadJob(jobPath);
+  if (job.phase === 'target-local-health') {
+    job = transition(jobPath, job.jobId, 'relay-pending');
+  }
+  health = await waitForRelay(job, health);
+  atomicJson(path.join(job.dataRoot, 'last-known-good.json'), job.targetSelection);
+  job = transition(jobPath, job.jobId, 'completed', { error: null });
   releaseLock(job, worker);
   cleanupLaunchdWorker(job);
 }
 
-main().catch(error => {
+main().catch(async error => {
   fail(error.stack || error.message || String(error));
+  if (!acquiredOwnership) {
+    process.exitCode = 1;
+    return;
+  }
   let failedJob = null;
   try {
     const jobIndex = process.argv.indexOf('--job');
     const jobPath = jobIndex >= 0 ? path.resolve(process.argv[jobIndex + 1]) : null;
     if (jobPath) {
       failedJob = loadJob(jobPath);
-      if (failedJob.phase !== 'target-started' && failedJob.phase !== 'failed') {
-        failedJob = transition(jobPath, failedJob.jobId, 'failed', {
-          error: String(error.message || error).slice(0, 512),
-        });
+      if (!TERMINAL_PHASES.has(failedJob.phase)) {
+        const beforeShutdown = [
+          'created',
+          'worker-starting',
+          'worker-ready',
+        ].includes(failedJob.phase);
+        if (beforeShutdown) {
+          failedJob = transition(jobPath, failedJob.jobId, 'failed', {
+            error: String(error.message || error).slice(0, 512),
+          });
+        } else if (failedJob.phase === 'shutdown-requested') {
+          // The old Agent may already be draining. Leave this phase durable so
+          // the stable launcher can reclaim the stale worker and continue.
+        } else if (error.preventRollback) {
+          failedJob = transition(jobPath, failedJob.jobId, 'repair-required', {
+            error: String(error.message || error).slice(0, 512),
+          });
+        } else if (failedJob.worker) {
+          try {
+            failedJob = await rollback(
+              jobPath,
+              failedJob,
+              failedJob.worker,
+              String(error.message || error),
+            );
+          } catch (rollbackError) {
+            fail(String(rollbackError.message || rollbackError));
+            failedJob = loadJob(jobPath);
+          }
+        }
       }
     }
   } catch (recordError) {
     fail(`could not persist worker failure: ${recordError.message}`);
   }
-  if (failedJob?.worker) {
+  if (
+    failedJob?.worker
+    && TERMINAL_PHASES.has(failedJob.phase)
+    && fs.existsSync(path.join(failedJob.dataRoot, 'upgrades', 'active.lock'))
+  ) {
     try {
       releaseLock(failedJob, failedJob.worker);
     } catch (releaseError) {