@outerlayer/cli 0.19.1 → 0.22.0

package/dist/{chunk-F2KXHIFI.js → chunk-22DYXLU7.js}

@@ -1,33 +1,32 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { LEASE_SECONDS, releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startSender, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, AttemptLogBook, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, ALREADY_RELEASED } from './chunk-B6SCN5NY.js';
-import { writePendingUpdate } from './chunk-MTP2QEVO.js';
-import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-S5PCA3SN.js';
+import { releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, commandLogOffset, execCommandLine, startSender, startHostServices, finishJob, isContainerTracked, probeCommand, appendCommandLog, judgeProbe, EXIT_CODE_FILE, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, AttemptLogBook, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, listTrackedJobs, restartSender, ALREADY_RELEASED } from './chunk-FGW2774C.js';
 import { decideInstalledSwitch } from './chunk-3JVYMD4G.js';
-import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-YETTWTAY.js';
-import { writeJobFile, readJobFile, readJobFileChecked } from './chunk-SADUQSO4.js';
 import { compareVersions, satisfiesRange } from './chunk-I2UOAV7X.js';
-import { RUNNER_PROTOCOL } from './chunk-BWSUPRDK.js';
-import { currentSkillCommand, formerSkillProblem, placeholderQueues, placeholderProblem } from './chunk-GEMQLBMW.js';
+import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-YETTWTAY.js';
+import { RUNNER_PROTOCOL } from './chunk-TPKYTORP.js';
+import { writePendingUpdate } from './chunk-KPBQYEND.js';
+import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-2J5JGLW5.js';
+import { currentSkillCommand, formerSkillProblem, placeholderQueues, placeholderProblem } from './chunk-WMHYIBJW.js';
 import { freeBytes, formatBytes } from './chunk-YYYRBYWG.js';
-import { writeMarker, readMarker, currentVersion, removeMarker, entryIn, versionDir, startedThroughCurrent, switchCurrent, cleanVersions, isInLayout, stagingDir } from './chunk-AMFWYESY.js';
+import { writeMarker, readMarker, currentVersion, removeMarker, entryIn, versionDir, startedThroughCurrent, switchCurrent, cleanVersions, isInLayout, stagingDir } from './chunk-4NM3I3VV.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
-import { provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, measureDisk, setUpRunnerCgroup, parseCpuList, ensureTunnel, keepGhConfigCurrent, createContainerRuntime } from './chunk-3ZR7Y23Y.js';
-import { spawnRun } from './chunk-VMJPFQNE.js';
-import { builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, BUILD_LABEL, RUNNER_LABEL } from './chunk-3UZTIRLF.js';
-import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, BuildTokens, classifyRefusal, refusalCode, createRunnerLog, formatClock } from './chunk-XZ4H4X5F.js';
-import { checkFit } from './chunk-QEFKA2QD.js';
-import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-FBTGMSQF.js';
-import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-P53VEAOF.js';
+import { checkFit } from './chunk-I66MQDXH.js';
+import { LEASE_SECONDS, provisionContainer, leaseExpiresAtMs, deliverContainerHooks, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandExec, measureDisk, setUpRunnerCgroup, createContainerRuntime, parseCpuList, ensureTunnel, keepGhConfigCurrent, filesAreCopied } from './chunk-VX74ND77.js';
+import { writeJobFile, ensureRunnerIdentity, readJobFile, builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, spawnRun, BUILD_LABEL, RUNNER_LABEL, readJobFileChecked } from './chunk-WERZMIKA.js';
+import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, BuildTokens, classifyRefusal, refusalCode, createRunnerLog, formatClock } from './chunk-C3OVGKVJ.js';
+import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-2HT2XGJX.js';
+import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-2CMJLYV6.js';
 import { exceedsDisk } from './chunk-7RQ73DLO.js';
-import { claimPidFile, readPidFile, releasePidFile } from './chunk-ECODQ7II.js';
+import { claimPidFile, readPidFile, releasePidFile } from './chunk-ZFLOGMUC.js';
 import { startTimeOf, isSameProcess } from './chunk-JBJTC3UU.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
-import { loadHostKey, ensureHostKey, signerFor } from './chunk-M2ZWWTWZ.js';
-import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, spoolDir, RUNNER_FILE_MODE, rootfsDir } from './chunk-N2PXGQRL.js';
-import { parseClaimIdentity, parseClaimedRepositories, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-BQXFBI7T.js';
+import { loadHostKey, ensureHostKey, signerFor } from './chunk-KDHYLBRR.js';
+import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, RUNNER_FILE_MODE, jobsDir, pidFilePath, spoolDir, rootfsDir } from './chunk-KIGSGZBB.js';
+import { parseClaimIdentity, parseClaimedRepositories, parseUnreachable, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-V7MQKOZY.js';
+import { COMMAND_LOST_REASON, CONTAINER_STOPPED_REASON } from './chunk-GISNHSYL.js';
 import { randomBytes } from 'crypto';
-import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, writeFileSync, readFileSync, rmSync, renameSync, closeSync, openSync, constants, lstatSync } from 'fs';
+import { mkdirSync, existsSync, chmodSync, writeFileSync, renameSync, statSync, readdirSync, readFileSync, rmSync, closeSync, openSync, constants, lstatSync } from 'fs';
 import { join, dirname } from 'path';
 import { totalmem, constants as constants$1 } from 'os';
 
@@ -85,6 +84,9 @@ var HEARTBEAT_TIMEOUT_MS = 1e4;
 var ERROR_CODE_MAX = 100;
 var ERROR_MESSAGE_MAX = 500;
 var BLOCKED_REASON_MAX = 500;
+var TAKES_PATTERNS_MAX = 50;
+var TAKES_PATTERN_CHARS_MAX = 200;
+var WAITING_ITEMS_MAX = 200;
 var UNREACHABLE_MESSAGE = "the request did not reach the platform";
 function noPlatformMessage(status) {
   return `the request got HTTP ${status} with no error message from the platform`;
@@ -112,6 +114,35 @@ function workRequestFromFailure(err) {
 function workRequestFromRetryFailure(outcome) {
   return shapeWorkRequestFailure(outcome.code, outcome.status, outcome.message);
 }
+function stopsOf(data) {
+  const raw = data?.stops;
+  if (!Array.isArray(raw)) return [];
+  const stops = [];
+  for (const entry of raw) {
+    const stop = entry;
+    if (stop === null || typeof stop !== "object" || typeof stop.claimId !== "string") continue;
+    stops.push({
+      claimId: stop.claimId,
+      requestedByName: typeof stop.requestedByName === "string" ? stop.requestedByName : "",
+      note: typeof stop.note === "string" && stop.note !== "" ? stop.note : null
+    });
+  }
+  return stops;
+}
+function takesBody(takes) {
+  let truncated = false;
+  const list = (patterns) => {
+    const all = (patterns ?? []).filter((pattern) => typeof pattern === "string");
+    if (all.length > TAKES_PATTERNS_MAX) truncated = true;
+    return all.slice(0, TAKES_PATTERNS_MAX).map((pattern) => {
+      if (pattern.length > TAKES_PATTERN_CHARS_MAX) truncated = true;
+      return cutForGateway(pattern, TAKES_PATTERN_CHARS_MAX);
+    }).filter((pattern) => pattern !== "");
+  };
+  const include = list(takes.repos.include);
+  const exclude = list(takes.repos.exclude);
+  return { queues: takes.queues, repos: { include, exclude }, truncated };
+}
 async function sendHeartbeat(args, fetchImpl) {
   const body = {
     host: args.host,
@@ -132,7 +163,10 @@ async function sendHeartbeat(args, fetchImpl) {
     // Free disk on the runner's file system, in bytes. Sent only while held, so a host with room sends nothing about its disk.
     lowDisk: args.lowDisk === void 0 ? void 0 : { free: args.lowDisk.free, needed: args.lowDisk.needed },
     // Why the runner claims nothing. Sent only while it holds, so a runner that takes work says nothing about it.
-    blocked: args.blocked === void 0 ? void 0 : { reason: cutForGateway(args.blocked.reason, BLOCKED_REASON_MAX) }
+    blocked: args.blocked === void 0 ? void 0 : { reason: cutForGateway(args.blocked.reason, BLOCKED_REASON_MAX) },
+    state: args.state,
+    waitingItems: args.waitingItems === void 0 ? void 0 : args.waitingItems.slice(0, WAITING_ITEMS_MAX),
+    takes: args.takes === void 0 ? void 0 : takesBody(args.takes)
   };
   try {
     const response = await postJson(
@@ -146,11 +180,22 @@ async function sendHeartbeat(args, fetchImpl) {
       await signerFor(args.configPath)
     );
     const runnerVersion = response.data?.runnerVersion;
-    return typeof runnerVersion === "string" ? { ok: true, runnerVersion } : { ok: true };
+    const stops = stopsOf(response.data);
+    return typeof runnerVersion === "string" ? { ok: true, runnerVersion, stops } : { ok: true, stops };
   } catch (err) {
     return { ok: false, reason: describeCallFailure(err) };
   }
 }
+
+// src/runner/host-state.ts
+function hostState(flags) {
+  if (flags.draining) return "draining";
+  if (flags.restarting) return "restarting";
+  if (flags.holdingForUpdate) return "holding-for-update";
+  if (flags.blocked) return "blocked";
+  if (flags.lowDisk) return "low-disk";
+  return "taking-work";
+}
 var OUTCOMES = ["updated", "rolled_back", "verification_failed", "not_published", "retrying"];
 function lastUpdatePath(configPath) {
   return join(runnerDir(configPath), "last-update.json");
@@ -930,6 +975,7 @@ var QUEUE_ORDER = ["amend", "implement"];
 var TRACKER_REPOSITORY_HOSTS = {
   github: "github.com"
 };
+var PROBE_INTERVAL_MS = 3e3;
 var HOUSEKEEPING_INTERVAL_MS = 36e5;
 function logOf(state) {
   state.deps.log ??= createRunnerLog(state.deps.now ?? (() => /* @__PURE__ */ new Date()));
@@ -1042,6 +1088,7 @@ async function sendTickHeartbeat(state, workRequest) {
     let next = workRequest;
     for (; ; ) {
       reportRepeating(state, "the heartbeat's previous call", void 0, "line");
+      const waitingItems = state.draining === true ? runningItems(state) : [];
       const result = await sendHeartbeat(
         {
           configPath: state.config.configPath,
@@ -1056,12 +1103,25 @@ async function sendTickHeartbeat(state, workRequest) {
           workRequest: next,
           lastUpdate: readLastUpdate(state.config.configPath),
           lowDisk: state.lowDisk,
-          blocked: state.blocked
+          blocked: state.blocked,
+          state: hostState({
+            draining: state.draining === true,
+            restarting: state.restarting === true,
+            holdingForUpdate: state.holdingForUpdate === true,
+            blocked: state.blocked !== void 0,
+            lowDisk: state.lowDisk !== void 0
+          }),
+          // Left out unless a draining runner has builds to wait on, so the gateway stores null.
+          waitingItems: waitingItems.length > 0 ? waitingItems : void 0,
+          takes: { queues: state.config.queues, repos: { include: state.config.repos?.include, exclude: state.config.repos?.exclude } }
         },
         state.deps.fetchImpl
       );
       reportRepeating(state, "the heartbeat", result.ok ? void 0 : `heartbeat failed: ${result.reason}`);
-      if (result.ok) considerNamedVersion(state, result.runnerVersion);
+      if (result.ok) {
+        considerNamedVersion(state, result.runnerVersion);
+        applyStops(state, result.stops);
+      }
       next = state.queuedWorkRequest;
       state.queuedWorkRequest = void 0;
       if (next === void 0) break;
@@ -1070,6 +1130,27 @@ async function sendTickHeartbeat(state, workRequest) {
     state.heartbeating = false;
   }
 }
+function applyStops(state, stops) {
+  for (const stop of stops) {
+    const held = [...state.jobs.values()].find((entry) => entry.job.claimId === stop.claimId);
+    const active = held !== void 0 && !held.ending ? held : void 0;
+    if (active === void 0) {
+      reportRepeating(
+        state,
+        `stop of claim ${stop.claimId}`,
+        held === void 0 ? `ignoring a stop for claim ${stop.claimId}: this runner holds no build under that claim` : `#${held.job.item} a stop was asked for; the build is already ending`,
+        "line"
+      );
+      continue;
+    }
+    const who = `stopped by ${stop.requestedByName === "" ? "someone" : stop.requestedByName} through OuterLayer`;
+    const reason = stop.note === null ? who : `${who}: ${stop.note}`;
+    logOf(state).line(`#${active.job.item} ${reason} \xB7 ending the build`);
+    void endJob(state, active, "stopped", { reason }).catch((err) => {
+      logOf(state).error(`#${active.job.item} could not be stopped: ${describeCallFailure(err)}`);
+    });
+  }
+}
 function ownVersionOf(deps) {
   return (deps.cliVersion ?? resolveVersion()).split(" ")[0] ?? "";
 }
@@ -1388,7 +1469,7 @@ async function housekeepingPass(state) {
   state.housekeepingRunning = true;
   try {
     state.deps.housekeeping ??= {};
-    state.deps.housekeeping.docker ??= createHousekeepingDocker({ runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) });
+    state.deps.housekeeping.docker ??= createHousekeepingDocker({ run: state.deps.housekeeping.run, runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) });
     const result = await runHousekeeping({
       configPath: state.config.configPath,
       keepDays: config.jobsKeepDays,
@@ -1405,9 +1486,17 @@ async function housekeepingPass(state) {
   }
 }
 async function runTick(state) {
+  state.restarting = false;
+  state.holdingForUpdate = false;
   reloadConfig(state);
-  if (restartOntoInstalled(state)) return;
-  if (await updateWhenIdle(state)) return;
+  if (restartOntoInstalled(state)) {
+    state.restarting = true;
+    return;
+  }
+  if (await updateWhenIdle(state)) {
+    state.holdingForUpdate = true;
+    return;
+  }
   if (holdWhenBlocked(state)) return;
   if (holdOffForDisk(state)) return;
   const log = logOf(state);
@@ -1460,10 +1549,10 @@ async function runTick(state) {
   );
 }
 var CLAIM_REFUSAL_CODES = new Set(
-  "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous pull_request_repository_not_connected branch_claimed_elsewhere".split(" ")
+  "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous pull_request_repository_not_connected repository_not_reachable branch_claimed_elsewhere".split(" ")
 );
 async function tryClaimAndStart(state, item, queue) {
-  const { postJson: postJson2 } = await import('./work-cmd-5I3J2XQQ.js');
+  const { postJson: postJson2 } = await import('./work-cmd-L6WMPAA2.js');
   const log = logOf(state);
   const hostKey = await loadHostKey(state.config.configPath);
   const claim = await retrySend(
@@ -1567,6 +1656,9 @@ async function tryClaimAndStart(state, item, queue) {
     return { kind: "unreadable_repositories" };
   }
   reportRepeating(state, "the repositories", void 0);
+  for (const entry of parseUnreachable(data.unreachable)) {
+    log.line(`#${item.number}: ${entry.repository} is left out of this build because its GitHub App installation is suspended`);
+  }
   try {
     await startJob(state, item, queue, claimId, itemKey, branch, identity, repositories);
   } catch (err) {
@@ -1697,7 +1789,8 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
       aborted: () => active.ending,
       deadlineMs: startedAt.getTime() + state.config.timeLimitMinutes * 6e4,
       fetchImpl: state.deps.fetchImpl,
-      now: state.deps.now
+      now: state.deps.now,
+      leaseExpiryMs: () => leaseExpiresAtMs(active.job)
     }).catch((err) => ({ ok: false, reason: `provision failed: ${describeCallFailure(err)}`, aborted: false }));
     const provisionTook = formatDuration(nowOf(state).getTime() - provisionStartedMs);
     if (active.ending) return;
@@ -1710,6 +1803,11 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     active.job.container = provisioned.state;
     active.job.environment = provisioned.environment;
     installCommitHooks(active.job, log, tag);
+    const hooksDelivered = await deliverContainerHooks(runtimeOf(state), provisioned.state, dirPath);
+    if (!hooksDelivered.ok) {
+      await endOnProvisionFailure(state, active, tokens, provisionTook, hooksDelivered.reason);
+      return;
+    }
   } else if (vmMode) {
     const provisioned = await provisionMicroVm({
       config: state.config,
@@ -1941,14 +2039,16 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
       const composed = microVmCommandLine({ job, cmdline, runtime, config: state.config, cli: join(runtime.cliBuild().distDir, "index.js") });
       running = runCommand(composed.line, { ...composed.env, OUTERLAYER_JOB_DIR: dirPath }, dirPath, join(dirPath, "command.log"), state.deps.spawnFn);
     } else if (job.container !== void 0) {
-      const composed = containerCommandLine({ job, cmdline, runtime: runtimeOf(state), config: state.config });
-      running = runCommand(
-        composed.line,
-        { ...composed.env, OUTERLAYER_JOB_DIR: dirPath },
-        dirPath,
-        join(dirPath, "command.log"),
-        state.deps.spawnFn
-      );
+      const runtime = runtimeOf(state);
+      const exec = containerCommandExec({ job, cmdline, runtime, config: state.config });
+      job.container.tracking = "container";
+      job.container.commandLogBase = commandLogOffset(dirPath, 0);
+      writeJobFile(jobFilePath, job);
+      const started = await runtime.host.execDetached(exec);
+      if (started.status !== 0) {
+        const said = started.stderr.trim().split("\n")[0] ?? "";
+        throw new Error(`docker exec exited ${started.status}${said === "" ? "" : `: ${said}`}`);
+      }
     } else {
       const build = job.build;
       const commandEnv = hostCommandEnvironment(jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), build, state.config, identity?.hooksPath);
@@ -1959,12 +2059,13 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     await endJob(state, active, "failed", { stage: "command" });
     return;
   }
-  active.cmd = running;
-  job = { ...job, stage: "command", pid: running.pid, start: running.start ?? void 0 };
+  active.cmd = running ?? null;
+  job = { ...job, stage: "command", ...running === void 0 ? {} : { pid: running.pid, start: running.start ?? void 0 } };
   active.job = job;
   writeJobFile(jobFilePath, job);
   const commandStartedMs = nowOf(state).getTime();
-  log.line(`${tag} command started \xB7 pid ${running.pid}`);
+  active.commandStartedMs = commandStartedMs;
+  log.line(running === void 0 ? `${tag} command started \xB7 in container ${job.container.container}` : `${tag} command started \xB7 pid ${running.pid}`);
   const sender = await startSender({
     config: state.config,
     job,
@@ -1976,6 +2077,10 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     active.job = job;
     writeJobFile(jobFilePath, job);
   }
+  if (running === void 0) {
+    void probeBuild(state, active);
+    return;
+  }
   running.onExit.then(({ code, signal, error }) => {
     if (active.ending) return;
     if (error !== void 0) {
@@ -2122,12 +2227,84 @@ async function renewLeaseIfDue(state, active, nowMs) {
     active.job.lastRenewAt = new Date(nowMs).toISOString();
     writeJobFile(join(active.dirPath, "job.json"), active.job);
     log.line(`${tag} lease renewed \xB7 expires ${formatClock(new Date(nowMs + LEASE_SECONDS * 1e3), new Date(nowMs))}`);
+    if (active.job.container !== void 0) state.containerRuntime?.leases.get(active.job.container.buildId)?.publish();
   } else {
     log.error(`${tag} renew did not land \xB7 retrying in ${RENEW_INTERVAL_SECONDS / 60}m`);
   }
   active.lastRenewAtMs = nowMs;
   return true;
 }
+var COMMAND_START_GRACE_MS = 3e4;
+async function endStoppedBuild(state, active, where) {
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  const read = where === "stopped" ? await readStoppedBuild(state, active.dirPath, active.job) : { code: void 0, readable: true };
+  if (active.ending) return;
+  active.job.container.stopped = true;
+  if (read.code !== void 0) {
+    recordCommandExit(state, active, read.code);
+    return;
+  }
+  log.line(
+    `${tag} the build's container ${where === "missing" ? "is gone" : "stopped"} and its volume ${read.readable ? "holds no exit code" : "could not be read"} \xB7 ending job as interrupted`
+  );
+  void endJob(state, active, "interrupted", { stage: "command", reason: COMMAND_LOST_REASON });
+}
+async function probeBuild(state, active) {
+  const container = active.job.container;
+  if (container === void 0 || !isContainerTracked(active.job) || active.ending || active.probing) return;
+  active.probing = true;
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  try {
+    const runtime = runtimeOf(state);
+    for (; ; ) {
+      const offset = commandLogOffset(active.dirPath, container.commandLogBase);
+      const probe = await probeCommand(runtime, container, offset);
+      if (active.ending) return;
+      if (probe === void 0) {
+        const where = await runtime.host.state(container.container);
+        if (where === "stopped" || where === "missing") {
+          await endStoppedBuild(state, active, where);
+        } else if (active.probeFailing !== true) {
+          active.probeFailing = true;
+          log.error(`${tag} the build's command could not be read from its container (${where}) \xB7 trying again`);
+        }
+        return;
+      }
+      active.probeFailing = false;
+      appendCommandLog(active.dirPath, probe.bytes);
+      if (probe.bytes.length > 0 && offset + probe.bytes.length < probe.size) continue;
+      const starting = active.commandStartedMs !== void 0 && nowOf(state).getTime() - active.commandStartedMs < COMMAND_START_GRACE_MS;
+      const verdict = starting && probe.exitCode === void 0 && !probe.alive ? { kind: "running" } : judgeProbe(probe, active.commandMissed === true);
+      active.commandMissed = verdict.kind === "recheck";
+      if (verdict.kind === "exited") {
+        recordCommandExit(state, active, verdict.code);
+      } else if (verdict.kind === "lost") {
+        log.line(`${tag} command gone, no exit code recorded \xB7 ending job as interrupted`);
+        void endJob(state, active, "interrupted", { stage: "command", reason: COMMAND_LOST_REASON });
+      }
+      return;
+    }
+  } catch (err) {
+    log.error(`${tag} reading the build's command failed: ${describeCallFailure(err)}`);
+  } finally {
+    active.probing = false;
+  }
+}
+function recordCommandExit(state, active, code) {
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  const file = join(active.dirPath, EXIT_CODE_FILE);
+  writeFileSync(`${file}.partial`, `${code}
+`, { mode: RUNNER_FILE_MODE });
+  renameSync(`${file}.partial`, file);
+  const took = active.commandStartedMs === void 0 ? "" : ` (${formatDuration(nowOf(state).getTime() - active.commandStartedMs)})`;
+  log.line(`${tag} command exited ${code}${took}`);
+  active.job.exitCode = code;
+  active.job.commandEndReason = jobNotEndedByRunnerReason({ code, signal: null }, active.job);
+  void endJob(state, active, code === 0 ? "ok" : "failed");
+}
 async function watchTick(state, active, nowMs, logMtimeMs) {
   if (active.ending) {
     if (active.job.stage === "checks" || active.job.stage === "judge") await renewLeaseIfDue(state, active, nowMs);
@@ -2136,6 +2313,7 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
   const log = logOf(state);
   const tag = `#${active.job.item}`;
   const pid = active.job.pid;
+  const started = pid !== void 0 || isContainerTracked(active.job);
   if (pid !== void 0 && !isGroupAlive(pid)) {
     if (active.cmd === null) {
       const recorded = outcomeFromExitCode(active.dirPath);
@@ -2162,16 +2340,22 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
     }
     if (refusal?.kind === "failed") {
       log.error(`${tag} a token request was refused: ${refusal.reason} \xB7 ending job as failed`);
-      await endJob(state, active, "failed", { stage: pid === void 0 ? "provision" : "command", reason: refusal.reason });
+      await endJob(state, active, "failed", { stage: started ? "command" : "provision", reason: refusal.reason });
       return;
     }
   }
+  const unreachable = active.job.container === void 0 ? void 0 : runtimeOf(state).relayFatal.get(active.job.container.buildId);
+  if (unreachable !== void 0) {
+    log.error(`${tag} ${unreachable} \xB7 ending job as failed`);
+    await endJob(state, active, "failed", { stage: started ? "command" : "provision", reason: unreachable });
+    return;
+  }
   if (isTimedOut(new Date(active.job.startedAt).getTime(), nowMs, state.config.timeLimitMinutes)) {
     log.line(`${tag} past the ${state.config.timeLimitMinutes}m time limit \xB7 ending job as timed_out`);
     await endJob(state, active, "timed_out");
     return;
   }
-  if (pid === void 0) return;
+  if (!started) return;
   if (active.job.container !== void 0) {
     const limit = state.config.build?.disk ?? "40g";
     const used = await measureDisk(runtimeOf(state), active.job.container);
@@ -2219,6 +2403,117 @@ async function probeCredentials(config, fetchImpl = fetch) {
     return { ok: false, fatal: false, reason: describeCallFailure(err) };
   }
 }
+async function rejoinContainerBuild(state, adopted) {
+  const { job, dirPath } = adopted;
+  const config = state.config;
+  const container = job.container;
+  const cores = container.cpuset === void 0 ? void 0 : parseCpuList(container.cpuset);
+  if (cores !== void 0) runtimeOf(state).cpus.hold(container.buildId, cores);
+  const tokens = buildTokensFor(state, adopted, job.item);
+  runtimeOf(state).credentials.set(dirPath, tokens);
+  await ensureTunnel(runtimeOf(state), container, dirPath, config, job.itemKey, job.branch, job.repositories, (line) => logOf(state).line(`#${job.item} ${line}`), () => leaseExpiresAtMs(adopted.job));
+  try {
+    await tokens.token("read");
+    keepGhConfigCurrent(
+      tokens,
+      container.buildDir,
+      (err) => logOf(state).error(`#${job.item} the read token could not be written for gh: ${describeCallFailure(err)}`),
+      filesAreCopied(container) ? {
+        runtime: runtimeOf(state),
+        state: container,
+        secrets: () => tokens.issuedTokens(),
+        onFailure: (reason) => logOf(state).error(`#${job.item} ${reason}`)
+      } : void 0
+    );
+  } catch (err) {
+    logOf(state).error(`#${job.item} could not give the adopted build a read token: ${describeCallFailure(err)}`);
+  }
+}
+async function adoptTrackedBuilds(state, jobsDirPath) {
+  const log = logOf(state);
+  const attemptLogs = attemptLogsOf(state);
+  const runtime = runtimeOf(state);
+  for (const { dirPath, job, known } of listTrackedJobs(jobsDirPath)) {
+    if (!known) continue;
+    const container = job.container;
+    const where = await runtime.host.state(container.container).catch(() => "unknown");
+    if (where === "stopped" || where === "missing") {
+      await finishTrackedWhileAway(state, dirPath, job, where);
+      continue;
+    }
+    const adopted = { job, dirPath, lastRenewAtMs: 0, cmd: null, cpuSample: null, ending: false };
+    state.jobs.set(dirPath, adopted);
+    attemptLogs.open(job.item, dirPath);
+    await rejoinContainerBuild(state, adopted);
+    if (!await renewLeaseIfDue(state, adopted, nowOf(state).getTime())) continue;
+    const sender = await restartSender({ config: state.config, job: adopted.job, jobDir: dirPath, deps: { spawnFn: state.deps.spawnFn, runtime, log, sleep: state.deps.sleep } });
+    adopted.job = sender === void 0 ? { ...adopted.job, sender: void 0 } : { ...adopted.job, sender };
+    writeJobFile(join(dirPath, "job.json"), adopted.job);
+    const line = `adopted job ${job.queue} #${job.item} (claim ${shortClaim(job.claimId)}) \xB7 command tracked in container ${container.container}`;
+    log.line(line);
+    attemptLogs.note(job.item, line);
+    void probeBuild(state, adopted);
+  }
+}
+async function readStoppedBuild(state, dirPath, job) {
+  const runtime = runtimeOf(state);
+  const container = job.container;
+  runtime.leases.get(container.buildId)?.stop();
+  try {
+    for (let attempt = 1; attempt <= STOPPED_READ_ATTEMPTS; attempt++) {
+      const started = await runtime.host.start(container.container);
+      if (started.status === 0) {
+        const code = await readVolumeExitCode(runtime, dirPath, container);
+        if (code !== "unreadable") {
+          await runtime.host.stop(container.container);
+          return { code, readable: true };
+        }
+        await runtime.host.stop(container.container).catch(() => void 0);
+      }
+      if (attempt < STOPPED_READ_ATTEMPTS) await (state.deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(STOPPED_READ_RETRY_MS);
+    }
+    logOf(state).error(`#${job.item} the stopped build's volume could not be read after ${STOPPED_READ_ATTEMPTS} tries`);
+    return { code: void 0, readable: false };
+  } catch (err) {
+    logOf(state).error(`#${job.item} the stopped build's volume could not be read: ${describeCallFailure(err)}`);
+    return { code: void 0, readable: false };
+  }
+}
+var STOPPED_READ_ATTEMPTS = 3;
+var STOPPED_READ_RETRY_MS = 1e3;
+async function readVolumeExitCode(runtime, dirPath, container) {
+  for (; ; ) {
+    const offset = commandLogOffset(dirPath, container.commandLogBase);
+    const probe = await probeCommand(runtime, container, offset);
+    if (probe === void 0) return "unreadable";
+    appendCommandLog(dirPath, probe.bytes);
+    if (probe.bytes.length > 0 && offset + probe.bytes.length < probe.size) continue;
+    return probe.exitCode;
+  }
+}
+async function finishTrackedWhileAway(state, dirPath, job, where) {
+  const log = logOf(state);
+  const container = job.container;
+  const read = where === "stopped" ? await readStoppedBuild(state, dirPath, job) : { code: void 0, readable: true };
+  const code = read.code;
+  container.stopped = true;
+  let outcome = "interrupted";
+  let ending = { reason: CONTAINER_STOPPED_REASON };
+  if (code !== void 0) {
+    const file = join(dirPath, EXIT_CODE_FILE);
+    writeFileSync(`${file}.partial`, `${code}
+`, { mode: RUNNER_FILE_MODE });
+    renameSync(`${file}.partial`, file);
+    job.exitCode = code;
+    job.commandEndReason = jobNotEndedByRunnerReason({ code, signal: null }, job);
+    outcome = code === 0 ? "ok" : "failed";
+    ending = void 0;
+  }
+  const finished = await finishJob(job, dirPath, outcome, state.config, finishDepsOf(state), ending);
+  log.line(
+    `recovered job ${finished.queue} #${finished.item} (claim ${shortClaim(finished.claimId)}) as ${outcome}: ${code === void 0 ? `its container ${where === "missing" ? "is gone" : "stopped"} and its volume ${read.readable ? "holds no exit code" : "could not be read"}` : `its command recorded exit ${code}`}`
+  );
+}
 async function start(configPath, deps = { fetchImpl: fetch }) {
   const log = deps.log ??= createRunnerLog(deps.now ?? (() => /* @__PURE__ */ new Date()));
   const update = deps.update;
@@ -2226,7 +2521,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const outcome = await onStartUnderMarker({
       configPath,
       own: ownVersionOf(deps),
-      check: update.check ? () => update.check(configPath) : async () => (await import('./check-W2JGXN3B.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
+      check: update.check ? () => update.check(configPath) : async () => (await import('./check-3YHMJQ3P.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
       now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
       log
     });
@@ -2239,6 +2534,12 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   const hookProblem = checkHooks(config);
   if (hookProblem) throw new RunnerStartError(hookProblem);
   await ensureHostKey(configPath);
+  if (builtinKind(config.hooks.provision) === "container" || builtinKind(config.hooks.provision) === "vm") {
+    await ensureRunnerIdentity(configPath, {
+      docker: deps.runnerIdentity ?? (deps.container?.host === void 0 ? void 0 : { holdsObjects: async () => false }),
+      log: (line) => log.error(line)
+    }).catch((err) => log.error(`the runner's Docker identity could not be stored: ${describeCallFailure(err)} \xB7 using the names its folder's path gives`));
+  }
   const probe = await probeCredentials(config, deps.fetchImpl);
   if (!probe.ok && probe.fatal) throw new RunnerStartError(probe.reason);
   if (!probe.ok) log.error(`the gateway did not answer at start: ${probe.reason} \xB7 starting anyway`);
@@ -2269,23 +2570,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const adopted = { job, dirPath, lastRenewAtMs: nowMs(), cmd: null, cpuSample: null, ending: false };
     state.jobs.set(dirPath, adopted);
     attemptLogs.open(job.item, dirPath);
-    if (job.container !== void 0) {
-      const cores = job.container.cpuset === void 0 ? void 0 : parseCpuList(job.container.cpuset);
-      if (cores !== void 0) runtimeOf(state).cpus.hold(job.container.buildId, cores);
-      const tokens = buildTokensFor(state, adopted, job.item);
-      runtimeOf(state).credentials.set(dirPath, tokens);
-      await ensureTunnel(runtimeOf(state), job.container, dirPath, config, job.itemKey, job.branch, job.repositories);
-      try {
-        await tokens.token("read");
-        keepGhConfigCurrent(
-          tokens,
-          job.container.buildDir,
-          (err) => logOf(state).error(`#${job.item} the read token could not be written for gh: ${describeCallFailure(err)}`)
-        );
-      } catch (err) {
-        logOf(state).error(`#${job.item} could not give the adopted build a read token: ${describeCallFailure(err)}`);
-      }
-    }
+    if (job.container !== void 0) await rejoinContainerBuild(state, adopted);
     if (job.build !== void 0 && !runtimeOf(state).tunnels.has(job.build.buildId)) {
       await startHostServices({ runtime: runtimeOf(state), config, build: job.build, jobDir: dirPath, itemKey: job.itemKey, process: job.build.workDir !== void 0 }).catch((err) => {
         logOf(state).error(`the tunnel of #${job.item} could not start again: ${describeCallFailure(err)}`);
@@ -2295,6 +2580,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     log.line(adoptedLine);
     attemptLogs.note(job.item, adoptedLine);
   }
+  await adoptTrackedBuilds(state, jobsDir(configPath));
   for (const { job, released } of await retryFailedReleases(jobsDir(configPath), config, {
     fetchImpl: deps.fetchImpl,
     now: deps.now,
@@ -2312,12 +2598,11 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   }
   let intakeClosed = false;
   let closed = false;
-  let draining = false;
   let stopping = false;
   let exited = false;
   let interrupts = 0;
   state.pollTimerSeconds = state.config.pollSeconds;
-  const pollTimer = setInterval(() => void tick(state), state.pollTimerSeconds * 1e3);
+  let pollTimer = setInterval(() => void tick(state), state.pollTimerSeconds * 1e3);
   const hooksKind = builtinKind(config.hooks.provision);
   const housekeepingTimer = hooksKind === "container" || hooksKind === "vm" ? (() => {
     const everyMs = deps.housekeeping?.intervalMs ?? HOUSEKEEPING_INTERVAL_MS;
@@ -2337,6 +2622,9 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
       });
     }
   }, deps.watchIntervalMs ?? 3e4);
+  const probeTimer = setInterval(() => {
+    for (const active of state.jobs.values()) void probeBuild(state, active);
+  }, deps.probeIntervalMs ?? PROBE_INTERVAL_MS);
   const onSigterm = () => void drain();
   const onSigusr1 = () => void stopNow();
   const onSigint = () => {
@@ -2357,6 +2645,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     if (intakeClosed) return;
     intakeClosed = true;
     clearInterval(pollTimer);
+    pollTimer = setInterval(() => void sendTickHeartbeat(state, void 0), (state.pollTimerSeconds ?? state.config.pollSeconds) * 1e3);
     clearInterval(housekeepingTimer);
   };
   const close = () => {
@@ -2364,18 +2653,21 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     detachSignals();
     if (closed) return;
     closed = true;
+    clearInterval(pollTimer);
     clearInterval(watchTimer);
+    clearInterval(probeTimer);
   };
   const finish2 = () => {
     if (exited) return;
     exited = true;
     close();
+    for (const relay of state.containerRuntime?.relays.values() ?? []) void relay.close();
     releasePidFile(pidFilePath(configPath), self);
     log.line("runner exited");
   };
   const drain = async (onWaiting) => {
-    if (draining) return;
-    draining = true;
+    if (state.draining === true) return;
+    state.draining = true;
     stopIntake();
     let passes = 0;
     while (state.jobs.size > 0 && !exited) {
@@ -2397,7 +2689,9 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     if (stopping) return;
     stopping = true;
     stopIntake();
+    clearInterval(pollTimer);
     clearInterval(watchTimer);
+    clearInterval(probeTimer);
     log.line(`stopping: ending ${state.jobs.size} running job(s) now`);
     await Promise.all([...state.jobs.values()].map((active) => endJob(state, active, "stopped")));
     finish2();
@@ -2483,4 +2777,4 @@ function runnerProcessState(configPath) {
   return isSameProcess(entry.pid, entry.start) ? { state: "running", pid: entry.pid } : { state: "stale", pid: entry.pid };
 }
 
-export { RunnerStartError, listJobFiles, probeCredentials, requestStop, runnerProcessState, start, tick, watchTick };
+export { RunnerStartError, listJobFiles, probeBuild, probeCredentials, requestStop, runnerProcessState, start, tick, watchTick };