@outerlayer/cli 0.20.0 → 0.22.0

package/dist/{chunk-AD3EYI7P.js → chunk-22DYXLU7.js}

@@ -1,33 +1,32 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { LEASE_SECONDS, releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startSender, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, AttemptLogBook, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, ALREADY_RELEASED } from './chunk-B6SCN5NY.js';
-import { writeJobFile, readJobFile, readJobFileChecked } from './chunk-SADUQSO4.js';
-import { writePendingUpdate } from './chunk-MTP2QEVO.js';
-import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-S5PCA3SN.js';
+import { releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, commandLogOffset, execCommandLine, startSender, startHostServices, finishJob, isContainerTracked, probeCommand, appendCommandLog, judgeProbe, EXIT_CODE_FILE, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, AttemptLogBook, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, listTrackedJobs, restartSender, ALREADY_RELEASED } from './chunk-FGW2774C.js';
 import { decideInstalledSwitch } from './chunk-3JVYMD4G.js';
-import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-YETTWTAY.js';
 import { compareVersions, satisfiesRange } from './chunk-I2UOAV7X.js';
-import { RUNNER_PROTOCOL } from './chunk-ZX4I2MEK.js';
-import { currentSkillCommand, formerSkillProblem, placeholderQueues, placeholderProblem } from './chunk-GEMQLBMW.js';
+import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-YETTWTAY.js';
+import { RUNNER_PROTOCOL } from './chunk-TPKYTORP.js';
+import { writePendingUpdate } from './chunk-KPBQYEND.js';
+import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-2J5JGLW5.js';
+import { currentSkillCommand, formerSkillProblem, placeholderQueues, placeholderProblem } from './chunk-WMHYIBJW.js';
 import { freeBytes, formatBytes } from './chunk-YYYRBYWG.js';
-import { writeMarker, readMarker, currentVersion, removeMarker, entryIn, versionDir, startedThroughCurrent, switchCurrent, cleanVersions, isInLayout, stagingDir } from './chunk-AMFWYESY.js';
+import { writeMarker, readMarker, currentVersion, removeMarker, entryIn, versionDir, startedThroughCurrent, switchCurrent, cleanVersions, isInLayout, stagingDir } from './chunk-4NM3I3VV.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
-import { provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, measureDisk, setUpRunnerCgroup, parseCpuList, ensureTunnel, keepGhConfigCurrent, createContainerRuntime } from './chunk-3ZR7Y23Y.js';
-import { spawnRun } from './chunk-VMJPFQNE.js';
-import { builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, BUILD_LABEL, RUNNER_LABEL } from './chunk-3UZTIRLF.js';
-import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, BuildTokens, classifyRefusal, refusalCode, createRunnerLog, formatClock } from './chunk-XZ4H4X5F.js';
-import { checkFit } from './chunk-QEFKA2QD.js';
-import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-FBTGMSQF.js';
-import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-P53VEAOF.js';
+import { checkFit } from './chunk-I66MQDXH.js';
+import { LEASE_SECONDS, provisionContainer, leaseExpiresAtMs, deliverContainerHooks, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandExec, measureDisk, setUpRunnerCgroup, createContainerRuntime, parseCpuList, ensureTunnel, keepGhConfigCurrent, filesAreCopied } from './chunk-VX74ND77.js';
+import { writeJobFile, ensureRunnerIdentity, readJobFile, builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, spawnRun, BUILD_LABEL, RUNNER_LABEL, readJobFileChecked } from './chunk-WERZMIKA.js';
+import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, BuildTokens, classifyRefusal, refusalCode, createRunnerLog, formatClock } from './chunk-C3OVGKVJ.js';
+import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-2HT2XGJX.js';
+import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-2CMJLYV6.js';
 import { exceedsDisk } from './chunk-7RQ73DLO.js';
-import { claimPidFile, readPidFile, releasePidFile } from './chunk-ECODQ7II.js';
+import { claimPidFile, readPidFile, releasePidFile } from './chunk-ZFLOGMUC.js';
 import { startTimeOf, isSameProcess } from './chunk-JBJTC3UU.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
-import { loadHostKey, ensureHostKey, signerFor } from './chunk-M2ZWWTWZ.js';
-import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, spoolDir, RUNNER_FILE_MODE, rootfsDir } from './chunk-N2PXGQRL.js';
-import { parseClaimIdentity, parseClaimedRepositories, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-BQXFBI7T.js';
+import { loadHostKey, ensureHostKey, signerFor } from './chunk-KDHYLBRR.js';
+import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, RUNNER_FILE_MODE, jobsDir, pidFilePath, spoolDir, rootfsDir } from './chunk-KIGSGZBB.js';
+import { parseClaimIdentity, parseClaimedRepositories, parseUnreachable, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-V7MQKOZY.js';
+import { COMMAND_LOST_REASON, CONTAINER_STOPPED_REASON } from './chunk-GISNHSYL.js';
 import { randomBytes } from 'crypto';
-import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, writeFileSync, readFileSync, rmSync, renameSync, closeSync, openSync, constants, lstatSync } from 'fs';
+import { mkdirSync, existsSync, chmodSync, writeFileSync, renameSync, statSync, readdirSync, readFileSync, rmSync, closeSync, openSync, constants, lstatSync } from 'fs';
 import { join, dirname } from 'path';
 import { totalmem, constants as constants$1 } from 'os';
 
@@ -976,6 +975,7 @@ var QUEUE_ORDER = ["amend", "implement"];
 var TRACKER_REPOSITORY_HOSTS = {
   github: "github.com"
 };
+var PROBE_INTERVAL_MS = 3e3;
 var HOUSEKEEPING_INTERVAL_MS = 36e5;
 function logOf(state) {
   state.deps.log ??= createRunnerLog(state.deps.now ?? (() => /* @__PURE__ */ new Date()));
@@ -1469,7 +1469,7 @@ async function housekeepingPass(state) {
   state.housekeepingRunning = true;
   try {
     state.deps.housekeeping ??= {};
-    state.deps.housekeeping.docker ??= createHousekeepingDocker({ runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) });
+    state.deps.housekeeping.docker ??= createHousekeepingDocker({ run: state.deps.housekeeping.run, runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) });
     const result = await runHousekeeping({
       configPath: state.config.configPath,
       keepDays: config.jobsKeepDays,
@@ -1549,10 +1549,10 @@ async function runTick(state) {
   );
 }
 var CLAIM_REFUSAL_CODES = new Set(
-  "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous pull_request_repository_not_connected branch_claimed_elsewhere".split(" ")
+  "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous pull_request_repository_not_connected repository_not_reachable branch_claimed_elsewhere".split(" ")
 );
 async function tryClaimAndStart(state, item, queue) {
-  const { postJson: postJson2 } = await import('./work-cmd-5I3J2XQQ.js');
+  const { postJson: postJson2 } = await import('./work-cmd-L6WMPAA2.js');
   const log = logOf(state);
   const hostKey = await loadHostKey(state.config.configPath);
   const claim = await retrySend(
@@ -1656,6 +1656,9 @@ async function tryClaimAndStart(state, item, queue) {
     return { kind: "unreadable_repositories" };
   }
   reportRepeating(state, "the repositories", void 0);
+  for (const entry of parseUnreachable(data.unreachable)) {
+    log.line(`#${item.number}: ${entry.repository} is left out of this build because its GitHub App installation is suspended`);
+  }
   try {
     await startJob(state, item, queue, claimId, itemKey, branch, identity, repositories);
   } catch (err) {
@@ -1786,7 +1789,8 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
       aborted: () => active.ending,
       deadlineMs: startedAt.getTime() + state.config.timeLimitMinutes * 6e4,
       fetchImpl: state.deps.fetchImpl,
-      now: state.deps.now
+      now: state.deps.now,
+      leaseExpiryMs: () => leaseExpiresAtMs(active.job)
     }).catch((err) => ({ ok: false, reason: `provision failed: ${describeCallFailure(err)}`, aborted: false }));
     const provisionTook = formatDuration(nowOf(state).getTime() - provisionStartedMs);
     if (active.ending) return;
@@ -1799,6 +1803,11 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     active.job.container = provisioned.state;
     active.job.environment = provisioned.environment;
     installCommitHooks(active.job, log, tag);
+    const hooksDelivered = await deliverContainerHooks(runtimeOf(state), provisioned.state, dirPath);
+    if (!hooksDelivered.ok) {
+      await endOnProvisionFailure(state, active, tokens, provisionTook, hooksDelivered.reason);
+      return;
+    }
   } else if (vmMode) {
     const provisioned = await provisionMicroVm({
       config: state.config,
@@ -2030,14 +2039,16 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
       const composed = microVmCommandLine({ job, cmdline, runtime, config: state.config, cli: join(runtime.cliBuild().distDir, "index.js") });
       running = runCommand(composed.line, { ...composed.env, OUTERLAYER_JOB_DIR: dirPath }, dirPath, join(dirPath, "command.log"), state.deps.spawnFn);
     } else if (job.container !== void 0) {
-      const composed = containerCommandLine({ job, cmdline, runtime: runtimeOf(state), config: state.config });
-      running = runCommand(
-        composed.line,
-        { ...composed.env, OUTERLAYER_JOB_DIR: dirPath },
-        dirPath,
-        join(dirPath, "command.log"),
-        state.deps.spawnFn
-      );
+      const runtime = runtimeOf(state);
+      const exec = containerCommandExec({ job, cmdline, runtime, config: state.config });
+      job.container.tracking = "container";
+      job.container.commandLogBase = commandLogOffset(dirPath, 0);
+      writeJobFile(jobFilePath, job);
+      const started = await runtime.host.execDetached(exec);
+      if (started.status !== 0) {
+        const said = started.stderr.trim().split("\n")[0] ?? "";
+        throw new Error(`docker exec exited ${started.status}${said === "" ? "" : `: ${said}`}`);
+      }
     } else {
       const build = job.build;
       const commandEnv = hostCommandEnvironment(jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), build, state.config, identity?.hooksPath);
@@ -2048,12 +2059,13 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     await endJob(state, active, "failed", { stage: "command" });
     return;
   }
-  active.cmd = running;
-  job = { ...job, stage: "command", pid: running.pid, start: running.start ?? void 0 };
+  active.cmd = running ?? null;
+  job = { ...job, stage: "command", ...running === void 0 ? {} : { pid: running.pid, start: running.start ?? void 0 } };
   active.job = job;
   writeJobFile(jobFilePath, job);
   const commandStartedMs = nowOf(state).getTime();
-  log.line(`${tag} command started \xB7 pid ${running.pid}`);
+  active.commandStartedMs = commandStartedMs;
+  log.line(running === void 0 ? `${tag} command started \xB7 in container ${job.container.container}` : `${tag} command started \xB7 pid ${running.pid}`);
   const sender = await startSender({
     config: state.config,
     job,
@@ -2065,6 +2077,10 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
     active.job = job;
     writeJobFile(jobFilePath, job);
   }
+  if (running === void 0) {
+    void probeBuild(state, active);
+    return;
+  }
   running.onExit.then(({ code, signal, error }) => {
     if (active.ending) return;
     if (error !== void 0) {
@@ -2211,12 +2227,84 @@ async function renewLeaseIfDue(state, active, nowMs) {
     active.job.lastRenewAt = new Date(nowMs).toISOString();
     writeJobFile(join(active.dirPath, "job.json"), active.job);
     log.line(`${tag} lease renewed \xB7 expires ${formatClock(new Date(nowMs + LEASE_SECONDS * 1e3), new Date(nowMs))}`);
+    if (active.job.container !== void 0) state.containerRuntime?.leases.get(active.job.container.buildId)?.publish();
   } else {
     log.error(`${tag} renew did not land \xB7 retrying in ${RENEW_INTERVAL_SECONDS / 60}m`);
   }
   active.lastRenewAtMs = nowMs;
   return true;
 }
+var COMMAND_START_GRACE_MS = 3e4;
+async function endStoppedBuild(state, active, where) {
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  const read = where === "stopped" ? await readStoppedBuild(state, active.dirPath, active.job) : { code: void 0, readable: true };
+  if (active.ending) return;
+  active.job.container.stopped = true;
+  if (read.code !== void 0) {
+    recordCommandExit(state, active, read.code);
+    return;
+  }
+  log.line(
+    `${tag} the build's container ${where === "missing" ? "is gone" : "stopped"} and its volume ${read.readable ? "holds no exit code" : "could not be read"} \xB7 ending job as interrupted`
+  );
+  void endJob(state, active, "interrupted", { stage: "command", reason: COMMAND_LOST_REASON });
+}
+async function probeBuild(state, active) {
+  const container = active.job.container;
+  if (container === void 0 || !isContainerTracked(active.job) || active.ending || active.probing) return;
+  active.probing = true;
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  try {
+    const runtime = runtimeOf(state);
+    for (; ; ) {
+      const offset = commandLogOffset(active.dirPath, container.commandLogBase);
+      const probe = await probeCommand(runtime, container, offset);
+      if (active.ending) return;
+      if (probe === void 0) {
+        const where = await runtime.host.state(container.container);
+        if (where === "stopped" || where === "missing") {
+          await endStoppedBuild(state, active, where);
+        } else if (active.probeFailing !== true) {
+          active.probeFailing = true;
+          log.error(`${tag} the build's command could not be read from its container (${where}) \xB7 trying again`);
+        }
+        return;
+      }
+      active.probeFailing = false;
+      appendCommandLog(active.dirPath, probe.bytes);
+      if (probe.bytes.length > 0 && offset + probe.bytes.length < probe.size) continue;
+      const starting = active.commandStartedMs !== void 0 && nowOf(state).getTime() - active.commandStartedMs < COMMAND_START_GRACE_MS;
+      const verdict = starting && probe.exitCode === void 0 && !probe.alive ? { kind: "running" } : judgeProbe(probe, active.commandMissed === true);
+      active.commandMissed = verdict.kind === "recheck";
+      if (verdict.kind === "exited") {
+        recordCommandExit(state, active, verdict.code);
+      } else if (verdict.kind === "lost") {
+        log.line(`${tag} command gone, no exit code recorded \xB7 ending job as interrupted`);
+        void endJob(state, active, "interrupted", { stage: "command", reason: COMMAND_LOST_REASON });
+      }
+      return;
+    }
+  } catch (err) {
+    log.error(`${tag} reading the build's command failed: ${describeCallFailure(err)}`);
+  } finally {
+    active.probing = false;
+  }
+}
+function recordCommandExit(state, active, code) {
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  const file = join(active.dirPath, EXIT_CODE_FILE);
+  writeFileSync(`${file}.partial`, `${code}
+`, { mode: RUNNER_FILE_MODE });
+  renameSync(`${file}.partial`, file);
+  const took = active.commandStartedMs === void 0 ? "" : ` (${formatDuration(nowOf(state).getTime() - active.commandStartedMs)})`;
+  log.line(`${tag} command exited ${code}${took}`);
+  active.job.exitCode = code;
+  active.job.commandEndReason = jobNotEndedByRunnerReason({ code, signal: null }, active.job);
+  void endJob(state, active, code === 0 ? "ok" : "failed");
+}
 async function watchTick(state, active, nowMs, logMtimeMs) {
   if (active.ending) {
     if (active.job.stage === "checks" || active.job.stage === "judge") await renewLeaseIfDue(state, active, nowMs);
@@ -2225,6 +2313,7 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
   const log = logOf(state);
   const tag = `#${active.job.item}`;
   const pid = active.job.pid;
+  const started = pid !== void 0 || isContainerTracked(active.job);
   if (pid !== void 0 && !isGroupAlive(pid)) {
     if (active.cmd === null) {
       const recorded = outcomeFromExitCode(active.dirPath);
@@ -2251,16 +2340,22 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
     }
     if (refusal?.kind === "failed") {
       log.error(`${tag} a token request was refused: ${refusal.reason} \xB7 ending job as failed`);
-      await endJob(state, active, "failed", { stage: pid === void 0 ? "provision" : "command", reason: refusal.reason });
+      await endJob(state, active, "failed", { stage: started ? "command" : "provision", reason: refusal.reason });
       return;
     }
   }
+  const unreachable = active.job.container === void 0 ? void 0 : runtimeOf(state).relayFatal.get(active.job.container.buildId);
+  if (unreachable !== void 0) {
+    log.error(`${tag} ${unreachable} \xB7 ending job as failed`);
+    await endJob(state, active, "failed", { stage: started ? "command" : "provision", reason: unreachable });
+    return;
+  }
   if (isTimedOut(new Date(active.job.startedAt).getTime(), nowMs, state.config.timeLimitMinutes)) {
     log.line(`${tag} past the ${state.config.timeLimitMinutes}m time limit \xB7 ending job as timed_out`);
     await endJob(state, active, "timed_out");
     return;
   }
-  if (pid === void 0) return;
+  if (!started) return;
   if (active.job.container !== void 0) {
     const limit = state.config.build?.disk ?? "40g";
     const used = await measureDisk(runtimeOf(state), active.job.container);
@@ -2308,6 +2403,117 @@ async function probeCredentials(config, fetchImpl = fetch) {
     return { ok: false, fatal: false, reason: describeCallFailure(err) };
   }
 }
+async function rejoinContainerBuild(state, adopted) {
+  const { job, dirPath } = adopted;
+  const config = state.config;
+  const container = job.container;
+  const cores = container.cpuset === void 0 ? void 0 : parseCpuList(container.cpuset);
+  if (cores !== void 0) runtimeOf(state).cpus.hold(container.buildId, cores);
+  const tokens = buildTokensFor(state, adopted, job.item);
+  runtimeOf(state).credentials.set(dirPath, tokens);
+  await ensureTunnel(runtimeOf(state), container, dirPath, config, job.itemKey, job.branch, job.repositories, (line) => logOf(state).line(`#${job.item} ${line}`), () => leaseExpiresAtMs(adopted.job));
+  try {
+    await tokens.token("read");
+    keepGhConfigCurrent(
+      tokens,
+      container.buildDir,
+      (err) => logOf(state).error(`#${job.item} the read token could not be written for gh: ${describeCallFailure(err)}`),
+      filesAreCopied(container) ? {
+        runtime: runtimeOf(state),
+        state: container,
+        secrets: () => tokens.issuedTokens(),
+        onFailure: (reason) => logOf(state).error(`#${job.item} ${reason}`)
+      } : void 0
+    );
+  } catch (err) {
+    logOf(state).error(`#${job.item} could not give the adopted build a read token: ${describeCallFailure(err)}`);
+  }
+}
+async function adoptTrackedBuilds(state, jobsDirPath) {
+  const log = logOf(state);
+  const attemptLogs = attemptLogsOf(state);
+  const runtime = runtimeOf(state);
+  for (const { dirPath, job, known } of listTrackedJobs(jobsDirPath)) {
+    if (!known) continue;
+    const container = job.container;
+    const where = await runtime.host.state(container.container).catch(() => "unknown");
+    if (where === "stopped" || where === "missing") {
+      await finishTrackedWhileAway(state, dirPath, job, where);
+      continue;
+    }
+    const adopted = { job, dirPath, lastRenewAtMs: 0, cmd: null, cpuSample: null, ending: false };
+    state.jobs.set(dirPath, adopted);
+    attemptLogs.open(job.item, dirPath);
+    await rejoinContainerBuild(state, adopted);
+    if (!await renewLeaseIfDue(state, adopted, nowOf(state).getTime())) continue;
+    const sender = await restartSender({ config: state.config, job: adopted.job, jobDir: dirPath, deps: { spawnFn: state.deps.spawnFn, runtime, log, sleep: state.deps.sleep } });
+    adopted.job = sender === void 0 ? { ...adopted.job, sender: void 0 } : { ...adopted.job, sender };
+    writeJobFile(join(dirPath, "job.json"), adopted.job);
+    const line = `adopted job ${job.queue} #${job.item} (claim ${shortClaim(job.claimId)}) \xB7 command tracked in container ${container.container}`;
+    log.line(line);
+    attemptLogs.note(job.item, line);
+    void probeBuild(state, adopted);
+  }
+}
+async function readStoppedBuild(state, dirPath, job) {
+  const runtime = runtimeOf(state);
+  const container = job.container;
+  runtime.leases.get(container.buildId)?.stop();
+  try {
+    for (let attempt = 1; attempt <= STOPPED_READ_ATTEMPTS; attempt++) {
+      const started = await runtime.host.start(container.container);
+      if (started.status === 0) {
+        const code = await readVolumeExitCode(runtime, dirPath, container);
+        if (code !== "unreadable") {
+          await runtime.host.stop(container.container);
+          return { code, readable: true };
+        }
+        await runtime.host.stop(container.container).catch(() => void 0);
+      }
+      if (attempt < STOPPED_READ_ATTEMPTS) await (state.deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(STOPPED_READ_RETRY_MS);
+    }
+    logOf(state).error(`#${job.item} the stopped build's volume could not be read after ${STOPPED_READ_ATTEMPTS} tries`);
+    return { code: void 0, readable: false };
+  } catch (err) {
+    logOf(state).error(`#${job.item} the stopped build's volume could not be read: ${describeCallFailure(err)}`);
+    return { code: void 0, readable: false };
+  }
+}
+var STOPPED_READ_ATTEMPTS = 3;
+var STOPPED_READ_RETRY_MS = 1e3;
+async function readVolumeExitCode(runtime, dirPath, container) {
+  for (; ; ) {
+    const offset = commandLogOffset(dirPath, container.commandLogBase);
+    const probe = await probeCommand(runtime, container, offset);
+    if (probe === void 0) return "unreadable";
+    appendCommandLog(dirPath, probe.bytes);
+    if (probe.bytes.length > 0 && offset + probe.bytes.length < probe.size) continue;
+    return probe.exitCode;
+  }
+}
+async function finishTrackedWhileAway(state, dirPath, job, where) {
+  const log = logOf(state);
+  const container = job.container;
+  const read = where === "stopped" ? await readStoppedBuild(state, dirPath, job) : { code: void 0, readable: true };
+  const code = read.code;
+  container.stopped = true;
+  let outcome = "interrupted";
+  let ending = { reason: CONTAINER_STOPPED_REASON };
+  if (code !== void 0) {
+    const file = join(dirPath, EXIT_CODE_FILE);
+    writeFileSync(`${file}.partial`, `${code}
+`, { mode: RUNNER_FILE_MODE });
+    renameSync(`${file}.partial`, file);
+    job.exitCode = code;
+    job.commandEndReason = jobNotEndedByRunnerReason({ code, signal: null }, job);
+    outcome = code === 0 ? "ok" : "failed";
+    ending = void 0;
+  }
+  const finished = await finishJob(job, dirPath, outcome, state.config, finishDepsOf(state), ending);
+  log.line(
+    `recovered job ${finished.queue} #${finished.item} (claim ${shortClaim(finished.claimId)}) as ${outcome}: ${code === void 0 ? `its container ${where === "missing" ? "is gone" : "stopped"} and its volume ${read.readable ? "holds no exit code" : "could not be read"}` : `its command recorded exit ${code}`}`
+  );
+}
 async function start(configPath, deps = { fetchImpl: fetch }) {
   const log = deps.log ??= createRunnerLog(deps.now ?? (() => /* @__PURE__ */ new Date()));
   const update = deps.update;
@@ -2315,7 +2521,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const outcome = await onStartUnderMarker({
       configPath,
       own: ownVersionOf(deps),
-      check: update.check ? () => update.check(configPath) : async () => (await import('./check-BR5D27CX.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
+      check: update.check ? () => update.check(configPath) : async () => (await import('./check-3YHMJQ3P.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
       now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
       log
     });
@@ -2328,6 +2534,12 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   const hookProblem = checkHooks(config);
   if (hookProblem) throw new RunnerStartError(hookProblem);
   await ensureHostKey(configPath);
+  if (builtinKind(config.hooks.provision) === "container" || builtinKind(config.hooks.provision) === "vm") {
+    await ensureRunnerIdentity(configPath, {
+      docker: deps.runnerIdentity ?? (deps.container?.host === void 0 ? void 0 : { holdsObjects: async () => false }),
+      log: (line) => log.error(line)
+    }).catch((err) => log.error(`the runner's Docker identity could not be stored: ${describeCallFailure(err)} \xB7 using the names its folder's path gives`));
+  }
   const probe = await probeCredentials(config, deps.fetchImpl);
   if (!probe.ok && probe.fatal) throw new RunnerStartError(probe.reason);
   if (!probe.ok) log.error(`the gateway did not answer at start: ${probe.reason} \xB7 starting anyway`);
@@ -2358,23 +2570,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const adopted = { job, dirPath, lastRenewAtMs: nowMs(), cmd: null, cpuSample: null, ending: false };
     state.jobs.set(dirPath, adopted);
     attemptLogs.open(job.item, dirPath);
-    if (job.container !== void 0) {
-      const cores = job.container.cpuset === void 0 ? void 0 : parseCpuList(job.container.cpuset);
-      if (cores !== void 0) runtimeOf(state).cpus.hold(job.container.buildId, cores);
-      const tokens = buildTokensFor(state, adopted, job.item);
-      runtimeOf(state).credentials.set(dirPath, tokens);
-      await ensureTunnel(runtimeOf(state), job.container, dirPath, config, job.itemKey, job.branch, job.repositories);
-      try {
-        await tokens.token("read");
-        keepGhConfigCurrent(
-          tokens,
-          job.container.buildDir,
-          (err) => logOf(state).error(`#${job.item} the read token could not be written for gh: ${describeCallFailure(err)}`)
-        );
-      } catch (err) {
-        logOf(state).error(`#${job.item} could not give the adopted build a read token: ${describeCallFailure(err)}`);
-      }
-    }
+    if (job.container !== void 0) await rejoinContainerBuild(state, adopted);
     if (job.build !== void 0 && !runtimeOf(state).tunnels.has(job.build.buildId)) {
       await startHostServices({ runtime: runtimeOf(state), config, build: job.build, jobDir: dirPath, itemKey: job.itemKey, process: job.build.workDir !== void 0 }).catch((err) => {
         logOf(state).error(`the tunnel of #${job.item} could not start again: ${describeCallFailure(err)}`);
@@ -2384,6 +2580,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     log.line(adoptedLine);
     attemptLogs.note(job.item, adoptedLine);
   }
+  await adoptTrackedBuilds(state, jobsDir(configPath));
   for (const { job, released } of await retryFailedReleases(jobsDir(configPath), config, {
     fetchImpl: deps.fetchImpl,
     now: deps.now,
@@ -2425,6 +2622,9 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
       });
     }
   }, deps.watchIntervalMs ?? 3e4);
+  const probeTimer = setInterval(() => {
+    for (const active of state.jobs.values()) void probeBuild(state, active);
+  }, deps.probeIntervalMs ?? PROBE_INTERVAL_MS);
   const onSigterm = () => void drain();
   const onSigusr1 = () => void stopNow();
   const onSigint = () => {
@@ -2455,11 +2655,13 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     closed = true;
     clearInterval(pollTimer);
     clearInterval(watchTimer);
+    clearInterval(probeTimer);
   };
   const finish2 = () => {
     if (exited) return;
     exited = true;
     close();
+    for (const relay of state.containerRuntime?.relays.values() ?? []) void relay.close();
     releasePidFile(pidFilePath(configPath), self);
     log.line("runner exited");
   };
@@ -2489,6 +2691,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     stopIntake();
     clearInterval(pollTimer);
     clearInterval(watchTimer);
+    clearInterval(probeTimer);
     log.line(`stopping: ending ${state.jobs.size} running job(s) now`);
     await Promise.all([...state.jobs.values()].map((active) => endJob(state, active, "stopped")));
     finish2();
@@ -2574,4 +2777,4 @@ function runnerProcessState(configPath) {
   return isSameProcess(entry.pid, entry.start) ? { state: "running", pid: entry.pid } : { state: "stale", pid: entry.pid };
 }
 
-export { RunnerStartError, listJobFiles, probeCredentials, requestStop, runnerProcessState, start, tick, watchTick };
+export { RunnerStartError, listJobFiles, probeBuild, probeCredentials, requestStop, runnerProcessState, start, tick, watchTick };

package/dist/{chunk-P53VEAOF.js → chunk-2CMJLYV6.js}

@@ -3,7 +3,7 @@ import { createRequire } from 'node:module';
 import { MAX_ALLOW_HOSTS, allowHostProblem } from './chunk-UXYLWVTQ.js';
 import { parseSize } from './chunk-7RQ73DLO.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
-import { isRefusedInConfig, resolveDestinations, isReservedVariable } from './chunk-BQXFBI7T.js';
+import { isRefusedInConfig, resolveDestinations, isReservedVariable } from './chunk-V7MQKOZY.js';
 import { existsSync, statSync, accessSync, constants, readFileSync } from 'fs';
 import { hostname } from 'os';
 import { join } from 'path';

package/dist/{chunk-FBTGMSQF.js → chunk-2HT2XGJX.js}

@@ -1,16 +1,16 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { detectActiveSession, autoConnect } from './chunk-IF2OSI5T.js';
-import { workPageUrl } from './chunk-CG36IONK.js';
+import { detectActiveSession, autoConnect } from './chunk-QOALB6ZD.js';
+import { workPageUrl } from './chunk-JRCSET55.js';
 import { sanitizeTerminal } from './chunk-G3IJ3N6F.js';
-import { resolveRepoIdentity } from './chunk-QNLX477Q.js';
-import { C } from './chunk-ANPNT6PM.js';
-import { parseTrackerNumber, trackerNumberError } from './chunk-WZBHBH3J.js';
-import { defaultFetch } from './chunk-FVMYKL6L.js';
+import { defaultFetch } from './chunk-VWLOQ6FI.js';
 import { resolveGatewayCredentials, unauthorizedMessage } from './chunk-NTRCK4NH.js';
 import { readRepoConnection } from './chunk-BJ3KTMDH.js';
+import { resolveRepoIdentity } from './chunk-AEKSX4EY.js';
+import { C } from './chunk-ANPNT6PM.js';
+import { parseTrackerNumber, trackerNumberError } from './chunk-WZBHBH3J.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
-import { signRunnerRequest } from './chunk-M2ZWWTWZ.js';
+import { signRunnerRequest } from './chunk-KDHYLBRR.js';
 
 createRequire(import.meta.url);
 
@@ -378,6 +378,11 @@ async function runWorkBuild(opts) {
   const cwd = opts.cwd ?? process.cwd();
   const env = opts.env ?? process.env;
   const home = opts.home ?? homeFromEnv(env);
+  if (opts.local && opts.instructions !== void 0) {
+    throw new WorkCommandError(
+      "--instructions applies to host builds: with --local you build the item yourself, so give your instructions in your own session"
+    );
+  }
   await connectIfOneFits(opts);
   const { url, apiKey, appId } = resolveCredentials(opts);
   const fetchImpl = opts.fetchImpl ?? defaultFetch;
@@ -404,9 +409,11 @@ async function runWorkBuild(opts) {
     if (!opts.quiet) process.stdout.write(output2 + "\n");
     return { data: data2, output: output2 };
   }
-  const body = itemNumber !== void 0 ? { workItemId: String(itemNumber), note: opts.note, ...facts } : {
+  const instructions = opts.instructions !== void 0 ? { instructions: opts.instructions } : {};
+  const body = itemNumber !== void 0 ? { workItemId: String(itemNumber), note: opts.note, ...instructions, ...facts } : {
     subject: resolveWorkSubject({ issue: opts.issue, tracker: opts.tracker, repo: opts.repo, cwd }),
     note: opts.note,
+    ...instructions,
     ...facts
   };
   const { data } = await postJsonWithRetry(url, "/v1/work-items/build", apiKey, appId, body, fetchImpl, retries, sleep);
@@ -421,7 +428,7 @@ async function runWorkLinkSession(opts) {
     return await linkSession(opts);
   } catch (err) {
     if (opts.logFailures) {
-      const { logHookError } = await import('./hook-fast-RL5E5OXE.js');
+      const { logHookError } = await import('./hook-fast-WFB2ZMRB.js');
       const reason = err instanceof Error ? err.message : String(err);
       logHookError(opts.home ?? homeFromEnv(opts.env ?? process.env), new Error(`work link-session #${opts.number} failed: ${reason}`));
     }

package/dist/{chunk-S5PCA3SN.js → chunk-2J5JGLW5.js}

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { RUNNER_DIR_MODE, RUNNER_FILE_MODE } from './chunk-N2PXGQRL.js';
+import { RUNNER_DIR_MODE, RUNNER_FILE_MODE } from './chunk-KIGSGZBB.js';
 import { readFileSync, mkdirSync, writeFileSync, renameSync } from 'fs';
 import { join } from 'path';
 

package/dist/{chunk-R22NZSGA.js → chunk-34XORKFD.js}

@@ -1,25 +1,25 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { uploadSpooledSteps } from './chunk-KYS3TQAG.js';
+import { uploadSpooledSteps } from './chunk-G2WME4RG.js';
 import { LaunchFilter } from './chunk-ESVW4MJY.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
 import { migrateLegacyArtifactSidecars, planArtifactUpload, artifactsStatePath, sanitizeTerminal, acquireArtifactSpoolLock, artifactBlobsDir, resolveArtifactTurnIndex, isArtifactTombstoned, ARTIFACT_RETRY_MAX_AGE_MS, writeArtifactSpoolState, reclaimUnneededArtifactBlobs, neededArtifactShas } from './chunk-G3IJ3N6F.js';
-import { writeSyncStatus, spoolDir } from './chunk-PSN33ERW.js';
-import { hookExecSpoolPath } from './chunk-F7CU5ABH.js';
-import { applyReportedCost } from './chunk-BKO6JEZI.js';
-import { setCustomScrubbers, setLocalIdentity, tailTranscript, enrichSessionRepo, scanAll, setRepoRoot, resolveRepoIdentity, sessionIdFromPath, scrubDeep, SOURCE_ADAPTERS } from './chunk-QNLX477Q.js';
-import { C } from './chunk-ANPNT6PM.js';
-import { readLaunchSpoolState, launchStatusIsDecisive, launchedItemsBySession, launchesPath } from './chunk-T2Q7ISPE.js';
-import { defaultFetch } from './chunk-FVMYKL6L.js';
+import { defaultFetch } from './chunk-VWLOQ6FI.js';
 import { resolveGatewayCredentials, unauthorizedMessage } from './chunk-NTRCK4NH.js';
+import { writeSyncStatus, spoolDir } from './chunk-6IO62VJS.js';
 import { readLatestProvenanceForSession, readContextProvenance } from './chunk-ZJSNLVCE.js';
+import { hookExecSpoolPath } from './chunk-F7CU5ABH.js';
+import { applyReportedCost } from './chunk-BKO6JEZI.js';
 import { readRepoConnection, findGitDirs } from './chunk-BJ3KTMDH.js';
+import { setCustomScrubbers, setLocalIdentity, tailTranscript, enrichSessionRepo, scanAll, setRepoRoot, resolveRepoIdentity, sessionIdFromPath, scrubDeep, SOURCE_ADAPTERS } from './chunk-AEKSX4EY.js';
 import { readCloudConfig, cloudConfigPath, writeCloudConfig } from './chunk-EABW6AJQ.js';
+import { C } from './chunk-ANPNT6PM.js';
+import { readLaunchSpoolState, launchStatusIsDecisive, launchedItemsBySession, launchesPath } from './chunk-T2Q7ISPE.js';
 import { cliInvocation } from './chunk-CRB3POP4.js';
 import { removeJournalHooks } from './chunk-TUSYOL2O.js';
 import { CAPTURE_TIERS, inferArtifactKind, bannedPathsForTier, downconvertSession, isHumanUserTurn, turnCountBound, EVENT_TYPES } from './chunk-7VUK5RFS.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
-import { signerFor, signRunnerRequest } from './chunk-M2ZWWTWZ.js';
+import { signerFor, signRunnerRequest } from './chunk-KDHYLBRR.js';
 import { dirname, basename, join } from 'path';
 import { statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, renameSync, existsSync, rmSync } from 'fs';
 import { execFileSync } from 'child_process';