@outerlayer/cli 0.5.1 → 0.7.0

package/dist/{chunk-UWH5DSUP.js → chunk-AJKEHABI.js}

@@ -1,28 +1,32 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { DOCKERD_LOG } from './chunk-CBPPJSAR.js';
-import { installScrubContext } from './chunk-ZOC7JSYC.js';
-import { AssetError, describeCallFailure, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createBuildDisk, machineConfig, refusalCode, resolveAgentUser, AgentUserError, runBuildSteps, removeStagedContext, stageGovernedContext, shellQuote, ADDED_USER, createRunnerLog, createContainerRuntime, stopContainer, syncInContainer, formatDuration, cleanupContainer, retrySend, formatClock, GH_CONFIG_YML, ghHostsYml, isLiteralAddress, deleteBuildDisk } from './chunk-T52PKE2Y.js';
-import { CLAUDE_CODE_VERSION } from './chunk-Z62HNKEJ.js';
+import { writeJobFile, readJobFile } from './chunk-T6OIJQRT.js';
+import { senderPidfile } from './chunk-2VO25HHO.js';
+import { DOCKERD_LOG } from './chunk-WVGSB3Z2.js';
+import { installScrubContext } from './chunk-6ZZHWY2Z.js';
+import { AssetError, describeCallFailure, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createBuildDisk, machineConfig, refusalCode, resolveAgentUser, AgentUserError, runBuildSteps, removeStagedContext, stageGovernedContext, shellQuote, createRunnerLog, createContainerRuntime, stopContainer, cleanupContainer, outOfMemoryReason, syncInContainer, formatDuration, retrySend, formatClock, containerSenderCommand, GH_CONFIG_YML, ghHostsYml, ADDED_USER, isLiteralAddress, signalInContainer, startCheckContainer, lifecycleSteps, lifecycleFailureReason, deleteBuildDisk, senderSignalScript } from './chunk-E4PIFIC3.js';
+import { CLAUDE_CODE_VERSION, readBranchHead, fetchCommit } from './chunk-CEBIA5FD.js';
 import { isSameProcess, startTimeOf } from './chunk-N5FOE5PS.js';
 import { childEnvironment } from './chunk-774HEQL6.js';
-import { waitForAgent, vmExecArgs } from './chunk-W4SQCOZU.js';
-import { socketPathProblem, vsockLinks, linkVsockSockets } from './chunk-B2G7JHHB.js';
-import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-RMAZ2NWA.js';
-import { planArtifactUpload } from './chunk-WOBNRVQT.js';
-import { scrubText } from './chunk-EX6M5JPR.js';
-import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm } from './chunk-3DD6AOWC.js';
+import { waitForAgent, vmExecArgs } from './chunk-ZOAEPWGL.js';
+import { socketPathProblem, vsockLinks, linkVsockSockets } from './chunk-R7STSBLC.js';
+import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-DGAIVMMN.js';
+import { planArtifactUpload } from './chunk-ZYHKJKSU.js';
+import { scrubText } from './chunk-D6FXM5OW.js';
+import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm, DEFAULT_CHECK_TIME_LIMIT_MINUTES } from './chunk-DTXOJDEH.js';
 import { parseSize } from './chunk-F6GFZXZ3.js';
-import { claudeCredential, buildEnvironment, JOB_MOUNT, CHECKOUT_DIR, parseIsolation, proxyEnvironment, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, withHooksPath, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT } from './chunk-AUFG23AA.js';
+import { claudeCredential, buildEnvironment, JOB_MOUNT, CHECKOUT_DIR, parseIsolation, proxyEnvironment, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, withTunnelRecord, RELAY_PORT } from './chunk-OD2VTIHN.js';
+import { withHooksPath, authorEnvironment, commitIdentityRecord } from './chunk-HKMPVPO7.js';
+import { parsePolicy } from './chunk-7ZR5UPMB.js';
 import { readCloudConfig } from './chunk-EABW6AJQ.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
 import { signerFor } from './chunk-M5POMKKW.js';
-import { runnerDir, buildDir, vsockPath, RUNNER_DIR_MODE, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
+import { runnerDir, buildDir, vsockPath, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
 import { spawn, spawnSync } from 'child_process';
-import { randomBytes } from 'crypto';
-import { mkdirSync, chmodSync, writeFileSync, appendFileSync, statSync, accessSync, constants, existsSync, readFileSync, openSync, closeSync, rmSync, readdirSync } from 'fs';
-import { join, isAbsolute, dirname, delimiter } from 'path';
+import { randomBytes, createHash } from 'crypto';
+import { mkdirSync, chmodSync, writeFileSync, appendFileSync, statSync, accessSync, constants, openSync, existsSync, readFileSync, closeSync, rmSync, readdirSync, lstatSync, realpathSync } from 'fs';
+import { join, isAbsolute, dirname, delimiter, sep } from 'path';
 
 createRequire(import.meta.url);
 var DEFAULT_MEMORY = "8g";
@@ -141,12 +145,20 @@ async function provisionMicroVm(input) {
   };
   input.save(state, environment);
   const bail = async (reason, aborted = false) => {
+    let why = reason;
+    if (!aborted && state.cgroup !== void 0) {
+      try {
+        if (runtime.vm.cgroup.oomKills(state.cgroup) > 0) why = outOfMemoryReason(config.build?.memory);
+      } catch (err) {
+        log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
+      }
+    }
     try {
       await cleanupMicroVm(runtime, state);
     } catch (err) {
       log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
     }
-    return aborted ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
+    return aborted ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(why);
   };
   if (input.aborted()) return bail("", true);
   let imageConfig;
@@ -192,9 +204,15 @@ async function provisionMicroVm(input) {
     return bail(`the build's disk and machine could not be made: ${describeCallFailure(err)}`);
   }
   if (input.aborted()) return bail("", true);
+  try {
+    state.cgroup = vm.cgroup.create(buildId, limits.memoryBytes);
+  } catch (err) {
+    return bail(`the build's memory limit could not be set: ${describeCallFailure(err)}`);
+  }
+  if (state.cgroup !== void 0) input.save(state, environment);
   const consolePath = join(jobDir, "vm-console.log");
   try {
-    state.firecracker = await vm.firecracker.spawn({ binary: assets.firecracker, configPath, consolePath });
+    state.firecracker = await vm.firecracker.spawn({ binary: assets.firecracker, configPath, consolePath, cgroup: state.cgroup });
   } catch (err) {
     return bail(`Firecracker did not start: ${describeCallFailure(err)}`);
   }
@@ -315,7 +333,7 @@ function agentVersionOf(toolsJson) {
   }
   return CLAUDE_CODE_VERSION;
 }
-function microVmCommandLine(input) {
+function vmExecFor(input, argv) {
   const state = input.job.microvm;
   const env = buildEnvironment({
     config: input.config,
@@ -334,13 +352,17 @@ function microVmCommandLine(input) {
     cwd: CHECKOUT_DIR,
     env: { ...state.imageEnv, ...env.literals },
     pass: Object.keys(env.passed).filter((name) => env.passed[name] !== void 0),
-    argv: ["/bin/sh", "-c", input.cmdline]
-  };
-  return {
-    line: [process.execPath, input.cli, "vm-exec", ...vmExecArgs(options)].map(shellQuote).join(" "),
-    options,
-    env: { ...childEnvironment(input.runtime.hostEnv), ...env.passed }
+    argv
   };
+  return { options, env: { ...childEnvironment(input.runtime.hostEnv), ...env.passed } };
+}
+function microVmCommandLine(input) {
+  const { options, env } = vmExecFor(input, ["/bin/sh", "-c", input.cmdline]);
+  return { line: [process.execPath, input.cli, "vm-exec", ...vmExecArgs(options)].map(shellQuote).join(" "), options, env };
+}
+function microVmSenderCommand(input) {
+  const { options, env } = vmExecFor(input, ["outerlayer", "session-sender"]);
+  return { command: process.execPath, args: [input.cli, "vm-exec", ...vmExecArgs(options)], env };
 }
 function isRunning(runtime, state) {
   return state.firecracker !== void 0 && runtime.vm.firecracker.alive(state.firecracker);
@@ -391,6 +413,13 @@ async function cleanupMicroVm(runtime, state, jobDir) {
       failures.push(`the microVM (pid ${state.firecracker.pid}) was not stopped: ${describeCallFailure(err)}`);
     }
   }
+  if (state.cgroup !== void 0) {
+    try {
+      runtime.vm.cgroup.remove(state.cgroup);
+    } catch (err) {
+      failures.push(`the build's cgroup ${state.cgroup} was not removed: ${describeCallFailure(err)}`);
+    }
+  }
   if (state.buildDisk !== void 0) deleteBuildDisk(state.buildDisk);
   const tunnel = runtime.tunnels.get(state.buildId);
   if (tunnel !== void 0) {
@@ -583,70 +612,6 @@ async function closeHostBuild(runtime, build) {
   rmSync(build.buildDir, { recursive: true, force: true });
   if (build.workDir !== void 0) rmSync(build.workDir, { recursive: true, force: true });
 }
-var ENDED = "ended before provision finished";
-async function provisionProcess(input) {
-  const { config, runtime, item, jobDir } = input;
-  if (claudeCredential(runtime.hostEnv) === void 0) return { ok: false, reason: AGENT_CREDENTIAL_MISSING };
-  const repository = item.specRepository;
-  if (repository === void 0 || repository === null || repository === "") {
-    return { ok: false, reason: "the work item names no repository to clone" };
-  }
-  const buildId = randomBytes(6).toString("hex");
-  const buildDir2 = buildDir(config.configPath, buildId);
-  const workDir = join(jobDir, "work");
-  const state = { buildId, buildDir: buildDir2, workDir };
-  const environment = sharedUserEnvironment(input.cliVersion);
-  input.save(state, environment);
-  const bail = async (reason) => {
-    try {
-      await cleanupProcess(runtime, state);
-    } catch (err) {
-      input.log.error(`#${item.number} cleanup failed: ${describeCallFailure(err)}`);
-    }
-    return { ok: false, reason };
-  };
-  if (input.aborted()) return bail(ENDED);
-  mkdirSync(buildDir2, { recursive: true });
-  chmodSync(buildDir2, 493);
-  try {
-    await startHostServices({ runtime, config, build: state, jobDir, itemKey: input.itemKey, process: true });
-  } catch (err) {
-    return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
-  }
-  input.save(state, environment);
-  if (input.aborted()) return bail(ENDED);
-  const checkout = join(workDir, "checkout");
-  const remote = runtime.remoteFor?.(repository) ?? `https://github.com/${repository}.git`;
-  const cloned = await runtime.clone({ remote, dest: checkout });
-  if (!cloned.ok) return bail(cloned.reason);
-  if (input.aborted()) return bail(ENDED);
-  return { ok: true, workdir: checkout, state, environment };
-}
-function cleanupProcess(runtime, state) {
-  return closeHostBuild(runtime, state);
-}
-function readJobFile(path) {
-  if (!existsSync(path)) return null;
-  try {
-    return JSON.parse(readFileSync(path, "utf8"));
-  } catch {
-    return null;
-  }
-}
-function readJobFileChecked(path) {
-  if (!existsSync(path)) return { kind: "missing" };
-  try {
-    const parsed = JSON.parse(readFileSync(path, "utf8"));
-    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { kind: "unreadable" };
-    return { kind: "ok", job: parsed };
-  } catch {
-    return { kind: "unreadable" };
-  }
-}
-function writeJobFile(path, job) {
-  mkdirSync(dirname(path), { recursive: true, mode: RUNNER_DIR_MODE });
-  writeFileSync(path, JSON.stringify(job, null, 2), { mode: RUNNER_FILE_MODE });
-}
 
 // src/runner/verdict.ts
 var VERDICT_SETTLE_MS = 5 * 6e4;
@@ -720,6 +685,308 @@ async function settleVerdict(deps) {
     await deps.sleep(everyMs);
   }
 }
+var ENDED = "ended before provision finished";
+async function provisionProcess(input) {
+  const { config, runtime, item, jobDir } = input;
+  if (claudeCredential(runtime.hostEnv) === void 0) return { ok: false, reason: AGENT_CREDENTIAL_MISSING };
+  const repository = item.specRepository;
+  if (repository === void 0 || repository === null || repository === "") {
+    return { ok: false, reason: "the work item names no repository to clone" };
+  }
+  const buildId = randomBytes(6).toString("hex");
+  const buildDir2 = buildDir(config.configPath, buildId);
+  const workDir = join(jobDir, "work");
+  const state = { buildId, buildDir: buildDir2, workDir };
+  const environment = sharedUserEnvironment(input.cliVersion);
+  input.save(state, environment);
+  const bail = async (reason) => {
+    try {
+      await cleanupProcess(runtime, state);
+    } catch (err) {
+      input.log.error(`#${item.number} cleanup failed: ${describeCallFailure(err)}`);
+    }
+    return { ok: false, reason };
+  };
+  if (input.aborted()) return bail(ENDED);
+  mkdirSync(buildDir2, { recursive: true });
+  chmodSync(buildDir2, 493);
+  try {
+    await startHostServices({ runtime, config, build: state, jobDir, itemKey: input.itemKey, process: true });
+  } catch (err) {
+    return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
+  }
+  input.save(state, environment);
+  if (input.aborted()) return bail(ENDED);
+  const checkout = join(workDir, "checkout");
+  const remote = runtime.remoteFor?.(repository) ?? `https://github.com/${repository}.git`;
+  const cloned = await runtime.clone({ remote, dest: checkout });
+  if (!cloned.ok) return bail(cloned.reason);
+  if (input.aborted()) return bail(ENDED);
+  return { ok: true, workdir: checkout, state, environment };
+}
+function cleanupProcess(runtime, state) {
+  return closeHostBuild(runtime, state);
+}
+var MAX_VALIDATOR_FILES = 20;
+var VALIDATORS_DIR = ".outerlayer/validators";
+var POLICY_FILE = ".outerlayer/policy.yaml";
+var LIMIT_KEY_TAIL = " (runner.checks.timeLimitMinutes)";
+function lastLine(text2) {
+  return text2.split(/\r\n|\r|\n/).map((line) => line.trim()).filter((line) => line !== "").at(-1);
+}
+function readInside(root, rel) {
+  const path = join(root, rel);
+  try {
+    if (!lstatSync(path).isFile()) return void 0;
+    const inside = realpathSync(path);
+    const base = realpathSync(root);
+    return inside.startsWith(base + sep) ? readFileSync(inside, "utf8") : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function validatorFilesIn(root) {
+  const dir = join(root, VALIDATORS_DIR);
+  let names;
+  try {
+    if (!lstatSync(dir).isDirectory()) return [];
+    names = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const files = [];
+  for (const name of names.filter((entry) => /\.ya?ml$/.test(entry)).sort().slice(0, MAX_VALIDATOR_FILES)) {
+    const path = `${VALIDATORS_DIR}/${name}`;
+    const content = readInside(root, path);
+    if (content !== void 0) files.push({ path, content });
+  }
+  return files;
+}
+function checksIn(root, log, tag) {
+  const policy = readInside(root, POLICY_FILE);
+  const loaded = parsePolicy(policy === void 0 ? null : { path: POLICY_FILE, content: policy }, validatorFilesIn(root));
+  for (const problem of loaded.problems) log.error(`${tag} ${problem.file}: ${problem.problem}`);
+  const hostChecks = loaded.customs.filter((def) => def.runWhere === "host" && def.level !== "off" && def.declaresEmit !== null && def.command !== void 0);
+  return hostChecks.map((def) => ({ name: def.declaresEmit, command: def.command }));
+}
+async function readDefinitions(input, repository, scratch, token) {
+  const { config, job, deps } = input;
+  const tag = `#${job.item}`;
+  const stage = deps.stage ?? stageGovernedContext;
+  const controlPlaneUrl = (controlPlane) => deps.runtime.remoteFor?.(controlPlane) ?? `${deps.runtime.github.gitUrl}/${controlPlane}.git`;
+  const staged = await stage({
+    config,
+    repository,
+    buildId: `checks-${randomBytes(4).toString("hex")}`,
+    buildDir: scratch,
+    fetchImpl: deps.fetchImpl,
+    controlPlaneUrl
+  });
+  if (staged.kind === "refused") return { ok: false, reason: staged.reason };
+  if (staged.kind === "staged") {
+    try {
+      return { ok: true, checks: checksIn(join(staged.dir, "tree"), deps.log, tag) };
+    } finally {
+      rmSync(staged.dir, { recursive: true, force: true });
+    }
+  }
+  const repo = join(scratch, "repo");
+  const remote = deps.runtime.remoteFor?.(repository) ?? `${deps.runtime.github.gitUrl}/${repository}.git`;
+  const commit = job.environment?.recipe?.sourceCommit ?? await deps.runtime.readHead(remote, scratch);
+  if (commit === void 0) return { ok: false, reason: "the repository's default branch could not be read" };
+  const fetched = await (deps.fetchCommit ?? fetchCommit)({ remote, dir: repo, commit, token });
+  if (!fetched.ok) return { ok: false, reason: fetched.reason };
+  return { ok: true, checks: checksIn(repo, deps.log, tag) };
+}
+function timeoutAfter(ms, sleep) {
+  if (sleep !== void 0) return { done: sleep(ms).then(() => "timeout"), cancel: () => void 0 };
+  let timer;
+  const done = new Promise((resolve) => {
+    timer = setTimeout(() => resolve("timeout"), ms);
+    timer.unref();
+  });
+  return { done, cancel: () => clearTimeout(timer) };
+}
+function noCleanEnvironment(job) {
+  const ran = job.microvm !== void 0 ? "builtin:vm" : job.build?.workDir !== void 0 ? "builtin:process" : "a provision hook of the host's own";
+  return `the host has no clean environment for checks: this build ran under ${ran}, and checks run only in a new container from a container build's image`;
+}
+async function runHostChecks(input) {
+  const { config, job, jobDir, deps } = input;
+  const { runtime, log } = deps;
+  const tag = `#${job.item}`;
+  const now = deps.now ?? (() => /* @__PURE__ */ new Date());
+  const repository = job.repository ?? job.container?.repository;
+  if (repository === void 0) return;
+  const jobFilePath = join(jobDir, "job.json");
+  const tokens = runtime.credentials.get(jobDir);
+  let readToken;
+  try {
+    readToken = await tokens?.token("read");
+  } catch (err) {
+    log.error(`${tag} no read token for the checks: ${describeCallFailure(err)}`);
+  }
+  const scratch = join(jobDir, "checks", "definitions");
+  rmSync(scratch, { recursive: true, force: true });
+  mkdirSync(scratch, { recursive: true });
+  let definitions;
+  try {
+    definitions = await readDefinitions(input, repository, scratch, readToken);
+  } catch (err) {
+    definitions = { ok: false, reason: describeCallFailure(err) };
+  } finally {
+    removeStagedContext(scratch);
+  }
+  if (!definitions.ok) {
+    log.line(`${tag} checks not run: the definitions could not be read: ${definitions.reason}`);
+    rmSync(scratch, { recursive: true, force: true });
+    return;
+  }
+  const checks = definitions.checks;
+  if (checks.length === 0) {
+    rmSync(scratch, { recursive: true, force: true });
+    return;
+  }
+  log.line(`${tag} checks started (${checks.map((check) => check.name).join(", ")})`);
+  const startedMs = now().getTime();
+  let signer;
+  const record = async (check, result, sentence, keyTail = "") => {
+    signer ??= await signerFor(config.configPath);
+    const body = sentence === void 0 ? void 0 : cutReasonLine(`${scrubReason(sentence, config) ?? "check failed"}${keyTail}`);
+    const clientEmitId = `host-${createHash("sha256").update(`${job.claimId}:${job.startedAt}:${check.name}`).digest("hex").slice(0, 40)}`;
+    const sent = await retrySend(
+      () => postJson(
+        config.url,
+        "/v1/emitted-results",
+        config.apiKey,
+        config.appId,
+        { schemaVersion: 1, emit: { clientEmitId, name: check.name, result, ...body === void 0 ? {} : { body }, emittedAt: now().toISOString(), item: { number: Number(job.item) } } },
+        deps.fetchImpl,
+        void 0,
+        signer
+      ),
+      { now: deps.now }
+    );
+    if (sent.ok) log.line(`${tag} check ${check.name} ${result}${body === void 0 ? "" : `: ${body}`}`);
+    else log.error(`${tag} check ${check.name} ${result} could not be recorded: ${sent.reason}`);
+  };
+  const failAll = async (reason, keyTail = "") => {
+    log.error(`${tag} checks failed: ${reason}${keyTail}`);
+    for (const check of checks) await record(check, "fail", reason, keyTail);
+  };
+  let container;
+  try {
+    if (job.container === void 0) return await failAll(noCleanEnvironment(job));
+    if (tokens === void 0) return await failAll("the build's repository tokens are no longer held, so the branch cannot be fetched");
+    const branch = job.branch;
+    if (branch === void 0) return await failAll("the build pushed no branch: the claim names none");
+    const remote = runtime.remoteFor?.(repository) ?? `${runtime.github.gitUrl}/${repository}.git`;
+    const read = await (deps.readBranch ?? readBranchHead)({ remote, dir: scratch, branch, token: readToken });
+    if (read.kind === "absent") return await failAll(`the build pushed no branch: ${branch} does not exist on ${repository}`);
+    if (read.kind === "unreadable") return await failAll(`the branch ${branch} could not be read: ${read.reason}`);
+    const limitMinutes = config.checks?.timeLimitMinutes ?? DEFAULT_CHECK_TIME_LIMIT_MINUTES;
+    const minutes = `${limitMinutes} ${limitMinutes === 1 ? "minute" : "minutes"}`;
+    const environmentLog = join(jobDir, "checks", "environment.log");
+    const setup = async () => {
+      const started = await startCheckContainer({
+        config,
+        runtime,
+        job,
+        jobDir,
+        tokens,
+        log,
+        save: (state) => {
+          job.checkContainer = state;
+          writeJobFile(jobFilePath, job);
+        }
+      });
+      if (!started.ok) return `the check environment could not be made: ${started.reason}`;
+      container = started.container;
+      const made = started.container;
+      const steps = [
+        {
+          argv: ["git", "config", "--global", `url.http://127.0.0.1:${RELAY_PORT}/github.com/.insteadOf`, "https://github.com/"],
+          workdir: "/build/work",
+          failed: (status) => `configuring git failed (exit ${status})`
+        },
+        {
+          argv: ["git", "clone", "--filter=blob:none", "--single-branch", "--branch", branch, `https://github.com/${repository}.git`, CHECKOUT_DIR],
+          workdir: "/build/work",
+          failed: (status) => `the clone of ${branch} failed (exit ${status})`
+        }
+      ];
+      for (const step of steps) {
+        const ran = await made.exec(step.argv, step.workdir, { logPath: environmentLog });
+        if (ran.status !== 0) return `the check environment could not be prepared: ${step.failed(ran.status)}`;
+      }
+      for (const step of lifecycleSteps(job.container?.lifecycle ?? {})) {
+        log.line(`${tag} checks lifecycle ${step.name} (${step.source})`);
+        const ran = await made.exec(step.argv, CHECKOUT_DIR, { logPath: environmentLog });
+        if (ran.status !== 0) return `the check environment could not be prepared: ${lifecycleFailureReason(step, ran.status)}`;
+      }
+      return made;
+    };
+    const setupTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+    let settled;
+    try {
+      const outcome = await Promise.race([setup(), setupTimer.done]);
+      if (outcome === "timeout") return await failAll(`the check environment setup ran out of time after ${minutes}`, LIMIT_KEY_TAIL);
+      settled = outcome;
+    } finally {
+      setupTimer.cancel();
+    }
+    if (typeof settled === "string") return await failAll(settled);
+    const ready = settled;
+    let usable = true;
+    for (const check of checks) {
+      if (!usable) {
+        await record(check, "fail", "the check environment could not be restarted after an earlier check ran out of time");
+        continue;
+      }
+      const checkStartedMs = now().getTime();
+      const running = ready.exec(["/bin/sh", "-c", check.command], CHECKOUT_DIR, { logPath: join(jobDir, "checks", `${check.name}.log`) });
+      running.catch(() => void 0);
+      const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+      let outcome;
+      try {
+        outcome = await Promise.race([running, timer.done]);
+      } catch (err) {
+        timer.cancel();
+        await record(check, "fail", `the command could not be run: ${describeCallFailure(err)}`);
+        continue;
+      }
+      timer.cancel();
+      if (outcome === "timeout") {
+        await record(check, "fail", `ran out of time after ${minutes}`, LIMIT_KEY_TAIL);
+        const restarting = ready.restart().catch(() => false);
+        const restartTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+        usable = await Promise.race([restarting, restartTimer.done]) === true;
+        restartTimer.cancel();
+        continue;
+      }
+      if (outcome.status === 0) {
+        log.line(`${tag} check ${check.name} ran (${formatDuration(now().getTime() - checkStartedMs)})`);
+        await record(check, "pass");
+        continue;
+      }
+      const last = lastLine(outcome.stderr) ?? lastLine(outcome.stdout);
+      const scrubbed = last === void 0 ? void 0 : scrubReason(last, config);
+      await record(check, "fail", `exited ${outcome.status}${scrubbed === void 0 ? " with no output" : `: ${cutReasonLine(scrubbed, 140)}`}`);
+    }
+  } catch (err) {
+    log.error(`${tag} checks stopped: ${describeCallFailure(err)}`);
+  } finally {
+    if (container !== void 0 || job.checkContainer !== void 0) {
+      const state = container?.state ?? job.checkContainer;
+      await cleanupContainer(runtime, state).catch((err) => log.error(`${tag} removing the check environment failed: ${describeCallFailure(err)}`));
+      delete job.checkContainer;
+      writeJobFile(jobFilePath, job);
+    }
+    rmSync(scratch, { recursive: true, force: true });
+    if (existsSync(scratch)) log.error(`${tag} the checks' scratch directory could not be removed`);
+    log.line(`${tag} checks done (${formatDuration(now().getTime() - startedMs)})`);
+  }
+}
 
 // src/runner/job.ts
 var LEASE_SECONDS = 900;
@@ -1167,6 +1434,11 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
   const jobFilePath = join(jobDirPath, "job.json");
   const logPath = join(jobDirPath, "command.log");
   const tag = `#${job.item}`;
+  if (job.sender !== void 0) {
+    await stopSender({ job, deps: { spawnFn, runtime: deps.container, log } });
+    delete job.sender;
+    writeJobFile(jobFilePath, job);
+  }
   const build = job.container === void 0 ? void 0 : { state: job.container, runtime: deps.container ?? createContainerRuntime() };
   const vmBuild = job.microvm === void 0 ? void 0 : { state: job.microvm, runtime: deps.container ?? createContainerRuntime() };
   if (build !== void 0 && job.pid !== void 0 && isGroupAlive(job.pid)) {
@@ -1179,7 +1451,7 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
     await stopProcessGroup(job.pid);
   }
   if (outcome !== void 0 && job.endStage === void 0) {
-    const TEARDOWN_STAGES = ["sync", "cleanup", "report"];
+    const TEARDOWN_STAGES = ["checks", "sync", "cleanup", "report"];
     const staleTeardownStage = job.stage !== void 0 && TEARDOWN_STAGES.includes(job.stage);
     job.endStage = ending?.stage ?? (staleTeardownStage ? job.exitCode !== void 0 ? "command" : void 0 : job.stage);
     job.endExitCode = ending?.exitCode ?? (outcome === "ok" || outcome === "failed" ? job.exitCode : void 0);
@@ -1189,6 +1461,44 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
     job.reason = scrubReason(ending.reason, config);
     writeJobFile(jobFilePath, job);
   }
+  if (job.checkContainer !== void 0 && job.checksRan === true) {
+    try {
+      await cleanupContainer(deps.container ?? createContainerRuntime(), job.checkContainer);
+      delete job.checkContainer;
+      writeJobFile(jobFilePath, job);
+    } catch (err) {
+      log.error(`${tag} removing an earlier attempt's check environment failed: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+  }
+  if (outcome === "ok" && !job.cleanupRan && job.checksRan !== true) {
+    setStage(job, jobDirPath, "checks");
+    job.checksRan = true;
+    writeJobFile(jobFilePath, job);
+    try {
+      await runHostChecks({
+        config,
+        job,
+        jobDir: jobDirPath,
+        deps: { fetchImpl: deps.fetchImpl, runtime: deps.container ?? createContainerRuntime(), log, now: deps.now, ...deps.hostChecks }
+      });
+    } catch (err) {
+      log.error(`${tag} the host's checks stopped: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+  }
+  let outOfMemory;
+  if (vmBuild?.state.cgroup !== void 0 && outcome !== void 0 && outcome !== "ok" && ending?.reason === void 0) {
+    try {
+      if (vmBuild.runtime.vm.cgroup.oomKills(vmBuild.state.cgroup) > 0) {
+        outOfMemory = outOfMemoryReason(config.build?.memory);
+      }
+    } catch (err) {
+      log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+    if (outOfMemory !== void 0) {
+      job.reason = scrubReason(outOfMemory, config);
+      writeJobFile(jobFilePath, job);
+    }
+  }
   const hookLimits = () => hookLimitsFor(config, now().getTime(), deps.now);
   let tunnelRecord;
   if (!job.cleanupRan) {
@@ -1352,6 +1662,8 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
     const fromHook = readReasonLine(jobDirPath, config);
     if (verdictReason !== void 0) {
       job.reason = verdictReason;
+    } else if (outOfMemory !== void 0) {
+      job.reason = scrubReason(outOfMemory, config);
     } else if (ending?.reason === void 0) {
       job.reason = (outcome === "ok" ? void 0 : held?.reason) ?? (isolated ? fromHook ?? job.reason : fromHook);
     }
@@ -1470,4 +1782,109 @@ async function recoverLostJobs(jobsDirPath, config, deps) {
   return recovered;
 }
 
-export { ALREADY_RELEASED, LEASE_SECONDS, RENEW_INTERVAL_SECONDS, deliverCommitHooks, describeHookEnd, execArgv, execCommandLine, finishJob, hookLimitsFor, hostCommandEnvironment, hostEnvironment, isGroupAlive, isIdle, isTimedOut, isUnfinished, jobEnv, listAdoptableJobs, listenOnBroker, microVmCommandLine, needsRenew, outcomeFromExitCode, provisionEnvironment, provisionMicroVm, provisionProcess, readCpuUsec, readJobFile, readJobFileChecked, readProvisionOut, recoverLostJobs, releaseLease, renewLease, resolveHostBuild, retryFailedReleases, runCommand, runHook, setStage, startHostServices, stopOrphanedMicroVms, unfinishedJobs, workdirProblem, writeJobFile };
+// src/runner/sender.ts
+var STOP_WAIT_MS = 3e4;
+var POLL_MS = 200;
+var BUILD_HOME = "/build/home";
+async function startSender(input) {
+  const { config, job, jobDir, deps } = input;
+  const tag = `#${job.item}`;
+  if (job.itemKey === void 0) return void 0;
+  const spawnFn = deps.spawnFn ?? spawn;
+  try {
+    const launch = launchFor(config, job, jobDir, deps);
+    const fd = openSync(join(jobDir, "sender.log"), "a", RUNNER_FILE_MODE);
+    let pid;
+    try {
+      const child = spawnFn(launch.command, launch.args, { env: launch.env, cwd: launch.cwd, detached: true, stdio: ["ignore", fd, fd] });
+      child.on("error", (err) => deps.log.error(`${tag} session sender failed: ${describeCallFailure(err)}`));
+      child.on("exit", (code, signal) => deps.log.line(`${tag} session sender exited \xB7 ${signal ?? `code ${code}`} \xB7 see sender.log`));
+      child.unref?.();
+      pid = child.pid;
+    } finally {
+      closeSync(fd);
+    }
+    if (pid === void 0) {
+      deps.log.line(`${tag} session sender did not start: no pid`);
+      return void 0;
+    }
+    deps.log.line(`${tag} session sender started \xB7 pid ${pid}`);
+    return { pid, start: startTimeOf(pid) ?? void 0 };
+  } catch (err) {
+    deps.log.error(`${tag} session sender did not start: ${describeCallFailure(err)} \xB7 continuing`);
+    return void 0;
+  }
+}
+function launchFor(config, job, jobDir, deps) {
+  if (job.microvm !== void 0) {
+    const runtime = deps.runtime ?? createContainerRuntime();
+    const cli = deps.cliPath ?? join(runtime.cliBuild().distDir, "index.js");
+    return { ...microVmSenderCommand({ job, runtime, config, cli }), cwd: jobDir };
+  }
+  if (job.container !== void 0) {
+    return { ...containerSenderCommand({ job, runtime: deps.runtime ?? createContainerRuntime(), config }), cwd: jobDir };
+  }
+  const env = jobEnv(config, job.item, job.itemKey, jobDir, process.env, job.branch, job.commitIdentity?.author);
+  const build = job.build;
+  const hostEnv = build === void 0 ? env : hostCommandEnvironment(env, build, config, job.commitIdentity?.hooksPath);
+  if (build?.exec !== void 0) {
+    return { command: build.exec, args: execArgv(job.workdir, ["outerlayer", "session-sender"]), env: hostEnv, cwd: jobDir };
+  }
+  return { command: process.execPath, args: [deps.cliPath ?? process.argv[1] ?? "outerlayer", "session-sender"], env: hostEnv, cwd: job.workdir ?? jobDir };
+}
+function isLive(sender) {
+  if (sender.start !== void 0) return isSameProcess(sender.pid, sender.start);
+  try {
+    process.kill(sender.pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function signalHost(sender, signal) {
+  if (!isLive(sender)) return;
+  try {
+    process.kill(-sender.pid, signal);
+  } catch {
+    try {
+      process.kill(sender.pid, signal);
+    } catch {
+    }
+  }
+}
+async function signalInGuest(runtime, state, pidfile, signal) {
+  await runtime.vm.exec(state.vsockPath, {
+    argv: ["/bin/sh", "-c", senderSignalScript(pidfile, signal)],
+    user: state.agentUser ?? ADDED_USER,
+    cwd: "/",
+    env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" }
+  });
+}
+async function stopSender(input) {
+  const { job, deps } = input;
+  const sender = job.sender;
+  if (sender === void 0) return;
+  const tag = `#${job.item}`;
+  const sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  const inContainer = job.container === void 0 ? void 0 : { state: job.container, runtime: deps.runtime ?? createContainerRuntime(), pidfile: senderPidfile(BUILD_HOME, Number(job.item)) };
+  const inGuest = job.microvm === void 0 ? void 0 : { state: job.microvm, runtime: deps.runtime ?? createContainerRuntime(), pidfile: senderPidfile(BUILD_HOME, Number(job.item)) };
+  const signalInside = async (signal) => {
+    if (inContainer !== void 0) await signalInContainer(inContainer.runtime, inContainer.state, inContainer.pidfile, signal).catch(() => void 0);
+    if (inGuest !== void 0) await signalInGuest(inGuest.runtime, inGuest.state, inGuest.pidfile, signal).catch(() => void 0);
+  };
+  try {
+    if (inContainer !== void 0 || inGuest !== void 0) await signalInside("TERM");
+    else signalHost(sender, "SIGTERM");
+    const deadline = (deps.waitMs ?? STOP_WAIT_MS) / POLL_MS;
+    for (let waited = 0; isLive(sender) && waited < deadline; waited += 1) await sleep(POLL_MS);
+    if (isLive(sender)) {
+      deps.log.error(`${tag} session sender did not stop in time \xB7 killing it`);
+      await signalInside("KILL");
+      signalHost(sender, "SIGKILL");
+    }
+  } catch (err) {
+    deps.log.error(`${tag} stopping the session sender failed: ${describeCallFailure(err)} \xB7 continuing`);
+  }
+}
+
+export { ALREADY_RELEASED, LEASE_SECONDS, RENEW_INTERVAL_SECONDS, deliverCommitHooks, describeHookEnd, execArgv, execCommandLine, finishJob, hookLimitsFor, hostCommandEnvironment, hostEnvironment, isGroupAlive, isIdle, isTimedOut, isUnfinished, jobEnv, listAdoptableJobs, listenOnBroker, microVmCommandLine, needsRenew, outcomeFromExitCode, provisionEnvironment, provisionMicroVm, provisionProcess, readCpuUsec, readProvisionOut, recoverLostJobs, releaseLease, renewLease, resolveHostBuild, retryFailedReleases, runCommand, runHook, setStage, startHostServices, startSender, stopOrphanedMicroVms, unfinishedJobs, workdirProblem };