@outerlayer/cli 0.6.0 → 0.8.0

package/dist/{chunk-VMJ3MMBX.js → chunk-7VNURP6M.js}

@@ -1,30 +1,33 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { writeJobFile, readJobFile } from './chunk-T6OIJQRT.js';
-import { senderPidfile } from './chunk-ZILGTTY6.js';
-import { DOCKERD_LOG } from './chunk-CBPPJSAR.js';
-import { installScrubContext } from './chunk-UMFWVGJO.js';
-import { AssetError, describeCallFailure, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createBuildDisk, machineConfig, refusalCode, resolveAgentUser, AgentUserError, runBuildSteps, removeStagedContext, stageGovernedContext, shellQuote, createRunnerLog, createContainerRuntime, stopContainer, syncInContainer, formatDuration, cleanupContainer, retrySend, formatClock, containerSenderCommand, GH_CONFIG_YML, ghHostsYml, ADDED_USER, isLiteralAddress, signalInContainer, deleteBuildDisk, senderSignalScript } from './chunk-RZSF5B46.js';
-import { CLAUDE_CODE_VERSION } from './chunk-Z62HNKEJ.js';
+import { senderPidfile } from './chunk-3S4P55NE.js';
+import { DOCKERD_LOG } from './chunk-WVGSB3Z2.js';
+import { installScrubContext } from './chunk-AIL2R4SI.js';
+import { AssetError, describeCallFailure, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createBuildDisk, machineConfig, refusalCode, resolveAgentUser, AgentUserError, runBuildSteps, removeStagedContext, stageGovernedContext, shellQuote, createRunnerLog, createContainerRuntime, stopContainer, cleanupContainer, outOfMemoryReason, formatDuration, retrySend, formatClock, containerSenderCommand, syncInContainer, GH_CONFIG_YML, ghHostsYml, ADDED_USER, isLiteralAddress, signalInContainer, deleteBuildDisk, startCheckContainer, lifecycleSteps, lifecycleFailureReason, senderSignalScript } from './chunk-CM5QTKM4.js';
+import { readBranchHead, fetchCommit } from './chunk-TTNFGMHO.js';
 import { isSameProcess, startTimeOf } from './chunk-N5FOE5PS.js';
-import { childEnvironment } from './chunk-774HEQL6.js';
-import { waitForAgent, vmExecArgs } from './chunk-W4SQCOZU.js';
-import { socketPathProblem, vsockLinks, linkVsockSockets } from './chunk-B2G7JHHB.js';
-import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-BNMCXPG7.js';
-import { planArtifactUpload } from './chunk-3DEVYTNQ.js';
-import { scrubText } from './chunk-ANK2B36Y.js';
-import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm } from './chunk-3DD6AOWC.js';
-import { parseSize } from './chunk-F6GFZXZ3.js';
-import { claudeCredential, buildEnvironment, JOB_MOUNT, CHECKOUT_DIR, parseIsolation, proxyEnvironment, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, withHooksPath, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT } from './chunk-AUFG23AA.js';
+import { childEnvironment } from './chunk-U3GOI2B4.js';
+import { CLAUDE_CODE_VERSION } from './chunk-CX22S5BA.js';
+import { waitForAgent, vmExecArgs } from './chunk-ZOAEPWGL.js';
+import { socketPathProblem, vsockLinks, linkVsockSockets } from './chunk-R7STSBLC.js';
+import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-5QLNZGAQ.js';
+import { planArtifactUpload } from './chunk-ZYHKJKSU.js';
+import { scrubText } from './chunk-GAYWY4GI.js';
+import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm, DEFAULT_CHECK_TIME_LIMIT_MINUTES } from './chunk-PWBILI5P.js';
+import { parseSize } from './chunk-7RQ73DLO.js';
+import { claudeCredential, buildEnvironment, JOB_MOUNT, CHECKOUT_DIR, parseIsolation, proxyEnvironment, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, withTunnelRecord, RELAY_PORT, checkEnvironment } from './chunk-OD2VTIHN.js';
+import { withHooksPath, authorEnvironment, commitIdentityRecord } from './chunk-HKMPVPO7.js';
+import { parsePolicy } from './chunk-7ZR5UPMB.js';
 import { readCloudConfig } from './chunk-EABW6AJQ.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
 import { signerFor } from './chunk-M5POMKKW.js';
 import { runnerDir, buildDir, vsockPath, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
 import { spawn, spawnSync } from 'child_process';
-import { randomBytes } from 'crypto';
-import { mkdirSync, chmodSync, writeFileSync, appendFileSync, statSync, accessSync, constants, openSync, existsSync, readFileSync, closeSync, rmSync, readdirSync } from 'fs';
-import { join, isAbsolute, dirname, delimiter } from 'path';
+import { randomBytes, createHash } from 'crypto';
+import { mkdirSync, chmodSync, writeFileSync, appendFileSync, statSync, accessSync, constants, openSync, existsSync, readFileSync, closeSync, rmSync, readdirSync, lstatSync, realpathSync } from 'fs';
+import { join, isAbsolute, dirname, delimiter, sep } from 'path';
 
 createRequire(import.meta.url);
 var DEFAULT_MEMORY = "8g";
@@ -119,7 +122,9 @@ async function provisionMicroVm(input) {
     repository,
     imageId: report.imageId,
     containerEnv: report.apply.containerEnv,
-    builtIns: true
+    builtIns: true,
+    lifecycle: report.apply.lifecycle,
+    ...report.docker === true ? { docker: true } : {}
   };
   const env = buildEnvironment({
     config,
@@ -143,12 +148,20 @@ async function provisionMicroVm(input) {
   };
   input.save(state, environment);
   const bail = async (reason, aborted = false) => {
+    let why = reason;
+    if (!aborted && state.cgroup !== void 0) {
+      try {
+        if (runtime.vm.cgroup.oomKills(state.cgroup) > 0) why = outOfMemoryReason(config.build?.memory);
+      } catch (err) {
+        log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
+      }
+    }
     try {
       await cleanupMicroVm(runtime, state);
     } catch (err) {
       log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
     }
-    return aborted ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
+    return aborted ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(why);
   };
   if (input.aborted()) return bail("", true);
   let imageConfig;
@@ -393,6 +406,147 @@ async function copyHookErrors(runtime, state, jobDir) {
   } catch {
   }
 }
+async function startCheckMicroVm(input) {
+  const { config, runtime, job, jobDir, tokens, log } = input;
+  const { vm } = runtime;
+  const tag = `#${job.item}`;
+  const built = job.microvm;
+  const repository = built?.repository ?? job.repository;
+  if (built === void 0 || built.imageId === void 0 || repository === void 0 || job.branch === void 0) {
+    return { ok: false, reason: "the build's image, repository or branch is not recorded" };
+  }
+  const branch = job.branch;
+  const runnerDirPath = runnerDir(config.configPath);
+  const tooDeep = socketPathProblem(buildDir(config.configPath, "0".repeat(12)));
+  if (tooDeep !== void 0) return { ok: false, reason: tooDeep };
+  const buildId = randomBytes(6).toString("hex");
+  const buildDir2 = buildDir(config.configPath, buildId);
+  const agentUser = built.agentUser ?? ADDED_USER;
+  const state = {
+    buildId,
+    buildDir: buildDir2,
+    repository,
+    imageId: built.imageId,
+    agentUser,
+    containerEnv: built.containerEnv,
+    imageEnv: built.imageEnv,
+    lifecycle: built.lifecycle,
+    ...built.docker === true ? { docker: true } : {}
+  };
+  const checksDir = join(jobDir, "checks");
+  mkdirSync(checksDir, { recursive: true });
+  const consolePath = join(checksDir, "vm-console.log");
+  input.save(state);
+  const fail = async (reason) => {
+    try {
+      await cleanupMicroVm(runtime, state);
+    } catch (err) {
+      log.error(`${tag} cleanup of the check environment failed: ${describeCallFailure(err)}`);
+    }
+    return { ok: false, reason };
+  };
+  const abandoned = () => input.aborted?.() === true;
+  const ABANDONED = "the check environment was abandoned before it was ready";
+  let assets;
+  try {
+    assets = await vm.assets(runnerDirPath);
+  } catch (err) {
+    return fail(err instanceof AssetError ? err.message : `the microVM's assets could not be had: ${describeCallFailure(err)}`);
+  }
+  if (abandoned()) return fail(ABANDONED);
+  try {
+    state.rootfs = await ensureRootfs({ host: vm.rootfs, runnerDir: runnerDirPath, imageId: built.imageId });
+  } catch (err) {
+    return fail(`the image could not be converted to a root disk: ${describeCallFailure(err)}`);
+  }
+  if (abandoned()) return fail(ABANDONED);
+  state.buildDisk = join(buildDir2, "build.ext4");
+  state.vsockPath = vsockPath(buildDir2);
+  input.save(state);
+  mkdirSync(buildDir2, { recursive: true, mode: 448 });
+  try {
+    const gitRemote = gitRemoteFor(runtime, { repository, branch, tokens, buildDir: buildDir2, log: (line) => log.line(`${tag} ${line}`), readOnly: true });
+    runtime.tunnels.set(buildId, await startServicesFor(runtime, config, buildDir2, checksDir, void 0, false, gitRemote));
+  } catch (err) {
+    return fail(`the tunnel could not start: ${describeCallFailure(err)}`);
+  }
+  if (abandoned()) return fail(ABANDONED);
+  const limits = {
+    cpus: config.build?.cpus ?? DEFAULT_CPUS,
+    memoryBytes: parseSize(config.build?.memory ?? DEFAULT_MEMORY) ?? parseSize(DEFAULT_MEMORY),
+    diskBytes: parseSize(config.build?.disk ?? DEFAULT_DISK) ?? parseSize(DEFAULT_DISK)
+  };
+  const links = vsockLinks({ destinations: config.build?.destinations?.length ?? 0, builtIns: false });
+  const configPath = join(buildDir2, "vm.json");
+  try {
+    linkVsockSockets({ buildDir: buildDir2, vsockPath: state.vsockPath, links });
+    await createBuildDisk({ host: vm.rootfs, runnerDir: runnerDirPath, bytes: limits.diskBytes, dest: state.buildDisk });
+    writeFileSync(
+      configPath,
+      JSON.stringify(
+        machineConfig(limits, { kernel: assets.kernel, rootfs: state.rootfs, buildDisk: state.buildDisk, vsock: state.vsockPath, relayPorts: links.map((link) => link.port), docker: built.docker }),
+        null,
+        2
+      ),
+      { mode: 384 }
+    );
+  } catch (err) {
+    return fail(`the check environment's disk and machine could not be made: ${describeCallFailure(err)}`);
+  }
+  if (abandoned()) return fail(ABANDONED);
+  try {
+    state.cgroup = vm.cgroup.create(buildId, limits.memoryBytes);
+  } catch (err) {
+    return fail(`the check environment's memory limit could not be set: ${describeCallFailure(err)}`);
+  }
+  if (state.cgroup !== void 0) input.save(state);
+  const boot = async () => {
+    try {
+      state.firecracker = await vm.firecracker.spawn({ binary: assets.firecracker, configPath, consolePath, cgroup: state.cgroup });
+    } catch (err) {
+      return `Firecracker did not start: ${describeCallFailure(err)}`;
+    }
+    input.save(state);
+    if (abandoned()) return ABANDONED;
+    const ready = await waitForAgent(state.vsockPath, { attempts: BOOT_ATTEMPTS, delayMs: BOOT_POLL_MS, sleep: runtime.sleep });
+    if (!ready) {
+      const why = vm.firecracker.alive(state.firecracker) ? "its exec agent did not answer" : "it exited before its exec agent answered";
+      return `the microVM did not start: ${why}; its console is in ${consolePath}`;
+    }
+    if (built.docker === true) return waitForDocker(runtime, state, agentUser, () => /* @__PURE__ */ new Date());
+    return void 0;
+  };
+  const failed = await boot();
+  if (failed !== void 0) return fail(failed);
+  if (abandoned()) return fail(ABANDONED);
+  const env = checkEnvironment({ config, item: job.item, repository, hostEnv: runtime.hostEnv, containerEnv: built.containerEnv ?? {} });
+  return {
+    ok: true,
+    environment: {
+      state,
+      exec: async (argv, workdir, opts) => {
+        const ran = await vm.exec(state.vsockPath, { argv, user: agentUser, cwd: workdir, env: execEnvironment(state.imageEnv, env) });
+        if (opts?.logPath !== void 0) appendFileSync(opts.logPath, ran.stdout + ran.stderr);
+        return ran;
+      },
+      restart: async () => {
+        try {
+          await vm.firecracker.stop(state.firecracker);
+          rmSync(state.vsockPath, { force: true });
+          return await boot() === void 0;
+        } catch {
+          return false;
+        }
+      }
+    }
+  };
+}
+async function stopBuildMicroVm(runtime, state, jobDir) {
+  await copyHookErrors(runtime, state, jobDir);
+  if (state.firecracker === void 0) return;
+  await runtime.vm.firecracker.stop(state.firecracker);
+  if (runtime.vm.firecracker.alive(state.firecracker)) throw new Error(`the microVM (pid ${state.firecracker.pid}) is still running after it was stopped`);
+}
 async function cleanupMicroVm(runtime, state, jobDir) {
   const failures = [];
   if (jobDir !== void 0) await copyHookErrors(runtime, state, jobDir);
@@ -422,8 +576,8 @@ async function cleanupMicroVm(runtime, state, jobDir) {
 async function stopOrphanedMicroVms(input) {
   const stopped = [];
   for (const { job } of input.jobs) {
-    const state = job.microvm;
-    if (state === void 0) continue;
+    const states = [job.microvm, job.checkMicrovm].filter((state) => state !== void 0);
+    if (states.length === 0) continue;
     if (job.pid !== void 0 && job.start !== void 0 && isSameProcess(job.pid, job.start)) {
       try {
         process.kill(-job.pid, "SIGKILL");
@@ -432,9 +586,11 @@ async function stopOrphanedMicroVms(input) {
       const deadline = Date.now() + 2e3;
       while (isSameProcess(job.pid, job.start) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10));
     }
-    if (state.firecracker !== void 0 && input.runtime.vm.firecracker.alive(state.firecracker)) {
-      await input.runtime.vm.firecracker.stop(state.firecracker);
-      stopped.push(state.buildId);
+    for (const state of states) {
+      if (state.firecracker !== void 0 && input.runtime.vm.firecracker.alive(state.firecracker)) {
+        await input.runtime.vm.firecracker.stop(state.firecracker);
+        stopped.push(state.buildId);
+      }
     }
   }
   return stopped;
@@ -717,6 +873,297 @@ async function provisionProcess(input) {
 function cleanupProcess(runtime, state) {
   return closeHostBuild(runtime, state);
 }
+var MAX_VALIDATOR_FILES = 20;
+var VALIDATORS_DIR = ".outerlayer/validators";
+var POLICY_FILE = ".outerlayer/policy.yaml";
+var LIMIT_KEY_TAIL = " (runner.checks.timeLimitMinutes)";
+function lastLine(text2) {
+  return text2.split(/\r\n|\r|\n/).map((line) => line.trim()).filter((line) => line !== "").at(-1);
+}
+function readInside(root, rel) {
+  const path = join(root, rel);
+  try {
+    if (!lstatSync(path).isFile()) return void 0;
+    const inside = realpathSync(path);
+    const base = realpathSync(root);
+    return inside.startsWith(base + sep) ? readFileSync(inside, "utf8") : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function validatorFilesIn(root) {
+  const dir = join(root, VALIDATORS_DIR);
+  let names;
+  try {
+    if (!lstatSync(dir).isDirectory()) return [];
+    names = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const files = [];
+  for (const name of names.filter((entry) => /\.ya?ml$/.test(entry)).sort().slice(0, MAX_VALIDATOR_FILES)) {
+    const path = `${VALIDATORS_DIR}/${name}`;
+    const content = readInside(root, path);
+    if (content !== void 0) files.push({ path, content });
+  }
+  return files;
+}
+function checksIn(root, log, tag) {
+  const policy = readInside(root, POLICY_FILE);
+  const loaded = parsePolicy(policy === void 0 ? null : { path: POLICY_FILE, content: policy }, validatorFilesIn(root));
+  for (const problem of loaded.problems) log.error(`${tag} ${problem.file}: ${problem.problem}`);
+  const hostChecks = loaded.customs.filter((def) => def.runWhere === "host" && def.level !== "off" && def.declaresEmit !== null && def.command !== void 0);
+  return hostChecks.map((def) => ({ name: def.declaresEmit, command: def.command }));
+}
+async function readDefinitions(input, repository, scratch, token) {
+  const { config, job, deps } = input;
+  const tag = `#${job.item}`;
+  const stage = deps.stage ?? stageGovernedContext;
+  const controlPlaneUrl = (controlPlane) => deps.runtime.remoteFor?.(controlPlane) ?? `${deps.runtime.github.gitUrl}/${controlPlane}.git`;
+  const staged = await stage({
+    config,
+    repository,
+    buildId: `checks-${randomBytes(4).toString("hex")}`,
+    buildDir: scratch,
+    fetchImpl: deps.fetchImpl,
+    controlPlaneUrl
+  });
+  if (staged.kind === "refused") return { ok: false, reason: staged.reason };
+  if (staged.kind === "staged") {
+    try {
+      return { ok: true, checks: checksIn(join(staged.dir, "tree"), deps.log, tag) };
+    } finally {
+      rmSync(staged.dir, { recursive: true, force: true });
+    }
+  }
+  const repo = join(scratch, "repo");
+  const remote = deps.runtime.remoteFor?.(repository) ?? `${deps.runtime.github.gitUrl}/${repository}.git`;
+  const commit = job.environment?.recipe?.sourceCommit ?? await deps.runtime.readHead(remote, scratch);
+  if (commit === void 0) return { ok: false, reason: "the repository's default branch could not be read" };
+  const fetched = await (deps.fetchCommit ?? fetchCommit)({ remote, dir: repo, commit, token });
+  if (!fetched.ok) return { ok: false, reason: fetched.reason };
+  return { ok: true, checks: checksIn(repo, deps.log, tag) };
+}
+function timeoutAfter(ms, sleep) {
+  if (sleep !== void 0) return { done: sleep(ms).then(() => "timeout"), cancel: () => void 0 };
+  let timer;
+  const done = new Promise((resolve) => {
+    timer = setTimeout(() => resolve("timeout"), ms);
+    timer.unref();
+  });
+  return { done, cancel: () => clearTimeout(timer) };
+}
+function noCleanEnvironment(job) {
+  const ran = job.build?.workDir !== void 0 ? "builtin:process" : "a provision hook of the host's own";
+  return `the host has no clean environment for checks: this build ran under ${ran}, and checks run only in a new container or microVM from a build's image`;
+}
+async function runHostChecks(input) {
+  const { config, job, jobDir, deps } = input;
+  const { runtime, log } = deps;
+  const tag = `#${job.item}`;
+  const now = deps.now ?? (() => /* @__PURE__ */ new Date());
+  const repository = job.repository ?? job.container?.repository;
+  if (repository === void 0) return;
+  const jobFilePath = join(jobDir, "job.json");
+  const tokens = runtime.credentials.get(jobDir);
+  let readToken;
+  try {
+    readToken = await tokens?.token("read");
+  } catch (err) {
+    log.error(`${tag} no read token for the checks: ${describeCallFailure(err)}`);
+  }
+  const scratch = join(jobDir, "checks", "definitions");
+  rmSync(scratch, { recursive: true, force: true });
+  mkdirSync(scratch, { recursive: true });
+  let definitions;
+  try {
+    definitions = await readDefinitions(input, repository, scratch, readToken);
+  } catch (err) {
+    definitions = { ok: false, reason: describeCallFailure(err) };
+  } finally {
+    removeStagedContext(scratch);
+  }
+  if (!definitions.ok) {
+    log.line(`${tag} checks not run: the definitions could not be read: ${definitions.reason}`);
+    rmSync(scratch, { recursive: true, force: true });
+    return;
+  }
+  const checks = definitions.checks;
+  if (checks.length === 0) {
+    rmSync(scratch, { recursive: true, force: true });
+    return;
+  }
+  log.line(`${tag} checks started (${checks.map((check) => check.name).join(", ")})`);
+  const startedMs = now().getTime();
+  let signer;
+  const record = async (check, result, sentence, keyTail = "") => {
+    signer ??= await signerFor(config.configPath);
+    const body = sentence === void 0 ? void 0 : cutReasonLine(`${scrubReason(sentence, config) ?? "check failed"}${keyTail}`);
+    const clientEmitId = `host-${createHash("sha256").update(`${job.claimId}:${job.startedAt}:${check.name}`).digest("hex").slice(0, 40)}`;
+    const sent = await retrySend(
+      () => postJson(
+        config.url,
+        "/v1/emitted-results",
+        config.apiKey,
+        config.appId,
+        { schemaVersion: 1, emit: { clientEmitId, name: check.name, result, ...body === void 0 ? {} : { body }, emittedAt: now().toISOString(), item: { number: Number(job.item) } } },
+        deps.fetchImpl,
+        void 0,
+        signer
+      ),
+      { now: deps.now }
+    );
+    if (sent.ok) log.line(`${tag} check ${check.name} ${result}${body === void 0 ? "" : `: ${body}`}`);
+    else log.error(`${tag} check ${check.name} ${result} could not be recorded: ${sent.reason}`);
+  };
+  const failAll = async (reason, keyTail = "") => {
+    log.error(`${tag} checks failed: ${reason}${keyTail}`);
+    for (const check of checks) await record(check, "fail", reason, keyTail);
+  };
+  try {
+    if (job.container === void 0 && job.microvm === void 0) return await failAll(noCleanEnvironment(job));
+    if (tokens === void 0) return await failAll("the build's repository tokens are no longer held, so the branch cannot be fetched");
+    const branch = job.branch;
+    if (branch === void 0) return await failAll("the build pushed no branch: the claim names none");
+    const remote = runtime.remoteFor?.(repository) ?? `${runtime.github.gitUrl}/${repository}.git`;
+    const read = await (deps.readBranch ?? readBranchHead)({ remote, dir: scratch, branch, token: readToken });
+    if (read.kind === "absent") return await failAll(`the build pushed no branch: ${branch} does not exist on ${repository}`);
+    if (read.kind === "unreadable") return await failAll(`the branch ${branch} could not be read: ${read.reason}`);
+    const limitMinutes = config.checks?.timeLimitMinutes ?? DEFAULT_CHECK_TIME_LIMIT_MINUTES;
+    const minutes = `${limitMinutes} ${limitMinutes === 1 ? "minute" : "minutes"}`;
+    const environmentLog = join(jobDir, "checks", "environment.log");
+    if (job.microvm !== void 0) {
+      if (deps.endBuildMicroVm === void 0) return await failAll("the runner has no way to stop the build's microVM, so no check environment was made");
+      try {
+        await deps.endBuildMicroVm();
+      } catch (err) {
+        return await failAll(`the build's microVM could not be stopped, so no check environment was made: ${describeCallFailure(err)}`);
+      }
+    }
+    let abandoned = false;
+    const setup = async () => {
+      const started = job.microvm !== void 0 ? await startCheckMicroVm({
+        config,
+        runtime,
+        job,
+        jobDir,
+        tokens,
+        log,
+        aborted: () => abandoned,
+        save: (state) => {
+          if (abandoned) return;
+          job.checkMicrovm = state;
+          writeJobFile(jobFilePath, job);
+        }
+      }) : await startCheckContainer({
+        config,
+        runtime,
+        job,
+        jobDir,
+        tokens,
+        log,
+        save: (state) => {
+          job.checkContainer = state;
+          writeJobFile(jobFilePath, job);
+        }
+      });
+      if (!started.ok) return `the check environment could not be made: ${started.reason}`;
+      const made = "container" in started ? started.container : started.environment;
+      const steps = [
+        {
+          argv: ["git", "config", "--global", `url.http://127.0.0.1:${RELAY_PORT}/github.com/.insteadOf`, "https://github.com/"],
+          workdir: "/build/work",
+          failed: (status) => `configuring git failed (exit ${status})`
+        },
+        {
+          argv: ["git", "clone", "--filter=blob:none", "--single-branch", "--branch", branch, `https://github.com/${repository}.git`, CHECKOUT_DIR],
+          workdir: "/build/work",
+          failed: (status) => `the clone of ${branch} failed (exit ${status})`
+        }
+      ];
+      for (const step of steps) {
+        if (abandoned) return "the check environment setup ran out of time";
+        const ran = await made.exec(step.argv, step.workdir, { logPath: environmentLog });
+        if (ran.status !== 0) return `the check environment could not be prepared: ${step.failed(ran.status)}`;
+      }
+      for (const step of lifecycleSteps((job.container ?? job.microvm)?.lifecycle ?? {})) {
+        log.line(`${tag} checks lifecycle ${step.name} (${step.source})`);
+        const ran = await made.exec(step.argv, CHECKOUT_DIR, { logPath: environmentLog });
+        if (ran.status !== 0) return `the check environment could not be prepared: ${lifecycleFailureReason(step, ran.status)}`;
+      }
+      return made;
+    };
+    const setupTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+    let settled;
+    try {
+      const outcome = await Promise.race([
+        setup().catch((err) => `the check environment could not be made: ${describeCallFailure(err)}`),
+        setupTimer.done
+      ]);
+      if (outcome === "timeout") {
+        abandoned = true;
+        return await failAll(`the check environment setup ran out of time after ${minutes}`, LIMIT_KEY_TAIL);
+      }
+      settled = outcome;
+    } finally {
+      setupTimer.cancel();
+    }
+    if (typeof settled === "string") return await failAll(settled);
+    const ready = settled;
+    let usable = true;
+    for (const check of checks) {
+      if (!usable) {
+        await record(check, "fail", "the check environment could not be restarted after an earlier check ran out of time");
+        continue;
+      }
+      const checkStartedMs = now().getTime();
+      const running = ready.exec(["/bin/sh", "-c", check.command], CHECKOUT_DIR, { logPath: join(jobDir, "checks", `${check.name}.log`) });
+      running.catch(() => void 0);
+      const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+      let outcome;
+      try {
+        outcome = await Promise.race([running, timer.done]);
+      } catch (err) {
+        timer.cancel();
+        await record(check, "fail", `the command could not be run: ${describeCallFailure(err)}`);
+        continue;
+      }
+      timer.cancel();
+      if (outcome === "timeout") {
+        await record(check, "fail", `ran out of time after ${minutes}`, LIMIT_KEY_TAIL);
+        const restarting = ready.restart().catch(() => false);
+        const restartTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
+        usable = await Promise.race([restarting, restartTimer.done]) === true;
+        restartTimer.cancel();
+        continue;
+      }
+      if (outcome.status === 0) {
+        log.line(`${tag} check ${check.name} ran (${formatDuration(now().getTime() - checkStartedMs)})`);
+        await record(check, "pass");
+        continue;
+      }
+      const last = lastLine(outcome.stderr) ?? lastLine(outcome.stdout);
+      const scrubbed = last === void 0 ? void 0 : scrubReason(last, config);
+      await record(check, "fail", `exited ${outcome.status}${scrubbed === void 0 ? " with no output" : `: ${cutReasonLine(scrubbed, 140)}`}`);
+    }
+  } catch (err) {
+    log.error(`${tag} checks stopped: ${describeCallFailure(err)}`);
+  } finally {
+    if (job.checkContainer !== void 0) {
+      await cleanupContainer(runtime, job.checkContainer).catch((err) => log.error(`${tag} removing the check environment failed: ${describeCallFailure(err)}`));
+      delete job.checkContainer;
+      writeJobFile(jobFilePath, job);
+    }
+    if (job.checkMicrovm !== void 0) {
+      await cleanupMicroVm(runtime, job.checkMicrovm).catch((err) => log.error(`${tag} removing the check environment failed: ${describeCallFailure(err)}`));
+      delete job.checkMicrovm;
+      writeJobFile(jobFilePath, job);
+    }
+    rmSync(scratch, { recursive: true, force: true });
+    if (existsSync(scratch)) log.error(`${tag} the checks' scratch directory could not be removed`);
+    log.line(`${tag} checks done (${formatDuration(now().getTime() - startedMs)})`);
+  }
+}
 
 // src/runner/job.ts
 var LEASE_SECONDS = 900;
@@ -1181,7 +1628,7 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
     await stopProcessGroup(job.pid);
   }
   if (outcome !== void 0 && job.endStage === void 0) {
-    const TEARDOWN_STAGES = ["sync", "cleanup", "report"];
+    const TEARDOWN_STAGES = ["checks", "sync", "cleanup", "report"];
     const staleTeardownStage = job.stage !== void 0 && TEARDOWN_STAGES.includes(job.stage);
     job.endStage = ending?.stage ?? (staleTeardownStage ? job.exitCode !== void 0 ? "command" : void 0 : job.stage);
     job.endExitCode = ending?.exitCode ?? (outcome === "ok" || outcome === "failed" ? job.exitCode : void 0);
@@ -1191,11 +1638,74 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
     job.reason = scrubReason(ending.reason, config);
     writeJobFile(jobFilePath, job);
   }
+  const uploadSessions = async (workdir) => {
+    if (job.container === void 0 && job.microvm === void 0 && job.build?.exec === void 0 && workdirProblem(workdir) !== void 0) {
+      log.line(`${tag} sync skipped: workdir ${workdir} is gone`);
+    } else if (job.itemKey === void 0) {
+      log.line(`${tag} sync skipped: the job file holds no item key`);
+    } else {
+      log.line(`${tag} sync started`);
+      const startedMs = now().getTime();
+      const sync = build !== void 0 ? await syncInContainer({ config, runtime: build.runtime, job, jobDir: jobDirPath }) : vmBuild !== void 0 ? await syncInMicroVm({ config, runtime: vmBuild.runtime, job, jobDir: jobDirPath }) : job.build?.exec !== void 0 ? await runSyncViaExec(config, job, jobDirPath, spawnFn) : await runSync(config, job.item, job.itemKey, workdir, jobDirPath, spawnFn, job.branch, job.commitIdentity?.author);
+      const took = formatDuration(now().getTime() - startedMs);
+      if (sync.ok) log.line(`${tag} sync ok (${took})`);
+      else log.error(`${tag} sync failed: ${sync.reason ?? "no reason given"} \xB7 continuing`);
+      const left = spooledArtifactIds(deps.home ?? homeFromEnv());
+      if (left.length > 0) {
+        job.spooledArtifacts = left.slice(0, MAX_NAMED_SPOOLED);
+        const named = job.spooledArtifacts.join(", ");
+        log.error(
+          `${tag} ${left.length} spooled artifact(s) not uploaded: ${named}${left.length > MAX_NAMED_SPOOLED ? `, and ${left.length - MAX_NAMED_SPOOLED} more` : ""}`
+        );
+      }
+    }
+  };
+  const endBuildMicroVm = vmBuild === void 0 ? void 0 : async () => {
+    if (job.workdir && job.sessionsSynced !== true) {
+      await uploadSessions(job.workdir);
+      job.sessionsSynced = true;
+      writeJobFile(jobFilePath, job);
+    }
+    await stopBuildMicroVm(vmBuild.runtime, vmBuild.state, jobDirPath);
+  };
+  if (job.checkContainer !== void 0 && job.checksRan === true) {
+    try {
+      await cleanupContainer(deps.container ?? createContainerRuntime(), job.checkContainer);
+      delete job.checkContainer;
+      writeJobFile(jobFilePath, job);
+    } catch (err) {
+      log.error(`${tag} removing an earlier attempt's check environment failed: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+  }
+  if (job.checkMicrovm !== void 0 && job.checksRan === true) {
+    try {
+      await cleanupMicroVm(deps.container ?? createContainerRuntime(), job.checkMicrovm);
+      delete job.checkMicrovm;
+      writeJobFile(jobFilePath, job);
+    } catch (err) {
+      log.error(`${tag} removing an earlier attempt's check environment failed: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+  }
+  if (outcome === "ok" && !job.cleanupRan && job.checksRan !== true) {
+    setStage(job, jobDirPath, "checks");
+    job.checksRan = true;
+    writeJobFile(jobFilePath, job);
+    try {
+      await runHostChecks({
+        config,
+        job,
+        jobDir: jobDirPath,
+        deps: { fetchImpl: deps.fetchImpl, runtime: deps.container ?? createContainerRuntime(), log, now: deps.now, endBuildMicroVm, ...deps.hostChecks }
+      });
+    } catch (err) {
+      log.error(`${tag} the host's checks stopped: ${describeCallFailure(err)} \xB7 continuing`);
+    }
+  }
   let outOfMemory;
   if (vmBuild?.state.cgroup !== void 0 && outcome !== void 0 && outcome !== "ok" && ending?.reason === void 0) {
     try {
       if (vmBuild.runtime.vm.cgroup.oomKills(vmBuild.state.cgroup) > 0) {
-        outOfMemory = `the build ran out of memory: the kernel killed its microVM at the limit of runner.build.memory (${config.build?.memory ?? "8g"}) plus 256 MiB for Firecracker. Raise runner.build.memory, or lower runner.concurrency`;
+        outOfMemory = outOfMemoryReason(config.build?.memory);
       }
     } catch (err) {
       log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
@@ -1210,26 +1720,8 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
   if (!job.cleanupRan) {
     if (job.workdir) {
       setStage(job, jobDirPath, "sync");
-      if (job.container === void 0 && job.microvm === void 0 && job.build?.exec === void 0 && workdirProblem(job.workdir) !== void 0) {
-        log.line(`${tag} sync skipped: workdir ${job.workdir} is gone`);
-      } else if (job.itemKey === void 0) {
-        log.line(`${tag} sync skipped: the job file holds no item key`);
-      } else {
-        log.line(`${tag} sync started`);
-        const startedMs = now().getTime();
-        const sync = build !== void 0 ? await syncInContainer({ config, runtime: build.runtime, job, jobDir: jobDirPath }) : vmBuild !== void 0 ? await syncInMicroVm({ config, runtime: vmBuild.runtime, job, jobDir: jobDirPath }) : job.build?.exec !== void 0 ? await runSyncViaExec(config, job, jobDirPath, spawnFn) : await runSync(config, job.item, job.itemKey, job.workdir, jobDirPath, spawnFn, job.branch, job.commitIdentity?.author);
-        const took = formatDuration(now().getTime() - startedMs);
-        if (sync.ok) log.line(`${tag} sync ok (${took})`);
-        else log.error(`${tag} sync failed: ${sync.reason ?? "no reason given"} \xB7 continuing`);
-        const left = spooledArtifactIds(deps.home ?? homeFromEnv());
-        if (left.length > 0) {
-          job.spooledArtifacts = left.slice(0, MAX_NAMED_SPOOLED);
-          const named = job.spooledArtifacts.join(", ");
-          log.error(
-            `${tag} ${left.length} spooled artifact(s) not uploaded: ${named}${left.length > MAX_NAMED_SPOOLED ? `, and ${left.length - MAX_NAMED_SPOOLED} more` : ""}`
-          );
-        }
-      }
+      if (job.sessionsSynced === true) log.line(`${tag} sync done before the build's microVM stopped`);
+      else await uploadSessions(job.workdir);
     }
     setStage(job, jobDirPath, "cleanup");
     try {