@outerlayer/cli 0.5.0 → 0.6.0

package/dist/{chunk-L24G4PLP.js → chunk-43J3M6KB.js}

@@ -1,30 +1,78 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { LEASE_SECONDS, releaseLease, isUnfinished, writeJobFile, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, readJobFile, hookLimitsFor } from './chunk-RWOTHXYE.js';
-import { RUNNER_PROTOCOL } from './chunk-QDQEUVUF.js';
 import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-ABIWDUMS.js';
 import { readMarker, currentVersion, removeMarker, writeMarker, entryIn, versionDir, switchCurrent, cleanVersions, isInLayout, startedThroughCurrent, stagingDir } from './chunk-YYYXRJUW.js';
+import { LEASE_SECONDS, releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startSender, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, ALREADY_RELEASED } from './chunk-VMJ3MMBX.js';
+import { writeJobFile, readJobFile, readJobFileChecked } from './chunk-T6OIJQRT.js';
+import { RUNNER_PROTOCOL } from './chunk-AIW6GLET.js';
+import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-UGP76IZ5.js';
+import { placeholderQueues, placeholderProblem } from './chunk-MXB5VB4Y.js';
+import { freeBytes, formatBytes } from './chunk-YYYRBYWG.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
-import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, formatClock, measureDisk, createRunnerLog, ensureTunnel, keepGhConfigCurrent, createContainerRuntime } from './chunk-4NMBDRBS.js';
+import { checkFit } from './chunk-OUUINEYB.js';
+import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, formatClock, measureDisk, createRunnerLog, setUpRunnerCgroup, ensureTunnel, keepGhConfigCurrent, createContainerRuntime, runnerLabelValue, BUILD_LABEL, RUNNER_LABEL } from './chunk-RZSF5B46.js';
 import { startTimeOf, claimPidFile, readPidFile, isSameProcess, releasePidFile } from './chunk-N5FOE5PS.js';
-import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-RMAZ2NWA.js';
-import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks } from './chunk-5Y3QQRUH.js';
+import { spawnRun } from './chunk-774HEQL6.js';
+import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-BNMCXPG7.js';
+import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind } from './chunk-3DD6AOWC.js';
 import { exceedsDisk } from './chunk-F6GFZXZ3.js';
-import { parseClaimIdentity, sharedUserEnvironment, JOB_MOUNT } from './chunk-AUFG23AA.js';
+import { parseClaimIdentity, sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-AUFG23AA.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
 import { loadHostKey, ensureHostKey, signerFor } from './chunk-M5POMKKW.js';
-import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
+import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, spoolDir, RUNNER_FILE_MODE, rootfsDir } from './chunk-ZM2IMMYN.js';
 import { HOOKS_DIR_NAME, writeCommitHooks } from './chunk-XCCLFVXM.js';
 import { randomBytes } from 'crypto';
-import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync } from 'fs';
+import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync, closeSync, openSync, constants, lstatSync } from 'fs';
 import { join, dirname } from 'path';
+import { totalmem } from 'os';
 
 createRequire(import.meta.url);
+function openKvm() {
+  try {
+    closeSync(openSync("/dev/kvm", constants.O_RDWR));
+    return "usable";
+  } catch (err) {
+    return err.code === "ENOENT" ? "absent" : "no-permission";
+  }
+}
+function realBlockedProbes() {
+  return { env: process.env, kvm: openKvm, totalMemoryBytes: totalmem };
+}
+var DEFAULT_BUILD_MEMORY = "8g";
+function checkBlocked(config, probes) {
+  const placeholders = placeholderQueues(config);
+  if (placeholders.length > 0) return { reason: `${placeholderProblem(placeholders)}. The runner takes work again once the config is edited` };
+  const kind = builtinKind(config.hooks.provision);
+  if (kind === "container" || kind === "vm") {
+    const fit = checkFit({
+      concurrency: config.concurrency,
+      buildMemory: config.build?.memory ?? DEFAULT_BUILD_MEMORY,
+      usableBytes: probes.totalMemoryBytes()
+    });
+    if (!fit.fits) return { reason: `${fit.message}. Lower runner.concurrency or runner.build.memory` };
+  }
+  const kvm = kind === "vm" ? probes.kvm() : "usable";
+  if (kvm === "absent") {
+    return { reason: "/dev/kvm does not exist on this machine, so builtin:vm cannot start a microVM. Use a host with KVM enabled, or builtin:container hooks" };
+  }
+  if (kvm !== "usable") {
+    return {
+      reason: "the runner cannot open /dev/kvm, so builtin:vm cannot start a microVM. Add the runner's account to the kvm group, then restart the service, because a service reads its groups only when it starts"
+    };
+  }
+  if (kind !== "own" && claudeCredential(probes.env) === void 0) {
+    return {
+      reason: "there is no Claude credential in the runner's environment. Put CLAUDE_CODE_OAUTH_TOKEN (from `claude setup-token`) or ANTHROPIC_API_KEY in the runner.env beside the config, then restart the service"
+    };
+  }
+  return void 0;
+}
 
 // src/runner/heartbeat.ts
 var HEARTBEAT_TIMEOUT_MS = 1e4;
 var ERROR_CODE_MAX = 100;
 var ERROR_MESSAGE_MAX = 500;
+var BLOCKED_REASON_MAX = 500;
 var UNREACHABLE_MESSAGE = "the request did not reach the platform";
 function noPlatformMessage(status) {
   return `the request got HTTP ${status} with no error message from the platform`;
@@ -68,7 +116,11 @@ async function sendHeartbeat(args, fetchImpl) {
       outcome: args.lastUpdate.outcome,
       reason: args.lastUpdate.reason,
       ...args.lastUpdate.at !== void 0 ? { at: args.lastUpdate.at } : {}
-    }
+    },
+    // Free disk on the runner's file system, in bytes. Sent only while held, so a host with room sends nothing about its disk.
+    lowDisk: args.lowDisk === void 0 ? void 0 : { free: args.lowDisk.free, needed: args.lowDisk.needed },
+    // Why the runner claims nothing. Sent only while it holds, so a runner that takes work says nothing about it.
+    blocked: args.blocked === void 0 ? void 0 : { reason: cutForGateway(args.blocked.reason, BLOCKED_REASON_MAX) }
   };
   try {
     const response = await postJson(
@@ -371,9 +423,9 @@ function retryFailure(version, err) {
   }
   return { outcome: "retrying", reason: messageOf(err) };
 }
-function finish(args, failure) {
-  let update = { from: args.own, to: args.version, outcome: failure.outcome, reason: failure.reason, at: isoAt(args.now) };
-  if (failure.outcome === "retrying") {
+function finish(args, failure2) {
+  let update = { from: args.own, to: args.version, outcome: failure2.outcome, reason: failure2.reason, at: isoAt(args.now) };
+  if (failure2.outcome === "retrying") {
     const previous = readLastUpdate(args.configPath);
     const attempts = previous?.to === args.version && previous.outcome === "retrying" ? (previous.attempts ?? 1) + 1 : 1;
     update = { ...update, attempts };
@@ -383,16 +435,16 @@ function finish(args, failure) {
   } catch (err) {
     args.log.error(`could not record the update attempt: ${messageOf(err)}`);
   }
-  if (failure.outcome === "retrying") {
+  if (failure2.outcome === "retrying") {
     return {
       kind: "retrying",
-      reason: failure.reason,
-      detail: failure.detail ?? failure.reason,
+      reason: failure2.reason,
+      detail: failure2.detail ?? failure2.reason,
       attempts: update.attempts ?? 1,
       retryAtMs: retryAfterMs(update)
     };
   }
-  return { kind: "failed", outcome: failure.outcome, reason: failure.reason };
+  return { kind: "failed", outcome: failure2.outcome, reason: failure2.reason };
 }
 async function earlyFailure(args) {
   const current = currentVersion(args.configPath);
@@ -423,8 +475,8 @@ async function earlyFailure(args) {
   return void 0;
 }
 async function preflightUpdate(args) {
-  const failure = await earlyFailure(args);
-  return failure === void 0 ? void 0 : finish(args, failure);
+  const failure2 = await earlyFailure(args);
+  return failure2 === void 0 ? void 0 : finish(args, failure2);
 }
 async function performUpdate(args) {
   const { configPath, own, version, deps } = args;
@@ -525,12 +577,325 @@ function settleUpdate(args) {
   }
 }
 
+// src/runner/housekeeping/docker.ts
+var KIND_LABEL = "ai.outerlayer.kind";
+function failure(result, what) {
+  return new Error(`${what}: ${result.stderr.trim() || `docker exited ${result.status}`}`);
+}
+function labelled(stdout, runnerId) {
+  const found = [];
+  for (const line of stdout.split("\n")) {
+    const [key = "", build = "", runner = ""] = line.replace(/\r?\n$/, "").split("	");
+    if (key.trim() === "" || build === "") continue;
+    if (runner !== "" && runner !== runnerId) continue;
+    found.push({ key: key.trim(), build, owned: runner === runnerId });
+  }
+  return found;
+}
+function decimalUnits() {
+  return { b: 1, kb: 1e3, mb: 1e6, gb: 1e9, tb: 1e12, pb: 1e15 };
+}
+function parsePrunedBytes(stdout) {
+  const match = /^Total(?: reclaimed space)?:\s*([\d.]+)\s*([A-Za-z]*)\s*$/im.exec(stdout);
+  if (match === null) return 0;
+  const unit = decimalUnits()[match[2].toLowerCase() || "b"];
+  return unit === void 0 ? 0 : Math.round(Number(match[1]) * unit);
+}
+function createHousekeepingDocker(deps) {
+  const run = deps.run ?? spawnRun;
+  return {
+    async listBuildContainers() {
+      const result = await run("docker", [
+        "ps",
+        "--all",
+        "--no-trunc",
+        "--filter",
+        `label=${BUILD_LABEL}`,
+        "--format",
+        `{{.ID}}	{{.Label "${BUILD_LABEL}"}}	{{.Label "${RUNNER_LABEL}"}}`
+      ]);
+      if (result.status !== 0) throw failure(result, "docker ps");
+      return labelled(result.stdout, deps.runnerId).map(({ key, build, owned }) => ({ id: key, build, owned }));
+    },
+    async listBuildVolumes() {
+      const result = await run("docker", [
+        "volume",
+        "ls",
+        "--filter",
+        `label=${BUILD_LABEL}`,
+        "--format",
+        `{{.Name}}	{{.Label "${BUILD_LABEL}"}}	{{.Label "${RUNNER_LABEL}"}}`
+      ]);
+      if (result.status !== 0) throw failure(result, "docker volume ls");
+      return labelled(result.stdout, deps.runnerId).map(({ key, build, owned }) => ({ name: key, build, owned }));
+    },
+    async removeContainer(id) {
+      const result = await run("docker", ["rm", "--force", "--volumes", id]);
+      if (result.status !== 0) throw failure(result, `docker rm ${id}`);
+    },
+    async removeVolume(name) {
+      const result = await run("docker", ["volume", "rm", "--force", name]);
+      if (result.status !== 0) throw failure(result, `docker volume rm ${name}`);
+    },
+    async imageState(ref) {
+      const result = await run("docker", ["image", "inspect", "--format", "{{.Id}}", ref]);
+      if (result.status === 0) return "present";
+      return /no such image|not found/i.test(result.stderr) ? "missing" : "unknown";
+    },
+    async listDanglingImages() {
+      const listed = await run("docker", [
+        "image",
+        "ls",
+        "--all",
+        "--no-trunc",
+        "--quiet",
+        "--filter",
+        "dangling=true",
+        "--filter",
+        `label=${KIND_LABEL}`
+      ]);
+      if (listed.status !== 0) throw failure(listed, "docker image ls");
+      const ids = [...new Set(listed.stdout.split("\n").map((l) => l.trim()).filter(Boolean))];
+      if (ids.length === 0) return [];
+      const inspected = await run("docker", ["image", "inspect", ...ids]);
+      if (inspected.status !== 0) throw failure(inspected, "docker image inspect");
+      return JSON.parse(inspected.stdout).map((row) => ({
+        id: row.Id,
+        size: row.Size ?? 0,
+        labels: row.Config?.Labels ?? {}
+      }));
+    },
+    async removeImage(id) {
+      const result = await run("docker", ["rmi", id]);
+      if (result.status !== 0) throw failure(result, `docker rmi ${id}`);
+    },
+    async builderPrune(keepBytes) {
+      const result = await run("docker", ["builder", "prune", "--force", "--keep-storage", String(keepBytes)]);
+      if (result.status !== 0) throw failure(result, "docker builder prune");
+      return { freedBytes: parsePrunedBytes(result.stdout) };
+    }
+  };
+}
+
+// src/runner/housekeeping/select.ts
+var DAY_MS = 864e5;
+function spoolMaxAgeMs() {
+  return 24 * 36e5;
+}
+function selectJobDirs(entries, nowMs, keepDays) {
+  return entries.filter(
+    (entry) => (entry.state === "finished" || entry.state === "no_job_file") && nowMs - entry.mtimeMs > keepDays * DAY_MS
+  );
+}
+function liveBuildIds(jobs) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const job of jobs) {
+    for (const id of [job.container?.buildId, job.microvm?.buildId, job.build?.buildId]) if (id !== void 0) ids.add(id);
+  }
+  return ids;
+}
+function finishedBuildIds(jobs) {
+  return liveBuildIds(jobs);
+}
+function liveImageIds(jobs) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const job of jobs) {
+    for (const id of [job.container?.imageId, job.microvm?.imageId]) if (id !== void 0) ids.add(id);
+  }
+  return ids;
+}
+function selectOrphans(items, live, finished) {
+  return items.filter((item) => !live.has(item.build) && (item.owned || finished.has(item.build)));
+}
+function rootDiskImageId(fileName) {
+  if (!fileName.endsWith(".ext4") || fileName.includes(".partial-")) return void 0;
+  const id = fileName.slice(0, -".ext4".length);
+  return id === "" ? void 0 : id;
+}
+function selectSpool(files, nowMs, buildRunning) {
+  return buildRunning ? [] : files.filter((file) => nowMs - file.mtimeMs > spoolMaxAgeMs());
+}
+
+// src/runner/housekeeping/run.ts
+function sizeOf(path) {
+  try {
+    const stats = lstatSync(path);
+    if (!stats.isDirectory()) return stats.size;
+    return readdirSync(path).reduce((sum, name) => sum + sizeOf(join(path, name)), 0);
+  } catch {
+    return 0;
+  }
+}
+function listDir(path) {
+  try {
+    return readdirSync(path);
+  } catch {
+    return [];
+  }
+}
+function mtimeOf(path) {
+  try {
+    return statSync(path).mtimeMs;
+  } catch {
+    return void 0;
+  }
+}
+function readJobs(jobsPath) {
+  const entries = [];
+  const unfinished = [];
+  const finished = [];
+  const unreadable = [];
+  for (const dirent of existsSync(jobsPath) ? readdirSync(jobsPath, { withFileTypes: true }) : []) {
+    if (!dirent.isDirectory()) continue;
+    const dirPath = join(jobsPath, dirent.name);
+    const mtimeMs = mtimeOf(dirPath);
+    if (mtimeMs === void 0) continue;
+    const read = readJobFileChecked(join(dirPath, "job.json"));
+    let state;
+    if (read.kind === "missing") {
+      state = "no_job_file";
+    } else if (read.kind === "unreadable") {
+      unreadable.push(dirent.name);
+      state = "unreadable";
+    } else if (isUnfinished(read.job)) {
+      unfinished.push(read.job);
+      state = "unfinished";
+    } else {
+      finished.push(read.job);
+      state = read.job.releaseFailed === void 0 || read.job.releaseFailed === ALREADY_RELEASED ? "finished" : "release_failed";
+    }
+    entries.push({ name: dirent.name, mtimeMs, state });
+  }
+  return { entries, unfinished, finished, unreadable };
+}
+async function removeDanglingImages(docker, log, live) {
+  let freed = 0;
+  for (const image of await docker.listDanglingImages()) {
+    if (image.labels[KIND_LABEL] === void 0) continue;
+    if (live.has(image.id)) continue;
+    try {
+      await docker.removeImage(image.id);
+      freed += image.size;
+      log.line(`housekeeping: removed dangling image ${image.id.replace(/^sha256:/, "").slice(0, 12)} (${formatBytes(image.size)}), a rebuild replaced it`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove dangling image ${image.id}: ${describeCallFailure(err)}`);
+    }
+  }
+  return freed;
+}
+async function runHousekeeping(options) {
+  const { docker, log } = options;
+  const nowMs = options.now().getTime();
+  const root = runnerDir(options.configPath);
+  let freed = 0;
+  let listed;
+  try {
+    const containers = await docker.listBuildContainers();
+    listed = { containers, volumes: await docker.listBuildVolumes() };
+  } catch (err) {
+    log.error(`housekeeping: Docker could not be asked, so nothing in Docker is removed this pass: ${describeCallFailure(err)}`);
+  }
+  const jobs = readJobs(jobsDir(options.configPath));
+  for (const name of jobs.unreadable) {
+    log.error(`housekeeping: job file ${name}/job.json does not parse, so nothing in Docker is removed this pass`);
+  }
+  const canTell = jobs.unreadable.length === 0;
+  for (const entry of selectJobDirs(jobs.entries, nowMs, options.keepDays)) {
+    const path = join(jobsDir(options.configPath), entry.name);
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { recursive: true, force: true });
+      freed += size;
+      log.line(`housekeeping: removed job directory ${entry.name} (${formatBytes(size)}), last changed ${formatDuration(nowMs - entry.mtimeMs)} ago`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove job directory ${entry.name}: ${describeCallFailure(err)}`);
+    }
+  }
+  const live = liveBuildIds(jobs.unfinished);
+  const finished = finishedBuildIds(jobs.finished);
+  if (canTell && listed !== void 0) {
+    for (const container of selectOrphans(listed.containers, live, finished)) {
+      try {
+        await docker.removeContainer(container.id);
+        log.line(`housekeeping: removed container ${container.id.slice(0, 12)}, no unfinished build ${container.build} names it`);
+      } catch (err) {
+        log.error(`housekeeping: could not remove container ${container.id}: ${describeCallFailure(err)}`);
+      }
+    }
+    for (const volume of selectOrphans(listed.volumes, live, finished)) {
+      try {
+        await docker.removeVolume(volume.name);
+        log.line(`housekeeping: removed volume ${volume.name}, no unfinished build ${volume.build} names it`);
+      } catch (err) {
+        log.error(`housekeeping: could not remove volume ${volume.name}: ${describeCallFailure(err)}`);
+      }
+    }
+  }
+  if (canTell && listed !== void 0) {
+    freed += await removeStaleRootDisks(root, jobs.unfinished, options);
+    try {
+      freed += await removeDanglingImages(docker, log, liveImageIds(jobs.unfinished));
+    } catch (err) {
+      log.error(`housekeeping: dangling images could not be listed: ${describeCallFailure(err)}`);
+    }
+    if (options.buildCacheMax !== "off") {
+      try {
+        const pruned = await docker.builderPrune(options.buildCacheMax);
+        freed += pruned.freedBytes;
+        log.line(`housekeeping: pruned the build cache down to ${formatBytes(options.buildCacheMax)}, freed ${formatBytes(pruned.freedBytes)}`);
+      } catch (err) {
+        log.error(`housekeeping: the build cache could not be pruned: ${describeCallFailure(err)}`);
+      }
+    }
+  }
+  const spoolPath = spoolDir(root);
+  const spoolFiles = listDir(spoolPath).flatMap((name) => {
+    const mtimeMs = mtimeOf(join(spoolPath, name));
+    return mtimeMs === void 0 ? [] : [{ name, mtimeMs }];
+  });
+  for (const file of selectSpool(spoolFiles, nowMs, jobs.unfinished.length > 0)) {
+    const path = join(spoolPath, file.name);
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { recursive: true, force: true });
+      freed += size;
+      log.line(`housekeeping: removed spool file ${file.name} (${formatBytes(size)}), last changed ${formatDuration(nowMs - file.mtimeMs)} ago and no build is running`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove spool file ${file.name}: ${describeCallFailure(err)}`);
+    }
+  }
+  log.line(`housekeeping: pass finished, freed ${formatBytes(freed)}`);
+  return { freedBytes: freed };
+}
+async function removeStaleRootDisks(root, unfinished, options) {
+  const dir = rootfsDir(root);
+  const inUse = new Set(unfinished.flatMap((job) => job.microvm?.rootfs === void 0 ? [] : [job.microvm.rootfs]));
+  let freed = 0;
+  for (const name of listDir(dir)) {
+    const imageId = rootDiskImageId(name);
+    if (imageId === void 0) continue;
+    const path = join(dir, name);
+    if (inUse.has(path)) continue;
+    if (await options.docker.imageState(`sha256:${imageId}`) !== "missing") continue;
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { force: true });
+      freed += size;
+      options.log.line(`housekeeping: removed root disk ${name} (${formatBytes(size)}), Docker no longer has its image`);
+    } catch (err) {
+      options.log.error(`housekeeping: could not remove root disk ${name}: ${describeCallFailure(err)}`);
+    }
+  }
+  return freed;
+}
+
 // src/runner/loop.ts
 var QUEUE_ORDER = ["amend", "implement"];
 var TRACKER_REPOSITORY_HOSTS = {
   github: "github.com",
   gitlab: "gitlab.com"
 };
+var HOUSEKEEPING_INTERVAL_MS = 36e5;
 function logOf(state) {
   state.deps.log ??= createRunnerLog(state.deps.now ?? (() => /* @__PURE__ */ new Date()));
   return state.deps.log;
@@ -645,7 +1010,9 @@ async function sendTickHeartbeat(state, workRequest) {
           slotsUsed: state.jobs.size,
           slotsTotal: state.config.concurrency,
           workRequest: next,
-          lastUpdate: readLastUpdate(state.config.configPath)
+          lastUpdate: readLastUpdate(state.config.configPath),
+          lowDisk: state.lowDisk,
+          blocked: state.blocked
         },
         state.deps.fetchImpl
       );
@@ -769,9 +1136,84 @@ async function tick(state) {
   }
   await sendTickHeartbeat(state, state.lastWorkRequest);
 }
+function holdWhenBlocked(state) {
+  const probes = {
+    ...realBlockedProbes(),
+    ...state.deps.container?.hostEnv === void 0 ? {} : { env: state.deps.container.hostEnv },
+    ...state.deps.blocked
+  };
+  const blocked = checkBlocked(state.config, probes);
+  const log = logOf(state);
+  if (blocked !== void 0) {
+    if (state.blocked?.reason !== blocked.reason) log.line(`claiming nothing: ${blocked.reason}`);
+    state.blocked = blocked;
+    return true;
+  }
+  if (state.blocked !== void 0) {
+    state.blocked = void 0;
+    log.line("the runner can run its work again \xB7 taking work");
+  }
+  return false;
+}
+function holdOffForDisk(state) {
+  const minimum = state.config.housekeeping?.minFreeDisk ?? 0;
+  const root = runnerDir(state.config.configPath);
+  const free = minimum === 0 ? Number.POSITIVE_INFINITY : (state.deps.freeBytes ?? freeBytes)(root) ?? Number.POSITIVE_INFINITY;
+  const log = logOf(state);
+  if (free < minimum) {
+    if (state.lowDisk === void 0) {
+      log.line(
+        `holding off new work: ${formatBytes(free)} free on the runner's file system, ${formatBytes(minimum)} needed (runner.housekeeping.minFreeDisk) \xB7 claiming again once there is room`
+      );
+      recordHousekeeping(state, root, { holding: { free, needed: minimum, since: nowOf(state).toISOString() } });
+    }
+    state.lowDisk = { free, needed: minimum };
+    return true;
+  }
+  if (state.lowDisk !== void 0) {
+    state.lowDisk = void 0;
+    log.line("free disk is back above the minimum \xB7 taking work again");
+    recordHousekeeping(state, root, { holding: void 0 });
+  } else if (!state.holdRecordChecked) {
+    if (readHousekeepingRecord(root).holding !== void 0) recordHousekeeping(state, root, { holding: void 0 });
+  }
+  state.holdRecordChecked = true;
+  return false;
+}
+function recordHousekeeping(state, root, patch) {
+  try {
+    writeHousekeepingRecord(root, patch);
+  } catch (err) {
+    logOf(state).error(`could not write housekeeping.json: ${describeCallFailure(err)}`);
+  }
+}
+async function housekeepingPass(state) {
+  const config = state.config.housekeeping;
+  if (config === void 0 || state.housekeepingRunning) return;
+  state.housekeepingRunning = true;
+  try {
+    state.deps.housekeeping ??= {};
+    state.deps.housekeeping.docker ??= createHousekeepingDocker({ runnerId: runnerLabelValue(state.config.configPath) });
+    const result = await runHousekeeping({
+      configPath: state.config.configPath,
+      keepDays: config.jobsKeepDays,
+      buildCacheMax: config.buildCacheMax,
+      docker: state.deps.housekeeping.docker,
+      now: state.deps.now ?? (() => /* @__PURE__ */ new Date()),
+      log: logOf(state)
+    });
+    recordHousekeeping(state, runnerDir(state.config.configPath), { lastRunAt: nowOf(state).toISOString(), freedBytes: result.freedBytes });
+  } catch (err) {
+    logOf(state).error(`housekeeping failed: ${describeCallFailure(err)}`);
+  } finally {
+    state.housekeepingRunning = false;
+  }
+}
 async function runTick(state) {
   reloadConfig(state);
   if (await updateWhenIdle(state)) return;
+  if (holdWhenBlocked(state)) return;
+  if (holdOffForDisk(state)) return;
   const log = logOf(state);
   const repoFilter = new RepoFilter(state.config.repos);
   let filtered = 0;
@@ -825,7 +1267,7 @@ var CLAIM_REFUSAL_CODES = new Set(
   "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous branch_claimed_elsewhere".split(" ")
 );
 async function tryClaimAndStart(state, item, queue) {
-  const { postJson: postJson2 } = await import('./work-cmd-NUR4CAG6.js');
+  const { postJson: postJson2 } = await import('./work-cmd-OVNF44FY.js');
   const log = logOf(state);
   const hostKey = await loadHostKey(state.config.configPath);
   const claim = await retrySend(
@@ -1304,6 +1746,17 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity)
   writeJobFile(jobFilePath, job);
   const commandStartedMs = nowOf(state).getTime();
   log.line(`${tag} command started \xB7 pid ${running.pid}`);
+  const sender = await startSender({
+    config: state.config,
+    job,
+    jobDir: dirPath,
+    deps: { spawnFn: state.deps.spawnFn, runtime: job.microvm !== void 0 || job.container !== void 0 ? runtimeOf(state) : void 0, log }
+  });
+  if (sender !== void 0) {
+    job = { ...active.job, sender };
+    active.job = job;
+    writeJobFile(jobFilePath, job);
+  }
   running.onExit.then(({ code, signal, error }) => {
     if (active.ending) return;
     if (error !== void 0) {
@@ -1525,7 +1978,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const outcome = await onStartUnderMarker({
       configPath,
       own: ownVersionOf(deps),
-      check: update.check ? () => update.check(configPath) : async () => (await import('./check-AXCXDYO2.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
+      check: update.check ? () => update.check(configPath) : async () => (await import('./check-CCVPBN7A.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
       now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
       log
     });
@@ -1544,6 +1997,11 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   mkdirSync(jobsDir(configPath), { recursive: true, mode: RUNNER_DIR_MODE });
   const self = { pid: process.pid, start: startTimeOf(process.pid) ?? "" };
   claimPidFile(pidFilePath(configPath), self);
+  const { delegated, refusal } = setUpRunnerCgroup();
+  if (refusal !== void 0) log.error(`the kernel refused a cgroup step for the runner's service: ${refusal}`);
+  if (isBuiltinVm(config.hooks.provision) && !delegated) {
+    log.line("microVM builds run with no memory limit of their own: the runner's service is not delegated its cgroup (Delegate=yes in the unit, cgroup v2)");
+  }
   let configMtimeMs;
   try {
     configMtimeMs = statSync(configPath).mtimeMs;
@@ -1606,6 +2064,12 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   let interrupts = 0;
   state.pollTimerSeconds = state.config.pollSeconds;
   const pollTimer = setInterval(() => void tick(state), state.pollTimerSeconds * 1e3);
+  const hooksKind = builtinKind(config.hooks.provision);
+  const housekeepingTimer = hooksKind === "container" || hooksKind === "vm" ? (() => {
+    const everyMs = deps.housekeeping?.intervalMs ?? HOUSEKEEPING_INTERVAL_MS;
+    void housekeepingPass(state);
+    return setInterval(() => void housekeepingPass(state), everyMs);
+  })() : void 0;
   const watchTimer = setInterval(() => {
     const now = nowMs();
     for (const active of state.jobs.values()) {
@@ -1639,6 +2103,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     if (intakeClosed) return;
     intakeClosed = true;
     clearInterval(pollTimer);
+    clearInterval(housekeepingTimer);
   };
   const close = () => {
     stopIntake();

package/dist/{chunk-FC7TMRNA.js → chunk-4NLKY4WR.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { detectMachine, supportsContainerBuilds, reachDockerStep, describeMachineDocker, supportsMicroVmBuilds } from './chunk-RJ467PDQ.js';
-import { MIN_ENGINE_MAJOR } from './chunk-4NMBDRBS.js';
+import { detectMachine, supportsContainerBuilds, reachDockerStep, describeMachineDocker, supportsMicroVmBuilds } from './chunk-RBOMWNOG.js';
+import { MIN_ENGINE_MAJOR } from './chunk-RZSF5B46.js';
 import { RUNNER_DIR_MODE, runnerDir, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
 import { existsSync, readFileSync, mkdirSync, writeFileSync } from 'fs';
 import { dirname } from 'path';

package/dist/{lima-XN65D7GN.js → chunk-77MM5RYD.js}

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import './chunk-Z5GEFMRW.js';
 import { spawnSync, spawn } from 'child_process';
 import { statSync, accessSync, constants } from 'fs';
 import { delimiter, resolve } from 'path';

package/dist/{chunk-D2KWEAN6.js → chunk-7DBI6SNT.js}

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { sanitizeTerminal } from './chunk-WOBNRVQT.js';
+import { sanitizeTerminal } from './chunk-3DEVYTNQ.js';
 import { DEFAULT_DASHBOARD } from './chunk-USO2DKBD.js';
 import { readCloudConfig } from './chunk-EABW6AJQ.js';
 

package/dist/{chunk-QDQEUVUF.js → chunk-AIW6GLET.js}

@@ -4,6 +4,6 @@ import { createRequire } from 'node:module';
 createRequire(import.meta.url);
 
 // src/runner/protocol.ts
-var RUNNER_PROTOCOL = 12;
+var RUNNER_PROTOCOL = 14;
 
 export { RUNNER_PROTOCOL };