@outerlayer/cli 0.4.1 → 0.5.1

package/dist/{chunk-QY22NZUW.js → chunk-F4NU4H3V.js}

@@ -1,22 +1,25 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { LEASE_SECONDS, releaseLease, isUnfinished, writeJobFile, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, readJobFile, hookLimitsFor } from './chunk-DM3VFDS3.js';
-import { RUNNER_PROTOCOL } from './chunk-QDQEUVUF.js';
+import { LEASE_SECONDS, releaseLease, isUnfinished, writeJobFile, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, readJobFile, hookLimitsFor, readJobFileChecked, ALREADY_RELEASED } from './chunk-UWH5DSUP.js';
+import { RUNNER_PROTOCOL } from './chunk-ZXBITXTS.js';
+import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-UGP76IZ5.js';
 import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-ABIWDUMS.js';
 import { readMarker, currentVersion, removeMarker, writeMarker, entryIn, versionDir, switchCurrent, cleanVersions, isInLayout, startedThroughCurrent, stagingDir } from './chunk-YYYXRJUW.js';
+import { freeBytes, formatBytes } from './chunk-YYYRBYWG.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
-import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, formatClock, measureDisk, createRunnerLog, ensureTunnel, keepGhConfigCurrent, createContainerRuntime } from './chunk-HZRNMGLF.js';
+import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, formatClock, measureDisk, createRunnerLog, ensureTunnel, keepGhConfigCurrent, createContainerRuntime, runnerLabelValue, BUILD_LABEL, RUNNER_LABEL } from './chunk-T52PKE2Y.js';
 import { startTimeOf, claimPidFile, readPidFile, isSameProcess, releasePidFile } from './chunk-N5FOE5PS.js';
-import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-W5FVUZXV.js';
-import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks } from './chunk-5Y3QQRUH.js';
+import { spawnRun } from './chunk-774HEQL6.js';
+import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-RMAZ2NWA.js';
+import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind } from './chunk-3DD6AOWC.js';
 import { exceedsDisk } from './chunk-F6GFZXZ3.js';
 import { parseClaimIdentity, sharedUserEnvironment, JOB_MOUNT } from './chunk-AUFG23AA.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
 import { loadHostKey, ensureHostKey, signerFor } from './chunk-M5POMKKW.js';
-import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, RUNNER_FILE_MODE } from './chunk-ZM2IMMYN.js';
+import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, spoolDir, RUNNER_FILE_MODE, rootfsDir } from './chunk-ZM2IMMYN.js';
 import { HOOKS_DIR_NAME, writeCommitHooks } from './chunk-XCCLFVXM.js';
 import { randomBytes } from 'crypto';
-import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync } from 'fs';
+import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync, lstatSync } from 'fs';
 import { join, dirname } from 'path';
 
 createRequire(import.meta.url);
@@ -68,7 +71,9 @@ async function sendHeartbeat(args, fetchImpl) {
       outcome: args.lastUpdate.outcome,
       reason: args.lastUpdate.reason,
       ...args.lastUpdate.at !== void 0 ? { at: args.lastUpdate.at } : {}
-    }
+    },
+    // Free disk on the runner's file system, in bytes. Sent only while held, so a host with room sends nothing about its disk.
+    lowDisk: args.lowDisk === void 0 ? void 0 : { free: args.lowDisk.free, needed: args.lowDisk.needed }
   };
   try {
     const response = await postJson(
@@ -371,9 +376,9 @@ function retryFailure(version, err) {
   }
   return { outcome: "retrying", reason: messageOf(err) };
 }
-function finish(args, failure) {
-  let update = { from: args.own, to: args.version, outcome: failure.outcome, reason: failure.reason, at: isoAt(args.now) };
-  if (failure.outcome === "retrying") {
+function finish(args, failure2) {
+  let update = { from: args.own, to: args.version, outcome: failure2.outcome, reason: failure2.reason, at: isoAt(args.now) };
+  if (failure2.outcome === "retrying") {
     const previous = readLastUpdate(args.configPath);
     const attempts = previous?.to === args.version && previous.outcome === "retrying" ? (previous.attempts ?? 1) + 1 : 1;
     update = { ...update, attempts };
@@ -383,16 +388,16 @@ function finish(args, failure) {
   } catch (err) {
     args.log.error(`could not record the update attempt: ${messageOf(err)}`);
   }
-  if (failure.outcome === "retrying") {
+  if (failure2.outcome === "retrying") {
     return {
       kind: "retrying",
-      reason: failure.reason,
-      detail: failure.detail ?? failure.reason,
+      reason: failure2.reason,
+      detail: failure2.detail ?? failure2.reason,
       attempts: update.attempts ?? 1,
       retryAtMs: retryAfterMs(update)
     };
   }
-  return { kind: "failed", outcome: failure.outcome, reason: failure.reason };
+  return { kind: "failed", outcome: failure2.outcome, reason: failure2.reason };
 }
 async function earlyFailure(args) {
   const current = currentVersion(args.configPath);
@@ -423,8 +428,8 @@ async function earlyFailure(args) {
   return void 0;
 }
 async function preflightUpdate(args) {
-  const failure = await earlyFailure(args);
-  return failure === void 0 ? void 0 : finish(args, failure);
+  const failure2 = await earlyFailure(args);
+  return failure2 === void 0 ? void 0 : finish(args, failure2);
 }
 async function performUpdate(args) {
   const { configPath, own, version, deps } = args;
@@ -525,12 +530,325 @@ function settleUpdate(args) {
   }
 }
 
+// src/runner/housekeeping/docker.ts
+var KIND_LABEL = "ai.outerlayer.kind";
+function failure(result, what) {
+  return new Error(`${what}: ${result.stderr.trim() || `docker exited ${result.status}`}`);
+}
+function labelled(stdout, runnerId) {
+  const found = [];
+  for (const line of stdout.split("\n")) {
+    const [key = "", build = "", runner = ""] = line.replace(/\r?\n$/, "").split("	");
+    if (key.trim() === "" || build === "") continue;
+    if (runner !== "" && runner !== runnerId) continue;
+    found.push({ key: key.trim(), build, owned: runner === runnerId });
+  }
+  return found;
+}
+function decimalUnits() {
+  return { b: 1, kb: 1e3, mb: 1e6, gb: 1e9, tb: 1e12, pb: 1e15 };
+}
+function parsePrunedBytes(stdout) {
+  const match = /^Total(?: reclaimed space)?:\s*([\d.]+)\s*([A-Za-z]*)\s*$/im.exec(stdout);
+  if (match === null) return 0;
+  const unit = decimalUnits()[match[2].toLowerCase() || "b"];
+  return unit === void 0 ? 0 : Math.round(Number(match[1]) * unit);
+}
+function createHousekeepingDocker(deps) {
+  const run = deps.run ?? spawnRun;
+  return {
+    async listBuildContainers() {
+      const result = await run("docker", [
+        "ps",
+        "--all",
+        "--no-trunc",
+        "--filter",
+        `label=${BUILD_LABEL}`,
+        "--format",
+        `{{.ID}}	{{.Label "${BUILD_LABEL}"}}	{{.Label "${RUNNER_LABEL}"}}`
+      ]);
+      if (result.status !== 0) throw failure(result, "docker ps");
+      return labelled(result.stdout, deps.runnerId).map(({ key, build, owned }) => ({ id: key, build, owned }));
+    },
+    async listBuildVolumes() {
+      const result = await run("docker", [
+        "volume",
+        "ls",
+        "--filter",
+        `label=${BUILD_LABEL}`,
+        "--format",
+        `{{.Name}}	{{.Label "${BUILD_LABEL}"}}	{{.Label "${RUNNER_LABEL}"}}`
+      ]);
+      if (result.status !== 0) throw failure(result, "docker volume ls");
+      return labelled(result.stdout, deps.runnerId).map(({ key, build, owned }) => ({ name: key, build, owned }));
+    },
+    async removeContainer(id) {
+      const result = await run("docker", ["rm", "--force", "--volumes", id]);
+      if (result.status !== 0) throw failure(result, `docker rm ${id}`);
+    },
+    async removeVolume(name) {
+      const result = await run("docker", ["volume", "rm", "--force", name]);
+      if (result.status !== 0) throw failure(result, `docker volume rm ${name}`);
+    },
+    async imageState(ref) {
+      const result = await run("docker", ["image", "inspect", "--format", "{{.Id}}", ref]);
+      if (result.status === 0) return "present";
+      return /no such image|not found/i.test(result.stderr) ? "missing" : "unknown";
+    },
+    async listDanglingImages() {
+      const listed = await run("docker", [
+        "image",
+        "ls",
+        "--all",
+        "--no-trunc",
+        "--quiet",
+        "--filter",
+        "dangling=true",
+        "--filter",
+        `label=${KIND_LABEL}`
+      ]);
+      if (listed.status !== 0) throw failure(listed, "docker image ls");
+      const ids = [...new Set(listed.stdout.split("\n").map((l) => l.trim()).filter(Boolean))];
+      if (ids.length === 0) return [];
+      const inspected = await run("docker", ["image", "inspect", ...ids]);
+      if (inspected.status !== 0) throw failure(inspected, "docker image inspect");
+      return JSON.parse(inspected.stdout).map((row) => ({
+        id: row.Id,
+        size: row.Size ?? 0,
+        labels: row.Config?.Labels ?? {}
+      }));
+    },
+    async removeImage(id) {
+      const result = await run("docker", ["rmi", id]);
+      if (result.status !== 0) throw failure(result, `docker rmi ${id}`);
+    },
+    async builderPrune(keepBytes) {
+      const result = await run("docker", ["builder", "prune", "--force", "--keep-storage", String(keepBytes)]);
+      if (result.status !== 0) throw failure(result, "docker builder prune");
+      return { freedBytes: parsePrunedBytes(result.stdout) };
+    }
+  };
+}
+
+// src/runner/housekeeping/select.ts
+var DAY_MS = 864e5;
+function spoolMaxAgeMs() {
+  return 24 * 36e5;
+}
+function selectJobDirs(entries, nowMs, keepDays) {
+  return entries.filter(
+    (entry) => (entry.state === "finished" || entry.state === "no_job_file") && nowMs - entry.mtimeMs > keepDays * DAY_MS
+  );
+}
+function liveBuildIds(jobs) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const job of jobs) {
+    for (const id of [job.container?.buildId, job.microvm?.buildId, job.build?.buildId]) if (id !== void 0) ids.add(id);
+  }
+  return ids;
+}
+function finishedBuildIds(jobs) {
+  return liveBuildIds(jobs);
+}
+function liveImageIds(jobs) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const job of jobs) {
+    for (const id of [job.container?.imageId, job.microvm?.imageId]) if (id !== void 0) ids.add(id);
+  }
+  return ids;
+}
+function selectOrphans(items, live, finished) {
+  return items.filter((item) => !live.has(item.build) && (item.owned || finished.has(item.build)));
+}
+function rootDiskImageId(fileName) {
+  if (!fileName.endsWith(".ext4") || fileName.includes(".partial-")) return void 0;
+  const id = fileName.slice(0, -".ext4".length);
+  return id === "" ? void 0 : id;
+}
+function selectSpool(files, nowMs, buildRunning) {
+  return buildRunning ? [] : files.filter((file) => nowMs - file.mtimeMs > spoolMaxAgeMs());
+}
+
+// src/runner/housekeeping/run.ts
+function sizeOf(path) {
+  try {
+    const stats = lstatSync(path);
+    if (!stats.isDirectory()) return stats.size;
+    return readdirSync(path).reduce((sum, name) => sum + sizeOf(join(path, name)), 0);
+  } catch {
+    return 0;
+  }
+}
+function listDir(path) {
+  try {
+    return readdirSync(path);
+  } catch {
+    return [];
+  }
+}
+function mtimeOf(path) {
+  try {
+    return statSync(path).mtimeMs;
+  } catch {
+    return void 0;
+  }
+}
+function readJobs(jobsPath) {
+  const entries = [];
+  const unfinished = [];
+  const finished = [];
+  const unreadable = [];
+  for (const dirent of existsSync(jobsPath) ? readdirSync(jobsPath, { withFileTypes: true }) : []) {
+    if (!dirent.isDirectory()) continue;
+    const dirPath = join(jobsPath, dirent.name);
+    const mtimeMs = mtimeOf(dirPath);
+    if (mtimeMs === void 0) continue;
+    const read = readJobFileChecked(join(dirPath, "job.json"));
+    let state;
+    if (read.kind === "missing") {
+      state = "no_job_file";
+    } else if (read.kind === "unreadable") {
+      unreadable.push(dirent.name);
+      state = "unreadable";
+    } else if (isUnfinished(read.job)) {
+      unfinished.push(read.job);
+      state = "unfinished";
+    } else {
+      finished.push(read.job);
+      state = read.job.releaseFailed === void 0 || read.job.releaseFailed === ALREADY_RELEASED ? "finished" : "release_failed";
+    }
+    entries.push({ name: dirent.name, mtimeMs, state });
+  }
+  return { entries, unfinished, finished, unreadable };
+}
+async function removeDanglingImages(docker, log, live) {
+  let freed = 0;
+  for (const image of await docker.listDanglingImages()) {
+    if (image.labels[KIND_LABEL] === void 0) continue;
+    if (live.has(image.id)) continue;
+    try {
+      await docker.removeImage(image.id);
+      freed += image.size;
+      log.line(`housekeeping: removed dangling image ${image.id.replace(/^sha256:/, "").slice(0, 12)} (${formatBytes(image.size)}), a rebuild replaced it`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove dangling image ${image.id}: ${describeCallFailure(err)}`);
+    }
+  }
+  return freed;
+}
+async function runHousekeeping(options) {
+  const { docker, log } = options;
+  const nowMs = options.now().getTime();
+  const root = runnerDir(options.configPath);
+  let freed = 0;
+  let listed;
+  try {
+    const containers = await docker.listBuildContainers();
+    listed = { containers, volumes: await docker.listBuildVolumes() };
+  } catch (err) {
+    log.error(`housekeeping: Docker could not be asked, so nothing in Docker is removed this pass: ${describeCallFailure(err)}`);
+  }
+  const jobs = readJobs(jobsDir(options.configPath));
+  for (const name of jobs.unreadable) {
+    log.error(`housekeeping: job file ${name}/job.json does not parse, so nothing in Docker is removed this pass`);
+  }
+  const canTell = jobs.unreadable.length === 0;
+  for (const entry of selectJobDirs(jobs.entries, nowMs, options.keepDays)) {
+    const path = join(jobsDir(options.configPath), entry.name);
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { recursive: true, force: true });
+      freed += size;
+      log.line(`housekeeping: removed job directory ${entry.name} (${formatBytes(size)}), last changed ${formatDuration(nowMs - entry.mtimeMs)} ago`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove job directory ${entry.name}: ${describeCallFailure(err)}`);
+    }
+  }
+  const live = liveBuildIds(jobs.unfinished);
+  const finished = finishedBuildIds(jobs.finished);
+  if (canTell && listed !== void 0) {
+    for (const container of selectOrphans(listed.containers, live, finished)) {
+      try {
+        await docker.removeContainer(container.id);
+        log.line(`housekeeping: removed container ${container.id.slice(0, 12)}, no unfinished build ${container.build} names it`);
+      } catch (err) {
+        log.error(`housekeeping: could not remove container ${container.id}: ${describeCallFailure(err)}`);
+      }
+    }
+    for (const volume of selectOrphans(listed.volumes, live, finished)) {
+      try {
+        await docker.removeVolume(volume.name);
+        log.line(`housekeeping: removed volume ${volume.name}, no unfinished build ${volume.build} names it`);
+      } catch (err) {
+        log.error(`housekeeping: could not remove volume ${volume.name}: ${describeCallFailure(err)}`);
+      }
+    }
+  }
+  if (canTell && listed !== void 0) {
+    freed += await removeStaleRootDisks(root, jobs.unfinished, options);
+    try {
+      freed += await removeDanglingImages(docker, log, liveImageIds(jobs.unfinished));
+    } catch (err) {
+      log.error(`housekeeping: dangling images could not be listed: ${describeCallFailure(err)}`);
+    }
+    if (options.buildCacheMax !== "off") {
+      try {
+        const pruned = await docker.builderPrune(options.buildCacheMax);
+        freed += pruned.freedBytes;
+        log.line(`housekeeping: pruned the build cache down to ${formatBytes(options.buildCacheMax)}, freed ${formatBytes(pruned.freedBytes)}`);
+      } catch (err) {
+        log.error(`housekeeping: the build cache could not be pruned: ${describeCallFailure(err)}`);
+      }
+    }
+  }
+  const spoolPath = spoolDir(root);
+  const spoolFiles = listDir(spoolPath).flatMap((name) => {
+    const mtimeMs = mtimeOf(join(spoolPath, name));
+    return mtimeMs === void 0 ? [] : [{ name, mtimeMs }];
+  });
+  for (const file of selectSpool(spoolFiles, nowMs, jobs.unfinished.length > 0)) {
+    const path = join(spoolPath, file.name);
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { recursive: true, force: true });
+      freed += size;
+      log.line(`housekeeping: removed spool file ${file.name} (${formatBytes(size)}), last changed ${formatDuration(nowMs - file.mtimeMs)} ago and no build is running`);
+    } catch (err) {
+      log.error(`housekeeping: could not remove spool file ${file.name}: ${describeCallFailure(err)}`);
+    }
+  }
+  log.line(`housekeeping: pass finished, freed ${formatBytes(freed)}`);
+  return { freedBytes: freed };
+}
+async function removeStaleRootDisks(root, unfinished, options) {
+  const dir = rootfsDir(root);
+  const inUse = new Set(unfinished.flatMap((job) => job.microvm?.rootfs === void 0 ? [] : [job.microvm.rootfs]));
+  let freed = 0;
+  for (const name of listDir(dir)) {
+    const imageId = rootDiskImageId(name);
+    if (imageId === void 0) continue;
+    const path = join(dir, name);
+    if (inUse.has(path)) continue;
+    if (await options.docker.imageState(`sha256:${imageId}`) !== "missing") continue;
+    const size = sizeOf(path);
+    try {
+      rmSync(path, { force: true });
+      freed += size;
+      options.log.line(`housekeeping: removed root disk ${name} (${formatBytes(size)}), Docker no longer has its image`);
+    } catch (err) {
+      options.log.error(`housekeeping: could not remove root disk ${name}: ${describeCallFailure(err)}`);
+    }
+  }
+  return freed;
+}
+
 // src/runner/loop.ts
 var QUEUE_ORDER = ["amend", "implement"];
 var TRACKER_REPOSITORY_HOSTS = {
   github: "github.com",
   gitlab: "gitlab.com"
 };
+var HOUSEKEEPING_INTERVAL_MS = 36e5;
 function logOf(state) {
   state.deps.log ??= createRunnerLog(state.deps.now ?? (() => /* @__PURE__ */ new Date()));
   return state.deps.log;
@@ -645,7 +963,8 @@ async function sendTickHeartbeat(state, workRequest) {
           slotsUsed: state.jobs.size,
           slotsTotal: state.config.concurrency,
           workRequest: next,
-          lastUpdate: readLastUpdate(state.config.configPath)
+          lastUpdate: readLastUpdate(state.config.configPath),
+          lowDisk: state.lowDisk
         },
         state.deps.fetchImpl
       );
@@ -769,9 +1088,64 @@ async function tick(state) {
   }
   await sendTickHeartbeat(state, state.lastWorkRequest);
 }
+function holdOffForDisk(state) {
+  const minimum = state.config.housekeeping?.minFreeDisk ?? 0;
+  const root = runnerDir(state.config.configPath);
+  const free = minimum === 0 ? Number.POSITIVE_INFINITY : (state.deps.freeBytes ?? freeBytes)(root) ?? Number.POSITIVE_INFINITY;
+  const log = logOf(state);
+  if (free < minimum) {
+    if (state.lowDisk === void 0) {
+      log.line(
+        `holding off new work: ${formatBytes(free)} free on the runner's file system, ${formatBytes(minimum)} needed (runner.housekeeping.minFreeDisk) \xB7 claiming again once there is room`
+      );
+      recordHousekeeping(state, root, { holding: { free, needed: minimum, since: nowOf(state).toISOString() } });
+    }
+    state.lowDisk = { free, needed: minimum };
+    return true;
+  }
+  if (state.lowDisk !== void 0) {
+    state.lowDisk = void 0;
+    log.line("free disk is back above the minimum \xB7 taking work again");
+    recordHousekeeping(state, root, { holding: void 0 });
+  } else if (!state.holdRecordChecked) {
+    if (readHousekeepingRecord(root).holding !== void 0) recordHousekeeping(state, root, { holding: void 0 });
+  }
+  state.holdRecordChecked = true;
+  return false;
+}
+function recordHousekeeping(state, root, patch) {
+  try {
+    writeHousekeepingRecord(root, patch);
+  } catch (err) {
+    logOf(state).error(`could not write housekeeping.json: ${describeCallFailure(err)}`);
+  }
+}
+async function housekeepingPass(state) {
+  const config = state.config.housekeeping;
+  if (config === void 0 || state.housekeepingRunning) return;
+  state.housekeepingRunning = true;
+  try {
+    state.deps.housekeeping ??= {};
+    state.deps.housekeeping.docker ??= createHousekeepingDocker({ runnerId: runnerLabelValue(state.config.configPath) });
+    const result = await runHousekeeping({
+      configPath: state.config.configPath,
+      keepDays: config.jobsKeepDays,
+      buildCacheMax: config.buildCacheMax,
+      docker: state.deps.housekeeping.docker,
+      now: state.deps.now ?? (() => /* @__PURE__ */ new Date()),
+      log: logOf(state)
+    });
+    recordHousekeeping(state, runnerDir(state.config.configPath), { lastRunAt: nowOf(state).toISOString(), freedBytes: result.freedBytes });
+  } catch (err) {
+    logOf(state).error(`housekeeping failed: ${describeCallFailure(err)}`);
+  } finally {
+    state.housekeepingRunning = false;
+  }
+}
 async function runTick(state) {
   reloadConfig(state);
   if (await updateWhenIdle(state)) return;
+  if (holdOffForDisk(state)) return;
   const log = logOf(state);
   const repoFilter = new RepoFilter(state.config.repos);
   let filtered = 0;
@@ -825,7 +1199,7 @@ var CLAIM_REFUSAL_CODES = new Set(
   "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous branch_claimed_elsewhere".split(" ")
 );
 async function tryClaimAndStart(state, item, queue) {
-  const { postJson: postJson2 } = await import('./work-cmd-QAUZ7MTD.js');
+  const { postJson: postJson2 } = await import('./work-cmd-NUR4CAG6.js');
   const log = logOf(state);
   const hostKey = await loadHostKey(state.config.configPath);
   const claim = await retrySend(
@@ -1525,7 +1899,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const outcome = await onStartUnderMarker({
       configPath,
       own: ownVersionOf(deps),
-      check: update.check ? () => update.check(configPath) : async () => (await import('./check-7MFH6HNC.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
+      check: update.check ? () => update.check(configPath) : async () => (await import('./check-4Q33B5ZE.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
       now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
       log
     });
@@ -1606,6 +1980,12 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   let interrupts = 0;
   state.pollTimerSeconds = state.config.pollSeconds;
   const pollTimer = setInterval(() => void tick(state), state.pollTimerSeconds * 1e3);
+  const hooksKind = builtinKind(config.hooks.provision);
+  const housekeepingTimer = hooksKind === "container" || hooksKind === "vm" ? (() => {
+    const everyMs = deps.housekeeping?.intervalMs ?? HOUSEKEEPING_INTERVAL_MS;
+    void housekeepingPass(state);
+    return setInterval(() => void housekeepingPass(state), everyMs);
+  })() : void 0;
   const watchTimer = setInterval(() => {
     const now = nowMs();
     for (const active of state.jobs.values()) {
@@ -1639,6 +2019,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     if (intakeClosed) return;
     intakeClosed = true;
     clearInterval(pollTimer);
+    clearInterval(housekeepingTimer);
   };
   const close = () => {
     stopIntake();

package/dist/{chunk-TWNWNS2Q.js → chunk-H4CK2YK4.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { isLinkedRepository } from './chunk-SY4GK4SP.js';
+import { isLinkedRepository } from './chunk-D2KWEAN6.js';
 import { readStatuslineState, STATE_FRESH_MS } from './chunk-F47JBIAW.js';
-import { resolveCredentials, getJson, WorkCommandError } from './chunk-W5FVUZXV.js';
-import { sanitizeTerminal } from './chunk-WFJ65NUD.js';
-import { resolveRepoIdentity, compareVersions, SUPPORTED_VERSIONS } from './chunk-4C4THABW.js';
+import { resolveCredentials, getJson, WorkCommandError } from './chunk-RMAZ2NWA.js';
+import { sanitizeTerminal } from './chunk-WOBNRVQT.js';
+import { resolveRepoIdentity, compareVersions, SUPPORTED_VERSIONS } from './chunk-EX6M5JPR.js';
 import { C } from './chunk-YRYMNBGQ.js';
 import { readSettings, settingsPath, isOurStatuslineSlot, statuslineWrappedOriginal, REGISTERED_EVENTS, isOurHookCommand, ourCommandBins, cliBinResolvable, ourDirectScriptBins, wrappedOriginalCommand } from './chunk-5QRP3MVS.js';
 import { readPin, readContextManifest, readDegraded, readSourceCache } from './chunk-WIOZAJ2W.js';
@@ -1004,7 +1004,7 @@ function doctorExitCode(checks) {
 
 // src/doctor-report.ts
 async function gatherDoctorChecks() {
-  const { defaultRunnerConfigPath, hasRunnerBlock, loadRunnerConfig, runnerVmOf } = await import('./config-XVJYZ4IQ.js');
+  const { defaultRunnerConfigPath, hasRunnerBlock, loadRunnerConfig, runnerVmOf } = await import('./config-HL2AIR3K.js');
   const configPath = defaultRunnerConfigPath();
   const checks = runDoctor({ runnerHost: hasRunnerBlock(configPath) });
   const credential = await credentialChecks();
@@ -1016,7 +1016,7 @@ async function gatherDoctorChecks() {
     const { runnerVmCheck } = await import('./doctor-53F2PYY7.js');
     checks.push(runnerVmCheck(configPath, { lima: realLima(), home: homeFromEnv(), instance: runnerVm.instance }));
   } else if (hasRunnerBlock(configPath)) {
-    const { runnerHostChecks } = await import('./doctor-ZLGZYMDK.js');
+    const { runnerHostChecks } = await import('./doctor-MOA3RIEP.js');
     const hooks = (() => {
       try {
         return loadRunnerConfig(configPath).hooks;
@@ -1041,8 +1041,8 @@ async function gatherDoctorChecks() {
     })();
     const runner = (() => {
       try {
-        const { concurrency, build } = loadRunnerConfig(configPath);
-        return { concurrency, buildMemory: build?.memory ?? "8g" };
+        const { concurrency, build, housekeeping } = loadRunnerConfig(configPath);
+        return { concurrency, buildMemory: build?.memory ?? "8g", minFreeDisk: housekeeping?.minFreeDisk };
       } catch {
         return void 0;
       }

package/dist/{chunk-7WERLFVR.js → chunk-JLDZTG3A.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { detectActiveSession } from './chunk-BFESLKPP.js';
-import { resolveRepoIdentity } from './chunk-4C4THABW.js';
+import { resolveRepoIdentity } from './chunk-EX6M5JPR.js';
 import { parseTrackerNumber, trackerNumberError } from './chunk-WZBHBH3J.js';
 import { defaultFetch } from './chunk-BCLJSQCV.js';
 import { unauthorizedMessage, resolveGatewayCredentials } from './chunk-BOLTI6LR.js';

package/dist/{chunk-Y7KJLXYN.js → chunk-OR6AEEDO.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { dashboardBase, isRepositoryLinked, workPageUrl, notLinkedText } from './chunk-SY4GK4SP.js';
-import { sanitizeTerminal } from './chunk-WFJ65NUD.js';
-import { remoteLookupFailed } from './chunk-4C4THABW.js';
+import { dashboardBase, isRepositoryLinked, workPageUrl, notLinkedText } from './chunk-D2KWEAN6.js';
+import { sanitizeTerminal } from './chunk-WOBNRVQT.js';
+import { remoteLookupFailed } from './chunk-EX6M5JPR.js';
 import { resolveGatewayCredentials } from './chunk-BOLTI6LR.js';
 import { findGitDirs, readRepoConnection, writeRepoConnection } from './chunk-BJ3KTMDH.js';
 import { createInterface } from 'readline/promises';

package/dist/{chunk-W5FVUZXV.js → chunk-RMAZ2NWA.js}

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { detectActiveSession } from './chunk-BFESLKPP.js';
-import { sanitizeTerminal } from './chunk-WFJ65NUD.js';
-import { resolveRepoIdentity } from './chunk-4C4THABW.js';
+import { sanitizeTerminal } from './chunk-WOBNRVQT.js';
+import { resolveRepoIdentity } from './chunk-EX6M5JPR.js';
 import { parseTrackerNumber, trackerNumberError } from './chunk-WZBHBH3J.js';
 import { defaultFetch } from './chunk-BCLJSQCV.js';
 import { resolveGatewayCredentials, unauthorizedMessage } from './chunk-BOLTI6LR.js';

package/dist/{chunk-3TFPDVTI.js → chunk-RTXKYLCA.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { MIN_ENGINE_MAJOR } from './chunk-HZRNMGLF.js';
-import { firecrackerArch, MICROVM_ARCHES, guestAssetsPinned } from './chunk-ZNA27WEV.js';
+import { MIN_ENGINE_MAJOR } from './chunk-T52PKE2Y.js';
+import { firecrackerArch, MICROVM_ARCHES, guestAssetsPinned } from './chunk-Z62HNKEJ.js';
 import { spawnSync } from 'child_process';
 import { existsSync, accessSync, constants, statSync, readFileSync } from 'fs';