@outerlayer/cli 0.5.1 → 0.7.0

package/dist/{chunk-F4NU4H3V.js → chunk-AII7CKTH.js}

@@ -1,33 +1,79 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
-import { LEASE_SECONDS, releaseLease, isUnfinished, writeJobFile, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, readJobFile, hookLimitsFor, readJobFileChecked, ALREADY_RELEASED } from './chunk-UWH5DSUP.js';
-import { RUNNER_PROTOCOL } from './chunk-ZXBITXTS.js';
-import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-UGP76IZ5.js';
 import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-ABIWDUMS.js';
-import { readMarker, currentVersion, removeMarker, writeMarker, entryIn, versionDir, switchCurrent, cleanVersions, isInLayout, startedThroughCurrent, stagingDir } from './chunk-YYYXRJUW.js';
+import { readMarker, currentVersion, removeMarker, writeMarker, entryIn, versionDir, switchCurrent, cleanVersions, isInLayout, startedThroughCurrent, stagingDir } from './chunk-TI7B7UEO.js';
+import { LEASE_SECONDS, releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, provisionEnvironment, readProvisionOut, resolveHostBuild, workdirProblem, listenOnBroker, hostEnvironment, hostCommandEnvironment, execArgv, microVmCommandLine, runCommand, execCommandLine, startSender, startHostServices, finishJob, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, hookLimitsFor, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, ALREADY_RELEASED } from './chunk-AJKEHABI.js';
+import { writeJobFile, readJobFile, readJobFileChecked } from './chunk-T6OIJQRT.js';
+import { RUNNER_PROTOCOL } from './chunk-AIW6GLET.js';
+import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-UGP76IZ5.js';
+import { placeholderQueues, placeholderProblem } from './chunk-6IYLDODX.js';
 import { freeBytes, formatBytes } from './chunk-YYYRBYWG.js';
 import { RepoFilter } from './chunk-TBT347UY.js';
-import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, formatClock, measureDisk, createRunnerLog, ensureTunnel, keepGhConfigCurrent, createContainerRuntime, runnerLabelValue, BUILD_LABEL, RUNNER_LABEL } from './chunk-T52PKE2Y.js';
+import { checkFit } from './chunk-OUUINEYB.js';
+import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, provisionContainer, lifecycleSteps, lifecycleFailureReason, prepareBranch, containerCommandLine, BuildTokens, classifyRefusal, refusalCode, measureDisk, createRunnerLog, setUpRunnerCgroup, ensureTunnel, keepGhConfigCurrent, createContainerRuntime, formatClock, runnerLabelValue, BUILD_LABEL, RUNNER_LABEL } from './chunk-E4PIFIC3.js';
 import { startTimeOf, claimPidFile, readPidFile, isSameProcess, releasePidFile } from './chunk-N5FOE5PS.js';
 import { spawnRun } from './chunk-774HEQL6.js';
-import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-RMAZ2NWA.js';
-import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind } from './chunk-3DD6AOWC.js';
+import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-DGAIVMMN.js';
+import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind } from './chunk-DTXOJDEH.js';
 import { exceedsDisk } from './chunk-F6GFZXZ3.js';
-import { parseClaimIdentity, sharedUserEnvironment, JOB_MOUNT } from './chunk-AUFG23AA.js';
+import { sharedUserEnvironment, JOB_MOUNT, claudeCredential } from './chunk-OD2VTIHN.js';
+import { parseClaimIdentity } from './chunk-HKMPVPO7.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
 import { loadHostKey, ensureHostKey, signerFor } from './chunk-M5POMKKW.js';
 import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, jobsDir, pidFilePath, spoolDir, RUNNER_FILE_MODE, rootfsDir } from './chunk-ZM2IMMYN.js';
 import { HOOKS_DIR_NAME, writeCommitHooks } from './chunk-XCCLFVXM.js';
 import { randomBytes } from 'crypto';
-import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync, lstatSync } from 'fs';
+import { mkdirSync, existsSync, chmodSync, statSync, readdirSync, readFileSync, rmSync, renameSync, writeFileSync, closeSync, openSync, constants, lstatSync } from 'fs';
 import { join, dirname } from 'path';
+import { totalmem } from 'os';
 
 createRequire(import.meta.url);
+function openKvm() {
+  try {
+    closeSync(openSync("/dev/kvm", constants.O_RDWR));
+    return "usable";
+  } catch (err) {
+    return err.code === "ENOENT" ? "absent" : "no-permission";
+  }
+}
+function realBlockedProbes() {
+  return { env: process.env, kvm: openKvm, totalMemoryBytes: totalmem };
+}
+var DEFAULT_BUILD_MEMORY = "8g";
+function checkBlocked(config, probes) {
+  const placeholders = placeholderQueues(config);
+  if (placeholders.length > 0) return { reason: `${placeholderProblem(placeholders)}. The runner takes work again once the config is edited` };
+  const kind = builtinKind(config.hooks.provision);
+  if (kind === "container" || kind === "vm") {
+    const fit = checkFit({
+      concurrency: config.concurrency,
+      buildMemory: config.build?.memory ?? DEFAULT_BUILD_MEMORY,
+      usableBytes: probes.totalMemoryBytes()
+    });
+    if (!fit.fits) return { reason: `${fit.message}. Lower runner.concurrency or runner.build.memory` };
+  }
+  const kvm = kind === "vm" ? probes.kvm() : "usable";
+  if (kvm === "absent") {
+    return { reason: "/dev/kvm does not exist on this machine, so builtin:vm cannot start a microVM. Use a host with KVM enabled, or builtin:container hooks" };
+  }
+  if (kvm !== "usable") {
+    return {
+      reason: "the runner cannot open /dev/kvm, so builtin:vm cannot start a microVM. Add the runner's account to the kvm group, then restart the service, because a service reads its groups only when it starts"
+    };
+  }
+  if (kind !== "own" && claudeCredential(probes.env) === void 0) {
+    return {
+      reason: "there is no Claude credential in the runner's environment. Put CLAUDE_CODE_OAUTH_TOKEN (from `claude setup-token`) or ANTHROPIC_API_KEY in the runner.env beside the config, then restart the service"
+    };
+  }
+  return void 0;
+}
 
 // src/runner/heartbeat.ts
 var HEARTBEAT_TIMEOUT_MS = 1e4;
 var ERROR_CODE_MAX = 100;
 var ERROR_MESSAGE_MAX = 500;
+var BLOCKED_REASON_MAX = 500;
 var UNREACHABLE_MESSAGE = "the request did not reach the platform";
 function noPlatformMessage(status) {
   return `the request got HTTP ${status} with no error message from the platform`;
@@ -73,7 +119,9 @@ async function sendHeartbeat(args, fetchImpl) {
       ...args.lastUpdate.at !== void 0 ? { at: args.lastUpdate.at } : {}
     },
     // Free disk on the runner's file system, in bytes. Sent only while held, so a host with room sends nothing about its disk.
-    lowDisk: args.lowDisk === void 0 ? void 0 : { free: args.lowDisk.free, needed: args.lowDisk.needed }
+    lowDisk: args.lowDisk === void 0 ? void 0 : { free: args.lowDisk.free, needed: args.lowDisk.needed },
+    // Why the runner claims nothing. Sent only while it holds, so a runner that takes work says nothing about it.
+    blocked: args.blocked === void 0 ? void 0 : { reason: cutForGateway(args.blocked.reason, BLOCKED_REASON_MAX) }
   };
   try {
     const response = await postJson(
@@ -643,7 +691,7 @@ function selectJobDirs(entries, nowMs, keepDays) {
 function liveBuildIds(jobs) {
   const ids = /* @__PURE__ */ new Set();
   for (const job of jobs) {
-    for (const id of [job.container?.buildId, job.microvm?.buildId, job.build?.buildId]) if (id !== void 0) ids.add(id);
+    for (const id of [job.container?.buildId, job.microvm?.buildId, job.build?.buildId, job.checkContainer?.buildId]) if (id !== void 0) ids.add(id);
   }
   return ids;
 }
@@ -964,7 +1012,8 @@ async function sendTickHeartbeat(state, workRequest) {
           slotsTotal: state.config.concurrency,
           workRequest: next,
           lastUpdate: readLastUpdate(state.config.configPath),
-          lowDisk: state.lowDisk
+          lowDisk: state.lowDisk,
+          blocked: state.blocked
         },
         state.deps.fetchImpl
       );
@@ -1088,6 +1137,25 @@ async function tick(state) {
   }
   await sendTickHeartbeat(state, state.lastWorkRequest);
 }
+function holdWhenBlocked(state) {
+  const probes = {
+    ...realBlockedProbes(),
+    ...state.deps.container?.hostEnv === void 0 ? {} : { env: state.deps.container.hostEnv },
+    ...state.deps.blocked
+  };
+  const blocked = checkBlocked(state.config, probes);
+  const log = logOf(state);
+  if (blocked !== void 0) {
+    if (state.blocked?.reason !== blocked.reason) log.line(`claiming nothing: ${blocked.reason}`);
+    state.blocked = blocked;
+    return true;
+  }
+  if (state.blocked !== void 0) {
+    state.blocked = void 0;
+    log.line("the runner can run its work again \xB7 taking work");
+  }
+  return false;
+}
 function holdOffForDisk(state) {
   const minimum = state.config.housekeeping?.minFreeDisk ?? 0;
   const root = runnerDir(state.config.configPath);
@@ -1145,6 +1213,7 @@ async function housekeepingPass(state) {
 async function runTick(state) {
   reloadConfig(state);
   if (await updateWhenIdle(state)) return;
+  if (holdWhenBlocked(state)) return;
   if (holdOffForDisk(state)) return;
   const log = logOf(state);
   const repoFilter = new RepoFilter(state.config.repos);
@@ -1199,7 +1268,7 @@ var CLAIM_REFUSAL_CODES = new Set(
   "pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous branch_claimed_elsewhere".split(" ")
 );
 async function tryClaimAndStart(state, item, queue) {
-  const { postJson: postJson2 } = await import('./work-cmd-NUR4CAG6.js');
+  const { postJson: postJson2 } = await import('./work-cmd-6UMCFTAW.js');
   const log = logOf(state);
   const hostKey = await loadHostKey(state.config.configPath);
   const claim = await retrySend(
@@ -1327,6 +1396,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity)
     itemKey,
     branch,
     commitIdentity: identity,
+    ...item.specRepository === void 0 || item.specRepository === null || item.specRepository === "" ? {} : { repository: item.specRepository },
     startedAt: startedAt.toISOString()
   };
   writeJobFile(jobFilePath, job);
@@ -1678,6 +1748,17 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity)
   writeJobFile(jobFilePath, job);
   const commandStartedMs = nowOf(state).getTime();
   log.line(`${tag} command started \xB7 pid ${running.pid}`);
+  const sender = await startSender({
+    config: state.config,
+    job,
+    jobDir: dirPath,
+    deps: { spawnFn: state.deps.spawnFn, runtime: job.microvm !== void 0 || job.container !== void 0 ? runtimeOf(state) : void 0, log }
+  });
+  if (sender !== void 0) {
+    job = { ...active.job, sender };
+    active.job = job;
+    writeJobFile(jobFilePath, job);
+  }
   running.onExit.then(({ code, signal, error }) => {
     if (active.ending) return;
     if (error !== void 0) {
@@ -1785,8 +1866,39 @@ async function endJob(state, active, outcome, ending) {
     state.jobs.delete(active.dirPath);
   }
 }
+async function renewLeaseIfDue(state, active, nowMs) {
+  const log = logOf(state);
+  const tag = `#${active.job.item}`;
+  if (!needsRenew(active.lastRenewAtMs, nowMs)) return true;
+  const renew = await retrySend(() => renewLease(state.config, active.job.item, state.deps.fetchImpl), {
+    now: state.deps.now,
+    sleep: state.deps.sleep
+  });
+  const result = renew.ok ? renew.value : "retry_later";
+  if (result === "lease_lost") {
+    if (active.ending) {
+      log.error(`${tag} lease lost while the checks ran`);
+      return false;
+    }
+    log.error(`${tag} lease lost \xB7 ending job as lease_lost`);
+    await endJob(state, active, "lease_lost");
+    return false;
+  }
+  if (result === "renewed") {
+    active.job.lastRenewAt = new Date(nowMs).toISOString();
+    writeJobFile(join(active.dirPath, "job.json"), active.job);
+    log.line(`${tag} lease renewed \xB7 expires ${formatClock(new Date(nowMs + LEASE_SECONDS * 1e3), new Date(nowMs))}`);
+  } else {
+    log.error(`${tag} renew did not land \xB7 retrying in ${RENEW_INTERVAL_SECONDS / 60}m`);
+  }
+  active.lastRenewAtMs = nowMs;
+  return true;
+}
 async function watchTick(state, active, nowMs, logMtimeMs) {
-  if (active.ending) return;
+  if (active.ending) {
+    if (active.job.stage === "checks") await renewLeaseIfDue(state, active, nowMs);
+    return;
+  }
   const log = logOf(state);
   const tag = `#${active.job.item}`;
   const pid = active.job.pid;
@@ -1804,26 +1916,7 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
     }
     return;
   }
-  if (needsRenew(active.lastRenewAtMs, nowMs)) {
-    const renew = await retrySend(() => renewLease(state.config, active.job.item, state.deps.fetchImpl), {
-      now: state.deps.now,
-      sleep: state.deps.sleep
-    });
-    const result = renew.ok ? renew.value : "retry_later";
-    if (result === "lease_lost") {
-      log.error(`${tag} lease lost \xB7 ending job as lease_lost`);
-      await endJob(state, active, "lease_lost");
-      return;
-    }
-    if (result === "renewed") {
-      active.job.lastRenewAt = new Date(nowMs).toISOString();
-      writeJobFile(join(active.dirPath, "job.json"), active.job);
-      log.line(`${tag} lease renewed \xB7 expires ${formatClock(new Date(nowMs + LEASE_SECONDS * 1e3), new Date(nowMs))}`);
-    } else {
-      log.error(`${tag} renew did not land \xB7 retrying in ${RENEW_INTERVAL_SECONDS / 60}m`);
-    }
-    active.lastRenewAtMs = nowMs;
-  }
+  if (!await renewLeaseIfDue(state, active, nowMs)) return;
   const tokens = active.job.container === void 0 && active.job.microvm === void 0 ? void 0 : runtimeOf(state).credentials.get(active.dirPath);
   if (tokens !== void 0) {
     await tokens.renewReadIfDue();
@@ -1899,7 +1992,7 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
     const outcome = await onStartUnderMarker({
       configPath,
       own: ownVersionOf(deps),
-      check: update.check ? () => update.check(configPath) : async () => (await import('./check-4Q33B5ZE.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
+      check: update.check ? () => update.check(configPath) : async () => (await import('./check-ASZL5BEO.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl }),
       now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
       log
     });
@@ -1918,6 +2011,11 @@ async function start(configPath, deps = { fetchImpl: fetch }) {
   mkdirSync(jobsDir(configPath), { recursive: true, mode: RUNNER_DIR_MODE });
   const self = { pid: process.pid, start: startTimeOf(process.pid) ?? "" };
   claimPidFile(pidFilePath(configPath), self);
+  const { delegated, refusal } = setUpRunnerCgroup();
+  if (refusal !== void 0) log.error(`the kernel refused a cgroup step for the runner's service: ${refusal}`);
+  if (isBuiltinVm(config.hooks.provision) && !delegated) {
+    log.line("microVM builds run with no memory limit of their own: the runner's service is not delegated its cgroup (Delegate=yes in the unit, cgroup v2)");
+  }
   let configMtimeMs;
   try {
     configMtimeMs = statSync(configPath).mtimeMs;

package/dist/{chunk-ZXBITXTS.js → chunk-AIW6GLET.js}

@@ -4,6 +4,6 @@ import { createRequire } from 'node:module';
 createRequire(import.meta.url);
 
 // src/runner/protocol.ts
-var RUNNER_PROTOCOL = 13;
+var RUNNER_PROTOCOL = 14;
 
 export { RUNNER_PROTOCOL };