@wardby/cli 0.5.3 → 0.7.0

package/dist/providers/jobs/kubernetes.d.ts

@@ -1,9 +1,10 @@
 import { type Readable } from "node:stream";
 import type { KubernetesJobConfig } from "../../config/providers.js";
+import { type Power } from "../../core/power.js";
 import { type KubernetesApi } from "./kubernetes-api.js";
 import { type ServiceStateReporter } from "./service-state.js";
 import type { CodingProvider } from "../../coding/provider.js";
-import type { JobHandle, JobResult, JobSpec, JobStatus, WorkspaceJobLauncher } from "./types.js";
+import { type JobHandle, type JobResult, type JobSpec, type JobStatus, type WorkspaceJobLauncher } from "./types.js";
 export interface KubernetesClusterInfo {
     /** The proxy Service's ClusterIP: the enforcement witness's address (8787 reachable, 8788 not). */
     proxyIp: string;
@@ -19,6 +20,11 @@ export interface KubernetesJobLauncherOptions {
      * the gate probes the address `provision` re-reads for that launch, not this memoized one.
      */
     preflight?: () => Promise<KubernetesClusterInfo | void>;
+    /**
+     * Low-power mode: `launch()` wakes the (possibly scaled-to-zero) coding proxy before its preflight,
+     * and `warmUp()` defers the preflight while the proxy is asleep. Default `DISABLED_POWER` (no-op).
+     */
+    power?: Power;
     now?: () => number;
     sleep?: (milliseconds: number) => Promise<void>;
     readyTimeoutMs?: number;
@@ -67,6 +73,7 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
     /** Timeout of one streak exec: ENFORCEMENT_BLOCKED_STREAK probes plus the gaps between them. */
     private readonly streakExecTimeoutMs;
     private preflightResult?;
+    private readonly power;
     constructor(options: KubernetesJobLauncherOptions);
     /**
      * Services run as native sidecars in the run pod (kubernetes-isolation.ts), for either provider:
@@ -94,7 +101,35 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
     materializeWorkspace(handle: JobHandle, destination: string): Promise<void>;
     stop(handle: JobHandle, _reason?: string): Promise<void>;
     remove(handle: JobHandle): Promise<void>;
-    /** Memoized: the preflight (or, without one, a single proxy-witness read) runs once per launcher. */
+    /**
+     * Low-power mode: whether the proxy is scaled to zero. A failed check is logged by code only (the
+     * leading reason code, else the error's class name -- never the raw message, which could carry
+     * cluster detail) and counts as awake, so warm-up runs the preflight as it would without the mode.
+     */
+    private proxyAsleep;
+    /**
+     * Starts the memoized cluster preflight (`runPreflight`) right away, instead of waiting for the
+     * first `launch()`, so a long-running server process's own start-up pays the preflight's cost (a
+     * canary pod on `kind`: ~20s) rather than that process's first coding run. Never throws: it only
+     * reports the outcome (an info line on success, a warning naming the failure code otherwise) so
+     * an operator sees a misconfigured cluster at boot. Any `launch()` already awaiting this same
+     * attempt (concurrent with this call) still fails exactly as it does today when the preflight
+     * failed — it shares the same rejected promise. But unlike a preflight `launch()` starts on its
+     * own, a start-up failure here is **not** left stuck memoized: a transient cluster problem at
+     * process start must not fail every coding run until restart, so the failed attempt is cleared
+     * (only if nothing has already started a newer one — compared by identity, never by state) and
+     * the *next* `launch()` runs the preflight fresh. A successful warm-up stays memoized as always.
+     * In low-power mode, while the proxy is asleep, warm-up is skipped (the memo stays unset) rather
+     * than waking the proxy only for the preflight; the first `launch()` wakes it and runs the preflight.
+     */
+    warmUp(): Promise<void>;
+    /**
+     * Memoized: the preflight (or, without one, a single proxy-witness read) runs once per launcher —
+     * including its failure, which stays memoized and fails every subsequent `launch()` immediately,
+     * for a preflight `launch()` itself started (there was no `warmUp()`, or `warmUp()`'s own failed
+     * attempt was already cleared). `warmUp()` is the one caller that clears a failed attempt instead
+     * of leaving it stuck; see its own doc comment.
+     */
     private runPreflight;
     /**
      * Waits until the run's NetworkPolicy is enforced on this pod: the keeper (same network namespace
@@ -130,6 +165,10 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
      * must not open the gate at the initial proof either.
      */
     private waitForPolicyEnforcement;
+    /** True when `found` is a requeued record for this spec, which launch() replaces with a fresh one. */
+    private isRequeued;
+    /** Tells the operator once that a quota refused a pod while no quota is configured for the pre-launch check. */
+    private hintQuotaUnconfigured;
     private existingLaunch;
     private provision;
     private createIfMissing;
@@ -143,6 +182,7 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
     private refresh;
     /** Reads only a fixed diagnostic code from a failed worker's bounded log tail; never keeps anything else. */
     private readWorkerDiagnostic;
+    readTranscript(handle: JobHandle, maxBytes: number): Promise<string | undefined>;
     private readResultArtifact;
     /**
      * Streams the keeper's workspace through the strict extractor into `staging`.

package/dist/providers/jobs/kubernetes.js

@@ -29,10 +29,11 @@ import { finished } from "node:stream/promises";
 import { createHash } from "node:crypto";
 import { isIP } from "node:net";
 import { logger } from "../../core/logger.js";
-import { MAX_CODING_ARTIFACT_BYTES, MAX_DEBUG_TRACE_LINE_BYTES, parseCodingAgentOutputJson, } from "../../coding/protocol.js";
+import { DISABLED_POWER, wakeTimeoutMs } from "../../core/power.js";
+import { MAX_CODING_ARTIFACT_BYTES, MAX_CODING_INPUT_BYTES, MAX_DEBUG_TRACE_LINE_BYTES, parseCodingWorkerOutputJson, } from "../../coding/protocol.js";
 import { CODING_PROXY_DENY_PORT, CODING_PROXY_PORT } from "./docker-isolation.js";
 import { parseWorkerDiagnosticLine } from "./docker.js";
-import { KubernetesAlreadyExistsError, KubernetesConflictError } from "./kubernetes-api.js";
+import { KubernetesAlreadyExistsError, KubernetesConflictError, KubernetesQuotaExceededError, } from "./kubernetes-api.js";
 import { ENFORCEMENT_PROBE_DENY_REACHABLE, ENFORCEMENT_PROBE_PROVEN, ENFORCEMENT_PROBE_DENY_REFUSED, ENFORCEMENT_PROBE_PROXY_UNREACHABLE, KEEPER_CONTAINER, KEEPER_SEEDED_MARKER, KUBERNETES_ISOLATION_ERROR, STORAGE_ROOT, TOOL_RUNNER_CONTAINER, WORKER_CONTAINER, assertRunNetworkPolicyMatches, assertRunPodMatches, buildCapabilitySecret, buildRunNetworkPolicy, buildRunPod, enforcementStreakScript, kubernetesRunNames, kubernetesRunNamesForToken, runLabels, serviceContainerName, validateKubernetesSpec, } from "./kubernetes-isolation.js";
 import { claudeToolSetup } from "./claude-tool-setup.js";
 import { podQuotaUsage, quotaShortfall } from "./kubernetes-quota.js";
@@ -40,6 +41,7 @@ import { readProxyWitness } from "./kubernetes-witness.js";
 import { safeExtract } from "./safe-extract.js";
 import { reportServiceState, } from "./service-state.js";
 import { serviceUnreadyError } from "../../coding/services/wording.js";
+import { JobCapacityError, } from "./types.js";
 import { replaceDirectoryFromStaging } from "./workspace-swap.js";
 import { collectExclusions, normalizeCollectExclusions, tarExcludeArgs, } from "../../coding/collect-exclude.js";
 const BACKEND = "kubernetes";
@@ -183,10 +185,22 @@ const TOKEN = /^[a-f0-9]{20}$/;
 const WORKSPACE_STORAGE = `${STORAGE_ROOT}/workspace`;
 const INPUT_STORAGE = `${STORAGE_ROOT}/input`;
 const RESULT_PATH = `${STORAGE_ROOT}/output/result.json`;
+const TRANSCRIPT_PATH = `${STORAGE_ROOT}/output/session.jsonl`;
 /** Stands in for the proxy ClusterIP when building a pod only to count its resources. */
 const QUOTA_PROBE_PROXY_IP = "10.0.0.1";
+/** A Kubernetes object name (RFC 1123 subdomain). */
+const DNS_1123_NAME = /^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/;
 const kubernetesLog = logger.child({ module: "kubernetes-jobs" });
-const PHASES = new Set(["provisioning", "active", "succeeded", "failed", "stopped", "lost", "removed"]);
+const PHASES = new Set([
+    "provisioning",
+    "requeued",
+    "active",
+    "succeeded",
+    "failed",
+    "stopped",
+    "lost",
+    "removed",
+]);
 const TERMINAL_PHASES = new Set(["succeeded", "failed", "stopped", "lost", "removed"]);
 // ---------------------------------------------------------------------------
 // Pure helpers: the lifecycle rules.
@@ -205,6 +219,8 @@ function statusFor(record) {
     switch (record.phase) {
         case "provisioning":
             return { state: "pending" };
+        case "requeued":
+            return { state: "pending", capacityRefused: true };
         case "active":
             return { state: "running" };
         case "succeeded":
@@ -386,9 +402,11 @@ export class KubernetesJobLauncher {
     /** Timeout of one streak exec: ENFORCEMENT_BLOCKED_STREAK probes plus the gaps between them. */
     streakExecTimeoutMs;
     preflightResult;
+    power;
     constructor(options) {
         this.options = options;
         this.api = options.api;
+        this.power = options.power ?? DISABLED_POWER;
         this.namespace = options.config.namespace;
         this.workspaceRoot = resolve(options.workspaceRoot);
         if (this.workspaceRoot === resolve("/"))
@@ -495,13 +513,19 @@ export class KubernetesJobLauncher {
                 this.warn(`kubernetes_spec_rejected: ${error.message}`);
             throw error;
         }
+        // Low-power mode: the proxy may be scaled to zero; wake it before the preflight reads it. Guarded
+        // (not just a no-op await) so mode off keeps launch() joining an in-flight warm-up synchronously.
+        if (this.power.enabled) {
+            await this.power.ensureAwake("proxy", { timeoutMs: wakeTimeoutMs(this.readyTimeoutMs), runId: spec.runId });
+        }
         await this.runPreflight();
         const names = kubernetesRunNames(spec.runId);
         const handle = { backend: BACKEND, id: `${this.namespace}/${names.token}` };
         const specHash = stableSpecHash(spec);
         const existing = await this.readRecord(names);
-        if (existing)
+        if (existing && !this.isRequeued(existing, specHash)) {
             return this.existingLaunch(existing.record, specHash, handle);
+        }
         const createdAt = this.now();
         const record = {
             schemaVersion: RECORD_SCHEMA_VERSION,
@@ -513,22 +537,54 @@ export class KubernetesJobLauncher {
             phase: "provisioning",
             collectExclude: spec.collectExclude ?? collectExclusions([]),
         };
-        try {
-            // The record is the idempotency fence before any other cluster side effect.
-            await this.api.createConfigMap(this.namespace, this.recordConfigMap(names, record));
+        if (existing) {
+            // Replaces the requeued record; a conflict means another replica got there first.
+            try {
+                await this.api.replaceConfigMap(this.namespace, names.record, this.recordConfigMap(names, record, existing.resourceVersion));
+            }
+            catch (error) {
+                if (!(error instanceof KubernetesConflictError))
+                    throw error;
+                const raced = await this.readRecord(names);
+                if (!raced)
+                    throw error;
+                if (!this.isRequeued(raced, specHash))
+                    return this.existingLaunch(raced.record, specHash, handle);
+                // Requeued again by the other replica (its attempt was refused too): start over from that record.
+                return this.launch(spec);
+            }
         }
-        catch (error) {
-            if (!(error instanceof KubernetesAlreadyExistsError))
-                throw error;
-            const raced = await this.readRecord(names);
-            if (!raced)
-                throw error;
-            return this.existingLaunch(raced.record, specHash, handle);
+        else {
+            try {
+                // The record is the idempotency fence before any other cluster side effect. A quota refusing
+                // this create itself (a count/configmaps quota) is not a capacity refusal: it fails the run.
+                await this.api.createConfigMap(this.namespace, this.recordConfigMap(names, record));
+            }
+            catch (error) {
+                if (!(error instanceof KubernetesAlreadyExistsError))
+                    throw error;
+                const raced = await this.readRecord(names);
+                if (!raced)
+                    throw error;
+                if (!this.isRequeued(raced, specHash))
+                    return this.existingLaunch(raced.record, specHash, handle);
+                // Another replica requeued it between our read and create: start over from the requeued record.
+                return this.launch(spec);
+            }
         }
         try {
             await this.provision(spec, names, record);
         }
         catch (error) {
+            if (error instanceof KubernetesQuotaExceededError) {
+                // Capacity, not failure: leave a record a later launch() provisions afresh.
+                await this.updateRecord(names, () => ({ phase: "requeued" })).catch(() => undefined);
+                await this.cleanupRun(names, 0);
+                this.hintQuotaUnconfigured(error.quotaName);
+                throw new JobCapacityError("kubernetes_quota_exceeded: the namespace ResourceQuota has no room for this run", {
+                    cause: error,
+                });
+            }
             // Record the failure before deleting the pod, so a replica that observes "no pod" in between
             // can't record `lost` first and have this more specific outcome dropped.
             await this.updateRecord(names, () => ({
@@ -618,7 +674,69 @@ export class KubernetesJobLauncher {
     }
     // -------------------------------------------------------------------------
     // Launch
-    /** Memoized: the preflight (or, without one, a single proxy-witness read) runs once per launcher. */
+    /**
+     * Low-power mode: whether the proxy is scaled to zero. A failed check is logged by code only (the
+     * leading reason code, else the error's class name -- never the raw message, which could carry
+     * cluster detail) and counts as awake, so warm-up runs the preflight as it would without the mode.
+     */
+    async proxyAsleep() {
+        try {
+            return await this.power.isAsleep("proxy");
+        }
+        catch (error) {
+            const code = (error instanceof Error && /^[a-z][a-z0-9_]*(?=:)/.exec(error.message)?.[0]) ||
+                (error instanceof Error ? error.name : "unknown");
+            this.warn(`kubernetes_preflight_power_check_failed: ${code}`);
+            return false;
+        }
+    }
+    /**
+     * Starts the memoized cluster preflight (`runPreflight`) right away, instead of waiting for the
+     * first `launch()`, so a long-running server process's own start-up pays the preflight's cost (a
+     * canary pod on `kind`: ~20s) rather than that process's first coding run. Never throws: it only
+     * reports the outcome (an info line on success, a warning naming the failure code otherwise) so
+     * an operator sees a misconfigured cluster at boot. Any `launch()` already awaiting this same
+     * attempt (concurrent with this call) still fails exactly as it does today when the preflight
+     * failed — it shares the same rejected promise. But unlike a preflight `launch()` starts on its
+     * own, a start-up failure here is **not** left stuck memoized: a transient cluster problem at
+     * process start must not fail every coding run until restart, so the failed attempt is cleared
+     * (only if nothing has already started a newer one — compared by identity, never by state) and
+     * the *next* `launch()` runs the preflight fresh. A successful warm-up stays memoized as always.
+     * In low-power mode, while the proxy is asleep, warm-up is skipped (the memo stays unset) rather
+     * than waking the proxy only for the preflight; the first `launch()` wakes it and runs the preflight.
+     */
+    async warmUp() {
+        if (this.power.enabled && (await this.proxyAsleep())) {
+            kubernetesLog.info("kubernetes_preflight_deferred: proxy asleep (low-power mode); the first coding run runs the preflight");
+            return;
+        }
+        const attempt = this.runPreflight();
+        try {
+            await attempt;
+            kubernetesLog.info("kubernetes_preflight_warm_up_succeeded: cluster preflight passed at start-up");
+        }
+        catch (error) {
+            // Only clear the memo if it still holds this exact attempt: a concurrent launch() or warmUp()
+            // may have already seen this same rejection and started a fresh one (which must survive), and
+            // nothing else could have replaced this attempt with anything but a newer one, since
+            // runPreflight() never overwrites an already-set, still-pending-or-resolved promise.
+            if (this.preflightResult === attempt)
+                this.preflightResult = undefined;
+            // Same code-only convention as every other failure this launcher logs (e.g. launch()'s
+            // kubernetes_spec_rejected above): the error's own message is the fixed reason code set by
+            // runPreflight's errorWithCode (KUBERNETES_ISOLATION_ERROR) — never the raw cause, which could
+            // carry cluster detail.
+            const code = error instanceof Error ? error.message : String(error);
+            this.warn(`kubernetes_preflight_warm_up_failed: ${code}`);
+        }
+    }
+    /**
+     * Memoized: the preflight (or, without one, a single proxy-witness read) runs once per launcher —
+     * including its failure, which stays memoized and fails every subsequent `launch()` immediately,
+     * for a preflight `launch()` itself started (there was no `warmUp()`, or `warmUp()`'s own failed
+     * attempt was already cleared). `warmUp()` is the one caller that clears a failed attempt instead
+     * of leaving it stuck; see its own doc comment.
+     */
     runPreflight() {
         this.preflightResult ??= (async () => {
             try {
@@ -688,6 +806,22 @@ export class KubernetesJobLauncher {
             await this.sleep(ENFORCEMENT_POLL_MS);
         }
     }
+    /** True when `found` is a requeued record for this spec, which launch() replaces with a fresh one. */
+    isRequeued(found, specHash) {
+        if (found.record.phase !== "requeued")
+            return false;
+        if (found.record.specHash !== specHash)
+            throw new Error("job_spec_conflict");
+        return true;
+    }
+    /** Tells the operator once that a quota refused a pod while no quota is configured for the pre-launch check. */
+    hintQuotaUnconfigured(refusedQuota) {
+        if (this.options.config.resourceQuota)
+            return;
+        const name = refusedQuota !== undefined && DNS_1123_NAME.test(refusedQuota) ? refusedQuota : undefined;
+        this.warnOnce(`kubernetes_resource_quota_unconfigured: a pod was refused by ${name ? `ResourceQuota ${name}` : "a ResourceQuota"}; ` +
+            `set KUBERNETES_RESOURCE_QUOTA=${name ?? "<name>"} and grant the launcher get on that ResourceQuota so runs wait before they start`);
+    }
     existingLaunch(record, specHash, handle) {
         if (record.specHash !== specHash)
             throw new Error("job_spec_conflict");
@@ -862,14 +996,14 @@ export class KubernetesJobLauncher {
         const staging = await mkdtemp(join(tmpdir(), "wardby-k8s-input-"));
         try {
             const source = await lstat(inputArtifact);
-            if (!source.isFile() || source.size > MAX_CODING_ARTIFACT_BYTES) {
+            if (!source.isFile() || source.size > MAX_CODING_INPUT_BYTES) {
                 throw new Error("kubernetes_input_artifact_invalid");
             }
             const copy = join(staging, "input.json");
             await copyFile(inputArtifact, copy);
             // Re-check the copy: the source could have changed between lstat and copy.
             const copied = await lstat(copy);
-            if (!copied.isFile() || copied.size > MAX_CODING_ARTIFACT_BYTES) {
+            if (!copied.isFile() || copied.size > MAX_CODING_INPUT_BYTES) {
                 throw new Error("kubernetes_input_artifact_invalid");
             }
             await this.seedDirectory(names, staging, INPUT_STORAGE, timeoutMs);
@@ -887,7 +1021,8 @@ export class KubernetesJobLauncher {
     // Observation and collection
     /** Observes a non-terminal run against its pod and persists any transition; returns the stored record. */
     async refresh(names, record) {
-        if (isTerminal(record.phase))
+        // A requeued run has no pod by design; observing it would record it lost.
+        if (isTerminal(record.phase) || record.phase === "requeued")
             return record;
         const pod = await this.api.readPod(this.namespace, names.pod);
         const observed = observePod(pod, record, this.now(), this.readyTimeoutMs);
@@ -915,6 +1050,30 @@ export class KubernetesJobLauncher {
         }
         return undefined;
     }
+    async readTranscript(handle, maxBytes) {
+        const names = this.namesFor(handle);
+        const found = names && (await this.readRecord(names));
+        if (!names || !found)
+            return undefined;
+        const collector = new BoundedCollector(maxBytes);
+        let exitCode;
+        try {
+            exitCode = await this.api.exec(this.namespace, names.pod, KEEPER_CONTAINER, ["head", "-c", String(maxBytes + 1), TRANSCRIPT_PATH], { stdout: collector, timeoutMs: SHORT_EXEC_TIMEOUT_MS });
+            if (!collector.writableEnded)
+                collector.end();
+            await finished(collector);
+        }
+        catch {
+            // The pod is gone (a stopped run) or unreachable: there is nothing to collect.
+            collector.destroy();
+            return undefined;
+        }
+        if (exitCode !== 0)
+            return undefined;
+        if (collector.exceeded)
+            throw new Error("transcript_too_large");
+        return collector.text();
+    }
     async readResultArtifact(names, runId) {
         const collector = new BoundedCollector(MAX_CODING_ARTIFACT_BYTES);
         let raw;
@@ -935,7 +1094,7 @@ export class KubernetesJobLauncher {
         }
         let output;
         try {
-            output = parseCodingAgentOutputJson(raw);
+            output = parseCodingWorkerOutputJson(raw);
         }
         catch (error) {
             throw errorWithCode("kubernetes_result_artifact_invalid", error);

package/dist/providers/jobs/metadata-probes.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Cloud metadata endpoints a coding or native worker pod must never reach: they hand out node or
+ * workload credentials. The instance metadata service (GCE, EC2 IPv4), the EKS Pod Identity agent (IPv4
+ * and IPv6), and EC2's IPv6 IMDS endpoint. Probed on port 80, where every one of them serves. A pod without
+ * IPv6 fails the last connect immediately, which counts as unreachable — correct either way.
+ * src/sandbox/fetch-policy.ts blocks the same addresses for in-process fetches.
+ */
+export declare const CLOUD_METADATA_PROBES: readonly {
+    host: string;
+    port: number;
+}[];

package/dist/providers/jobs/metadata-probes.js

@@ -0,0 +1,13 @@
+/**
+ * Cloud metadata endpoints a coding or native worker pod must never reach: they hand out node or
+ * workload credentials. The instance metadata service (GCE, EC2 IPv4), the EKS Pod Identity agent (IPv4
+ * and IPv6), and EC2's IPv6 IMDS endpoint. Probed on port 80, where every one of them serves. A pod without
+ * IPv6 fails the last connect immediately, which counts as unreachable — correct either way.
+ * src/sandbox/fetch-policy.ts blocks the same addresses for in-process fetches.
+ */
+export const CLOUD_METADATA_PROBES = Object.freeze([
+    Object.freeze({ host: "169.254.169.254", port: 80 }),
+    Object.freeze({ host: "169.254.170.23", port: 80 }),
+    Object.freeze({ host: "fd00:ec2::254", port: 80 }),
+    Object.freeze({ host: "fd00:ec2::23", port: 80 }),
+]);

package/dist/providers/jobs/types.d.ts

@@ -21,6 +21,14 @@ export interface JobSpec {
     /** Required only for Claude's credential-free repository tool container. */
     toolImage?: string;
     inputArtifact: string;
+    /** Set only when the run's dependency cache is on (CodingRun.dependencyCache); absent otherwise, never false. */
+    dependencyCache?: true;
+    /**
+     * Claude Code only, set only when the run keeps its session (CodingRun.sessionResume); absent otherwise,
+     * never false. The worker's SDK config dir, where the session file grows, gets its own size-limited
+     * scratch so a long session never fills the worker's shared /tmp.
+     */
+    sessionResume?: true;
     timeoutSec: number;
     limits: JobResourceLimits;
     labels: Record<string, string>;
@@ -42,6 +50,12 @@ export interface JobHandle {
 }
 export type JobStatus = {
     state: "pending" | "running";
+    /**
+     * Only ever on `pending`: the backend refused this job's launch for capacity (JobCapacityError)
+     * and nothing of it is running, nor will it start on its own. A caller that finds its job so
+     * may return the run to its queue and later `launch` the same spec again.
+     */
+    capacityRefused?: true;
 } | {
     state: "succeeded";
 } | {
@@ -61,11 +75,24 @@ export interface JobResult {
     /** Output-schema failure locations (`path:code`) that passed SAFE_CODING_OUTPUT_ISSUE; never values. */
     diagnosticIssues?: string[];
 }
+/**
+ * The backend has no room for this job right now (Kubernetes: the namespace ResourceQuota). No
+ * workload of the job is left running; the backend may keep a record of the refusal (Kubernetes
+ * does: its run record, which `status` then reports as `pending` with `capacityRefused`). The
+ * caller may launch the same spec again later.
+ */
+export declare class JobCapacityError extends Error {
+    readonly code = "coding_capacity_exceeded";
+    constructor(message: string, options?: ErrorOptions);
+}
 /**
  * Lifecycle invariants implemented by every launcher:
  *
  * - `launch` is idempotent by runId for an identical spec and rejects a
  *   different spec for the same runId.
+ * - `launch` may reject with JobCapacityError when the backend has no room for
+ *   the job; nothing of it is then left running, and a later `launch` of the
+ *   same spec provisions it afresh.
  * - pending may become running or terminal; running may become terminal;
  *   terminal states never change.
  * - `collect` is repeatable for a terminal job until `remove` succeeds.
@@ -88,9 +115,24 @@ export interface JobLauncher {
      * itself stays the authority.
      */
     hasCapacityFor?: (spec: JobSpec) => Promise<boolean>;
+    /**
+     * Starts any one-time start-up check (e.g. Kubernetes's memoized cluster preflight) right away,
+     * instead of waiting for the first `launch()`. Never throws — a failure is only logged — and
+     * `launch()` still runs and reports the same check itself, memoized exactly as if this had never
+     * been called. Composition roots call this, fire-and-forget, only from a long-running server
+     * process's own start-up (`wardby serve`, the MCP server, `wardby scheduler`); never from a
+     * one-shot CLI command. Absent on a launcher with nothing to warm up (e.g. Docker).
+     */
+    warmUp?: () => Promise<void>;
     launch: (spec: JobSpec) => Promise<JobHandle>;
     status: (handle: JobHandle) => Promise<JobStatus>;
     collect: (handle: JobHandle) => Promise<JobResult>;
+    /**
+     * The run's session transcript file (the worker's session.jsonl), at most `maxBytes`; undefined
+     * when there is none or the job's storage is gone (a stopped job). Throws `transcript_too_large`
+     * when larger. Never cached in the job record: it is not redacted yet. Call before `remove`.
+     */
+    readTranscript?: (handle: JobHandle, maxBytes: number) => Promise<string | undefined>;
     stop: (handle: JobHandle, reason?: string) => Promise<void>;
     remove: (handle: JobHandle) => Promise<void>;
 }

package/dist/providers/jobs/types.js

@@ -3,4 +3,16 @@
  * values are carried in a versioned input artifact, never expanded into an
  * arbitrary environment map or shell command.
  */
-export {};
+/**
+ * The backend has no room for this job right now (Kubernetes: the namespace ResourceQuota). No
+ * workload of the job is left running; the backend may keep a record of the refusal (Kubernetes
+ * does: its run record, which `status` then reports as `pending` with `capacityRefused`). The
+ * caller may launch the same spec again later.
+ */
+export class JobCapacityError extends Error {
+    code = "coding_capacity_exceeded";
+    constructor(message, options) {
+        super(message, options);
+        this.name = "JobCapacityError";
+    }
+}

package/dist/providers/llm/anthropic.js

@@ -6,6 +6,7 @@
  */
 import Anthropic from "@anthropic-ai/sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
+import { simulatorModelClientOptions } from "../../config/test-simulators.js";
 export function anthropicCredentialsPresent(env = process.env) {
     return Boolean(env.ANTHROPIC_API_KEY);
 }
@@ -14,6 +15,9 @@ export class AnthropicLlmProvider extends ClaudeLlmProvider {
         if (!client && !apiKey) {
             throw new Error("ANTHROPIC_API_KEY is not set — required by the Anthropic LlmProvider adapter.");
         }
-        super(client ?? new Anthropic({ apiKey }), lookup);
+        // WARDBY_TEST_SIMULATORS (test only): the scripted model, with a placeholder key the
+        // credential-free fetch drops; the real key never goes there.
+        const simulator = client ? null : simulatorModelClientOptions("anthropic");
+        super(client ?? new Anthropic(simulator ?? { apiKey }), lookup);
     }
 }

package/dist/providers/llm/bedrock.d.ts

@@ -1,15 +1,3 @@
-/**
- * Bedrock-Claude adapter for the LlmProvider seam. `@anthropic-ai/bedrock-sdk`
- * is imported only here; all protocol/pricing-dispatch logic lives in the
- * shared ClaudeLlmProvider base class (claude-provider.ts) — AnthropicBedrock
- * exposes the identical .messages.stream() surface as @anthropic-ai/sdk, so
- * this file's only job is constructing the right client.
- *
- * Named BedrockClaudeLlmProvider, not BedrockLlmProvider: Bedrock hosts
- * multiple model vendors, and this adapter only ever speaks the
- * Anthropic-native protocol for Claude models. A future non-Claude Bedrock
- * adapter gets its own name, not a rename of this one.
- */
 import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
 import type { CatalogLookup } from "./catalog-lookup.js";

package/dist/providers/llm/bedrock.js

@@ -10,6 +10,7 @@
  * Anthropic-native protocol for Claude models. A future non-Claude Bedrock
  * adapter gets its own name, not a rename of this one.
  */
+import { refuseBedrockUnderTestSimulators } from "../../config/test-simulators.js";
 import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
 export function bedrockCredentialsPresent(env = process.env) {
@@ -23,6 +24,9 @@ export class BedrockClaudeLlmProvider extends ClaudeLlmProvider {
         if (!client && !region) {
             throw new Error("BEDROCK_REGION (or AWS_REGION) is required by the Bedrock Claude LlmProvider adapter.");
         }
+        // WARDBY_TEST_SIMULATORS (test only) refuses Bedrock: AWS-signed calls cannot go to the simulator.
+        if (!client)
+            refuseBedrockUnderTestSimulators();
         super(client ?? new AnthropicBedrock({ awsRegion: region }), lookup);
     }
 }

package/dist/providers/llm/claude-messages.d.ts

@@ -106,5 +106,7 @@ export type ClaudeStreamEvent = {
 } | {
     type: "message_stop";
 };
-export declare function mapClaudeStream(events: AsyncIterable<ClaudeStreamEvent>, priceUsd: (usage: LlmUsage) => number): AsyncIterable<LlmStreamEvent>;
+export declare function mapClaudeStream(events: AsyncIterable<ClaudeStreamEvent>, priceUsd: (usage: LlmUsage) => number, 
+/** Called at message_start: the response has started, and output may be billed from here on. */
+onResponseStarted?: () => void): AsyncIterable<LlmStreamEvent>;
 export {};

package/dist/providers/llm/claude-messages.js

@@ -98,7 +98,9 @@ export function withCacheBreakpoints(req) {
     });
     return { ...req, ...(system ? { system } : {}), messages };
 }
-export async function* mapClaudeStream(events, priceUsd) {
+export async function* mapClaudeStream(events, priceUsd, 
+/** Called at message_start: the response has started, and output may be billed from here on. */
+onResponseStarted) {
     const toolBuffers = new Map();
     let start = {};
     let outputTokens = 0;
@@ -106,6 +108,7 @@ export async function* mapClaudeStream(events, priceUsd) {
     for await (const ev of events) {
         switch (ev.type) {
             case "message_start":
+                onResponseStarted?.();
                 start = ev.message.usage ?? {};
                 outputTokens = ev.message.usage?.output_tokens ?? 0;
                 break;

package/dist/providers/llm/claude-provider.d.ts

@@ -5,7 +5,7 @@
  * stays exactly what each concrete adapter (anthropic.ts, bedrock.ts) has in
  * common, and nothing more.
  */
-import type { LlmMessage, LlmRequest, LlmStreamEvent, LlmToolDef } from "./types.js";
+import type { LlmMessage, LlmRequest, LlmStreamEvent, LlmStreamHooks, LlmToolDef } from "./types.js";
 import { type CatalogLookup } from "./catalog-lookup.js";
 import type { CatalogEntry } from "./catalog-types.js";
 import type { CatalogLlmAdapter } from "./routing.js";
@@ -27,7 +27,7 @@ export declare class ClaudeLlmProvider implements CatalogLlmAdapter {
     private readonly lookup;
     constructor(client: ClaudeMessagesClient, lookup?: CatalogLookup);
     withEntry(entry: CatalogEntry): ClaudeLlmProvider;
-    stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable<LlmStreamEvent>;
+    stream(req: LlmRequest, signal?: AbortSignal, hooks?: LlmStreamHooks): AsyncIterable<LlmStreamEvent>;
     countTokens(model: string, messages: LlmMessage[], tools?: LlmToolDef[]): Promise<number>;
     priceUsd(model: string, usage: {
         inputTokens: number;

package/dist/providers/llm/claude-provider.js

@@ -21,14 +21,14 @@ export class ClaudeLlmProvider {
     withEntry(entry) {
         return new ClaudeLlmProvider(this.client, pinnedLookup(entry));
     }
-    async *stream(req, signal) {
+    async *stream(req, signal, hooks) {
         // Defence in depth behind config-time validation: a level the model
         // doesn't accept would fail the whole call, so it is dropped here.
         const { effort, ...rest } = req;
         const accepted = effort && this.lookup(req.model).efforts.includes(effort);
         const claudeReq = withCacheBreakpoints(toClaudeRequest(accepted ? { ...rest, effort } : rest, DEFAULT_MAX_TOKENS));
         const raw = this.client.messages.stream({ model: req.model, ...claudeReq }, { signal });
-        yield* mapClaudeStream(raw, (usage) => this.priceUsd(req.model, usage));
+        yield* mapClaudeStream(raw, (usage) => this.priceUsd(req.model, usage), hooks?.onResponseStarted);
     }
     async countTokens(model, messages, tools) {
         this.lookup(model); // fail-closed on a model the catalog does not serve