@wardby/cli 0.6.0 → 0.8.0

package/dist/providers/jobs/kubernetes-api.js

@@ -11,6 +11,16 @@ export class KubernetesForbiddenError extends Error {
         this.name = "KubernetesForbiddenError";
     }
 }
+/** A create the namespace ResourceQuota refused (HTTP 403 "exceeded quota"). */
+export class KubernetesQuotaExceededError extends Error {
+    /** The refusing quota's name as the API server worded it; unvalidated, and nothing else of the refusal is kept. */
+    quotaName;
+    constructor(message, options) {
+        super(message, options);
+        this.name = "KubernetesQuotaExceededError";
+        this.quotaName = options?.quotaName;
+    }
+}
 /** Stale resourceVersion on replace. */
 export class KubernetesConflictError extends Error {
     constructor(message) {

package/dist/providers/jobs/kubernetes-client.js

@@ -5,7 +5,7 @@
  */
 import { ApiException, AppsV1Api, AuthorizationV1Api, CoreV1Api, Exec, KubeConfig, NetworkingV1Api, PatchStrategy, setHeaderOptions, VersionApi, } from "@kubernetes/client-node";
 import { Writable } from "node:stream";
-import { KubernetesAlreadyExistsError, KubernetesConflictError, KubernetesForbiddenError, KubernetesNotFoundError, } from "./kubernetes-api.js";
+import { KubernetesAlreadyExistsError, KubernetesConflictError, KubernetesForbiddenError, KubernetesNotFoundError, KubernetesQuotaExceededError, } from "./kubernetes-api.js";
 function statusCode(error) {
     return error instanceof ApiException ? error.code : undefined;
 }
@@ -44,6 +44,11 @@ async function deleteIgnoringMissing(remove) {
         throw error;
     }
 }
+/** The error's message plus its body as text, for matching only; never copied into a thrown message. */
+function bodyText(error) {
+    const { body, message } = error;
+    return `${message}\n${typeof body === "string" ? body : JSON.stringify(body ?? "")}`;
+}
 async function create(what, write) {
     try {
         return await write();
@@ -54,6 +59,10 @@ async function create(what, write) {
             throw new KubernetesAlreadyExistsError(what);
         if (code === 404)
             throw new KubernetesNotFoundError(what);
+        if (code === 403 && /exceeded quota/i.test(bodyText(error))) {
+            const quotaName = /exceeded quota: ([^\s,"\\]+)/i.exec(bodyText(error))?.[1];
+            throw new KubernetesQuotaExceededError(`exceeded quota: ${what}`, { cause: error, quotaName });
+        }
         throw error;
     }
 }

package/dist/providers/jobs/kubernetes-isolation.d.ts

@@ -1,6 +1,7 @@
 import type { V1Container, V1NetworkPolicy, V1Pod, V1Secret } from "@kubernetes/client-node";
 import { type ResolvedCodingService } from "../../coding/services/catalog.js";
 import type { JobSpec } from "./types.js";
+import { CLAUDE_CONFIG_PATH, CLAUDE_CONFIG_SCRATCH_MIB } from "./docker-isolation.js";
 import { type KubernetesPlatform } from "./kubernetes-platform.js";
 export declare const KUBERNETES_ISOLATION_ERROR = "kubernetes_isolation_unsupported";
 /** Extra seconds past timeoutSec before Kubernetes kills the pod (keeper included). */
@@ -12,6 +13,8 @@ export declare const WORKER_CONTAINER = "worker";
 export declare const TOOL_RUNNER_CONTAINER = "tool-runner";
 /** The run Secret's key holding the tool runner's setup (claude-tool-setup.ts). */
 export declare const TOOL_SETUP_SECRET_KEY = "tool-setup";
+/** The run Secret's key for the MCP token (a run with MCP tools only); the agent container alone reads it. */
+export declare const MCP_TOKEN_SECRET_KEY = "mcp-token";
 /** Where the tool runner listens and the agent's relay connects (claude-tool-runner/main.mjs). */
 export declare const TOOL_SOCKET_PATH = "/run/wardby/tool/runner.sock";
 /** Every service sidecar's container name starts with this (service-postgres, service-redis, ...). */
@@ -51,6 +54,7 @@ export declare const ENFORCEMENT_PROBE_PROXY_UNREACHABLE = 4;
  * blocking the path — the port is reachable-but-unserved and witnesses nothing.
  */
 export declare const ENFORCEMENT_PROBE_DENY_REFUSED = 5;
+export { CLAUDE_CONFIG_PATH, CLAUDE_CONFIG_SCRATCH_MIB };
 export interface KubernetesRunNames {
     token: string;
     runSha: string;
@@ -80,7 +84,7 @@ export interface RunPodOptions {
 }
 export declare function buildRunPod(spec: JobSpec, options: RunPodOptions): V1Pod;
 export declare function buildRunNetworkPolicy(spec: JobSpec, namespace: string): V1NetworkPolicy;
-export declare function buildCapabilitySecret(spec: JobSpec, namespace: string, capability: string, toolSetup?: string): V1Secret;
+export declare function buildCapabilitySecret(spec: JobSpec, namespace: string, capability: string, toolSetup?: string, mcpToken?: string): V1Secret;
 /** A copy of a container's resources with every quantity in one rendering ("1" and "1000m" alike); see normalizeResources. */
 export declare function canonicalResources(r?: V1Container["resources"]): V1Container["resources"];
 /**

package/dist/providers/jobs/kubernetes-isolation.js

@@ -7,7 +7,8 @@
 import { createHash } from "node:crypto";
 import { isIP } from "node:net";
 import { StoredCodingServicesSchema } from "../../coding/services/catalog.js";
-import { CLAUDE_TOOL_SETUP_ENV, CODING_PROXY_ALIAS, CODING_PROXY_DENY_PORT, CODING_PROXY_PORT, CODING_WORKER_GID, CODING_WORKER_UID, WORKER_STOP_GRACE_SECONDS, claudeToolLimits, isRepositoryDigest, } from "./docker-isolation.js";
+import { MCP_TOKEN_ENV } from "../../coding/mcp-token.js";
+import { CLAUDE_CONFIG_PATH, CLAUDE_CONFIG_SCRATCH_MIB, CLAUDE_TOOL_SETUP_ENV, CODING_PROXY_ALIAS, CODING_PROXY_DENY_PORT, CODING_PROXY_PORT, CODING_WORKER_GID, CODING_WORKER_UID, WORKER_STOP_GRACE_SECONDS, claudeToolLimits, isRepositoryDigest, } from "./docker-isolation.js";
 import { GVISOR_RUNTIME_CLASS, KubernetesPlatformError, STORAGE_INIT_EPHEMERAL_MIB, TOOL_RUNNER_EPHEMERAL_MIB, WORKER_EPHEMERAL_MIB, conformResources, describeMib, normalizePlatformMetadata, platformProfile, podEphemeralStorageMib, } from "./kubernetes-platform.js";
 export const KUBERNETES_ISOLATION_ERROR = "kubernetes_isolation_unsupported";
 /** Extra seconds past timeoutSec before Kubernetes kills the pod (keeper included). */
@@ -19,6 +20,8 @@ export const WORKER_CONTAINER = "worker";
 export const TOOL_RUNNER_CONTAINER = "tool-runner";
 /** The run Secret's key holding the tool runner's setup (claude-tool-setup.ts). */
 export const TOOL_SETUP_SECRET_KEY = "tool-setup";
+/** The run Secret's key for the MCP token (a run with MCP tools only); the agent container alone reads it. */
+export const MCP_TOKEN_SECRET_KEY = "mcp-token";
 /** Where the tool runner listens and the agent's relay connects (claude-tool-runner/main.mjs). */
 export const TOOL_SOCKET_PATH = "/run/wardby/tool/runner.sock";
 const CODEX_ENTRYPOINT = "/opt/wardby/coding-worker/main.js";
@@ -97,6 +100,9 @@ const STORAGE_DIRECTORIES = ["workspace", "input", "output"];
  * sandbox (GKE Autopilot annotates it `share: pod`), so the agent's relay can connect.
  */
 const TOOL_SOCKET_VOLUME = "tool-socket";
+/** Session resume only: the Claude worker's SDK config dir (docker-isolation.ts CLAUDE_CONFIG_SCRATCH_MIB). */
+const CLAUDE_CONFIG_VOLUME = "claude-config";
+export { CLAUDE_CONFIG_PATH, CLAUDE_CONFIG_SCRATCH_MIB };
 const TOOL_SOCKET_VOLUME_MIB = 1;
 /**
  * `reason` is a short, fixed code naming which check failed (never a value from the spec itself,
@@ -166,6 +172,11 @@ export function validateKubernetesSpec(spec) {
     else if (spec.toolImage !== undefined) {
         throw isolationError("tool-image-set");
     }
+    if (spec.sessionResume !== undefined && (spec.sessionResume !== true || spec.provider !== "claude-code")) {
+        throw isolationError("session-resume-not-claude");
+    }
+    if (spec.mcpTools !== undefined && spec.mcpTools !== true)
+        throw isolationError("mcp-tools-invalid");
     if (!isRegistryDigest(spec.image))
         throw isolationError("image-not-registry-digest");
     if (spec.services !== undefined) {
@@ -307,6 +318,10 @@ export function buildRunPod(spec, options) {
         env: [
             { name: "WARDBY_PROXY_URL", value: `http://${CODING_PROXY_ALIAS}:${CODING_PROXY_PORT}` },
             { name: "WARDBY_RUN_CAPABILITY", valueFrom: { secretKeyRef: { name: names.secret, key: "capability" } } },
+            // A run with MCP tools only: the agent's bearer for the proxy's /mcp route. Never the tool runner's.
+            ...(spec.mcpTools
+                ? [{ name: MCP_TOKEN_ENV, valueFrom: { secretKeyRef: { name: names.secret, key: MCP_TOKEN_SECRET_KEY } } }]
+                : []),
         ],
         securityContext: containerSecurity(),
         resources: conformResources(profile, {
@@ -322,6 +337,8 @@ export function buildRunPod(spec, options) {
                 { name: TOOL_SOCKET_VOLUME, mountPath: "/run/wardby/tool" },
                 { name: "tmp", mountPath: "/tmp" },
                 { name: "home", mountPath: "/home/wardby" },
+                // Session resume: the SDK's config dir, where the session file grows, off /tmp's budget.
+                ...(spec.sessionResume ? [{ name: CLAUDE_CONFIG_VOLUME, mountPath: CLAUDE_CONFIG_PATH }] : []),
             ]
             : [
                 { name: "storage", mountPath: "/workspace", subPath: "workspace" },
@@ -414,6 +431,14 @@ export function buildRunPod(spec, options) {
                         { name: TOOL_SOCKET_VOLUME, emptyDir: { medium: "Memory", sizeLimit: `${TOOL_SOCKET_VOLUME_MIB}Mi` } },
                         { name: "tool-tmp", emptyDir: { medium: "Memory", sizeLimit: `${toolScratchMb}Mi` } },
                         { name: "tool-home", emptyDir: { medium: "Memory", sizeLimit: `${toolScratchMb}Mi` } },
+                        ...(spec.sessionResume
+                            ? [
+                                {
+                                    name: CLAUDE_CONFIG_VOLUME,
+                                    emptyDir: { medium: "Memory", sizeLimit: `${CLAUDE_CONFIG_SCRATCH_MIB}Mi` },
+                                },
+                            ]
+                            : []),
                     ]
                     : []),
                 ...serviceVolumes(services),
@@ -448,14 +473,18 @@ export function buildRunNetworkPolicy(spec, namespace) {
         },
     };
 }
-export function buildCapabilitySecret(spec, namespace, capability, toolSetup) {
+export function buildCapabilitySecret(spec, namespace, capability, toolSetup, mcpToken) {
     const names = kubernetesRunNames(spec.runId);
     return {
         apiVersion: "v1",
         kind: "Secret",
         type: "Opaque",
         metadata: { name: names.secret, namespace, labels: runLabels(spec.runId) },
-        stringData: { capability, ...(toolSetup !== undefined ? { [TOOL_SETUP_SECRET_KEY]: toolSetup } : {}) },
+        stringData: {
+            capability,
+            ...(toolSetup !== undefined ? { [TOOL_SETUP_SECRET_KEY]: toolSetup } : {}),
+            ...(mcpToken !== undefined ? { [MCP_TOKEN_SECRET_KEY]: mcpToken } : {}),
+        },
     };
 }
 /**

package/dist/providers/jobs/kubernetes.d.ts

@@ -4,7 +4,7 @@ import { type Power } from "../../core/power.js";
 import { type KubernetesApi } from "./kubernetes-api.js";
 import { type ServiceStateReporter } from "./service-state.js";
 import type { CodingProvider } from "../../coding/provider.js";
-import type { JobHandle, JobResult, JobSpec, JobStatus, WorkspaceJobLauncher } from "./types.js";
+import { type JobHandle, type JobResult, type JobSpec, type JobStatus, type WorkspaceJobLauncher } from "./types.js";
 export interface KubernetesClusterInfo {
     /** The proxy Service's ClusterIP: the enforcement witness's address (8787 reachable, 8788 not). */
     proxyIp: string;
@@ -14,6 +14,11 @@ export interface KubernetesJobLauncherOptions {
     config: KubernetesJobConfig;
     workspaceRoot: string;
     resolveCapability: (runId: string) => Promise<string>;
+    /**
+     * The run's MCP token (the capability vault), asked for only when the spec has mcpTools; it goes into
+     * the run's Secret next to the capability. Absent = a run with MCP tools can't be launched.
+     */
+    resolveMcpToken?: (runId: string) => Promise<string | undefined>;
     /**
      * Cluster preflight run once before the first launch; failure fails every launch. It may return the
      * proxy ClusterIP it validated; otherwise the launcher reads the proxy witness once itself. Either way
@@ -165,6 +170,10 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
      * must not open the gate at the initial proof either.
      */
     private waitForPolicyEnforcement;
+    /** True when `found` is a requeued record for this spec, which launch() replaces with a fresh one. */
+    private isRequeued;
+    /** Tells the operator once that a quota refused a pod while no quota is configured for the pre-launch check. */
+    private hintQuotaUnconfigured;
     private existingLaunch;
     private provision;
     private createIfMissing;
@@ -178,6 +187,7 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
     private refresh;
     /** Reads only a fixed diagnostic code from a failed worker's bounded log tail; never keeps anything else. */
     private readWorkerDiagnostic;
+    readTranscript(handle: JobHandle, maxBytes: number): Promise<string | undefined>;
     private readResultArtifact;
     /**
      * Streams the keeper's workspace through the strict extractor into `staging`.

package/dist/providers/jobs/kubernetes.js

@@ -30,17 +30,19 @@ import { createHash } from "node:crypto";
 import { isIP } from "node:net";
 import { logger } from "../../core/logger.js";
 import { DISABLED_POWER, wakeTimeoutMs } from "../../core/power.js";
-import { MAX_CODING_ARTIFACT_BYTES, MAX_CODING_INPUT_BYTES, MAX_DEBUG_TRACE_LINE_BYTES, parseCodingAgentOutputJson, } from "../../coding/protocol.js";
+import { MAX_CODING_ARTIFACT_BYTES, MAX_CODING_INPUT_BYTES, MAX_DEBUG_TRACE_LINE_BYTES, parseCodingWorkerOutputJson, } from "../../coding/protocol.js";
 import { CODING_PROXY_DENY_PORT, CODING_PROXY_PORT } from "./docker-isolation.js";
 import { parseWorkerDiagnosticLine } from "./docker.js";
-import { KubernetesAlreadyExistsError, KubernetesConflictError } from "./kubernetes-api.js";
+import { KubernetesAlreadyExistsError, KubernetesConflictError, KubernetesQuotaExceededError, } from "./kubernetes-api.js";
 import { ENFORCEMENT_PROBE_DENY_REACHABLE, ENFORCEMENT_PROBE_PROVEN, ENFORCEMENT_PROBE_DENY_REFUSED, ENFORCEMENT_PROBE_PROXY_UNREACHABLE, KEEPER_CONTAINER, KEEPER_SEEDED_MARKER, KUBERNETES_ISOLATION_ERROR, STORAGE_ROOT, TOOL_RUNNER_CONTAINER, WORKER_CONTAINER, assertRunNetworkPolicyMatches, assertRunPodMatches, buildCapabilitySecret, buildRunNetworkPolicy, buildRunPod, enforcementStreakScript, kubernetesRunNames, kubernetesRunNamesForToken, runLabels, serviceContainerName, validateKubernetesSpec, } from "./kubernetes-isolation.js";
 import { claudeToolSetup } from "./claude-tool-setup.js";
+import { isMcpToken } from "../../coding/mcp-token.js";
 import { podQuotaUsage, quotaShortfall } from "./kubernetes-quota.js";
 import { readProxyWitness } from "./kubernetes-witness.js";
 import { safeExtract } from "./safe-extract.js";
 import { reportServiceState, } from "./service-state.js";
 import { serviceUnreadyError } from "../../coding/services/wording.js";
+import { JobCapacityError, } from "./types.js";
 import { replaceDirectoryFromStaging } from "./workspace-swap.js";
 import { collectExclusions, normalizeCollectExclusions, tarExcludeArgs, } from "../../coding/collect-exclude.js";
 const BACKEND = "kubernetes";
@@ -184,10 +186,22 @@ const TOKEN = /^[a-f0-9]{20}$/;
 const WORKSPACE_STORAGE = `${STORAGE_ROOT}/workspace`;
 const INPUT_STORAGE = `${STORAGE_ROOT}/input`;
 const RESULT_PATH = `${STORAGE_ROOT}/output/result.json`;
+const TRANSCRIPT_PATH = `${STORAGE_ROOT}/output/session.jsonl`;
 /** Stands in for the proxy ClusterIP when building a pod only to count its resources. */
 const QUOTA_PROBE_PROXY_IP = "10.0.0.1";
+/** A Kubernetes object name (RFC 1123 subdomain). */
+const DNS_1123_NAME = /^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/;
 const kubernetesLog = logger.child({ module: "kubernetes-jobs" });
-const PHASES = new Set(["provisioning", "active", "succeeded", "failed", "stopped", "lost", "removed"]);
+const PHASES = new Set([
+    "provisioning",
+    "requeued",
+    "active",
+    "succeeded",
+    "failed",
+    "stopped",
+    "lost",
+    "removed",
+]);
 const TERMINAL_PHASES = new Set(["succeeded", "failed", "stopped", "lost", "removed"]);
 // ---------------------------------------------------------------------------
 // Pure helpers: the lifecycle rules.
@@ -206,6 +220,8 @@ function statusFor(record) {
     switch (record.phase) {
         case "provisioning":
             return { state: "pending" };
+        case "requeued":
+            return { state: "pending", capacityRefused: true };
         case "active":
             return { state: "running" };
         case "succeeded":
@@ -508,8 +524,9 @@ export class KubernetesJobLauncher {
         const handle = { backend: BACKEND, id: `${this.namespace}/${names.token}` };
         const specHash = stableSpecHash(spec);
         const existing = await this.readRecord(names);
-        if (existing)
+        if (existing && !this.isRequeued(existing, specHash)) {
             return this.existingLaunch(existing.record, specHash, handle);
+        }
         const createdAt = this.now();
         const record = {
             schemaVersion: RECORD_SCHEMA_VERSION,
@@ -521,22 +538,54 @@ export class KubernetesJobLauncher {
             phase: "provisioning",
             collectExclude: spec.collectExclude ?? collectExclusions([]),
         };
-        try {
-            // The record is the idempotency fence before any other cluster side effect.
-            await this.api.createConfigMap(this.namespace, this.recordConfigMap(names, record));
+        if (existing) {
+            // Replaces the requeued record; a conflict means another replica got there first.
+            try {
+                await this.api.replaceConfigMap(this.namespace, names.record, this.recordConfigMap(names, record, existing.resourceVersion));
+            }
+            catch (error) {
+                if (!(error instanceof KubernetesConflictError))
+                    throw error;
+                const raced = await this.readRecord(names);
+                if (!raced)
+                    throw error;
+                if (!this.isRequeued(raced, specHash))
+                    return this.existingLaunch(raced.record, specHash, handle);
+                // Requeued again by the other replica (its attempt was refused too): start over from that record.
+                return this.launch(spec);
+            }
         }
-        catch (error) {
-            if (!(error instanceof KubernetesAlreadyExistsError))
-                throw error;
-            const raced = await this.readRecord(names);
-            if (!raced)
-                throw error;
-            return this.existingLaunch(raced.record, specHash, handle);
+        else {
+            try {
+                // The record is the idempotency fence before any other cluster side effect. A quota refusing
+                // this create itself (a count/configmaps quota) is not a capacity refusal: it fails the run.
+                await this.api.createConfigMap(this.namespace, this.recordConfigMap(names, record));
+            }
+            catch (error) {
+                if (!(error instanceof KubernetesAlreadyExistsError))
+                    throw error;
+                const raced = await this.readRecord(names);
+                if (!raced)
+                    throw error;
+                if (!this.isRequeued(raced, specHash))
+                    return this.existingLaunch(raced.record, specHash, handle);
+                // Another replica requeued it between our read and create: start over from the requeued record.
+                return this.launch(spec);
+            }
         }
         try {
             await this.provision(spec, names, record);
         }
         catch (error) {
+            if (error instanceof KubernetesQuotaExceededError) {
+                // Capacity, not failure: leave a record a later launch() provisions afresh.
+                await this.updateRecord(names, () => ({ phase: "requeued" })).catch(() => undefined);
+                await this.cleanupRun(names, 0);
+                this.hintQuotaUnconfigured(error.quotaName);
+                throw new JobCapacityError("kubernetes_quota_exceeded: the namespace ResourceQuota has no room for this run", {
+                    cause: error,
+                });
+            }
             // Record the failure before deleting the pod, so a replica that observes "no pod" in between
             // can't record `lost` first and have this more specific outcome dropped.
             await this.updateRecord(names, () => ({
@@ -758,6 +807,22 @@ export class KubernetesJobLauncher {
             await this.sleep(ENFORCEMENT_POLL_MS);
         }
     }
+    /** True when `found` is a requeued record for this spec, which launch() replaces with a fresh one. */
+    isRequeued(found, specHash) {
+        if (found.record.phase !== "requeued")
+            return false;
+        if (found.record.specHash !== specHash)
+            throw new Error("job_spec_conflict");
+        return true;
+    }
+    /** Tells the operator once that a quota refused a pod while no quota is configured for the pre-launch check. */
+    hintQuotaUnconfigured(refusedQuota) {
+        if (this.options.config.resourceQuota)
+            return;
+        const name = refusedQuota !== undefined && DNS_1123_NAME.test(refusedQuota) ? refusedQuota : undefined;
+        this.warnOnce(`kubernetes_resource_quota_unconfigured: a pod was refused by ${name ? `ResourceQuota ${name}` : "a ResourceQuota"}; ` +
+            `set KUBERNETES_RESOURCE_QUOTA=${name ?? "<name>"} and grant the launcher get on that ResourceQuota so runs wait before they start`);
+    }
     existingLaunch(record, specHash, handle) {
         if (record.specHash !== specHash)
             throw new Error("job_spec_conflict");
@@ -774,6 +839,10 @@ export class KubernetesJobLauncher {
         const capability = await this.options.resolveCapability(spec.runId);
         if (!CAPABILITY.test(capability))
             throw new Error("kubernetes_capability_invalid");
+        const mcpToken = spec.mcpTools ? await this.options.resolveMcpToken?.(spec.runId) : undefined;
+        if (spec.mcpTools && !isMcpToken(mcpToken)) {
+            throw new Error("kubernetes_mcp_token_invalid");
+        }
         // Re-read (not just the ClusterIP): the gate below reads "deny port unreachable" as evidence,
         // which is only meaningful against a Service that actually exposes it with a ready backend.
         const witness = await readProxyWitness(this.api, this.namespace, proxyService);
@@ -787,7 +856,7 @@ export class KubernetesJobLauncher {
         });
         const policy = buildRunNetworkPolicy(spec, this.namespace);
         const toolSetup = spec.provider === "claude-code" ? claudeToolSetup(spec, capability) : undefined;
-        await this.createIfMissing(() => this.api.createSecret(this.namespace, buildCapabilitySecret(spec, this.namespace, capability, toolSetup)));
+        await this.createIfMissing(() => this.api.createSecret(this.namespace, buildCapabilitySecret(spec, this.namespace, capability, toolSetup, mcpToken)));
         await this.createIfMissing(() => this.api.createNetworkPolicy(this.namespace, policy));
         await this.createIfMissing(() => this.api.createPod(this.namespace, pod));
         await this.waitForKeeper(names, spec.runId, (spec.services ?? []).map((service) => service.name), spec.provider === "claude-code");
@@ -957,7 +1026,8 @@ export class KubernetesJobLauncher {
     // Observation and collection
     /** Observes a non-terminal run against its pod and persists any transition; returns the stored record. */
     async refresh(names, record) {
-        if (isTerminal(record.phase))
+        // A requeued run has no pod by design; observing it would record it lost.
+        if (isTerminal(record.phase) || record.phase === "requeued")
             return record;
         const pod = await this.api.readPod(this.namespace, names.pod);
         const observed = observePod(pod, record, this.now(), this.readyTimeoutMs);
@@ -985,6 +1055,30 @@ export class KubernetesJobLauncher {
         }
         return undefined;
     }
+    async readTranscript(handle, maxBytes) {
+        const names = this.namesFor(handle);
+        const found = names && (await this.readRecord(names));
+        if (!names || !found)
+            return undefined;
+        const collector = new BoundedCollector(maxBytes);
+        let exitCode;
+        try {
+            exitCode = await this.api.exec(this.namespace, names.pod, KEEPER_CONTAINER, ["head", "-c", String(maxBytes + 1), TRANSCRIPT_PATH], { stdout: collector, timeoutMs: SHORT_EXEC_TIMEOUT_MS });
+            if (!collector.writableEnded)
+                collector.end();
+            await finished(collector);
+        }
+        catch {
+            // The pod is gone (a stopped run) or unreachable: there is nothing to collect.
+            collector.destroy();
+            return undefined;
+        }
+        if (exitCode !== 0)
+            return undefined;
+        if (collector.exceeded)
+            throw new Error("transcript_too_large");
+        return collector.text();
+    }
     async readResultArtifact(names, runId) {
         const collector = new BoundedCollector(MAX_CODING_ARTIFACT_BYTES);
         let raw;
@@ -1005,7 +1099,7 @@ export class KubernetesJobLauncher {
         }
         let output;
         try {
-            output = parseCodingAgentOutputJson(raw);
+            output = parseCodingWorkerOutputJson(raw);
         }
         catch (error) {
             throw errorWithCode("kubernetes_result_artifact_invalid", error);

package/dist/providers/jobs/types.d.ts

@@ -21,6 +21,20 @@ export interface JobSpec {
     /** Required only for Claude's credential-free repository tool container. */
     toolImage?: string;
     inputArtifact: string;
+    /** Set only when the run's dependency cache is on (CodingRun.dependencyCache); absent otherwise, never false. */
+    dependencyCache?: true;
+    /**
+     * Claude Code only, set only when the run keeps its session (CodingRun.sessionResume); absent otherwise,
+     * never false. The worker's SDK config dir, where the session file grows, gets its own size-limited
+     * scratch so a long session never fills the worker's shared /tmp.
+     */
+    sessionResume?: true;
+    /**
+     * Set only when the run has pinned MCP tools (remote MCP servers); absent otherwise, never false.
+     * The launcher then delivers the run's MCP token, from the capability vault, as WARDBY_MCP_TOKEN to
+     * the agent container only (never Claude's tool runner). The token itself is never in the spec.
+     */
+    mcpTools?: true;
     timeoutSec: number;
     limits: JobResourceLimits;
     labels: Record<string, string>;
@@ -42,6 +56,12 @@ export interface JobHandle {
 }
 export type JobStatus = {
     state: "pending" | "running";
+    /**
+     * Only ever on `pending`: the backend refused this job's launch for capacity (JobCapacityError)
+     * and nothing of it is running, nor will it start on its own. A caller that finds its job so
+     * may return the run to its queue and later `launch` the same spec again.
+     */
+    capacityRefused?: true;
 } | {
     state: "succeeded";
 } | {
@@ -61,11 +81,24 @@ export interface JobResult {
     /** Output-schema failure locations (`path:code`) that passed SAFE_CODING_OUTPUT_ISSUE; never values. */
     diagnosticIssues?: string[];
 }
+/**
+ * The backend has no room for this job right now (Kubernetes: the namespace ResourceQuota). No
+ * workload of the job is left running; the backend may keep a record of the refusal (Kubernetes
+ * does: its run record, which `status` then reports as `pending` with `capacityRefused`). The
+ * caller may launch the same spec again later.
+ */
+export declare class JobCapacityError extends Error {
+    readonly code = "coding_capacity_exceeded";
+    constructor(message: string, options?: ErrorOptions);
+}
 /**
  * Lifecycle invariants implemented by every launcher:
  *
  * - `launch` is idempotent by runId for an identical spec and rejects a
  *   different spec for the same runId.
+ * - `launch` may reject with JobCapacityError when the backend has no room for
+ *   the job; nothing of it is then left running, and a later `launch` of the
+ *   same spec provisions it afresh.
  * - pending may become running or terminal; running may become terminal;
  *   terminal states never change.
  * - `collect` is repeatable for a terminal job until `remove` succeeds.
@@ -100,6 +133,12 @@ export interface JobLauncher {
     launch: (spec: JobSpec) => Promise<JobHandle>;
     status: (handle: JobHandle) => Promise<JobStatus>;
     collect: (handle: JobHandle) => Promise<JobResult>;
+    /**
+     * The run's session transcript file (the worker's session.jsonl), at most `maxBytes`; undefined
+     * when there is none or the job's storage is gone (a stopped job). Throws `transcript_too_large`
+     * when larger. Never cached in the job record: it is not redacted yet. Call before `remove`.
+     */
+    readTranscript?: (handle: JobHandle, maxBytes: number) => Promise<string | undefined>;
     stop: (handle: JobHandle, reason?: string) => Promise<void>;
     remove: (handle: JobHandle) => Promise<void>;
 }

package/dist/providers/jobs/types.js

@@ -3,4 +3,16 @@
  * values are carried in a versioned input artifact, never expanded into an
  * arbitrary environment map or shell command.
  */
-export {};
+/**
+ * The backend has no room for this job right now (Kubernetes: the namespace ResourceQuota). No
+ * workload of the job is left running; the backend may keep a record of the refusal (Kubernetes
+ * does: its run record, which `status` then reports as `pending` with `capacityRefused`). The
+ * caller may launch the same spec again later.
+ */
+export class JobCapacityError extends Error {
+    code = "coding_capacity_exceeded";
+    constructor(message, options) {
+        super(message, options);
+        this.name = "JobCapacityError";
+    }
+}

package/dist/providers/llm/anthropic.js

@@ -6,6 +6,7 @@
  */
 import Anthropic from "@anthropic-ai/sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
+import { simulatorModelClientOptions } from "../../config/test-simulators.js";
 export function anthropicCredentialsPresent(env = process.env) {
     return Boolean(env.ANTHROPIC_API_KEY);
 }
@@ -14,6 +15,9 @@ export class AnthropicLlmProvider extends ClaudeLlmProvider {
         if (!client && !apiKey) {
             throw new Error("ANTHROPIC_API_KEY is not set — required by the Anthropic LlmProvider adapter.");
         }
-        super(client ?? new Anthropic({ apiKey }), lookup);
+        // WARDBY_TEST_SIMULATORS (test only): the scripted model, with a placeholder key the
+        // credential-free fetch drops; the real key never goes there.
+        const simulator = client ? null : simulatorModelClientOptions("anthropic");
+        super(client ?? new Anthropic(simulator ?? { apiKey }), lookup);
     }
 }

package/dist/providers/llm/bedrock.d.ts

@@ -1,15 +1,3 @@
-/**
- * Bedrock-Claude adapter for the LlmProvider seam. `@anthropic-ai/bedrock-sdk`
- * is imported only here; all protocol/pricing-dispatch logic lives in the
- * shared ClaudeLlmProvider base class (claude-provider.ts) — AnthropicBedrock
- * exposes the identical .messages.stream() surface as @anthropic-ai/sdk, so
- * this file's only job is constructing the right client.
- *
- * Named BedrockClaudeLlmProvider, not BedrockLlmProvider: Bedrock hosts
- * multiple model vendors, and this adapter only ever speaks the
- * Anthropic-native protocol for Claude models. A future non-Claude Bedrock
- * adapter gets its own name, not a rename of this one.
- */
 import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
 import type { CatalogLookup } from "./catalog-lookup.js";

package/dist/providers/llm/bedrock.js

@@ -10,6 +10,7 @@
  * Anthropic-native protocol for Claude models. A future non-Claude Bedrock
  * adapter gets its own name, not a rename of this one.
  */
+import { refuseBedrockUnderTestSimulators } from "../../config/test-simulators.js";
 import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk";
 import { ClaudeLlmProvider } from "./claude-provider.js";
 export function bedrockCredentialsPresent(env = process.env) {
@@ -23,6 +24,9 @@ export class BedrockClaudeLlmProvider extends ClaudeLlmProvider {
         if (!client && !region) {
             throw new Error("BEDROCK_REGION (or AWS_REGION) is required by the Bedrock Claude LlmProvider adapter.");
         }
+        // WARDBY_TEST_SIMULATORS (test only) refuses Bedrock: AWS-signed calls cannot go to the simulator.
+        if (!client)
+            refuseBedrockUnderTestSimulators();
         super(client ?? new AnthropicBedrock({ awsRegion: region }), lookup);
     }
 }

package/dist/providers/llm/claude-messages.d.ts

@@ -106,5 +106,7 @@ export type ClaudeStreamEvent = {
 } | {
     type: "message_stop";
 };
-export declare function mapClaudeStream(events: AsyncIterable<ClaudeStreamEvent>, priceUsd: (usage: LlmUsage) => number): AsyncIterable<LlmStreamEvent>;
+export declare function mapClaudeStream(events: AsyncIterable<ClaudeStreamEvent>, priceUsd: (usage: LlmUsage) => number, 
+/** Called at message_start: the response has started, and output may be billed from here on. */
+onResponseStarted?: () => void): AsyncIterable<LlmStreamEvent>;
 export {};

package/dist/providers/llm/claude-messages.js

@@ -98,7 +98,9 @@ export function withCacheBreakpoints(req) {
     });
     return { ...req, ...(system ? { system } : {}), messages };
 }
-export async function* mapClaudeStream(events, priceUsd) {
+export async function* mapClaudeStream(events, priceUsd, 
+/** Called at message_start: the response has started, and output may be billed from here on. */
+onResponseStarted) {
     const toolBuffers = new Map();
     let start = {};
     let outputTokens = 0;
@@ -106,6 +108,7 @@ export async function* mapClaudeStream(events, priceUsd) {
     for await (const ev of events) {
         switch (ev.type) {
             case "message_start":
+                onResponseStarted?.();
                 start = ev.message.usage ?? {};
                 outputTokens = ev.message.usage?.output_tokens ?? 0;
                 break;

package/dist/providers/llm/claude-provider.d.ts

@@ -5,7 +5,7 @@
  * stays exactly what each concrete adapter (anthropic.ts, bedrock.ts) has in
  * common, and nothing more.
  */
-import type { LlmMessage, LlmRequest, LlmStreamEvent, LlmToolDef } from "./types.js";
+import type { LlmMessage, LlmRequest, LlmStreamEvent, LlmStreamHooks, LlmToolDef } from "./types.js";
 import { type CatalogLookup } from "./catalog-lookup.js";
 import type { CatalogEntry } from "./catalog-types.js";
 import type { CatalogLlmAdapter } from "./routing.js";
@@ -27,7 +27,7 @@ export declare class ClaudeLlmProvider implements CatalogLlmAdapter {
     private readonly lookup;
     constructor(client: ClaudeMessagesClient, lookup?: CatalogLookup);
     withEntry(entry: CatalogEntry): ClaudeLlmProvider;
-    stream(req: LlmRequest, signal?: AbortSignal): AsyncIterable<LlmStreamEvent>;
+    stream(req: LlmRequest, signal?: AbortSignal, hooks?: LlmStreamHooks): AsyncIterable<LlmStreamEvent>;
     countTokens(model: string, messages: LlmMessage[], tools?: LlmToolDef[]): Promise<number>;
     priceUsd(model: string, usage: {
         inputTokens: number;