@wardby/cli 0.1.0

package/dist/providers/jobs/kubernetes-preflight.js

@@ -0,0 +1,339 @@
+/**
+ * Fail-closed cluster preflight for JOB_LAUNCHER=kubernetes. Proves the
+ * namespace exists, the proxy Service is a usable enforcement witness (both
+ * ports exposed with a ready backend), the worker image is a registry digest,
+ * and — with a canary pod built from the real run pod and run NetworkPolicy —
+ * that DNS, the internet, the metadata server and **the proxy's deny port**
+ * are unreachable while the proxy itself is reachable. Any other state, error,
+ * or timeout fails the check; the whole preflight, cleanup included, is bounded.
+ */
+import { randomBytes } from "node:crypto";
+import { CODING_PROXY_DENY_PORT, CODING_PROXY_PORT } from "./docker-isolation.js";
+import { KUBERNETES_ISOLATION_ERROR, WORKER_CONTAINER, buildRunNetworkPolicy, buildRunPod, isRegistryDigest, } from "./kubernetes-isolation.js";
+import { KubernetesPlatformError, assertPlatformConfig, platformProfile } from "./kubernetes-platform.js";
+import { readProxyWitness } from "./kubernetes-witness.js";
+const DEFAULT_TIMEOUT_MS = 90_000;
+const DEFAULT_CLEANUP_TIMEOUT_MS = 15_000;
+const POLL_INTERVAL_MS = 500;
+const LOG_TAIL_LINES = 20;
+const LOG_LIMIT_BYTES = 4096;
+const CANARY_LINE_PREFIX = '{"wardbyCanary":';
+const EXPECTED = { dns: false, proxyDeny: false, internet: false, metadata: false, proxy: true };
+const CANARY_KEYS = Object.keys(EXPECTED).sort();
+/**
+ * Runs in the worker image under the run policy; prints exactly one `{"wardbyCanary":{...}}` line.
+ * `dns` checks in-pod resolution; `proxyDeny` checks the policy itself — a TCP connect to the *same*
+ * proxy pod on a port no run policy permits, which a namespace-wide allow-all policy would reopen.
+ * Probing one destination on two ports is what makes the result decisive: `proxy` true with
+ * `proxyDeny` false can only mean a policy is enforced and port-scoped.
+ *
+ * `proxyDeny` is false for exactly one socket outcome: a **timeout**, i.e. the packet was dropped.
+ * A refusal (RST) is not a denial — it proves the SYN reached the destination host, so the deny
+ * port is merely unserved — and a connect obviously is not either. This mirrors the launch gate's
+ * probe (`enforcementProbeScript`), which had to make the same distinction after a live cluster
+ * with no NetworkPolicy at all passed a boolean version of this check; preflight and gate must
+ * agree on what counts as evidence, or one of them is vacuous. `internet`, `metadata` and `proxy`
+ * stay plain reachability, where "connected" is the only thing that matters.
+ *
+ * CNIs program a new pod's policy a few seconds after it starts, so the script first waits (up to
+ * 20 s) for the deny-port connect to be *dropped*; if it never is, `proxyDeny` reports true.
+ */
+export const CANARY_SCRIPT = [
+    'const net = require("node:net");',
+    'const dns = require("node:dns").promises;',
+    "const tcp = (host, port) =>",
+    "  new Promise((done) => {",
+    "    const socket = net.connect({ host, port, timeout: 3000 });",
+    '    socket.once("connect", () => {',
+    "      socket.destroy();",
+    '      done("connect");',
+    "    });",
+    '    socket.once("timeout", () => {',
+    "      socket.destroy();",
+    '      done("timeout");',
+    "    });",
+    '    socket.once("error", () => done("error"));',
+    "  });",
+    "// Plain reachability, for the probes where only a completed connection matters.",
+    'const reachable = async (host, port) => (await tcp(host, port)) === "connect";',
+    "// Only a dropped packet is a policy denial: a refusal proves the SYN reached the host.",
+    'const dropped = async (host, port) => (await tcp(host, port)) === "timeout";',
+    "const settle = async () => {",
+    "  const until = Date.now() + 20000;",
+    "  while (Date.now() < until) {",
+    `    if (await dropped(process.env.WARDBY_CANARY_PROXY_IP, ${CODING_PROXY_DENY_PORT})) return;`,
+    "    await new Promise((wake) => setTimeout(wake, 500));",
+    "  }",
+    "};",
+    "(async () => {",
+    "  await settle();",
+    "  const wardbyCanary = {",
+    '    dns: await dns.lookup("kubernetes.default.svc.cluster.local").then(',
+    "      () => true,",
+    "      () => false,",
+    "    ),",
+    `    proxyDeny: !(await dropped(process.env.WARDBY_CANARY_PROXY_IP, ${CODING_PROXY_DENY_PORT})),`,
+    '    internet: await reachable("1.1.1.1", 443),',
+    '    metadata: await reachable("169.254.169.254", 80),',
+    `    proxy: await reachable(process.env.WARDBY_CANARY_PROXY_IP, ${CODING_PROXY_PORT}),`,
+    "  };",
+    "  console.log(JSON.stringify({ wardbyCanary }));",
+    "})();",
+].join("\n");
+/** A failed check. Carries no cause for canary-output failures so no log text can leave this module. */
+class PreflightFailure extends Error {
+}
+function failure(check, cause) {
+    const message = `${KUBERNETES_ISOLATION_ERROR}:${check}`;
+    return cause === undefined ? new PreflightFailure(message) : new PreflightFailure(message, { cause });
+}
+/** Runs one check; any error that is not already a check failure becomes this check's failure. */
+async function runCheck(check, action) {
+    try {
+        return await action();
+    }
+    catch (error) {
+        if (error instanceof PreflightFailure)
+            throw error;
+        throw failure(check, error);
+    }
+}
+/** Resolves with the action's result, or rejects with `onTimeout()` after `ms`. Never leaves a timer behind. */
+async function within(action, ms, onTimeout) {
+    let timer;
+    const expired = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(onTimeout()), ms);
+    });
+    try {
+        return await Promise.race([action, expired]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/** Parses only the fixed canary line and requires exactly the five booleans. */
+function parseCanary(log) {
+    const lines = log
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith(CANARY_LINE_PREFIX));
+    if (lines.length !== 1)
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(lines[0]);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof parsed !== "object" || parsed === null || Object.keys(parsed).length !== 1)
+        return undefined;
+    const result = parsed.wardbyCanary;
+    if (typeof result !== "object" || result === null || Array.isArray(result))
+        return undefined;
+    const keys = Object.keys(result).sort();
+    if (keys.join(",") !== CANARY_KEYS.join(","))
+        return undefined;
+    const record = result;
+    if (!keys.every((key) => typeof record[key] === "boolean"))
+        return undefined;
+    return record;
+}
+function canaryPasses(result) {
+    return (result !== undefined &&
+        result.dns === EXPECTED.dns &&
+        result.proxyDeny === EXPECTED.proxyDeny &&
+        result.internet === EXPECTED.internet &&
+        result.metadata === EXPECTED.metadata &&
+        result.proxy === EXPECTED.proxy);
+}
+/** Deletes the pod (grace 0), and only then its policy, so the canary never runs unpoliced. */
+async function deleteCanary(api, state, cleanupTimeoutMs) {
+    const timedOut = () => new Error("canary_cleanup_timeout");
+    if (state.podName)
+        await within(api.deletePod(state.namespace, state.podName, 0), cleanupTimeoutMs, timedOut);
+    if (state.policyName) {
+        await within(api.deleteNetworkPolicy(state.namespace, state.policyName), cleanupTimeoutMs, timedOut);
+    }
+}
+/** Issues one create; a create that settles after the preflight finished removes what it made. */
+async function tracked(state, create, lateCleanup) {
+    if (state.finished)
+        throw failure("timeout");
+    state.createInFlight = true;
+    try {
+        return await create();
+    }
+    finally {
+        state.createInFlight = false;
+        if (state.finished)
+            await lateCleanup().catch(() => undefined);
+    }
+}
+async function runCanary(options, state, proxyIp, timeoutMs, cleanupTimeoutMs) {
+    const { api, config, workerImage } = options;
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((ms) => new Promise((done) => setTimeout(done, ms)));
+    const namespace = config.namespace;
+    const timeoutSec = Math.max(1, Math.ceil(timeoutMs / 1000));
+    const spec = {
+        kind: "coding-agent",
+        runId: `preflight-${randomBytes(8).toString("hex")}`,
+        provider: "codex",
+        image: workerImage,
+        inputArtifact: "",
+        timeoutSec,
+        limits: { cpus: 0.25, memoryMb: 128, pids: 64, diskMb: 64 },
+        labels: {},
+    };
+    const { pod, policy } = await runCheck("canary", async () => {
+        const pod = buildRunPod(spec, {
+            namespace,
+            proxyIp,
+            runtimeClassName: config.runtimeClassName,
+            platform: config.platform,
+        });
+        const podSpec = pod.spec;
+        const worker = podSpec.containers.find((container) => container.name === WORKER_CONTAINER);
+        if (!worker)
+            throw failure("canary");
+        worker.env = [{ name: "WARDBY_CANARY_PROXY_IP", value: proxyIp }];
+        worker.command = ["node", "-e", CANARY_SCRIPT];
+        podSpec.containers = [worker];
+        // The canary has no collection window: Kubernetes kills it at the preflight's own deadline.
+        podSpec.activeDeadlineSeconds = timeoutSec;
+        return { pod, policy: buildRunNetworkPolicy(spec, namespace) };
+    });
+    const podName = pod.metadata.name;
+    const policyName = policy.metadata.name;
+    const lateCleanup = () => deleteCanary(api, state, cleanupTimeoutMs);
+    await runCheck("canary", async () => {
+        // The policy exists before the pod so the canary never runs unpoliced.
+        state.policyName = policyName;
+        await tracked(state, () => api.createNetworkPolicy(namespace, policy), lateCleanup);
+        state.podName = podName;
+        await tracked(state, () => api.createPod(namespace, pod), lateCleanup);
+        const deadline = now() + timeoutMs;
+        for (;;) {
+            if (state.finished)
+                throw failure("timeout");
+            const current = await api.readPod(namespace, podName);
+            if (!current)
+                throw failure("canary");
+            const status = current.status?.containerStatuses?.find((entry) => entry.name === WORKER_CONTAINER);
+            const terminated = status?.state?.terminated;
+            if (terminated) {
+                if (terminated.exitCode !== 0)
+                    throw failure("canary");
+                break;
+            }
+            if (current.status?.phase === "Failed" || current.status?.phase === "Succeeded")
+                throw failure("canary");
+            if (now() >= deadline)
+                throw failure("timeout");
+            await sleep(POLL_INTERVAL_MS);
+        }
+        let log;
+        try {
+            log = await api.readLogTail(namespace, podName, WORKER_CONTAINER, LOG_TAIL_LINES, LOG_LIMIT_BYTES);
+        }
+        catch {
+            throw failure("canary");
+        }
+        if (!canaryPasses(parseCanary(log)))
+            throw failure("canary");
+    });
+}
+async function runChecks(options, state, passed, timeoutMs, cleanupTimeoutMs) {
+    const { api, config, workerImage } = options;
+    // Pure configuration, checked first: a deployment that cannot work is refused before
+    // a single API call, with a message naming the setting and the limit.
+    await runCheck("platform", async () => {
+        assertPlatformConfig(platformProfile(config.platform), {
+            runtimeClassName: config.runtimeClassName,
+            maxDiskMb: options.maxDiskMb,
+        });
+    });
+    passed.push("platform");
+    await runCheck("namespace", async () => {
+        if (!(await api.readNamespace(config.namespace)))
+            throw failure("namespace");
+    });
+    passed.push("namespace");
+    const proxyIp = await runCheck("proxy-service", async () => {
+        const witness = await readProxyWitness(api, config.namespace, config.proxyService);
+        return witness.clusterIp;
+    });
+    passed.push("proxy-service");
+    if (!isRegistryDigest(workerImage))
+        throw failure("worker-image");
+    passed.push("worker-image");
+    await runCanary(options, state, proxyIp, timeoutMs, cleanupTimeoutMs);
+    passed.push("canary");
+    return proxyIp;
+}
+/** Throws kubernetes_isolation_unsupported:<check> on the first failed check; returns the checks that passed. */
+export async function kubernetesPreflight(options) {
+    return (await runKubernetesPreflight(options)).checks;
+}
+/**
+ * Throws kubernetes_isolation_unsupported:<check> on the first failed check
+ * (`:timeout` when the whole preflight exceeds `timeoutMs`); returns the checks that passed and
+ * the proxy ClusterIP it validated.
+ */
+export async function runKubernetesPreflight(options) {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const cleanupTimeoutMs = options.cleanupTimeoutMs ?? DEFAULT_CLEANUP_TIMEOUT_MS;
+    const state = { namespace: options.config.namespace, createInFlight: false, finished: false };
+    const passed = [];
+    let outcome;
+    let failed = false;
+    let proxyIp = "";
+    try {
+        proxyIp = await within(runChecks(options, state, passed, timeoutMs, cleanupTimeoutMs), timeoutMs, () => failure("timeout"));
+    }
+    catch (error) {
+        failed = true;
+        outcome = error;
+    }
+    finally {
+        state.finished = true;
+        // A create still in flight owns cleanup (see `tracked`); otherwise delete what exists now.
+        if (!state.createInFlight) {
+            try {
+                await deleteCanary(options.api, state, cleanupTimeoutMs);
+            }
+            catch {
+                // A canary that cannot be cleaned up is an unexpected cluster state: fail closed.
+                if (!failed) {
+                    failed = true;
+                    outcome = failure("canary");
+                }
+            }
+        }
+    }
+    if (failed)
+        throw outcome;
+    return { checks: passed, proxyIp };
+}
+function shortMessage(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const firstLine = message.split("\n", 1)[0].trim();
+    // A KubernetesPlatformError is single-line by construction and never carries canary output (it is
+    // thrown from pure configuration checks before any cluster call), so the 200-char cap that exists
+    // to keep arbitrary API-error text and canary noise out of an operator-facing line does not need to
+    // apply here — and truncating it cuts the remediation clause an operator needs most.
+    if (error instanceof KubernetesPlatformError)
+        return firstLine || "unknown error";
+    return firstLine.length > 200 ? `${firstLine.slice(0, 200)}…` : firstLine || "unknown error";
+}
+/**
+ * One operator-facing line for a failed preflight: the check code, plus the
+ * short message of a cause only where the preflight attached one (API errors;
+ * never canary output). Anything else (e.g. a kubeconfig load error) is shown briefly.
+ */
+export function describePreflightFailure(error) {
+    if (!(error instanceof PreflightFailure))
+        return shortMessage(error);
+    return error.cause === undefined ? error.message : `${error.message} (${shortMessage(error.cause)})`;
+}

package/dist/providers/jobs/kubernetes-witness.d.ts

@@ -0,0 +1,10 @@
+import type { KubernetesApi } from "./kubernetes-api.js";
+export declare const PROXY_WITNESS_UNUSABLE = "kubernetes_proxy_witness_unusable";
+export declare class ProxyWitnessError extends Error {
+    constructor(detail: string);
+}
+export interface ProxyWitness {
+    /** The proxy Service's ClusterIP: the address both probed ports live on. */
+    clusterIp: string;
+}
+export declare function readProxyWitness(api: KubernetesApi, namespace: string, service: string): Promise<ProxyWitness>;

package/dist/providers/jobs/kubernetes-witness.js

@@ -0,0 +1,193 @@
+/**
+ * The proxy Service as the NetworkPolicy enforcement witness.
+ *
+ * The preflight canary and the per-launch gate both prove a run's policy is
+ * enforced by requiring a connect to the proxy's deny port to be *refused*
+ * while a connect to the proxy port succeeds. This read is the configuration
+ * cross-check for that: the Service must expose both ports, and at least one
+ * ready endpoint must serve both, or the deployment is misconfigured and says
+ * so by name rather than failing later at a gate.
+ *
+ * This is a control-plane cross-check only: it does not itself keep a probe
+ * from being vacuous. That is what the two-port probe (measured in one exec,
+ * at the instant of the blocked observation) proves; see kubernetes.ts.
+ *
+ * It also checks the one thing the probe cannot observe about itself: WHO
+ * dropped the packet. A timeout on the deny port proves the SYN was dropped
+ * somewhere on the path, not that the run pod's own egress policy dropped it.
+ * The attribution follows from the proxy's own NetworkPolicy admitting run
+ * pods on the deny port at its ingress — because ingress is enforced at the
+ * destination, a proxy policy that did NOT admit 8788 would drop the packet
+ * there and every run would read "blocked" with no egress policy at all. That
+ * was falsified on a live cluster: a proxy policy admitting 8787 only made a
+ * prober with no policy and full internet egress read PROVEN. The rule was
+ * asserted by manifest and checked nowhere, so one deleted line in an overlay
+ * silently voided the gate. It is verified here instead.
+ *
+ * Both reads are in wardby's own namespace, which the launcher Role permits.
+ * (EndpointSlice is the durable successor to the core/v1 Endpoints read here;
+ * migrating is a follow-up and needs its own resourceNames grant.)
+ */
+import { isIP } from "node:net";
+import { CODING_PROXY_DENY_PORT, CODING_PROXY_PORT } from "./docker-isolation.js";
+import { PROXY_POD_LABEL, RUN_COMPONENT_LABEL } from "./kubernetes-isolation.js";
+export const PROXY_WITNESS_UNUSABLE = "kubernetes_proxy_witness_unusable";
+export class ProxyWitnessError extends Error {
+    constructor(detail) {
+        super(`${PROXY_WITNESS_UNUSABLE}: ${detail}`);
+        this.name = "ProxyWitnessError";
+    }
+}
+const REQUIRED_PORTS = [CODING_PROXY_PORT, CODING_PROXY_DENY_PORT];
+/** core/v1 defaults an omitted port protocol to TCP; only count a port toward either
+ * required-port check when it is TCP (or silent on protocol), so a UDP/SCTP declaration
+ * of 8788 can never satisfy a check that exists to prove something is TCP-listening there. */
+function isTcp(protocol) {
+    return protocol === undefined || protocol === "TCP";
+}
+/**
+ * Whether a NetworkPolicy's `ingress[]` section has any bearing on ingress traffic at all.
+ * Per `V1NetworkPolicySpec`'s generated docs: if `policyTypes` is absent it defaults based on the
+ * rules present, and "all policies (whether or not they contain an ingress section) are assumed to
+ * affect ingress" — so an absent list still counts. But an explicit list that omits `"Ingress"`
+ * (e.g. an egress-only `["Egress"]` policy) makes any `ingress[]` content inert: Kubernetes never
+ * consults it, so a rule that reads as "admits the deny port" there proves nothing about what is
+ * actually admitted.
+ */
+function affectsIngress(policyTypes) {
+    return policyTypes === undefined || policyTypes.includes("Ingress");
+}
+/** Whether a Service selector requires every key/value in `labels` (extra keys on the selector are
+ * fine — they only narrow which pods it routes to further). Service selectors are plain label maps,
+ * not `LabelSelector` objects, so this is direct equality-per-key, not `selectsPodLabeled`'s
+ * matchLabels/matchExpressions shape. */
+function selectorRequires(selector, labels) {
+    return Object.entries(labels).every(([key, value]) => selector?.[key] === value);
+}
+/** A selector we can evaluate: plain matchLabels, no matchExpressions we would have to guess at. */
+function selectsPodLabeled(selector, labels) {
+    if (!selector)
+        return false;
+    // An expression-based selector may well be correct, but we cannot prove it here: fail closed.
+    if (Array.isArray(selector.matchExpressions) && selector.matchExpressions.length > 0)
+        return false;
+    const required = Object.entries(selector.matchLabels ?? {});
+    return required.every(([key, value]) => labels[key] === value);
+}
+/** Whether a peer admits pods carrying `labels` in this same namespace. */
+function peerAdmits(peer, labels) {
+    // A namespaceSelector re-scopes the peer to other namespaces; run pods are in this one. An
+    // ipBlock never matches a pod selector at all. Neither can be counted as admitting run pods.
+    if (peer.namespaceSelector || peer.ipBlock)
+        return false;
+    return selectsPodLabeled(peer.podSelector, labels);
+}
+/** Whether a rule's port list covers `port` over TCP (absent/empty = every port, per the API). */
+function ruleCoversPort(rule, port) {
+    const ports = rule.ports ?? [];
+    if (ports.length === 0)
+        return true;
+    return ports.some((entry) => {
+        if (!isTcp(entry.protocol))
+            return false;
+        // `port` may be a named port, which we cannot resolve to a number here: fail closed.
+        if (typeof entry.port !== "number")
+            return false;
+        const endPort = typeof entry.endPort === "number" ? entry.endPort : entry.port;
+        return entry.port <= port && port <= endPort;
+    });
+}
+/**
+ * The rule's source list, under either spelling. `@kubernetes/client-node` deserializes the wire
+ * field `from` to `_from` (it collides with a TypeScript keyword), so a policy read back from a
+ * real API server has `_from` while a hand-written manifest object has `from`. Reading only one
+ * spelling would leave the other `undefined`, which this function reads as "no restriction" — so
+ * getting this wrong fails OPEN and would let any policy at all satisfy the check.
+ */
+function ruleSources(rule) {
+    return rule._from ?? rule.from ?? [];
+}
+/** Whether a rule admits pods carrying `labels` on `port` (absent/empty `from` = every source). */
+function ruleAdmits(rule, labels, port) {
+    if (!ruleCoversPort(rule, port))
+        return false;
+    const from = ruleSources(rule);
+    return from.length === 0 || from.some((peer) => peerAdmits(peer, labels));
+}
+/**
+ * Proves the attribution precondition: the proxy's own policy admits run pods on the deny port,
+ * so the run pod's egress policy is the only thing left that can drop that packet.
+ */
+async function assertDenyPortAdmitted(api, namespace, service) {
+    const where = `NetworkPolicy ${namespace}/${service}`;
+    const policy = await api.readNetworkPolicy(namespace, service);
+    if (!policy) {
+        throw new ProxyWitnessError(`${where} was not found, so nothing proves the proxy admits run pods on port ${CODING_PROXY_DENY_PORT}; ` +
+            "without that rule the proxy's own ingress drops the probe and every run reads as enforced");
+    }
+    if (!affectsIngress(policy.spec?.policyTypes)) {
+        throw new ProxyWitnessError(`${where} has policyTypes ${JSON.stringify(policy.spec?.policyTypes)} without "Ingress", so its ingress ` +
+            "rules are inert and prove nothing about what the proxy admits at its ingress");
+    }
+    if (!selectsPodLabeled(policy.spec?.podSelector, PROXY_POD_LABEL)) {
+        throw new ProxyWitnessError(`${where} does not select the proxy pods (${JSON.stringify(PROXY_POD_LABEL)}), so its rules govern nothing`);
+    }
+    const admitted = (policy.spec?.ingress ?? []).some((rule) => ruleAdmits(rule, RUN_COMPONENT_LABEL, CODING_PROXY_DENY_PORT));
+    if (!admitted) {
+        throw new ProxyWitnessError(`${where} has no ingress rule that admits ${JSON.stringify(RUN_COMPONENT_LABEL)} on port ` +
+            `${CODING_PROXY_DENY_PORT}/TCP. The gate reads a dropped connect to that port as proof the run's ` +
+            "own egress policy is enforced; if the proxy does not admit it, the proxy's ingress drops it instead " +
+            "and every run reads as enforced while having no egress policy at all");
+    }
+}
+export async function readProxyWitness(api, namespace, service) {
+    const where = `Service ${namespace}/${service}`;
+    const found = await api.readService(namespace, service);
+    if (!found)
+        throw new ProxyWitnessError(`${where} was not found`);
+    if (!selectorRequires(found.spec?.selector, PROXY_POD_LABEL)) {
+        throw new ProxyWitnessError(`${where} has selector ${JSON.stringify(found.spec?.selector ?? {})}, which does not require ` +
+            `${JSON.stringify(PROXY_POD_LABEL)}; the endpoints it routes to are not provably the pods the checked ` +
+            "NetworkPolicy governs");
+    }
+    const clusterIp = found.spec?.clusterIP;
+    if (!clusterIp || clusterIp === "None" || isIP(clusterIp) === 0) {
+        throw new ProxyWitnessError(`${where} has no usable ClusterIP`);
+    }
+    const exposed = new Set((found.spec?.ports ?? []).filter((port) => isTcp(port.protocol)).map((port) => port.port));
+    for (const required of REQUIRED_PORTS) {
+        if (!exposed.has(required))
+            throw new ProxyWitnessError(`${where} does not expose port ${required}`);
+    }
+    const endpoints = await api.readEndpoints(namespace, service);
+    if (!endpoints)
+        throw new ProxyWitnessError(`${where} has no Endpoints`);
+    // Endpoints subset ports are the container's targetPorts, not the Service's own port
+    // numbers checked above. Comparing them directly here is only correct because the
+    // manifest keeps targetPort == port for both the proxy and deny ports; a future re-map
+    // (Service port != container targetPort) fails closed here rather than trusting a port
+    // that merely happens to share a number with the deny port.
+    let hasReadyAddress = false;
+    const readyPorts = new Set();
+    for (const subset of endpoints.subsets ?? []) {
+        const ready = (subset.addresses ?? []).some((address) => typeof address.ip === "string" && isIP(address.ip) !== 0);
+        if (!ready)
+            continue;
+        hasReadyAddress = true;
+        for (const port of subset.ports ?? []) {
+            if (typeof port.port === "number" && isTcp(port.protocol))
+                readyPorts.add(port.port);
+        }
+    }
+    if (!hasReadyAddress)
+        throw new ProxyWitnessError(`${where} has no ready endpoint address`);
+    if (readyPorts.size === 0)
+        throw new ProxyWitnessError(`${where} has a ready endpoint that declares no ports`);
+    for (const required of REQUIRED_PORTS) {
+        if (!readyPorts.has(required)) {
+            throw new ProxyWitnessError(`${where} has no ready endpoint serving port ${required}`);
+        }
+    }
+    await assertDenyPortAdmitted(api, namespace, service);
+    return { clusterIp };
+}

package/dist/providers/jobs/kubernetes.d.ts

@@ -0,0 +1,127 @@
+import { type Readable } from "node:stream";
+import type { KubernetesJobConfig } from "../../config/providers.js";
+import { type KubernetesApi } from "./kubernetes-api.js";
+import type { JobHandle, JobResult, JobSpec, JobStatus, WorkspaceJobLauncher } from "./types.js";
+export interface KubernetesClusterInfo {
+    /** The proxy Service's ClusterIP: the enforcement witness's address (8787 reachable, 8788 not). */
+    proxyIp: string;
+}
+export interface KubernetesJobLauncherOptions {
+    api: KubernetesApi;
+    config: KubernetesJobConfig;
+    workspaceRoot: string;
+    resolveCapability: (runId: string) => Promise<string>;
+    /**
+     * Cluster preflight run once before the first launch; failure fails every launch. It may return the
+     * proxy ClusterIP it validated; otherwise the launcher reads the proxy witness once itself. Either way
+     * the gate probes the address `provision` re-reads for that launch, not this memoized one.
+     */
+    preflight?: () => Promise<KubernetesClusterInfo | void>;
+    now?: () => number;
+    sleep?: (milliseconds: number) => Promise<void>;
+    readyTimeoutMs?: number;
+    /** How long to wait for the run's NetworkPolicy to be enforced before seeding. Default 30_000. */
+    enforcementTimeoutMs?: number;
+    createArchive?: (directory: string) => {
+        stream: Readable;
+        done: Promise<number>;
+    };
+    onWarning?: (message: string) => void;
+}
+/** Archives a trusted local directory with the host's `tar`, under a minimal environment. */
+export declare function hostTarArchive(directory: string): {
+    stream: Readable;
+    done: Promise<number>;
+};
+export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
+    private readonly options;
+    private readonly api;
+    private readonly namespace;
+    private readonly workspaceRoot;
+    private readonly now;
+    private readonly sleep;
+    private readonly readyTimeoutMs;
+    private readonly createArchive;
+    private readonly warn;
+    private readonly enforcementTimeoutMs;
+    private preflightResult?;
+    constructor(options: KubernetesJobLauncherOptions);
+    /**
+     * Every per-run object name derives from the run id, so the handle is known before any cluster call.
+     * The executor persists it first; a crash mid-launch then still has a handle to clean up with (the
+     * launcher itself has no `list` permission and so can never rediscover stray objects).
+     */
+    plannedHandle(spec: JobSpec): JobHandle | undefined;
+    launch(spec: JobSpec): Promise<JobHandle>;
+    status(handle: JobHandle): Promise<JobStatus>;
+    collect(handle: JobHandle): Promise<JobResult>;
+    materializeWorkspace(handle: JobHandle, destination: string): Promise<void>;
+    stop(handle: JobHandle, _reason?: string): Promise<void>;
+    remove(handle: JobHandle): Promise<void>;
+    /** Memoized: the preflight (or, without one, a single proxy-witness read) runs once per launcher. */
+    private runPreflight;
+    /**
+     * Waits until the run's NetworkPolicy is enforced on this pod: the keeper (same network namespace
+     * as the worker) must, on ENFORCEMENT_BLOCKED_STREAK consecutive probes 500 ms apart, reach the
+     * proxy on CODING_PROXY_PORT and fail to reach it on CODING_PROXY_DENY_PORT. Anything else resets
+     * the streak; the wall-clock bound still applies.
+     *
+     * "Fail to reach" means the connect *timed out* — the packet was dropped. A refusal (RST) is not
+     * a denial: it proves the SYN reached the destination host, so the deny port is merely unserved
+     * and witnesses nothing. That is its own verdict, below.
+     *
+     * The verdict at the bound is taken from the *last* probe, not from whether any probe was ever
+     * unavailable: an early blip while the pod's networking came up must not send an operator looking
+     * at the proxy when the real problem is an unenforced policy. The three non-proven outcomes stay
+     * distinct because they send an operator to three different places: the CNI, the proxy pod, and
+     * the deny listener.
+     *
+     * `stage` only changes which error codes a bound produces (see `EnforcementStage`); the probe
+     * machinery itself — the script, the streak, the poll interval, the wall-clock bound — is identical
+     * for the initial proof and for `provision`'s later re-confirmation immediately before the marker
+     * write. That re-confirmation deliberately requires the same full `ENFORCEMENT_BLOCKED_STREAK`
+     * consecutive proven probes, not a single one-shot probe: seeding (the window it's guarding) can run
+     * for real hours, so the few hundred extra milliseconds a streak costs in the common, still-enforced
+     * case is negligible, while a single probe could pass on a lucky sample the same way one dropped SYN
+     * must not open the gate at the initial proof either.
+     */
+    private waitForPolicyEnforcement;
+    private existingLaunch;
+    private provision;
+    private createIfMissing;
+    private waitForKeeper;
+    private runWorkspace;
+    /** Streams a trusted local directory into `destination` inside the keeper. */
+    private seedDirectory;
+    private seedInput;
+    private cleanupRun;
+    /** Observes a non-terminal run against its pod and persists any transition; returns the stored record. */
+    private refresh;
+    /** Reads only a fixed diagnostic code from a failed worker's bounded log tail; never keeps anything else. */
+    private readWorkerDiagnostic;
+    private readResultArtifact;
+    /**
+     * Streams the keeper's workspace through the strict extractor into `staging`.
+     * The transfer is bounded by `timeoutMs` (see transferBudgetMs): on expiry the
+     * extraction is aborted. On any failure the output stream this method created
+     * is destroyed, since the exec's own timeout does not end it.
+     */
+    private extractWorkspace;
+    /**
+     * Time allowed for one workspace/input transfer: the run's full timeout
+     * (deadlineAt - createdAt), measured from when the transfer starts. The
+     * remaining run deadline is not used because collection legitimately
+     * happens after a run has finished close to (or past) its deadline.
+     */
+    private transferBudgetMs;
+    private namesFor;
+    private recordConfigMap;
+    private readRecord;
+    /**
+     * Applies a transition with optimistic concurrency, retrying on conflict.
+     * `transition` sees the freshly read record and returns the new phase/result
+     * (or `undefined` for no change). A terminal record never moves to another
+     * phase except `removed`; it may only have its result filled in.
+     */
+    private updateRecord;
+}