@telorun/k8s-runner 0.13.0 → 0.15.0

package/src/config.ts

@@ -46,31 +46,49 @@ export interface LimitCeilings {
 }
 
 /**
- * On-cluster image-build settings. The runner ALWAYS prebuilds a self-contained
- * per-app image (controllers + module manifests baked in via `telo install`) and
- * the session pod runs it directly — there is no in-pod install fallback, so a
- * registry repository (`RUNNER_IMAGE_REPOSITORY`) is required.
+ * Which routing layer the runner publishes per-session routes on.
+ *
+ *  - `auto`    — resolve from what is CONFIGURED, then from what the cluster
+ *                serves; refuse at boot when the answer is genuinely ambiguous.
+ *  - `ingress` — `networking.k8s.io/v1` Ingress.
+ *  - `gateway` — `gateway.networking.k8s.io/v1` HTTPRoute.
+ *  - `none`    — publish nothing; the session is logs-only even with a base
+ *                domain set.
  */
-export interface ImageBuildConfig {
-  /** Registry repository the per-app image is pushed to / pulled from; the tag
-   *  is the bundle content hash (e.g. `registry.telo-runner.svc:5000/telo-sessions`). */
-  repository: string;
-  /** Namespace the trusted Kaniko build Jobs run in (registry egress lives here). */
+export type SessionRoutingMode = "auto" | "ingress" | "gateway" | "none";
+
+const ROUTING_MODES: readonly SessionRoutingMode[] = ["auto", "ingress", "gateway", "none"];
+
+/** The Gateway a session's HTTPRoute attaches to. Never detected: picking one of
+ *  several Gateways is the same guess as picking one of two routing APIs, and a
+ *  route attached to the wrong Gateway fails exactly as silently as no route. */
+export interface GatewayParentConfig {
+  name: string;
+  /** Defaults to the session namespace — mirroring Gateway API's own
+   *  `parentRefs[].namespace` default, which is the route's own namespace. */
   namespace: string;
-  /** Builder image — Kaniko (or a compatible `--context`/`--destination` executor). */
-  builderImage: string;
-  /** Build Job `activeDeadlineSeconds` and the runner's wait budget. */
-  timeoutSeconds: number;
-  /** Push/pull to an insecure (HTTP / self-signed) registry — the in-cluster default. */
-  insecureRegistry: boolean;
-  /** HTTP(S) base for a best-effort manifest existence check (skip build on hit);
-   *  undefined → always build. */
-  registryApiUrl?: string;
-  /** Optional dockerconfig Secret (in the build namespace) Kaniko pushes with. */
-  pushSecretName?: string;
-  /** Optional dockerconfig Secret (in the session namespace) the kubelet uses to
-   *  pull per-app images from a private registry. */
-  imagePullSecret?: string;
+  /** Optional listener name, when the Gateway has several and only one should
+   *  carry session traffic. */
+  sectionName?: string;
+}
+
+export interface SessionRoutingConfig {
+  mode: SessionRoutingMode;
+  /** Wildcard base domain for per-session hosts; unset → logs-only whatever the
+   *  mode, since without it no host can be constructed at all. */
+  baseDomain?: string;
+  /** IngressClass for created session Ingresses (ingress mode). */
+  ingressClassName?: string;
+  /** `kubernetes.io/tls` Secret (in the session namespace) the per-session
+   *  Ingress presents so an upstream (e.g. Cloudflare Full (Strict)) can validate
+   *  the origin. Must cover `*.<baseDomain>`. INGRESS MODE ONLY — under Gateway
+   *  API the certificate belongs to the Gateway's listener, so setting this in
+   *  gateway mode is refused at boot rather than silently ignored. */
+  tlsSecretName?: string;
+  gateway?: GatewayParentConfig;
+  /** How long a published route may go unclaimed by any controller before it is
+   *  reported `unprogrammed`. */
+  routeReadyTimeoutMs: number;
 }
 
 export interface K8sRunnerConfig extends RunnerCoreConfig {
@@ -78,28 +96,24 @@ export interface K8sRunnerConfig extends RunnerCoreConfig {
    *  with these. */
   displayName: string;
   description: string;
-  /** Namespace where session Pods/Services/Ingresses are created. */
+  /** Namespace where session Pods/Services and routing objects are created. */
   sessionNamespace: string;
   /** Default image for spawned session Pods (telorun/node). Baked into the runner
    *  image as `RUNNER_IMAGE` at build time (the kernel version the runner was
    *  built against); the literal fallback below is for a source checkout only. */
   defaultImage: string;
-  /** Small image for the bundle/build-context fetch initContainer (needs wget + tar). */
+  /** Small image for the bundle-fetch initContainer (needs wget + tar). */
   initImage: string;
+  /** Optional dockerconfig Secret (in the session namespace) the kubelet pulls
+   *  session images with — the kernel image, and any operator catalog image
+   *  (`RUNNER_APPS`) held in a private registry. */
+  imagePullSecret?: string;
   /** Optional sandbox RuntimeClass (gvisor/kata). Unset → cluster default (runc). */
   runtimeClass?: string;
-  /** Wildcard base domain for per-session ingress; unset → logs-only. */
-  sessionIngressBaseDomain?: string;
-  /** Optional IngressClass name for created session Ingresses. */
-  sessionIngressClassName?: string;
-  /** Optional `kubernetes.io/tls` Secret (in the session namespace) the per-session
-   *  Ingress presents so an upstream (e.g. Cloudflare Full (Strict)) can validate
-   *  the origin. Must cover the wildcard `*.<sessionIngressBaseDomain>`. Unset → no TLS block. */
-  sessionIngressTlsSecretName?: string;
-  /** On-cluster image build — the only delivery path (always present). */
-  build: ImageBuildConfig;
-  /** Runner's own in-cluster base URL, used to build the bundle/build-context
-   *  fetch URL the initContainer curls (e.g. http://k8s-runner.telo-runner:8062). */
+  /** How per-session routes are published. */
+  sessionRouting: SessionRoutingConfig;
+  /** Runner's own in-cluster base URL, used to build the bundle fetch URL the
+   *  initContainer curls (e.g. http://k8s-runner.telo-runner:8062). */
   selfUrl: string;
   /** Label applied to every session object, used for orphan reaping. */
   managedByLabel: string;
@@ -128,6 +142,69 @@ function parseTagRegex(raw: string | undefined, field: string): RegExp[] | undef
   }
 }
 
+/** Parse the routing mode, THROWING on an unrecognized one rather than degrading
+ *  to a default — a typo'd mode that silently fell back to `auto` is the same
+ *  class of failure this whole mechanism exists to remove. */
+function parseRoutingMode(raw: string | undefined): SessionRoutingMode {
+  const v = raw?.trim();
+  if (!v) return "auto";
+  if ((ROUTING_MODES as readonly string[]).includes(v)) return v as SessionRoutingMode;
+  throw new RunnerConfigError(
+    `SESSION_ROUTING_MODE must be one of ${ROUTING_MODES.join(" | ")} (got '${v}').`,
+  );
+}
+
+function loadSessionRoutingConfig(
+  env: NodeJS.ProcessEnv,
+  sessionNamespace: string,
+): SessionRoutingConfig {
+  const mode = parseRoutingMode(env.SESSION_ROUTING_MODE);
+  const gatewayName = env.SESSION_GATEWAY_NAME?.trim() || undefined;
+  const tlsSecretName = env.SESSION_INGRESS_TLS_SECRET?.trim() || undefined;
+
+  // A certificate configured for a layer that will never present it is a silent
+  // downgrade to plaintext origin traffic, so it is refused rather than ignored.
+  // This is only the FAST PATH, for the mode that states the layer outright: the
+  // authoritative refusal is at routing resolution, because under `auto` a named
+  // Gateway decides the layer and nothing here can see that yet.
+  if (mode === "gateway" && tlsSecretName) {
+    throw new RunnerConfigError(
+      "SESSION_INGRESS_TLS_SECRET is set but SESSION_ROUTING_MODE=gateway. Under Gateway API the " +
+        "origin certificate belongs to the Gateway listener's own `tls.certificateRefs`, not to the " +
+        "per-session route. Move the Secret reference to the Gateway and unset this.",
+    );
+  }
+  if (mode === "gateway" && !gatewayName) {
+    throw new RunnerConfigError(
+      "SESSION_ROUTING_MODE=gateway requires SESSION_GATEWAY_NAME — the Gateway each session's " +
+        "HTTPRoute attaches to. Set it (and SESSION_GATEWAY_NAMESPACE when the Gateway is not in " +
+        `'${sessionNamespace}').`,
+    );
+  }
+
+  return {
+    mode,
+    baseDomain: env.SESSION_ROUTING_BASE_DOMAIN?.trim() || undefined,
+    ingressClassName: env.SESSION_INGRESS_CLASS?.trim() || undefined,
+    tlsSecretName,
+    ...(gatewayName
+      ? {
+          gateway: {
+            name: gatewayName,
+            namespace: env.SESSION_GATEWAY_NAMESPACE?.trim() || sessionNamespace,
+            sectionName: env.SESSION_GATEWAY_SECTION_NAME?.trim() || undefined,
+          },
+        }
+      : {}),
+    routeReadyTimeoutMs:
+      parsePositiveInt(
+        env.SESSION_ROUTE_READY_TIMEOUT_SECONDS,
+        60,
+        "SESSION_ROUTE_READY_TIMEOUT_SECONDS",
+      ) * 1000,
+  };
+}
+
 function loadBaseImageCatalogConfig(env: NodeJS.ProcessEnv): BaseImageCatalogConfig {
   return {
     enabled: parseBool(env.RUNNER_BASE_IMAGE_CATALOG_ENABLED, true, "RUNNER_BASE_IMAGE_CATALOG_ENABLED"),
@@ -162,50 +239,35 @@ export function loadK8sRunnerConfig(env: NodeJS.ProcessEnv): K8sRunnerConfig {
     );
   }
 
-  const repository = env.RUNNER_IMAGE_REPOSITORY?.trim();
-  if (!repository) {
-    throw new RunnerConfigError(
-      "RUNNER_IMAGE_REPOSITORY env var is required. The runner prebuilds a per-app session image for " +
-        "every run (there is no in-pod install fallback); set it to the registry repository the built " +
-        "images are pushed to / pulled from (e.g. registry.example.com/telo-sessions). The tag is the " +
-        "bundle content hash.",
-    );
-  }
-  const build: ImageBuildConfig = {
-    repository: repository.replace(/\/+$/, ""),
-    namespace: env.RUNNER_BUILD_NAMESPACE?.trim() || "telo-builds",
-    builderImage: env.RUNNER_BUILDER_IMAGE?.trim() || "gcr.io/kaniko-project/executor:latest",
-    timeoutSeconds: parsePositiveInt(
-      env.RUNNER_BUILD_TIMEOUT_SECONDS,
-      600,
-      "RUNNER_BUILD_TIMEOUT_SECONDS",
-    ),
-    insecureRegistry: env.RUNNER_REGISTRY_INSECURE?.trim() === "true",
-    registryApiUrl: env.RUNNER_REGISTRY_API_URL?.trim() || undefined,
-    pushSecretName: env.RUNNER_REGISTRY_PUSH_SECRET?.trim() || undefined,
-    imagePullSecret: env.RUNNER_IMAGE_PULL_SECRET?.trim() || undefined,
-  };
+  const sessionNamespace = env.RUNNER_SESSION_NAMESPACE?.trim() || "telo-sessions";
 
   return {
     ...loadCoreConfig(env, { port: DEFAULT_PORT }),
     displayName: env.RUNNER_DISPLAY_NAME?.trim() || "Telo Runner",
     description:
       env.RUNNER_DESCRIPTION?.trim() || "Runs the Telo application in a cloud environment",
-    sessionNamespace: env.RUNNER_SESSION_NAMESPACE?.trim() || "telo-sessions",
+    sessionNamespace,
     defaultImage: env.RUNNER_IMAGE?.trim() || "telorun/node:latest-slim",
     initImage: env.RUNNER_INIT_IMAGE?.trim() || "busybox:stable",
+    imagePullSecret: env.RUNNER_IMAGE_PULL_SECRET?.trim() || undefined,
     runtimeClass: env.RUNNER_RUNTIME_CLASS?.trim() || undefined,
-    sessionIngressBaseDomain: env.SESSION_INGRESS_BASE_DOMAIN?.trim() || undefined,
-    sessionIngressClassName: env.SESSION_INGRESS_CLASS?.trim() || undefined,
-    sessionIngressTlsSecretName: env.SESSION_INGRESS_TLS_SECRET?.trim() || undefined,
-    build,
+    sessionRouting: loadSessionRoutingConfig(env, sessionNamespace),
     selfUrl: selfUrl.replace(/\/+$/, ""),
     managedByLabel: env.RUNNER_MANAGED_BY?.trim() || "telo-k8s-runner",
+    // Sized for a pod that RESOLVES ITS OWN module closure. They used to be
+    // 50m / 100Mi / 512Mi, which described a different workload: the closure
+    // arrived in image layers, so the pod only ran it. It now downloads,
+    // unpacks and resolves it in-pod, into an emptyDir that counts against
+    // ephemeral-storage — so the old numbers meant an OOMKill on the memory
+    // ceiling and an eviction on the storage one, for the ordinary case. The
+    // watch path already ran this workload and already carried the roomier
+    // ceiling; these match it. `appLimits` stays separate: it still differs in
+    // TTL, and both remain the operator's policy to tighten.
     limits: {
-      cpu: env.RUNNER_MAX_CPU?.trim() || "50m",
-      memory: env.RUNNER_MAX_MEMORY?.trim() || "100Mi",
+      cpu: env.RUNNER_MAX_CPU?.trim() || "500m",
+      memory: env.RUNNER_MAX_MEMORY?.trim() || "512Mi",
       ttlSeconds: parsePositiveInt(env.RUNNER_MAX_TTL_SECONDS, 3600, "RUNNER_MAX_TTL_SECONDS"),
-      ephemeralStorage: env.RUNNER_MAX_EPHEMERAL_STORAGE?.trim() || "512Mi",
+      ephemeralStorage: env.RUNNER_MAX_EPHEMERAL_STORAGE?.trim() || "1Gi",
     },
     appLimits: {
       cpu: env.RUNNER_APP_MAX_CPU?.trim() || "500m",

package/src/k8s/api-error.test.ts

@@ -0,0 +1,89 @@
+import { ApiException } from "@kubernetes/client-node";
+import { describe, expect, it } from "vitest";
+
+import { apiFailure, apiReason } from "./api-error.js";
+
+/** A 403 with an arbitrary admission message. `body` is the raw JSON text,
+ *  which is what the client-node deserializer leaves for a status code it does
+ *  not recognize — exactly the 403 case ("Unknown API Status Code!"). */
+function forbidden(message: string): ApiException<string> {
+  const body = JSON.stringify({
+    kind: "Status",
+    apiVersion: "v1",
+    status: "Failure",
+    message,
+    reason: "Forbidden",
+    code: 403,
+  });
+  return new ApiException<string>(403, "Unknown API Status Code!", body, {
+    "audit-id": "240bebaf-17aa-4684-8732-210b5b720f5a",
+    "x-kubernetes-pf-flowschema-uid": "f895735a-d707-400c-88de-23e24406fa8c",
+  });
+}
+
+const RBAC_DENIAL =
+  'pods is forbidden: User "system:serviceaccount:telo-runner:telo-k8s-runner" cannot create resource "pods" in the namespace "telo-sessions"';
+const QUOTA_DENIAL =
+  'pods "telo-run-abc" is forbidden: exceeded quota: telo-k8s-runner-sessions, requested: pods=1, used: pods=32, limited: pods=32';
+
+describe("apiReason", () => {
+  it("keeps the status and reason, and nothing else the exception carried", () => {
+    const reason = apiReason(forbidden(RBAC_DENIAL));
+    expect(reason).toContain("Forbidden");
+    expect(reason).toContain("403");
+    // The whole point: no cluster identities, no audit id, no headers.
+    expect(reason).not.toContain("system:serviceaccount");
+    expect(reason).not.toContain("audit-id");
+    expect(reason).not.toContain("flowschema");
+  });
+
+  it("names RBAC as the fix for a rejection only the operator can clear", () => {
+    expect(apiReason(forbidden(RBAC_DENIAL))).toContain("RBAC");
+  });
+
+  it("tells a quota rejection from an RBAC one — same code, same reason", () => {
+    // The chart ships a 32-pod ResourceQuota enabled by default, so the 33rd
+    // session is a 403 `Forbidden` that has nothing to do with RBAC. Sending
+    // that operator to re-check a Role is the wrong action at the one moment
+    // the message is read under load.
+    const reason = apiReason(forbidden(QUOTA_DENIAL));
+    expect(reason).toContain("quota");
+    expect(reason).not.toContain("RBAC");
+    // The admission message is classified, never echoed.
+    expect(reason).not.toContain("telo-k8s-runner-sessions");
+  });
+
+  it("stays neutral on a 403 whose message identifies neither", () => {
+    const reason = apiReason(forbidden("denied by a validating webhook"));
+    expect(reason).toContain("rejected the request");
+    expect(reason).not.toContain("RBAC");
+    expect(reason).not.toContain("quota");
+  });
+
+  it("reads a parsed status body as well as a raw one", () => {
+    const parsed = new ApiException<unknown>(404, "Not Found", { reason: "NotFound" }, {});
+    expect(apiReason(parsed)).toContain("NotFound");
+  });
+
+  it("reports an unreachable apiserver as unreachable, not as a rejection", () => {
+    // A system error's `code` is a STRING and a DOMException's is a small
+    // number, both in the field an HTTP status also lands in. Reading either as
+    // a status claimed the apiserver refused something it was never asked.
+    const refused = Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" });
+    const aborted = Object.assign(new Error("The operation was aborted"), { code: 20 });
+    for (const err of [refused, aborted, new Error("socket hang up")]) {
+      expect(apiReason(err)).toBe("the Kubernetes API could not be reached");
+    }
+  });
+});
+
+describe("apiFailure", () => {
+  it("carries the raw exception as `cause` so the runner log keeps the detail", () => {
+    const raw = forbidden(RBAC_DENIAL);
+    const failure = apiFailure(raw, "create", "could not create the session pod");
+    expect(failure.stage).toBe("create");
+    expect(failure.message).toContain("could not create the session pod");
+    expect(failure.message).not.toContain("system:serviceaccount");
+    expect(failure.cause).toBe(raw);
+  });
+});

package/src/k8s/api-error.ts

@@ -0,0 +1,128 @@
+import { SessionStartError, type StartFailureStage } from "@telorun/runner-core";
+
+/**
+ * A Kubernetes API rejection, phrased for the person who clicked Run.
+ *
+ * `ApiException.message` is a full HTTP dump — status line, the raw `Status`
+ * body and every response header — and a start failure's message travels
+ * verbatim to the client as the session's terminal `failed` status. That put
+ * the runner's ServiceAccount name, the request's audit id and its flowschema
+ * UIDs on an end user's screen, none of which they can act on.
+ *
+ * So the client sees the operation, the HTTP status and the API's own one-word
+ * `reason` — enough for an operator to know what was refused — and never the
+ * body or the headers. The raw exception rides along as the error's `cause`,
+ * which the runner's log serializer records, so nothing is swallowed: the detail
+ * moves to the log, it does not disappear.
+ */
+export function apiFailure(
+  err: unknown,
+  stage: StartFailureStage,
+  action: string,
+): SessionStartError {
+  return withCause(
+    new SessionStartError("start_failed", stage, `${action}: ${apiReason(err)}`),
+    err,
+  );
+}
+
+/** Attach the original error as `cause` so a log serializer can reach it.
+ *  Every wrap on a failure path goes through here — a rewrap that drops the
+ *  cause silently undoes the whole point of summarizing the message. */
+export function withCause<E extends Error>(error: E, cause: unknown): E {
+  error.cause = cause;
+  return error;
+}
+
+/** The client-safe half of a Kubernetes API error: what was refused, and who
+ *  can act on it — never the `Status` message, which names cluster identities. */
+export function apiReason(err: unknown): string {
+  const code = statusCode(err);
+  // No HTTP status means the request never got an answer: a refused connection,
+  // a DNS failure, an abort. Those carry a `code` too (`ECONNREFUSED`, an
+  // `ABORT_ERR` number), which is why `statusCode` accepts only what an HTTP
+  // response could have produced — reporting `HTTP ECONNREFUSED` claimed the
+  // apiserver rejected something when it was never reached, which is exactly
+  // the case where the difference matters.
+  if (code === undefined) return "the Kubernetes API could not be reached";
+
+  const reason = statusReason(err);
+  const detail = reason ? `${reason}, HTTP ${code}` : `HTTP ${code}`;
+  const cause = refusalCause(err);
+  if (cause === "quota") {
+    return `the session namespace's resource quota is exhausted (${detail})`;
+  }
+  if (cause === "rbac") {
+    return `the runner is not permitted to do this (${detail}). The cluster operator must check the runner's RBAC.`;
+  }
+  return `the Kubernetes API rejected the request (${detail})`;
+}
+
+/**
+ * Which of the two things a `403 Forbidden` means here.
+ *
+ * The status code alone cannot say: an RBAC denial and a `ResourceQuota`
+ * rejection are both `403` with `reason: Forbidden`, and this chart ships a
+ * quota (32 pods) enabled by default — so blaming RBAC on every 403 tells the
+ * operator to re-check a Role at exactly the moment the cluster is simply full.
+ * The two also differ in when they happen: an RBAC 403 is an install-time
+ * mistake, a quota 403 arrives under load.
+ *
+ * The only thing that separates them is the admission message, which is why it
+ * is READ here and never echoed: the classification crosses the boundary, the
+ * text does not. An unrecognized 403 degrades to the neutral wording rather than
+ * to a guess.
+ */
+function refusalCause(err: unknown): "quota" | "rbac" | undefined {
+  const code = statusCode(err);
+  if (code !== 401 && code !== 403) return undefined;
+  const message = statusMessage(err);
+  if (message && /exceeded quota|forbidden: failed quota/i.test(message)) return "quota";
+  if (!message || /\bis forbidden: User\b|\bcannot \w+ resource\b/i.test(message)) return "rbac";
+  return undefined;
+}
+
+/** The API `Status` object, however the client-node deserializer left it: a
+ *  parsed object for a status code it recognizes, the raw JSON text for one it
+ *  does not (which is the 403 case, reported as "Unknown API Status Code!"). */
+function status(err: unknown): Record<string, unknown> | undefined {
+  const body = (err as { body?: unknown })?.body;
+  const parsed = typeof body === "string" ? parseJson(body) : body;
+  return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : undefined;
+}
+
+/** `Forbidden`, `NotFound`, `Invalid`, … — the one word safe to pass on. */
+function statusReason(err: unknown): string | undefined {
+  const reason = status(err)?.reason;
+  return typeof reason === "string" && reason.trim() !== "" ? reason : undefined;
+}
+
+/** Read for classification only — it names cluster identities and never leaves. */
+function statusMessage(err: unknown): string | undefined {
+  const message = status(err)?.message;
+  return typeof message === "string" ? message : undefined;
+}
+
+function parseJson(text: string): unknown {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * The HTTP status an error carries, or `undefined` when it carries none.
+ *
+ * The number check is the whole point: `code` is also where Node puts a system
+ * error's string code and where a `DOMException` puts its legacy numeric one, so
+ * an unreachable apiserver and an aborted request both arrive here looking like
+ * a status. Only a value in the HTTP range is one.
+ */
+export function statusCode(err: unknown): number | undefined {
+  const e = err as { statusCode?: unknown; code?: unknown; response?: { statusCode?: unknown } };
+  for (const candidate of [e?.statusCode, e?.code, e?.response?.statusCode]) {
+    if (typeof candidate === "number" && candidate >= 100 && candidate <= 599) return candidate;
+  }
+  return undefined;
+}