@telorun/k8s-runner 0.14.0 → 0.15.0

package/src/k8s/routing/route-health.test.ts

@@ -0,0 +1,215 @@
+import { describe, expect, it } from "vitest";
+
+import type { RouteState } from "@telorun/runner-core";
+
+import { watchRouteHealth } from "./route-health.js";
+import type { PublishedRoute, RouteVerdict, SessionRouter } from "./session-router.js";
+
+const routes: PublishedRoute[] = [
+  { host: "8080-abc.telo.run", port: 8080 },
+  { host: "9090-abc.telo.run", port: 9090 },
+];
+
+function router(verdicts: (route: PublishedRoute) => RouteVerdict): SessionRouter {
+  return {
+    layer: "gateway",
+    publish: async () => routes,
+    verdictFor: async (_id, route) => verdicts(route),
+    unclaimedReason: () => "nobody claimed it",
+  };
+}
+
+function collect() {
+  const seen: Array<{ host: string; state: RouteState; reason?: string }> = [];
+  return {
+    seen,
+    onState: (route: PublishedRoute, state: RouteState, reason?: string) =>
+      seen.push({ host: route.host, state, ...(reason ? { reason } : {}) }),
+  };
+}
+
+describe("watchRouteHealth", () => {
+  it("reports pending up front, then programmed per host", async () => {
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: router(() => ({ kind: "programmed" })),
+      sessionId: "abc",
+      routes,
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 1_000,
+      intervalMs: 1,
+    });
+    expect(seen).toEqual([
+      { host: "8080-abc.telo.run", state: "pending" },
+      { host: "9090-abc.telo.run", state: "pending" },
+      { host: "8080-abc.telo.run", state: "programmed" },
+      { host: "9090-abc.telo.run", state: "programmed" },
+    ]);
+  });
+
+  it("carries a refusal's reason straight through", async () => {
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: router(() => ({ kind: "rejected", reason: "NotAllowedByListeners" })),
+      sessionId: "abc",
+      routes: [routes[0]],
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 1_000,
+      intervalMs: 1,
+    });
+    expect(seen.at(-1)).toEqual({
+      host: "8080-abc.telo.run",
+      state: "unprogrammed",
+      reason: "NotAllowedByListeners",
+    });
+  });
+
+  it("calls a route that stays pending unprogrammed at the deadline — the silent-404 case", async () => {
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: router(() => ({ kind: "pending" })),
+      sessionId: "abc",
+      routes: [routes[0]],
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 5,
+      intervalMs: 1,
+    });
+    expect(seen.at(-1)).toEqual({
+      host: "8080-abc.telo.run",
+      state: "unprogrammed",
+      reason: "nobody claimed it",
+    });
+  });
+
+  it("settles each host independently", async () => {
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: router((route) =>
+        route.port === 8080 ? { kind: "programmed" } : { kind: "rejected", reason: "no listener" },
+      ),
+      sessionId: "abc",
+      routes,
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 1_000,
+      intervalMs: 1,
+    });
+    expect(seen.filter((s) => s.state !== "pending")).toEqual([
+      { host: "8080-abc.telo.run", state: "programmed" },
+      { host: "9090-abc.telo.run", state: "unprogrammed", reason: "no listener" },
+    ]);
+  });
+
+  it("keeps polling through a transient API error instead of declaring the route dead", async () => {
+    let calls = 0;
+    const flaky: SessionRouter = {
+      layer: "ingress",
+      publish: async () => routes,
+      verdictFor: async () => {
+        calls += 1;
+        if (calls < 3) throw new Error("apiserver hiccup");
+        return { kind: "programmed" };
+      },
+      unclaimedReason: () => "nobody claimed it",
+    };
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: flaky,
+      sessionId: "abc",
+      routes: [routes[0]],
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 1_000,
+      intervalMs: 1,
+    });
+    expect(seen.at(-1)).toEqual({ host: "8080-abc.telo.run", state: "programmed" });
+  });
+
+  it("reports the READ failure at the deadline, not the Gateway advice", async () => {
+    // A permanent read failure (no RBAC, an unserved API version) polls out the
+    // whole deadline. Reporting `unclaimedReason()` there sends an operator to
+    // inspect the Gateway when the runner could not read its own object.
+    const denied: SessionRouter = {
+      layer: "gateway",
+      publish: async () => routes,
+      verdictFor: async () => {
+        throw Object.assign(new Error("httproutes is forbidden"), { code: 403 });
+      },
+      unclaimedReason: () => "no controller claimed the session HTTPRoute",
+    };
+    const seenErrors: unknown[] = [];
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: denied,
+      sessionId: "abc",
+      routes: [routes[0]],
+      onState,
+      onReadError: (err) => seenErrors.push(err),
+      signal: new AbortController().signal,
+      timeoutMs: 5,
+      intervalMs: 1,
+    });
+    expect(seen.at(-1)?.state).toBe("unprogrammed");
+    expect(seen.at(-1)?.reason).toMatch(/route status could not be read/);
+    expect(seen.at(-1)?.reason).not.toMatch(/claimed the session HTTPRoute/);
+    // Reported once, not once per poll.
+    expect(seenErrors).toHaveLength(1);
+  });
+
+  it("does not blame a transient read failure once a verdict was obtained", async () => {
+    let calls = 0;
+    const flaky: SessionRouter = {
+      layer: "gateway",
+      publish: async () => routes,
+      verdictFor: async () => {
+        calls += 1;
+        if (calls === 1) throw new Error("apiserver hiccup");
+        return { kind: "pending" };
+      },
+      unclaimedReason: () => "nobody claimed it",
+    };
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: flaky,
+      sessionId: "abc",
+      routes: [routes[0]],
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 8,
+      intervalMs: 1,
+    });
+    expect(seen.at(-1)?.reason).toBe("nobody claimed it");
+  });
+
+  it("stops on abort without reporting a verdict it never reached", async () => {
+    const abort = new AbortController();
+    const { seen, onState } = collect();
+    abort.abort();
+    await watchRouteHealth({
+      router: router(() => ({ kind: "pending" })),
+      sessionId: "abc",
+      routes,
+      onState,
+      signal: abort.signal,
+      timeoutMs: 1_000,
+      intervalMs: 1,
+    });
+    expect(seen.every((s) => s.state === "pending")).toBe(true);
+  });
+
+  it("does nothing when nothing was published", async () => {
+    const { seen, onState } = collect();
+    await watchRouteHealth({
+      router: router(() => ({ kind: "programmed" })),
+      sessionId: "abc",
+      routes: [],
+      onState,
+      signal: new AbortController().signal,
+      timeoutMs: 1_000,
+    });
+    expect(seen).toEqual([]);
+  });
+});

package/src/k8s/routing/route-health.ts

@@ -0,0 +1,95 @@
+import { abortableDelay, type RouteState } from "@telorun/runner-core";
+
+import { apiReason } from "../api-error.js";
+import type { PublishedRoute, SessionRouter } from "./session-router.js";
+
+export interface WatchRouteHealthOptions {
+  router: SessionRouter;
+  sessionId: string;
+  /** The hosts `publish` reported. Empty → nothing to watch. */
+  routes: PublishedRoute[];
+  /** Receives every transition for one host. */
+  onState: (route: PublishedRoute, state: RouteState, reason?: string) => void;
+  /** Aborts the watch — wire to the session's teardown. */
+  signal: AbortSignal;
+  /** How long a route may stay unclaimed before it is reported `unprogrammed`. */
+  timeoutMs: number;
+  /** Poll interval while waiting for a verdict. */
+  intervalMs?: number;
+  /** Receives a read failure once, so a permanent one (a missing RBAC grant, an
+   *  unserved API version) reaches the runner's log and not only the client. */
+  onReadError?: (err: unknown) => void;
+}
+
+const DEFAULT_INTERVAL_MS = 2_000;
+
+/**
+ * Watch whether the routing layer actually programmed each published host.
+ *
+ * This is the check whose absence made an unroutable cluster indistinguishable
+ * from a healthy one: the runner already dials the workload's own address, which
+ * proves the app is listening and says nothing about whether traffic can reach
+ * it. A session whose routing objects are reconciled by no controller reported
+ * every port reachable while every public URL 404'd.
+ *
+ * REPORTS, never fails the session: a slow controller and an absent one look the
+ * same for the first few seconds, and killing a healthy session over routing that
+ * arrives late would trade a visible problem for a worse one.
+ */
+export async function watchRouteHealth(options: WatchRouteHealthOptions): Promise<void> {
+  const { router, sessionId, routes, onState, signal } = options;
+  if (routes.length === 0) return;
+  const interval = options.intervalMs ?? DEFAULT_INTERVAL_MS;
+  const deadline = Date.now() + options.timeoutMs;
+
+  const pending = new Map(routes.map((r) => [r.host, r]));
+  for (const route of routes) onState(route, "pending");
+
+  // A read failure is NOT a verdict, but it is also not nothing: a permanent one
+  // (no RBAC on the routing objects, an API version this runner did not resolve)
+  // polls out the whole deadline and would otherwise be reported as
+  // `unclaimedReason()` — sending an operator to inspect the Gateway when the
+  // runner could not read its own object. So the last one is retained and wins
+  // at the deadline.
+  let lastReadError: unknown;
+  let reportedReadError = false;
+
+  while (pending.size > 0 && !signal.aborted) {
+    for (const [host, route] of [...pending]) {
+      if (signal.aborted) return;
+      let verdict;
+      try {
+        verdict = await router.verdictFor(sessionId, route);
+      } catch (err) {
+        lastReadError = err;
+        if (!reportedReadError) {
+          reportedReadError = true;
+          options.onReadError?.(err);
+        }
+        continue;
+      }
+      // A verdict of ANY kind proves the read path works, so an earlier failure
+      // was transient and must not decide the message at the deadline.
+      lastReadError = undefined;
+      if (verdict.kind === "programmed") {
+        pending.delete(host);
+        onState(route, "programmed");
+      } else if (verdict.kind === "rejected") {
+        pending.delete(host);
+        onState(route, "unprogrammed", verdict.reason);
+      }
+    }
+    if (pending.size === 0) return;
+    if (Date.now() >= deadline) break;
+    await abortableDelay(interval, signal);
+  }
+
+  if (signal.aborted) return;
+  const reason =
+    lastReadError !== undefined
+      ? `route status could not be read: ${apiReason(lastReadError)}`
+      : router.unclaimedReason();
+  for (const route of pending.values()) {
+    onState(route, "unprogrammed", reason);
+  }
+}

package/src/k8s/routing/routing-mode.test.ts

@@ -0,0 +1,218 @@
+import { describe, expect, it } from "vitest";
+
+import type { K8sRunnerConfig, SessionRoutingConfig } from "../../config.js";
+import { RunnerConfigError } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { resolveRouting } from "./routing-mode.js";
+
+/** A cluster: which Gateway API versions it serves, and how many IngressClasses
+ *  exist. Those are the two signals `auto` may fall back on. */
+function fakeKube(cluster: {
+  gatewayVersions?: string[];
+  ingressClasses?: number;
+  defaultIngressClass?: boolean;
+}): KubeClient {
+  return {
+    apis: {
+      getAPIVersions: async () => ({
+        groups: cluster.gatewayVersions
+          ? [
+              {
+                name: "gateway.networking.k8s.io",
+                versions: cluster.gatewayVersions.map((version) => ({ version })),
+                preferredVersion: { version: cluster.gatewayVersions[0] },
+              },
+            ]
+          : [],
+      }),
+    },
+    networking: {
+      listIngressClass: async () => ({
+        items: Array.from({ length: cluster.ingressClasses ?? 0 }, (_unused, i) => ({
+          metadata:
+            cluster.defaultIngressClass && i === 0
+              ? { annotations: { "ingressclass.kubernetes.io/is-default-class": "true" } }
+              : {},
+        })),
+      }),
+    },
+  } as unknown as KubeClient;
+}
+
+function configWith(routing: Partial<SessionRoutingConfig>): K8sRunnerConfig {
+  return {
+    sessionNamespace: "telo-sessions",
+    sessionRouting: {
+      mode: "auto",
+      baseDomain: "telo.run",
+      routeReadyTimeoutMs: 60_000,
+      ...routing,
+    },
+  } as K8sRunnerConfig;
+}
+
+describe("resolveRouting — logs-only", () => {
+  it("is none when the mode says so", async () => {
+    const resolved = await resolveRouting(fakeKube({}), configWith({ mode: "none" }));
+    expect(resolved).toMatchObject({ layer: "none" });
+  });
+
+  it("is none without a base domain, whatever the mode — no domain, no constructible host", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ ingressClasses: 1 }),
+      configWith({ mode: "ingress", baseDomain: undefined }),
+    );
+    expect(resolved).toMatchObject({ layer: "none" });
+  });
+});
+
+describe("resolveRouting — explicit modes", () => {
+  it("takes ingress at its word", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1"], ingressClasses: 0 }),
+      configWith({ mode: "ingress" }),
+    );
+    expect(resolved).toEqual({ layer: "ingress" });
+  });
+
+  it("prefers the GA Gateway API version when several are served", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1beta1", "v1"] }),
+      configWith({ mode: "gateway", gateway: { name: "gw", namespace: "gw-ns" } }),
+    );
+    expect(resolved).toEqual({ layer: "gateway", apiVersion: "v1" });
+  });
+
+  it("falls back to v1beta1 where that is all the cluster serves", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1beta1"] }),
+      configWith({ mode: "gateway", gateway: { name: "gw", namespace: "gw-ns" } }),
+    );
+    expect(resolved).toEqual({ layer: "gateway", apiVersion: "v1beta1" });
+  });
+
+  it("refuses gateway mode on a cluster that does not serve the API", async () => {
+    await expect(
+      resolveRouting(
+        fakeKube({ ingressClasses: 1 }),
+        configWith({ mode: "gateway", gateway: { name: "gw", namespace: "gw-ns" } }),
+      ),
+    ).rejects.toBeInstanceOf(RunnerConfigError);
+  });
+});
+
+describe("resolveRouting — auto", () => {
+  it("resolves on the CONFIGURED target before the installed one: a named Gateway wins even where Ingress is usable", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1"], ingressClasses: 3 }),
+      configWith({ gateway: { name: "gw", namespace: "gw-ns" } }),
+    );
+    expect(resolved).toEqual({ layer: "gateway", apiVersion: "v1" });
+  });
+
+  it("takes a configured IngressClass even where the Gateway API is installed — CRDs present is not intent", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1"], ingressClasses: 1 }),
+      configWith({ ingressClassName: "nginx" }),
+    );
+    expect(resolved).toEqual({ layer: "ingress" });
+  });
+
+  it("picks the only usable layer when nothing is configured", async () => {
+    const resolved = await resolveRouting(fakeKube({ ingressClasses: 1 }), configWith({}));
+    expect(resolved).toEqual({ layer: "ingress" });
+  });
+
+  it("refuses when both layers are usable and nothing says which", async () => {
+    await expect(
+      resolveRouting(fakeKube({ gatewayVersions: ["v1"], ingressClasses: 2 }), configWith({})),
+    ).rejects.toThrow(/cannot choose/);
+  });
+
+  it("refuses a Gateway-only cluster with no Gateway named — picking one would fail as silently as publishing none", async () => {
+    await expect(
+      resolveRouting(fakeKube({ gatewayVersions: ["v1"] }), configWith({})),
+    ).rejects.toThrow(/SESSION_GATEWAY_NAME/);
+  });
+
+  it("refuses a cluster with no usable layer at all rather than publishing into the void", async () => {
+    await expect(resolveRouting(fakeKube({}), configWith({}))).rejects.toThrow(
+      /no usable routing layer/,
+    );
+  });
+
+  it("does not read an IngressClass count as usable when the API refused the list", async () => {
+    const kube = {
+      apis: { getAPIVersions: async () => ({ groups: [] }) },
+      networking: {
+        listIngressClass: async () => {
+          throw new Error("forbidden");
+        },
+      },
+    } as unknown as KubeClient;
+    // A permission failure must not be reported as a fact about the cluster:
+    // "no IngressClass exists" sends an operator to install a controller they
+    // already have.
+    await expect(resolveRouting(kube, configWith({}))).rejects.toThrow(
+      /listing IngressClasses failed/,
+    );
+    await expect(resolveRouting(kube, configWith({}))).rejects.not.toThrow(
+      /no IngressClass exists/,
+    );
+  });
+
+  it("takes a DEFAULT-marked IngressClass as intent, even beside the Gateway API", async () => {
+    // The cluster has nominated where an unqualified Ingress goes, and
+    // unqualified is what this runner creates. Without this, every cluster
+    // shipping the Gateway CRDs beside a default ingress controller would
+    // refuse to boot after the upgrade, having worked before it.
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1"], ingressClasses: 1, defaultIngressClass: true }),
+      configWith({}),
+    );
+    expect(resolved).toEqual({ layer: "ingress" });
+  });
+
+  it("still refuses when several classes exist and none is marked default", async () => {
+    await expect(
+      resolveRouting(
+        fakeKube({ gatewayVersions: ["v1"], ingressClasses: 2, defaultIngressClass: false }),
+        configWith({}),
+      ),
+    ).rejects.toThrow(/cannot choose/);
+  });
+
+  it("lets a named Gateway win over a default IngressClass — the more specific configuration", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ gatewayVersions: ["v1"], ingressClasses: 1, defaultIngressClass: true }),
+      configWith({ gateway: { name: "gw", namespace: "gw-ns" } }),
+    );
+    expect(resolved).toEqual({ layer: "gateway", apiVersion: "v1" });
+  });
+});
+
+describe("resolveRouting — origin TLS against the RESOLVED layer", () => {
+  it("refuses an ingress TLS secret when auto resolves to Gateway API", async () => {
+    // The config-time check only sees `mode`, and `auto` is the default: a named
+    // Gateway decides the layer later, so this combination booted, routed on
+    // Gateway API and ignored the certificate — plaintext origin traffic under a
+    // configuration that reads as TLS-configured.
+    await expect(
+      resolveRouting(
+        fakeKube({ gatewayVersions: ["v1"] }),
+        configWith({
+          gateway: { name: "gw", namespace: "gw-ns" },
+          tlsSecretName: "origin-tls",
+        }),
+      ),
+    ).rejects.toThrow(/belongs to the Gateway listener/);
+  });
+
+  it("allows an ingress TLS secret when the resolved layer is ingress", async () => {
+    const resolved = await resolveRouting(
+      fakeKube({ ingressClasses: 1 }),
+      configWith({ ingressClassName: "nginx", tlsSecretName: "origin-tls" }),
+    );
+    expect(resolved).toEqual({ layer: "ingress" });
+  });
+});