@telorun/k8s-runner 0.13.0 → 0.15.0

package/src/k8s/routing/gateway-router.test.ts

@@ -0,0 +1,236 @@
+import { describe, expect, it } from "vitest";
+
+import type { PortMapping } from "@telorun/runner-core";
+
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { createGatewayRouter } from "./gateway-router.js";
+
+const sessionId = "hzyayvabgyvz";
+
+const config = {
+  sessionNamespace: "telo-sessions",
+  managedByLabel: "telo-k8s-runner",
+  sessionRouting: {
+    mode: "gateway",
+    baseDomain: "telo.run",
+    routeReadyTimeoutMs: 60_000,
+    gateway: { name: "public", namespace: "gateway-system" },
+  },
+} as K8sRunnerConfig;
+
+interface Recorded {
+  created: Record<string, unknown>[];
+  deleted: string[];
+}
+
+/** A KubeClient stub carrying only what the gateway router touches. */
+function fakeKube(opts: {
+  existing?: { metadata?: { name?: string } }[];
+  getResult?: (name: string) => unknown;
+} = {}): { kube: KubeClient; rec: Recorded } {
+  const rec: Recorded = { created: [], deleted: [] };
+  const kube = {
+    custom: {
+      createNamespacedCustomObject: async ({ body }: { body: Record<string, unknown> }) => {
+        rec.created.push(body);
+        return body;
+      },
+      listNamespacedCustomObject: async () => ({ items: opts.existing ?? [] }),
+      deleteNamespacedCustomObject: async ({ name }: { name: string }) => {
+        rec.deleted.push(name);
+        return {};
+      },
+      getNamespacedCustomObject: async ({ name }: { name: string }) => {
+        if (opts.getResult) return opts.getResult(name);
+        throw Object.assign(new Error("not found"), { code: 404 });
+      },
+      replaceNamespacedCustomObject: async ({ body }: { body: Record<string, unknown> }) => body,
+    },
+  } as unknown as KubeClient;
+  return { kube, rec };
+}
+
+const ports: PortMapping[] = [
+  { port: 8080, protocol: "tcp" },
+  { port: 9090, protocol: "tcp" },
+  { port: 5000, protocol: "udp" },
+];
+
+const publishArgs = {
+  sessionId,
+  serviceName: `telo-run-${sessionId}`,
+  podName: "pod",
+  podUid: "uid",
+  ports,
+};
+
+describe("createGatewayRouter.publish", () => {
+  it("creates ONE HTTPRoute PER PORT — hostnames are route-scoped, so a shared route would collapse every host onto one backend", async () => {
+    const { kube, rec } = fakeKube();
+    const routes = await createGatewayRouter(kube, config, "v1").publish(publishArgs);
+
+    expect(rec.created).toHaveLength(2);
+    for (const body of rec.created) {
+      const spec = body.spec as { hostnames: string[]; rules: { backendRefs: unknown[] }[] };
+      expect(spec.hostnames).toHaveLength(1);
+      expect(spec.rules).toHaveLength(1);
+      expect(spec.rules[0].backendRefs).toHaveLength(1);
+    }
+    // Each host is bound to its own port's backend, never a sibling's.
+    const pairs = rec.created.map((b) => {
+      const spec = b.spec as {
+        hostnames: string[];
+        rules: { backendRefs: { name: string; port: number }[] }[];
+      };
+      return [spec.hostnames[0], spec.rules[0].backendRefs[0].port];
+    });
+    expect(pairs).toEqual([
+      [`8080-${sessionId}.telo.run`, 8080],
+      [`9090-${sessionId}.telo.run`, 9090],
+    ]);
+    expect(routes).toEqual([
+      { host: `8080-${sessionId}.telo.run`, port: 8080 },
+      { host: `9090-${sessionId}.telo.run`, port: 9090 },
+    ]);
+  });
+
+  it("skips udp ports — they are not HTTP-routable", async () => {
+    const { kube, rec } = fakeKube();
+    await createGatewayRouter(kube, config, "v1").publish({
+      ...publishArgs,
+      ports: [{ port: 5000, protocol: "udp" }],
+    });
+    expect(rec.created).toEqual([]);
+  });
+
+  it("attaches to the configured Gateway, carrying the listener when one is named", async () => {
+    const { kube, rec } = fakeKube();
+    const sectioned = {
+      ...config,
+      sessionRouting: {
+        ...config.sessionRouting,
+        gateway: { name: "public", namespace: "gateway-system", sectionName: "https" },
+      },
+    } as K8sRunnerConfig;
+    await createGatewayRouter(kube, sectioned, "v1").publish({ ...publishArgs, ports: [ports[0]] });
+    expect((rec.created[0].spec as { parentRefs: unknown[] }).parentRefs).toEqual([
+      {
+        group: "gateway.networking.k8s.io",
+        kind: "Gateway",
+        name: "public",
+        namespace: "gateway-system",
+        sectionName: "https",
+      },
+    ]);
+  });
+
+  it("deletes the route of a port a reload dropped — one object per port means a stale one keeps routing", async () => {
+    const { kube, rec } = fakeKube({
+      existing: [
+        { metadata: { name: `telo-run-${sessionId}-8080` } },
+        { metadata: { name: `telo-run-${sessionId}-9090` } },
+      ],
+    });
+    await createGatewayRouter(kube, config, "v1").publish({
+      ...publishArgs,
+      ports: [{ port: 8080, protocol: "tcp" }],
+    });
+    expect(rec.deleted).toEqual([`telo-run-${sessionId}-9090`]);
+  });
+
+  it("posts at the served API version", async () => {
+    const { kube, rec } = fakeKube();
+    await createGatewayRouter(kube, config, "v1beta1").publish({ ...publishArgs, ports: [ports[0]] });
+    expect(rec.created[0].apiVersion).toBe("gateway.networking.k8s.io/v1beta1");
+  });
+});
+
+describe("createGatewayRouter.verdictFor", () => {
+  const route = { host: `8080-${sessionId}.telo.run`, port: 8080 };
+
+  function verdict(status: unknown) {
+    const { kube } = fakeKube({ getResult: () => ({ status }) });
+    return createGatewayRouter(kube, config, "v1").verdictFor(sessionId, route);
+  }
+
+  it("is pending while no controller has written status", async () => {
+    await expect(verdict(undefined)).resolves.toEqual({ kind: "pending" });
+    await expect(verdict({ parents: [] })).resolves.toEqual({ kind: "pending" });
+  });
+
+  it("is programmed once a parent accepted and resolved it", async () => {
+    await expect(
+      verdict({
+        parents: [
+          {
+            conditions: [
+              { type: "Accepted", status: "True" },
+              { type: "ResolvedRefs", status: "True" },
+            ],
+          },
+        ],
+      }),
+    ).resolves.toEqual({ kind: "programmed" });
+  });
+
+  it("reports the controller's own reason for a refusal — the actionable half", async () => {
+    await expect(
+      verdict({
+        parents: [
+          {
+            conditions: [
+              {
+                type: "Accepted",
+                status: "False",
+                reason: "NotAllowedByListeners",
+                message: "no listener admits routes from this namespace",
+              },
+            ],
+          },
+        ],
+      }),
+    ).resolves.toEqual({
+      kind: "rejected",
+      reason: "NotAllowedByListeners: no listener admits routes from this namespace",
+    });
+  });
+
+  it("reports an unresolvable backend rather than calling the route programmed", async () => {
+    await expect(
+      verdict({
+        parents: [
+          {
+            conditions: [
+              { type: "Accepted", status: "True" },
+              { type: "ResolvedRefs", status: "False", reason: "BackendNotFound" },
+            ],
+          },
+        ],
+      }),
+    ).resolves.toEqual({ kind: "rejected", reason: "BackendNotFound" });
+  });
+
+  it("lets one accepting parent win over a sibling's refusal — traffic arrives either way", async () => {
+    await expect(
+      verdict({
+        parents: [
+          { conditions: [{ type: "Accepted", status: "False", reason: "NoMatchingParent" }] },
+          {
+            conditions: [
+              { type: "Accepted", status: "True" },
+              { type: "ResolvedRefs", status: "True" },
+            ],
+          },
+        ],
+      }),
+    ).resolves.toEqual({ kind: "programmed" });
+  });
+
+  it("is pending when the object is not visible yet", async () => {
+    const { kube } = fakeKube();
+    await expect(
+      createGatewayRouter(kube, config, "v1").verdictFor(sessionId, route),
+    ).resolves.toEqual({ kind: "pending" });
+  });
+});

package/src/k8s/routing/gateway-router.ts

@@ -0,0 +1,219 @@
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { is404 } from "../pod-status.js";
+import { createOrReplace } from "./create-or-replace.js";
+import { hostForPort } from "./session-endpoints.js";
+import { podOwnerRef, sessionObjectName } from "./session-service.js";
+import type {
+  PublishedRoute,
+  PublishRouteArgs,
+  RouteVerdict,
+  SessionRouter,
+} from "./session-router.js";
+
+const GROUP = "gateway.networking.k8s.io";
+const PLURAL = "httproutes";
+
+/**
+ * ONE HTTPRoute PER PORT, and that is forced by the API rather than chosen:
+ * `hostnames` is a property of the whole HTTPRoute, while a `rule` selects on
+ * path/header/method — never on host. A single route carrying every session
+ * hostname plus one rule per port would therefore send every host to whichever
+ * rule matched `/` first, silently collapsing all of a session's ports onto one.
+ *
+ * An Ingress rule owns its host, which is why that layer needs only one object.
+ */
+function routeName(sessionId: string, port: number): string {
+  return `${sessionObjectName(sessionId)}-${port}`;
+}
+
+interface RouteCondition {
+  type?: string;
+  status?: string;
+  reason?: string;
+  message?: string;
+}
+
+interface RouteParentStatus {
+  conditions?: RouteCondition[];
+}
+
+interface HttpRouteObject {
+  metadata?: { name?: string; resourceVersion?: string };
+  status?: { parents?: RouteParentStatus[] };
+}
+
+function condition(parent: RouteParentStatus, type: string): RouteCondition | undefined {
+  return parent.conditions?.find((c) => c.type === type);
+}
+
+/** Phrase a refusal in the controller's own words — every cause here is fixed in
+ *  the cluster (a listener that does not admit this namespace, a hostname outside
+ *  the listener's own), so the controller's `reason` is the actionable part. */
+function refusal(c: RouteCondition): string {
+  const reason = c.reason ?? "rejected";
+  return c.message ? `${reason}: ${c.message}` : reason;
+}
+
+export function createGatewayRouter(
+  kube: KubeClient,
+  config: K8sRunnerConfig,
+  apiVersion: string,
+): SessionRouter {
+  const ns = config.sessionNamespace;
+  const gateway = config.sessionRouting.gateway;
+  if (!gateway) {
+    // Unreachable through the factory, which refuses this at boot; kept so the
+    // invariant is stated where the field is read rather than assumed.
+    throw new Error("gateway routing selected without a configured Gateway parent");
+  }
+  const base = { group: GROUP, version: apiVersion, namespace: ns, plural: PLURAL };
+
+  // Arrow consts, not declarations: a hoisted `function` resets the narrowing of
+  // `gateway` above, since TypeScript cannot prove it is not called earlier.
+  const buildRoute = (args: PublishRouteArgs, port: number): Record<string, unknown> => {
+    return {
+      apiVersion: `${GROUP}/${apiVersion}`,
+      kind: "HTTPRoute",
+      metadata: {
+        name: routeName(args.sessionId, port),
+        namespace: ns,
+        labels: {
+          "app.kubernetes.io/managed-by": config.managedByLabel,
+          "telo.run/session-id": args.sessionId,
+        },
+        ownerReferences: [podOwnerRef(args.podName, args.podUid)],
+      },
+      spec: {
+        parentRefs: [
+          {
+            group: GROUP,
+            kind: "Gateway",
+            name: gateway.name,
+            namespace: gateway.namespace,
+            ...(gateway.sectionName ? { sectionName: gateway.sectionName } : {}),
+          },
+        ],
+        hostnames: [hostForPort(config, args.sessionId, port)],
+        rules: [
+          {
+            matches: [{ path: { type: "PathPrefix", value: "/" } }],
+            backendRefs: [{ name: args.serviceName, port }],
+          },
+        ],
+      },
+    };
+  };
+
+  const listSessionRoutes = async (sessionId: string): Promise<HttpRouteObject[]> => {
+    const list = (await kube.custom.listNamespacedCustomObject({
+      ...base,
+      labelSelector: `telo.run/session-id=${sessionId}`,
+    })) as { items?: HttpRouteObject[] };
+    return list.items ?? [];
+  };
+
+  return {
+    layer: "gateway",
+
+    async publish(args: PublishRouteArgs): Promise<PublishedRoute[]> {
+      const tcp = args.ports.filter((p) => p.protocol === "tcp");
+      const desired = new Set(tcp.map((p) => routeName(args.sessionId, p.port)));
+
+      for (const p of tcp) {
+        const body = buildRoute(args, p.port);
+        const name = routeName(args.sessionId, p.port);
+        await createOrReplace(
+          () => kube.custom.createNamespacedCustomObject({ ...base, body }),
+          async () => {
+            const existing = (await kube.custom.getNamespacedCustomObject({
+              ...base,
+              name,
+            })) as HttpRouteObject;
+            await kube.custom.replaceNamespacedCustomObject({
+              ...base,
+              name,
+              body: {
+                ...body,
+                metadata: {
+                  ...(body.metadata as Record<string, unknown>),
+                  resourceVersion: existing.metadata?.resourceVersion,
+                },
+              },
+            });
+          },
+        );
+      }
+
+      // A reload that DROPS a port must delete that port's route. In ingress mode
+      // the single object is replaced and the stale rule vanishes with it; here
+      // each port owns an object, so a removed one would keep routing to a port
+      // nothing listens on.
+      for (const existing of await listSessionRoutes(args.sessionId)) {
+        const name = existing.metadata?.name;
+        if (!name || desired.has(name)) continue;
+        try {
+          await kube.custom.deleteNamespacedCustomObject({ ...base, name });
+        } catch (err) {
+          if (!is404(err)) throw err;
+        }
+      }
+
+      return tcp.map((p) => ({
+        host: hostForPort(config, args.sessionId, p.port),
+        port: p.port,
+      }));
+    },
+
+    /**
+     * Gateway API is the layer that can actually answer this: a controller writes
+     * `Accepted` and `ResolvedRefs` per parent, so a route refused because the
+     * Gateway's listener does not admit this namespace says exactly that
+     * (`NotAllowedByListeners`) instead of timing out with no cause.
+     */
+    async verdictFor(sessionId: string, route: PublishedRoute): Promise<RouteVerdict> {
+      let obj: HttpRouteObject;
+      try {
+        obj = (await kube.custom.getNamespacedCustomObject({
+          ...base,
+          name: routeName(sessionId, route.port),
+        })) as HttpRouteObject;
+      } catch (err) {
+        if (is404(err)) return { kind: "pending" };
+        throw err;
+      }
+      const parents = obj.status?.parents ?? [];
+      if (parents.length === 0) return { kind: "pending" };
+
+      let rejected: string | undefined;
+      for (const parent of parents) {
+        const accepted = condition(parent, "Accepted");
+        const resolved = condition(parent, "ResolvedRefs");
+        if (accepted?.status === "False") {
+          rejected ??= refusal(accepted);
+          continue;
+        }
+        if (resolved?.status === "False") {
+          rejected ??= refusal(resolved);
+          continue;
+        }
+        // A route may attach to several parents; one that programmed it is enough
+        // for traffic to arrive, so a positive verdict wins over a sibling's refusal.
+        if (accepted?.status === "True" && resolved?.status !== "False") {
+          return { kind: "programmed" };
+        }
+      }
+      return rejected ? { kind: "rejected", reason: rejected } : { kind: "pending" };
+    },
+
+    unclaimedReason(): string {
+      const section = gateway.sectionName ? ` (listener '${gateway.sectionName}')` : "";
+      return (
+        `no controller claimed the session HTTPRoute for Gateway ` +
+        `'${gateway.namespace}/${gateway.name}'${section}. Check the Gateway exists, is programmed, ` +
+        `and that a listener admits routes from namespace '${ns}' ` +
+        `(spec.listeners[].allowedRoutes.namespaces).`
+      );
+    },
+  };
+}

package/src/k8s/routing/index.ts

@@ -0,0 +1,44 @@
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { createGatewayRouter } from "./gateway-router.js";
+import { createIngressRouter } from "./ingress-router.js";
+import { resolveRouting, type ResolvedRouting } from "./routing-mode.js";
+import type { SessionRouter } from "./session-router.js";
+
+export { createOrReplace } from "./create-or-replace.js";
+export { buildSessionIngress } from "./ingress-router.js";
+export { watchRouteHealth, type WatchRouteHealthOptions } from "./route-health.js";
+export {
+  detectRoutingSupport,
+  resolveRouting,
+  type ClusterRoutingSupport,
+  type ResolvedRouting,
+} from "./routing-mode.js";
+export { endpointsFor, hostForPort } from "./session-endpoints.js";
+export type {
+  PublishedRoute,
+  PublishRouteArgs,
+  RouteVerdict,
+  SessionRouter,
+} from "./session-router.js";
+export { buildSessionService, podOwnerRef, sessionObjectName } from "./session-service.js";
+
+/**
+ * Resolve the cluster's routing layer and build the router for it. Called ONCE at
+ * boot: an unresolvable configuration must fail the runner rather than every
+ * session it would otherwise accept and leave unroutable.
+ *
+ * `undefined` means logs-only — the honest answer when there is deliberately
+ * nothing to publish, distinct from a failure, which throws.
+ */
+export async function createSessionRouter(
+  kube: KubeClient,
+  config: K8sRunnerConfig,
+): Promise<{ router?: SessionRouter; resolved: ResolvedRouting }> {
+  const resolved = await resolveRouting(kube, config);
+  if (resolved.layer === "none") return { resolved };
+  if (resolved.layer === "gateway") {
+    return { router: createGatewayRouter(kube, config, resolved.apiVersion), resolved };
+  }
+  return { router: createIngressRouter(kube, config), resolved };
+}