@telorun/k8s-runner 0.14.0 → 0.15.1

package/src/k8s/routing/gateway-router.ts

@@ -0,0 +1,219 @@
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { is404 } from "../pod-status.js";
+import { createOrReplace } from "./create-or-replace.js";
+import { hostForPort } from "./session-endpoints.js";
+import { podOwnerRef, sessionObjectName } from "./session-service.js";
+import type {
+  PublishedRoute,
+  PublishRouteArgs,
+  RouteVerdict,
+  SessionRouter,
+} from "./session-router.js";
+
+const GROUP = "gateway.networking.k8s.io";
+const PLURAL = "httproutes";
+
+/**
+ * ONE HTTPRoute PER PORT, and that is forced by the API rather than chosen:
+ * `hostnames` is a property of the whole HTTPRoute, while a `rule` selects on
+ * path/header/method — never on host. A single route carrying every session
+ * hostname plus one rule per port would therefore send every host to whichever
+ * rule matched `/` first, silently collapsing all of a session's ports onto one.
+ *
+ * An Ingress rule owns its host, which is why that layer needs only one object.
+ */
+function routeName(sessionId: string, port: number): string {
+  return `${sessionObjectName(sessionId)}-${port}`;
+}
+
+interface RouteCondition {
+  type?: string;
+  status?: string;
+  reason?: string;
+  message?: string;
+}
+
+interface RouteParentStatus {
+  conditions?: RouteCondition[];
+}
+
+interface HttpRouteObject {
+  metadata?: { name?: string; resourceVersion?: string };
+  status?: { parents?: RouteParentStatus[] };
+}
+
+function condition(parent: RouteParentStatus, type: string): RouteCondition | undefined {
+  return parent.conditions?.find((c) => c.type === type);
+}
+
+/** Phrase a refusal in the controller's own words — every cause here is fixed in
+ *  the cluster (a listener that does not admit this namespace, a hostname outside
+ *  the listener's own), so the controller's `reason` is the actionable part. */
+function refusal(c: RouteCondition): string {
+  const reason = c.reason ?? "rejected";
+  return c.message ? `${reason}: ${c.message}` : reason;
+}
+
+export function createGatewayRouter(
+  kube: KubeClient,
+  config: K8sRunnerConfig,
+  apiVersion: string,
+): SessionRouter {
+  const ns = config.sessionNamespace;
+  const gateway = config.sessionRouting.gateway;
+  if (!gateway) {
+    // Unreachable through the factory, which refuses this at boot; kept so the
+    // invariant is stated where the field is read rather than assumed.
+    throw new Error("gateway routing selected without a configured Gateway parent");
+  }
+  const base = { group: GROUP, version: apiVersion, namespace: ns, plural: PLURAL };
+
+  // Arrow consts, not declarations: a hoisted `function` resets the narrowing of
+  // `gateway` above, since TypeScript cannot prove it is not called earlier.
+  const buildRoute = (args: PublishRouteArgs, port: number): Record<string, unknown> => {
+    return {
+      apiVersion: `${GROUP}/${apiVersion}`,
+      kind: "HTTPRoute",
+      metadata: {
+        name: routeName(args.sessionId, port),
+        namespace: ns,
+        labels: {
+          "app.kubernetes.io/managed-by": config.managedByLabel,
+          "telo.run/session-id": args.sessionId,
+        },
+        ownerReferences: [podOwnerRef(args.podName, args.podUid)],
+      },
+      spec: {
+        parentRefs: [
+          {
+            group: GROUP,
+            kind: "Gateway",
+            name: gateway.name,
+            namespace: gateway.namespace,
+            ...(gateway.sectionName ? { sectionName: gateway.sectionName } : {}),
+          },
+        ],
+        hostnames: [hostForPort(config, args.sessionId, port)],
+        rules: [
+          {
+            matches: [{ path: { type: "PathPrefix", value: "/" } }],
+            backendRefs: [{ name: args.serviceName, port }],
+          },
+        ],
+      },
+    };
+  };
+
+  const listSessionRoutes = async (sessionId: string): Promise<HttpRouteObject[]> => {
+    const list = (await kube.custom.listNamespacedCustomObject({
+      ...base,
+      labelSelector: `telo.run/session-id=${sessionId}`,
+    })) as { items?: HttpRouteObject[] };
+    return list.items ?? [];
+  };
+
+  return {
+    layer: "gateway",
+
+    async publish(args: PublishRouteArgs): Promise<PublishedRoute[]> {
+      const tcp = args.ports.filter((p) => p.protocol === "tcp");
+      const desired = new Set(tcp.map((p) => routeName(args.sessionId, p.port)));
+
+      for (const p of tcp) {
+        const body = buildRoute(args, p.port);
+        const name = routeName(args.sessionId, p.port);
+        await createOrReplace(
+          () => kube.custom.createNamespacedCustomObject({ ...base, body }),
+          async () => {
+            const existing = (await kube.custom.getNamespacedCustomObject({
+              ...base,
+              name,
+            })) as HttpRouteObject;
+            await kube.custom.replaceNamespacedCustomObject({
+              ...base,
+              name,
+              body: {
+                ...body,
+                metadata: {
+                  ...(body.metadata as Record<string, unknown>),
+                  resourceVersion: existing.metadata?.resourceVersion,
+                },
+              },
+            });
+          },
+        );
+      }
+
+      // A reload that DROPS a port must delete that port's route. In ingress mode
+      // the single object is replaced and the stale rule vanishes with it; here
+      // each port owns an object, so a removed one would keep routing to a port
+      // nothing listens on.
+      for (const existing of await listSessionRoutes(args.sessionId)) {
+        const name = existing.metadata?.name;
+        if (!name || desired.has(name)) continue;
+        try {
+          await kube.custom.deleteNamespacedCustomObject({ ...base, name });
+        } catch (err) {
+          if (!is404(err)) throw err;
+        }
+      }
+
+      return tcp.map((p) => ({
+        host: hostForPort(config, args.sessionId, p.port),
+        port: p.port,
+      }));
+    },
+
+    /**
+     * Gateway API is the layer that can actually answer this: a controller writes
+     * `Accepted` and `ResolvedRefs` per parent, so a route refused because the
+     * Gateway's listener does not admit this namespace says exactly that
+     * (`NotAllowedByListeners`) instead of timing out with no cause.
+     */
+    async verdictFor(sessionId: string, route: PublishedRoute): Promise<RouteVerdict> {
+      let obj: HttpRouteObject;
+      try {
+        obj = (await kube.custom.getNamespacedCustomObject({
+          ...base,
+          name: routeName(sessionId, route.port),
+        })) as HttpRouteObject;
+      } catch (err) {
+        if (is404(err)) return { kind: "pending" };
+        throw err;
+      }
+      const parents = obj.status?.parents ?? [];
+      if (parents.length === 0) return { kind: "pending" };
+
+      let rejected: string | undefined;
+      for (const parent of parents) {
+        const accepted = condition(parent, "Accepted");
+        const resolved = condition(parent, "ResolvedRefs");
+        if (accepted?.status === "False") {
+          rejected ??= refusal(accepted);
+          continue;
+        }
+        if (resolved?.status === "False") {
+          rejected ??= refusal(resolved);
+          continue;
+        }
+        // A route may attach to several parents; one that programmed it is enough
+        // for traffic to arrive, so a positive verdict wins over a sibling's refusal.
+        if (accepted?.status === "True" && resolved?.status !== "False") {
+          return { kind: "programmed" };
+        }
+      }
+      return rejected ? { kind: "rejected", reason: rejected } : { kind: "pending" };
+    },
+
+    unclaimedReason(): string {
+      const section = gateway.sectionName ? ` (listener '${gateway.sectionName}')` : "";
+      return (
+        `no controller claimed the session HTTPRoute for Gateway ` +
+        `'${gateway.namespace}/${gateway.name}'${section}. Check the Gateway exists, is programmed, ` +
+        `and that a listener admits routes from namespace '${ns}' ` +
+        `(spec.listeners[].allowedRoutes.namespaces).`
+      );
+    },
+  };
+}

package/src/k8s/routing/index.ts

@@ -0,0 +1,44 @@
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { createGatewayRouter } from "./gateway-router.js";
+import { createIngressRouter } from "./ingress-router.js";
+import { resolveRouting, type ResolvedRouting } from "./routing-mode.js";
+import type { SessionRouter } from "./session-router.js";
+
+export { createOrReplace } from "./create-or-replace.js";
+export { buildSessionIngress } from "./ingress-router.js";
+export { watchRouteHealth, type WatchRouteHealthOptions } from "./route-health.js";
+export {
+  detectRoutingSupport,
+  resolveRouting,
+  type ClusterRoutingSupport,
+  type ResolvedRouting,
+} from "./routing-mode.js";
+export { endpointsFor, hostForPort } from "./session-endpoints.js";
+export type {
+  PublishedRoute,
+  PublishRouteArgs,
+  RouteVerdict,
+  SessionRouter,
+} from "./session-router.js";
+export { buildSessionService, podOwnerRef, sessionObjectName } from "./session-service.js";
+
+/**
+ * Resolve the cluster's routing layer and build the router for it. Called ONCE at
+ * boot: an unresolvable configuration must fail the runner rather than every
+ * session it would otherwise accept and leave unroutable.
+ *
+ * `undefined` means logs-only — the honest answer when there is deliberately
+ * nothing to publish, distinct from a failure, which throws.
+ */
+export async function createSessionRouter(
+  kube: KubeClient,
+  config: K8sRunnerConfig,
+): Promise<{ router?: SessionRouter; resolved: ResolvedRouting }> {
+  const resolved = await resolveRouting(kube, config);
+  if (resolved.layer === "none") return { resolved };
+  if (resolved.layer === "gateway") {
+    return { router: createGatewayRouter(kube, config, resolved.apiVersion), resolved };
+  }
+  return { router: createIngressRouter(kube, config), resolved };
+}

package/src/k8s/routing/ingress-router.test.ts

@@ -0,0 +1,236 @@
+import { describe, expect, it } from "vitest";
+
+import type { PortMapping } from "@telorun/runner-core";
+
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { buildSessionIngress, createIngressRouter } from "./ingress-router.js";
+import { endpointsFor } from "./session-endpoints.js";
+
+const config = {
+  sessionNamespace: "telo-sessions",
+  sessionRouting: {
+    mode: "ingress",
+    baseDomain: "telo.run",
+    ingressClassName: "nginx",
+    routeReadyTimeoutMs: 60_000,
+  },
+  managedByLabel: "telo-k8s-runner",
+} as K8sRunnerConfig;
+
+const sessionId = "hzyayvabgyvz";
+const ports: PortMapping[] = [
+  { port: 8080, protocol: "tcp" },
+  { port: 9090, protocol: "tcp" },
+  { port: 5000, protocol: "udp" },
+];
+
+function withRouting(patch: Partial<K8sRunnerConfig["sessionRouting"]>): K8sRunnerConfig {
+  return { ...config, sessionRouting: { ...config.sessionRouting, ...patch } } as K8sRunnerConfig;
+}
+
+describe("endpointsFor", () => {
+  it("gives every tcp port a <port>-<id>.<domain> host + url, udp host-less", () => {
+    expect(endpointsFor(config, sessionId, ports, true)).toEqual([
+      {
+        host: `8080-${sessionId}.telo.run`,
+        port: 8080,
+        protocol: "tcp",
+        url: `https://8080-${sessionId}.telo.run`,
+      },
+      {
+        host: `9090-${sessionId}.telo.run`,
+        port: 9090,
+        protocol: "tcp",
+        url: `https://9090-${sessionId}.telo.run`,
+      },
+      { host: "", port: 5000, protocol: "udp" },
+    ]);
+  });
+
+  it("leaves hosts blank when no routing base domain is configured", () => {
+    expect(endpointsFor(withRouting({ baseDomain: undefined }), sessionId, ports, true)).toEqual([
+      { host: "", port: 8080, protocol: "tcp" },
+      { host: "", port: 9090, protocol: "tcp" },
+      { host: "", port: 5000, protocol: "udp" },
+    ]);
+  });
+
+  it("advertises no url when nothing publishes routes, even with a base domain set", () => {
+    // `SESSION_ROUTING_MODE=none` with a domain configured is documented and
+    // valid. Keying on the domain alone handed the editor a URL nothing serves,
+    // and the route watch never runs in that mode — so no event said so either.
+    expect(endpointsFor(withRouting({ mode: "none" }), sessionId, ports, false)).toEqual([
+      { host: "", port: 8080, protocol: "tcp" },
+      { host: "", port: 9090, protocol: "tcp" },
+      { host: "", port: 5000, protocol: "udp" },
+    ]);
+  });
+
+  it("keeps the host scheme identical whatever layer routes it", () => {
+    const viaGateway = withRouting({ mode: "gateway", gateway: { name: "gw", namespace: "gw-ns" } });
+    expect(endpointsFor(viaGateway, sessionId, ports, true)).toEqual(
+      endpointsFor(config, sessionId, ports, true),
+    );
+  });
+});
+
+describe("buildSessionIngress", () => {
+  it("emits one rule per tcp port to the matching service port", () => {
+    const { ingress, hosts } = buildSessionIngress(
+      config,
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      ports,
+    );
+    expect(hosts).toEqual([`8080-${sessionId}.telo.run`, `9090-${sessionId}.telo.run`]);
+    expect(ingress.spec?.ingressClassName).toBe("nginx");
+    expect(ingress.spec?.rules).toEqual([
+      {
+        host: `8080-${sessionId}.telo.run`,
+        http: {
+          paths: [
+            {
+              path: "/",
+              pathType: "Prefix",
+              backend: { service: { name: `telo-run-${sessionId}`, port: { number: 8080 } } },
+            },
+          ],
+        },
+      },
+      {
+        host: `9090-${sessionId}.telo.run`,
+        http: {
+          paths: [
+            {
+              path: "/",
+              pathType: "Prefix",
+              backend: { service: { name: `telo-run-${sessionId}`, port: { number: 9090 } } },
+            },
+          ],
+        },
+      },
+    ]);
+  });
+
+  it("produces no rules when there are no tcp ports", () => {
+    const { ingress, hosts } = buildSessionIngress(
+      config,
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      [{ port: 5000, protocol: "udp" }],
+    );
+    expect(hosts).toEqual([]);
+    expect(ingress.spec?.rules).toEqual([]);
+  });
+
+  it("omits the tls block when no tls secret is configured", () => {
+    const { ingress } = buildSessionIngress(
+      config,
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      ports,
+    );
+    expect(ingress.spec?.tls).toBeUndefined();
+  });
+
+  it("presents the configured tls secret for every tcp host", () => {
+    const { ingress } = buildSessionIngress(
+      withRouting({ tlsSecretName: "telo-origin-tls" }),
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      ports,
+    );
+    expect(ingress.spec?.tls).toEqual([
+      {
+        hosts: [`8080-${sessionId}.telo.run`, `9090-${sessionId}.telo.run`],
+        secretName: "telo-origin-tls",
+      },
+    ]);
+  });
+
+  it("omits the tls block when there are no tcp hosts to secure", () => {
+    const { ingress } = buildSessionIngress(
+      withRouting({ tlsSecretName: "telo-origin-tls" }),
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      [{ port: 5000, protocol: "udp" }],
+    );
+    expect(ingress.spec?.tls).toBeUndefined();
+  });
+
+  it("carries the session label and the pod ownerReference every routing object does", () => {
+    const { ingress } = buildSessionIngress(
+      config,
+      sessionId,
+      `telo-run-${sessionId}`,
+      "pod",
+      "uid",
+      ports,
+    );
+    expect(ingress.metadata?.labels?.["telo.run/session-id"]).toBe(sessionId);
+    expect(ingress.metadata?.ownerReferences?.[0]).toMatchObject({ kind: "Pod", name: "pod" });
+  });
+});
+
+describe("createIngressRouter.publish", () => {
+  function fakeKube() {
+    const rec = { created: 0, deleted: [] as string[] };
+    const kube = {
+      networking: {
+        createNamespacedIngress: async () => {
+          rec.created += 1;
+          return {};
+        },
+        deleteNamespacedIngress: async ({ name }: { name: string }) => {
+          rec.deleted.push(name);
+          return {};
+        },
+      },
+    } as unknown as KubeClient;
+    return { kube, rec };
+  }
+
+  const args = {
+    sessionId,
+    serviceName: `telo-run-${sessionId}`,
+    podName: "pod",
+    podUid: "uid",
+  };
+
+  it("deletes the session Ingress when a reload leaves no routable port", async () => {
+    // Returning early left the previous Ingress serving a port nothing listens
+    // on — the drop-all case the per-port gateway cleanup already handled.
+    const { kube, rec } = fakeKube();
+    const routes = await createIngressRouter(kube, config).publish({
+      ...args,
+      ports: [{ port: 5000, protocol: "udp" }],
+    });
+    expect(routes).toEqual([]);
+    expect(rec.created).toBe(0);
+    expect(rec.deleted).toEqual([`telo-run-${sessionId}`]);
+  });
+
+  it("treats an already-absent Ingress as nothing to delete", async () => {
+    const kube = {
+      networking: {
+        deleteNamespacedIngress: async () => {
+          throw Object.assign(new Error("not found"), { code: 404 });
+        },
+      },
+    } as unknown as KubeClient;
+    await expect(
+      createIngressRouter(kube, config).publish({ ...args, ports: [] }),
+    ).resolves.toEqual([]);
+  });
+});

package/src/k8s/routing/ingress-router.ts

@@ -0,0 +1,157 @@
+import type { V1Ingress } from "@kubernetes/client-node";
+
+import type { PortMapping } from "@telorun/runner-core";
+
+import type { K8sRunnerConfig } from "../../config.js";
+import type { KubeClient } from "../client.js";
+import { is404 } from "../pod-status.js";
+import { createOrReplace } from "./create-or-replace.js";
+import { hostForPort } from "./session-endpoints.js";
+import { podOwnerRef, sessionObjectName } from "./session-service.js";
+import type {
+  PublishedRoute,
+  PublishRouteArgs,
+  RouteVerdict,
+  SessionRouter,
+} from "./session-router.js";
+
+/** One Ingress carries every host for a session: unlike Gateway API, an Ingress
+ *  rule is scoped to its own host, so per-host backends fit in one object. */
+export function buildSessionIngress(
+  config: K8sRunnerConfig,
+  sessionId: string,
+  serviceName: string,
+  podName: string,
+  podUid: string,
+  ports: PortMapping[],
+): { ingress: V1Ingress; hosts: string[] } {
+  const rules = ports
+    .filter((p) => p.protocol === "tcp")
+    .map((p) => ({
+      host: hostForPort(config, sessionId, p.port),
+      http: {
+        paths: [
+          {
+            path: "/",
+            pathType: "Prefix" as const,
+            backend: { service: { name: serviceName, port: { number: p.port } } },
+          },
+        ],
+      },
+    }));
+  const ingress: V1Ingress = {
+    apiVersion: "networking.k8s.io/v1",
+    kind: "Ingress",
+    metadata: {
+      name: sessionObjectName(sessionId),
+      namespace: config.sessionNamespace,
+      labels: {
+        "app.kubernetes.io/managed-by": config.managedByLabel,
+        "telo.run/session-id": sessionId,
+      },
+      ownerReferences: [podOwnerRef(podName, podUid)],
+    },
+    spec: {
+      ...(config.sessionRouting.ingressClassName
+        ? { ingressClassName: config.sessionRouting.ingressClassName }
+        : {}),
+      // Present the predefined cert (e.g. a Cloudflare Origin cert) so an upstream
+      // in Full (Strict) mode can validate the origin. Only meaningful with routable
+      // hosts; a single wildcard `*.<domain>` Secret covers every session host.
+      ...(config.sessionRouting.tlsSecretName && rules.length > 0
+        ? {
+            tls: [
+              {
+                hosts: rules.map((r) => r.host),
+                secretName: config.sessionRouting.tlsSecretName,
+              },
+            ],
+          }
+        : {}),
+      rules,
+    },
+  };
+  return { ingress, hosts: rules.map((r) => r.host) };
+}
+
+export function createIngressRouter(kube: KubeClient, config: K8sRunnerConfig): SessionRouter {
+  const ns = config.sessionNamespace;
+
+  return {
+    layer: "ingress",
+
+    async publish(args: PublishRouteArgs): Promise<PublishedRoute[]> {
+      const { ingress } = buildSessionIngress(
+        config,
+        args.sessionId,
+        args.serviceName,
+        args.podName,
+        args.podUid,
+        args.ports,
+      );
+      const name = ingress.metadata!.name!;
+      const rules = ingress.spec?.rules ?? [];
+      // No tcp ports → nothing HTTP-routable. DELETE rather than return: a reload
+      // that drops the last routed port would otherwise leave the previous
+      // Ingress serving a port nothing listens on. (The Service stays for any udp.)
+      if (rules.length === 0) {
+        try {
+          await kube.networking.deleteNamespacedIngress({ name, namespace: ns });
+        } catch (err) {
+          if (!is404(err)) throw err;
+        }
+        return [];
+      }
+      await createOrReplace(
+        () => kube.networking.createNamespacedIngress({ namespace: ns, body: ingress }),
+        async () => {
+          const existing = await kube.networking.readNamespacedIngress({ name, namespace: ns });
+          await kube.networking.replaceNamespacedIngress({
+            name,
+            namespace: ns,
+            body: {
+              ...ingress,
+              metadata: {
+                ...ingress.metadata,
+                resourceVersion: existing.metadata?.resourceVersion,
+              },
+            },
+          });
+        },
+      );
+      return args.ports
+        .filter((p) => p.protocol === "tcp")
+        .map((p) => ({ host: hostForPort(config, args.sessionId, p.port), port: p.port }));
+    },
+
+    /**
+     * Ingress offers no rejection signal — a controller that refuses a rule simply
+     * never writes status — so this only ever answers `programmed` or `pending`,
+     * and an unclaimed Ingress is caught by the deadline rather than by a reason
+     * the cluster gave. That is the weaker half of the two layers, and it is a
+     * property of the API, not of this implementation.
+     */
+    async verdictFor(sessionId: string): Promise<RouteVerdict> {
+      try {
+        const ingress = await kube.networking.readNamespacedIngress({
+          name: sessionObjectName(sessionId),
+          namespace: ns,
+        });
+        const assigned = ingress.status?.loadBalancer?.ingress ?? [];
+        return assigned.length > 0 ? { kind: "programmed" } : { kind: "pending" };
+      } catch (err) {
+        if (is404(err)) return { kind: "pending" };
+        throw err;
+      }
+    },
+
+    unclaimedReason(): string {
+      const cls = config.sessionRouting.ingressClassName;
+      return (
+        `no ingress controller claimed the session Ingress` +
+        (cls ? ` for class '${cls}'` : " (no ingressClassName configured)") +
+        `. Check that an Ingress controller is installed and watching namespace '${ns}'.`
+      );
+    },
+  };
+}