@telorun/k8s-runner 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -14
- package/dist/config.d.ts +44 -9
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +51 -4
- package/dist/config.js.map +1 -1
- package/dist/k8s/backend.d.ts +4 -0
- package/dist/k8s/backend.d.ts.map +1 -1
- package/dist/k8s/backend.js +55 -17
- package/dist/k8s/backend.js.map +1 -1
- package/dist/k8s/client.d.ts +7 -1
- package/dist/k8s/client.d.ts.map +1 -1
- package/dist/k8s/client.js +3 -1
- package/dist/k8s/client.js.map +1 -1
- package/dist/k8s/pod-status.d.ts +5 -0
- package/dist/k8s/pod-status.d.ts.map +1 -1
- package/dist/k8s/pod-status.js +7 -0
- package/dist/k8s/pod-status.js.map +1 -1
- package/dist/k8s/routing/create-or-replace.d.ts +5 -0
- package/dist/k8s/routing/create-or-replace.d.ts.map +1 -0
- package/dist/k8s/routing/create-or-replace.js +15 -0
- package/dist/k8s/routing/create-or-replace.js.map +1 -0
- package/dist/k8s/routing/gateway-router.d.ts +5 -0
- package/dist/k8s/routing/gateway-router.d.ts.map +1 -0
- package/dist/k8s/routing/gateway-router.js +178 -0
- package/dist/k8s/routing/gateway-router.js.map +1 -0
- package/dist/k8s/routing/index.d.ts +24 -0
- package/dist/k8s/routing/index.d.ts.map +1 -0
- package/dist/k8s/routing/index.js +27 -0
- package/dist/k8s/routing/index.js.map +1 -0
- package/dist/k8s/routing/ingress-router.d.ts +13 -0
- package/dist/k8s/routing/ingress-router.d.ts.map +1 -0
- package/dist/k8s/routing/ingress-router.js +125 -0
- package/dist/k8s/routing/ingress-router.js.map +1 -0
- package/dist/k8s/routing/route-health.d.ts +34 -0
- package/dist/k8s/routing/route-health.d.ts.map +1 -0
- package/dist/k8s/routing/route-health.js +77 -0
- package/dist/k8s/routing/route-health.js.map +1 -0
- package/dist/k8s/routing/routing-mode.d.ts +45 -0
- package/dist/k8s/routing/routing-mode.d.ts.map +1 -0
- package/dist/k8s/routing/routing-mode.js +141 -0
- package/dist/k8s/routing/routing-mode.js.map +1 -0
- package/dist/k8s/routing/session-endpoints.d.ts +25 -0
- package/dist/k8s/routing/session-endpoints.d.ts.map +1 -0
- package/dist/k8s/routing/session-endpoints.js +36 -0
- package/dist/k8s/routing/session-endpoints.js.map +1 -0
- package/dist/k8s/routing/session-router.d.ts +57 -0
- package/dist/k8s/routing/session-router.d.ts.map +1 -0
- package/dist/k8s/routing/session-router.js +2 -0
- package/dist/k8s/routing/session-router.js.map +1 -0
- package/dist/k8s/routing/session-service.d.ts +14 -0
- package/dist/k8s/routing/session-service.d.ts.map +1 -0
- package/dist/k8s/routing/session-service.js +42 -0
- package/dist/k8s/routing/session-service.js.map +1 -0
- package/dist/k8s/watch-session.d.ts +3 -0
- package/dist/k8s/watch-session.d.ts.map +1 -1
- package/dist/k8s/watch-session.js +82 -75
- package/dist/k8s/watch-session.js.map +1 -1
- package/dist/k8s/workspace-configmap.d.ts.map +1 -1
- package/dist/k8s/workspace-configmap.js +3 -12
- package/dist/k8s/workspace-configmap.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +34 -1
- package/dist/server.js.map +1 -1
- package/package.json +2 -2
- package/src/config.ts +116 -13
- package/src/k8s/backend.ts +73 -23
- package/src/k8s/client.ts +17 -1
- package/src/k8s/pod-status.ts +8 -0
- package/src/k8s/routing/create-or-replace.ts +16 -0
- package/src/k8s/routing/gateway-router.test.ts +236 -0
- package/src/k8s/routing/gateway-router.ts +219 -0
- package/src/k8s/routing/index.ts +44 -0
- package/src/k8s/routing/ingress-router.test.ts +236 -0
- package/src/k8s/routing/ingress-router.ts +157 -0
- package/src/k8s/routing/route-health.test.ts +215 -0
- package/src/k8s/routing/route-health.ts +95 -0
- package/src/k8s/routing/routing-mode.test.ts +218 -0
- package/src/k8s/routing/routing-mode.ts +191 -0
- package/src/k8s/routing/session-endpoints.ts +45 -0
- package/src/k8s/routing/session-router.ts +59 -0
- package/src/k8s/routing/session-service.ts +55 -0
- package/src/k8s/watch-session.ts +82 -64
- package/src/k8s/workspace-configmap.ts +3 -15
- package/src/server.ts +38 -1
- package/dist/k8s/ingress.d.ts +0 -15
- package/dist/k8s/ingress.d.ts.map +0 -1
- package/dist/k8s/ingress.js +0 -102
- package/dist/k8s/ingress.js.map +0 -1
- package/src/k8s/ingress.test.ts +0 -146
- package/src/k8s/ingress.ts +0 -127
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import type { K8sRunnerConfig } from "../../config.js";
|
|
2
|
+
import { RunnerConfigError } from "../../config.js";
|
|
3
|
+
import { apiReason } from "../api-error.js";
|
|
4
|
+
import type { KubeClient } from "../client.js";
|
|
5
|
+
|
|
6
|
+
const GATEWAY_GROUP = "gateway.networking.k8s.io";
|
|
7
|
+
/** Preference order among served Gateway API versions. `v1` is GA; plenty of
|
|
8
|
+
* clusters still serve only `v1beta1`, and an HTTPRoute posted at a version the
|
|
9
|
+
* apiserver does not serve is a 404 at publish time rather than a bad route. */
|
|
10
|
+
const GATEWAY_VERSIONS = ["v1", "v1beta1"];
|
|
11
|
+
|
|
12
|
+
export type ResolvedRouting =
|
|
13
|
+
| { layer: "none"; reason: string }
|
|
14
|
+
| { layer: "ingress" }
|
|
15
|
+
| { layer: "gateway"; apiVersion: string };
|
|
16
|
+
|
|
17
|
+
/** The annotation by which a cluster nominates the class unqualified Ingresses
|
|
18
|
+
* go to — which is exactly what this runner creates when no class is set. */
|
|
19
|
+
const DEFAULT_CLASS_ANNOTATION = "ingressclass.kubernetes.io/is-default-class";
|
|
20
|
+
|
|
21
|
+
export interface ClusterRoutingSupport {
|
|
22
|
+
/** Served Gateway API version, preferred first; undefined when the group is
|
|
23
|
+
* not served at all. */
|
|
24
|
+
gatewayApiVersion?: string;
|
|
25
|
+
/** Number of IngressClasses, or **undefined when the list could not be read**.
|
|
26
|
+
* `networking.k8s.io` is always served, so the API being present is no
|
|
27
|
+
* evidence anyone can route with it — an installed controller registers a
|
|
28
|
+
* class, and that is the usable signal. Zero and unknown are kept apart
|
|
29
|
+
* because a refusal phrased from them is a different claim: one is about the
|
|
30
|
+
* cluster, the other about this runner's permissions. */
|
|
31
|
+
ingressClassCount?: number;
|
|
32
|
+
/** Set when at least one IngressClass carries the default-class annotation. */
|
|
33
|
+
hasDefaultIngressClass: boolean;
|
|
34
|
+
/** Why the IngressClass list could not be read, when it could not. */
|
|
35
|
+
ingressClassError?: unknown;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** What the cluster can actually route with. Both probes fail SOFT — discovery
|
|
39
|
+
* and IngressClass listing are the runner's least important permissions — but a
|
|
40
|
+
* failure is REPORTED rather than folded into a zero, so nothing downstream can
|
|
41
|
+
* turn a permission error into a claim about the cluster. */
|
|
42
|
+
export async function detectRoutingSupport(kube: KubeClient): Promise<ClusterRoutingSupport> {
|
|
43
|
+
let gatewayApiVersion: string | undefined;
|
|
44
|
+
try {
|
|
45
|
+
const groups = await kube.apis.getAPIVersions();
|
|
46
|
+
const group = groups.groups?.find((g) => g.name === GATEWAY_GROUP);
|
|
47
|
+
const served = new Set((group?.versions ?? []).map((v) => v.version));
|
|
48
|
+
gatewayApiVersion =
|
|
49
|
+
GATEWAY_VERSIONS.find((v) => served.has(v)) ?? group?.preferredVersion?.version;
|
|
50
|
+
} catch {
|
|
51
|
+
gatewayApiVersion = undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
const classes = await kube.networking.listIngressClass();
|
|
56
|
+
const items = classes.items ?? [];
|
|
57
|
+
return {
|
|
58
|
+
gatewayApiVersion,
|
|
59
|
+
ingressClassCount: items.length,
|
|
60
|
+
hasDefaultIngressClass: items.some(
|
|
61
|
+
(c) => c.metadata?.annotations?.[DEFAULT_CLASS_ANNOTATION] === "true",
|
|
62
|
+
),
|
|
63
|
+
};
|
|
64
|
+
} catch (err) {
|
|
65
|
+
return { gatewayApiVersion, hasDefaultIngressClass: false, ingressClassError: err };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Decide the routing layer ONCE, at boot, so a misconfiguration fails the runner
|
|
71
|
+
* instead of 404-ing every session it later accepts.
|
|
72
|
+
*
|
|
73
|
+
* `auto` resolves on what is CONFIGURED before what is INSTALLED, and that order
|
|
74
|
+
* is the whole point: Gateway API CRDs are frequently present without being the
|
|
75
|
+
* intended path, so "both APIs available" is not evidence of intent and must not
|
|
76
|
+
* decide anything. Only a cluster that offers two usable layers and was told
|
|
77
|
+
* nothing is genuinely ambiguous — and there the answer is to refuse, because
|
|
78
|
+
* guessing wrong publishes routes nothing reconciles, which is silent.
|
|
79
|
+
*/
|
|
80
|
+
export async function resolveRouting(
|
|
81
|
+
kube: KubeClient,
|
|
82
|
+
config: K8sRunnerConfig,
|
|
83
|
+
): Promise<ResolvedRouting> {
|
|
84
|
+
const routing = config.sessionRouting;
|
|
85
|
+
|
|
86
|
+
if (routing.mode === "none") {
|
|
87
|
+
return { layer: "none", reason: "SESSION_ROUTING_MODE=none" };
|
|
88
|
+
}
|
|
89
|
+
// The base domain is what makes a host constructible at all, so it stays the
|
|
90
|
+
// master switch it has always been — no domain, no routing, whatever the mode.
|
|
91
|
+
if (!routing.baseDomain) {
|
|
92
|
+
return {
|
|
93
|
+
layer: "none",
|
|
94
|
+
reason: "SESSION_ROUTING_BASE_DOMAIN is unset, so no session host can be constructed",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const support = await detectRoutingSupport(kube);
|
|
99
|
+
const resolved = decideLayer(config, support);
|
|
100
|
+
assertTlsMatchesLayer(config, resolved);
|
|
101
|
+
return resolved;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** The origin certificate is refused HERE as well as at config load, because
|
|
105
|
+
* under `auto` the effective layer is not known until now: a named Gateway wins,
|
|
106
|
+
* so `mode: auto` + a Gateway + `SESSION_INGRESS_TLS_SECRET` passed the
|
|
107
|
+
* config-time check and then routed on Gateway API with the certificate
|
|
108
|
+
* ignored — plaintext origin traffic under a configuration that reads as
|
|
109
|
+
* TLS-configured, which is the silent downgrade the check exists to prevent. */
|
|
110
|
+
function assertTlsMatchesLayer(config: K8sRunnerConfig, resolved: ResolvedRouting): void {
|
|
111
|
+
if (resolved.layer !== "gateway" || !config.sessionRouting.tlsSecretName) return;
|
|
112
|
+
throw new RunnerConfigError(
|
|
113
|
+
"SESSION_INGRESS_TLS_SECRET is set but session routing resolved to Gateway API. Under Gateway " +
|
|
114
|
+
"API the origin certificate belongs to the Gateway listener's own `tls.certificateRefs`, not " +
|
|
115
|
+
"to the per-session route, so this Secret would be silently ignored. Move the reference to " +
|
|
116
|
+
"the Gateway and unset this, or set SESSION_ROUTING_MODE=ingress.",
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function decideLayer(config: K8sRunnerConfig, support: ClusterRoutingSupport): ResolvedRouting {
|
|
121
|
+
const routing = config.sessionRouting;
|
|
122
|
+
|
|
123
|
+
if (routing.mode === "ingress") return { layer: "ingress" };
|
|
124
|
+
|
|
125
|
+
if (routing.mode === "gateway") {
|
|
126
|
+
if (!support.gatewayApiVersion) {
|
|
127
|
+
throw new RunnerConfigError(
|
|
128
|
+
`SESSION_ROUTING_MODE=gateway but the cluster does not serve '${GATEWAY_GROUP}'. ` +
|
|
129
|
+
"Install the Gateway API CRDs, or set SESSION_ROUTING_MODE=ingress.",
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
return { layer: "gateway", apiVersion: support.gatewayApiVersion };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// auto — configured target first.
|
|
136
|
+
if (routing.gateway) {
|
|
137
|
+
if (!support.gatewayApiVersion) {
|
|
138
|
+
throw new RunnerConfigError(
|
|
139
|
+
`SESSION_GATEWAY_NAME is set but the cluster does not serve '${GATEWAY_GROUP}'. ` +
|
|
140
|
+
"Install the Gateway API CRDs, or unset SESSION_GATEWAY_NAME to route via Ingress.",
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
return { layer: "gateway", apiVersion: support.gatewayApiVersion };
|
|
144
|
+
}
|
|
145
|
+
if (routing.ingressClassName) return { layer: "ingress" };
|
|
146
|
+
// A DEFAULT-marked IngressClass is configuration, not mere availability: the
|
|
147
|
+
// cluster has nominated where an unqualified Ingress goes, and unqualified is
|
|
148
|
+
// exactly what this runner creates. It therefore ranks with a named Gateway
|
|
149
|
+
// and a configured class rather than with "an IngressClass exists" — without
|
|
150
|
+
// which every cluster that ships the Gateway CRDs beside a default ingress
|
|
151
|
+
// controller would refuse to boot after this upgrade, having worked before.
|
|
152
|
+
if (support.hasDefaultIngressClass) return { layer: "ingress" };
|
|
153
|
+
|
|
154
|
+
const gatewayUsable = Boolean(support.gatewayApiVersion);
|
|
155
|
+
const ingressUsable = (support.ingressClassCount ?? 0) > 0;
|
|
156
|
+
|
|
157
|
+
if (gatewayUsable && ingressUsable) {
|
|
158
|
+
throw new RunnerConfigError(
|
|
159
|
+
`SESSION_ROUTING_MODE=auto cannot choose: this cluster serves '${GATEWAY_GROUP}' AND has ` +
|
|
160
|
+
`${support.ingressClassCount} IngressClass(es) with none marked default, and nothing says ` +
|
|
161
|
+
"which should carry session traffic. Set SESSION_ROUTING_MODE=ingress (with " +
|
|
162
|
+
"SESSION_INGRESS_CLASS) or SESSION_ROUTING_MODE=gateway (with SESSION_GATEWAY_NAME).",
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
if (gatewayUsable) {
|
|
166
|
+
throw new RunnerConfigError(
|
|
167
|
+
`SESSION_ROUTING_MODE=auto resolved to Gateway API, but SESSION_GATEWAY_NAME is unset — a ` +
|
|
168
|
+
"route has to name the Gateway it attaches to, and picking one of several would fail as " +
|
|
169
|
+
"silently as publishing none. Set SESSION_GATEWAY_NAME (and SESSION_GATEWAY_NAMESPACE " +
|
|
170
|
+
`when the Gateway is not in '${config.sessionNamespace}').`,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
if (ingressUsable) return { layer: "ingress" };
|
|
174
|
+
|
|
175
|
+
// Say which of the two it is. Reporting a permission failure as "no
|
|
176
|
+
// IngressClass exists" is a claim about the cluster drawn from a fact about
|
|
177
|
+
// this runner's access, and it sends an operator to install a controller they
|
|
178
|
+
// already have.
|
|
179
|
+
if (support.ingressClassError !== undefined) {
|
|
180
|
+
throw new RunnerConfigError(
|
|
181
|
+
"SESSION_ROUTING_BASE_DOMAIN is set but the routing layer could not be resolved: listing " +
|
|
182
|
+
`IngressClasses failed (${apiReason(support.ingressClassError)}) and '${GATEWAY_GROUP}' is ` +
|
|
183
|
+
"not served. Grant `list` on `ingressclasses`, or set SESSION_ROUTING_MODE explicitly.",
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
throw new RunnerConfigError(
|
|
187
|
+
"SESSION_ROUTING_BASE_DOMAIN is set but this cluster offers no usable routing layer: no " +
|
|
188
|
+
`IngressClass exists and '${GATEWAY_GROUP}' is not served. Install an Ingress controller or ` +
|
|
189
|
+
"the Gateway API, or set SESSION_ROUTING_MODE=none to run logs-only.",
|
|
190
|
+
);
|
|
191
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { PortMapping, RunnerEndpoint } from "@telorun/runner-core";
|
|
2
|
+
|
|
3
|
+
import type { K8sRunnerConfig } from "../../config.js";
|
|
4
|
+
|
|
5
|
+
/** Host fronting a single tcp port: `<port>-<sessionId>.<domain>`. The port rides
|
|
6
|
+
* as a leading label (no dots), so it stays a single label under the base domain
|
|
7
|
+
* — matching the docker runner's proxy scheme and compatible with a single-label
|
|
8
|
+
* wildcard cert (`*.<domain>`).
|
|
9
|
+
*
|
|
10
|
+
* Shared by both routing layers on purpose: the host scheme is what a client is
|
|
11
|
+
* told and what DNS is configured for, so it must not vary with the cluster's
|
|
12
|
+
* choice of Ingress or Gateway API. */
|
|
13
|
+
export function hostForPort(config: K8sRunnerConfig, sessionId: string, port: number): string {
|
|
14
|
+
return `${port}-${sessionId}.${config.sessionRouting.baseDomain}`;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Endpoints announced on the `running` status. Every tcp port is fronted by its
|
|
19
|
+
* own per-session host (served on 443) and carries an external `url`. udp ports
|
|
20
|
+
* aren't HTTP-routable, so they keep the host-less form. Without a base domain,
|
|
21
|
+
* host is left blank for the client adapter to fill (parity with docker).
|
|
22
|
+
*
|
|
23
|
+
* `routed` is what says a URL will be served, and it is separate from the base
|
|
24
|
+
* domain because the two can disagree: `SESSION_ROUTING_MODE=none` with a domain
|
|
25
|
+
* configured is a valid, documented setup that publishes nothing. Keying on the
|
|
26
|
+
* domain alone handed the editor a URL nothing serves — and the route watch
|
|
27
|
+
* never runs in that mode, so no `route` event would have said so either.
|
|
28
|
+
*/
|
|
29
|
+
export function endpointsFor(
|
|
30
|
+
config: K8sRunnerConfig,
|
|
31
|
+
sessionId: string,
|
|
32
|
+
ports: PortMapping[],
|
|
33
|
+
routed: boolean,
|
|
34
|
+
): RunnerEndpoint[] {
|
|
35
|
+
if (!routed || !config.sessionRouting.baseDomain || ports.length === 0) {
|
|
36
|
+
return ports.map((p) => ({ host: "", port: p.port, protocol: p.protocol }));
|
|
37
|
+
}
|
|
38
|
+
return ports.map((p) => {
|
|
39
|
+
if (p.protocol !== "tcp") {
|
|
40
|
+
return { host: "", port: p.port, protocol: p.protocol };
|
|
41
|
+
}
|
|
42
|
+
const host = hostForPort(config, sessionId, p.port);
|
|
43
|
+
return { host, port: p.port, protocol: p.protocol, url: `https://${host}` };
|
|
44
|
+
});
|
|
45
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { PortMapping } from "@telorun/runner-core";
|
|
2
|
+
|
|
3
|
+
/** One published host and the session port behind it. */
|
|
4
|
+
export interface PublishedRoute {
|
|
5
|
+
host: string;
|
|
6
|
+
port: number;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface PublishRouteArgs {
|
|
10
|
+
sessionId: string;
|
|
11
|
+
/** The Service the routing object points at. */
|
|
12
|
+
serviceName: string;
|
|
13
|
+
/** Owner of every object created — see `podOwnerRef`. */
|
|
14
|
+
podName: string;
|
|
15
|
+
podUid: string;
|
|
16
|
+
/** Every port the session must carry. Non-tcp ports are ignored: they are not
|
|
17
|
+
* HTTP-routable, and the Service still exposes them. */
|
|
18
|
+
ports: PortMapping[];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* What a routing layer has to say about one published host.
|
|
23
|
+
*
|
|
24
|
+
* `pending` is a real answer, not an absence: a controller that has not yet
|
|
25
|
+
* written status is indistinguishable from one that will never claim the route,
|
|
26
|
+
* and only time separates them.
|
|
27
|
+
*/
|
|
28
|
+
export type RouteVerdict =
|
|
29
|
+
| { kind: "programmed" }
|
|
30
|
+
| { kind: "pending" }
|
|
31
|
+
| { kind: "rejected"; reason: string };
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* How this cluster publishes a session's public routes.
|
|
35
|
+
*
|
|
36
|
+
* The seam exists because Ingress and Gateway API are both current: neither is a
|
|
37
|
+
* migration target for the other, clusters ship one, the other, or both, and a
|
|
38
|
+
* runner that hardcodes either is unroutable on half of them. It deliberately
|
|
39
|
+
* stays inside the k8s backend — docker publishes host ports through its own
|
|
40
|
+
* proxy and has no use for it, and `runner-core` owns the backend-neutral `/v1`
|
|
41
|
+
* contract, where "Ingress or HTTPRoute" is not a distinction that exists.
|
|
42
|
+
*/
|
|
43
|
+
export interface SessionRouter {
|
|
44
|
+
/** Names the layer for diagnostics. */
|
|
45
|
+
readonly layer: "ingress" | "gateway";
|
|
46
|
+
|
|
47
|
+
/** Create-or-replace this session's routing objects. Returns the hosts it
|
|
48
|
+
* published, which is what the route watch then asks about. */
|
|
49
|
+
publish(args: PublishRouteArgs): Promise<PublishedRoute[]>;
|
|
50
|
+
|
|
51
|
+
/** This layer's current verdict on one published host. Asked repeatedly by the
|
|
52
|
+
* route watch until it settles or the deadline passes. */
|
|
53
|
+
verdictFor(sessionId: string, route: PublishedRoute): Promise<RouteVerdict>;
|
|
54
|
+
|
|
55
|
+
/** What to tell an operator when a route is still `pending` at the deadline.
|
|
56
|
+
* Layer-specific because the causes are: no controller for the IngressClass,
|
|
57
|
+
* versus a Gateway that never claimed the route. */
|
|
58
|
+
unclaimedReason(): string;
|
|
59
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { V1OwnerReference, V1Service } from "@kubernetes/client-node";
|
|
2
|
+
|
|
3
|
+
import type { PortMapping } from "@telorun/runner-core";
|
|
4
|
+
|
|
5
|
+
import type { K8sRunnerConfig } from "../../config.js";
|
|
6
|
+
|
|
7
|
+
/** OwnerReference to the session Pod so every routing object is garbage collected
|
|
8
|
+
* when the Pod dies — essential for sub-minute sessions that would otherwise leak
|
|
9
|
+
* routing objects. Both routing layers use it: an HTTPRoute is namespaced like an
|
|
10
|
+
* Ingress, so the same ownership works unchanged. */
|
|
11
|
+
export function podOwnerRef(podName: string, podUid: string): V1OwnerReference {
|
|
12
|
+
return {
|
|
13
|
+
apiVersion: "v1",
|
|
14
|
+
kind: "Pod",
|
|
15
|
+
name: podName,
|
|
16
|
+
uid: podUid,
|
|
17
|
+
controller: true,
|
|
18
|
+
blockOwnerDeletion: true,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** The name every routing object for one session shares. */
|
|
23
|
+
export function sessionObjectName(sessionId: string): string {
|
|
24
|
+
return `telo-run-${sessionId}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The Service both routing layers point at — the routing layer decides how
|
|
28
|
+
* traffic ARRIVES, never what it arrives at, so this is layer-neutral. */
|
|
29
|
+
export function buildSessionService(
|
|
30
|
+
config: K8sRunnerConfig,
|
|
31
|
+
sessionId: string,
|
|
32
|
+
podName: string,
|
|
33
|
+
podUid: string,
|
|
34
|
+
ports: PortMapping[],
|
|
35
|
+
): V1Service {
|
|
36
|
+
return {
|
|
37
|
+
apiVersion: "v1",
|
|
38
|
+
kind: "Service",
|
|
39
|
+
metadata: {
|
|
40
|
+
name: sessionObjectName(sessionId),
|
|
41
|
+
namespace: config.sessionNamespace,
|
|
42
|
+
labels: { "app.kubernetes.io/managed-by": config.managedByLabel },
|
|
43
|
+
ownerReferences: [podOwnerRef(podName, podUid)],
|
|
44
|
+
},
|
|
45
|
+
spec: {
|
|
46
|
+
selector: { "telo.run/session-id": sessionId },
|
|
47
|
+
ports: ports.map((p) => ({
|
|
48
|
+
name: `p${p.port}`,
|
|
49
|
+
port: p.port,
|
|
50
|
+
targetPort: p.port,
|
|
51
|
+
protocol: p.protocol.toUpperCase(),
|
|
52
|
+
})),
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
package/src/k8s/watch-session.ts
CHANGED
|
@@ -25,7 +25,15 @@ import type { K8sRunnerConfig } from "../config.js";
|
|
|
25
25
|
import { clampLimits } from "../limits.js";
|
|
26
26
|
import { apiFailure, apiReason } from "./api-error.js";
|
|
27
27
|
import type { KubeClient } from "./client.js";
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
buildSessionService,
|
|
30
|
+
createOrReplace,
|
|
31
|
+
endpointsFor,
|
|
32
|
+
sessionObjectName,
|
|
33
|
+
watchRouteHealth,
|
|
34
|
+
type PublishedRoute,
|
|
35
|
+
type SessionRouter,
|
|
36
|
+
} from "./routing/index.js";
|
|
29
37
|
import { buildWatchPod, inspectPortFor, WORKSPACE_PORT } from "./pod-spec.js";
|
|
30
38
|
import { deletePod, is404, msg, podPhase, podStatus, provisionMessage } from "./pod-status.js";
|
|
31
39
|
import { ensureWorkspaceConfigMap } from "./workspace-configmap.js";
|
|
@@ -49,6 +57,8 @@ interface ResizableSocket {
|
|
|
49
57
|
export interface WatchSessionDeps {
|
|
50
58
|
kube: KubeClient;
|
|
51
59
|
config: K8sRunnerConfig;
|
|
60
|
+
/** Resolved at boot; absent → logs-only. */
|
|
61
|
+
router?: SessionRouter;
|
|
52
62
|
}
|
|
53
63
|
|
|
54
64
|
/**
|
|
@@ -64,7 +74,7 @@ export async function startWatchSession(
|
|
|
64
74
|
deps: WatchSessionDeps,
|
|
65
75
|
spec: BackendStartSpec,
|
|
66
76
|
): Promise<BackendSession> {
|
|
67
|
-
const { kube, config } = deps;
|
|
77
|
+
const { kube, config, router } = deps;
|
|
68
78
|
const ns = config.sessionNamespace;
|
|
69
79
|
const limits = clampLimits(config.appLimits, undefined);
|
|
70
80
|
|
|
@@ -86,6 +96,9 @@ export async function startWatchSession(
|
|
|
86
96
|
stdins: Map<string, PassThrough>;
|
|
87
97
|
abort: AbortController;
|
|
88
98
|
stopWatch: () => void;
|
|
99
|
+
/** Aborts the previous route-health watch when a reload republishes routes —
|
|
100
|
+
* the old host set is no longer what the session serves. */
|
|
101
|
+
routeAbort?: AbortController;
|
|
89
102
|
}
|
|
90
103
|
let runtime: PodRuntime | null = null;
|
|
91
104
|
|
|
@@ -166,22 +179,22 @@ export async function startWatchSession(
|
|
|
166
179
|
const agent = agentEndpoint();
|
|
167
180
|
spec.onStatus({
|
|
168
181
|
kind: "running",
|
|
169
|
-
endpoints: endpointsFor(config, spec.sessionId, allPorts(apps)),
|
|
182
|
+
endpoints: endpointsFor(config, spec.sessionId, allPorts(apps), Boolean(router)),
|
|
170
183
|
...(agent ? { agent } : {}),
|
|
171
184
|
});
|
|
172
185
|
}
|
|
173
186
|
|
|
174
187
|
/** Where this session's co-resident agent answers. The pod's containers share
|
|
175
188
|
* one network namespace, so the agent rides the session's own Service and
|
|
176
|
-
*
|
|
189
|
+
* routing objects — the port is simply in the routed set, and the host follows the
|
|
177
190
|
* same `<port>-<sessionId>` scheme every app port does. */
|
|
178
191
|
function agentEndpoint(): RunnerEndpoint | undefined {
|
|
179
192
|
const port = spec.agent?.port;
|
|
180
193
|
if (port === undefined) return undefined;
|
|
181
|
-
return endpointsFor(config, spec.sessionId, [{ port, protocol: "tcp" }])[0];
|
|
194
|
+
return endpointsFor(config, spec.sessionId, [{ port, protocol: "tcp" }], Boolean(router))[0];
|
|
182
195
|
}
|
|
183
196
|
|
|
184
|
-
/** Every port the session's Service and
|
|
197
|
+
/** Every port the session's Service and routing objects must carry: the apps' declared
|
|
185
198
|
* ports plus the agent's. Kept apart from `allPorts`, which answers what the
|
|
186
199
|
* APPLICATIONS declare — that set is what the client is told about and what a
|
|
187
200
|
* reload re-reads, and the agent belongs in neither. */
|
|
@@ -322,8 +335,8 @@ export async function startWatchSession(
|
|
|
322
335
|
* which matters, because a manifest the runner could not parse would otherwise
|
|
323
336
|
* have to leave routing alone and report, on the hot path of every save.
|
|
324
337
|
*
|
|
325
|
-
* A pod's `containerPort` list is documentation; the Service and the
|
|
326
|
-
* are what make a port reachable, so this needs no pod recreate.
|
|
338
|
+
* A pod's `containerPort` list is documentation; the Service and the routing
|
|
339
|
+
* objects are what make a port reachable, so this needs no pod recreate.
|
|
327
340
|
*/
|
|
328
341
|
async function applyPortsResolved(appName: string, frame: DebugFrame): Promise<void> {
|
|
329
342
|
const declared = portsResolvedFrom(frame);
|
|
@@ -365,10 +378,10 @@ export async function startWatchSession(
|
|
|
365
378
|
}
|
|
366
379
|
spec.onEndpoints(appName, {
|
|
367
380
|
...(accepted.length > 0
|
|
368
|
-
? { added: endpointsFor(config, spec.sessionId, accepted) }
|
|
381
|
+
? { added: endpointsFor(config, spec.sessionId, accepted, Boolean(router)) }
|
|
369
382
|
: {}),
|
|
370
383
|
...(removed.length > 0
|
|
371
|
-
? { removed: endpointsFor(config, spec.sessionId, removed) }
|
|
384
|
+
? { removed: endpointsFor(config, spec.sessionId, removed, Boolean(router)) }
|
|
372
385
|
: {}),
|
|
373
386
|
...(rejected.length > 0
|
|
374
387
|
? {
|
|
@@ -488,21 +501,25 @@ export async function startWatchSession(
|
|
|
488
501
|
}
|
|
489
502
|
|
|
490
503
|
/**
|
|
491
|
-
* Create or re-patch the Service and
|
|
492
|
-
* port set. Adding a `ports:` entry is as ordinary an edit as
|
|
493
|
-
* import, and a container may bind any port regardless of what the
|
|
494
|
-
* declares — so without this the app listens and is simply
|
|
495
|
-
*
|
|
496
|
-
* documentation; the Service and
|
|
497
|
-
* is why this needs no pod recreate.
|
|
504
|
+
* Create or re-patch the Service and the routing objects for the session's
|
|
505
|
+
* whole declared port set. Adding a `ports:` entry is as ordinary an edit as
|
|
506
|
+
* adding an import, and a container may bind any port regardless of what the
|
|
507
|
+
* pod spec declares — so without this the app listens and is simply
|
|
508
|
+
* unreachable: no route, no error, no event. The pod's `containerPort` list is
|
|
509
|
+
* documentation; the Service and the routing objects are what make a port
|
|
510
|
+
* reachable, which is why this needs no pod recreate.
|
|
511
|
+
*
|
|
512
|
+
* An EMPTY routed set still reaches the router, because dropping the last
|
|
513
|
+
* declared port is a reload like any other and its routes must go with it —
|
|
514
|
+
* returning early here left them serving a port nothing listens on.
|
|
498
515
|
*/
|
|
499
516
|
async function publishEndpoints(rt: PodRuntime, forApps: BackendAppSpec[]): Promise<void> {
|
|
500
|
-
if (!
|
|
517
|
+
if (!router) return;
|
|
501
518
|
const ports = routedPorts(forApps);
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
519
|
+
const serviceName = sessionObjectName(spec.sessionId);
|
|
520
|
+
if (ports.length > 0) {
|
|
521
|
+
const service = buildSessionService(config, spec.sessionId, rt.name, rt.uid, ports);
|
|
522
|
+
await createOrReplace(
|
|
506
523
|
() => kube.core.createNamespacedService({ namespace: ns, body: service }),
|
|
507
524
|
async () => {
|
|
508
525
|
// A Service replace must carry the assigned `clusterIP` and the current
|
|
@@ -527,37 +544,51 @@ export async function startWatchSession(
|
|
|
527
544
|
},
|
|
528
545
|
});
|
|
529
546
|
},
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
spec.sessionId,
|
|
547
|
+
);
|
|
548
|
+
}
|
|
549
|
+
const published = await router.publish({
|
|
550
|
+
sessionId: spec.sessionId,
|
|
534
551
|
serviceName,
|
|
535
|
-
rt.name,
|
|
536
|
-
rt.uid,
|
|
552
|
+
podName: rt.name,
|
|
553
|
+
podUid: rt.uid,
|
|
537
554
|
ports,
|
|
538
|
-
);
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
555
|
+
});
|
|
556
|
+
watchRoutes(rt, router, published, forApps);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/** Verify the routing layer programmed what was just published, and report each
|
|
560
|
+
* host. A watch session republishes on every reload, so the previous watch is
|
|
561
|
+
* dropped and a fresh one started against the new host set. */
|
|
562
|
+
function watchRoutes(
|
|
563
|
+
rt: PodRuntime,
|
|
564
|
+
sessionRouter: SessionRouter,
|
|
565
|
+
published: PublishedRoute[],
|
|
566
|
+
forApps: BackendAppSpec[],
|
|
567
|
+
): void {
|
|
568
|
+
rt.routeAbort?.abort();
|
|
569
|
+
if (published.length === 0) return;
|
|
570
|
+
const abort = new AbortController();
|
|
571
|
+
rt.routeAbort = abort;
|
|
572
|
+
// A port belongs to at most one app (the session-wide uniqueness rule); the
|
|
573
|
+
// co-resident agent's port belongs to none, and reports with no `app`.
|
|
574
|
+
const ownerOf = new Map<number, string>();
|
|
575
|
+
for (const app of forApps) {
|
|
576
|
+
for (const p of app.ports) ownerOf.set(p.port, app.name);
|
|
577
|
+
}
|
|
578
|
+
void watchRouteHealth({
|
|
579
|
+
router: sessionRouter,
|
|
580
|
+
sessionId: spec.sessionId,
|
|
581
|
+
routes: published,
|
|
582
|
+
onState: (route, state, reason) =>
|
|
583
|
+
spec.onRoute(ownerOf.get(route.port), {
|
|
584
|
+
host: route.host,
|
|
585
|
+
port: route.port,
|
|
586
|
+
state,
|
|
587
|
+
reason,
|
|
588
|
+
}),
|
|
589
|
+
signal: AbortSignal.any([abort.signal, rt.abort.signal]),
|
|
590
|
+
timeoutMs: config.sessionRouting.routeReadyTimeoutMs,
|
|
591
|
+
});
|
|
561
592
|
}
|
|
562
593
|
|
|
563
594
|
async function teardownPod(): Promise<void> {
|
|
@@ -639,14 +670,6 @@ export async function startWatchSession(
|
|
|
639
670
|
return app.io === "tty" ? "tty" : real;
|
|
640
671
|
}
|
|
641
672
|
|
|
642
|
-
async function upsert(create: () => Promise<unknown>, replace: () => Promise<unknown>) {
|
|
643
|
-
try {
|
|
644
|
-
await create();
|
|
645
|
-
} catch (err) {
|
|
646
|
-
if (!isConflict(err)) throw err;
|
|
647
|
-
await replace();
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
673
|
}
|
|
651
674
|
|
|
652
675
|
function allPorts(apps: BackendAppSpec[]): PortMapping[] {
|
|
@@ -668,11 +691,6 @@ function freshPodName(sessionId: string): string {
|
|
|
668
691
|
return `telo-watch-${sessionId}-${Date.now().toString(36)}${podSequence.toString(36)}`;
|
|
669
692
|
}
|
|
670
693
|
|
|
671
|
-
function isConflict(err: unknown): boolean {
|
|
672
|
-
const e = err as { statusCode?: number; code?: number; response?: { statusCode?: number } };
|
|
673
|
-
return (e?.statusCode ?? e?.code ?? e?.response?.statusCode) === 409;
|
|
674
|
-
}
|
|
675
|
-
|
|
676
694
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
677
695
|
return new Promise((resolve, reject) => {
|
|
678
696
|
const timer = setTimeout(resolve, ms);
|
|
@@ -4,6 +4,7 @@ import { workspaceAppManifest, WORKSPACE_APP_FILENAME } from "@telorun/runner-co
|
|
|
4
4
|
|
|
5
5
|
import type { K8sRunnerConfig } from "../config.js";
|
|
6
6
|
import type { KubeClient } from "./client.js";
|
|
7
|
+
import { is404, isConflict } from "./pod-status.js";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* The workspace application's manifest, delivered to the pod as a ConfigMap.
|
|
@@ -39,7 +40,7 @@ export async function ensureWorkspaceConfigMap(
|
|
|
39
40
|
await kube.core.readNamespacedConfigMap({ name, namespace: config.sessionNamespace });
|
|
40
41
|
return name;
|
|
41
42
|
} catch (err) {
|
|
42
|
-
if (!
|
|
43
|
+
if (!is404(err)) throw err;
|
|
43
44
|
}
|
|
44
45
|
try {
|
|
45
46
|
await kube.core.createNamespacedConfigMap({
|
|
@@ -64,19 +65,6 @@ export async function ensureWorkspaceConfigMap(
|
|
|
64
65
|
return name;
|
|
65
66
|
}
|
|
66
67
|
|
|
67
|
-
function statusOf(err: unknown): number | undefined {
|
|
68
|
-
const e = err as { statusCode?: number; code?: number; response?: { statusCode?: number } };
|
|
69
|
-
return e?.statusCode ?? e?.code ?? e?.response?.statusCode;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function isNotFound(err: unknown): boolean {
|
|
73
|
-
return statusOf(err) === 404;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function isConflict(err: unknown): boolean {
|
|
77
|
-
return statusOf(err) === 409;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
68
|
/**
|
|
81
69
|
* Delete workspace-app ConfigMaps no live pod mounts.
|
|
82
70
|
*
|
|
@@ -126,7 +114,7 @@ export async function sweepWorkspaceConfigMaps(
|
|
|
126
114
|
await kube.core.deleteNamespacedConfigMap({ name, namespace: config.sessionNamespace });
|
|
127
115
|
removed += 1;
|
|
128
116
|
} catch (err) {
|
|
129
|
-
if (!
|
|
117
|
+
if (!is404(err)) log.warn({ err, name }, "failed to delete a stale workspace ConfigMap");
|
|
130
118
|
}
|
|
131
119
|
}
|
|
132
120
|
if (removed > 0) log.info({ removed }, "swept stale workspace-app ConfigMaps");
|