@telorun/k8s-runner 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -14
- package/dist/config.d.ts +44 -9
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +51 -4
- package/dist/config.js.map +1 -1
- package/dist/k8s/backend.d.ts +4 -0
- package/dist/k8s/backend.d.ts.map +1 -1
- package/dist/k8s/backend.js +55 -17
- package/dist/k8s/backend.js.map +1 -1
- package/dist/k8s/client.d.ts +7 -1
- package/dist/k8s/client.d.ts.map +1 -1
- package/dist/k8s/client.js +3 -1
- package/dist/k8s/client.js.map +1 -1
- package/dist/k8s/pod-status.d.ts +5 -0
- package/dist/k8s/pod-status.d.ts.map +1 -1
- package/dist/k8s/pod-status.js +7 -0
- package/dist/k8s/pod-status.js.map +1 -1
- package/dist/k8s/routing/create-or-replace.d.ts +5 -0
- package/dist/k8s/routing/create-or-replace.d.ts.map +1 -0
- package/dist/k8s/routing/create-or-replace.js +15 -0
- package/dist/k8s/routing/create-or-replace.js.map +1 -0
- package/dist/k8s/routing/gateway-router.d.ts +5 -0
- package/dist/k8s/routing/gateway-router.d.ts.map +1 -0
- package/dist/k8s/routing/gateway-router.js +178 -0
- package/dist/k8s/routing/gateway-router.js.map +1 -0
- package/dist/k8s/routing/index.d.ts +24 -0
- package/dist/k8s/routing/index.d.ts.map +1 -0
- package/dist/k8s/routing/index.js +27 -0
- package/dist/k8s/routing/index.js.map +1 -0
- package/dist/k8s/routing/ingress-router.d.ts +13 -0
- package/dist/k8s/routing/ingress-router.d.ts.map +1 -0
- package/dist/k8s/routing/ingress-router.js +125 -0
- package/dist/k8s/routing/ingress-router.js.map +1 -0
- package/dist/k8s/routing/route-health.d.ts +34 -0
- package/dist/k8s/routing/route-health.d.ts.map +1 -0
- package/dist/k8s/routing/route-health.js +77 -0
- package/dist/k8s/routing/route-health.js.map +1 -0
- package/dist/k8s/routing/routing-mode.d.ts +45 -0
- package/dist/k8s/routing/routing-mode.d.ts.map +1 -0
- package/dist/k8s/routing/routing-mode.js +141 -0
- package/dist/k8s/routing/routing-mode.js.map +1 -0
- package/dist/k8s/routing/session-endpoints.d.ts +25 -0
- package/dist/k8s/routing/session-endpoints.d.ts.map +1 -0
- package/dist/k8s/routing/session-endpoints.js +36 -0
- package/dist/k8s/routing/session-endpoints.js.map +1 -0
- package/dist/k8s/routing/session-router.d.ts +57 -0
- package/dist/k8s/routing/session-router.d.ts.map +1 -0
- package/dist/k8s/routing/session-router.js +2 -0
- package/dist/k8s/routing/session-router.js.map +1 -0
- package/dist/k8s/routing/session-service.d.ts +14 -0
- package/dist/k8s/routing/session-service.d.ts.map +1 -0
- package/dist/k8s/routing/session-service.js +42 -0
- package/dist/k8s/routing/session-service.js.map +1 -0
- package/dist/k8s/watch-session.d.ts +3 -0
- package/dist/k8s/watch-session.d.ts.map +1 -1
- package/dist/k8s/watch-session.js +82 -75
- package/dist/k8s/watch-session.js.map +1 -1
- package/dist/k8s/workspace-configmap.d.ts.map +1 -1
- package/dist/k8s/workspace-configmap.js +3 -12
- package/dist/k8s/workspace-configmap.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +34 -1
- package/dist/server.js.map +1 -1
- package/package.json +2 -2
- package/src/config.ts +116 -13
- package/src/k8s/backend.ts +73 -23
- package/src/k8s/client.ts +17 -1
- package/src/k8s/pod-status.ts +8 -0
- package/src/k8s/routing/create-or-replace.ts +16 -0
- package/src/k8s/routing/gateway-router.test.ts +236 -0
- package/src/k8s/routing/gateway-router.ts +219 -0
- package/src/k8s/routing/index.ts +44 -0
- package/src/k8s/routing/ingress-router.test.ts +236 -0
- package/src/k8s/routing/ingress-router.ts +157 -0
- package/src/k8s/routing/route-health.test.ts +215 -0
- package/src/k8s/routing/route-health.ts +95 -0
- package/src/k8s/routing/routing-mode.test.ts +218 -0
- package/src/k8s/routing/routing-mode.ts +191 -0
- package/src/k8s/routing/session-endpoints.ts +45 -0
- package/src/k8s/routing/session-router.ts +59 -0
- package/src/k8s/routing/session-service.ts +55 -0
- package/src/k8s/watch-session.ts +82 -64
- package/src/k8s/workspace-configmap.ts +3 -15
- package/src/server.ts +38 -1
- package/dist/k8s/ingress.d.ts +0 -15
- package/dist/k8s/ingress.d.ts.map +0 -1
- package/dist/k8s/ingress.js +0 -102
- package/dist/k8s/ingress.js.map +0 -1
- package/src/k8s/ingress.test.ts +0 -146
- package/src/k8s/ingress.ts +0 -127
package/src/config.ts
CHANGED
|
@@ -45,12 +45,58 @@ export interface LimitCeilings {
|
|
|
45
45
|
ephemeralStorage: string;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* Which routing layer the runner publishes per-session routes on.
|
|
50
|
+
*
|
|
51
|
+
* - `auto` — resolve from what is CONFIGURED, then from what the cluster
|
|
52
|
+
* serves; refuse at boot when the answer is genuinely ambiguous.
|
|
53
|
+
* - `ingress` — `networking.k8s.io/v1` Ingress.
|
|
54
|
+
* - `gateway` — `gateway.networking.k8s.io/v1` HTTPRoute.
|
|
55
|
+
* - `none` — publish nothing; the session is logs-only even with a base
|
|
56
|
+
* domain set.
|
|
57
|
+
*/
|
|
58
|
+
export type SessionRoutingMode = "auto" | "ingress" | "gateway" | "none";
|
|
59
|
+
|
|
60
|
+
const ROUTING_MODES: readonly SessionRoutingMode[] = ["auto", "ingress", "gateway", "none"];
|
|
61
|
+
|
|
62
|
+
/** The Gateway a session's HTTPRoute attaches to. Never detected: picking one of
|
|
63
|
+
* several Gateways is the same guess as picking one of two routing APIs, and a
|
|
64
|
+
* route attached to the wrong Gateway fails exactly as silently as no route. */
|
|
65
|
+
export interface GatewayParentConfig {
|
|
66
|
+
name: string;
|
|
67
|
+
/** Defaults to the session namespace — mirroring Gateway API's own
|
|
68
|
+
* `parentRefs[].namespace` default, which is the route's own namespace. */
|
|
69
|
+
namespace: string;
|
|
70
|
+
/** Optional listener name, when the Gateway has several and only one should
|
|
71
|
+
* carry session traffic. */
|
|
72
|
+
sectionName?: string;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface SessionRoutingConfig {
|
|
76
|
+
mode: SessionRoutingMode;
|
|
77
|
+
/** Wildcard base domain for per-session hosts; unset → logs-only whatever the
|
|
78
|
+
* mode, since without it no host can be constructed at all. */
|
|
79
|
+
baseDomain?: string;
|
|
80
|
+
/** IngressClass for created session Ingresses (ingress mode). */
|
|
81
|
+
ingressClassName?: string;
|
|
82
|
+
/** `kubernetes.io/tls` Secret (in the session namespace) the per-session
|
|
83
|
+
* Ingress presents so an upstream (e.g. Cloudflare Full (Strict)) can validate
|
|
84
|
+
* the origin. Must cover `*.<baseDomain>`. INGRESS MODE ONLY — under Gateway
|
|
85
|
+
* API the certificate belongs to the Gateway's listener, so setting this in
|
|
86
|
+
* gateway mode is refused at boot rather than silently ignored. */
|
|
87
|
+
tlsSecretName?: string;
|
|
88
|
+
gateway?: GatewayParentConfig;
|
|
89
|
+
/** How long a published route may go unclaimed by any controller before it is
|
|
90
|
+
* reported `unprogrammed`. */
|
|
91
|
+
routeReadyTimeoutMs: number;
|
|
92
|
+
}
|
|
93
|
+
|
|
48
94
|
export interface K8sRunnerConfig extends RunnerCoreConfig {
|
|
49
95
|
/** Identity advertised on `/v1/capabilities` — studio labels the runner
|
|
50
96
|
* with these. */
|
|
51
97
|
displayName: string;
|
|
52
98
|
description: string;
|
|
53
|
-
/** Namespace where session Pods/Services
|
|
99
|
+
/** Namespace where session Pods/Services and routing objects are created. */
|
|
54
100
|
sessionNamespace: string;
|
|
55
101
|
/** Default image for spawned session Pods (telorun/node). Baked into the runner
|
|
56
102
|
* image as `RUNNER_IMAGE` at build time (the kernel version the runner was
|
|
@@ -64,14 +110,8 @@ export interface K8sRunnerConfig extends RunnerCoreConfig {
|
|
|
64
110
|
imagePullSecret?: string;
|
|
65
111
|
/** Optional sandbox RuntimeClass (gvisor/kata). Unset → cluster default (runc). */
|
|
66
112
|
runtimeClass?: string;
|
|
67
|
-
/**
|
|
68
|
-
|
|
69
|
-
/** Optional IngressClass name for created session Ingresses. */
|
|
70
|
-
sessionIngressClassName?: string;
|
|
71
|
-
/** Optional `kubernetes.io/tls` Secret (in the session namespace) the per-session
|
|
72
|
-
* Ingress presents so an upstream (e.g. Cloudflare Full (Strict)) can validate
|
|
73
|
-
* the origin. Must cover the wildcard `*.<sessionIngressBaseDomain>`. Unset → no TLS block. */
|
|
74
|
-
sessionIngressTlsSecretName?: string;
|
|
113
|
+
/** How per-session routes are published. */
|
|
114
|
+
sessionRouting: SessionRoutingConfig;
|
|
75
115
|
/** Runner's own in-cluster base URL, used to build the bundle fetch URL the
|
|
76
116
|
* initContainer curls (e.g. http://k8s-runner.telo-runner:8062). */
|
|
77
117
|
selfUrl: string;
|
|
@@ -102,6 +142,69 @@ function parseTagRegex(raw: string | undefined, field: string): RegExp[] | undef
|
|
|
102
142
|
}
|
|
103
143
|
}
|
|
104
144
|
|
|
145
|
+
/** Parse the routing mode, THROWING on an unrecognized one rather than degrading
|
|
146
|
+
* to a default — a typo'd mode that silently fell back to `auto` is the same
|
|
147
|
+
* class of failure this whole mechanism exists to remove. */
|
|
148
|
+
function parseRoutingMode(raw: string | undefined): SessionRoutingMode {
|
|
149
|
+
const v = raw?.trim();
|
|
150
|
+
if (!v) return "auto";
|
|
151
|
+
if ((ROUTING_MODES as readonly string[]).includes(v)) return v as SessionRoutingMode;
|
|
152
|
+
throw new RunnerConfigError(
|
|
153
|
+
`SESSION_ROUTING_MODE must be one of ${ROUTING_MODES.join(" | ")} (got '${v}').`,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function loadSessionRoutingConfig(
|
|
158
|
+
env: NodeJS.ProcessEnv,
|
|
159
|
+
sessionNamespace: string,
|
|
160
|
+
): SessionRoutingConfig {
|
|
161
|
+
const mode = parseRoutingMode(env.SESSION_ROUTING_MODE);
|
|
162
|
+
const gatewayName = env.SESSION_GATEWAY_NAME?.trim() || undefined;
|
|
163
|
+
const tlsSecretName = env.SESSION_INGRESS_TLS_SECRET?.trim() || undefined;
|
|
164
|
+
|
|
165
|
+
// A certificate configured for a layer that will never present it is a silent
|
|
166
|
+
// downgrade to plaintext origin traffic, so it is refused rather than ignored.
|
|
167
|
+
// This is only the FAST PATH, for the mode that states the layer outright: the
|
|
168
|
+
// authoritative refusal is at routing resolution, because under `auto` a named
|
|
169
|
+
// Gateway decides the layer and nothing here can see that yet.
|
|
170
|
+
if (mode === "gateway" && tlsSecretName) {
|
|
171
|
+
throw new RunnerConfigError(
|
|
172
|
+
"SESSION_INGRESS_TLS_SECRET is set but SESSION_ROUTING_MODE=gateway. Under Gateway API the " +
|
|
173
|
+
"origin certificate belongs to the Gateway listener's own `tls.certificateRefs`, not to the " +
|
|
174
|
+
"per-session route. Move the Secret reference to the Gateway and unset this.",
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
if (mode === "gateway" && !gatewayName) {
|
|
178
|
+
throw new RunnerConfigError(
|
|
179
|
+
"SESSION_ROUTING_MODE=gateway requires SESSION_GATEWAY_NAME — the Gateway each session's " +
|
|
180
|
+
"HTTPRoute attaches to. Set it (and SESSION_GATEWAY_NAMESPACE when the Gateway is not in " +
|
|
181
|
+
`'${sessionNamespace}').`,
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return {
|
|
186
|
+
mode,
|
|
187
|
+
baseDomain: env.SESSION_ROUTING_BASE_DOMAIN?.trim() || undefined,
|
|
188
|
+
ingressClassName: env.SESSION_INGRESS_CLASS?.trim() || undefined,
|
|
189
|
+
tlsSecretName,
|
|
190
|
+
...(gatewayName
|
|
191
|
+
? {
|
|
192
|
+
gateway: {
|
|
193
|
+
name: gatewayName,
|
|
194
|
+
namespace: env.SESSION_GATEWAY_NAMESPACE?.trim() || sessionNamespace,
|
|
195
|
+
sectionName: env.SESSION_GATEWAY_SECTION_NAME?.trim() || undefined,
|
|
196
|
+
},
|
|
197
|
+
}
|
|
198
|
+
: {}),
|
|
199
|
+
routeReadyTimeoutMs:
|
|
200
|
+
parsePositiveInt(
|
|
201
|
+
env.SESSION_ROUTE_READY_TIMEOUT_SECONDS,
|
|
202
|
+
60,
|
|
203
|
+
"SESSION_ROUTE_READY_TIMEOUT_SECONDS",
|
|
204
|
+
) * 1000,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
105
208
|
function loadBaseImageCatalogConfig(env: NodeJS.ProcessEnv): BaseImageCatalogConfig {
|
|
106
209
|
return {
|
|
107
210
|
enabled: parseBool(env.RUNNER_BASE_IMAGE_CATALOG_ENABLED, true, "RUNNER_BASE_IMAGE_CATALOG_ENABLED"),
|
|
@@ -136,19 +239,19 @@ export function loadK8sRunnerConfig(env: NodeJS.ProcessEnv): K8sRunnerConfig {
|
|
|
136
239
|
);
|
|
137
240
|
}
|
|
138
241
|
|
|
242
|
+
const sessionNamespace = env.RUNNER_SESSION_NAMESPACE?.trim() || "telo-sessions";
|
|
243
|
+
|
|
139
244
|
return {
|
|
140
245
|
...loadCoreConfig(env, { port: DEFAULT_PORT }),
|
|
141
246
|
displayName: env.RUNNER_DISPLAY_NAME?.trim() || "Telo Runner",
|
|
142
247
|
description:
|
|
143
248
|
env.RUNNER_DESCRIPTION?.trim() || "Runs the Telo application in a cloud environment",
|
|
144
|
-
sessionNamespace
|
|
249
|
+
sessionNamespace,
|
|
145
250
|
defaultImage: env.RUNNER_IMAGE?.trim() || "telorun/node:latest-slim",
|
|
146
251
|
initImage: env.RUNNER_INIT_IMAGE?.trim() || "busybox:stable",
|
|
147
252
|
imagePullSecret: env.RUNNER_IMAGE_PULL_SECRET?.trim() || undefined,
|
|
148
253
|
runtimeClass: env.RUNNER_RUNTIME_CLASS?.trim() || undefined,
|
|
149
|
-
|
|
150
|
-
sessionIngressClassName: env.SESSION_INGRESS_CLASS?.trim() || undefined,
|
|
151
|
-
sessionIngressTlsSecretName: env.SESSION_INGRESS_TLS_SECRET?.trim() || undefined,
|
|
254
|
+
sessionRouting: loadSessionRoutingConfig(env, sessionNamespace),
|
|
152
255
|
selfUrl: selfUrl.replace(/\/+$/, ""),
|
|
153
256
|
managedByLabel: env.RUNNER_MANAGED_BY?.trim() || "telo-k8s-runner",
|
|
154
257
|
// Sized for a pod that RESOLVES ITS OWN module closure. They used to be
|
package/src/k8s/backend.ts
CHANGED
|
@@ -17,7 +17,14 @@ import type { K8sRunnerConfig } from "../config.js";
|
|
|
17
17
|
import { clampLimits } from "../limits.js";
|
|
18
18
|
import { apiFailure, apiReason, withCause } from "./api-error.js";
|
|
19
19
|
import type { KubeClient } from "./client.js";
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
buildSessionService,
|
|
22
|
+
endpointsFor,
|
|
23
|
+
hostForPort,
|
|
24
|
+
watchRouteHealth,
|
|
25
|
+
type PublishedRoute,
|
|
26
|
+
type SessionRouter,
|
|
27
|
+
} from "./routing/index.js";
|
|
21
28
|
import { buildAppPod, buildSessionPod, INSPECT_PORT } from "./pod-spec.js";
|
|
22
29
|
import {
|
|
23
30
|
deletePod,
|
|
@@ -50,10 +57,13 @@ export interface K8sBackendDeps {
|
|
|
50
57
|
kube: KubeClient;
|
|
51
58
|
config: K8sRunnerConfig;
|
|
52
59
|
bundleStore: BundleStore;
|
|
60
|
+
/** How this cluster publishes session routes, resolved once at boot. Absent →
|
|
61
|
+
* logs-only: no base domain, or `SESSION_ROUTING_MODE=none`. */
|
|
62
|
+
router?: SessionRouter;
|
|
53
63
|
}
|
|
54
64
|
|
|
55
65
|
export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
56
|
-
const { kube, config, bundleStore } = deps;
|
|
66
|
+
const { kube, config, bundleStore, router } = deps;
|
|
57
67
|
const ns = config.sessionNamespace;
|
|
58
68
|
|
|
59
69
|
async function probe(probeConfig: ProbeConfig): Promise<AvailabilityReport> {
|
|
@@ -82,7 +92,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
82
92
|
// outlives its runs — so it takes its own path rather than accreting
|
|
83
93
|
// branches through this one.
|
|
84
94
|
if (spec.mode === "watch") {
|
|
85
|
-
return startWatchSession({ kube, config }, spec);
|
|
95
|
+
return startWatchSession({ kube, config, router }, spec);
|
|
86
96
|
}
|
|
87
97
|
return startRunSession(spec);
|
|
88
98
|
}
|
|
@@ -155,7 +165,9 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
155
165
|
let startDeadline: NodeJS.Timeout | undefined;
|
|
156
166
|
let podIP: string | undefined;
|
|
157
167
|
const debugAbort = new AbortController();
|
|
158
|
-
|
|
168
|
+
// Aborts every background watcher this session started: the per-port
|
|
169
|
+
// reachability probes and the route-health watch.
|
|
170
|
+
const watchersAbort = new AbortController();
|
|
159
171
|
|
|
160
172
|
const stdin = new PassThrough();
|
|
161
173
|
const stdout = new Writable({
|
|
@@ -178,7 +190,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
178
190
|
clearStartDeadline();
|
|
179
191
|
abortWatch();
|
|
180
192
|
debugAbort.abort();
|
|
181
|
-
|
|
193
|
+
watchersAbort.abort();
|
|
182
194
|
bundleStore.drop(spec.sessionId);
|
|
183
195
|
spec.onStatus(status);
|
|
184
196
|
try {
|
|
@@ -197,7 +209,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
197
209
|
const flipRunning = (): void => {
|
|
198
210
|
if (readyFlipped || finished) return;
|
|
199
211
|
readyFlipped = true;
|
|
200
|
-
spec.onStatus({ kind: "running", endpoints: endpointsFor(config, spec.sessionId, app.ports) });
|
|
212
|
+
spec.onStatus({ kind: "running", endpoints: endpointsFor(config, spec.sessionId, app.ports, Boolean(router)) });
|
|
201
213
|
};
|
|
202
214
|
|
|
203
215
|
let resolveRunning!: () => void;
|
|
@@ -327,13 +339,52 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
327
339
|
);
|
|
328
340
|
}
|
|
329
341
|
|
|
330
|
-
if (
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
342
|
+
if (router && app.ports.length > 0) {
|
|
343
|
+
let published: PublishedRoute[] = [];
|
|
344
|
+
try {
|
|
345
|
+
published = await publishRoutes(
|
|
346
|
+
deps,
|
|
347
|
+
router,
|
|
348
|
+
spec.sessionId,
|
|
349
|
+
podName,
|
|
350
|
+
podUid,
|
|
351
|
+
app.ports,
|
|
336
352
|
);
|
|
353
|
+
} catch (err) {
|
|
354
|
+
// A publish failure is reported on the ROUTE channel, not only as a
|
|
355
|
+
// terminal line: the endpoints were already advertised on the `running`
|
|
356
|
+
// status, so leaving this to a log line is the silence this whole change
|
|
357
|
+
// exists to remove.
|
|
358
|
+
const reason = `could not publish the session route: ${apiReason(err)}`;
|
|
359
|
+
for (const p of app.ports) {
|
|
360
|
+
if (p.protocol !== "tcp") continue;
|
|
361
|
+
spec.onRoute(appName, {
|
|
362
|
+
host: hostForPort(config, spec.sessionId, p.port),
|
|
363
|
+
port: p.port,
|
|
364
|
+
state: "unprogrammed",
|
|
365
|
+
reason,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
spec.onOutput(appName, Buffer.from(`\r\n[runner] ${reason}\r\n`), "tty");
|
|
369
|
+
}
|
|
370
|
+
// Verify the routing layer actually PROGRAMMED what we published. Without
|
|
371
|
+
// this a cluster whose routing objects nothing reconciles is indistinguishable
|
|
372
|
+
// from a healthy one: reachability below dials the pod directly and stays
|
|
373
|
+
// green while every public URL 404s.
|
|
374
|
+
void watchRouteHealth({
|
|
375
|
+
router,
|
|
376
|
+
sessionId: spec.sessionId,
|
|
377
|
+
routes: published,
|
|
378
|
+
onState: (route, state, reason) =>
|
|
379
|
+
spec.onRoute(appName, { host: route.host, port: route.port, state, reason }),
|
|
380
|
+
onReadError: (err) =>
|
|
381
|
+
spec.onOutput(
|
|
382
|
+
appName,
|
|
383
|
+
Buffer.from(`\r\n[runner] could not read route status: ${apiReason(err)}\r\n`),
|
|
384
|
+
"tty",
|
|
385
|
+
),
|
|
386
|
+
signal: watchersAbort.signal,
|
|
387
|
+
timeoutMs: config.sessionRouting.routeReadyTimeoutMs,
|
|
337
388
|
});
|
|
338
389
|
}
|
|
339
390
|
|
|
@@ -372,7 +423,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
372
423
|
if (tcpPorts.length > 0) {
|
|
373
424
|
void (async () => {
|
|
374
425
|
const ip = await resolvePodIP();
|
|
375
|
-
if (
|
|
426
|
+
if (watchersAbort.signal.aborted) return;
|
|
376
427
|
if (!ip) {
|
|
377
428
|
// Couldn't resolve the pod IP to probe — report unverified rather than
|
|
378
429
|
// leaving the badge spinning forever.
|
|
@@ -383,7 +434,7 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
383
434
|
host: ip,
|
|
384
435
|
ports: tcpPorts,
|
|
385
436
|
onState: (port, state) => spec.onReachability(appName, port, state),
|
|
386
|
-
signal:
|
|
437
|
+
signal: watchersAbort.signal,
|
|
387
438
|
});
|
|
388
439
|
})();
|
|
389
440
|
}
|
|
@@ -443,28 +494,27 @@ export function createKubernetesBackend(deps: K8sBackendDeps): RunnerBackend {
|
|
|
443
494
|
return { probe, start, reapOrphans };
|
|
444
495
|
}
|
|
445
496
|
|
|
446
|
-
|
|
497
|
+
/** Create the session Service and publish its routes on whichever layer the
|
|
498
|
+
* cluster resolved to. Returns the hosts published, for the route watch. */
|
|
499
|
+
async function publishRoutes(
|
|
447
500
|
deps: K8sBackendDeps,
|
|
501
|
+
router: SessionRouter,
|
|
448
502
|
sessionId: string,
|
|
449
503
|
podName: string,
|
|
450
504
|
podUid: string,
|
|
451
505
|
ports: PortMapping[],
|
|
452
|
-
): Promise<
|
|
506
|
+
): Promise<PublishedRoute[]> {
|
|
453
507
|
const { kube, config } = deps;
|
|
454
508
|
const ns = config.sessionNamespace;
|
|
455
509
|
const service = buildSessionService(config, sessionId, podName, podUid, ports);
|
|
456
510
|
await kube.core.createNamespacedService({ namespace: ns, body: service });
|
|
457
|
-
|
|
458
|
-
config,
|
|
511
|
+
return router.publish({
|
|
459
512
|
sessionId,
|
|
460
|
-
service.metadata!.name!,
|
|
513
|
+
serviceName: service.metadata!.name!,
|
|
461
514
|
podName,
|
|
462
515
|
podUid,
|
|
463
516
|
ports,
|
|
464
|
-
);
|
|
465
|
-
// No tcp ports → no HTTP-routable rules; the Service still exists for any udp.
|
|
466
|
-
if (!ingress.spec?.rules?.length) return;
|
|
467
|
-
await kube.networking.createNamespacedIngress({ namespace: ns, body: ingress });
|
|
517
|
+
});
|
|
468
518
|
}
|
|
469
519
|
|
|
470
520
|
async function clusterReachable(kube: KubeClient): Promise<boolean> {
|
package/src/k8s/client.ts
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ApisApi,
|
|
3
|
+
Attach,
|
|
4
|
+
CoreV1Api,
|
|
5
|
+
CustomObjectsApi,
|
|
6
|
+
KubeConfig,
|
|
7
|
+
NetworkingV1Api,
|
|
8
|
+
Watch,
|
|
9
|
+
} from "@kubernetes/client-node";
|
|
2
10
|
|
|
3
11
|
export interface KubeClient {
|
|
4
12
|
kc: KubeConfig;
|
|
5
13
|
core: CoreV1Api;
|
|
6
14
|
networking: NetworkingV1Api;
|
|
15
|
+
/** Gateway API has no typed client in `@kubernetes/client-node`, so HTTPRoutes
|
|
16
|
+
* are read and written as unstructured custom objects. */
|
|
17
|
+
custom: CustomObjectsApi;
|
|
18
|
+
/** API-group discovery, used to decide whether Gateway API is served at all
|
|
19
|
+
* when the routing mode is `auto`. */
|
|
20
|
+
apis: ApisApi;
|
|
7
21
|
attach: Attach;
|
|
8
22
|
watch: Watch;
|
|
9
23
|
}
|
|
@@ -24,6 +38,8 @@ export function createKubeClient(): KubeClient {
|
|
|
24
38
|
kc,
|
|
25
39
|
core: kc.makeApiClient(CoreV1Api),
|
|
26
40
|
networking: kc.makeApiClient(NetworkingV1Api),
|
|
41
|
+
custom: kc.makeApiClient(CustomObjectsApi),
|
|
42
|
+
apis: kc.makeApiClient(ApisApi),
|
|
27
43
|
attach: new Attach(kc),
|
|
28
44
|
watch: new Watch(kc),
|
|
29
45
|
};
|
package/src/k8s/pod-status.ts
CHANGED
|
@@ -132,6 +132,14 @@ export function is404(err: unknown): boolean {
|
|
|
132
132
|
return statusCode(err) === 404;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
/** A name-already-taken rejection — what makes a create idempotent by falling
|
|
136
|
+
* through to a replace. Reads the status through `statusCode` like every other
|
|
137
|
+
* classification here, so an `ECONNREFUSED` or a `DOMException`'s legacy number
|
|
138
|
+
* in `code` cannot be mistaken for one. */
|
|
139
|
+
export function isConflict(err: unknown): boolean {
|
|
140
|
+
return statusCode(err) === 409;
|
|
141
|
+
}
|
|
142
|
+
|
|
135
143
|
export function msg(err: unknown): string {
|
|
136
144
|
if (err instanceof Error) return err.message;
|
|
137
145
|
if (typeof err === "string") return err;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { isConflict } from "../pod-status.js";
|
|
2
|
+
|
|
3
|
+
/** A create that yields to an existing object of the same name. Every routing
|
|
4
|
+
* object is republished on reload (an app's declared port set changes), so the
|
|
5
|
+
* write has to be idempotent rather than create-once. */
|
|
6
|
+
export async function createOrReplace(
|
|
7
|
+
create: () => Promise<unknown>,
|
|
8
|
+
replace: () => Promise<unknown>,
|
|
9
|
+
): Promise<void> {
|
|
10
|
+
try {
|
|
11
|
+
await create();
|
|
12
|
+
} catch (err) {
|
|
13
|
+
if (!isConflict(err)) throw err;
|
|
14
|
+
await replace();
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
|
|
3
|
+
import type { PortMapping } from "@telorun/runner-core";
|
|
4
|
+
|
|
5
|
+
import type { K8sRunnerConfig } from "../../config.js";
|
|
6
|
+
import type { KubeClient } from "../client.js";
|
|
7
|
+
import { createGatewayRouter } from "./gateway-router.js";
|
|
8
|
+
|
|
9
|
+
const sessionId = "hzyayvabgyvz";
|
|
10
|
+
|
|
11
|
+
const config = {
|
|
12
|
+
sessionNamespace: "telo-sessions",
|
|
13
|
+
managedByLabel: "telo-k8s-runner",
|
|
14
|
+
sessionRouting: {
|
|
15
|
+
mode: "gateway",
|
|
16
|
+
baseDomain: "telo.run",
|
|
17
|
+
routeReadyTimeoutMs: 60_000,
|
|
18
|
+
gateway: { name: "public", namespace: "gateway-system" },
|
|
19
|
+
},
|
|
20
|
+
} as K8sRunnerConfig;
|
|
21
|
+
|
|
22
|
+
interface Recorded {
|
|
23
|
+
created: Record<string, unknown>[];
|
|
24
|
+
deleted: string[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** A KubeClient stub carrying only what the gateway router touches. */
|
|
28
|
+
function fakeKube(opts: {
|
|
29
|
+
existing?: { metadata?: { name?: string } }[];
|
|
30
|
+
getResult?: (name: string) => unknown;
|
|
31
|
+
} = {}): { kube: KubeClient; rec: Recorded } {
|
|
32
|
+
const rec: Recorded = { created: [], deleted: [] };
|
|
33
|
+
const kube = {
|
|
34
|
+
custom: {
|
|
35
|
+
createNamespacedCustomObject: async ({ body }: { body: Record<string, unknown> }) => {
|
|
36
|
+
rec.created.push(body);
|
|
37
|
+
return body;
|
|
38
|
+
},
|
|
39
|
+
listNamespacedCustomObject: async () => ({ items: opts.existing ?? [] }),
|
|
40
|
+
deleteNamespacedCustomObject: async ({ name }: { name: string }) => {
|
|
41
|
+
rec.deleted.push(name);
|
|
42
|
+
return {};
|
|
43
|
+
},
|
|
44
|
+
getNamespacedCustomObject: async ({ name }: { name: string }) => {
|
|
45
|
+
if (opts.getResult) return opts.getResult(name);
|
|
46
|
+
throw Object.assign(new Error("not found"), { code: 404 });
|
|
47
|
+
},
|
|
48
|
+
replaceNamespacedCustomObject: async ({ body }: { body: Record<string, unknown> }) => body,
|
|
49
|
+
},
|
|
50
|
+
} as unknown as KubeClient;
|
|
51
|
+
return { kube, rec };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const ports: PortMapping[] = [
|
|
55
|
+
{ port: 8080, protocol: "tcp" },
|
|
56
|
+
{ port: 9090, protocol: "tcp" },
|
|
57
|
+
{ port: 5000, protocol: "udp" },
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
const publishArgs = {
|
|
61
|
+
sessionId,
|
|
62
|
+
serviceName: `telo-run-${sessionId}`,
|
|
63
|
+
podName: "pod",
|
|
64
|
+
podUid: "uid",
|
|
65
|
+
ports,
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
describe("createGatewayRouter.publish", () => {
|
|
69
|
+
it("creates ONE HTTPRoute PER PORT — hostnames are route-scoped, so a shared route would collapse every host onto one backend", async () => {
|
|
70
|
+
const { kube, rec } = fakeKube();
|
|
71
|
+
const routes = await createGatewayRouter(kube, config, "v1").publish(publishArgs);
|
|
72
|
+
|
|
73
|
+
expect(rec.created).toHaveLength(2);
|
|
74
|
+
for (const body of rec.created) {
|
|
75
|
+
const spec = body.spec as { hostnames: string[]; rules: { backendRefs: unknown[] }[] };
|
|
76
|
+
expect(spec.hostnames).toHaveLength(1);
|
|
77
|
+
expect(spec.rules).toHaveLength(1);
|
|
78
|
+
expect(spec.rules[0].backendRefs).toHaveLength(1);
|
|
79
|
+
}
|
|
80
|
+
// Each host is bound to its own port's backend, never a sibling's.
|
|
81
|
+
const pairs = rec.created.map((b) => {
|
|
82
|
+
const spec = b.spec as {
|
|
83
|
+
hostnames: string[];
|
|
84
|
+
rules: { backendRefs: { name: string; port: number }[] }[];
|
|
85
|
+
};
|
|
86
|
+
return [spec.hostnames[0], spec.rules[0].backendRefs[0].port];
|
|
87
|
+
});
|
|
88
|
+
expect(pairs).toEqual([
|
|
89
|
+
[`8080-${sessionId}.telo.run`, 8080],
|
|
90
|
+
[`9090-${sessionId}.telo.run`, 9090],
|
|
91
|
+
]);
|
|
92
|
+
expect(routes).toEqual([
|
|
93
|
+
{ host: `8080-${sessionId}.telo.run`, port: 8080 },
|
|
94
|
+
{ host: `9090-${sessionId}.telo.run`, port: 9090 },
|
|
95
|
+
]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("skips udp ports — they are not HTTP-routable", async () => {
|
|
99
|
+
const { kube, rec } = fakeKube();
|
|
100
|
+
await createGatewayRouter(kube, config, "v1").publish({
|
|
101
|
+
...publishArgs,
|
|
102
|
+
ports: [{ port: 5000, protocol: "udp" }],
|
|
103
|
+
});
|
|
104
|
+
expect(rec.created).toEqual([]);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("attaches to the configured Gateway, carrying the listener when one is named", async () => {
|
|
108
|
+
const { kube, rec } = fakeKube();
|
|
109
|
+
const sectioned = {
|
|
110
|
+
...config,
|
|
111
|
+
sessionRouting: {
|
|
112
|
+
...config.sessionRouting,
|
|
113
|
+
gateway: { name: "public", namespace: "gateway-system", sectionName: "https" },
|
|
114
|
+
},
|
|
115
|
+
} as K8sRunnerConfig;
|
|
116
|
+
await createGatewayRouter(kube, sectioned, "v1").publish({ ...publishArgs, ports: [ports[0]] });
|
|
117
|
+
expect((rec.created[0].spec as { parentRefs: unknown[] }).parentRefs).toEqual([
|
|
118
|
+
{
|
|
119
|
+
group: "gateway.networking.k8s.io",
|
|
120
|
+
kind: "Gateway",
|
|
121
|
+
name: "public",
|
|
122
|
+
namespace: "gateway-system",
|
|
123
|
+
sectionName: "https",
|
|
124
|
+
},
|
|
125
|
+
]);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("deletes the route of a port a reload dropped — one object per port means a stale one keeps routing", async () => {
|
|
129
|
+
const { kube, rec } = fakeKube({
|
|
130
|
+
existing: [
|
|
131
|
+
{ metadata: { name: `telo-run-${sessionId}-8080` } },
|
|
132
|
+
{ metadata: { name: `telo-run-${sessionId}-9090` } },
|
|
133
|
+
],
|
|
134
|
+
});
|
|
135
|
+
await createGatewayRouter(kube, config, "v1").publish({
|
|
136
|
+
...publishArgs,
|
|
137
|
+
ports: [{ port: 8080, protocol: "tcp" }],
|
|
138
|
+
});
|
|
139
|
+
expect(rec.deleted).toEqual([`telo-run-${sessionId}-9090`]);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("posts at the served API version", async () => {
|
|
143
|
+
const { kube, rec } = fakeKube();
|
|
144
|
+
await createGatewayRouter(kube, config, "v1beta1").publish({ ...publishArgs, ports: [ports[0]] });
|
|
145
|
+
expect(rec.created[0].apiVersion).toBe("gateway.networking.k8s.io/v1beta1");
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
describe("createGatewayRouter.verdictFor", () => {
|
|
150
|
+
const route = { host: `8080-${sessionId}.telo.run`, port: 8080 };
|
|
151
|
+
|
|
152
|
+
function verdict(status: unknown) {
|
|
153
|
+
const { kube } = fakeKube({ getResult: () => ({ status }) });
|
|
154
|
+
return createGatewayRouter(kube, config, "v1").verdictFor(sessionId, route);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
it("is pending while no controller has written status", async () => {
|
|
158
|
+
await expect(verdict(undefined)).resolves.toEqual({ kind: "pending" });
|
|
159
|
+
await expect(verdict({ parents: [] })).resolves.toEqual({ kind: "pending" });
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it("is programmed once a parent accepted and resolved it", async () => {
|
|
163
|
+
await expect(
|
|
164
|
+
verdict({
|
|
165
|
+
parents: [
|
|
166
|
+
{
|
|
167
|
+
conditions: [
|
|
168
|
+
{ type: "Accepted", status: "True" },
|
|
169
|
+
{ type: "ResolvedRefs", status: "True" },
|
|
170
|
+
],
|
|
171
|
+
},
|
|
172
|
+
],
|
|
173
|
+
}),
|
|
174
|
+
).resolves.toEqual({ kind: "programmed" });
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it("reports the controller's own reason for a refusal — the actionable half", async () => {
|
|
178
|
+
await expect(
|
|
179
|
+
verdict({
|
|
180
|
+
parents: [
|
|
181
|
+
{
|
|
182
|
+
conditions: [
|
|
183
|
+
{
|
|
184
|
+
type: "Accepted",
|
|
185
|
+
status: "False",
|
|
186
|
+
reason: "NotAllowedByListeners",
|
|
187
|
+
message: "no listener admits routes from this namespace",
|
|
188
|
+
},
|
|
189
|
+
],
|
|
190
|
+
},
|
|
191
|
+
],
|
|
192
|
+
}),
|
|
193
|
+
).resolves.toEqual({
|
|
194
|
+
kind: "rejected",
|
|
195
|
+
reason: "NotAllowedByListeners: no listener admits routes from this namespace",
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("reports an unresolvable backend rather than calling the route programmed", async () => {
|
|
200
|
+
await expect(
|
|
201
|
+
verdict({
|
|
202
|
+
parents: [
|
|
203
|
+
{
|
|
204
|
+
conditions: [
|
|
205
|
+
{ type: "Accepted", status: "True" },
|
|
206
|
+
{ type: "ResolvedRefs", status: "False", reason: "BackendNotFound" },
|
|
207
|
+
],
|
|
208
|
+
},
|
|
209
|
+
],
|
|
210
|
+
}),
|
|
211
|
+
).resolves.toEqual({ kind: "rejected", reason: "BackendNotFound" });
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it("lets one accepting parent win over a sibling's refusal — traffic arrives either way", async () => {
|
|
215
|
+
await expect(
|
|
216
|
+
verdict({
|
|
217
|
+
parents: [
|
|
218
|
+
{ conditions: [{ type: "Accepted", status: "False", reason: "NoMatchingParent" }] },
|
|
219
|
+
{
|
|
220
|
+
conditions: [
|
|
221
|
+
{ type: "Accepted", status: "True" },
|
|
222
|
+
{ type: "ResolvedRefs", status: "True" },
|
|
223
|
+
],
|
|
224
|
+
},
|
|
225
|
+
],
|
|
226
|
+
}),
|
|
227
|
+
).resolves.toEqual({ kind: "programmed" });
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
it("is pending when the object is not visible yet", async () => {
|
|
231
|
+
const { kube } = fakeKube();
|
|
232
|
+
await expect(
|
|
233
|
+
createGatewayRouter(kube, config, "v1").verdictFor(sessionId, route),
|
|
234
|
+
).resolves.toEqual({ kind: "pending" });
|
|
235
|
+
});
|
|
236
|
+
});
|