alasio 8.0.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/src/api-server.js +18 -0
- package/dist/cli/src/api-server.js.map +1 -0
- package/dist/cli/src/bound-harnesses.js +98 -0
- package/dist/cli/src/bound-harnesses.js.map +1 -0
- package/dist/cli/src/branches.js +79 -22
- package/dist/cli/src/branches.js.map +1 -1
- package/dist/cli/src/cluster/docker.js +40 -15
- package/dist/cli/src/cluster/docker.js.map +1 -1
- package/dist/cli/src/cluster/host.js +38 -8
- package/dist/cli/src/cluster/host.js.map +1 -1
- package/dist/cli/src/cluster/k3s.js +105 -14
- package/dist/cli/src/cluster/k3s.js.map +1 -1
- package/dist/cli/src/cluster/node.js +142 -33
- package/dist/cli/src/cluster/node.js.map +1 -1
- package/dist/cli/src/cluster/stargz.js +92 -0
- package/dist/cli/src/cluster/stargz.js.map +1 -0
- package/dist/cli/src/commands/branch.js +5 -4
- package/dist/cli/src/commands/branch.js.map +1 -1
- package/dist/cli/src/commands/common.js +11 -5
- package/dist/cli/src/commands/common.js.map +1 -1
- package/dist/cli/src/commands/folder.js +14 -0
- package/dist/cli/src/commands/folder.js.map +1 -0
- package/dist/cli/src/commands/lake.js +1 -1
- package/dist/cli/src/commands/lake.js.map +1 -1
- package/dist/cli/src/commands/login-relay.js +91 -0
- package/dist/cli/src/commands/login-relay.js.map +1 -0
- package/dist/cli/src/commands/login.js +11 -10
- package/dist/cli/src/commands/login.js.map +1 -1
- package/dist/cli/src/commands/logs.js +89 -6
- package/dist/cli/src/commands/logs.js.map +1 -1
- package/dist/cli/src/commands/restart.js +10 -4
- package/dist/cli/src/commands/restart.js.map +1 -1
- package/dist/cli/src/commands/restore-filer-store.js +27 -10
- package/dist/cli/src/commands/restore-filer-store.js.map +1 -1
- package/dist/cli/src/commands/status.js +119 -20
- package/dist/cli/src/commands/status.js.map +1 -1
- package/dist/cli/src/commands/uninstall.js +8 -9
- package/dist/cli/src/commands/uninstall.js.map +1 -1
- package/dist/cli/src/commands/up.js +20 -9
- package/dist/cli/src/commands/up.js.map +1 -1
- package/dist/cli/src/commands/upgrade.js +22 -6
- package/dist/cli/src/commands/upgrade.js.map +1 -1
- package/dist/cli/src/commands.js +2 -1
- package/dist/cli/src/commands.js.map +1 -1
- package/dist/cli/src/config.js +4 -1
- package/dist/cli/src/config.js.map +1 -1
- package/dist/cli/src/cpu-baseline.js +8 -4
- package/dist/cli/src/cpu-baseline.js.map +1 -1
- package/dist/cli/src/folders.js +63 -0
- package/dist/cli/src/folders.js.map +1 -0
- package/dist/cli/src/gateway-calls.js +64 -0
- package/dist/cli/src/gateway-calls.js.map +1 -0
- package/dist/cli/src/install.js +45 -31
- package/dist/cli/src/install.js.map +1 -1
- package/dist/cli/src/kept-releases.js +27 -20
- package/dist/cli/src/kept-releases.js.map +1 -1
- package/dist/cli/src/kube/api.js +61 -15
- package/dist/cli/src/kube/api.js.map +1 -1
- package/dist/cli/src/kube/apply.js +122 -81
- package/dist/cli/src/kube/apply.js.map +1 -1
- package/dist/cli/src/kube/rollout.js +43 -21
- package/dist/cli/src/kube/rollout.js.map +1 -1
- package/dist/cli/src/main.js +2 -1
- package/dist/cli/src/main.js.map +1 -1
- package/dist/cli/src/manifests/admission.js +558 -222
- package/dist/cli/src/manifests/admission.js.map +1 -1
- package/dist/cli/src/manifests/alasio.js +180 -428
- package/dist/cli/src/manifests/alasio.js.map +1 -1
- package/dist/cli/src/manifests/branch.js +16 -27
- package/dist/cli/src/manifests/branch.js.map +1 -1
- package/dist/cli/src/manifests/codex-login.js +101 -0
- package/dist/cli/src/manifests/codex-login.js.map +1 -0
- package/dist/cli/src/manifests/collector.js +83 -62
- package/dist/cli/src/manifests/collector.js.map +1 -1
- package/dist/cli/src/manifests/common.js +82 -25
- package/dist/cli/src/manifests/common.js.map +1 -1
- package/dist/cli/src/manifests/config.js +137 -105
- package/dist/cli/src/manifests/config.js.map +1 -1
- package/dist/cli/src/manifests/disruption.js +18 -3
- package/dist/cli/src/manifests/disruption.js.map +1 -1
- package/dist/cli/src/manifests/folders.js +131 -0
- package/dist/cli/src/manifests/folders.js.map +1 -0
- package/dist/cli/src/manifests/grafana.js +4 -2
- package/dist/cli/src/manifests/grafana.js.map +1 -1
- package/dist/cli/src/manifests/harness-pools.js +236 -0
- package/dist/cli/src/manifests/harness-pools.js.map +1 -0
- package/dist/cli/src/manifests/image-holder.js +61 -0
- package/dist/cli/src/manifests/image-holder.js.map +1 -0
- package/dist/cli/src/manifests/index.js +14 -7
- package/dist/cli/src/manifests/index.js.map +1 -1
- package/dist/cli/src/manifests/keda.js +4 -2
- package/dist/cli/src/manifests/keda.js.map +1 -1
- package/dist/cli/src/manifests/lake.js +18 -20
- package/dist/cli/src/manifests/lake.js.map +1 -1
- package/dist/cli/src/manifests/log.js +84 -19
- package/dist/cli/src/manifests/log.js.map +1 -1
- package/dist/cli/src/manifests/neon.js +60 -80
- package/dist/cli/src/manifests/neon.js.map +1 -1
- package/dist/cli/src/manifests/network-policies.js +232 -94
- package/dist/cli/src/manifests/network-policies.js.map +1 -1
- package/dist/cli/src/manifests/node-agent.js +42 -15
- package/dist/cli/src/manifests/node-agent.js.map +1 -1
- package/dist/cli/src/manifests/object-store.js +99 -25
- package/dist/cli/src/manifests/object-store.js.map +1 -1
- package/dist/cli/src/manifests/sandbox-gateway.js +761 -0
- package/dist/cli/src/manifests/sandbox-gateway.js.map +1 -0
- package/dist/cli/src/manifests/scalers.js +17 -5
- package/dist/cli/src/manifests/scalers.js.map +1 -1
- package/dist/cli/src/manifests/sizing.js +17 -9
- package/dist/cli/src/manifests/sizing.js.map +1 -1
- package/dist/cli/src/manifests/valkey.js +30 -10
- package/dist/cli/src/manifests/valkey.js.map +1 -1
- package/dist/cli/src/manifests/volume-lock.js +43 -0
- package/dist/cli/src/manifests/volume-lock.js.map +1 -0
- package/dist/cli/src/manifests/vpa.js +15 -2
- package/dist/cli/src/manifests/vpa.js.map +1 -1
- package/dist/cli/src/manifests/workspace-storage.js +182 -94
- package/dist/cli/src/manifests/workspace-storage.js.map +1 -1
- package/dist/cli/src/neon-storage.js +54 -16
- package/dist/cli/src/neon-storage.js.map +1 -1
- package/dist/cli/src/release.js +9 -9
- package/dist/cli/src/release.js.map +1 -1
- package/dist/cli/src/setup-pod.js +44 -0
- package/dist/cli/src/setup-pod.js.map +1 -0
- package/dist/cli/src/target.js +2 -2
- package/dist/cli/src/target.js.map +1 -1
- package/dist/cli/src/telegram.js +13 -4
- package/dist/cli/src/telegram.js.map +1 -1
- package/dist/cli/src/wakes.js +68 -0
- package/dist/cli/src/wakes.js.map +1 -0
- package/dist/cli/src/webhook.js +5 -2
- package/dist/cli/src/webhook.js.map +1 -1
- package/dist/cluster/node/pins.json +11 -5
- package/dist/neon/control/branches.js +13 -6
- package/dist/neon/control/branches.js.map +1 -1
- package/dist/neon/control/endpoints.js +81 -10
- package/dist/neon/control/endpoints.js.map +1 -1
- package/dist/neon/control/secrets.js +24 -8
- package/dist/neon/control/secrets.js.map +1 -1
- package/dist/src/binder/bindings.js +195 -0
- package/dist/src/binder/bindings.js.map +1 -0
- package/dist/src/binder/names.js +63 -0
- package/dist/src/binder/names.js.map +1 -0
- package/dist/src/binder/tokens.js +71 -0
- package/dist/src/binder/tokens.js.map +1 -0
- package/dist/src/branch/names.js +20 -6
- package/dist/src/branch/names.js.map +1 -1
- package/dist/src/events/names.js +28 -0
- package/dist/src/events/names.js.map +1 -0
- package/dist/src/gateway/names.js +240 -0
- package/dist/src/gateway/names.js.map +1 -0
- package/dist/src/harness/names.js +67 -0
- package/dist/src/harness/names.js.map +1 -0
- package/dist/src/kube/client.js +212 -0
- package/dist/src/kube/client.js.map +1 -0
- package/dist/src/kube/sandbox-shape.js +14 -68
- package/dist/src/kube/sandbox-shape.js.map +1 -1
- package/dist/src/kube/started-map.js +12 -0
- package/dist/src/kube/started-map.js.map +1 -0
- package/dist/src/node-agent/names.js +62 -5
- package/dist/src/node-agent/names.js.map +1 -1
- package/dist/src/sandbox/names.js +12 -20
- package/dist/src/sandbox/names.js.map +1 -1
- package/dist/src/sandbox/otlp-resource.js +173 -0
- package/dist/src/sandbox/otlp-resource.js.map +1 -0
- package/dist/src/sandbox/telemetry.js +43 -0
- package/dist/src/sandbox/telemetry.js.map +1 -0
- package/dist/src/telegram/bot-commands.js +17 -0
- package/dist/src/telegram/bot-commands.js.map +1 -0
- package/dist/src/telegram/stop.js +12 -0
- package/dist/src/telegram/stop.js.map +1 -0
- package/dist/src/telemetry/config.js +131 -0
- package/dist/src/telemetry/config.js.map +1 -0
- package/dist/src/telemetry/forward.js +154 -0
- package/dist/src/telemetry/forward.js.map +1 -0
- package/package.json +1 -1
- package/dist/cli/src/manifests/agent-sandbox.js +0 -92
- package/dist/cli/src/manifests/agent-sandbox.js.map +0 -1
- package/dist/cli/src/manifests/juicefs-csi.js +0 -320
- package/dist/cli/src/manifests/juicefs-csi.js.map +0 -1
- package/dist/cli/src/manifests/sandboxes.agents.x-k8s.io.json +0 -6295
- package/dist/src/kube/config.js +0 -145
- package/dist/src/kube/config.js.map +0 -1
|
@@ -0,0 +1,761 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sandbox service (src/gateway/, src/binder/): session workspaces' sandboxes as pools
|
|
3
|
+
* of pods already running, each bound to one workspace while it has work, through the
|
|
4
|
+
* gateway.
|
|
5
|
+
*
|
|
6
|
+
* - The gateway: alasio's image, as its own program, the one door to a pool's pod, of main's
|
|
7
|
+
* sessions and of every branch environment's, which it finds by their namespaces; its
|
|
8
|
+
* configuration, rendered here; its keys, made by the stack's setup
|
|
9
|
+
* (neon/control/kube-setup.ts); a Service of it, which alasio binds through and KEDA asks
|
|
10
|
+
* each pool's counts of, and a headless one its replicas find each other by. In each
|
|
11
|
+
* environment's sessions' namespace (sandboxPoolObjects) it may bind, claim and delete the
|
|
12
|
+
* pods of its pools, hold their Leases, and read their Deployments, whose templates a
|
|
13
|
+
* release's snapshot records; and it finds the node agents.
|
|
14
|
+
* - A pool of each of a session's internets, `none` and `full`, in each environment: a
|
|
15
|
+
* Deployment in its sessions' namespace of pods of the agent image under the sessions' runtime, each its own directory
|
|
16
|
+
* on its node (src/node-agent/names.ts ATTACH_ROOT, by `subPathExpr` of its namespace and its name) at /attach,
|
|
17
|
+
* where its workspace is attached once it is bound, its bayma started once its binding's
|
|
18
|
+
* token is beside it (sandbox/agent/start.sh). KEDA keeps `min` free, and makes more, to
|
|
19
|
+
* `max`, for the binds that wait; a bound pod is out of its Deployment, which replaces it.
|
|
20
|
+
* A pod that ends is ended: it serves one binding. Its template changes as a whole: the old
|
|
21
|
+
* free pods go as the new come, as none is ready before it is bound. A bound pod is evicted
|
|
22
|
+
* by no drain (./disruption.ts boundBudget).
|
|
23
|
+
* - The session telemetry receiver (src/receiver/): alasio's image, as its own program,
|
|
24
|
+
* watching the pools' Leases of every environment, which bound pods' bayma exports to.
|
|
25
|
+
* - The node's mount of workspace storage, a pod of each node's own, held through upgrades of
|
|
26
|
+
* the node agent, which attaches workspaces from it: a new one takes its mount over before
|
|
27
|
+
* the old goes.
|
|
28
|
+
*/
|
|
29
|
+
import { createHash } from "node:crypto";
|
|
30
|
+
import { isDeepStrictEqual } from "node:util";
|
|
31
|
+
import { Schema } from "effect";
|
|
32
|
+
import { freeLabels, GATEWAY_READINESS, STOP_GRACE_SECONDS } from "../../../src/binder/names.js";
|
|
33
|
+
import { EVENTS_BROKERS_VARIABLE, EVENTS_PASSWORD_FILE_VARIABLE, EVENTS_USERNAME_VARIABLE } from "../../../src/events/names.js";
|
|
34
|
+
import { MAIN_ENVIRONMENT_NAME } from "../../../src/branch/names.js";
|
|
35
|
+
import { BINDING_KEY_VARIABLE, CALLER_KEY_VARIABLE, GATEWAY_CONFIG_VARIABLE, GatewayConfig as GatewayConfigSchema, FOLDERS_NAMESPACE_VARIABLE, RECEIVER_PORT_VARIABLE, SESSIONS_NAMESPACE_VARIABLE, templateHash, } from "../../../src/gateway/names.js";
|
|
36
|
+
import { BAYMA_CONTAINER, BAYMA_STOP_SECONDS, CPU_FEATURES_ANNOTATION, NET_MODE_LABEL, SESSION_TEMPLATE_ANNOTATION, STOP_WAIT_ARG } from "../../../src/kube/sandbox-shape.js";
|
|
37
|
+
import { ATTACH_ROOT, goferOf, gvisorRuntimeClass, NODE_AGENT_PORT, NODE_AGENT_TOKEN_VARIABLE, WORKSPACES_ROOT } from "../../../src/node-agent/names.js";
|
|
38
|
+
import { memoryBytes } from "../capacity.js";
|
|
39
|
+
import { NODE_PINS } from "../cluster/node.js";
|
|
40
|
+
import { imageReference } from "../images.js";
|
|
41
|
+
import { READY_ONCE_BOUND_ANNOTATION } from "../kube/rollout.js";
|
|
42
|
+
import { sessionsNamespace, stopWaitSeconds } from "./alasio.js";
|
|
43
|
+
import { folderTemplate, foldersRun } from "./folders.js";
|
|
44
|
+
import { harnessesConfig } from "./harness-pools.js";
|
|
45
|
+
import { accountRole, APPLIED_AFTER_ANNOTATION, appliedAfter, componentName, given, imagePullSecrets, labels, MAIN_ENVIRONMENT, NAMESPACE, otelEnv, RELEASE, replicasOf, restrictedContainer, restrictedPod, selectorLabels, sha256, TINI_SUBREAPER, } from "./common.js";
|
|
46
|
+
import { resumedConditionPolicy, sandboxPodPolicy } from "./admission.js";
|
|
47
|
+
import { durationSeconds } from "./config.js";
|
|
48
|
+
import { LOG_BROKERS, logPrincipalSecret } from "./log.js";
|
|
49
|
+
import { boundBudget } from "./disruption.js";
|
|
50
|
+
import { FILER_STORE } from "./object-store.js";
|
|
51
|
+
import { priority } from "./priorities.js";
|
|
52
|
+
import { scaledObject } from "./scalers.js";
|
|
53
|
+
import { verticalPodAutoscaler } from "./sizing.js";
|
|
54
|
+
import { JUICEFS_ADMIN, VALKEY, VALKEY_ADDRESS, WORKSPACES_CREDENTIALS, workspacesBucketUrl } from "./workspace-storage.js";
|
|
55
|
+
/** The gateway's name, its pods' component, and its Service's; the headless Service its replicas find each other by. */
|
|
56
|
+
export const GATEWAY = componentName("sandbox-gateway");
|
|
57
|
+
export const GATEWAY_COMPONENT = "sandbox-gateway";
|
|
58
|
+
const GATEWAY_PEERS = componentName("sandbox-gateway-peers");
|
|
59
|
+
/** The port the gateway serves on. */
|
|
60
|
+
export const GATEWAY_PORT = 8080;
|
|
61
|
+
/** The receiver's name and its pods' component. */
|
|
62
|
+
export const RECEIVER = componentName("receiver");
|
|
63
|
+
export const RECEIVER_COMPONENT = "receiver";
|
|
64
|
+
/** The node's mount of workspace storage, its DaemonSet's name and its pods' component. */
|
|
65
|
+
export const WORKSPACE_MOUNT = componentName("workspace-mount");
|
|
66
|
+
export const WORKSPACE_MOUNT_COMPONENT = "workspace-mount";
|
|
67
|
+
/** The port every pool pod's bayma serves MCP on. */
|
|
68
|
+
export const POOL_POD_PORT = 7290;
|
|
69
|
+
/** Each pool's name, by its sessions' internet. */
|
|
70
|
+
export const POOLS = { none: "sandboxes-none", full: "sandboxes-full" };
|
|
71
|
+
/** The component of a pool's pods: a session's. */
|
|
72
|
+
export const POOL_COMPONENT = "session";
|
|
73
|
+
/** The pools of `environment`, by its sessions' internet. */
|
|
74
|
+
export const poolsOf = (config, environment) => {
|
|
75
|
+
const namespace = sessionsNamespace(config, environment);
|
|
76
|
+
return { none: { name: POOLS.none, namespace }, full: { name: POOLS.full, namespace } };
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Each sandbox pool's template before the current one, as the gateway's configuration it
|
|
80
|
+
* replaces, `liveGateway` (its ConfigMap's `config.json`), carried it: the one it rendered, where
|
|
81
|
+
* the template changes, or else the one it carried; none where it rendered none, as where its
|
|
82
|
+
* sessions were not snapshotted. A template is carried for one change of it, and no more
|
|
83
|
+
* (./alasio.ts carriedFrom); and pinned into the sandbox pods'
|
|
84
|
+
* policy for the restores of the snapshots taken on it, read back from the cluster: the
|
|
85
|
+
* ConfigMap is that trust's anchor, which only the operator, by `alasio up`, writes. Pure, for tests.
|
|
86
|
+
*/
|
|
87
|
+
export function carriedPools(config, liveGateway, kept) {
|
|
88
|
+
const none = { none: null, full: null };
|
|
89
|
+
if (!poolsSnapshotted(kept.cpuBaseline) || liveGateway === undefined)
|
|
90
|
+
return none;
|
|
91
|
+
const held = Schema.decodeUnknownOption(Schema.fromJsonString(GatewayConfigSchema))(liveGateway);
|
|
92
|
+
if (held._tag === "None" || held.value.snapshots === null)
|
|
93
|
+
return none;
|
|
94
|
+
const pools = poolsOf(config, MAIN_ENVIRONMENT);
|
|
95
|
+
const carry = (mode) => {
|
|
96
|
+
const { current, previous } = held.value.snapshots?.templates[mode] ?? { current: {}, previous: null };
|
|
97
|
+
const now = JSON.parse(JSON.stringify(poolTemplate(config, pools[mode], mode, kept.cpuBaseline)));
|
|
98
|
+
return (isDeepStrictEqual(current, now) ? previous : current);
|
|
99
|
+
};
|
|
100
|
+
return { none: carry("none"), full: carry("full") };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The CPU baseline the installation whose gateway's configuration is `liveGateway` (as
|
|
104
|
+
* carriedPools takes it) holds sessions to: what its pools' current template sets, if
|
|
105
|
+
* anything, as only a pool of snapshotted sessions is held to one. Pure, for tests.
|
|
106
|
+
*/
|
|
107
|
+
export function recordedCpuBaseline(liveGateway) {
|
|
108
|
+
if (liveGateway === undefined)
|
|
109
|
+
return null;
|
|
110
|
+
const held = Schema.decodeUnknownOption(Schema.fromJsonString(GatewayConfigSchema))(liveGateway);
|
|
111
|
+
if (held._tag === "None")
|
|
112
|
+
return null;
|
|
113
|
+
const recorded = held.value.snapshots?.templates.none.current?.metadata?.annotations?.[CPU_FEATURES_ANNOTATION];
|
|
114
|
+
return recorded ? recorded.split(",") : null;
|
|
115
|
+
}
|
|
116
|
+
/** Whether the sandbox pools' sessions are snapshotted: where the nodes' CPU baseline is recorded, which they are held to. */
|
|
117
|
+
export const poolsSnapshotted = (cpuBaseline) => cpuBaseline !== null;
|
|
118
|
+
/**
|
|
119
|
+
* The template of a pool's pods, as the module says: a session's, under its internet `mode`.
|
|
120
|
+
* Where its sessions are snapshotted (poolsSnapshotted), as main's are by the gateway, it holds
|
|
121
|
+
* them to the nodes' CPU baseline `cpuBaseline`, its bayma holds a stop it is not given for the
|
|
122
|
+
* gateway's save (sessions.snapshots.stopWait), and it records its own hash
|
|
123
|
+
* (SESSION_TEMPLATE_ANNOTATION), which a snapshot taken on it names it by. Every environment's
|
|
124
|
+
* pools have the same.
|
|
125
|
+
*/
|
|
126
|
+
export function poolTemplate(config, pool, mode, cpuBaseline) {
|
|
127
|
+
const template = unhashedPoolTemplate(config, pool, mode, cpuBaseline);
|
|
128
|
+
if (!poolsSnapshotted(cpuBaseline))
|
|
129
|
+
return template;
|
|
130
|
+
return { ...template, metadata: { ...template.metadata, annotations: { ...template.metadata?.annotations, [SESSION_TEMPLATE_ANNOTATION]: templateHash(template) } } };
|
|
131
|
+
}
|
|
132
|
+
/** A pool's template, as poolTemplate says, but its hash. */
|
|
133
|
+
function unhashedPoolTemplate(config, pool, mode, cpuBaseline) {
|
|
134
|
+
const { sessions } = config;
|
|
135
|
+
const agent = imageReference(config.images.agent);
|
|
136
|
+
const snapshotted = poolsSnapshotted(cpuBaseline);
|
|
137
|
+
return {
|
|
138
|
+
metadata: {
|
|
139
|
+
labels: { "app.kubernetes.io/part-of": "alasio", "app.kubernetes.io/instance": RELEASE, "app.kubernetes.io/component": POOL_COMPONENT, [NET_MODE_LABEL]: mode, ...freeLabels(pool) },
|
|
140
|
+
// Under gVisor, /tmp is a tmpfs of the sandbox's own, by gVisor's mount hints, rather
|
|
141
|
+
// than the emptyDir's directory on the node: a snapshot of the sandbox holds it.
|
|
142
|
+
annotations: {
|
|
143
|
+
"dev.gvisor.spec.mount.tmp.share": "container",
|
|
144
|
+
"dev.gvisor.spec.mount.tmp.type": "tmpfs",
|
|
145
|
+
"dev.gvisor.spec.mount.tmp.options": "rw",
|
|
146
|
+
...(snapshotted ? { [CPU_FEATURES_ANNOTATION]: cpuBaseline.join(",") } : {}),
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
spec: {
|
|
150
|
+
...(sessions.runtimeClassName ? { runtimeClassName: sessions.runtimeClassName } : {}),
|
|
151
|
+
...priority("turn"),
|
|
152
|
+
// Ended by other than the gateway, bayma holds its stop for the gateway's save, then stops.
|
|
153
|
+
...(snapshotted ? { terminationGracePeriodSeconds: stopWaitSeconds(config) + BAYMA_STOP_SECONDS } : {}),
|
|
154
|
+
securityContext: restrictedPod(1000, 1000),
|
|
155
|
+
automountServiceAccountToken: false,
|
|
156
|
+
enableServiceLinks: false,
|
|
157
|
+
// Names resolved as its internet says, as a session's are: by none but itself without one.
|
|
158
|
+
dnsPolicy: "None",
|
|
159
|
+
dnsConfig: { nameservers: mode === "full" ? [...sessions.fullModeNameservers] : ["127.0.0.1"] },
|
|
160
|
+
...imagePullSecrets(config),
|
|
161
|
+
...given("nodeSelector", { ...sessions.nodeSelector }),
|
|
162
|
+
...given("tolerations", [...sessions.tolerations]),
|
|
163
|
+
containers: [{
|
|
164
|
+
name: BAYMA_CONTAINER,
|
|
165
|
+
image: agent,
|
|
166
|
+
imagePullPolicy: config.images.pullPolicy,
|
|
167
|
+
// bayma, and its hook as a release's snapshot runs it, once its workspace is attached.
|
|
168
|
+
command: ["/usr/bin/tini", "-s", "--", "/usr/local/bin/alasio-start", "node", "/opt/bayma/bayma.js"],
|
|
169
|
+
args: [
|
|
170
|
+
"mcp-http",
|
|
171
|
+
"--host",
|
|
172
|
+
"0.0.0.0",
|
|
173
|
+
"--port",
|
|
174
|
+
String(POOL_POD_PORT),
|
|
175
|
+
"--state-dir",
|
|
176
|
+
"/home/agent/.bayma",
|
|
177
|
+
"--token-file",
|
|
178
|
+
"/attach/token",
|
|
179
|
+
"--telemetry-token-file",
|
|
180
|
+
"/attach/token",
|
|
181
|
+
...(snapshotted ? [STOP_WAIT_ARG, String(stopWaitSeconds(config) * 1000)] : []),
|
|
182
|
+
],
|
|
183
|
+
workingDir: "/attach",
|
|
184
|
+
env: [
|
|
185
|
+
{ name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" } } },
|
|
186
|
+
{ name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
|
|
187
|
+
{ name: "HOME", value: "/home/agent" },
|
|
188
|
+
{ name: "USER", value: "agent" },
|
|
189
|
+
{ name: "BAYMA_PAYLOAD_DIR", value: "/opt/bayma/payload" },
|
|
190
|
+
],
|
|
191
|
+
ports: [{ name: "mcp", containerPort: POOL_POD_PORT }],
|
|
192
|
+
readinessProbe: { tcpSocket: { port: "mcp" }, periodSeconds: 2 },
|
|
193
|
+
securityContext: restrictedContainer(),
|
|
194
|
+
resources: sessions.resources,
|
|
195
|
+
volumeMounts: [
|
|
196
|
+
// Its own directory on its node, by its namespace and its name, as pods of two namespaces
|
|
197
|
+
// may share a name, which its workspace is attached under once it is bound.
|
|
198
|
+
{ name: "attach", mountPath: "/attach", subPathExpr: "$(POD_NAMESPACE)/$(POD_NAME)", mountPropagation: "HostToContainer" },
|
|
199
|
+
{ name: "tmp", mountPath: "/tmp" },
|
|
200
|
+
],
|
|
201
|
+
}],
|
|
202
|
+
volumes: [{ name: "attach", hostPath: { path: ATTACH_ROOT, type: "DirectoryOrCreate" } }, { name: "tmp", emptyDir: { sizeLimit: "2Gi" } }],
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
/** A pool's Deployment, and the ScaledObject that keeps its free pods. */
|
|
207
|
+
function poolObjects(config, pool, mode, cpuBaseline) {
|
|
208
|
+
const template = poolTemplate(config, pool, mode, cpuBaseline);
|
|
209
|
+
const deployment = {
|
|
210
|
+
apiVersion: "apps/v1",
|
|
211
|
+
kind: "Deployment",
|
|
212
|
+
// Ready, as `alasio up` and `alasio status` say it, once its pods run: each is ready only once bound.
|
|
213
|
+
metadata: { name: pool.name, namespace: pool.namespace, labels: labels(POOL_COMPONENT), annotations: { [READY_ONCE_BOUND_ANNOTATION]: "true" } },
|
|
214
|
+
spec: {
|
|
215
|
+
...replicasOf(config.sessions.pools),
|
|
216
|
+
selector: { matchLabels: { ...freeLabels(pool) } },
|
|
217
|
+
// None is ready before it is bound: the old free pods go as the new come.
|
|
218
|
+
strategy: { type: "Recreate" },
|
|
219
|
+
template,
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
// As many free as binds wait for, past `min`: each is claimed by one.
|
|
223
|
+
const waiting = {
|
|
224
|
+
type: "metrics-api",
|
|
225
|
+
metadata: { url: `http://${GATEWAY}.${NAMESPACE}.svc.cluster.local:${GATEWAY_PORT}/counts/${pool.namespace}/${pool.name}`, valueLocation: "waiting", targetValue: "1" },
|
|
226
|
+
};
|
|
227
|
+
// Sized as recommended alone: a sandbox's resources change only as a new pod is made.
|
|
228
|
+
return [deployment, ...scaledObject(deployment, config.sessions.pools, [waiting]), verticalPodAutoscaler(deployment, "Recommended")];
|
|
229
|
+
}
|
|
230
|
+
/** The gateway's configuration, as src/gateway/gateway.ts reads it. */
|
|
231
|
+
function gatewayConfig(config, carried) {
|
|
232
|
+
const pools = poolsOf(config, MAIN_ENVIRONMENT);
|
|
233
|
+
const templates = (mode) => ({ current: { ...poolTemplate(config, pools[mode], mode, carried.cpuBaseline) }, previous: carried.previousPools[mode] ? { ...carried.previousPools[mode] } : null });
|
|
234
|
+
return {
|
|
235
|
+
port: GATEWAY_PORT,
|
|
236
|
+
peers: `${GATEWAY_PEERS}.${NAMESPACE}.svc.cluster.local`,
|
|
237
|
+
podPort: POOL_POD_PORT,
|
|
238
|
+
waitSeconds: 120,
|
|
239
|
+
// A duration, as the configuration checks it.
|
|
240
|
+
releaseAfterSeconds: durationSeconds(config.sessions.snapshots.releaseAfter) ?? 0,
|
|
241
|
+
environment: { name: MAIN_ENVIRONMENT_NAME, namespace: sessionsNamespace(config, MAIN_ENVIRONMENT) },
|
|
242
|
+
// A new session's volume's size, as the configuration checks it.
|
|
243
|
+
quotaBytes: memoryBytes(config.sessions.storage.size),
|
|
244
|
+
// Under the releases the cluster keeps, each restored with a RuntimeClass of its own.
|
|
245
|
+
snapshots: poolsSnapshotted(carried.cpuBaseline)
|
|
246
|
+
? {
|
|
247
|
+
liveEverySeconds: durationSeconds(config.sessions.snapshots.liveEvery) ?? 0,
|
|
248
|
+
releases: carried.gvisorReleases.map((release) => ({ release, runtimeClassName: gvisorRuntimeClass(release), gofer: goferOf(release, NODE_PINS.gvisor.restoresThroughSocketSince) })),
|
|
249
|
+
templates: { none: templates("none"), full: templates("full") },
|
|
250
|
+
}
|
|
251
|
+
: null,
|
|
252
|
+
pools: { none: POOLS.none, full: POOLS.full },
|
|
253
|
+
nodeAgents: {
|
|
254
|
+
namespace: NAMESPACE,
|
|
255
|
+
selector: Object.entries(selectorLabels("node-agent")).map(([label, value]) => `${label}=${value}`).join(","),
|
|
256
|
+
port: NODE_AGENT_PORT,
|
|
257
|
+
},
|
|
258
|
+
receiver: { host: `${RECEIVER}.${NAMESPACE}.svc.cluster.local`, port: config.telemetry.receiverPort },
|
|
259
|
+
// Folder conversations' instances, of the host profile, where the installation has folder workspaces.
|
|
260
|
+
folders: foldersRun(config)
|
|
261
|
+
? { namespace: config.host.namespace, template: { ...folderTemplate(config) }, stateRoot: config.host.stateRoot, releaseAfterSeconds: durationSeconds(config.host.releaseAfter) ?? 0 }
|
|
262
|
+
: null,
|
|
263
|
+
// Conversations' harnesses' pools, where they run in pools of their own.
|
|
264
|
+
harnesses: harnessesConfig(config),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* What lets the ServiceAccount `account` find the branch environments, by their namespaces
|
|
269
|
+
* (src/gateway/environments.ts): a ClusterRole that reads namespaces, and nothing else of the
|
|
270
|
+
* cluster, and its binding.
|
|
271
|
+
*/
|
|
272
|
+
function environmentsFinder(account, component) {
|
|
273
|
+
const metadata = { name: account, labels: labels(component) };
|
|
274
|
+
return [
|
|
275
|
+
{ apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", metadata, rules: [{ apiGroups: [""], resources: ["namespaces"], verbs: ["get", "list", "watch"] }] },
|
|
276
|
+
{
|
|
277
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
278
|
+
kind: "ClusterRoleBinding",
|
|
279
|
+
metadata,
|
|
280
|
+
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: account },
|
|
281
|
+
subjects: [{ kind: "ServiceAccount", name: account, namespace: NAMESPACE }],
|
|
282
|
+
},
|
|
283
|
+
];
|
|
284
|
+
}
|
|
285
|
+
/** The ServiceAccount `account` of alasio's namespace. */
|
|
286
|
+
const serviceAccount = (account, component) => ({ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: account, namespace: NAMESPACE, labels: labels(component) } });
|
|
287
|
+
/** The user the gateway's ServiceAccount is to the API server, which the sandbox pods' policy names as the one maker of restores' pods. */
|
|
288
|
+
export const GATEWAY_USER = `system:serviceaccount:${NAMESPACE}:${GATEWAY}`;
|
|
289
|
+
/**
|
|
290
|
+
* The gateway's identity: its ServiceAccount, the node agents it finds, the branch
|
|
291
|
+
* environments it finds, and the nodes it reads whether a restore's pod may be scheduled onto;
|
|
292
|
+
* what it may do in each environment's pools is sandboxPoolObjects'.
|
|
293
|
+
*/
|
|
294
|
+
function gatewayIdentity() {
|
|
295
|
+
const nodes = { name: `${GATEWAY}-nodes`, labels: labels(GATEWAY_COMPONENT) };
|
|
296
|
+
const nodesRole = { apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", metadata: nodes, rules: [{ apiGroups: [""], resources: ["nodes"], verbs: ["get"] }] };
|
|
297
|
+
const nodesBinding = {
|
|
298
|
+
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
299
|
+
kind: "ClusterRoleBinding",
|
|
300
|
+
metadata: nodes,
|
|
301
|
+
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: nodes.name },
|
|
302
|
+
subjects: [{ kind: "ServiceAccount", name: GATEWAY, namespace: NAMESPACE }],
|
|
303
|
+
};
|
|
304
|
+
return [
|
|
305
|
+
serviceAccount(GATEWAY, GATEWAY_COMPONENT),
|
|
306
|
+
nodesRole,
|
|
307
|
+
nodesBinding,
|
|
308
|
+
// The node agents, by their pods, which it attaches workspaces through.
|
|
309
|
+
...accountRole(GATEWAY, GATEWAY_COMPONENT, NAMESPACE, [{ apiGroups: [""], resources: ["pods"], verbs: ["list"] }]),
|
|
310
|
+
...environmentsFinder(GATEWAY, GATEWAY_COMPONENT),
|
|
311
|
+
];
|
|
312
|
+
}
|
|
313
|
+
/** The gateway's Deployment, its configuration's ConfigMap, and its Services. */
|
|
314
|
+
function gatewayObjects(config, carried) {
|
|
315
|
+
const rendered = JSON.stringify(gatewayConfig(config, carried), null, 2);
|
|
316
|
+
const configMap = { apiVersion: "v1", kind: "ConfigMap", metadata: { name: GATEWAY, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) }, data: { "config.json": rendered } };
|
|
317
|
+
const key = (variable, field) => ({ name: variable, valueFrom: { secretKeyRef: { name: GATEWAY, key: field } } });
|
|
318
|
+
const deployment = {
|
|
319
|
+
apiVersion: "apps/v1",
|
|
320
|
+
kind: "Deployment",
|
|
321
|
+
metadata: { name: GATEWAY, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) },
|
|
322
|
+
spec: {
|
|
323
|
+
replicas: 1,
|
|
324
|
+
selector: { matchLabels: selectorLabels(GATEWAY_COMPONENT) },
|
|
325
|
+
template: {
|
|
326
|
+
metadata: { labels: labels(GATEWAY_COMPONENT), annotations: { "checksum/config": sha256(rendered) } },
|
|
327
|
+
spec: {
|
|
328
|
+
serviceAccountName: GATEWAY,
|
|
329
|
+
...priority("floor"),
|
|
330
|
+
// Its front's calls under way let end as it stops (src/binder/server.ts).
|
|
331
|
+
terminationGracePeriodSeconds: STOP_GRACE_SECONDS,
|
|
332
|
+
securityContext: restrictedPod(1000, 1000),
|
|
333
|
+
...imagePullSecrets(config),
|
|
334
|
+
containers: [{
|
|
335
|
+
name: GATEWAY_COMPONENT,
|
|
336
|
+
image: imageReference(config.images.alasio),
|
|
337
|
+
imagePullPolicy: config.images.pullPolicy,
|
|
338
|
+
args: ["node", "/opt/alasio/src/gateway/main.ts"],
|
|
339
|
+
env: [
|
|
340
|
+
{ name: GATEWAY_CONFIG_VARIABLE, value: "/etc/gateway/config.json" },
|
|
341
|
+
key(CALLER_KEY_VARIABLE, "caller-key"),
|
|
342
|
+
key(BINDING_KEY_VARIABLE, "binding-key"),
|
|
343
|
+
{ name: NODE_AGENT_TOKEN_VARIABLE, valueFrom: { secretKeyRef: { name: componentName("node-agent"), key: "token" } } },
|
|
344
|
+
{ name: "POD_IP", valueFrom: { fieldRef: { fieldPath: "status.podIP" } } },
|
|
345
|
+
// Every environment's events topic, which it writes as its principal on the log (src/events/).
|
|
346
|
+
{ name: EVENTS_BROKERS_VARIABLE, value: LOG_BROKERS },
|
|
347
|
+
{ name: EVENTS_USERNAME_VARIABLE, valueFrom: { secretKeyRef: { name: logPrincipalSecret("sandbox-gateway"), key: "username" } } },
|
|
348
|
+
{ name: EVENTS_PASSWORD_FILE_VARIABLE, value: "/run/alasio/log/password" },
|
|
349
|
+
...(otelEnv(config).length > 0 ? [{ name: "OTEL_SERVICE_NAME", value: GATEWAY }, ...otelEnv(config)] : []),
|
|
350
|
+
],
|
|
351
|
+
ports: [{ name: "gateway", containerPort: GATEWAY_PORT }],
|
|
352
|
+
readinessProbe: { httpGet: { path: "/healthz", port: "gateway" }, ...GATEWAY_READINESS },
|
|
353
|
+
securityContext: restrictedContainer(),
|
|
354
|
+
resources: config.sessions.gateway.resources,
|
|
355
|
+
volumeMounts: [{ name: "config", mountPath: "/etc/gateway", readOnly: true }, { name: "log", mountPath: "/run/alasio/log", readOnly: true }],
|
|
356
|
+
}],
|
|
357
|
+
volumes: [{ name: "config", configMap: { name: GATEWAY } }, { name: "log", secret: { secretName: logPrincipalSecret("sandbox-gateway"), items: [{ key: "password", path: "password" }] } }],
|
|
358
|
+
},
|
|
359
|
+
},
|
|
360
|
+
},
|
|
361
|
+
};
|
|
362
|
+
const service = (name, headless) => ({
|
|
363
|
+
apiVersion: "v1",
|
|
364
|
+
kind: "Service",
|
|
365
|
+
metadata: { name, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) },
|
|
366
|
+
spec: { ...(headless ? { clusterIP: "None" } : {}), selector: selectorLabels(GATEWAY_COMPONENT), ports: [{ name: "gateway", port: GATEWAY_PORT, targetPort: "gateway" }] },
|
|
367
|
+
});
|
|
368
|
+
return [configMap, deployment, verticalPodAutoscaler(deployment, "InPlace"), service(GATEWAY, false), service(GATEWAY_PEERS, true)];
|
|
369
|
+
}
|
|
370
|
+
/** The receiver: its identity, which finds the branch environments, whose pools' Leases sandboxPoolObjects lets it watch, its Deployment and its Service. */
|
|
371
|
+
function receiverObjects(config) {
|
|
372
|
+
const port = config.telemetry.receiverPort;
|
|
373
|
+
const deployment = {
|
|
374
|
+
apiVersion: "apps/v1",
|
|
375
|
+
kind: "Deployment",
|
|
376
|
+
metadata: { name: RECEIVER, namespace: NAMESPACE, labels: labels(RECEIVER_COMPONENT) },
|
|
377
|
+
spec: {
|
|
378
|
+
replicas: 1,
|
|
379
|
+
selector: { matchLabels: selectorLabels(RECEIVER_COMPONENT) },
|
|
380
|
+
template: {
|
|
381
|
+
metadata: { labels: labels(RECEIVER_COMPONENT) },
|
|
382
|
+
spec: {
|
|
383
|
+
serviceAccountName: RECEIVER,
|
|
384
|
+
...priority("floor"),
|
|
385
|
+
securityContext: restrictedPod(1000, 1000),
|
|
386
|
+
...imagePullSecrets(config),
|
|
387
|
+
containers: [{
|
|
388
|
+
name: RECEIVER_COMPONENT,
|
|
389
|
+
image: imageReference(config.images.alasio),
|
|
390
|
+
imagePullPolicy: config.images.pullPolicy,
|
|
391
|
+
args: ["node", "/opt/alasio/src/receiver/main.ts"],
|
|
392
|
+
env: [
|
|
393
|
+
{ name: RECEIVER_PORT_VARIABLE, value: String(port) },
|
|
394
|
+
{ name: BINDING_KEY_VARIABLE, valueFrom: { secretKeyRef: { name: GATEWAY, key: "binding-key" } } },
|
|
395
|
+
{ name: SESSIONS_NAMESPACE_VARIABLE, value: sessionsNamespace(config, MAIN_ENVIRONMENT) },
|
|
396
|
+
...(foldersRun(config) ? [{ name: FOLDERS_NAMESPACE_VARIABLE, value: config.host.namespace }] : []),
|
|
397
|
+
...(otelEnv(config).length > 0 ? [{ name: "OTEL_SERVICE_NAME", value: RECEIVER }, ...otelEnv(config)] : []),
|
|
398
|
+
],
|
|
399
|
+
ports: [{ name: "otlp-http", containerPort: port }],
|
|
400
|
+
securityContext: restrictedContainer(),
|
|
401
|
+
resources: config.sessions.receiver.resources,
|
|
402
|
+
}],
|
|
403
|
+
},
|
|
404
|
+
},
|
|
405
|
+
},
|
|
406
|
+
};
|
|
407
|
+
const service = {
|
|
408
|
+
apiVersion: "v1",
|
|
409
|
+
kind: "Service",
|
|
410
|
+
metadata: { name: RECEIVER, namespace: NAMESPACE, labels: labels(RECEIVER_COMPONENT) },
|
|
411
|
+
spec: { selector: selectorLabels(RECEIVER_COMPONENT), ports: [{ name: "otlp-http", port, targetPort: "otlp-http" }] },
|
|
412
|
+
};
|
|
413
|
+
return [serviceAccount(RECEIVER, RECEIVER_COMPONENT), ...environmentsFinder(RECEIVER, RECEIVER_COMPONENT), deployment, verticalPodAutoscaler(deployment, "InPlace"), service];
|
|
414
|
+
}
|
|
415
|
+
/** Where a workspace mount's pod keeps what is its own: its juicefs's log and pid, and whether it has handed its mount over. */
|
|
416
|
+
export const MOUNT_RUN = "/run/alasio";
|
|
417
|
+
/** Where JuiceFS keeps a mount's open handles on the node, which the mount that takes it over reads. */
|
|
418
|
+
export const JUICEFS_STATE = "/var/lib/alasio/juicefs-state";
|
|
419
|
+
/** The port each node's mount serves JuiceFS's client's metrics on, in its pod's own network, which the collector's scraper reads. */
|
|
420
|
+
export const JUICEFS_METRICS_PORT = 9567;
|
|
421
|
+
/** Where the node runs JuiceFS's command from: its static release, copied from alasio's image, under its digest. */
|
|
422
|
+
export const JUICEFS_BIN = "/var/lib/alasio/bin";
|
|
423
|
+
/** Where the node's mount of workspace storage caches its blocks. */
|
|
424
|
+
export const JUICEFS_CACHE = "/var/lib/alasio/juicefs-cache";
|
|
425
|
+
/** What JuiceFS's mount says once it has handed its FUSE session over to another mount of the same mount point, and ends. */
|
|
426
|
+
export const HANDED_OVER = "transfer FUSE session to others";
|
|
427
|
+
/**
|
|
428
|
+
* What JuiceFS's mount says once its own process serves the mount point: mounted afresh, or
|
|
429
|
+
* having taken another's FUSE session over, as it waits until the mount point names its process
|
|
430
|
+
* and no longer the one before.
|
|
431
|
+
*/
|
|
432
|
+
export const SERVING = "is ready at";
|
|
433
|
+
/**
|
|
434
|
+
* The Services the nodes' mount of workspace storage reaches by the addresses it resolves them to
|
|
435
|
+
* as it starts (workspaceMountScript): the metadata engine's, Valkey's, and the bundled object
|
|
436
|
+
* store's, where the bucket is its.
|
|
437
|
+
*/
|
|
438
|
+
export const mountServices = (config) => [VALKEY, ...(config.objectStore.bundled.enabled ? [componentName("seaweedfs")] : [])];
|
|
439
|
+
/** The annotation of the mount's template that changes, rolling it, where what it reaches by address does. */
|
|
440
|
+
export const MOUNT_SERVICES_ANNOTATION = "checksum/storage-services";
|
|
441
|
+
/**
|
|
442
|
+
* What the mount reaches by address, as a checksum: the UIDs of its Services, `uids`, and the
|
|
443
|
+
* URLs it resolves, so a Service made anew, whose address is another, or another bucket, rolls it.
|
|
444
|
+
*/
|
|
445
|
+
function mountServicesChecksum(config, uids) {
|
|
446
|
+
const reached = { metadata: VALKEY_ADDRESS, bucket: workspacesBucketUrl(config), services: Object.fromEntries(mountServices(config).map((name) => [name, uids[name] ?? null])) };
|
|
447
|
+
return createHash("sha256").update(JSON.stringify(reached)).digest("hex").slice(0, 16);
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* How long a mount's juicefs has to serve the node's mount once it starts: past what a takeover
|
|
451
|
+
* takes, juicefs's own waits in it bounded at some 70 s (the FUSE session's fd, then the old
|
|
452
|
+
* mount's end), and a fresh mount, a matter of seconds.
|
|
453
|
+
*/
|
|
454
|
+
export const MOUNT_SERVE_WAIT_SECONDS = 180;
|
|
455
|
+
export const MOUNT_PATHS = {
|
|
456
|
+
run: MOUNT_RUN,
|
|
457
|
+
bin: JUICEFS_BIN,
|
|
458
|
+
state: JUICEFS_STATE,
|
|
459
|
+
cache: JUICEFS_CACHE,
|
|
460
|
+
juicefs: "/usr/local/bin/juicefs",
|
|
461
|
+
mountinfo: "/proc/1/mountinfo",
|
|
462
|
+
inNode: "nsenter -t 1 -m --",
|
|
463
|
+
resolve: "getent hosts",
|
|
464
|
+
fuseConnections: "/sys/fs/fuse/connections",
|
|
465
|
+
};
|
|
466
|
+
/**
|
|
467
|
+
* The script of a workspace mount's pod: JuiceFS's mount of workspace storage at `root`, made
|
|
468
|
+
* in the node's own mount namespace, so it is the node's alone, and no container's mounts, torn
|
|
469
|
+
* down as their container ends, take it with them; its log copied out; and, once it ends, idle
|
|
470
|
+
* where it ended by handing its mount over, as it said, to the mount of the pod that replaces it,
|
|
471
|
+
* until its own pod is deleted, or, where it ended any other way, a failure, so kubelet starts it
|
|
472
|
+
* again and it mounts anew. Asked to end while it mounts, it has its mount end too.
|
|
473
|
+
*
|
|
474
|
+
* Its juicefs is JuiceFS's static release, of alasio's image, which runs as it is in the node's
|
|
475
|
+
* namespace: copied to the node, under its digest, never over the one a mount runs. A mount left
|
|
476
|
+
* at `root` whose juicefs is gone, as a pod killed with no successor leaves one, which the kernel
|
|
477
|
+
* answers as not connected, or whose juicefs answers nothing within 5 s, as one stopped or hung
|
|
478
|
+
* does, not even of what it holds in memory, is unmounted first, lazily, so the next mounts
|
|
479
|
+
* afresh.
|
|
480
|
+
*
|
|
481
|
+
* Its juicefs, taking a mount over or mounting afresh, serves within MOUNT_SERVE_WAIT_SECONDS
|
|
482
|
+
* (`serveWaitSeconds`), or is ended, and the mount fails, so kubelet starts it again, to mount
|
|
483
|
+
* afresh: a takeover whose old mount stopped answering waits, in juicefs's own reads of it, for
|
|
484
|
+
* as long as it does, and no juicefs serves the node's mount meanwhile. Ended so, or unmounted as
|
|
485
|
+
* one that answers nothing, the node's mount has its FUSE connection aborted first: a request on
|
|
486
|
+
* a session no process will answer, as a juicefs makes of the one it takes over and holds, waits
|
|
487
|
+
* where no signal ends it, KILL among them, until its connection is aborted, which fails it.
|
|
488
|
+
*
|
|
489
|
+
* In the node's mount namespace its juicefs reads the node's resolv.conf, which resolves none of
|
|
490
|
+
* the cluster's names, its network the pod's still: so the metadata engine's and the bucket's,
|
|
491
|
+
* where they are a Service's, are resolved here, by the pod's DNS, and it is given their
|
|
492
|
+
* addresses. A Service's address lasts as long as it does: `up` rolls the mount where one is made
|
|
493
|
+
* anew, or the bucket is another (MOUNT_SERVICES_ANNOTATION), and the pod that replaces it reaches
|
|
494
|
+
* the new address.
|
|
495
|
+
*
|
|
496
|
+
* Its juicefs, run in the node's mount namespace, is still this pod's process, of its cgroup: the
|
|
497
|
+
* pod's teardown kills it. So a pod hands its mount over before it is torn down, never after: the
|
|
498
|
+
* pod that replaces it starts beside it (maxSurge), takes its FUSE session over, and only then is
|
|
499
|
+
* it deleted, idle, its juicefs ended already, which its readiness keeps it from until then
|
|
500
|
+
* (workspaceMountReady). `paths` are MOUNT_PATHS but for tests.
|
|
501
|
+
*/
|
|
502
|
+
export function workspaceMountScript(root, cacheSizeMiB, paths = MOUNT_PATHS, serveWaitSeconds = MOUNT_SERVE_WAIT_SECONDS) {
|
|
503
|
+
const { run, bin, state, cache, juicefs, mountinfo, inNode, resolve, fuseConnections } = paths;
|
|
504
|
+
return [
|
|
505
|
+
// The FUSE connection of the node's mount at the root aborted, where its mount there, the
|
|
506
|
+
// topmost, is a FUSE file system's: every request on it fails, those no process will ever
|
|
507
|
+
// answer among them, as one a juicefs makes of the very session it takes over and holds,
|
|
508
|
+
// which leaves it waiting where no signal, not even KILL, ends it. Its connection is named by
|
|
509
|
+
// the minor of the mount's device, under fusectl's mount, made where the node has none.
|
|
510
|
+
"aborted() {",
|
|
511
|
+
` connection=$(awk -v root=${root} '$5 == root { connection = ""; for (i = 6; i < NF; i += 1) if ($i == "-") { if ($(i + 1) ~ /^fuse(\\.|$)/) { split($3, device, ":"); connection = device[2] }; break } } END { print connection }' ${mountinfo})`,
|
|
512
|
+
' [ -n "$connection" ] || return 0',
|
|
513
|
+
` grep -qF ' ${fuseConnections} ' ${mountinfo} || ${inNode} mount -t fusectl none ${fuseConnections}`,
|
|
514
|
+
` ${inNode} sh -c 'echo 1 > "$1"' sh "${fuseConnections}/$connection/abort" && echo "aborted the FUSE connection $connection of the node's mount, so what waits on it fails"`,
|
|
515
|
+
"}",
|
|
516
|
+
// Whether the node's mount at the root answers its .config within 5 s, which juicefs answers
|
|
517
|
+
// of its own memory, what the read said kept in $run/probed: where not, no process serves it,
|
|
518
|
+
// or one stopped or hung does. Not waited on past that: a read juicefs took but never
|
|
519
|
+
// answers waits where no signal ends it, until the connection is aborted, which frees it.
|
|
520
|
+
"answers() {",
|
|
521
|
+
` rm -f "$run/answered" "$run/probed"`,
|
|
522
|
+
` ( ${inNode} cat ${root}/.config > /dev/null 2> "$run/probed"; : > "$run/answered" ) &`,
|
|
523
|
+
` for tick in 1 2 3 4 5; do sleep 1; [ -e "$run/answered" ] && return 0; done`,
|
|
524
|
+
" return 1",
|
|
525
|
+
"}",
|
|
526
|
+
// `url`, its host a Service's name given as its address, by the pod's DNS.
|
|
527
|
+
"resolved() {",
|
|
528
|
+
' scheme=${1%%://*}; rest=${1#*://}; auth=""',
|
|
529
|
+
' case $rest in *@*) auth="${rest%@*}@"; rest=${rest##*@};; esac',
|
|
530
|
+
' host=${rest%%[:/]*}; tail=${rest#"$host"}; address=$host',
|
|
531
|
+
" case $host in *.svc|*.svc.cluster.local)",
|
|
532
|
+
` address=$(${resolve} "$host" | awk '{ print $1; exit }')`,
|
|
533
|
+
' [ -n "$address" ] || { echo "$host resolves to no address" >&2; return 1; };;',
|
|
534
|
+
" esac",
|
|
535
|
+
` printf '%s://%s%s%s\n' "$scheme" "$auth" "$address" "$tail"`,
|
|
536
|
+
"}",
|
|
537
|
+
// Told to stop (TERM, INT) at any time, the script's start among it: noted, and passed on to
|
|
538
|
+
// its juicefs, as it runs; before it runs, it is not started.
|
|
539
|
+
"pid=",
|
|
540
|
+
"termed=",
|
|
541
|
+
`trap 'termed=1; [ -z "$pid" ] || kill -TERM "$pid"' TERM INT`,
|
|
542
|
+
// The metadata engine's password given in META_PASSWORD, which JuiceFS reads, not in its
|
|
543
|
+
// address on the command line, which every process of the node's PIDs could read.
|
|
544
|
+
`META_PASSWORD=$(node -e 'process.stdout.write(decodeURIComponent(new URL(process.env.META_URL).password))') || exit 1`,
|
|
545
|
+
"export META_PASSWORD",
|
|
546
|
+
`meta=$(resolved "$(node -e 'const url = new URL(process.env.META_URL); url.password = ""; process.stdout.write(url.href)')") || exit 1`,
|
|
547
|
+
'bucket=$(resolved "$BUCKET") || exit 1',
|
|
548
|
+
`run=${run}`,
|
|
549
|
+
`mkdir -p "$run" ${bin} ${state} ${cache} || exit 1`,
|
|
550
|
+
`rm -f "$run/handed-over" "$run/expired"`,
|
|
551
|
+
// Made, empty, before the mount starts or the log is followed, which would not find it yet.
|
|
552
|
+
`: > "$run/juicefs.log"`,
|
|
553
|
+
// Named by its digest, read whole: a read that fails, as one of a layer lazily pulled may, stops
|
|
554
|
+
// the mount, which kubelet starts again, rather than run a juicefs of no digest, which is none.
|
|
555
|
+
`digest=$(sha256sum ${juicefs}) || { echo "could not read ${juicefs}, so the mount is not started" >&2; exit 1; }`,
|
|
556
|
+
`node_juicefs="${bin}/juicefs-$(printf '%.16s' "$digest")"`,
|
|
557
|
+
// Copied beside it under a name of its own, made by mktemp, and moved into place, so no
|
|
558
|
+
// pod's half-written copy is run: no $$, which Kubernetes makes $ in a container's command.
|
|
559
|
+
`if [ ! -x "$node_juicefs" ]; then`,
|
|
560
|
+
` copying=$(mktemp "${bin}/.juicefs.XXXXXX") && { cp ${juicefs} "$copying" && chmod 755 "$copying" && mv "$copying" "$node_juicefs"; } || { rm -f "$copying"; echo "could not copy ${juicefs} to the node, so the mount is not started" >&2; exit 1; }`,
|
|
561
|
+
"fi",
|
|
562
|
+
// A mount of a juicefs gone, which its kernel answers as no longer connected, or of one that
|
|
563
|
+
// answers nothing within 5 s: unmounted, so this one mounts afresh. Asked of its .config,
|
|
564
|
+
// which juicefs answers of its own memory, never its metadata engine's (JuiceFS's
|
|
565
|
+
// pkg/vfs/internal.go), so a juicefs that serves still, however slow its metadata engine
|
|
566
|
+
// is, or unreachable, as it is where `up` rolls the mount, is answered and taken over, its
|
|
567
|
+
// workspaces kept. Its takeover reads the same file (cmd/mount.go).
|
|
568
|
+
`if awk -v root=${root} '$5 == root { found = 1 } END { exit !found }' ${mountinfo}; then`,
|
|
569
|
+
" if ! answers; then",
|
|
570
|
+
" aborted",
|
|
571
|
+
` ${inNode} umount -l ${root} && echo "unmounted the node's mount, whose juicefs answered nothing in 5 s, not even its .config"`,
|
|
572
|
+
` elif grep -qi 'not connected' "$run/probed"; then`,
|
|
573
|
+
` ${inNode} umount -l ${root} && echo "unmounted the node's mount its juicefs left, which served nothing"`,
|
|
574
|
+
" fi",
|
|
575
|
+
"fi",
|
|
576
|
+
`${inNode} mkdir -p ${root}`,
|
|
577
|
+
`if [ -n "$termed" ]; then echo "told to stop before its juicefs started, so it starts none"; exit 0; fi`,
|
|
578
|
+
`TMPDIR=${state} ${inNode} "$node_juicefs" mount --no-bgjob --metrics 0.0.0.0:${JUICEFS_METRICS_PORT} --bucket "$bucket" --cache-dir ${cache} --cache-size ${cacheSizeMiB} "$meta" ${root} >> "$run/juicefs.log" 2>&1 &`,
|
|
579
|
+
"pid=$!",
|
|
580
|
+
// Told to stop as it started, before its PID was known: passed on now.
|
|
581
|
+
`[ -z "$termed" ] || kill -TERM "$pid"`,
|
|
582
|
+
`echo "$pid" > "$run/juicefs.pid"`,
|
|
583
|
+
`tail -n +1 -f "$run/juicefs.log" &`,
|
|
584
|
+
"tailer=$!",
|
|
585
|
+
// Ended where it serves nothing within its wait, its watcher's end awaited then (expired):
|
|
586
|
+
// what each of its goroutines waits on said, into its log, as Go says it on QUIT, of it and
|
|
587
|
+
// of each process it runs, while their main threads run still, as a KILLed process one of
|
|
588
|
+
// whose threads waits on yet says nothing. Then, where the node's mount answers nothing, as
|
|
589
|
+
// where it holds the session no process serves, its FUSE connection aborted, as what holds
|
|
590
|
+
// it up may be a request on it that nothing answers, which no signal ends; TERM, and KILL 5 s
|
|
591
|
+
// on; and the mount, dead, unmounted, so the next mounts afresh. Where it answers, it is the
|
|
592
|
+
// mount of the pod this replaces, served still, which the next takes over again.
|
|
593
|
+
"(",
|
|
594
|
+
` for second in $(seq ${serveWaitSeconds}); do grep -qF '${SERVING}' "$run/juicefs.log" && exit 0; kill -0 "$pid" 2>/dev/null || exit 0; sleep 1; done`,
|
|
595
|
+
` : > "$run/expired"`,
|
|
596
|
+
` echo "juicefs served nothing within ${serveWaitSeconds} s of starting, so it is ended, and the mount started again to mount afresh" >&2`,
|
|
597
|
+
` for proc in /proc/[0-9]*; do [ "$(sed 's/.*) //' "$proc/stat" 2>/dev/null | cut -d ' ' -f 2)" = "$pid" ] && kill -QUIT "\${proc#/proc/}" 2>/dev/null; done`,
|
|
598
|
+
` kill -QUIT "$pid" 2>/dev/null; sleep 2`,
|
|
599
|
+
" dead=",
|
|
600
|
+
" if ! answers; then aborted; dead=1; fi",
|
|
601
|
+
` kill -TERM "$pid" 2>/dev/null; sleep 5; kill -KILL "$pid" 2>/dev/null`,
|
|
602
|
+
` if [ -n "$dead" ]; then ${inNode} umount -l ${root} && echo "unmounted the node's mount, which no juicefs served"; fi`,
|
|
603
|
+
") &",
|
|
604
|
+
"watcher=$!",
|
|
605
|
+
`wait "$pid"; code=$?`,
|
|
606
|
+
// A TERM's wait ends as the trap runs; the mount's own end is waited for.
|
|
607
|
+
`while kill -0 "$pid" 2>/dev/null; do wait "$pid"; code=$?; done`,
|
|
608
|
+
"sleep 1",
|
|
609
|
+
`if [ -e "$run/expired" ]; then wait "$watcher"; fi`,
|
|
610
|
+
`kill "$tailer" "$watcher" 2>/dev/null`,
|
|
611
|
+
`if grep -qF '${HANDED_OVER}' "$run/juicefs.log"; then`,
|
|
612
|
+
` touch "$run/handed-over"`,
|
|
613
|
+
" echo 'the mount is handed over to the pod that replaces this one, which is idle until it is deleted'",
|
|
614
|
+
" trap 'exit 0' TERM INT",
|
|
615
|
+
" while :; do sleep 3600 & wait $!; done",
|
|
616
|
+
"fi",
|
|
617
|
+
`echo "juicefs mount ended ($code) without handing its mount over" >&2`,
|
|
618
|
+
`[ "$code" -ne 0 ] && exit "$code"`,
|
|
619
|
+
"exit 1",
|
|
620
|
+
].join("\n");
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* The readiness of a workspace mount's pod: its own juicefs runs, has said it serves the mount
|
|
624
|
+
* point (SERVING), and the node's mount table holds JuiceFS's mount at `root`: this pod's juicefs
|
|
625
|
+
* serves the node's mount point. The table alone holds the mount of the pod it replaces as its
|
|
626
|
+
* juicefs starts, before it takes that mount over, which the pod it replaces would be deleted
|
|
627
|
+
* for, its mount with it.
|
|
628
|
+
*
|
|
629
|
+
* Or it has handed its mount over, its part done: ready still, until its DaemonSet's update
|
|
630
|
+
* deletes it, once the pod that took its mount over is ready. Kubernetes keeps two of a
|
|
631
|
+
* DaemonSet's pods on a node, a surge's, only while the new is not ready and the old is; were
|
|
632
|
+
* neither ready, it deletes the old at once (the DaemonSet controller's podsShouldBeOnNode),
|
|
633
|
+
* whatever the update's maxUnavailable, and the node has no ready pod of its mount while the
|
|
634
|
+
* takeover runs on.
|
|
635
|
+
*/
|
|
636
|
+
export function workspaceMountReady(root, paths = MOUNT_PATHS) {
|
|
637
|
+
const { run, mountinfo } = paths;
|
|
638
|
+
const serving = [
|
|
639
|
+
`kill -0 "$(cat ${run}/juicefs.pid)" 2>/dev/null`,
|
|
640
|
+
`grep -qF '${SERVING}' ${run}/juicefs.log`,
|
|
641
|
+
`awk -v root=${root} '$5 == root { for (i = 6; i < NF; i += 1) if ($i == "-") { if ($(i + 1) == "fuse.juicefs") found = 1; break } } END { exit !found }' ${mountinfo}`,
|
|
642
|
+
].join(" && ");
|
|
643
|
+
return `[ -e ${run}/handed-over ] || { ${serving}; }`;
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* The node's mount of workspace storage, at WORKSPACES_ROOT, the node's own (workspaceMountScript),
|
|
647
|
+
* which the node agent attaches workspaces from. A mount held by a pod that goes takes every
|
|
648
|
+
* workspace attached from it with it, so a new pod of it starts beside the old (maxSurge), and
|
|
649
|
+
* its mount of the same mount point takes the old's FUSE session over, which needs them to share
|
|
650
|
+
* the node's PIDs and mount namespace; the old pod idles once it has handed its mount over, not
|
|
651
|
+
* ready, and is deleted once the new is ready. It runs alasio's image, for JuiceFS's static
|
|
652
|
+
* release, of the version the mount image is (../../../Dockerfile), privileged, as it enters
|
|
653
|
+
* the node's mount namespace.
|
|
654
|
+
*/
|
|
655
|
+
function workspaceMount(config, mountServices) {
|
|
656
|
+
const credential = (name, key) => ({ name, valueFrom: { secretKeyRef: { name: WORKSPACES_CREDENTIALS, key } } });
|
|
657
|
+
const nodePaths = [["bin", JUICEFS_BIN], ["state", JUICEFS_STATE], ["cache", JUICEFS_CACHE]];
|
|
658
|
+
return {
|
|
659
|
+
apiVersion: "apps/v1",
|
|
660
|
+
kind: "DaemonSet",
|
|
661
|
+
metadata: {
|
|
662
|
+
name: WORKSPACE_MOUNT,
|
|
663
|
+
namespace: NAMESPACE,
|
|
664
|
+
labels: labels(WORKSPACE_MOUNT_COMPONENT),
|
|
665
|
+
// Rolled only once the Valkeys it reaches are ready, its metadata engine's and the
|
|
666
|
+
// bucket's filer store's: a pod that takes its predecessor's FUSE session over while its
|
|
667
|
+
// metadata engine restarts may never serve it.
|
|
668
|
+
annotations: { [APPLIED_AFTER_ANNOTATION]: appliedAfter([{ kind: "StatefulSet", namespace: NAMESPACE, name: VALKEY }, { kind: "StatefulSet", namespace: NAMESPACE, name: FILER_STORE }]) },
|
|
669
|
+
},
|
|
670
|
+
spec: {
|
|
671
|
+
selector: { matchLabels: selectorLabels(WORKSPACE_MOUNT_COMPONENT) },
|
|
672
|
+
updateStrategy: { type: "RollingUpdate", rollingUpdate: { maxSurge: 1, maxUnavailable: 0 } },
|
|
673
|
+
template: {
|
|
674
|
+
metadata: {
|
|
675
|
+
labels: { ...labels(WORKSPACE_MOUNT_COMPONENT), ...JUICEFS_ADMIN },
|
|
676
|
+
annotations: { [MOUNT_SERVICES_ANNOTATION]: mountServicesChecksum(config, mountServices) },
|
|
677
|
+
},
|
|
678
|
+
spec: {
|
|
679
|
+
...priority("floor"),
|
|
680
|
+
...imagePullSecrets(config),
|
|
681
|
+
...given("nodeSelector", { ...config.sessions.nodeSelector }),
|
|
682
|
+
...given("tolerations", [...config.sessions.tolerations]),
|
|
683
|
+
automountServiceAccountToken: false,
|
|
684
|
+
// The mount that takes another's over signals it, by its PID on the node; and it enters the node's mount namespace by PID 1's.
|
|
685
|
+
hostPID: true,
|
|
686
|
+
containers: [{
|
|
687
|
+
name: WORKSPACE_MOUNT_COMPONENT,
|
|
688
|
+
image: imageReference(config.images.alasio),
|
|
689
|
+
imagePullPolicy: config.images.pullPolicy,
|
|
690
|
+
args: ["sh", "-c", workspaceMountScript(WORKSPACES_ROOT, config.workspaceStorage.cacheSizeMiB)],
|
|
691
|
+
env: [TINI_SUBREAPER, credential("META_URL", "metaurl"), credential("BUCKET", "bucket"), credential("ACCESS_KEY", "access-key"), credential("SECRET_KEY", "secret-key")],
|
|
692
|
+
ports: [{ name: "metrics", containerPort: JUICEFS_METRICS_PORT }],
|
|
693
|
+
readinessProbe: { exec: { command: ["sh", "-c", workspaceMountReady(WORKSPACES_ROOT)] }, periodSeconds: 2 },
|
|
694
|
+
securityContext: { privileged: true },
|
|
695
|
+
resources: config.workspaceStorage.mount.resources,
|
|
696
|
+
// The node's directories, where the pod reaches them as the node does.
|
|
697
|
+
volumeMounts: [...nodePaths.map(([name, path]) => ({ name, mountPath: path })), { name: "run", mountPath: MOUNT_RUN }],
|
|
698
|
+
}],
|
|
699
|
+
volumes: [...nodePaths.map(([name, path]) => ({ name, hostPath: { path, type: "DirectoryOrCreate" } })), { name: "run", emptyDir: {} }],
|
|
700
|
+
},
|
|
701
|
+
},
|
|
702
|
+
},
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* What the sandbox service has in `environment`'s sessions' namespace, main's or a branch
|
|
707
|
+
* environment's (./branch.ts): its pools, the budget of their bound pods, and what the gateway
|
|
708
|
+
* may do there, binding in its pools, and the receiver, watching their Leases.
|
|
709
|
+
*/
|
|
710
|
+
export function sandboxPoolObjects(config, environment, cpuBaseline) {
|
|
711
|
+
const pools = poolsOf(config, environment);
|
|
712
|
+
const namespace = pools.none.namespace;
|
|
713
|
+
return [
|
|
714
|
+
...accountRole(GATEWAY, GATEWAY_COMPONENT, namespace, [
|
|
715
|
+
// Its pools' pods, claimed by an update of the version read, and deleted as they are released; and a restore's, made, which the sandbox pods' policy holds to its pool's template.
|
|
716
|
+
{ apiGroups: [""], resources: ["pods"], verbs: ["get", "list", "watch", "create", "update", "delete"] },
|
|
717
|
+
// A restore's pod made ready once its sandbox is resumed, by its readiness gate's condition.
|
|
718
|
+
{ apiGroups: [""], resources: ["pods/status"], verbs: ["patch"] },
|
|
719
|
+
// The network policies that select a restore's pod, which its node's agent waits for in force before it resumes it.
|
|
720
|
+
{ apiGroups: ["networking.k8s.io"], resources: ["networkpolicies"], verbs: ["list"] },
|
|
721
|
+
{ apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] },
|
|
722
|
+
// Its pools' templates, which a release's snapshot records.
|
|
723
|
+
{ apiGroups: ["apps"], resources: ["deployments"], verbs: ["get"], resourceNames: [POOLS.none, POOLS.full] },
|
|
724
|
+
// What the API server tells of the pods it refused to make, as the sandbox pods' policy refused them (src/gateway/refusals.ts).
|
|
725
|
+
{ apiGroups: [""], resources: ["events"], verbs: ["list"] },
|
|
726
|
+
]),
|
|
727
|
+
...accountRole(RECEIVER, RECEIVER_COMPONENT, namespace, [{ apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["list", "watch"] }]),
|
|
728
|
+
...poolObjects(config, pools.none, "none", cpuBaseline),
|
|
729
|
+
...poolObjects(config, pools.full, "full", cpuBaseline),
|
|
730
|
+
boundBudget(namespace),
|
|
731
|
+
];
|
|
732
|
+
}
|
|
733
|
+
/** The sandbox service's objects: its gateway, its receiver, and the nodes' mount, main's alone, which serve every environment, and main's pools. */
|
|
734
|
+
export function sandboxServiceObjects(config, carried) {
|
|
735
|
+
const mount = workspaceMount(config, carried.mountServices);
|
|
736
|
+
const pools = poolsOf(config, MAIN_ENVIRONMENT);
|
|
737
|
+
return [
|
|
738
|
+
...gatewayIdentity(),
|
|
739
|
+
...gatewayObjects(config, carried),
|
|
740
|
+
// Every pod of every environment's sessions held to the pools' templates, as Pod Security cannot.
|
|
741
|
+
...sandboxPodPolicy([poolTemplate(config, pools.none, "none", carried.cpuBaseline), poolTemplate(config, pools.full, "full", carried.cpuBaseline)], {
|
|
742
|
+
sessionsNamespace: sessionsNamespace(config, MAIN_ENVIRONMENT),
|
|
743
|
+
// Main's restores, by the gateway alone, onto each pool's template and the one before it, under the releases kept.
|
|
744
|
+
restores: poolsSnapshotted(carried.cpuBaseline)
|
|
745
|
+
? {
|
|
746
|
+
restorer: GATEWAY_USER,
|
|
747
|
+
templates: ["none", "full"].flatMap((mode) => [poolTemplate(config, pools[mode], mode, carried.cpuBaseline), ...(carried.previousPools[mode] ? [carried.previousPools[mode]] : [])]),
|
|
748
|
+
runtimeClasses: carried.gvisorReleases.map(gvisorRuntimeClass),
|
|
749
|
+
}
|
|
750
|
+
: null,
|
|
751
|
+
}),
|
|
752
|
+
// Where main's sessions are snapshotted, what the gateway may set of a restore's pod's status.
|
|
753
|
+
...(poolsSnapshotted(carried.cpuBaseline) ? resumedConditionPolicy(componentName("resumed-condition"), GATEWAY_USER, pools.none.namespace) : []),
|
|
754
|
+
...receiverObjects(config),
|
|
755
|
+
...sandboxPoolObjects(config, MAIN_ENVIRONMENT, carried.cpuBaseline),
|
|
756
|
+
mount,
|
|
757
|
+
// Sized as recommended alone, as a pod of it is only replaced, by one that takes its mount over.
|
|
758
|
+
verticalPodAutoscaler(mount, "Recommended"),
|
|
759
|
+
];
|
|
760
|
+
}
|
|
761
|
+
//# sourceMappingURL=sandbox-gateway.js.map
|