alasio 8.0.0 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/dist/cli/src/api-server.js +18 -0
  2. package/dist/cli/src/api-server.js.map +1 -0
  3. package/dist/cli/src/bound-harnesses.js +98 -0
  4. package/dist/cli/src/bound-harnesses.js.map +1 -0
  5. package/dist/cli/src/branches.js +79 -22
  6. package/dist/cli/src/branches.js.map +1 -1
  7. package/dist/cli/src/cluster/docker.js +40 -15
  8. package/dist/cli/src/cluster/docker.js.map +1 -1
  9. package/dist/cli/src/cluster/host.js +38 -8
  10. package/dist/cli/src/cluster/host.js.map +1 -1
  11. package/dist/cli/src/cluster/k3s.js +105 -14
  12. package/dist/cli/src/cluster/k3s.js.map +1 -1
  13. package/dist/cli/src/cluster/node.js +142 -33
  14. package/dist/cli/src/cluster/node.js.map +1 -1
  15. package/dist/cli/src/cluster/stargz.js +92 -0
  16. package/dist/cli/src/cluster/stargz.js.map +1 -0
  17. package/dist/cli/src/commands/branch.js +5 -4
  18. package/dist/cli/src/commands/branch.js.map +1 -1
  19. package/dist/cli/src/commands/common.js +11 -5
  20. package/dist/cli/src/commands/common.js.map +1 -1
  21. package/dist/cli/src/commands/folder.js +14 -0
  22. package/dist/cli/src/commands/folder.js.map +1 -0
  23. package/dist/cli/src/commands/lake.js +1 -1
  24. package/dist/cli/src/commands/lake.js.map +1 -1
  25. package/dist/cli/src/commands/login-relay.js +91 -0
  26. package/dist/cli/src/commands/login-relay.js.map +1 -0
  27. package/dist/cli/src/commands/login.js +11 -10
  28. package/dist/cli/src/commands/login.js.map +1 -1
  29. package/dist/cli/src/commands/logs.js +89 -6
  30. package/dist/cli/src/commands/logs.js.map +1 -1
  31. package/dist/cli/src/commands/restart.js +10 -4
  32. package/dist/cli/src/commands/restart.js.map +1 -1
  33. package/dist/cli/src/commands/restore-filer-store.js +27 -10
  34. package/dist/cli/src/commands/restore-filer-store.js.map +1 -1
  35. package/dist/cli/src/commands/status.js +119 -20
  36. package/dist/cli/src/commands/status.js.map +1 -1
  37. package/dist/cli/src/commands/uninstall.js +8 -9
  38. package/dist/cli/src/commands/uninstall.js.map +1 -1
  39. package/dist/cli/src/commands/up.js +20 -9
  40. package/dist/cli/src/commands/up.js.map +1 -1
  41. package/dist/cli/src/commands/upgrade.js +22 -6
  42. package/dist/cli/src/commands/upgrade.js.map +1 -1
  43. package/dist/cli/src/commands.js +2 -1
  44. package/dist/cli/src/commands.js.map +1 -1
  45. package/dist/cli/src/config.js +4 -1
  46. package/dist/cli/src/config.js.map +1 -1
  47. package/dist/cli/src/cpu-baseline.js +8 -4
  48. package/dist/cli/src/cpu-baseline.js.map +1 -1
  49. package/dist/cli/src/folders.js +63 -0
  50. package/dist/cli/src/folders.js.map +1 -0
  51. package/dist/cli/src/gateway-calls.js +64 -0
  52. package/dist/cli/src/gateway-calls.js.map +1 -0
  53. package/dist/cli/src/install.js +45 -31
  54. package/dist/cli/src/install.js.map +1 -1
  55. package/dist/cli/src/kept-releases.js +27 -20
  56. package/dist/cli/src/kept-releases.js.map +1 -1
  57. package/dist/cli/src/kube/api.js +61 -15
  58. package/dist/cli/src/kube/api.js.map +1 -1
  59. package/dist/cli/src/kube/apply.js +122 -81
  60. package/dist/cli/src/kube/apply.js.map +1 -1
  61. package/dist/cli/src/kube/rollout.js +43 -21
  62. package/dist/cli/src/kube/rollout.js.map +1 -1
  63. package/dist/cli/src/main.js +2 -1
  64. package/dist/cli/src/main.js.map +1 -1
  65. package/dist/cli/src/manifests/admission.js +558 -222
  66. package/dist/cli/src/manifests/admission.js.map +1 -1
  67. package/dist/cli/src/manifests/alasio.js +180 -428
  68. package/dist/cli/src/manifests/alasio.js.map +1 -1
  69. package/dist/cli/src/manifests/branch.js +16 -27
  70. package/dist/cli/src/manifests/branch.js.map +1 -1
  71. package/dist/cli/src/manifests/codex-login.js +101 -0
  72. package/dist/cli/src/manifests/codex-login.js.map +1 -0
  73. package/dist/cli/src/manifests/collector.js +83 -62
  74. package/dist/cli/src/manifests/collector.js.map +1 -1
  75. package/dist/cli/src/manifests/common.js +82 -25
  76. package/dist/cli/src/manifests/common.js.map +1 -1
  77. package/dist/cli/src/manifests/config.js +137 -105
  78. package/dist/cli/src/manifests/config.js.map +1 -1
  79. package/dist/cli/src/manifests/disruption.js +18 -3
  80. package/dist/cli/src/manifests/disruption.js.map +1 -1
  81. package/dist/cli/src/manifests/folders.js +131 -0
  82. package/dist/cli/src/manifests/folders.js.map +1 -0
  83. package/dist/cli/src/manifests/grafana.js +4 -2
  84. package/dist/cli/src/manifests/grafana.js.map +1 -1
  85. package/dist/cli/src/manifests/harness-pools.js +236 -0
  86. package/dist/cli/src/manifests/harness-pools.js.map +1 -0
  87. package/dist/cli/src/manifests/image-holder.js +61 -0
  88. package/dist/cli/src/manifests/image-holder.js.map +1 -0
  89. package/dist/cli/src/manifests/index.js +14 -7
  90. package/dist/cli/src/manifests/index.js.map +1 -1
  91. package/dist/cli/src/manifests/keda.js +4 -2
  92. package/dist/cli/src/manifests/keda.js.map +1 -1
  93. package/dist/cli/src/manifests/lake.js +18 -20
  94. package/dist/cli/src/manifests/lake.js.map +1 -1
  95. package/dist/cli/src/manifests/log.js +84 -19
  96. package/dist/cli/src/manifests/log.js.map +1 -1
  97. package/dist/cli/src/manifests/neon.js +60 -80
  98. package/dist/cli/src/manifests/neon.js.map +1 -1
  99. package/dist/cli/src/manifests/network-policies.js +232 -94
  100. package/dist/cli/src/manifests/network-policies.js.map +1 -1
  101. package/dist/cli/src/manifests/node-agent.js +42 -15
  102. package/dist/cli/src/manifests/node-agent.js.map +1 -1
  103. package/dist/cli/src/manifests/object-store.js +99 -25
  104. package/dist/cli/src/manifests/object-store.js.map +1 -1
  105. package/dist/cli/src/manifests/sandbox-gateway.js +761 -0
  106. package/dist/cli/src/manifests/sandbox-gateway.js.map +1 -0
  107. package/dist/cli/src/manifests/scalers.js +17 -5
  108. package/dist/cli/src/manifests/scalers.js.map +1 -1
  109. package/dist/cli/src/manifests/sizing.js +17 -9
  110. package/dist/cli/src/manifests/sizing.js.map +1 -1
  111. package/dist/cli/src/manifests/valkey.js +30 -10
  112. package/dist/cli/src/manifests/valkey.js.map +1 -1
  113. package/dist/cli/src/manifests/volume-lock.js +43 -0
  114. package/dist/cli/src/manifests/volume-lock.js.map +1 -0
  115. package/dist/cli/src/manifests/vpa.js +15 -2
  116. package/dist/cli/src/manifests/vpa.js.map +1 -1
  117. package/dist/cli/src/manifests/workspace-storage.js +182 -94
  118. package/dist/cli/src/manifests/workspace-storage.js.map +1 -1
  119. package/dist/cli/src/neon-storage.js +54 -16
  120. package/dist/cli/src/neon-storage.js.map +1 -1
  121. package/dist/cli/src/release.js +9 -9
  122. package/dist/cli/src/release.js.map +1 -1
  123. package/dist/cli/src/setup-pod.js +44 -0
  124. package/dist/cli/src/setup-pod.js.map +1 -0
  125. package/dist/cli/src/target.js +2 -2
  126. package/dist/cli/src/target.js.map +1 -1
  127. package/dist/cli/src/telegram.js +13 -4
  128. package/dist/cli/src/telegram.js.map +1 -1
  129. package/dist/cli/src/wakes.js +68 -0
  130. package/dist/cli/src/wakes.js.map +1 -0
  131. package/dist/cli/src/webhook.js +5 -2
  132. package/dist/cli/src/webhook.js.map +1 -1
  133. package/dist/cluster/node/pins.json +11 -5
  134. package/dist/neon/control/branches.js +13 -6
  135. package/dist/neon/control/branches.js.map +1 -1
  136. package/dist/neon/control/endpoints.js +81 -10
  137. package/dist/neon/control/endpoints.js.map +1 -1
  138. package/dist/neon/control/secrets.js +24 -8
  139. package/dist/neon/control/secrets.js.map +1 -1
  140. package/dist/src/binder/bindings.js +195 -0
  141. package/dist/src/binder/bindings.js.map +1 -0
  142. package/dist/src/binder/names.js +63 -0
  143. package/dist/src/binder/names.js.map +1 -0
  144. package/dist/src/binder/tokens.js +71 -0
  145. package/dist/src/binder/tokens.js.map +1 -0
  146. package/dist/src/branch/names.js +20 -6
  147. package/dist/src/branch/names.js.map +1 -1
  148. package/dist/src/events/names.js +28 -0
  149. package/dist/src/events/names.js.map +1 -0
  150. package/dist/src/gateway/names.js +240 -0
  151. package/dist/src/gateway/names.js.map +1 -0
  152. package/dist/src/harness/names.js +67 -0
  153. package/dist/src/harness/names.js.map +1 -0
  154. package/dist/src/kube/client.js +212 -0
  155. package/dist/src/kube/client.js.map +1 -0
  156. package/dist/src/kube/sandbox-shape.js +14 -68
  157. package/dist/src/kube/sandbox-shape.js.map +1 -1
  158. package/dist/src/kube/started-map.js +12 -0
  159. package/dist/src/kube/started-map.js.map +1 -0
  160. package/dist/src/node-agent/names.js +62 -5
  161. package/dist/src/node-agent/names.js.map +1 -1
  162. package/dist/src/sandbox/names.js +12 -20
  163. package/dist/src/sandbox/names.js.map +1 -1
  164. package/dist/src/sandbox/otlp-resource.js +173 -0
  165. package/dist/src/sandbox/otlp-resource.js.map +1 -0
  166. package/dist/src/sandbox/telemetry.js +43 -0
  167. package/dist/src/sandbox/telemetry.js.map +1 -0
  168. package/dist/src/telegram/bot-commands.js +17 -0
  169. package/dist/src/telegram/bot-commands.js.map +1 -0
  170. package/dist/src/telegram/stop.js +12 -0
  171. package/dist/src/telegram/stop.js.map +1 -0
  172. package/dist/src/telemetry/config.js +131 -0
  173. package/dist/src/telemetry/config.js.map +1 -0
  174. package/dist/src/telemetry/forward.js +154 -0
  175. package/dist/src/telemetry/forward.js.map +1 -0
  176. package/package.json +1 -1
  177. package/dist/cli/src/manifests/agent-sandbox.js +0 -92
  178. package/dist/cli/src/manifests/agent-sandbox.js.map +0 -1
  179. package/dist/cli/src/manifests/juicefs-csi.js +0 -320
  180. package/dist/cli/src/manifests/juicefs-csi.js.map +0 -1
  181. package/dist/cli/src/manifests/sandboxes.agents.x-k8s.io.json +0 -6295
  182. package/dist/src/kube/config.js +0 -145
  183. package/dist/src/kube/config.js.map +0 -1
@@ -0,0 +1,761 @@
1
+ /**
2
+ * The sandbox service (src/gateway/, src/binder/): session workspaces' sandboxes as pools
3
+ * of pods already running, each bound to one workspace while it has work, through the
4
+ * gateway.
5
+ *
6
+ * - The gateway: alasio's image, as its own program, the one door to a pool's pod, of main's
7
+ * sessions and of every branch environment's, which it finds by their namespaces; its
8
+ * configuration, rendered here; its keys, made by the stack's setup
9
+ * (neon/control/kube-setup.ts); a Service of it, which alasio binds through and KEDA asks
10
+ * each pool's counts of, and a headless one its replicas find each other by. In each
11
+ * environment's sessions' namespace (sandboxPoolObjects) it may bind, claim and delete the
12
+ * pods of its pools, hold their Leases, and read their Deployments, whose templates a
13
+ * release's snapshot records; and it finds the node agents.
14
+ * - A pool of each of a session's internets, `none` and `full`, in each environment: a
15
+ * Deployment in its sessions' namespace of pods of the agent image under the sessions' runtime, each its own directory
16
+ * on its node (src/node-agent/names.ts ATTACH_ROOT, by `subPathExpr` of its namespace and its name) at /attach,
17
+ * where its workspace is attached once it is bound, its bayma started once its binding's
18
+ * token is beside it (sandbox/agent/start.sh). KEDA keeps `min` free, and makes more, to
19
+ * `max`, for the binds that wait; a bound pod is out of its Deployment, which replaces it.
20
+ * A pod that ends is ended: it serves one binding. Its template changes as a whole: the old
21
+ * free pods go as the new come, as none is ready before it is bound. A bound pod is evicted
22
+ * by no drain (./disruption.ts boundBudget).
23
+ * - The session telemetry receiver (src/receiver/): alasio's image, as its own program,
24
+ * watching the pools' Leases of every environment, which bound pods' bayma exports to.
25
+ * - The node's mount of workspace storage, a pod of each node's own, held through upgrades of
26
+ * the node agent, which attaches workspaces from it: a new one takes its mount over before
27
+ * the old goes.
28
+ */
29
+ import { createHash } from "node:crypto";
30
+ import { isDeepStrictEqual } from "node:util";
31
+ import { Schema } from "effect";
32
+ import { freeLabels, GATEWAY_READINESS, STOP_GRACE_SECONDS } from "../../../src/binder/names.js";
33
+ import { EVENTS_BROKERS_VARIABLE, EVENTS_PASSWORD_FILE_VARIABLE, EVENTS_USERNAME_VARIABLE } from "../../../src/events/names.js";
34
+ import { MAIN_ENVIRONMENT_NAME } from "../../../src/branch/names.js";
35
+ import { BINDING_KEY_VARIABLE, CALLER_KEY_VARIABLE, GATEWAY_CONFIG_VARIABLE, GatewayConfig as GatewayConfigSchema, FOLDERS_NAMESPACE_VARIABLE, RECEIVER_PORT_VARIABLE, SESSIONS_NAMESPACE_VARIABLE, templateHash, } from "../../../src/gateway/names.js";
36
+ import { BAYMA_CONTAINER, BAYMA_STOP_SECONDS, CPU_FEATURES_ANNOTATION, NET_MODE_LABEL, SESSION_TEMPLATE_ANNOTATION, STOP_WAIT_ARG } from "../../../src/kube/sandbox-shape.js";
37
+ import { ATTACH_ROOT, goferOf, gvisorRuntimeClass, NODE_AGENT_PORT, NODE_AGENT_TOKEN_VARIABLE, WORKSPACES_ROOT } from "../../../src/node-agent/names.js";
38
+ import { memoryBytes } from "../capacity.js";
39
+ import { NODE_PINS } from "../cluster/node.js";
40
+ import { imageReference } from "../images.js";
41
+ import { READY_ONCE_BOUND_ANNOTATION } from "../kube/rollout.js";
42
+ import { sessionsNamespace, stopWaitSeconds } from "./alasio.js";
43
+ import { folderTemplate, foldersRun } from "./folders.js";
44
+ import { harnessesConfig } from "./harness-pools.js";
45
+ import { accountRole, APPLIED_AFTER_ANNOTATION, appliedAfter, componentName, given, imagePullSecrets, labels, MAIN_ENVIRONMENT, NAMESPACE, otelEnv, RELEASE, replicasOf, restrictedContainer, restrictedPod, selectorLabels, sha256, TINI_SUBREAPER, } from "./common.js";
46
+ import { resumedConditionPolicy, sandboxPodPolicy } from "./admission.js";
47
+ import { durationSeconds } from "./config.js";
48
+ import { LOG_BROKERS, logPrincipalSecret } from "./log.js";
49
+ import { boundBudget } from "./disruption.js";
50
+ import { FILER_STORE } from "./object-store.js";
51
+ import { priority } from "./priorities.js";
52
+ import { scaledObject } from "./scalers.js";
53
+ import { verticalPodAutoscaler } from "./sizing.js";
54
+ import { JUICEFS_ADMIN, VALKEY, VALKEY_ADDRESS, WORKSPACES_CREDENTIALS, workspacesBucketUrl } from "./workspace-storage.js";
55
+ /** The gateway's name, its pods' component, and its Service's; the headless Service its replicas find each other by. */
56
+ export const GATEWAY = componentName("sandbox-gateway");
57
+ export const GATEWAY_COMPONENT = "sandbox-gateway";
58
+ const GATEWAY_PEERS = componentName("sandbox-gateway-peers");
59
+ /** The port the gateway serves on. */
60
+ export const GATEWAY_PORT = 8080;
61
+ /** The receiver's name and its pods' component. */
62
+ export const RECEIVER = componentName("receiver");
63
+ export const RECEIVER_COMPONENT = "receiver";
64
+ /** The node's mount of workspace storage, its DaemonSet's name and its pods' component. */
65
+ export const WORKSPACE_MOUNT = componentName("workspace-mount");
66
+ export const WORKSPACE_MOUNT_COMPONENT = "workspace-mount";
67
+ /** The port every pool pod's bayma serves MCP on. */
68
+ export const POOL_POD_PORT = 7290;
69
+ /** Each pool's name, by its sessions' internet. */
70
+ export const POOLS = { none: "sandboxes-none", full: "sandboxes-full" };
71
+ /** The component of a pool's pods: a session's. */
72
+ export const POOL_COMPONENT = "session";
73
+ /** The pools of `environment`, by its sessions' internet. */
74
+ export const poolsOf = (config, environment) => {
75
+ const namespace = sessionsNamespace(config, environment);
76
+ return { none: { name: POOLS.none, namespace }, full: { name: POOLS.full, namespace } };
77
+ };
78
+ /**
79
+ * Each sandbox pool's template before the current one, as the gateway's configuration it
80
+ * replaces, `liveGateway` (its ConfigMap's `config.json`), carried it: the one it rendered, where
81
+ * the template changes, or else the one it carried; none where it rendered none, as where its
82
+ * sessions were not snapshotted. A template is carried for one change of it, and no more
83
+ * (./alasio.ts carriedFrom); and pinned into the sandbox pods'
84
+ * policy for the restores of the snapshots taken on it, read back from the cluster: the
85
+ * ConfigMap is that trust's anchor, which only the operator, by `alasio up`, writes. Pure, for tests.
86
+ */
87
+ export function carriedPools(config, liveGateway, kept) {
88
+ const none = { none: null, full: null };
89
+ if (!poolsSnapshotted(kept.cpuBaseline) || liveGateway === undefined)
90
+ return none;
91
+ const held = Schema.decodeUnknownOption(Schema.fromJsonString(GatewayConfigSchema))(liveGateway);
92
+ if (held._tag === "None" || held.value.snapshots === null)
93
+ return none;
94
+ const pools = poolsOf(config, MAIN_ENVIRONMENT);
95
+ const carry = (mode) => {
96
+ const { current, previous } = held.value.snapshots?.templates[mode] ?? { current: {}, previous: null };
97
+ const now = JSON.parse(JSON.stringify(poolTemplate(config, pools[mode], mode, kept.cpuBaseline)));
98
+ return (isDeepStrictEqual(current, now) ? previous : current);
99
+ };
100
+ return { none: carry("none"), full: carry("full") };
101
+ }
102
+ /**
103
+ * The CPU baseline the installation whose gateway's configuration is `liveGateway` (as
104
+ * carriedPools takes it) holds sessions to: what its pools' current template sets, if
105
+ * anything, as only a pool of snapshotted sessions is held to one. Pure, for tests.
106
+ */
107
+ export function recordedCpuBaseline(liveGateway) {
108
+ if (liveGateway === undefined)
109
+ return null;
110
+ const held = Schema.decodeUnknownOption(Schema.fromJsonString(GatewayConfigSchema))(liveGateway);
111
+ if (held._tag === "None")
112
+ return null;
113
+ const recorded = held.value.snapshots?.templates.none.current?.metadata?.annotations?.[CPU_FEATURES_ANNOTATION];
114
+ return recorded ? recorded.split(",") : null;
115
+ }
116
+ /** Whether the sandbox pools' sessions are snapshotted: where the nodes' CPU baseline is recorded, which they are held to. */
117
+ export const poolsSnapshotted = (cpuBaseline) => cpuBaseline !== null;
118
+ /**
119
+ * The template of a pool's pods, as the module says: a session's, under its internet `mode`.
120
+ * Where its sessions are snapshotted (poolsSnapshotted), as main's are by the gateway, it holds
121
+ * them to the nodes' CPU baseline `cpuBaseline`, its bayma holds a stop it is not given for the
122
+ * gateway's save (sessions.snapshots.stopWait), and it records its own hash
123
+ * (SESSION_TEMPLATE_ANNOTATION), which a snapshot taken on it names it by. Every environment's
124
+ * pools have the same.
125
+ */
126
+ export function poolTemplate(config, pool, mode, cpuBaseline) {
127
+ const template = unhashedPoolTemplate(config, pool, mode, cpuBaseline);
128
+ if (!poolsSnapshotted(cpuBaseline))
129
+ return template;
130
+ return { ...template, metadata: { ...template.metadata, annotations: { ...template.metadata?.annotations, [SESSION_TEMPLATE_ANNOTATION]: templateHash(template) } } };
131
+ }
132
+ /** A pool's template, as poolTemplate says, but its hash. */
133
+ function unhashedPoolTemplate(config, pool, mode, cpuBaseline) {
134
+ const { sessions } = config;
135
+ const agent = imageReference(config.images.agent);
136
+ const snapshotted = poolsSnapshotted(cpuBaseline);
137
+ return {
138
+ metadata: {
139
+ labels: { "app.kubernetes.io/part-of": "alasio", "app.kubernetes.io/instance": RELEASE, "app.kubernetes.io/component": POOL_COMPONENT, [NET_MODE_LABEL]: mode, ...freeLabels(pool) },
140
+ // Under gVisor, /tmp is a tmpfs of the sandbox's own, by gVisor's mount hints, rather
141
+ // than the emptyDir's directory on the node: a snapshot of the sandbox holds it.
142
+ annotations: {
143
+ "dev.gvisor.spec.mount.tmp.share": "container",
144
+ "dev.gvisor.spec.mount.tmp.type": "tmpfs",
145
+ "dev.gvisor.spec.mount.tmp.options": "rw",
146
+ ...(snapshotted ? { [CPU_FEATURES_ANNOTATION]: cpuBaseline.join(",") } : {}),
147
+ },
148
+ },
149
+ spec: {
150
+ ...(sessions.runtimeClassName ? { runtimeClassName: sessions.runtimeClassName } : {}),
151
+ ...priority("turn"),
152
+ // Ended by other than the gateway, bayma holds its stop for the gateway's save, then stops.
153
+ ...(snapshotted ? { terminationGracePeriodSeconds: stopWaitSeconds(config) + BAYMA_STOP_SECONDS } : {}),
154
+ securityContext: restrictedPod(1000, 1000),
155
+ automountServiceAccountToken: false,
156
+ enableServiceLinks: false,
157
+ // Names resolved as its internet says, as a session's are: by none but itself without one.
158
+ dnsPolicy: "None",
159
+ dnsConfig: { nameservers: mode === "full" ? [...sessions.fullModeNameservers] : ["127.0.0.1"] },
160
+ ...imagePullSecrets(config),
161
+ ...given("nodeSelector", { ...sessions.nodeSelector }),
162
+ ...given("tolerations", [...sessions.tolerations]),
163
+ containers: [{
164
+ name: BAYMA_CONTAINER,
165
+ image: agent,
166
+ imagePullPolicy: config.images.pullPolicy,
167
+ // bayma, and its hook as a release's snapshot runs it, once its workspace is attached.
168
+ command: ["/usr/bin/tini", "-s", "--", "/usr/local/bin/alasio-start", "node", "/opt/bayma/bayma.js"],
169
+ args: [
170
+ "mcp-http",
171
+ "--host",
172
+ "0.0.0.0",
173
+ "--port",
174
+ String(POOL_POD_PORT),
175
+ "--state-dir",
176
+ "/home/agent/.bayma",
177
+ "--token-file",
178
+ "/attach/token",
179
+ "--telemetry-token-file",
180
+ "/attach/token",
181
+ ...(snapshotted ? [STOP_WAIT_ARG, String(stopWaitSeconds(config) * 1000)] : []),
182
+ ],
183
+ workingDir: "/attach",
184
+ env: [
185
+ { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" } } },
186
+ { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } },
187
+ { name: "HOME", value: "/home/agent" },
188
+ { name: "USER", value: "agent" },
189
+ { name: "BAYMA_PAYLOAD_DIR", value: "/opt/bayma/payload" },
190
+ ],
191
+ ports: [{ name: "mcp", containerPort: POOL_POD_PORT }],
192
+ readinessProbe: { tcpSocket: { port: "mcp" }, periodSeconds: 2 },
193
+ securityContext: restrictedContainer(),
194
+ resources: sessions.resources,
195
+ volumeMounts: [
196
+ // Its own directory on its node, by its namespace and its name, as pods of two namespaces
197
+ // may share a name, which its workspace is attached under once it is bound.
198
+ { name: "attach", mountPath: "/attach", subPathExpr: "$(POD_NAMESPACE)/$(POD_NAME)", mountPropagation: "HostToContainer" },
199
+ { name: "tmp", mountPath: "/tmp" },
200
+ ],
201
+ }],
202
+ volumes: [{ name: "attach", hostPath: { path: ATTACH_ROOT, type: "DirectoryOrCreate" } }, { name: "tmp", emptyDir: { sizeLimit: "2Gi" } }],
203
+ },
204
+ };
205
+ }
206
+ /** A pool's Deployment, and the ScaledObject that keeps its free pods. */
207
+ function poolObjects(config, pool, mode, cpuBaseline) {
208
+ const template = poolTemplate(config, pool, mode, cpuBaseline);
209
+ const deployment = {
210
+ apiVersion: "apps/v1",
211
+ kind: "Deployment",
212
+ // Ready, as `alasio up` and `alasio status` say it, once its pods run: each is ready only once bound.
213
+ metadata: { name: pool.name, namespace: pool.namespace, labels: labels(POOL_COMPONENT), annotations: { [READY_ONCE_BOUND_ANNOTATION]: "true" } },
214
+ spec: {
215
+ ...replicasOf(config.sessions.pools),
216
+ selector: { matchLabels: { ...freeLabels(pool) } },
217
+ // None is ready before it is bound: the old free pods go as the new come.
218
+ strategy: { type: "Recreate" },
219
+ template,
220
+ },
221
+ };
222
+ // As many free as binds wait for, past `min`: each is claimed by one.
223
+ const waiting = {
224
+ type: "metrics-api",
225
+ metadata: { url: `http://${GATEWAY}.${NAMESPACE}.svc.cluster.local:${GATEWAY_PORT}/counts/${pool.namespace}/${pool.name}`, valueLocation: "waiting", targetValue: "1" },
226
+ };
227
+ // Sized as recommended alone: a sandbox's resources change only as a new pod is made.
228
+ return [deployment, ...scaledObject(deployment, config.sessions.pools, [waiting]), verticalPodAutoscaler(deployment, "Recommended")];
229
+ }
230
+ /** The gateway's configuration, as src/gateway/gateway.ts reads it. */
231
+ function gatewayConfig(config, carried) {
232
+ const pools = poolsOf(config, MAIN_ENVIRONMENT);
233
+ const templates = (mode) => ({ current: { ...poolTemplate(config, pools[mode], mode, carried.cpuBaseline) }, previous: carried.previousPools[mode] ? { ...carried.previousPools[mode] } : null });
234
+ return {
235
+ port: GATEWAY_PORT,
236
+ peers: `${GATEWAY_PEERS}.${NAMESPACE}.svc.cluster.local`,
237
+ podPort: POOL_POD_PORT,
238
+ waitSeconds: 120,
239
+ // A duration, as the configuration checks it.
240
+ releaseAfterSeconds: durationSeconds(config.sessions.snapshots.releaseAfter) ?? 0,
241
+ environment: { name: MAIN_ENVIRONMENT_NAME, namespace: sessionsNamespace(config, MAIN_ENVIRONMENT) },
242
+ // A new session's volume's size, as the configuration checks it.
243
+ quotaBytes: memoryBytes(config.sessions.storage.size),
244
+ // Under the releases the cluster keeps, each restored with a RuntimeClass of its own.
245
+ snapshots: poolsSnapshotted(carried.cpuBaseline)
246
+ ? {
247
+ liveEverySeconds: durationSeconds(config.sessions.snapshots.liveEvery) ?? 0,
248
+ releases: carried.gvisorReleases.map((release) => ({ release, runtimeClassName: gvisorRuntimeClass(release), gofer: goferOf(release, NODE_PINS.gvisor.restoresThroughSocketSince) })),
249
+ templates: { none: templates("none"), full: templates("full") },
250
+ }
251
+ : null,
252
+ pools: { none: POOLS.none, full: POOLS.full },
253
+ nodeAgents: {
254
+ namespace: NAMESPACE,
255
+ selector: Object.entries(selectorLabels("node-agent")).map(([label, value]) => `${label}=${value}`).join(","),
256
+ port: NODE_AGENT_PORT,
257
+ },
258
+ receiver: { host: `${RECEIVER}.${NAMESPACE}.svc.cluster.local`, port: config.telemetry.receiverPort },
259
+ // Folder conversations' instances, of the host profile, where the installation has folder workspaces.
260
+ folders: foldersRun(config)
261
+ ? { namespace: config.host.namespace, template: { ...folderTemplate(config) }, stateRoot: config.host.stateRoot, releaseAfterSeconds: durationSeconds(config.host.releaseAfter) ?? 0 }
262
+ : null,
263
+ // Conversations' harnesses' pools, where they run in pools of their own.
264
+ harnesses: harnessesConfig(config),
265
+ };
266
+ }
267
+ /**
268
+ * What lets the ServiceAccount `account` find the branch environments, by their namespaces
269
+ * (src/gateway/environments.ts): a ClusterRole that reads namespaces, and nothing else of the
270
+ * cluster, and its binding.
271
+ */
272
+ function environmentsFinder(account, component) {
273
+ const metadata = { name: account, labels: labels(component) };
274
+ return [
275
+ { apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", metadata, rules: [{ apiGroups: [""], resources: ["namespaces"], verbs: ["get", "list", "watch"] }] },
276
+ {
277
+ apiVersion: "rbac.authorization.k8s.io/v1",
278
+ kind: "ClusterRoleBinding",
279
+ metadata,
280
+ roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: account },
281
+ subjects: [{ kind: "ServiceAccount", name: account, namespace: NAMESPACE }],
282
+ },
283
+ ];
284
+ }
285
+ /** The ServiceAccount `account` of alasio's namespace. */
286
+ const serviceAccount = (account, component) => ({ apiVersion: "v1", kind: "ServiceAccount", metadata: { name: account, namespace: NAMESPACE, labels: labels(component) } });
287
+ /** The user the gateway's ServiceAccount is to the API server, which the sandbox pods' policy names as the one maker of restores' pods. */
288
+ export const GATEWAY_USER = `system:serviceaccount:${NAMESPACE}:${GATEWAY}`;
289
+ /**
290
+ * The gateway's identity: its ServiceAccount, the node agents it finds, the branch
291
+ * environments it finds, and the nodes it reads whether a restore's pod may be scheduled onto;
292
+ * what it may do in each environment's pools is sandboxPoolObjects'.
293
+ */
294
+ function gatewayIdentity() {
295
+ const nodes = { name: `${GATEWAY}-nodes`, labels: labels(GATEWAY_COMPONENT) };
296
+ const nodesRole = { apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", metadata: nodes, rules: [{ apiGroups: [""], resources: ["nodes"], verbs: ["get"] }] };
297
+ const nodesBinding = {
298
+ apiVersion: "rbac.authorization.k8s.io/v1",
299
+ kind: "ClusterRoleBinding",
300
+ metadata: nodes,
301
+ roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: nodes.name },
302
+ subjects: [{ kind: "ServiceAccount", name: GATEWAY, namespace: NAMESPACE }],
303
+ };
304
+ return [
305
+ serviceAccount(GATEWAY, GATEWAY_COMPONENT),
306
+ nodesRole,
307
+ nodesBinding,
308
+ // The node agents, by their pods, which it attaches workspaces through.
309
+ ...accountRole(GATEWAY, GATEWAY_COMPONENT, NAMESPACE, [{ apiGroups: [""], resources: ["pods"], verbs: ["list"] }]),
310
+ ...environmentsFinder(GATEWAY, GATEWAY_COMPONENT),
311
+ ];
312
+ }
313
+ /** The gateway's Deployment, its configuration's ConfigMap, and its Services. */
314
+ function gatewayObjects(config, carried) {
315
+ const rendered = JSON.stringify(gatewayConfig(config, carried), null, 2);
316
+ const configMap = { apiVersion: "v1", kind: "ConfigMap", metadata: { name: GATEWAY, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) }, data: { "config.json": rendered } };
317
+ const key = (variable, field) => ({ name: variable, valueFrom: { secretKeyRef: { name: GATEWAY, key: field } } });
318
+ const deployment = {
319
+ apiVersion: "apps/v1",
320
+ kind: "Deployment",
321
+ metadata: { name: GATEWAY, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) },
322
+ spec: {
323
+ replicas: 1,
324
+ selector: { matchLabels: selectorLabels(GATEWAY_COMPONENT) },
325
+ template: {
326
+ metadata: { labels: labels(GATEWAY_COMPONENT), annotations: { "checksum/config": sha256(rendered) } },
327
+ spec: {
328
+ serviceAccountName: GATEWAY,
329
+ ...priority("floor"),
330
+ // Its front's calls under way let end as it stops (src/binder/server.ts).
331
+ terminationGracePeriodSeconds: STOP_GRACE_SECONDS,
332
+ securityContext: restrictedPod(1000, 1000),
333
+ ...imagePullSecrets(config),
334
+ containers: [{
335
+ name: GATEWAY_COMPONENT,
336
+ image: imageReference(config.images.alasio),
337
+ imagePullPolicy: config.images.pullPolicy,
338
+ args: ["node", "/opt/alasio/src/gateway/main.ts"],
339
+ env: [
340
+ { name: GATEWAY_CONFIG_VARIABLE, value: "/etc/gateway/config.json" },
341
+ key(CALLER_KEY_VARIABLE, "caller-key"),
342
+ key(BINDING_KEY_VARIABLE, "binding-key"),
343
+ { name: NODE_AGENT_TOKEN_VARIABLE, valueFrom: { secretKeyRef: { name: componentName("node-agent"), key: "token" } } },
344
+ { name: "POD_IP", valueFrom: { fieldRef: { fieldPath: "status.podIP" } } },
345
+ // Every environment's events topic, which it writes as its principal on the log (src/events/).
346
+ { name: EVENTS_BROKERS_VARIABLE, value: LOG_BROKERS },
347
+ { name: EVENTS_USERNAME_VARIABLE, valueFrom: { secretKeyRef: { name: logPrincipalSecret("sandbox-gateway"), key: "username" } } },
348
+ { name: EVENTS_PASSWORD_FILE_VARIABLE, value: "/run/alasio/log/password" },
349
+ ...(otelEnv(config).length > 0 ? [{ name: "OTEL_SERVICE_NAME", value: GATEWAY }, ...otelEnv(config)] : []),
350
+ ],
351
+ ports: [{ name: "gateway", containerPort: GATEWAY_PORT }],
352
+ readinessProbe: { httpGet: { path: "/healthz", port: "gateway" }, ...GATEWAY_READINESS },
353
+ securityContext: restrictedContainer(),
354
+ resources: config.sessions.gateway.resources,
355
+ volumeMounts: [{ name: "config", mountPath: "/etc/gateway", readOnly: true }, { name: "log", mountPath: "/run/alasio/log", readOnly: true }],
356
+ }],
357
+ volumes: [{ name: "config", configMap: { name: GATEWAY } }, { name: "log", secret: { secretName: logPrincipalSecret("sandbox-gateway"), items: [{ key: "password", path: "password" }] } }],
358
+ },
359
+ },
360
+ },
361
+ };
362
+ const service = (name, headless) => ({
363
+ apiVersion: "v1",
364
+ kind: "Service",
365
+ metadata: { name, namespace: NAMESPACE, labels: labels(GATEWAY_COMPONENT) },
366
+ spec: { ...(headless ? { clusterIP: "None" } : {}), selector: selectorLabels(GATEWAY_COMPONENT), ports: [{ name: "gateway", port: GATEWAY_PORT, targetPort: "gateway" }] },
367
+ });
368
+ return [configMap, deployment, verticalPodAutoscaler(deployment, "InPlace"), service(GATEWAY, false), service(GATEWAY_PEERS, true)];
369
+ }
370
+ /** The receiver: its identity, which finds the branch environments, whose pools' Leases sandboxPoolObjects lets it watch, its Deployment and its Service. */
371
+ function receiverObjects(config) {
372
+ const port = config.telemetry.receiverPort;
373
+ const deployment = {
374
+ apiVersion: "apps/v1",
375
+ kind: "Deployment",
376
+ metadata: { name: RECEIVER, namespace: NAMESPACE, labels: labels(RECEIVER_COMPONENT) },
377
+ spec: {
378
+ replicas: 1,
379
+ selector: { matchLabels: selectorLabels(RECEIVER_COMPONENT) },
380
+ template: {
381
+ metadata: { labels: labels(RECEIVER_COMPONENT) },
382
+ spec: {
383
+ serviceAccountName: RECEIVER,
384
+ ...priority("floor"),
385
+ securityContext: restrictedPod(1000, 1000),
386
+ ...imagePullSecrets(config),
387
+ containers: [{
388
+ name: RECEIVER_COMPONENT,
389
+ image: imageReference(config.images.alasio),
390
+ imagePullPolicy: config.images.pullPolicy,
391
+ args: ["node", "/opt/alasio/src/receiver/main.ts"],
392
+ env: [
393
+ { name: RECEIVER_PORT_VARIABLE, value: String(port) },
394
+ { name: BINDING_KEY_VARIABLE, valueFrom: { secretKeyRef: { name: GATEWAY, key: "binding-key" } } },
395
+ { name: SESSIONS_NAMESPACE_VARIABLE, value: sessionsNamespace(config, MAIN_ENVIRONMENT) },
396
+ ...(foldersRun(config) ? [{ name: FOLDERS_NAMESPACE_VARIABLE, value: config.host.namespace }] : []),
397
+ ...(otelEnv(config).length > 0 ? [{ name: "OTEL_SERVICE_NAME", value: RECEIVER }, ...otelEnv(config)] : []),
398
+ ],
399
+ ports: [{ name: "otlp-http", containerPort: port }],
400
+ securityContext: restrictedContainer(),
401
+ resources: config.sessions.receiver.resources,
402
+ }],
403
+ },
404
+ },
405
+ },
406
+ };
407
+ const service = {
408
+ apiVersion: "v1",
409
+ kind: "Service",
410
+ metadata: { name: RECEIVER, namespace: NAMESPACE, labels: labels(RECEIVER_COMPONENT) },
411
+ spec: { selector: selectorLabels(RECEIVER_COMPONENT), ports: [{ name: "otlp-http", port, targetPort: "otlp-http" }] },
412
+ };
413
+ return [serviceAccount(RECEIVER, RECEIVER_COMPONENT), ...environmentsFinder(RECEIVER, RECEIVER_COMPONENT), deployment, verticalPodAutoscaler(deployment, "InPlace"), service];
414
+ }
415
+ /** Where a workspace mount's pod keeps what is its own: its juicefs's log and pid, and whether it has handed its mount over. */
416
+ export const MOUNT_RUN = "/run/alasio";
417
+ /** Where JuiceFS keeps a mount's open handles on the node, which the mount that takes it over reads. */
418
+ export const JUICEFS_STATE = "/var/lib/alasio/juicefs-state";
419
+ /** The port each node's mount serves JuiceFS's client's metrics on, in its pod's own network, which the collector's scraper reads. */
420
+ export const JUICEFS_METRICS_PORT = 9567;
421
+ /** Where the node runs JuiceFS's command from: its static release, copied from alasio's image, under its digest. */
422
+ export const JUICEFS_BIN = "/var/lib/alasio/bin";
423
+ /** Where the node's mount of workspace storage caches its blocks. */
424
+ export const JUICEFS_CACHE = "/var/lib/alasio/juicefs-cache";
425
+ /** What JuiceFS's mount says once it has handed its FUSE session over to another mount of the same mount point, and ends. */
426
+ export const HANDED_OVER = "transfer FUSE session to others";
427
+ /**
428
+ * What JuiceFS's mount says once its own process serves the mount point: mounted afresh, or
429
+ * having taken another's FUSE session over, as it waits until the mount point names its process
430
+ * and no longer the one before.
431
+ */
432
+ export const SERVING = "is ready at";
433
+ /**
434
+ * The Services the nodes' mount of workspace storage reaches by the addresses it resolves them to
435
+ * as it starts (workspaceMountScript): the metadata engine's, Valkey's, and the bundled object
436
+ * store's, where the bucket is its.
437
+ */
438
+ export const mountServices = (config) => [VALKEY, ...(config.objectStore.bundled.enabled ? [componentName("seaweedfs")] : [])];
439
+ /** The annotation of the mount's template that changes, rolling it, where what it reaches by address does. */
440
+ export const MOUNT_SERVICES_ANNOTATION = "checksum/storage-services";
441
+ /**
442
+ * What the mount reaches by address, as a checksum: the UIDs of its Services, `uids`, and the
443
+ * URLs it resolves, so a Service made anew, whose address is another, or another bucket, rolls it.
444
+ */
445
+ function mountServicesChecksum(config, uids) {
446
+ const reached = { metadata: VALKEY_ADDRESS, bucket: workspacesBucketUrl(config), services: Object.fromEntries(mountServices(config).map((name) => [name, uids[name] ?? null])) };
447
+ return createHash("sha256").update(JSON.stringify(reached)).digest("hex").slice(0, 16);
448
+ }
449
+ /**
450
+ * How long a mount's juicefs has to serve the node's mount once it starts: past what a takeover
451
+ * takes, juicefs's own waits in it bounded at some 70 s (the FUSE session's fd, then the old
452
+ * mount's end), and a fresh mount, a matter of seconds.
453
+ */
454
+ export const MOUNT_SERVE_WAIT_SECONDS = 180;
455
+ export const MOUNT_PATHS = {
456
+ run: MOUNT_RUN,
457
+ bin: JUICEFS_BIN,
458
+ state: JUICEFS_STATE,
459
+ cache: JUICEFS_CACHE,
460
+ juicefs: "/usr/local/bin/juicefs",
461
+ mountinfo: "/proc/1/mountinfo",
462
+ inNode: "nsenter -t 1 -m --",
463
+ resolve: "getent hosts",
464
+ fuseConnections: "/sys/fs/fuse/connections",
465
+ };
466
+ /**
467
+ * The script of a workspace mount's pod: JuiceFS's mount of workspace storage at `root`, made
468
+ * in the node's own mount namespace, so it is the node's alone, and no container's mounts, torn
469
+ * down as their container ends, take it with them; its log copied out; and, once it ends, idle
470
+ * where it ended by handing its mount over, as it said, to the mount of the pod that replaces it,
471
+ * until its own pod is deleted, or, where it ended any other way, a failure, so kubelet starts it
472
+ * again and it mounts anew. Asked to end while it mounts, it has its mount end too.
473
+ *
474
+ * Its juicefs is JuiceFS's static release, of alasio's image, which runs as it is in the node's
475
+ * namespace: copied to the node, under its digest, never over the one a mount runs. A mount left
476
+ * at `root` whose juicefs is gone, as a pod killed with no successor leaves one, which the kernel
477
+ * answers as not connected, or whose juicefs answers nothing within 5 s, as one stopped or hung
478
+ * does, not even of what it holds in memory, is unmounted first, lazily, so the next mounts
479
+ * afresh.
480
+ *
481
+ * Its juicefs, taking a mount over or mounting afresh, serves within MOUNT_SERVE_WAIT_SECONDS
482
+ * (`serveWaitSeconds`), or is ended, and the mount fails, so kubelet starts it again, to mount
483
+ * afresh: a takeover whose old mount stopped answering waits, in juicefs's own reads of it, for
484
+ * as long as it does, and no juicefs serves the node's mount meanwhile. Ended so, or unmounted as
485
+ * one that answers nothing, the node's mount has its FUSE connection aborted first: a request on
486
+ * a session no process will answer, as a juicefs makes of the one it takes over and holds, waits
487
+ * where no signal ends it, KILL among them, until its connection is aborted, which fails it.
488
+ *
489
+ * In the node's mount namespace its juicefs reads the node's resolv.conf, which resolves none of
490
+ * the cluster's names, its network the pod's still: so the metadata engine's and the bucket's,
491
+ * where they are a Service's, are resolved here, by the pod's DNS, and it is given their
492
+ * addresses. A Service's address lasts as long as it does: `up` rolls the mount where one is made
493
+ * anew, or the bucket is another (MOUNT_SERVICES_ANNOTATION), and the pod that replaces it reaches
494
+ * the new address.
495
+ *
496
+ * Its juicefs, run in the node's mount namespace, is still this pod's process, of its cgroup: the
497
+ * pod's teardown kills it. So a pod hands its mount over before it is torn down, never after: the
498
+ * pod that replaces it starts beside it (maxSurge), takes its FUSE session over, and only then is
499
+ * it deleted, idle, its juicefs ended already, which its readiness keeps it from until then
500
+ * (workspaceMountReady). `paths` are MOUNT_PATHS but for tests.
501
+ */
502
+ export function workspaceMountScript(root, cacheSizeMiB, paths = MOUNT_PATHS, serveWaitSeconds = MOUNT_SERVE_WAIT_SECONDS) {
503
+ const { run, bin, state, cache, juicefs, mountinfo, inNode, resolve, fuseConnections } = paths;
504
+ return [
505
+ // The FUSE connection of the node's mount at the root aborted, where its mount there, the
506
+ // topmost, is a FUSE file system's: every request on it fails, those no process will ever
507
+ // answer among them, as one a juicefs makes of the very session it takes over and holds,
508
+ // which leaves it waiting where no signal, not even KILL, ends it. Its connection is named by
509
+ // the minor of the mount's device, under fusectl's mount, made where the node has none.
510
+ "aborted() {",
511
+ ` connection=$(awk -v root=${root} '$5 == root { connection = ""; for (i = 6; i < NF; i += 1) if ($i == "-") { if ($(i + 1) ~ /^fuse(\\.|$)/) { split($3, device, ":"); connection = device[2] }; break } } END { print connection }' ${mountinfo})`,
512
+ ' [ -n "$connection" ] || return 0',
513
+ ` grep -qF ' ${fuseConnections} ' ${mountinfo} || ${inNode} mount -t fusectl none ${fuseConnections}`,
514
+ ` ${inNode} sh -c 'echo 1 > "$1"' sh "${fuseConnections}/$connection/abort" && echo "aborted the FUSE connection $connection of the node's mount, so what waits on it fails"`,
515
+ "}",
516
+ // Whether the node's mount at the root answers its .config within 5 s, which juicefs answers
517
+ // of its own memory, what the read said kept in $run/probed: where not, no process serves it,
518
+ // or one stopped or hung does. Not waited on past that: a read juicefs took but never
519
+ // answers waits where no signal ends it, until the connection is aborted, which frees it.
520
+ "answers() {",
521
+ ` rm -f "$run/answered" "$run/probed"`,
522
+ ` ( ${inNode} cat ${root}/.config > /dev/null 2> "$run/probed"; : > "$run/answered" ) &`,
523
+ ` for tick in 1 2 3 4 5; do sleep 1; [ -e "$run/answered" ] && return 0; done`,
524
+ " return 1",
525
+ "}",
526
+ // `url`, its host a Service's name given as its address, by the pod's DNS.
527
+ "resolved() {",
528
+ ' scheme=${1%%://*}; rest=${1#*://}; auth=""',
529
+ ' case $rest in *@*) auth="${rest%@*}@"; rest=${rest##*@};; esac',
530
+ ' host=${rest%%[:/]*}; tail=${rest#"$host"}; address=$host',
531
+ " case $host in *.svc|*.svc.cluster.local)",
532
+ ` address=$(${resolve} "$host" | awk '{ print $1; exit }')`,
533
+ ' [ -n "$address" ] || { echo "$host resolves to no address" >&2; return 1; };;',
534
+ " esac",
535
+ ` printf '%s://%s%s%s\n' "$scheme" "$auth" "$address" "$tail"`,
536
+ "}",
537
+ // Told to stop (TERM, INT) at any time, the script's start among it: noted, and passed on to
538
+ // its juicefs, as it runs; before it runs, it is not started.
539
+ "pid=",
540
+ "termed=",
541
+ `trap 'termed=1; [ -z "$pid" ] || kill -TERM "$pid"' TERM INT`,
542
+ // The metadata engine's password given in META_PASSWORD, which JuiceFS reads, not in its
543
+ // address on the command line, which every process of the node's PIDs could read.
544
+ `META_PASSWORD=$(node -e 'process.stdout.write(decodeURIComponent(new URL(process.env.META_URL).password))') || exit 1`,
545
+ "export META_PASSWORD",
546
+ `meta=$(resolved "$(node -e 'const url = new URL(process.env.META_URL); url.password = ""; process.stdout.write(url.href)')") || exit 1`,
547
+ 'bucket=$(resolved "$BUCKET") || exit 1',
548
+ `run=${run}`,
549
+ `mkdir -p "$run" ${bin} ${state} ${cache} || exit 1`,
550
+ `rm -f "$run/handed-over" "$run/expired"`,
551
+ // Made, empty, before the mount starts or the log is followed, which would not find it yet.
552
+ `: > "$run/juicefs.log"`,
553
+ // Named by its digest, read whole: a read that fails, as one of a layer lazily pulled may, stops
554
+ // the mount, which kubelet starts again, rather than run a juicefs of no digest, which is none.
555
+ `digest=$(sha256sum ${juicefs}) || { echo "could not read ${juicefs}, so the mount is not started" >&2; exit 1; }`,
556
+ `node_juicefs="${bin}/juicefs-$(printf '%.16s' "$digest")"`,
557
+ // Copied beside it under a name of its own, made by mktemp, and moved into place, so no
558
+ // pod's half-written copy is run: no $$, which Kubernetes makes $ in a container's command.
559
+ `if [ ! -x "$node_juicefs" ]; then`,
560
+ ` copying=$(mktemp "${bin}/.juicefs.XXXXXX") && { cp ${juicefs} "$copying" && chmod 755 "$copying" && mv "$copying" "$node_juicefs"; } || { rm -f "$copying"; echo "could not copy ${juicefs} to the node, so the mount is not started" >&2; exit 1; }`,
561
+ "fi",
562
+ // A mount of a juicefs gone, which its kernel answers as no longer connected, or of one that
563
+ // answers nothing within 5 s: unmounted, so this one mounts afresh. Asked of its .config,
564
+ // which juicefs answers of its own memory, never its metadata engine's (JuiceFS's
565
+ // pkg/vfs/internal.go), so a juicefs that serves still, however slow its metadata engine
566
+ // is, or unreachable, as it is where `up` rolls the mount, is answered and taken over, its
567
+ // workspaces kept. Its takeover reads the same file (cmd/mount.go).
568
+ `if awk -v root=${root} '$5 == root { found = 1 } END { exit !found }' ${mountinfo}; then`,
569
+ " if ! answers; then",
570
+ " aborted",
571
+ ` ${inNode} umount -l ${root} && echo "unmounted the node's mount, whose juicefs answered nothing in 5 s, not even its .config"`,
572
+ ` elif grep -qi 'not connected' "$run/probed"; then`,
573
+ ` ${inNode} umount -l ${root} && echo "unmounted the node's mount its juicefs left, which served nothing"`,
574
+ " fi",
575
+ "fi",
576
+ `${inNode} mkdir -p ${root}`,
577
+ `if [ -n "$termed" ]; then echo "told to stop before its juicefs started, so it starts none"; exit 0; fi`,
578
+ `TMPDIR=${state} ${inNode} "$node_juicefs" mount --no-bgjob --metrics 0.0.0.0:${JUICEFS_METRICS_PORT} --bucket "$bucket" --cache-dir ${cache} --cache-size ${cacheSizeMiB} "$meta" ${root} >> "$run/juicefs.log" 2>&1 &`,
579
+ "pid=$!",
580
+ // Told to stop as it started, before its PID was known: passed on now.
581
+ `[ -z "$termed" ] || kill -TERM "$pid"`,
582
+ `echo "$pid" > "$run/juicefs.pid"`,
583
+ `tail -n +1 -f "$run/juicefs.log" &`,
584
+ "tailer=$!",
585
+ // Ended where it serves nothing within its wait, its watcher's end awaited then (expired):
586
+ // what each of its goroutines waits on said, into its log, as Go says it on QUIT, of it and
587
+ // of each process it runs, while their main threads run still, as a KILLed process one of
588
+ // whose threads waits on yet says nothing. Then, where the node's mount answers nothing, as
589
+ // where it holds the session no process serves, its FUSE connection aborted, as what holds
590
+ // it up may be a request on it that nothing answers, which no signal ends; TERM, and KILL 5 s
591
+ // on; and the mount, dead, unmounted, so the next mounts afresh. Where it answers, it is the
592
+ // mount of the pod this replaces, served still, which the next takes over again.
593
+ "(",
594
+ ` for second in $(seq ${serveWaitSeconds}); do grep -qF '${SERVING}' "$run/juicefs.log" && exit 0; kill -0 "$pid" 2>/dev/null || exit 0; sleep 1; done`,
595
+ ` : > "$run/expired"`,
596
+ ` echo "juicefs served nothing within ${serveWaitSeconds} s of starting, so it is ended, and the mount started again to mount afresh" >&2`,
597
+ ` for proc in /proc/[0-9]*; do [ "$(sed 's/.*) //' "$proc/stat" 2>/dev/null | cut -d ' ' -f 2)" = "$pid" ] && kill -QUIT "\${proc#/proc/}" 2>/dev/null; done`,
598
+ ` kill -QUIT "$pid" 2>/dev/null; sleep 2`,
599
+ " dead=",
600
+ " if ! answers; then aborted; dead=1; fi",
601
+ ` kill -TERM "$pid" 2>/dev/null; sleep 5; kill -KILL "$pid" 2>/dev/null`,
602
+ ` if [ -n "$dead" ]; then ${inNode} umount -l ${root} && echo "unmounted the node's mount, which no juicefs served"; fi`,
603
+ ") &",
604
+ "watcher=$!",
605
+ `wait "$pid"; code=$?`,
606
+ // A TERM's wait ends as the trap runs; the mount's own end is waited for.
607
+ `while kill -0 "$pid" 2>/dev/null; do wait "$pid"; code=$?; done`,
608
+ "sleep 1",
609
+ `if [ -e "$run/expired" ]; then wait "$watcher"; fi`,
610
+ `kill "$tailer" "$watcher" 2>/dev/null`,
611
+ `if grep -qF '${HANDED_OVER}' "$run/juicefs.log"; then`,
612
+ ` touch "$run/handed-over"`,
613
+ " echo 'the mount is handed over to the pod that replaces this one, which is idle until it is deleted'",
614
+ " trap 'exit 0' TERM INT",
615
+ " while :; do sleep 3600 & wait $!; done",
616
+ "fi",
617
+ `echo "juicefs mount ended ($code) without handing its mount over" >&2`,
618
+ `[ "$code" -ne 0 ] && exit "$code"`,
619
+ "exit 1",
620
+ ].join("\n");
621
+ }
622
+ /**
623
+ * The readiness of a workspace mount's pod: its own juicefs runs, has said it serves the mount
624
+ * point (SERVING), and the node's mount table holds JuiceFS's mount at `root`: this pod's juicefs
625
+ * serves the node's mount point. The table alone holds the mount of the pod it replaces as its
626
+ * juicefs starts, before it takes that mount over, which the pod it replaces would be deleted
627
+ * for, its mount with it.
628
+ *
629
+ * Or it has handed its mount over, its part done: ready still, until its DaemonSet's update
630
+ * deletes it, once the pod that took its mount over is ready. Kubernetes keeps two of a
631
+ * DaemonSet's pods on a node, a surge's, only while the new is not ready and the old is; were
632
+ * neither ready, it deletes the old at once (the DaemonSet controller's podsShouldBeOnNode),
633
+ * whatever the update's maxUnavailable, and the node has no ready pod of its mount while the
634
+ * takeover runs on.
635
+ */
636
+ export function workspaceMountReady(root, paths = MOUNT_PATHS) {
637
+ const { run, mountinfo } = paths;
638
+ const serving = [
639
+ `kill -0 "$(cat ${run}/juicefs.pid)" 2>/dev/null`,
640
+ `grep -qF '${SERVING}' ${run}/juicefs.log`,
641
+ `awk -v root=${root} '$5 == root { for (i = 6; i < NF; i += 1) if ($i == "-") { if ($(i + 1) == "fuse.juicefs") found = 1; break } } END { exit !found }' ${mountinfo}`,
642
+ ].join(" && ");
643
+ return `[ -e ${run}/handed-over ] || { ${serving}; }`;
644
+ }
645
+ /**
646
+ * The node's mount of workspace storage, at WORKSPACES_ROOT, the node's own (workspaceMountScript),
647
+ * which the node agent attaches workspaces from. A mount held by a pod that goes takes every
648
+ * workspace attached from it with it, so a new pod of it starts beside the old (maxSurge), and
649
+ * its mount of the same mount point takes the old's FUSE session over, which needs them to share
650
+ * the node's PIDs and mount namespace; the old pod idles once it has handed its mount over, not
651
+ * ready, and is deleted once the new is ready. It runs alasio's image, for JuiceFS's static
652
+ * release, of the version the mount image is (../../../Dockerfile), privileged, as it enters
653
+ * the node's mount namespace.
654
+ */
655
+ function workspaceMount(config, mountServices) {
656
+ const credential = (name, key) => ({ name, valueFrom: { secretKeyRef: { name: WORKSPACES_CREDENTIALS, key } } });
657
+ const nodePaths = [["bin", JUICEFS_BIN], ["state", JUICEFS_STATE], ["cache", JUICEFS_CACHE]];
658
+ return {
659
+ apiVersion: "apps/v1",
660
+ kind: "DaemonSet",
661
+ metadata: {
662
+ name: WORKSPACE_MOUNT,
663
+ namespace: NAMESPACE,
664
+ labels: labels(WORKSPACE_MOUNT_COMPONENT),
665
+ // Rolled only once the Valkeys it reaches are ready, its metadata engine's and the
666
+ // bucket's filer store's: a pod that takes its predecessor's FUSE session over while its
667
+ // metadata engine restarts may never serve it.
668
+ annotations: { [APPLIED_AFTER_ANNOTATION]: appliedAfter([{ kind: "StatefulSet", namespace: NAMESPACE, name: VALKEY }, { kind: "StatefulSet", namespace: NAMESPACE, name: FILER_STORE }]) },
669
+ },
670
+ spec: {
671
+ selector: { matchLabels: selectorLabels(WORKSPACE_MOUNT_COMPONENT) },
672
+ updateStrategy: { type: "RollingUpdate", rollingUpdate: { maxSurge: 1, maxUnavailable: 0 } },
673
+ template: {
674
+ metadata: {
675
+ labels: { ...labels(WORKSPACE_MOUNT_COMPONENT), ...JUICEFS_ADMIN },
676
+ annotations: { [MOUNT_SERVICES_ANNOTATION]: mountServicesChecksum(config, mountServices) },
677
+ },
678
+ spec: {
679
+ ...priority("floor"),
680
+ ...imagePullSecrets(config),
681
+ ...given("nodeSelector", { ...config.sessions.nodeSelector }),
682
+ ...given("tolerations", [...config.sessions.tolerations]),
683
+ automountServiceAccountToken: false,
684
+ // The mount that takes another's over signals it, by its PID on the node; and it enters the node's mount namespace by PID 1's.
685
+ hostPID: true,
686
+ containers: [{
687
+ name: WORKSPACE_MOUNT_COMPONENT,
688
+ image: imageReference(config.images.alasio),
689
+ imagePullPolicy: config.images.pullPolicy,
690
+ args: ["sh", "-c", workspaceMountScript(WORKSPACES_ROOT, config.workspaceStorage.cacheSizeMiB)],
691
+ env: [TINI_SUBREAPER, credential("META_URL", "metaurl"), credential("BUCKET", "bucket"), credential("ACCESS_KEY", "access-key"), credential("SECRET_KEY", "secret-key")],
692
+ ports: [{ name: "metrics", containerPort: JUICEFS_METRICS_PORT }],
693
+ readinessProbe: { exec: { command: ["sh", "-c", workspaceMountReady(WORKSPACES_ROOT)] }, periodSeconds: 2 },
694
+ securityContext: { privileged: true },
695
+ resources: config.workspaceStorage.mount.resources,
696
+ // The node's directories, where the pod reaches them as the node does.
697
+ volumeMounts: [...nodePaths.map(([name, path]) => ({ name, mountPath: path })), { name: "run", mountPath: MOUNT_RUN }],
698
+ }],
699
+ volumes: [...nodePaths.map(([name, path]) => ({ name, hostPath: { path, type: "DirectoryOrCreate" } })), { name: "run", emptyDir: {} }],
700
+ },
701
+ },
702
+ },
703
+ };
704
+ }
705
+ /**
706
+ * What the sandbox service has in `environment`'s sessions' namespace, main's or a branch
707
+ * environment's (./branch.ts): its pools, the budget of their bound pods, and what the gateway
708
+ * may do there, binding in its pools, and the receiver, watching their Leases.
709
+ */
710
+ export function sandboxPoolObjects(config, environment, cpuBaseline) {
711
+ const pools = poolsOf(config, environment);
712
+ const namespace = pools.none.namespace;
713
+ return [
714
+ ...accountRole(GATEWAY, GATEWAY_COMPONENT, namespace, [
715
+ // Its pools' pods, claimed by an update of the version read, and deleted as they are released; and a restore's, made, which the sandbox pods' policy holds to its pool's template.
716
+ { apiGroups: [""], resources: ["pods"], verbs: ["get", "list", "watch", "create", "update", "delete"] },
717
+ // A restore's pod made ready once its sandbox is resumed, by its readiness gate's condition.
718
+ { apiGroups: [""], resources: ["pods/status"], verbs: ["patch"] },
719
+ // The network policies that select a restore's pod, which its node's agent waits for in force before it resumes it.
720
+ { apiGroups: ["networking.k8s.io"], resources: ["networkpolicies"], verbs: ["list"] },
721
+ { apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] },
722
+ // Its pools' templates, which a release's snapshot records.
723
+ { apiGroups: ["apps"], resources: ["deployments"], verbs: ["get"], resourceNames: [POOLS.none, POOLS.full] },
724
+ // What the API server tells of the pods it refused to make, as the sandbox pods' policy refused them (src/gateway/refusals.ts).
725
+ { apiGroups: [""], resources: ["events"], verbs: ["list"] },
726
+ ]),
727
+ ...accountRole(RECEIVER, RECEIVER_COMPONENT, namespace, [{ apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["list", "watch"] }]),
728
+ ...poolObjects(config, pools.none, "none", cpuBaseline),
729
+ ...poolObjects(config, pools.full, "full", cpuBaseline),
730
+ boundBudget(namespace),
731
+ ];
732
+ }
733
+ /** The sandbox service's objects: its gateway, its receiver, and the nodes' mount, main's alone, which serve every environment, and main's pools. */
734
+ export function sandboxServiceObjects(config, carried) {
735
+ const mount = workspaceMount(config, carried.mountServices);
736
+ const pools = poolsOf(config, MAIN_ENVIRONMENT);
737
+ return [
738
+ ...gatewayIdentity(),
739
+ ...gatewayObjects(config, carried),
740
+ // Every pod of every environment's sessions held to the pools' templates, as Pod Security cannot.
741
+ ...sandboxPodPolicy([poolTemplate(config, pools.none, "none", carried.cpuBaseline), poolTemplate(config, pools.full, "full", carried.cpuBaseline)], {
742
+ sessionsNamespace: sessionsNamespace(config, MAIN_ENVIRONMENT),
743
+ // Main's restores, by the gateway alone, onto each pool's template and the one before it, under the releases kept.
744
+ restores: poolsSnapshotted(carried.cpuBaseline)
745
+ ? {
746
+ restorer: GATEWAY_USER,
747
+ templates: ["none", "full"].flatMap((mode) => [poolTemplate(config, pools[mode], mode, carried.cpuBaseline), ...(carried.previousPools[mode] ? [carried.previousPools[mode]] : [])]),
748
+ runtimeClasses: carried.gvisorReleases.map(gvisorRuntimeClass),
749
+ }
750
+ : null,
751
+ }),
752
+ // Where main's sessions are snapshotted, what the gateway may set of a restore's pod's status.
753
+ ...(poolsSnapshotted(carried.cpuBaseline) ? resumedConditionPolicy(componentName("resumed-condition"), GATEWAY_USER, pools.none.namespace) : []),
754
+ ...receiverObjects(config),
755
+ ...sandboxPoolObjects(config, MAIN_ENVIRONMENT, carried.cpuBaseline),
756
+ mount,
757
+ // Sized as recommended alone, as a pod of it is only replaced, by one that takes its mount over.
758
+ verticalPodAutoscaler(mount, "Recommended"),
759
+ ];
760
+ }
761
+ //# sourceMappingURL=sandbox-gateway.js.map