hammerkit 1.27.0-next.3 → 1.27.0-next.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.schema.json +188 -0
- package/dist/cache/backends/local-cache-backend.js +6 -0
- package/dist/cache/backends/registry-cache-backend.js +2 -2
- package/dist/cache/explain.js +17 -0
- package/dist/cli.js +2 -0
- package/dist/docker/item-network.js +13 -5
- package/dist/executer/deadline.js +17 -3
- package/dist/executer/deferred-restore.js +36 -0
- package/dist/executer/docker-service.js +30 -8
- package/dist/executer/docker-task.js +21 -1
- package/dist/executer/event-cache.js +20 -2
- package/dist/executer/execute-work-service.js +2 -0
- package/dist/executer/execute-work-task.js +10 -3
- package/dist/executer/execution-steps.js +25 -4
- package/dist/executer/reset-work-tree.js +4 -0
- package/dist/executer/scheduler/enqueue-next.js +57 -6
- package/dist/executer/scheduler/state-key.js +42 -5
- package/dist/executer/watch-loop.js +4 -1
- package/dist/harness/connect.js +20 -6
- package/dist/harness/mcp-command.js +22 -6
- package/dist/harness/mcp.js +63 -114
- package/dist/harness/opencode.js +158 -50
- package/dist/harness/pi.js +29 -0
- package/dist/harness/sandbox-client.js +168 -17
- package/dist/harness/service-tools.js +141 -0
- package/dist/harness/task-tools.js +221 -0
- package/dist/kubernetes/apply.js +27 -16
- package/dist/kubernetes/await-running-state.js +59 -13
- package/dist/kubernetes/claim-layout.js +71 -0
- package/dist/kubernetes/cluster-state.js +245 -0
- package/dist/kubernetes/ensure-http-route.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-deployment-exists.js +35 -12
- package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js +1 -1
- package/dist/kubernetes/ensure-kubernetes-service-exists.js +6 -9
- package/dist/kubernetes/ensure-persistent-data.js +30 -20
- package/dist/kubernetes/follow-pod-logs.js +182 -0
- package/dist/kubernetes/get-service-ip.js +2 -2
- package/dist/kubernetes/job-placement.js +36 -12
- package/dist/kubernetes/job-secrets.js +33 -19
- package/dist/kubernetes/kept-claims.js +22 -10
- package/dist/kubernetes/network-policy.js +85 -27
- package/dist/kubernetes/remove-persistent-data.js +3 -1
- package/dist/kubernetes/resources.js +25 -3
- package/dist/kubernetes/restore-kubernetes-data.js +9 -3
- package/dist/kubernetes/sandbox-runner.js +628 -0
- package/dist/kubernetes/secrets.js +7 -11
- package/dist/kubernetes/volumes.js +6 -3
- package/dist/main.js +2 -0
- package/dist/network/egress-proxy.js +45 -6
- package/dist/optimizer/calculate-checksum.js +3 -2
- package/dist/optimizer/check-key.js +94 -0
- package/dist/optimizer/check-keys.js +64 -0
- package/dist/optimizer/get-work-cache-stats.js +17 -3
- package/dist/optimizer/work-task-cache-description.js +51 -1
- package/dist/planner/utils/append-work-dependencies.js +10 -5
- package/dist/planner/utils/append-work-service.js +19 -1
- package/dist/planner/utils/append-work-task.js +18 -3
- package/dist/planner/utils/get-work-runtime.js +6 -1
- package/dist/planner/utils/normalize-path.js +19 -1
- package/dist/planner/utils/parse-work-mount.js +7 -1
- package/dist/planner/utils/parse-work-port.js +4 -1
- package/dist/planner/utils/task-arguments.js +3 -0
- package/dist/planner/validate.js +22 -0
- package/dist/planner/work-network.js +3 -1
- package/dist/planner/work-runtime-aggregate.js +32 -0
- package/dist/planner/work-runtime-docker.js +2 -1
- package/dist/planner/work-runtime-kubernetes.js +124 -48
- package/dist/planner/work-runtime-local.js +2 -1
- package/dist/planner/work-task.js +6 -1
- package/dist/program.js +31 -10
- package/dist/remote/device-login.js +7 -2
- package/dist/remote/host-config.js +69 -3
- package/dist/remote/remote-commands.js +30 -7
- package/dist/remote/remote-work-commands.js +426 -45
- package/dist/remote/server-client.js +15 -2
- package/dist/remote/token-source.js +64 -26
- package/dist/run-program.js +9 -0
- package/dist/schema/build-file-base-task-schema.js +8 -1
- package/dist/schema/build-file-container-service-schema.js +12 -0
- package/dist/schema/build-file-container-task-schema.js +16 -1
- package/dist/schema/build-file-include-schema.js +18 -4
- package/dist/schema/build-file-kubernetes-service-schema.js +2 -0
- package/dist/schema/build-file-task-schema.js +5 -1
- package/dist/schema/capabilities-schema.js +56 -0
- package/dist/schema/reference-parser.js +120 -11
- package/dist/schema/schema-parser.js +92 -41
- package/dist/schema/work-scope-parser.js +33 -10
- package/dist/secrets/command-secret-provider.js +3 -1
- package/dist/secrets/github-app-token-command.js +3 -1
- package/dist/secrets/secret-catalog.js +43 -19
- package/dist/server/accounts-api.js +3 -1
- package/dist/server/approvals-api.js +1 -1
- package/dist/server/checks-api.js +83 -0
- package/dist/server/checks.js +116 -0
- package/dist/server/definitions-api.js +66 -3
- package/dist/server/endpoints.js +739 -0
- package/dist/server/executor.js +57 -5
- package/dist/server/kubernetes-backend.js +367 -65
- package/dist/server/kubernetes-sandbox-storage.js +48 -3
- package/dist/server/oauth.js +12 -1
- package/dist/server/plan.js +188 -103
- package/dist/server/policy.js +32 -2
- package/dist/server/repo-access.js +5 -0
- package/dist/server/run-plan.js +50 -19
- package/dist/server/run-request.js +58 -3
- package/dist/server/run-worker.js +245 -12
- package/dist/server/runs.js +410 -27
- package/dist/server/sandbox-pod.js +28 -0
- package/dist/server/sandbox-storage.js +23 -0
- package/dist/server/sandboxes-api.js +10 -4
- package/dist/server/sandboxes.js +8 -0
- package/dist/server/server-command.js +39 -20
- package/dist/server/server-config.js +255 -14
- package/dist/server/server.js +269 -15
- package/dist/server/service-runs.js +82 -0
- package/dist/utils/abort-on-signals.js +46 -6
- package/dist/utils/tar.js +36 -0
- package/dist/utils/terminal-prompt.js +53 -0
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
|
|
3
|
+
exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.getPrivilegedNamespaceProblem = exports.getCapabilitiesProblem = exports.getPodSecurityLabels = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.removeServiceRunNamespace = exports.getServerIngressPolicy = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.SERVICE_RUN_LABEL = exports.getServiceRunNamespace = exports.getCallerNamespace = void 0;
|
|
4
4
|
const crypto_1 = require("crypto");
|
|
5
5
|
const fs_1 = require("fs");
|
|
6
6
|
const os_1 = require("os");
|
|
@@ -22,6 +22,8 @@ const POLICY_NAME = 'hammerkit';
|
|
|
22
22
|
const MANAGED_LABEL = 'hammerkit.dev/managed';
|
|
23
23
|
// how long the end of a run waits for its pods and claims to be gone
|
|
24
24
|
const RELEASE_TIMEOUT = 120000;
|
|
25
|
+
// how long the setup of a caller's namespace is taken as it is, before it is applied again
|
|
26
|
+
const NAMESPACE_SETUP_REUSE_MS = 5 * 60000;
|
|
25
27
|
// where a pod of the cluster finds the token and certificate of its ServiceAccount
|
|
26
28
|
const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
|
|
27
29
|
/**
|
|
@@ -32,6 +34,19 @@ function getCallerNamespace(prefix, owner) {
|
|
|
32
34
|
return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
|
|
33
35
|
}
|
|
34
36
|
exports.getCallerNamespace = getCallerNamespace;
|
|
37
|
+
/**
|
|
38
|
+
* The namespace of a service run: one of its own, as the services stay up for hours while
|
|
39
|
+
* the caller's other runs go on in theirs (one at a time, each removing what the one
|
|
40
|
+
* before left).
|
|
41
|
+
*/
|
|
42
|
+
function getServiceRunNamespace(prefix, run, wakes = 0) {
|
|
43
|
+
// a woken run starts in a namespace of its own, the one it slept in may still be removed
|
|
44
|
+
const name = wakes > 0 ? `up\0${run}\0${wakes}` : `up\0${run}`;
|
|
45
|
+
return `${prefix}${(0, crypto_1.createHash)('sha256').update(name).digest('hex').slice(0, 16)}`;
|
|
46
|
+
}
|
|
47
|
+
exports.getServiceRunNamespace = getServiceRunNamespace;
|
|
48
|
+
// the label of the namespace of a service run: the run label value of its run
|
|
49
|
+
exports.SERVICE_RUN_LABEL = 'hammerkit.dev/service-run';
|
|
35
50
|
// Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
|
|
36
51
|
// passes over the namespaces of a server named `hk-test` on the same cluster.
|
|
37
52
|
function isCallerNamespace(prefix, name) {
|
|
@@ -143,6 +158,64 @@ function getNamespacePolicies(namespace, deniedEgress, allowedEgress = []) {
|
|
|
143
158
|
];
|
|
144
159
|
}
|
|
145
160
|
exports.getNamespacePolicies = getNamespacePolicies;
|
|
161
|
+
/**
|
|
162
|
+
* What lets the server into the namespace of a service run, to the ports of its endpoints:
|
|
163
|
+
* policies add up, so this one opens the pods to the server's namespace besides what the
|
|
164
|
+
* others let in, limited ones too.
|
|
165
|
+
*/
|
|
166
|
+
// The server reaches the ports of the endpoints, from its own namespace, and no other port:
|
|
167
|
+
// what else the run starts (a database) stays for the pods of the run.
|
|
168
|
+
function getServerIngressPolicy(namespace, server, ports) {
|
|
169
|
+
return {
|
|
170
|
+
apiVersion: 'networking.k8s.io/v1',
|
|
171
|
+
kind: 'NetworkPolicy',
|
|
172
|
+
metadata: {
|
|
173
|
+
name: 'hammerkit-server-ingress',
|
|
174
|
+
namespace,
|
|
175
|
+
annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
|
|
176
|
+
labels: { [MANAGED_LABEL]: 'true' },
|
|
177
|
+
},
|
|
178
|
+
spec: {
|
|
179
|
+
podSelector: {},
|
|
180
|
+
policyTypes: ['Ingress'],
|
|
181
|
+
ingress: ports.length > 0
|
|
182
|
+
? [
|
|
183
|
+
{
|
|
184
|
+
_from: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': server } } }],
|
|
185
|
+
ports: [...new Set(ports)].map((port) => ({ protocol: 'TCP', port })),
|
|
186
|
+
},
|
|
187
|
+
]
|
|
188
|
+
: [],
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
exports.getServerIngressPolicy = getServerIngressPolicy;
|
|
193
|
+
// The namespace of a service run goes with all it holds; waits up to `timeout` for it.
|
|
194
|
+
async function removeServiceRunNamespace(instance, namespace, timeout) {
|
|
195
|
+
try {
|
|
196
|
+
await instance.coreApi.deleteNamespace({ name: namespace, propagationPolicy: 'Background' });
|
|
197
|
+
}
|
|
198
|
+
catch (e) {
|
|
199
|
+
if ((0, apply_1.statusCodeOf)(e) === 404) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
throw e;
|
|
203
|
+
}
|
|
204
|
+
const deadline = Date.now() + timeout;
|
|
205
|
+
while (Date.now() < deadline) {
|
|
206
|
+
try {
|
|
207
|
+
await instance.coreApi.readNamespace({ name: namespace });
|
|
208
|
+
}
|
|
209
|
+
catch (e) {
|
|
210
|
+
if ((0, apply_1.statusCodeOf)(e) === 404) {
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
throw e;
|
|
214
|
+
}
|
|
215
|
+
await (0, sleep_1.sleep)(1000);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
exports.removeServiceRunNamespace = removeServiceRunNamespace;
|
|
146
219
|
/**
|
|
147
220
|
* The server's rights in a caller's namespace: the operator's ClusterRole, bound there
|
|
148
221
|
* only, so the server needs no rights on Secrets or pods anywhere else.
|
|
@@ -167,22 +240,31 @@ exports.getRunRoleBinding = getRunRoleBinding;
|
|
|
167
240
|
* process of the server as on a host (see HostExecutor); what changes is where its
|
|
168
241
|
* container tasks and services go: into the caller's namespace, on the nodes of the
|
|
169
242
|
* operator's pools, with the secrets of providers read in the Job itself. The runs
|
|
170
|
-
* of
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
243
|
+
* of a caller execute side by side in their namespace, up to `runs.limits.running`:
|
|
244
|
+
* the objects of a run are named and found by its run label (getItemKey), so two runs
|
|
245
|
+
* of the same task have their own. What two runs share are the claims the server keeps
|
|
246
|
+
* (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
|
|
247
|
+
* a time (exclusive). When a run ends, everything made for it is removed, but for the
|
|
248
|
+
* kept claims, which go once they expired.
|
|
174
249
|
*/
|
|
175
250
|
class KubernetesExecutor {
|
|
176
251
|
constructor(config, processEnvs, options) {
|
|
177
252
|
this.config = config;
|
|
178
|
-
//
|
|
179
|
-
|
|
180
|
-
// the run that executes, or the last one waiting, per caller namespace
|
|
253
|
+
// per caller namespace, what changes objects of the whole namespace: its setup, the
|
|
254
|
+
// removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
|
|
181
255
|
this.queues = new Map();
|
|
256
|
+
// per caller namespace, the runs that execute in it: by run label, their volume scope
|
|
257
|
+
this.active = new Map();
|
|
182
258
|
// per caller namespace, when its kept claims are next looked at
|
|
183
259
|
this.sweeps = new Map();
|
|
184
260
|
// whether the kept claims a server left when it stopped were looked for
|
|
185
261
|
this.swept = false;
|
|
262
|
+
// the namespaces where no run of this process left anything: what a run makes goes at its
|
|
263
|
+
// end, unless its release failed
|
|
264
|
+
this.tidy = new Set();
|
|
265
|
+
// the namespaces of the service runs that execute, and whether those of earlier ones went
|
|
266
|
+
this.serviceRuns = new Set();
|
|
267
|
+
this.serviceRunsSwept = false;
|
|
186
268
|
this.access = getClusterAccess(config.backend, processEnvs);
|
|
187
269
|
this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
|
|
188
270
|
type: 'kubernetes',
|
|
@@ -203,17 +285,56 @@ class KubernetesExecutor {
|
|
|
203
285
|
this.log = options.log;
|
|
204
286
|
this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
|
|
205
287
|
}
|
|
206
|
-
execute(run, context) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
throw new Error('the run was cancelled');
|
|
288
|
+
async execute(run, context) {
|
|
289
|
+
if (run.up) {
|
|
290
|
+
if (run.source.type === 'sandbox') {
|
|
291
|
+
throw new Error('on a cluster a service run starts from a branch, push the sandbox and start it from there');
|
|
211
292
|
}
|
|
293
|
+
// in a namespace of its own, apart from the caller's runs and their claims
|
|
212
294
|
return this.host.execute(run, context);
|
|
213
|
-
}
|
|
295
|
+
}
|
|
296
|
+
const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
|
|
297
|
+
const label = (0, job_placement_1.getRunLabelValue)(run.id);
|
|
298
|
+
let runs = this.active.get(namespace);
|
|
299
|
+
if (!runs) {
|
|
300
|
+
runs = new Map();
|
|
301
|
+
this.active.set(namespace, runs);
|
|
302
|
+
}
|
|
303
|
+
runs.set(label, getVolumeScope(run));
|
|
304
|
+
try {
|
|
305
|
+
await this.host.execute(run, context);
|
|
306
|
+
}
|
|
307
|
+
finally {
|
|
308
|
+
runs.delete(label);
|
|
309
|
+
if (runs.size === 0 && this.active.get(namespace) === runs) {
|
|
310
|
+
this.active.delete(namespace);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
214
313
|
}
|
|
215
|
-
|
|
216
|
-
|
|
314
|
+
namespaceOf(run) {
|
|
315
|
+
const prefix = this.config.backend.namespacePrefix;
|
|
316
|
+
return run.up ? getServiceRunNamespace(prefix, run.id, run.wakes) : getCallerNamespace(prefix, run.owner);
|
|
317
|
+
}
|
|
318
|
+
// The runs of a scope share its kept claims, which one of them may change at a time:
|
|
319
|
+
// what a task finds in one is what the last run left there. Without kept claims, runs
|
|
320
|
+
// share no object and nothing is held.
|
|
321
|
+
exclusive(run) {
|
|
322
|
+
if (this.keep === 0 || run.up) {
|
|
323
|
+
return undefined;
|
|
324
|
+
}
|
|
325
|
+
return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
|
|
326
|
+
}
|
|
327
|
+
// The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
|
|
328
|
+
// are theirs to mount and stay. A server that keeps none mounts none.
|
|
329
|
+
scopesInUse(namespace, except) {
|
|
330
|
+
if (this.keep === 0) {
|
|
331
|
+
return new Set();
|
|
332
|
+
}
|
|
333
|
+
return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
|
|
334
|
+
.filter(([label]) => label !== except)
|
|
335
|
+
.map(([, scope]) => scope));
|
|
336
|
+
}
|
|
337
|
+
// What changes objects of a whole namespace happens one at a time, not what its runs do
|
|
217
338
|
enqueue(namespace, fn) {
|
|
218
339
|
const previous = this.queues.get(namespace) ?? Promise.resolve();
|
|
219
340
|
const execution = previous.then(fn);
|
|
@@ -227,54 +348,98 @@ class KubernetesExecutor {
|
|
|
227
348
|
return execution;
|
|
228
349
|
}
|
|
229
350
|
placement() {
|
|
230
|
-
const backend = this.config.backend;
|
|
231
351
|
return {
|
|
232
352
|
prepare: async (run, context, work) => {
|
|
233
|
-
const namespace =
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
353
|
+
const namespace = this.namespaceOf(run);
|
|
354
|
+
if (run.up) {
|
|
355
|
+
if (context.signal.aborted) {
|
|
356
|
+
throw new Error('the run was cancelled');
|
|
357
|
+
}
|
|
358
|
+
this.serviceRuns.add(namespace);
|
|
359
|
+
this.sweepServiceRuns();
|
|
360
|
+
// reached by the server at the Services of its endpoints
|
|
361
|
+
await this.namespaces.prepare(namespace, {
|
|
362
|
+
serviceRun: (0, job_placement_1.getRunLabelValue)(run.id),
|
|
363
|
+
server: this.namespace,
|
|
364
|
+
ports: (run.up.endpoints ?? []).map((endpoint) => endpoint.port),
|
|
365
|
+
quota: this.config.runs.up.quota,
|
|
366
|
+
});
|
|
367
|
+
context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
|
|
368
|
+
return { kubernetes: this.environmentOf(run, namespace), env: this.access.env };
|
|
246
369
|
}
|
|
370
|
+
const label = (0, job_placement_1.getRunLabelValue)(run.id);
|
|
371
|
+
let capabilitiesProblem;
|
|
372
|
+
await this.enqueue(namespace, async () => {
|
|
373
|
+
if (context.signal.aborted) {
|
|
374
|
+
throw new Error('the run was cancelled');
|
|
375
|
+
}
|
|
376
|
+
capabilitiesProblem = await this.namespaces.prepare(namespace);
|
|
377
|
+
// what an earlier run left when the server stopped before its end, or when its
|
|
378
|
+
// release failed: the objects of a run that does not execute. One that starts
|
|
379
|
+
// after this waits for it here before it makes any.
|
|
380
|
+
if (!this.tidy.has(namespace)) {
|
|
381
|
+
const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
|
|
382
|
+
await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
|
|
383
|
+
this.tidy.add(namespace);
|
|
384
|
+
}
|
|
385
|
+
// the claims earlier runs kept that expired; the others this run may mount
|
|
386
|
+
const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
|
|
387
|
+
now: new Date(),
|
|
388
|
+
keep: this.keep > 0,
|
|
389
|
+
timeout: RELEASE_TIMEOUT,
|
|
390
|
+
inUse: this.scopesInUse(namespace, label),
|
|
391
|
+
});
|
|
392
|
+
if (next) {
|
|
393
|
+
this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
|
|
394
|
+
}
|
|
395
|
+
});
|
|
247
396
|
this.sweepLeftClaims();
|
|
248
397
|
context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
kubeConfig: this.access.kubeConfig,
|
|
253
|
-
context: this.access.context,
|
|
254
|
-
httpRoutes: [],
|
|
255
|
-
storageClass: backend.storageClass,
|
|
256
|
-
volumeSize: backend.volumeSize,
|
|
257
|
-
maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
|
|
258
|
-
networkPolicyEnforced: backend.networkPolicy === 'enforced',
|
|
259
|
-
server: {
|
|
260
|
-
run: run.id,
|
|
261
|
-
namespace: this.namespace,
|
|
262
|
-
pools: backend.pools,
|
|
263
|
-
providers: backend.providers,
|
|
264
|
-
accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
|
|
265
|
-
...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
|
|
266
|
-
...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
|
|
267
|
-
},
|
|
398
|
+
return {
|
|
399
|
+
kubernetes: this.environmentOf(run, namespace, work, capabilitiesProblem),
|
|
400
|
+
env: this.access.env,
|
|
268
401
|
};
|
|
269
|
-
return { kubernetes, env: this.access.env };
|
|
270
402
|
},
|
|
271
403
|
release: async (run, context) => {
|
|
272
|
-
const namespace =
|
|
404
|
+
const namespace = this.namespaceOf(run);
|
|
405
|
+
if (run.up) {
|
|
406
|
+
try {
|
|
407
|
+
await removeServiceRunNamespace(this.instance, namespace, RELEASE_TIMEOUT);
|
|
408
|
+
}
|
|
409
|
+
catch (e) {
|
|
410
|
+
// at least nothing of it runs on: its objects are the server's to remove (runRole)
|
|
411
|
+
const forbidden = (0, apply_1.statusCodeOf)(e) === 403;
|
|
412
|
+
const text = forbidden
|
|
413
|
+
? `the server may not delete the namespace ${namespace}, its ClusterRole needs delete on namespaces (see the backend docs); removing what runs in it`
|
|
414
|
+
: `the namespace ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`;
|
|
415
|
+
this.log?.(text);
|
|
416
|
+
context.emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text });
|
|
417
|
+
try {
|
|
418
|
+
await removeRunObjects(this.instance, namespace, job_placement_1.RUN_LABEL, RELEASE_TIMEOUT);
|
|
419
|
+
}
|
|
420
|
+
catch (inner) {
|
|
421
|
+
context.emit({
|
|
422
|
+
type: 'log',
|
|
423
|
+
task: 'hammerkit',
|
|
424
|
+
stream: 'stderr',
|
|
425
|
+
text: `what the run made in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(inner)}`,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
// the next look removes the namespace
|
|
429
|
+
this.serviceRunsSwept = false;
|
|
430
|
+
}
|
|
431
|
+
finally {
|
|
432
|
+
this.serviceRuns.delete(namespace);
|
|
433
|
+
}
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
273
436
|
try {
|
|
274
437
|
const selector = `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run.id)}`;
|
|
275
438
|
await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
|
|
276
439
|
}
|
|
277
440
|
catch (e) {
|
|
441
|
+
// the next run removes what is left
|
|
442
|
+
this.tidy.delete(namespace);
|
|
278
443
|
context.emit({
|
|
279
444
|
type: 'log',
|
|
280
445
|
task: 'hammerkit',
|
|
@@ -300,8 +465,67 @@ class KubernetesExecutor {
|
|
|
300
465
|
},
|
|
301
466
|
};
|
|
302
467
|
}
|
|
303
|
-
// The
|
|
304
|
-
|
|
468
|
+
// The cluster as the process of a run sees it: the namespace its tasks and services go in.
|
|
469
|
+
environmentOf(run, namespace, work, capabilitiesProblem) {
|
|
470
|
+
const backend = this.config.backend;
|
|
471
|
+
return {
|
|
472
|
+
type: 'kubernetes',
|
|
473
|
+
namespace,
|
|
474
|
+
kubeConfig: this.access.kubeConfig,
|
|
475
|
+
context: this.access.context,
|
|
476
|
+
httpRoutes: [],
|
|
477
|
+
storageClass: backend.storageClass,
|
|
478
|
+
volumeSize: backend.volumeSize,
|
|
479
|
+
maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
|
|
480
|
+
networkPolicyEnforced: backend.networkPolicy === 'enforced',
|
|
481
|
+
server: {
|
|
482
|
+
run: run.id,
|
|
483
|
+
namespace: this.namespace,
|
|
484
|
+
pools: backend.pools,
|
|
485
|
+
providers: backend.providers,
|
|
486
|
+
accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
|
|
487
|
+
...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace, backend.sandboxes.runner) } : {}),
|
|
488
|
+
// the claims of a service run go with its namespace
|
|
489
|
+
...(this.keep > 0 && !run.up ? { volumeScope: getVolumeScope(run) } : {}),
|
|
490
|
+
...(backend.capabilities
|
|
491
|
+
? {
|
|
492
|
+
capabilities: {
|
|
493
|
+
runtimeClass: backend.capabilities.runtimeClass,
|
|
494
|
+
...(capabilitiesProblem ? { unavailable: capabilitiesProblem } : {}),
|
|
495
|
+
},
|
|
496
|
+
}
|
|
497
|
+
: {}),
|
|
498
|
+
},
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
// Once, on the first service run: the namespaces of service runs the server left when it
|
|
502
|
+
// stopped, whose services went with it
|
|
503
|
+
// Once when the server starts, and on the next service run after a namespace could not be
|
|
504
|
+
// removed: the namespaces of service runs that none of the server's runs is in, whose
|
|
505
|
+
// services went with the process of a server that stopped
|
|
506
|
+
sweepServiceRuns() {
|
|
507
|
+
if (this.serviceRunsSwept) {
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
this.serviceRunsSwept = true;
|
|
511
|
+
Promise.resolve()
|
|
512
|
+
.then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,${exports.SERVICE_RUN_LABEL}` }))
|
|
513
|
+
.then(async (namespaces) => {
|
|
514
|
+
for (const namespace of namespaces.items) {
|
|
515
|
+
const name = namespace.metadata?.name;
|
|
516
|
+
if (name &&
|
|
517
|
+
isCallerNamespace(this.config.backend.namespacePrefix, name) &&
|
|
518
|
+
!this.serviceRuns.has(name) &&
|
|
519
|
+
!namespace.metadata?.deletionTimestamp) {
|
|
520
|
+
this.log?.(`removing the namespace ${name} of a service run from before the server started`);
|
|
521
|
+
await removeServiceRunNamespace(this.instance, name, 0);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
})
|
|
525
|
+
.catch((e) => this.log?.(`the namespaces of service runs could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
|
|
526
|
+
}
|
|
527
|
+
// The kept claims of a namespace that expired go at the soonest expiry the server knows
|
|
528
|
+
// of, but those of a scope a run of the caller executes in
|
|
305
529
|
sweepLater(namespace, delay) {
|
|
306
530
|
const at = Date.now() + delay + 1000;
|
|
307
531
|
const pending = this.sweeps.get(namespace);
|
|
@@ -321,6 +545,7 @@ class KubernetesExecutor {
|
|
|
321
545
|
now: new Date(),
|
|
322
546
|
keep: this.keep > 0,
|
|
323
547
|
timeout: RELEASE_TIMEOUT,
|
|
548
|
+
inUse: this.scopesInUse(namespace),
|
|
324
549
|
});
|
|
325
550
|
if (removed.length > 0) {
|
|
326
551
|
this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
|
|
@@ -337,7 +562,7 @@ class KubernetesExecutor {
|
|
|
337
562
|
}
|
|
338
563
|
this.swept = true;
|
|
339
564
|
Promise.resolve()
|
|
340
|
-
.then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` }))
|
|
565
|
+
.then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true,!${exports.SERVICE_RUN_LABEL}` }))
|
|
341
566
|
.then((namespaces) => {
|
|
342
567
|
for (const namespace of namespaces.items) {
|
|
343
568
|
const name = namespace.metadata?.name;
|
|
@@ -352,11 +577,55 @@ class KubernetesExecutor {
|
|
|
352
577
|
}
|
|
353
578
|
}
|
|
354
579
|
exports.KubernetesExecutor = KubernetesExecutor;
|
|
580
|
+
const ENFORCE_LABEL = 'pod-security.kubernetes.io/enforce';
|
|
581
|
+
const AUDIT_LABEL = 'pod-security.kubernetes.io/audit';
|
|
582
|
+
/**
|
|
583
|
+
* The Pod Security level a caller's namespace is made with: `baseline`, no privileged pods,
|
|
584
|
+
* host paths or host network, whatever a build asks. `baseline` also refuses the capabilities
|
|
585
|
+
* of `backend.capabilities`, sandboxed or not: with them the namespace enforces `privileged`
|
|
586
|
+
* and audits `baseline`, and the operator's admission policy (see the docs on capabilities)
|
|
587
|
+
* refuses what `baseline` would to a pod outside the sandbox's RuntimeClass.
|
|
588
|
+
*/
|
|
589
|
+
function getPodSecurityLabels(backend) {
|
|
590
|
+
return backend.capabilities
|
|
591
|
+
? { [ENFORCE_LABEL]: 'privileged', [AUDIT_LABEL]: 'baseline' }
|
|
592
|
+
: { [ENFORCE_LABEL]: 'baseline' };
|
|
593
|
+
}
|
|
594
|
+
exports.getPodSecurityLabels = getPodSecurityLabels;
|
|
595
|
+
/**
|
|
596
|
+
* The server never changes the level of a namespace it made (that would take `patch` on every
|
|
597
|
+
* namespace of the cluster): one made before `backend.capabilities` is relabelled by the
|
|
598
|
+
* operator. Until then its tasks that add capabilities fail with this, the others run.
|
|
599
|
+
*/
|
|
600
|
+
function getCapabilitiesProblem(namespace, labels, backend) {
|
|
601
|
+
const enforce = labels?.[ENFORCE_LABEL];
|
|
602
|
+
if (!backend.capabilities || enforce === 'privileged') {
|
|
603
|
+
return undefined;
|
|
604
|
+
}
|
|
605
|
+
return (`the namespace ${namespace} was made before backend.capabilities and enforces ` +
|
|
606
|
+
`${enforce ? `Pod Security ${enforce}` : 'no Pod Security level of its own'}, not privileged: kubectl label namespace ${namespace} ` +
|
|
607
|
+
`${ENFORCE_LABEL}=privileged ${AUDIT_LABEL}=baseline --overwrite`);
|
|
608
|
+
}
|
|
609
|
+
exports.getCapabilitiesProblem = getCapabilitiesProblem;
|
|
610
|
+
/**
|
|
611
|
+
* A namespace made while the server had `backend.capabilities` admits privileged pods but for
|
|
612
|
+
* the operator's admission policy. Without capabilities the server runs nothing in it until the
|
|
613
|
+
* operator sets it back.
|
|
614
|
+
*/
|
|
615
|
+
function getPrivilegedNamespaceProblem(namespace, labels, backend) {
|
|
616
|
+
if (backend.capabilities || labels?.[ENFORCE_LABEL] !== 'privileged') {
|
|
617
|
+
return undefined;
|
|
618
|
+
}
|
|
619
|
+
return (`the namespace ${namespace} enforces Pod Security privileged, made while the server had ` +
|
|
620
|
+
`backend.capabilities, and the server runs nothing in it without them: kubectl label namespace ` +
|
|
621
|
+
`${namespace} ${ENFORCE_LABEL}=baseline ${AUDIT_LABEL}- --overwrite`);
|
|
622
|
+
}
|
|
623
|
+
exports.getPrivilegedNamespaceProblem = getPrivilegedNamespaceProblem;
|
|
355
624
|
// What may mount a claim a run kept: the runs of the same repository, as they share the
|
|
356
625
|
// cache, and on a sandbox the runs of that sandbox only, whose claim is on the same node or
|
|
357
626
|
// zone as the kept ones. The owner is the namespace's.
|
|
358
627
|
function getVolumeScope(run) {
|
|
359
|
-
if (run.source.type === 'sandbox') {
|
|
628
|
+
if (run.source.type === 'sandbox' && !run.source.check) {
|
|
360
629
|
return `sandbox-${(0, crypto_1.createHash)('sha256').update(run.source.id).digest('hex').slice(0, 24)}`;
|
|
361
630
|
}
|
|
362
631
|
return (0, executor_1.cacheNamespace)(run);
|
|
@@ -370,6 +639,8 @@ class CallerNamespaces {
|
|
|
370
639
|
constructor(instance, backend) {
|
|
371
640
|
this.instance = instance;
|
|
372
641
|
this.backend = backend;
|
|
642
|
+
// the namespaces this process set up, by their uid, and when
|
|
643
|
+
this.applied = new Map();
|
|
373
644
|
}
|
|
374
645
|
name(owner) {
|
|
375
646
|
return getCallerNamespace(this.backend.namespacePrefix, owner);
|
|
@@ -377,9 +648,15 @@ class CallerNamespaces {
|
|
|
377
648
|
// The caller's namespace: made by the server and kept apart from the others. The
|
|
378
649
|
// server binds its rights there, isolates the network of the pods, applies the
|
|
379
650
|
// operator's quota and gives containers without resources default requests, so they
|
|
380
|
-
// are admitted under a quota on compute. All of it is replaced
|
|
381
|
-
// namespace
|
|
382
|
-
|
|
651
|
+
// are admitted under a quota on compute. All of it is replaced when the server sets up
|
|
652
|
+
// a namespace it did not set up in the last `NAMESPACE_SETUP_REUSE_MS` (it reads the
|
|
653
|
+
// config once, at its start), so the namespace follows the config; a namespace made anew
|
|
654
|
+
// since (deleted and made again) is set up again at once. Returns what keeps the tasks that
|
|
655
|
+
// add capabilities from running in it (getCapabilitiesProblem), if anything.
|
|
656
|
+
async prepare(namespace,
|
|
657
|
+
// a service run's own namespace: its label, the namespace of the server that reaches its
|
|
658
|
+
// endpoints, and its quota
|
|
659
|
+
serviceRun) {
|
|
383
660
|
const backend = this.backend;
|
|
384
661
|
let existing;
|
|
385
662
|
try {
|
|
@@ -393,17 +670,33 @@ class CallerNamespaces {
|
|
|
393
670
|
if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
|
|
394
671
|
throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
|
|
395
672
|
}
|
|
673
|
+
const privileged = getPrivilegedNamespaceProblem(namespace, existing?.metadata?.labels, backend);
|
|
674
|
+
if (privileged) {
|
|
675
|
+
throw new Error(privileged);
|
|
676
|
+
}
|
|
677
|
+
// one made by another run of this server at the same time has the labels of its config
|
|
678
|
+
const problem = () => getCapabilitiesProblem(namespace, existing?.metadata?.labels ?? getPodSecurityLabels(backend), backend);
|
|
679
|
+
const started = Date.now();
|
|
680
|
+
const applied = this.applied.get(namespace);
|
|
681
|
+
if (existing &&
|
|
682
|
+
!existing.metadata?.deletionTimestamp &&
|
|
683
|
+
applied &&
|
|
684
|
+
applied.uid === existing.metadata?.uid &&
|
|
685
|
+
started - applied.at < NAMESPACE_SETUP_REUSE_MS) {
|
|
686
|
+
return problem();
|
|
687
|
+
}
|
|
688
|
+
this.applied.delete(namespace);
|
|
396
689
|
if (!existing) {
|
|
397
690
|
try {
|
|
398
|
-
await this.instance.coreApi.createNamespace({
|
|
691
|
+
existing = await this.instance.coreApi.createNamespace({
|
|
399
692
|
body: {
|
|
400
693
|
metadata: {
|
|
401
694
|
name: namespace,
|
|
402
695
|
labels: {
|
|
403
696
|
[MANAGED_LABEL]: 'true',
|
|
404
697
|
'hammerkit.dev/caller': namespace,
|
|
405
|
-
|
|
406
|
-
|
|
698
|
+
...(serviceRun ? { [exports.SERVICE_RUN_LABEL]: serviceRun.serviceRun } : {}),
|
|
699
|
+
...getPodSecurityLabels(backend),
|
|
407
700
|
},
|
|
408
701
|
},
|
|
409
702
|
},
|
|
@@ -419,7 +712,10 @@ class CallerNamespaces {
|
|
|
419
712
|
const [first, ...policies] = getNamespacePolicies(namespace, backend.deniedEgress, backend.allowedEgress);
|
|
420
713
|
// the API server may refuse what the binding allows for a moment after it was made
|
|
421
714
|
await untilAllowed(() => (0, apply_1.replace)(this.instance, first));
|
|
422
|
-
for (const policy of
|
|
715
|
+
for (const policy of [
|
|
716
|
+
...policies,
|
|
717
|
+
...(serviceRun ? [getServerIngressPolicy(namespace, serviceRun.server, serviceRun.ports)] : []),
|
|
718
|
+
]) {
|
|
423
719
|
await (0, apply_1.replace)(this.instance, policy);
|
|
424
720
|
}
|
|
425
721
|
const metadata = {
|
|
@@ -432,7 +728,7 @@ class CallerNamespaces {
|
|
|
432
728
|
apiVersion: 'v1',
|
|
433
729
|
kind: 'ResourceQuota',
|
|
434
730
|
metadata,
|
|
435
|
-
spec: { hard: backend.quota },
|
|
731
|
+
spec: { hard: serviceRun?.quota ?? backend.quota },
|
|
436
732
|
};
|
|
437
733
|
const defaults = backend.defaultResources;
|
|
438
734
|
const limits = {
|
|
@@ -450,7 +746,7 @@ class CallerNamespaces {
|
|
|
450
746
|
},
|
|
451
747
|
};
|
|
452
748
|
for (const [object, wanted] of [
|
|
453
|
-
[quota, !!backend.quota],
|
|
749
|
+
[quota, !!(serviceRun?.quota ?? backend.quota)],
|
|
454
750
|
[limits, !!defaults],
|
|
455
751
|
]) {
|
|
456
752
|
if (wanted) {
|
|
@@ -465,6 +761,11 @@ class CallerNamespaces {
|
|
|
465
761
|
});
|
|
466
762
|
}
|
|
467
763
|
}
|
|
764
|
+
const uid = existing?.metadata?.uid;
|
|
765
|
+
if (uid) {
|
|
766
|
+
this.applied.set(namespace, { uid, at: started });
|
|
767
|
+
}
|
|
768
|
+
return problem();
|
|
468
769
|
}
|
|
469
770
|
whoAmI() {
|
|
470
771
|
if (!this.subject) {
|
|
@@ -487,7 +788,7 @@ class CallerNamespaces {
|
|
|
487
788
|
exports.CallerNamespaces = CallerNamespaces;
|
|
488
789
|
// The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
|
|
489
790
|
// `root`, a source below it is the same path below `subPath` in the claim.
|
|
490
|
-
function getWorkspace(root, claim, namespace) {
|
|
791
|
+
function getWorkspace(root, claim, namespace, runner) {
|
|
491
792
|
if (claim.namespace !== namespace) {
|
|
492
793
|
throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
|
|
493
794
|
}
|
|
@@ -497,6 +798,7 @@ function getWorkspace(root, claim, namespace) {
|
|
|
497
798
|
subPath: claim.subPath,
|
|
498
799
|
label: claim.label,
|
|
499
800
|
...(claim.pod ? { pod: claim.pod } : {}),
|
|
801
|
+
...(runner ? { runner: true } : {}),
|
|
500
802
|
};
|
|
501
803
|
}
|
|
502
804
|
// Everything a run made in the caller's namespace, by the label selector of the run:
|