hammerkit 1.26.0 → 1.27.0-next.10

package/dist/server/kubernetes-backend.js

@@ -0,0 +1,603 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.awaitRunObjectsGone = exports.removeRunObjects = exports.CallerNamespaces = exports.getVolumeScope = exports.KubernetesExecutor = exports.getRunRoleBinding = exports.getNamespacePolicies = exports.getClusterAccess = exports.isCallerNamespace = exports.getCallerNamespace = void 0;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const os_1 = require("os");
+const path_1 = require("path");
+const net_1 = require("net");
+const apply_1 = require("../kubernetes/apply");
+const job_placement_1 = require("../kubernetes/job-placement");
+const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
+const log_1 = require("../log");
+const network_policy_1 = require("../kubernetes/network-policy");
+const sleep_1 = require("../utils/sleep");
+const version_1 = require("../version");
+const executor_1 = require("./executor");
+const kept_claims_1 = require("../kubernetes/kept-claims");
+const units_1 = require("../utils/units");
+const server_config_1 = require("./server-config");
+// the name of the quota and limit range the server keeps in a caller's namespace
+const POLICY_NAME = 'hammerkit';
+const MANAGED_LABEL = 'hammerkit.dev/managed';
+// how long the end of a run waits for its pods and claims to be gone
+const RELEASE_TIMEOUT = 120000;
+// where a pod of the cluster finds the token and certificate of its ServiceAccount
+const IN_CLUSTER_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';
+/**
+ * The namespace of a caller: the prefix and a hash of their issuer and subject. A
+ * subject is the caller's own, so nobody else's runs, claims or quota are in it.
+ */
+function getCallerNamespace(prefix, owner) {
+    return `${prefix}${(0, crypto_1.createHash)('sha256').update(`${owner.iss}\0${owner.sub}`).digest('hex').slice(0, 16)}`;
+}
+exports.getCallerNamespace = getCallerNamespace;
+// Whether a namespace is one getCallerNamespace makes with this prefix: a server named `hk`
+// passes over the namespaces of a server named `hk-test` on the same cluster.
+function isCallerNamespace(prefix, name) {
+    return name.startsWith(prefix) && /^[0-9a-f]{16}$/.test(name.slice(prefix.length));
+}
+exports.isCallerNamespace = isCallerNamespace;
+// How the server and the processes of its runs reach the cluster: a kubeconfig and
+// context the operator names, else the server's default kubeconfig, else the cluster
+// the server runs in (its pod's ServiceAccount, found through the variables
+// Kubernetes sets in every pod, which a run's process gets too).
+function getClusterAccess(backend, processEnvs) {
+    if (backend.kubeconfig) {
+        return { kubeConfig: backend.kubeconfig, context: backend.context, env: {} };
+    }
+    const fallback = processEnvs.KUBECONFIG?.split(process.platform === 'win32' ? ';' : ':')[0];
+    const kubeConfig = fallback || (0, path_1.join)(processEnvs.HOME ?? (0, os_1.homedir)(), '.kube', 'config');
+    if ((0, fs_1.existsSync)(kubeConfig)) {
+        return { kubeConfig, context: backend.context, env: {} };
+    }
+    const env = {};
+    for (const name of ['KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT']) {
+        const value = processEnvs[name];
+        if (value) {
+            env[name] = value;
+        }
+    }
+    return { context: backend.context, env };
+}
+exports.getClusterAccess = getClusterAccess;
+function readInClusterNamespace() {
+    try {
+        return (0, fs_1.readFileSync)(IN_CLUSTER_NAMESPACE, 'utf8').trim() || undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The NetworkPolicies of a caller's namespace. A namespace is no network boundary on its
+ * own: without them a task could connect to the pods of other callers, to the server
+ * and to the metadata endpoint of the cloud, which hands out the node's identity.
+ * - ingress of a pod with an open network: connections from the pods of its namespace
+ *   only (the services a task needs)
+ * - egress of a pod with an open network, and of an egress proxy: its namespace, DNS,
+ *   and any address but the denied ranges (link-local and private by default), plus the
+ *   allowed ones
+ * Policies add up, so neither selects the pods of a limited item: its own policy (see
+ * buildNetworkPolicy) is all it may reach, DNS left out, and all that may reach it. An
+ * egress proxy takes connections from its item only (buildProxyPolicy).
+ */
+function getNamespacePolicies(namespace, deniedEgress, allowedEgress = []) {
+    const metadata = (name) => ({
+        name,
+        namespace,
+        annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+        labels: { [MANAGED_LABEL]: 'true' },
+    });
+    const block = (cidr, except) => ({ ipBlock: { cidr, ...(except.length ? { except } : {}) } });
+    return [
+        {
+            apiVersion: 'networking.k8s.io/v1',
+            kind: 'NetworkPolicy',
+            metadata: metadata('hammerkit-ingress'),
+            spec: {
+                podSelector: { matchExpressions: [{ key: network_policy_1.NETWORK_LABEL, operator: 'DoesNotExist' }] },
+                policyTypes: ['Ingress'],
+                ingress: [{ _from: [{ podSelector: {} }] }],
+            },
+        },
+        {
+            apiVersion: 'networking.k8s.io/v1',
+            kind: 'NetworkPolicy',
+            metadata: metadata('hammerkit-egress'),
+            spec: {
+                podSelector: { matchExpressions: [{ key: network_policy_1.NETWORK_LABEL, operator: 'NotIn', values: [network_policy_1.NETWORK_LIMITED] }] },
+                policyTypes: ['Egress'],
+                egress: [
+                    { to: [{ podSelector: {} }] },
+                    // to whatever resolver the pod is given, a node-local cache included
+                    {
+                        ports: [
+                            { protocol: 'UDP', port: 53 },
+                            { protocol: 'TCP', port: 53 },
+                        ],
+                    },
+                    {
+                        to: [
+                            block('0.0.0.0/0', deniedEgress.filter((range) => (0, net_1.isIPv4)(range.split('/')[0]))),
+                            block('::/0', deniedEgress.filter((range) => !(0, net_1.isIPv4)(range.split('/')[0]))),
+                        ],
+                    },
+                    // the operator's exceptions, which rules add up to
+                    ...(allowedEgress.length > 0 ? [{ to: allowedEgress.map((cidr) => ({ ipBlock: { cidr } })) }] : []),
+                ],
+            },
+        },
+        {
+            apiVersion: 'networking.k8s.io/v1',
+            kind: 'NetworkPolicy',
+            metadata: metadata('hammerkit-proxy-ingress'),
+            spec: {
+                // an egress proxy takes only what its own policy lets in: one whose policy went
+                // before the pod ended takes nothing
+                podSelector: { matchLabels: { [network_policy_1.NETWORK_LABEL]: network_policy_1.NETWORK_PROXY } },
+                policyTypes: ['Ingress'],
+                ingress: [],
+            },
+        },
+    ];
+}
+exports.getNamespacePolicies = getNamespacePolicies;
+/**
+ * The server's rights in a caller's namespace: the operator's ClusterRole, bound there
+ * only, so the server needs no rights on Secrets or pods anywhere else.
+ */
+function getRunRoleBinding(namespace, role, subject) {
+    return {
+        apiVersion: 'rbac.authorization.k8s.io/v1',
+        kind: 'RoleBinding',
+        metadata: {
+            name: 'hammerkit-runs',
+            namespace,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: { [MANAGED_LABEL]: 'true' },
+        },
+        roleRef: { apiGroup: 'rbac.authorization.k8s.io', kind: 'ClusterRole', name: role },
+        subjects: [subject],
+    };
+}
+exports.getRunRoleBinding = getRunRoleBinding;
+/**
+ * Runs the tasks of runs as Jobs on a cluster. A run is planned and driven by a
+ * process of the server as on a host (see HostExecutor); what changes is where its
+ * container tasks and services go: into the caller's namespace, on the nodes of the
+ * operator's pools, with the secrets of providers read in the Job itself. The runs
+ * of a caller execute side by side in their namespace, up to `runs.limits.running`:
+ * the objects of a run are named and found by its run label (getItemKey), so two runs
+ * of the same task have their own. What two runs share are the claims the server keeps
+ * (`backend.keepVolumes`) of a scope (getVolumeScope): the runs of one scope go one at
+ * a time (exclusive). When a run ends, everything made for it is removed, but for the
+ * kept claims, which go once they expired.
+ */
+class KubernetesExecutor {
+    constructor(config, processEnvs, options) {
+        this.config = config;
+        // per caller namespace, what changes objects of the whole namespace: its setup, the
+        // removal of what a stopped server left, the sweeps of kept claims. The last one waiting.
+        this.queues = new Map();
+        // per caller namespace, the runs that execute in it: by run label, their volume scope
+        this.active = new Map();
+        // per caller namespace, when its kept claims are next looked at
+        this.sweeps = new Map();
+        // whether the kept claims a server left when it stopped were looked for
+        this.swept = false;
+        this.access = getClusterAccess(config.backend, processEnvs);
+        this.instance = (0, kubernetes_instance_1.createKubernetesInstances)({
+            type: 'kubernetes',
+            namespace: 'default',
+            kubeConfig: this.access.kubeConfig,
+            context: this.access.context,
+            httpRoutes: [],
+            networkPolicyEnforced: config.backend.networkPolicy === 'enforced',
+        });
+        const namespace = config.backend.namespace ?? readInClusterNamespace();
+        if (!namespace) {
+            throw new Error('backend.namespace: name the namespace of the server, where the credentials of the accounts are; ' +
+                'it is only found on its own in a pod of the cluster');
+        }
+        this.namespace = namespace;
+        this.namespaces = new CallerNamespaces(this.instance, config.backend);
+        this.keep = config.backend.keepVolumes === false ? 0 : (0, units_1.parseDuration)(config.backend.keepVolumes);
+        this.log = options.log;
+        this.host = new executor_1.HostExecutor({ ...options, placement: this.placement() });
+    }
+    async execute(run, context) {
+        const namespace = getCallerNamespace(this.config.backend.namespacePrefix, run.owner);
+        const label = (0, job_placement_1.getRunLabelValue)(run.id);
+        let runs = this.active.get(namespace);
+        if (!runs) {
+            runs = new Map();
+            this.active.set(namespace, runs);
+        }
+        runs.set(label, getVolumeScope(run));
+        try {
+            await this.host.execute(run, context);
+        }
+        finally {
+            runs.delete(label);
+            if (runs.size === 0 && this.active.get(namespace) === runs) {
+                this.active.delete(namespace);
+            }
+        }
+    }
+    // The runs of a scope share its kept claims, which one of them may change at a time:
+    // what a task finds in one is what the last run left there. Without kept claims, runs
+    // share no object and nothing is held.
+    exclusive(run) {
+        if (this.keep === 0) {
+            return undefined;
+        }
+        return `${getCallerNamespace(this.config.backend.namespacePrefix, run.owner)}/${getVolumeScope(run)}`;
+    }
+    // The scopes of the runs that execute in the namespace, but `except`'s: their kept claims
+    // are theirs to mount and stay. A server that keeps none mounts none.
+    scopesInUse(namespace, except) {
+        if (this.keep === 0) {
+            return new Set();
+        }
+        return new Set([...(this.active.get(namespace) ?? new Map()).entries()]
+            .filter(([label]) => label !== except)
+            .map(([, scope]) => scope));
+    }
+    // What changes objects of a whole namespace happens one at a time, not what its runs do
+    enqueue(namespace, fn) {
+        const previous = this.queues.get(namespace) ?? Promise.resolve();
+        const execution = previous.then(fn);
+        const settled = execution.then(() => undefined, () => undefined);
+        this.queues.set(namespace, settled);
+        settled.then(() => {
+            if (this.queues.get(namespace) === settled) {
+                this.queues.delete(namespace);
+            }
+        });
+        return execution;
+    }
+    placement() {
+        const backend = this.config.backend;
+        return {
+            prepare: async (run, context, work) => {
+                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
+                const label = (0, job_placement_1.getRunLabelValue)(run.id);
+                await this.enqueue(namespace, async () => {
+                    if (context.signal.aborted) {
+                        throw new Error('the run was cancelled');
+                    }
+                    await this.namespaces.prepare(namespace);
+                    // what an earlier run left when the server stopped before its end: the objects of
+                    // a run that does not execute. One that starts after this waits for it here
+                    // before it makes any.
+                    const running = [...new Set([label, ...(this.active.get(namespace)?.keys() ?? [])])];
+                    await removeRunObjects(this.instance, namespace, `${job_placement_1.RUN_LABEL},${job_placement_1.RUN_LABEL} notin (${running.join(',')})`, RELEASE_TIMEOUT);
+                    // the claims earlier runs kept that expired; the others this run may mount
+                    const { next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
+                        now: new Date(),
+                        keep: this.keep > 0,
+                        timeout: RELEASE_TIMEOUT,
+                        inUse: this.scopesInUse(namespace, label),
+                    });
+                    if (next) {
+                        this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+                    }
+                });
+                this.sweepLeftClaims();
+                context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `runs in the namespace ${namespace}` });
+                const kubernetes = {
+                    type: 'kubernetes',
+                    namespace,
+                    kubeConfig: this.access.kubeConfig,
+                    context: this.access.context,
+                    httpRoutes: [],
+                    storageClass: backend.storageClass,
+                    volumeSize: backend.volumeSize,
+                    maxVolumeSize: (0, server_config_1.getMaxVolumeSize)(backend),
+                    networkPolicyEnforced: backend.networkPolicy === 'enforced',
+                    server: {
+                        run: run.id,
+                        namespace: this.namespace,
+                        pools: backend.pools,
+                        providers: backend.providers,
+                        accounts: Object.fromEntries(Object.entries(this.config.accounts).flatMap(([name, account]) => account.kubernetes ? [[name, account.kubernetes]] : [])),
+                        ...(work?.claim ? { workspace: getWorkspace(work.root, work.claim, namespace) } : {}),
+                        ...(this.keep > 0 ? { volumeScope: getVolumeScope(run) } : {}),
+                    },
+                };
+                return { kubernetes, env: this.access.env };
+            },
+            release: async (run, context) => {
+                const namespace = getCallerNamespace(backend.namespacePrefix, run.owner);
+                try {
+                    const selector = `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run.id)}`;
+                    await removeRunObjects(this.instance, namespace, selector, RELEASE_TIMEOUT);
+                }
+                catch (e) {
+                    context.emit({
+                        type: 'log',
+                        task: 'hammerkit',
+                        stream: 'stderr',
+                        text: `what the run made in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`,
+                    });
+                }
+                if (this.keep > 0) {
+                    try {
+                        await (0, kept_claims_1.expireKeptClaims)(this.instance, namespace, (0, job_placement_1.getRunLabelValue)(run.id), new Date(Date.now() + this.keep));
+                    }
+                    catch (e) {
+                        // without an expiry, the next look at the namespace removes them
+                        context.emit({
+                            type: 'log',
+                            task: 'hammerkit',
+                            stream: 'stderr',
+                            text: `the claims the run kept in ${namespace} could not be given their expiry: ${(0, log_1.getErrorMessage)(e)}`,
+                        });
+                    }
+                    this.sweepLater(namespace, this.keep);
+                }
+            },
+        };
+    }
+    // The kept claims of a namespace that expired go at the soonest expiry the server knows
+    // of, but those of a scope a run of the caller executes in
+    sweepLater(namespace, delay) {
+        const at = Date.now() + delay + 1000;
+        const pending = this.sweeps.get(namespace);
+        if (pending && pending.at <= at) {
+            return;
+        }
+        clearTimeout(pending?.timer);
+        const timer = setTimeout(() => {
+            this.sweeps.delete(namespace);
+            this.enqueue(namespace, () => this.sweep(namespace)).catch((e) => this.log?.(`the kept claims in ${namespace} could not be removed: ${(0, log_1.getErrorMessage)(e)}`));
+        }, at - Date.now());
+        timer.unref();
+        this.sweeps.set(namespace, { at, timer });
+    }
+    async sweep(namespace) {
+        const { removed, next } = await (0, kept_claims_1.sweepKeptClaims)(this.instance, namespace, {
+            now: new Date(),
+            keep: this.keep > 0,
+            timeout: RELEASE_TIMEOUT,
+            inUse: this.scopesInUse(namespace),
+        });
+        if (removed.length > 0) {
+            this.log?.(`removed ${removed.length} kept claims in ${namespace}`);
+        }
+        if (next) {
+            this.sweepLater(namespace, Math.max(0, next.getTime() - Date.now()));
+        }
+    }
+    // Once, on the first run: the claims the server kept before it stopped, in the namespaces
+    // of other callers too, whose next run may be far off
+    sweepLeftClaims() {
+        if (this.swept) {
+            return;
+        }
+        this.swept = true;
+        Promise.resolve()
+            .then(() => this.instance.coreApi.listNamespace({ labelSelector: `${MANAGED_LABEL}=true` }))
+            .then((namespaces) => {
+            for (const namespace of namespaces.items) {
+                const name = namespace.metadata?.name;
+                if (name &&
+                    isCallerNamespace(this.config.backend.namespacePrefix, name) &&
+                    !namespace.metadata?.deletionTimestamp) {
+                    this.sweepLater(name, 0);
+                }
+            }
+        })
+            .catch((e) => this.log?.(`the namespaces of callers could not be listed: ${(0, log_1.getErrorMessage)(e)}`));
+    }
+}
+exports.KubernetesExecutor = KubernetesExecutor;
+// What may mount a claim a run kept: the runs of the same repository, as they share the
+// cache, and on a sandbox the runs of that sandbox only, whose claim is on the same node or
+// zone as the kept ones. The owner is the namespace's.
+function getVolumeScope(run) {
+    if (run.source.type === 'sandbox') {
+        return `sandbox-${(0, crypto_1.createHash)('sha256').update(run.source.id).digest('hex').slice(0, 24)}`;
+    }
+    return (0, executor_1.cacheNamespace)(run);
+}
+exports.getVolumeScope = getVolumeScope;
+/**
+ * The namespaces of callers on the cluster: named by the caller (getCallerNamespace),
+ * made on first use, with the operator's quota.
+ */
+class CallerNamespaces {
+    constructor(instance, backend) {
+        this.instance = instance;
+        this.backend = backend;
+    }
+    name(owner) {
+        return getCallerNamespace(this.backend.namespacePrefix, owner);
+    }
+    // The caller's namespace: made by the server and kept apart from the others. The
+    // server binds its rights there, isolates the network of the pods, applies the
+    // operator's quota and gives containers without resources default requests, so they
+    // are admitted under a quota on compute. All of it is replaced on every run, so the
+    // namespace follows the config.
+    async prepare(namespace) {
+        const backend = this.backend;
+        let existing;
+        try {
+            existing = await this.instance.coreApi.readNamespace({ name: namespace });
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                throw e;
+            }
+        }
+        if (existing && existing.metadata?.labels?.[MANAGED_LABEL] !== 'true') {
+            throw new Error(`the namespace ${namespace} was not made by hammerkit, the server does not run tasks in it`);
+        }
+        if (!existing) {
+            try {
+                await this.instance.coreApi.createNamespace({
+                    body: {
+                        metadata: {
+                            name: namespace,
+                            labels: {
+                                [MANAGED_LABEL]: 'true',
+                                'hammerkit.dev/caller': namespace,
+                                // no privileged pods, host paths or host network, whatever a build asks
+                                'pod-security.kubernetes.io/enforce': 'baseline',
+                            },
+                        },
+                    },
+                });
+            }
+            catch (createError) {
+                if ((0, apply_1.statusCodeOf)(createError) !== 409) {
+                    throw createError;
+                }
+            }
+        }
+        await (0, apply_1.replace)(this.instance, getRunRoleBinding(namespace, backend.runRole, await this.whoAmI()));
+        const [first, ...policies] = getNamespacePolicies(namespace, backend.deniedEgress, backend.allowedEgress);
+        // the API server may refuse what the binding allows for a moment after it was made
+        await untilAllowed(() => (0, apply_1.replace)(this.instance, first));
+        for (const policy of policies) {
+            await (0, apply_1.replace)(this.instance, policy);
+        }
+        const metadata = {
+            name: POLICY_NAME,
+            namespace,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: { [MANAGED_LABEL]: 'true' },
+        };
+        const quota = {
+            apiVersion: 'v1',
+            kind: 'ResourceQuota',
+            metadata,
+            spec: { hard: backend.quota },
+        };
+        const defaults = backend.defaultResources;
+        const limits = {
+            apiVersion: 'v1',
+            kind: 'LimitRange',
+            metadata,
+            spec: {
+                limits: [
+                    {
+                        type: 'Container',
+                        ...(defaults?.requests ? { defaultRequest: defaults.requests } : {}),
+                        ...(defaults?.limits ? { _default: defaults.limits } : {}),
+                    },
+                ],
+            },
+        };
+        for (const [object, wanted] of [
+            [quota, !!backend.quota],
+            [limits, !!defaults],
+        ]) {
+            if (wanted) {
+                await (0, apply_1.replace)(this.instance, object);
+            }
+            else {
+                // one the config had before
+                await this.instance.objectApi.delete(object).catch((e) => {
+                    if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                        throw e;
+                    }
+                });
+            }
+        }
+    }
+    whoAmI() {
+        if (!this.subject) {
+            this.subject = this.instance.authApi
+                .createSelfSubjectReview({ body: { apiVersion: 'authentication.k8s.io/v1', kind: 'SelfSubjectReview' } })
+                .then((review) => {
+                const name = review.status?.userInfo?.username;
+                if (!name) {
+                    throw new Error('the cluster did not say who the server is (SelfSubjectReview)');
+                }
+                return { kind: 'User', apiGroup: 'rbac.authorization.k8s.io', name };
+            });
+            this.subject.catch(() => {
+                this.subject = undefined;
+            });
+        }
+        return this.subject;
+    }
+}
+exports.CallerNamespaces = CallerNamespaces;
+// The claim of a sandbox as the Jobs of its run mount it: the run's copy of the files is
+// `root`, a source below it is the same path below `subPath` in the claim.
+function getWorkspace(root, claim, namespace) {
+    if (claim.namespace !== namespace) {
+        throw new Error(`the sandbox is in ${claim.namespace}, the run in ${namespace}`);
+    }
+    return {
+        root,
+        claim: claim.name,
+        subPath: claim.subPath,
+        label: claim.label,
+        ...(claim.pod ? { pod: claim.pod } : {}),
+    };
+}
+// Everything a run made in the caller's namespace, by the label selector of the run:
+// Jobs with their pods, Deployments (services and egress proxies), Services, claims,
+// Secrets and the NetworkPolicies of limited networks, which go last so no pod of the
+// run outlives its policy.
+// Deletes what a run made. The NetworkPolicies go last, once the pods are: a limited
+// pod's own policy is all that limits it.
+async function removeRunObjects(instance, namespace, labelSelector, timeout) {
+    const selector = { namespace, labelSelector };
+    await instance.batchApi.deleteCollectionNamespacedJob({ ...selector, propagationPolicy: 'Background' });
+    await instance.appsApi.deleteCollectionNamespacedDeployment({ ...selector, propagationPolicy: 'Background' });
+    await instance.coreApi.deleteCollectionNamespacedPod({ ...selector, gracePeriodSeconds: 0 });
+    await instance.coreApi.deleteCollectionNamespacedService(selector);
+    await instance.coreApi.deleteCollectionNamespacedSecret(selector);
+    await instance.coreApi.deleteCollectionNamespacedPersistentVolumeClaim(selector);
+    // the next run of the caller also uses the same claim names
+    await awaitRunObjectsGone(instance, namespace, labelSelector, timeout);
+    const policies = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', namespace, undefined, undefined, undefined, undefined, labelSelector);
+    for (const policy of policies.items) {
+        // the items of a list do not say their kind
+        const metadata = { name: policy.metadata?.name, namespace };
+        await instance.objectApi
+            .delete({ apiVersion: 'networking.k8s.io/v1', kind: 'NetworkPolicy', metadata })
+            .catch((e) => {
+            if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                throw e;
+            }
+        });
+    }
+}
+exports.removeRunObjects = removeRunObjects;
+async function untilAllowed(fn, attempts = 20) {
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) !== 403 || attempt >= attempts) {
+                throw e;
+            }
+            await (0, sleep_1.sleep)(500);
+        }
+    }
+}
+// Until the pods and claims of a run are gone: a claim is kept while a pod uses it.
+async function awaitRunObjectsGone(instance, namespace, labelSelector, timeout) {
+    const deadline = Date.now() + timeout;
+    for (;;) {
+        const pods = await instance.coreApi.listNamespacedPod({ namespace, labelSelector });
+        const claims = await instance.coreApi.listNamespacedPersistentVolumeClaim({ namespace, labelSelector });
+        const left = [...pods.items, ...claims.items].map((item) => `${item.kind ?? ''} ${item.metadata?.name}`.trim());
+        if (left.length === 0) {
+            return;
+        }
+        if (Date.now() > deadline) {
+            throw new Error(`still there after ${Math.round(timeout / 1000)}s: ${left.join(', ')}`);
+        }
+        await (0, sleep_1.sleep)(1000);
+    }
+}
+exports.awaitRunObjectsGone = awaitRunObjectsGone;
+//# sourceMappingURL=kubernetes-backend.js.map