hammerkit 1.26.0 → 1.27.0-next.10

package/dist/kubernetes/kept-claims.js

@@ -0,0 +1,203 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evictKeptClaims = exports.sweepKeptClaims = exports.expireKeptClaims = exports.recordClaimState = exports.forgetClaimState = exports.useClaimState = exports.isKeptClaim = exports.getClaimName = exports.EXPIRES_ANNOTATION = exports.STATE_ANNOTATION = exports.KEPT_BY_LABEL = exports.VOLUME_SCOPE_LABEL = void 0;
+const crypto_1 = require("crypto");
+const apply_1 = require("./apply");
+const sleep_1 = require("../utils/sleep");
+const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
+// On a server with `backend.keepVolumes`, the claim of a task stays after the run, so the
+// next run of the same source finds the outputs in it and mounts them without restoring
+// them from the cache. A kept claim has no run label (the end of the run removes what has
+// one), but these:
+// the trust of the source of the runs that may mount it (see getVolumeScope)
+exports.VOLUME_SCOPE_LABEL = 'hammerkit.dev/volume-scope';
+// the run label of the run that mounted it last; while it runs, the claim does not expire
+exports.KEPT_BY_LABEL = 'hammerkit.dev/kept-by';
+// the state key of the outputs in it, set once the task completed or they were restored
+exports.STATE_ANNOTATION = 'hammerkit.dev/state';
+// when it goes, set by the end of the run that mounted it last
+exports.EXPIRES_ANNOTATION = 'hammerkit.dev/expires';
+// The claim of a work item: a task's on a server that keeps them carries the scope, so a
+// run of another source never mounts it, and the runs of one scope go one at a time
+// (KubernetesExecutor.exclusive). Any other claim on a server is the run's own, as the
+// runs of a caller share their namespace. 63 characters at most, a pod's volume name.
+function getClaimName(work, kubernetes) {
+    const scope = kubernetes.server?.volumeScope;
+    if (scope && isTask(work)) {
+        return `hammerkit-${work.id()}-${(0, crypto_1.createHash)('sha256').update(scope).digest('hex').slice(0, 12)}`;
+    }
+    return 'hammerkit-' + (0, resources_1.getItemKey)(work, kubernetes);
+}
+exports.getClaimName = getClaimName;
+// whether the claim of the work item stays after the run
+function isKeptClaim(work, kubernetes) {
+    return !!kubernetes.server?.volumeScope && isTask(work);
+}
+exports.isKeptClaim = isKeptClaim;
+function isTask(work) {
+    return work.data.type === 'container-task';
+}
+async function readClaim(instance, namespace, name) {
+    try {
+        return await instance.coreApi.readNamespacedPersistentVolumeClaim({ name, namespace });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return null;
+        }
+        throw e;
+    }
+}
+// The state key of the outputs a kept claim holds, null when it holds none that are whole.
+// The claim is the run's from now: it does not expire or make way for another claim
+// before the run ended, so outputs the run found in it are still there when it mounts them.
+async function useClaimState(instance, kubernetes, name) {
+    let claim;
+    try {
+        claim = await instance.objectApi.patch({
+            apiVersion: 'v1',
+            kind: 'PersistentVolumeClaim',
+            metadata: {
+                name,
+                namespace: kubernetes.namespace,
+                labels: { [exports.KEPT_BY_LABEL]: (0, job_placement_1.getRunLabelValue)(kubernetes.server?.run ?? '') },
+                // null removes it
+                annotations: { [exports.EXPIRES_ANNOTATION]: null },
+            },
+        });
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return null;
+        }
+        throw e;
+    }
+    // one being removed holds nothing to mount, nor one of another source (ensured anyway
+    // by the hash of the source in its name)
+    if (claim.metadata?.deletionTimestamp ||
+        claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] !== kubernetes.server?.volumeScope) {
+        return null;
+    }
+    return claim.metadata?.annotations?.[exports.STATE_ANNOTATION] ?? null;
+}
+exports.useClaimState = useClaimState;
+async function annotate(instance, namespace, name, annotations) {
+    try {
+        await instance.objectApi.patch({
+            apiVersion: 'v1',
+            kind: 'PersistentVolumeClaim',
+            metadata: { name, namespace, annotations: annotations },
+        });
+        return true;
+    }
+    catch (e) {
+        if ((0, apply_1.statusCodeOf)(e) === 404) {
+            return false;
+        }
+        throw e;
+    }
+}
+// Before a task writes to its kept claim: the outputs in it stop being those of a state.
+// Whether the claim was there, with what an earlier run left in it.
+function forgetClaimState(instance, kubernetes, name) {
+    return annotate(instance, kubernetes.namespace, name, { [exports.STATE_ANNOTATION]: null });
+}
+exports.forgetClaimState = forgetClaimState;
+// Once the outputs of `stateKey` are whole in the kept claim
+async function recordClaimState(instance, kubernetes, name, stateKey) {
+    await annotate(instance, kubernetes.namespace, name, { [exports.STATE_ANNOTATION]: stateKey });
+}
+exports.recordClaimState = recordClaimState;
+// The kept claims in a namespace
+async function listKeptClaims(instance, namespace, labelSelector = exports.VOLUME_SCOPE_LABEL) {
+    const claims = await instance.coreApi.listNamespacedPersistentVolumeClaim({ namespace, labelSelector });
+    return claims.items.filter((claim) => !claim.metadata?.deletionTimestamp);
+}
+async function removeClaims(instance, namespace, claims, timeout) {
+    const names = [];
+    for (const claim of claims) {
+        const name = claim.metadata?.name;
+        if (!name) {
+            continue;
+        }
+        try {
+            // only as it was listed: one a run took for itself since (useClaimState) stays
+            await instance.coreApi.deleteNamespacedPersistentVolumeClaim({
+                name,
+                namespace,
+                body: { preconditions: { resourceVersion: claim.metadata?.resourceVersion } },
+            });
+            names.push(name);
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) === 404) {
+                names.push(name);
+            }
+            else if ((0, apply_1.statusCodeOf)(e) !== 409) {
+                throw e;
+            }
+        }
+    }
+    // a run mounts a claim of the same name again
+    const deadline = Date.now() + timeout;
+    for (const name of names) {
+        while (await readClaim(instance, namespace, name)) {
+            if (Date.now() > deadline) {
+                throw new Error(`the claim ${name} is still there after ${Math.round(timeout / 1000)}s`);
+            }
+            await (0, sleep_1.sleep)(1000);
+        }
+    }
+    return names;
+}
+// At the end of a run: the claims it kept go `keep` milliseconds from now, unless a run
+// mounts them before
+async function expireKeptClaims(instance, namespace, runLabel, expires) {
+    for (const claim of await listKeptClaims(instance, namespace, `${exports.KEPT_BY_LABEL}=${runLabel}`)) {
+        if (claim.metadata?.name) {
+            await annotate(instance, namespace, claim.metadata.name, { [exports.EXPIRES_ANNOTATION]: expires.toISOString() });
+        }
+    }
+}
+exports.expireKeptClaims = expireKeptClaims;
+// The kept claims that expired, and those a run left without an expiry because the server
+// stopped before its end (all of them when the server keeps none any more), but those of
+// the scopes `inUse`: a run of the scope executes and mounts them, or may in a moment.
+// The names of those removed, and when the next of the others expires.
+async function sweepKeptClaims(instance, namespace, options) {
+    const expired = [];
+    let next = null;
+    for (const claim of await listKeptClaims(instance, namespace)) {
+        if (options.inUse?.has(claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] ?? '')) {
+            continue;
+        }
+        const expires = Date.parse(claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION] ?? '');
+        if (!options.keep || !(expires > options.now.getTime())) {
+            expired.push(claim);
+        }
+        else if (next === null || expires < next) {
+            next = expires;
+        }
+    }
+    const removed = await removeClaims(instance, namespace, expired, options.timeout);
+    return { removed, next: next === null ? null : new Date(next) };
+}
+exports.sweepKeptClaims = sweepKeptClaims;
+// When a run needs room under the quota of its namespace: the kept claims it does not use
+// go, those of the other sources and those of its own a task of this run did not take (one
+// renamed or changed since). Of another scope, only one its last run released, which has an
+// expiry: one without is mounted by a run of the caller that executes now (or was left by
+// a server that stopped, which the next sweep removes). The names of those removed.
+async function evictKeptClaims(instance, kubernetes, timeout) {
+    const server = kubernetes.server;
+    if (!server?.volumeScope) {
+        return [];
+    }
+    const unused = await listKeptClaims(instance, kubernetes.namespace, `${exports.VOLUME_SCOPE_LABEL},${exports.KEPT_BY_LABEL}!=${(0, job_placement_1.getRunLabelValue)(server.run)}`);
+    const released = unused.filter((claim) => claim.metadata?.labels?.[exports.VOLUME_SCOPE_LABEL] === server.volumeScope ||
+        !!claim.metadata?.annotations?.[exports.EXPIRES_ANNOTATION]);
+    return removeClaims(instance, kubernetes.namespace, released, timeout);
+}
+exports.evictKeptClaims = evictKeptClaims;
+//# sourceMappingURL=kept-claims.js.map

package/dist/kubernetes/kubernetes-instance.js

@@ -12,23 +12,31 @@ function createKubernetesInstances(kubernetes) {
         else {
             kc.loadFromDefault();
         }
-        kc.setCurrentContext(kubernetes.context);
+        if (kubernetes.context) {
+            kc.setCurrentContext(kubernetes.context);
+        }
         const coreApi = kc.makeApiClient(client_node_1.CoreV1Api);
         const batchApi = kc.makeApiClient(client_node_1.BatchV1Api);
         const appsApi = kc.makeApiClient(client_node_1.AppsV1Api);
         const objectApi = kc.makeApiClient(client_node_1.KubernetesObjectApi);
+        const authApi = kc.makeApiClient(client_node_1.AuthenticationV1Api);
         const exec = new client_node_1.Exec(kc);
         return {
             objectApi,
+            log: new client_node_1.Log(kc),
             exec,
             appsApi,
             coreApi,
             batchApi,
+            authApi,
         };
     }
     catch (e) {
         if ((0, log_1.getErrorMessage)(e) === 'No active cluster!') {
-            if (kubernetes.kubeConfig) {
+            if (!kubernetes.context) {
+                throw new Error('No cluster found: no kubeconfig names a current context, and this is not a pod of a cluster');
+            }
+            else if (kubernetes.kubeConfig) {
                 throw new Error(`No cluster found for context ${kubernetes.context} in ${kubernetes.kubeConfig}`);
             }
             else {

package/dist/kubernetes/network-gate.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNetworkGateContainer = exports.HELPER_SECURITY_CONTEXT = exports.NETWORK_GATE_SCRIPT = exports.NETWORK_GATE_CONTAINER = exports.NETWORK_GATE_TIMEOUT_MS = void 0;
+const egress_proxy_1 = require("../network/egress-proxy");
+// Some network plugins (kind's kindnet among them) enforce a NetworkPolicy a
+// moment after a pod started, so a limited pod's first connections could
+// still get out. The gate is the pod's first init container: it waits until
+// the pod no longer reaches the cluster DNS, which the policy of every limited
+// item denies, so the item's own containers start only once the policy holds.
+// A policy that never takes hold fails the pod instead of running it open.
+// A script for `node -e` in the proxy's image, like the proxy.
+exports.NETWORK_GATE_TIMEOUT_MS = 60000;
+// the name of the gate's init container
+exports.NETWORK_GATE_CONTAINER = 'hammerkit-network';
+// Kept free of anything but Node's own modules.
+exports.NETWORK_GATE_SCRIPT = `
+const fs = require('fs')
+const net = require('net')
+const resolvConf = process.env.HAMMERKIT_GATE_RESOLV_CONF || '/etc/resolv.conf'
+const port = parseInt(process.env.HAMMERKIT_GATE_PORT || '53', 10)
+const deadline = Date.now() + parseInt(process.env.HAMMERKIT_GATE_TIMEOUT_MS || '${exports.NETWORK_GATE_TIMEOUT_MS}', 10)
+function fail(message) {
+  process.stderr.write('hammerkit: ' + message + '\\n')
+  try {
+    fs.writeFileSync('/dev/termination-log', 'hammerkit: ' + message)
+  } catch (e) {}
+  process.exit(1)
+}
+let conf = ''
+try {
+  conf = fs.readFileSync(resolvConf, 'utf8')
+} catch (e) {}
+const match = /^nameserver\\s+(\\S+)/m.exec(conf)
+if (!match) {
+  fail('the pod has no cluster DNS, so its network policy can not be checked')
+}
+const nameserver = match[1]
+function attempt() {
+  const socket = net.connect({ host: nameserver, port, timeout: 2000 })
+  socket.once('connect', () => {
+    socket.destroy()
+    if (Date.now() > deadline) {
+      fail('the network policy of this pod is not enforced, it still reaches ' + nameserver)
+    }
+    setTimeout(attempt, 500)
+  })
+  // dropped or refused: the policy holds
+  socket.once('timeout', () => process.exit(0))
+  socket.once('error', () => process.exit(0))
+}
+attempt()
+`;
+exports.HELPER_SECURITY_CONTEXT = {
+    runAsNonRoot: true,
+    // the image's `node` user
+    runAsUser: 1000,
+    allowPrivilegeEscalation: false,
+    readOnlyRootFilesystem: true,
+    capabilities: { drop: ['ALL'] },
+};
+function getNetworkGateContainer() {
+    return {
+        name: exports.NETWORK_GATE_CONTAINER,
+        image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        command: ['node', '-e', exports.NETWORK_GATE_SCRIPT],
+        securityContext: exports.HELPER_SECURITY_CONTEXT,
+        resources: { limits: { memory: '64Mi' } },
+    };
+}
+exports.getNetworkGateContainer = getNetworkGateContainer;
+//# sourceMappingURL=network-gate.js.map