hammerkit 2.0.0-next.7 → 2.0.0-next.8

package/dist/kubernetes/await-running-state.js

@@ -6,6 +6,7 @@ const abort_event_1 = require("../utils/abort-event");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
 const cluster_state_1 = require("./cluster-state");
+const quota_wait_1 = require("./quota-wait");
 // The waits below poll instead of using the client's Watch: Watch aborts every
 // request after 30s, which a pod or deployment still pulling its image outlasts.
 async function awaitRunningState(instance, env, name, phase, pollInterval = 1000) {
@@ -22,11 +23,16 @@ async function awaitRunningState(instance, env, name, phase, pollInterval = 1000
     }
 }
 exports.awaitRunningState = awaitRunningState;
-// Wait for a job to finish by polling its status: resolves once it succeeded,
-// rejects when it failed (its `Failed` condition, i.e. retries are exhausted).
-// On abort the job is deleted — with its pods — and an AbortError is thrown, so
-// a cancelled or timed-out task leaves nothing running on the cluster.
-async function awaitJobCompletion(instance, env, name, abort, pollInterval = 1000) {
+async function awaitJobCompletion(instance, env, name, abort, pollInterval = 1000, quota) {
+    const wait = quota
+        ? new quota_wait_1.QuotaWait(instance, env, {
+            subject: `the pod of job ${name}`,
+            timeout: quota.timeout,
+            onWaiting: quota.onWaiting,
+        })
+        : null;
+    // when the job last had a pod: a refusal before it is not one of the pod it waits for now
+    let hadPod = 0;
     for (;;) {
         if (abort.aborted) {
             await deleteJob(instance, env, name);
@@ -41,10 +47,50 @@ async function awaitJobCompletion(instance, env, name, abort, pollInterval = 100
         if (failed) {
             throw new Error(`job ${name} failed: ${failed.message ?? failed.reason ?? 'unknown reason'}`);
         }
+        if (wait) {
+            let refusal = null;
+            if (status?.active || status?.ready) {
+                hadPod = Date.now();
+            }
+            else {
+                // the job controller can't make the pod: no pod to wait on, only its events say why
+                refusal = await readJobRefusal(instance, env, name, job, hadPod);
+            }
+            try {
+                await wait.check(refusal);
+            }
+            catch (e) {
+                await deleteJob(instance, env, name);
+                throw e;
+            }
+        }
         await sleep(pollInterval, abort);
     }
 }
 exports.awaitJobCompletion = awaitJobCompletion;
+// The last `FailedCreate` of the job for its quota after `after`. Events the server may not list
+// leave the wait as it was.
+async function readJobRefusal(instance, env, name, job, after) {
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({
+            namespace: env.namespace,
+            fieldSelector: `involvedObject.kind=Job,involvedObject.name=${name},reason=FailedCreate`,
+        });
+        const uid = job.metadata?.uid;
+        const latest = events.items
+            .filter((event) => !uid || !event.involvedObject?.uid || event.involvedObject.uid === uid)
+            .filter((event) => eventTime(event) > after && (0, quota_wait_1.getQuotaRefusal)(event.message))
+            .sort((a, b) => eventTime(b) - eventTime(a))[0];
+        return (0, quota_wait_1.getQuotaRefusal)(latest?.message);
+    }
+    catch {
+        return null;
+    }
+}
+function eventTime(event) {
+    const at = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return at ? new Date(at).getTime() : 0;
+}
 function sleep(ms, abort) {
     if (abort.aborted) {
         return Promise.resolve();
@@ -77,6 +123,20 @@ async function deleteJob(instance, env, name) {
     }
 }
 exports.deleteJob = deleteJob;
+// The Deployment says when its ReplicaSet can't make a pod: `ReplicaFailure`, `FailedCreate`.
+function getDeploymentRefusal(deployment) {
+    const failure = deployment.status?.conditions?.find((condition) => condition.type === 'ReplicaFailure' && condition.status === 'True');
+    return (0, quota_wait_1.getQuotaRefusal)(failure?.message);
+}
+function getDeployQuotaWait(instance, env, name, readiness, now) {
+    return new quota_wait_1.QuotaWait(instance, env, {
+        subject: readiness.subject ?? name,
+        // without a limit of its own it waits as long as the run does
+        timeout: readiness.quotaTimeout ?? Number.POSITIVE_INFINITY,
+        onWaiting: (text) => readiness.onWaiting?.(text),
+        now,
+    });
+}
 // how long a reason that may still pass (an image pull a registry refused for now, a
 // container the runtime could not create) may last before it fails the service
 exports.PASSING_FAILURE_MS = 60000;
@@ -96,12 +156,14 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
     let said;
     // since when a pod has had a failure that may still pass, by pod
     const failingSince = new Map();
+    const quota = getDeployQuotaWait(instance, env, name, readiness, now);
     for (;;) {
         const deployment = await instance.appsApi.readNamespacedDeployment({ name, namespace: env.namespace });
         // available: ready for the Deployment's minReadySeconds
         if ((deployment.status?.availableReplicas ?? 0) > 0) {
             return;
         }
+        await quota.check(getDeploymentRefusal(deployment));
         const created = deployment.metadata?.creationTimestamp;
         const listed = await instance.coreApi.listNamespacedPod({
             namespace: env.namespace,
@@ -163,6 +225,7 @@ async function awaitDeployRollout(instance, env, name, readiness, pollInterval =
     let runningSince = null;
     let said;
     const failingSince = new Map();
+    const quota = getDeployQuotaWait(instance, env, name, readiness, now);
     for (;;) {
         if (readiness.signal?.aborted) {
             throw new abort_1.AbortError();
@@ -179,6 +242,7 @@ async function awaitDeployRollout(instance, env, name, readiness, pollInterval =
         if (stalled && (status?.observedGeneration ?? 0) >= readiness.generation) {
             throw new Error(`${timeoutMessage}: ${stalled.message ?? 'its rollout made no progress'}`);
         }
+        await quota.check(getDeploymentRefusal(deployment));
         const listed = await instance.coreApi.listNamespacedPod({
             namespace: env.namespace,
             labelSelector: readiness.podSelector,

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.getPodTemplateHash = exports.POD_ANNOTATION = exports.ROLLOUT_DEADLINE_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.getPodTemplateHash = exports.POD_ANNOTATION = exports.ROLLOUT_DEADLINE_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEBUG_CONTAINER_RESOURCES = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
@@ -17,10 +17,15 @@ const job_placement_1 = require("./job-placement");
 const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
 const service_rollouts_1 = require("./service-rollouts");
+const quota_wait_1 = require("./quota-wait");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
 // how often a starting container is checked: as the liveness probe
 const STARTUP_PERIOD_SECONDS = 5;
+exports.DEBUG_CONTAINER_RESOURCES = {
+    requests: { cpu: '10m', memory: '16Mi' },
+    limits: { cpu: '100m', memory: '256Mi' },
+};
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
 network, 
@@ -131,6 +136,9 @@ rollout = {}) {
                             command: ['sleep'],
                             args: ['3600'],
                             volumeMounts: persistence.mounts.map((m) => m.mount),
+                            // a shell to look at the service's files: without a size of its own it would get
+                            // the namespace's default, and count as much against its quota as the service
+                            resources: exports.DEBUG_CONTAINER_RESOURCES,
                             ...(0, job_placement_1.getContainerSecurity)(env),
                         },
                     ],
@@ -149,6 +157,7 @@ rollout = {}) {
         subject: `service ${service.name}`,
         onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
         ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
+        quotaTimeout: quota_wait_1.QUOTA_TIMEOUT,
     };
     if (env.server?.serviceRun) {
         await rollOut(instance, env, service, deployment, readiness, rollout);

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensurePersistentData = exports.packSource = exports.clearStateFiles = exports.ENSURE_OUTPUT_FILES = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = exports.applyWithinQuota = exports.QUOTA_TIMEOUT = exports.TRANSFER_POD_RESOURCES = void 0;
+const quota_wait_1 = require("./quota-wait");
+Object.defineProperty(exports, "QUOTA_TIMEOUT", { enumerable: true, get: function () { return quota_wait_1.QUOTA_TIMEOUT; } });
 const volumes_1 = require("./volumes");
 const apply_1 = require("./apply");
 const path_1 = require("path");
@@ -50,12 +52,9 @@ exports.TRANSFER_POD_RESOURCES = {
     requests: { cpu: '50m', memory: '64Mi' },
     limits: { memory: '512Mi' },
 };
-// how long a transfer pod waits for room under the quota of its namespace, such as the
-// pods of other tasks of the run ending, before its task fails
-exports.QUOTA_TIMEOUT = 10 * 60 * 1000;
 // A Job waits for room under a full quota, a pod is refused: so a transfer pod is
 // created again until the pods of the tasks it runs beside are done and make room
-async function applyWithinQuota(instance, podSpec, service, timeout = exports.QUOTA_TIMEOUT, pollInterval = 2000, 
+async function applyWithinQuota(instance, podSpec, service, timeout = quota_wait_1.QUOTA_TIMEOUT, pollInterval = 2000, 
 // a run stopped while it waits: it stops waiting
 signal) {
     const deadline = Date.now() + timeout;
@@ -155,7 +154,7 @@ signal, fn) {
     };
     service.status.write('debug', `create pod ${name}`);
     await removeStalePod(instance, env.namespace, name);
-    const pod = await applyWithinQuota(instance, podSpec, service, exports.QUOTA_TIMEOUT, 2000, signal);
+    const pod = await applyWithinQuota(instance, podSpec, service, quota_wait_1.QUOTA_TIMEOUT, 2000, signal);
     if (pod.status?.phase !== 'Running') {
         service.status.write('debug', `await running state ${name}`);
         await (0, await_running_state_1.awaitRunningState)(instance, env, name, 'Running');

package/dist/kubernetes/quota-wait.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QuotaWait = exports.exceedsQuota = exports.getQuotaRefusal = exports.QUOTA_TIMEOUT = void 0;
+const units_1 = require("../utils/units");
+// How long a pod the quota of its namespace refuses waits for room before what needs it fails:
+// a transfer pod in any case, a task's or service's only when no pod that will end holds the
+// room (see QuotaWait)
+exports.QUOTA_TIMEOUT = 10 * 60 * 1000;
+// What a refusal of the quota of the namespace says, from where it starts: `exceeded quota:
+// hammerkit, requested: limits.memory=4Gi, used: limits.memory=24Gi, limited: limits.memory=26Gi`
+function getQuotaRefusal(message) {
+    const index = message ? message.search(/exceeded quota/i) : -1;
+    return message && index >= 0 ? message.slice(index) : null;
+}
+exports.getQuotaRefusal = getQuotaRefusal;
+// A pod that asks for more of a resource than the quota allows at all never fits.
+function exceedsQuota(refusal) {
+    const requested = readAmounts(/requested: (\S+?)(?:,\s|$)/.exec(refusal)?.[1]);
+    const limited = readAmounts(/limited: (\S+?)(?:,\s|$)/.exec(refusal)?.[1]);
+    return [...requested].some(([resource, amount]) => {
+        const limit = limited.get(resource);
+        return limit !== undefined && amount > limit;
+    });
+}
+exports.exceedsQuota = exceedsQuota;
+// `limits.memory=4Gi,requests.cpu=1` → limits.memory 4Gi, requests.cpu 1
+function readAmounts(list = '') {
+    const amounts = new Map();
+    for (const entry of list.split(',')) {
+        const [resource, value] = entry.split('=');
+        const amount = (0, units_1.parseQuantity)(value);
+        if (resource && amount !== undefined) {
+            amounts.set(resource, amount);
+        }
+    }
+    return amounts;
+}
+// A pod that will end and free its room: one of a Job (a task) or a pod of its own (the copy of
+// a task's files). The pods of a Deployment (a service) stay as long as what needs them.
+function willEnd(pod) {
+    if (pod.metadata?.deletionTimestamp || (pod.status?.phase !== 'Pending' && pod.status?.phase !== 'Running')) {
+        return false;
+    }
+    return !(pod.metadata?.ownerReferences ?? []).some((owner) => owner.kind === 'ReplicaSet');
+}
+/**
+ * The wait of a pod the quota of its namespace refuses. It is told what the quota refused,
+ * fails at once when the pod asks for more than the quota allows at all, and after `timeout`
+ * goes on waiting only while a pod that will end (a task's, another run's) holds room in the
+ * namespace: when only services hold it, they wait for what waits for them, and it fails.
+ */
+class QuotaWait {
+    constructor(instance, env, options) {
+        this.instance = instance;
+        this.env = env;
+        this.options = options;
+        this.since = null;
+        this.told = '';
+    }
+    // Called on each poll with what the quota refused now, or null when nothing is refused.
+    async check(refusal) {
+        const now = this.options.now ?? Date.now;
+        if (!refusal) {
+            this.since = null;
+            return;
+        }
+        if (refusal !== this.told) {
+            this.told = refusal;
+            this.options.onWaiting?.(`waits for room under the quota of the namespace: ${refusal}`);
+        }
+        if (exceedsQuota(refusal)) {
+            throw new Error(`${this.options.subject} asks for more than the quota of the namespace allows: ${refusal}`);
+        }
+        this.since = this.since ?? now();
+        if (now() - this.since < this.options.timeout) {
+            return;
+        }
+        if (await this.othersWillEnd()) {
+            // checked again after another `timeout`
+            this.since = now();
+            return;
+        }
+        throw new Error(`${this.options.subject} did not fit the quota of the namespace within ${Math.round(this.options.timeout / 60000)} minutes, and only services hold its room: ${refusal}`);
+    }
+    async othersWillEnd() {
+        try {
+            const pods = await this.instance.coreApi.listNamespacedPod({ namespace: this.env.namespace });
+            return pods.items.some(willEnd);
+        }
+        catch {
+            // what it can not see may still end
+            return true;
+        }
+    }
+}
+exports.QuotaWait = QuotaWait;
+//# sourceMappingURL=quota-wait.js.map

package/dist/planner/work-runtime-kubernetes.js

@@ -385,7 +385,10 @@ function kubernetesTaskRuntime(task, kubernetes) {
                         try {
                             if (!pod.status?.succeeded) {
                                 try {
-                                    await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
+                                    await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort, 1000, {
+                                        onWaiting: (text) => task.status.write('warn', `${task.name} ${text}`),
+                                        timeout: ensure_persistent_data_1.QUOTA_TIMEOUT,
+                                    });
                                 }
                                 catch (e) {
                                     if (!(e instanceof abort_1.AbortError)) {

package/dist/server/capacity.js

@@ -7,41 +7,8 @@ const apply_1 = require("../kubernetes/apply");
 const job_placement_1 = require("../kubernetes/job-placement");
 const volumes_1 = require("../kubernetes/volumes");
 const network_policy_1 = require("../kubernetes/network-policy");
-const UNITS = {
-    n: 1e-9,
-    u: 1e-6,
-    m: 1e-3,
-    '': 1,
-    k: 1e3,
-    K: 1e3,
-    M: 1e6,
-    G: 1e9,
-    T: 1e12,
-    P: 1e15,
-    E: 1e18,
-    Ki: 1024,
-    Mi: 1024 ** 2,
-    Gi: 1024 ** 3,
-    Ti: 1024 ** 4,
-    Pi: 1024 ** 5,
-    Ei: 1024 ** 6,
-};
-// A quantity of Kubernetes (`250m`, `1.5`, `512Mi`, `12345678n`, `1e3`), as a number.
-function parseQuantity(value) {
-    if (value === undefined) {
-        return undefined;
-    }
-    if (typeof value === 'number') {
-        return value;
-    }
-    const match = /^([+-]?[0-9.]+(?:[eE][+-]?\d+)?)([a-zA-Z]*)$/.exec(value.trim());
-    if (!match || !(match[2] in UNITS)) {
-        return undefined;
-    }
-    const number = Number(match[1]);
-    return Number.isFinite(number) ? number * UNITS[match[2]] : undefined;
-}
-exports.parseQuantity = parseQuantity;
+const units_1 = require("../utils/units");
+Object.defineProperty(exports, "parseQuantity", { enumerable: true, get: function () { return units_1.parseQuantity; } });
 function add(into, cpu, memory) {
     if (cpu !== undefined) {
         into.cpu = (into.cpu ?? 0) + cpu;
@@ -182,8 +149,8 @@ class KubernetesCapacity {
                     use.pods++;
                     for (const container of pod.spec?.containers ?? []) {
                         const requests = container.resources?.requests ?? {};
-                        add(group.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
-                        add(use.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
+                        add(group.requested, (0, units_1.parseQuantity)(requests.cpu) ?? 0, (0, units_1.parseQuantity)(requests.memory) ?? 0);
+                        add(use.requested, (0, units_1.parseQuantity)(requests.cpu) ?? 0, (0, units_1.parseQuantity)(requests.memory) ?? 0);
                     }
                 }
             }
@@ -196,8 +163,8 @@ class KubernetesCapacity {
                     namespace,
                 });
                 const hard = quota.spec?.hard ?? {};
-                const cpu = parseQuantity(hard['requests.cpu'] ?? hard.cpu);
-                const memory = parseQuantity(hard['requests.memory'] ?? hard.memory);
+                const cpu = (0, units_1.parseQuantity)(hard['requests.cpu'] ?? hard.cpu);
+                const memory = (0, units_1.parseQuantity)(hard['requests.memory'] ?? hard.memory);
                 if (cpu !== undefined || memory !== undefined) {
                     group.quota = { ...(cpu !== undefined ? { cpu } : {}), ...(memory !== undefined ? { memory } : {}) };
                 }
@@ -215,10 +182,10 @@ class KubernetesCapacity {
                     for (const pod of metrics.items) {
                         const use = usesOfPods.get(pod.metadata?.name ?? '');
                         for (const container of pod.containers) {
-                            add(used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                            add(used, (0, units_1.parseQuantity)(container.usage.cpu), (0, units_1.parseQuantity)(container.usage.memory));
                             if (use) {
                                 use.used ?? (use.used = { cpu: 0, memory: 0 });
-                                add(use.used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                                add(use.used, (0, units_1.parseQuantity)(container.usage.cpu), (0, units_1.parseQuantity)(container.usage.memory));
                             }
                         }
                     }
@@ -248,7 +215,7 @@ class KubernetesCapacity {
             const allocatable = {};
             for (const node of nodes.items) {
                 const of = node.status?.allocatable ?? {};
-                add(allocatable, parseQuantity(of.cpu), parseQuantity(of.memory));
+                add(allocatable, (0, units_1.parseQuantity)(of.cpu), (0, units_1.parseQuantity)(of.memory));
             }
             machines = { count: nodes.items.length, allocatable };
             if (this.instance.metrics) {
@@ -256,7 +223,7 @@ class KubernetesCapacity {
                     const metrics = await this.instance.metrics.getNodeMetrics();
                     const nodesUsed = {};
                     for (const node of metrics.items) {
-                        add(nodesUsed, parseQuantity(node.usage.cpu), parseQuantity(node.usage.memory));
+                        add(nodesUsed, (0, units_1.parseQuantity)(node.usage.cpu), (0, units_1.parseQuantity)(node.usage.memory));
                     }
                     machines.used = nodesUsed;
                 }

package/dist/utils/units.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseCpus = exports.formatSize = exports.parseSize = exports.formatDuration = exports.parseDuration = void 0;
+exports.parseQuantity = exports.parseCpus = exports.formatSize = exports.parseSize = exports.formatDuration = exports.parseDuration = void 0;
 const DURATION_UNITS = {
     ms: 1,
     s: 1000,
@@ -97,4 +97,39 @@ function parseCpus(value) {
     return millicores / 1000;
 }
 exports.parseCpus = parseCpus;
+const QUANTITY_UNITS = {
+    n: 1e-9,
+    u: 1e-6,
+    m: 1e-3,
+    '': 1,
+    k: 1e3,
+    K: 1e3,
+    M: 1e6,
+    G: 1e9,
+    T: 1e12,
+    P: 1e15,
+    E: 1e18,
+    Ki: 1024,
+    Mi: 1024 ** 2,
+    Gi: 1024 ** 3,
+    Ti: 1024 ** 4,
+    Pi: 1024 ** 5,
+    Ei: 1024 ** 6,
+};
+// A quantity of Kubernetes (`250m`, `1.5`, `512Mi`, `12345678n`, `1e3`), as a number.
+function parseQuantity(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value === 'number') {
+        return value;
+    }
+    const match = /^([+-]?[0-9.]+(?:[eE][+-]?\d+)?)([a-zA-Z]*)$/.exec(value.trim());
+    if (!match || !(match[2] in QUANTITY_UNITS)) {
+        return undefined;
+    }
+    const number = Number(match[1]);
+    return Number.isFinite(number) ? number * QUANTITY_UNITS[match[2]] : undefined;
+}
+exports.parseQuantity = parseQuantity;
 //# sourceMappingURL=units.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "2.0.0-next.7",
+  "version": "2.0.0-next.8",
   "keywords": [
     "build",
     "build-tool",