hammerkit 2.0.0-next.6 → 2.0.0-next.8

package/dist/kubernetes/await-running-state.js

@@ -6,6 +6,7 @@ const abort_event_1 = require("../utils/abort-event");
 const apply_1 = require("./apply");
 const sleep_1 = require("../utils/sleep");
 const cluster_state_1 = require("./cluster-state");
+const quota_wait_1 = require("./quota-wait");
 // The waits below poll instead of using the client's Watch: Watch aborts every
 // request after 30s, which a pod or deployment still pulling its image outlasts.
 async function awaitRunningState(instance, env, name, phase, pollInterval = 1000) {
@@ -22,11 +23,16 @@ async function awaitRunningState(instance, env, name, phase, pollInterval = 1000
     }
 }
 exports.awaitRunningState = awaitRunningState;
-// Wait for a job to finish by polling its status: resolves once it succeeded,
-// rejects when it failed (its `Failed` condition, i.e. retries are exhausted).
-// On abort the job is deleted — with its pods — and an AbortError is thrown, so
-// a cancelled or timed-out task leaves nothing running on the cluster.
-async function awaitJobCompletion(instance, env, name, abort, pollInterval = 1000) {
+async function awaitJobCompletion(instance, env, name, abort, pollInterval = 1000, quota) {
+    const wait = quota
+        ? new quota_wait_1.QuotaWait(instance, env, {
+            subject: `the pod of job ${name}`,
+            timeout: quota.timeout,
+            onWaiting: quota.onWaiting,
+        })
+        : null;
+    // when the job last had a pod: a refusal before it is not one of the pod it waits for now
+    let hadPod = 0;
     for (;;) {
         if (abort.aborted) {
             await deleteJob(instance, env, name);
@@ -41,10 +47,50 @@ async function awaitJobCompletion(instance, env, name, abort, pollInterval = 100
         if (failed) {
             throw new Error(`job ${name} failed: ${failed.message ?? failed.reason ?? 'unknown reason'}`);
         }
+        if (wait) {
+            let refusal = null;
+            if (status?.active || status?.ready) {
+                hadPod = Date.now();
+            }
+            else {
+                // the job controller can't make the pod: no pod to wait on, only its events say why
+                refusal = await readJobRefusal(instance, env, name, job, hadPod);
+            }
+            try {
+                await wait.check(refusal);
+            }
+            catch (e) {
+                await deleteJob(instance, env, name);
+                throw e;
+            }
+        }
         await sleep(pollInterval, abort);
     }
 }
 exports.awaitJobCompletion = awaitJobCompletion;
+// The last `FailedCreate` of the job for its quota after `after`. Events the server may not list
+// leave the wait as it was.
+async function readJobRefusal(instance, env, name, job, after) {
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({
+            namespace: env.namespace,
+            fieldSelector: `involvedObject.kind=Job,involvedObject.name=${name},reason=FailedCreate`,
+        });
+        const uid = job.metadata?.uid;
+        const latest = events.items
+            .filter((event) => !uid || !event.involvedObject?.uid || event.involvedObject.uid === uid)
+            .filter((event) => eventTime(event) > after && (0, quota_wait_1.getQuotaRefusal)(event.message))
+            .sort((a, b) => eventTime(b) - eventTime(a))[0];
+        return (0, quota_wait_1.getQuotaRefusal)(latest?.message);
+    }
+    catch {
+        return null;
+    }
+}
+function eventTime(event) {
+    const at = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return at ? new Date(at).getTime() : 0;
+}
 function sleep(ms, abort) {
     if (abort.aborted) {
         return Promise.resolve();
@@ -77,6 +123,20 @@ async function deleteJob(instance, env, name) {
     }
 }
 exports.deleteJob = deleteJob;
+// The Deployment says when its ReplicaSet can't make a pod: `ReplicaFailure`, `FailedCreate`.
+function getDeploymentRefusal(deployment) {
+    const failure = deployment.status?.conditions?.find((condition) => condition.type === 'ReplicaFailure' && condition.status === 'True');
+    return (0, quota_wait_1.getQuotaRefusal)(failure?.message);
+}
+function getDeployQuotaWait(instance, env, name, readiness, now) {
+    return new quota_wait_1.QuotaWait(instance, env, {
+        subject: readiness.subject ?? name,
+        // without a limit of its own it waits as long as the run does
+        timeout: readiness.quotaTimeout ?? Number.POSITIVE_INFINITY,
+        onWaiting: (text) => readiness.onWaiting?.(text),
+        now,
+    });
+}
 // how long a reason that may still pass (an image pull a registry refused for now, a
 // container the runtime could not create) may last before it fails the service
 exports.PASSING_FAILURE_MS = 60000;
@@ -96,12 +156,14 @@ async function awaitDeployRunningState(instance, env, name, readiness, pollInter
     let said;
     // since when a pod has had a failure that may still pass, by pod
     const failingSince = new Map();
+    const quota = getDeployQuotaWait(instance, env, name, readiness, now);
     for (;;) {
         const deployment = await instance.appsApi.readNamespacedDeployment({ name, namespace: env.namespace });
         // available: ready for the Deployment's minReadySeconds
         if ((deployment.status?.availableReplicas ?? 0) > 0) {
             return;
         }
+        await quota.check(getDeploymentRefusal(deployment));
         const created = deployment.metadata?.creationTimestamp;
         const listed = await instance.coreApi.listNamespacedPod({
             namespace: env.namespace,
@@ -163,6 +225,7 @@ async function awaitDeployRollout(instance, env, name, readiness, pollInterval =
     let runningSince = null;
     let said;
     const failingSince = new Map();
+    const quota = getDeployQuotaWait(instance, env, name, readiness, now);
     for (;;) {
         if (readiness.signal?.aborted) {
             throw new abort_1.AbortError();
@@ -179,6 +242,7 @@ async function awaitDeployRollout(instance, env, name, readiness, pollInterval =
         if (stalled && (status?.observedGeneration ?? 0) >= readiness.generation) {
             throw new Error(`${timeoutMessage}: ${stalled.message ?? 'its rollout made no progress'}`);
         }
+        await quota.check(getDeploymentRefusal(deployment));
         const listed = await instance.coreApi.listNamespacedPod({
             namespace: env.namespace,
             labelSelector: readiness.podSelector,

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.getPodTemplateHash = exports.POD_ANNOTATION = exports.ROLLOUT_DEADLINE_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.getPodTemplateHash = exports.POD_ANNOTATION = exports.ROLLOUT_DEADLINE_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEBUG_CONTAINER_RESOURCES = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
@@ -17,10 +17,15 @@ const job_placement_1 = require("./job-placement");
 const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
 const service_rollouts_1 = require("./service-rollouts");
+const quota_wait_1 = require("./quota-wait");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
 // how often a starting container is checked: as the liveness probe
 const STARTUP_PERIOD_SECONDS = 5;
+exports.DEBUG_CONTAINER_RESOURCES = {
+    requests: { cpu: '10m', memory: '16Mi' },
+    limits: { cpu: '100m', memory: '256Mi' },
+};
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
 network, 
@@ -131,6 +136,9 @@ rollout = {}) {
                             command: ['sleep'],
                             args: ['3600'],
                             volumeMounts: persistence.mounts.map((m) => m.mount),
+                            // a shell to look at the service's files: without a size of its own it would get
+                            // the namespace's default, and count as much against its quota as the service
+                            resources: exports.DEBUG_CONTAINER_RESOURCES,
                             ...(0, job_placement_1.getContainerSecurity)(env),
                         },
                     ],
@@ -149,6 +157,7 @@ rollout = {}) {
         subject: `service ${service.name}`,
         onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
         ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
+        quotaTimeout: quota_wait_1.QUOTA_TIMEOUT,
     };
     if (env.server?.serviceRun) {
         await rollOut(instance, env, service, deployment, readiness, rollout);

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.packSource = exports.clearStateFiles = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = exports.applyWithinQuota = exports.QUOTA_TIMEOUT = exports.TRANSFER_POD_RESOURCES = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearStateFiles = exports.ENSURE_OUTPUT_FILES = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = exports.applyWithinQuota = exports.QUOTA_TIMEOUT = exports.TRANSFER_POD_RESOURCES = void 0;
+const quota_wait_1 = require("./quota-wait");
+Object.defineProperty(exports, "QUOTA_TIMEOUT", { enumerable: true, get: function () { return quota_wait_1.QUOTA_TIMEOUT; } });
 const volumes_1 = require("./volumes");
 const apply_1 = require("./apply");
 const path_1 = require("path");
@@ -50,12 +52,9 @@ exports.TRANSFER_POD_RESOURCES = {
     requests: { cpu: '50m', memory: '64Mi' },
     limits: { memory: '512Mi' },
 };
-// how long a transfer pod waits for room under the quota of its namespace, such as the
-// pods of other tasks of the run ending, before its task fails
-exports.QUOTA_TIMEOUT = 10 * 60 * 1000;
 // A Job waits for room under a full quota, a pod is refused: so a transfer pod is
 // created again until the pods of the tasks it runs beside are done and make room
-async function applyWithinQuota(instance, podSpec, service, timeout = exports.QUOTA_TIMEOUT, pollInterval = 2000, 
+async function applyWithinQuota(instance, podSpec, service, timeout = quota_wait_1.QUOTA_TIMEOUT, pollInterval = 2000, 
 // a run stopped while it waits: it stops waiting
 signal) {
     const deadline = Date.now() + timeout;
@@ -88,7 +87,11 @@ signal, fn) {
     // and takes no upload
     const own = persistence.mounts.filter((m) => !m.workspace);
     const mounts = type === 'read'
-        ? own.map((m) => m.mount)
+        ? // what a task stores are its outputs, which it mounts writable: its src and the
+            // outputs of its deps are not mounted, so an output inside a folder of its src
+            // (pkg/coverage/lcov.info) has a place in the pod even when the claim holds no src,
+            // after a restore into a new claim
+            own.filter((m) => service.data.type !== 'container-task' || !m.mount.readOnly).map((m) => m.mount)
         : own.reduce((array, v) => {
             if (!array.some((i) => i.name === v.mount.name)) {
                 array.push({
@@ -99,11 +102,13 @@ signal, fn) {
             }
             return array;
         }, []);
+    // only the claims it mounts: one of a dep it doesn't read would only bind where it goes
+    const volumes = persistence.volumes.filter((volume) => mounts.some((mount) => mount.name === volume.name));
     // Guarantee the claims exist right before scheduling the pod that mounts
     // them. ensurePersistentData also creates them, but store/restore reach this
     // function directly, and a claim shared with another work item can be removed
     // concurrently — without this the pod would hang Pending on "pvc not found".
-    for (const volume of persistence.volumes) {
+    for (const volume of volumes) {
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
     const name = (0, resources_1.getResourceName)(service, env, type === 'write' ? '-upload' : '-download');
@@ -111,7 +116,7 @@ signal, fn) {
     // commit): the pod is not one its Service sends to
     const serving = !!env.server?.serviceRun && service.data.type === 'container-service';
     // and on a service run, it goes where the pods that mount the same claims run
-    const claims = env.server?.serviceRun ? (0, volumes_1.getClaimPodSpec)(persistence.volumes) : { labels: {} };
+    const claims = env.server?.serviceRun ? (0, volumes_1.getClaimPodSpec)(volumes) : { labels: {} };
     const podSpec = {
         kind: 'Pod',
         apiVersion: 'v1',
@@ -144,12 +149,12 @@ signal, fn) {
                     ...(0, job_placement_1.getContainerSecurity)(env),
                 },
             ],
-            volumes: persistence.volumes,
+            volumes,
         },
     };
     service.status.write('debug', `create pod ${name}`);
     await removeStalePod(instance, env.namespace, name);
-    const pod = await applyWithinQuota(instance, podSpec, service, exports.QUOTA_TIMEOUT, 2000, signal);
+    const pod = await applyWithinQuota(instance, podSpec, service, quota_wait_1.QUOTA_TIMEOUT, 2000, signal);
     if (pod.status?.phase !== 'Running') {
         service.status.write('debug', `await running state ${name}`);
         await (0, await_running_state_1.awaitRunningState)(instance, env, name, 'Running');
@@ -329,6 +334,33 @@ async function prepareStateFiles(instance, env, work, pod, dir, inUse) {
     await clearStateDir(instance, env, pod, `${targetPath}/${dir}`);
     return true;
 }
+// Makes each of `files` (paths in the claim of `work`, see KubernetesPersistence.files) an
+// empty file unless it is one: Kubernetes would make a directory of a subPath that isn't
+// there, and the task could not write the file. A directory there is what an earlier
+// version left in a kept claim, and goes. A path ending in `/` is made a folder.
+exports.ENSURE_OUTPUT_FILES = `cd "$1" || exit 1
+shift
+for f; do
+  case "$f" in */) mkdir -p "$f" || exit 1; continue ;; esac
+  if [ -d "$f" ] && [ ! -L "$f" ]; then rm -rf "$f" || exit 1; fi
+  if [ ! -e "$f" ] && [ ! -L "$f" ]; then mkdir -p "$(dirname "$f")" && : > "$f" || exit 1; fi
+done`;
+async function ensureOutputFiles(instance, env, work, pod, files) {
+    if (files.length === 0) {
+        return;
+    }
+    const made = await execInPod(instance, env, pod, [
+        'sh',
+        '-c',
+        exports.ENSURE_OUTPUT_FILES,
+        'sh',
+        `/dev/hammerkit/${work.id()}`,
+        ...files,
+    ]);
+    if (made) {
+        throw new Error(`unable to make the output files ${files.join(', ')} of ${work.name}: ${made}`);
+    }
+}
 // what a failed upload, restore or task left: tar does not remove files the commit does not have
 async function clearStateDir(instance, env, pod, path) {
     const cleared = await execInPod(instance, env, pod, ['rm', '-rf', path]);
@@ -397,7 +429,7 @@ options = {}) {
     const files = persistence.volumes.some((volume) => volume.persistentVolumeClaim?.claimName === claim)
         ? options.files
         : undefined;
-    if (persistence.sources.length === 0 && !options.reset && !files) {
+    if (persistence.sources.length === 0 && persistence.files.length === 0 && !options.reset && !files) {
         return;
     }
     // the folders of the states the pods of the run mount, before the upload pod mounts the claim
@@ -447,6 +479,7 @@ options = {}) {
                 }
             });
         }
+        await ensureOutputFiles(instance, env, service, name, persistence.files);
         if (files) {
             const dir = `/dev/hammerkit/${service.id()}/${files.dir}`;
             const marked = await execInPod(instance, env, name, [

package/dist/kubernetes/quota-wait.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QuotaWait = exports.exceedsQuota = exports.getQuotaRefusal = exports.QUOTA_TIMEOUT = void 0;
+const units_1 = require("../utils/units");
+// How long a pod the quota of its namespace refuses waits for room before what needs it fails:
+// a transfer pod in any case, a task's or service's only when no pod that will end holds the
+// room (see QuotaWait)
+exports.QUOTA_TIMEOUT = 10 * 60 * 1000;
+// What a refusal of the quota of the namespace says, from where it starts: `exceeded quota:
+// hammerkit, requested: limits.memory=4Gi, used: limits.memory=24Gi, limited: limits.memory=26Gi`
+function getQuotaRefusal(message) {
+    const index = message ? message.search(/exceeded quota/i) : -1;
+    return message && index >= 0 ? message.slice(index) : null;
+}
+exports.getQuotaRefusal = getQuotaRefusal;
+// A pod that asks for more of a resource than the quota allows at all never fits.
+function exceedsQuota(refusal) {
+    const requested = readAmounts(/requested: (\S+?)(?:,\s|$)/.exec(refusal)?.[1]);
+    const limited = readAmounts(/limited: (\S+?)(?:,\s|$)/.exec(refusal)?.[1]);
+    return [...requested].some(([resource, amount]) => {
+        const limit = limited.get(resource);
+        return limit !== undefined && amount > limit;
+    });
+}
+exports.exceedsQuota = exceedsQuota;
+// `limits.memory=4Gi,requests.cpu=1` → limits.memory 4Gi, requests.cpu 1
+function readAmounts(list = '') {
+    const amounts = new Map();
+    for (const entry of list.split(',')) {
+        const [resource, value] = entry.split('=');
+        const amount = (0, units_1.parseQuantity)(value);
+        if (resource && amount !== undefined) {
+            amounts.set(resource, amount);
+        }
+    }
+    return amounts;
+}
+// A pod that will end and free its room: one of a Job (a task) or a pod of its own (the copy of
+// a task's files). The pods of a Deployment (a service) stay as long as what needs them.
+function willEnd(pod) {
+    if (pod.metadata?.deletionTimestamp || (pod.status?.phase !== 'Pending' && pod.status?.phase !== 'Running')) {
+        return false;
+    }
+    return !(pod.metadata?.ownerReferences ?? []).some((owner) => owner.kind === 'ReplicaSet');
+}
+/**
+ * The wait of a pod the quota of its namespace refuses. It is told what the quota refused,
+ * fails at once when the pod asks for more than the quota allows at all, and after `timeout`
+ * goes on waiting only while a pod that will end (a task's, another run's) holds room in the
+ * namespace: when only services hold it, they wait for what waits for them, and it fails.
+ */
+class QuotaWait {
+    constructor(instance, env, options) {
+        this.instance = instance;
+        this.env = env;
+        this.options = options;
+        this.since = null;
+        this.told = '';
+    }
+    // Called on each poll with what the quota refused now, or null when nothing is refused.
+    async check(refusal) {
+        const now = this.options.now ?? Date.now;
+        if (!refusal) {
+            this.since = null;
+            return;
+        }
+        if (refusal !== this.told) {
+            this.told = refusal;
+            this.options.onWaiting?.(`waits for room under the quota of the namespace: ${refusal}`);
+        }
+        if (exceedsQuota(refusal)) {
+            throw new Error(`${this.options.subject} asks for more than the quota of the namespace allows: ${refusal}`);
+        }
+        this.since = this.since ?? now();
+        if (now() - this.since < this.options.timeout) {
+            return;
+        }
+        if (await this.othersWillEnd()) {
+            // checked again after another `timeout`
+            this.since = now();
+            return;
+        }
+        throw new Error(`${this.options.subject} did not fit the quota of the namespace within ${Math.round(this.options.timeout / 60000)} minutes, and only services hold its room: ${refusal}`);
+    }
+    async othersWillEnd() {
+        try {
+            const pods = await this.instance.coreApi.listNamespacedPod({ namespace: this.env.namespace });
+            return pods.items.some(willEnd);
+        }
+        catch {
+            // what it can not see may still end
+            return true;
+        }
+    }
+}
+exports.QuotaWait = QuotaWait;
+//# sourceMappingURL=quota-wait.js.map

package/dist/kubernetes/restore-kubernetes-data.js

@@ -1,11 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreKubernetesData = exports.getRestoreTarget = void 0;
+exports.restoreKubernetesData = exports.getRestoreTarget = exports.RESTORE_FILE = void 0;
 const event_cache_1 = require("../executer/event-cache");
 const log_1 = require("../log");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
 const claim_layout_1 = require("./claim-layout");
+// Unpacks the archive of an output that is a file to `$1`: by its content, since the name in
+// the archive is the file's and the one in the claim may carry a hash (see getClaimSubPath)
+exports.RESTORE_FILE = 'mkdir -p "$(dirname "$1")" && rm -rf "$1" && tar xzf - -O > "$1"';
 // Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
 // archive of an output: the output's place in the claim, which its mount reads
 function getRestoreTarget(work, path, 
@@ -22,6 +25,13 @@ async function restoreKubernetesData(work, kubernetes, instance, environment, pa
 options = {}) {
     const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, options.stateKey);
     const stateDir = kubernetes.server?.serviceRun ? (0, volumes_1.getItemFilesDir)(work, options.stateKey) : undefined;
+    // an output that is a file: its archive holds it alone (see storeKubernetesData), and it
+    // takes the place of whatever is at its path in the claim
+    const files = new Set(work.data.type === 'container-task'
+        ? work.data.generates
+            .filter((generate) => generate.isFile && !generate.inherited)
+            .map((generate) => generate.path)
+        : []);
     // a run stopped while the pod waits for room under the quota
     const { signal } = environment.abortCtrl;
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', signal, async (name) => {
@@ -39,7 +49,13 @@ options = {}) {
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;
-            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['sh', '-c', 'mkdir -p "$1" && tar xzf - -C "$1"', 'sh', targetPath], process.stdout, process.stderr, environment.file.readStream(generatedArchive.filename), false, (status) => {
+            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', [
+                'sh',
+                '-c',
+                files.has(generatedArchive.path) ? exports.RESTORE_FILE : 'mkdir -p "$1" && tar xzf - -C "$1"',
+                'sh',
+                targetPath,
+            ], process.stdout, process.stderr, environment.file.readStream(generatedArchive.filename), false, (status) => {
                 work.status.write('debug', `upload ${generatedArchive.filename} to ${name}: ${status.status}`);
                 if (status.status !== 'Success') {
                     failure = status.message ?? status.reason ?? 'unknown';

package/dist/kubernetes/store-kubernetes-data.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.storeKubernetesData = void 0;
+const path_1 = require("path");
 const log_1 = require("../log");
 const event_cache_1 = require("../executer/event-cache");
 const volumes_1 = require("./volumes");
@@ -12,10 +13,18 @@ stateKey) {
     const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, stateKey);
     // a run stopped while the pod waits for room under the quota
     const { signal } = environment.abortCtrl;
+    // an output that is a file is mounted as one: its archive holds it by its name, as on Docker
+    const files = new Set(work.data.type === 'container-task'
+        ? work.data.generates
+            .filter((generate) => generate.isFile && !generate.inherited)
+            .map((generate) => generate.path)
+        : []);
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'read', signal, async (name) => {
         for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
             const writerStream = environment.file.createWriteStream(generatedArchive.filename);
-            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['tar', 'zcf', '-', '-C', generatedArchive.path, '.'], writerStream, process.stderr, null, false, (status) => {
+            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', files.has(generatedArchive.path)
+                ? ['tar', 'zcf', '-', '-C', path_1.posix.dirname(generatedArchive.path), path_1.posix.basename(generatedArchive.path)]
+                : ['tar', 'zcf', '-', '-C', generatedArchive.path, '.'], writerStream, process.stderr, null, false, (status) => {
                 work.status.write('debug', `archive ${generatedArchive.path} of ${name}: ${status}`);
                 writerStream.close();
             });

package/dist/kubernetes/volumes.js

@@ -59,6 +59,7 @@ stateKey) {
         sources: [],
         mounts: [],
         initCommands: [],
+        files: [],
     };
     const workspace = kubernetes.server?.workspace;
     for (const mount of work.data.mounts) {
@@ -79,8 +80,12 @@ stateKey) {
     }
     if (work.data.type === 'container-task') {
         for (const generate of work.data.generates) {
-            appendVolume(generate.inherited ?? work, persistence, generate.path, null, false, !!generate.inherited, kubernetes, dirOf(generate.inherited ?? work));
+            const subPath = appendVolume(generate.inherited ?? work, persistence, generate.path, null, generate.isFile, !!generate.inherited, kubernetes, dirOf(generate.inherited ?? work));
+            if (generate.isFile && !generate.inherited) {
+                persistence.files.push(subPath);
+            }
         }
+        persistence.files.push(...getNestedMountPlaces(persistence, getVolumeName(work, kubernetes)));
     }
     else {
         for (const volume of work.data.volumes) {
@@ -92,6 +97,27 @@ stateKey) {
     return persistence;
 }
 exports.getKubernetesPersistence = getKubernetesPersistence;
+// The places in the claim `claim` of the paths mounted inside a folder that is mounted read
+// only from it: an output of a task inside a folder of its src (pkg/coverage/lcov.info with a
+// src pkg). The container runtime makes the place of a mount where it isn't there, which it
+// can't in a folder mounted read only, and the pod fails to start (exit 128). Each is in the
+// claim's folder of the read-only path, a folder ending in `/`, or a file.
+function getNestedMountPlaces(persistence, claim) {
+    const own = persistence.mounts.filter((m) => !m.workspace);
+    const folders = own.filter((m) => m.mount.name === claim && m.mount.readOnly && m.cwd === m.mount.mountPath && m.mount.subPath);
+    const places = [];
+    for (const mount of own) {
+        const parent = folders
+            .filter((folder) => folder !== mount && mount.mount.mountPath.startsWith(`${folder.mount.mountPath}/`))
+            .sort((a, b) => b.mount.mountPath.length - a.mount.mountPath.length)[0];
+        if (!parent) {
+            continue;
+        }
+        const place = `${parent.mount.subPath}/${mount.mount.mountPath.slice(parent.mount.mountPath.length + 1)}`;
+        places.push(mount.cwd === mount.mount.mountPath ? `${place}/` : place);
+    }
+    return places;
+}
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes, 
 // the folder of the state of `sourceWork` the path is in, see getStateFilesDir
 stateDir, 
@@ -134,6 +160,7 @@ claimPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath)) {
             stateKey: '', // TODO
         });
     }
+    return subPath;
 }
 exports.appendVolume = appendVolume;
 /**

package/dist/planner/work-runtime-kubernetes.js

@@ -385,7 +385,10 @@ function kubernetesTaskRuntime(task, kubernetes) {
                         try {
                             if (!pod.status?.succeeded) {
                                 try {
-                                    await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
+                                    await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort, 1000, {
+                                        onWaiting: (text) => task.status.write('warn', `${task.name} ${text}`),
+                                        timeout: ensure_persistent_data_1.QUOTA_TIMEOUT,
+                                    });
                                 }
                                 catch (e) {
                                     if (!(e instanceof abort_1.AbortError)) {

package/dist/server/capacity.js

@@ -7,41 +7,8 @@ const apply_1 = require("../kubernetes/apply");
 const job_placement_1 = require("../kubernetes/job-placement");
 const volumes_1 = require("../kubernetes/volumes");
 const network_policy_1 = require("../kubernetes/network-policy");
-const UNITS = {
-    n: 1e-9,
-    u: 1e-6,
-    m: 1e-3,
-    '': 1,
-    k: 1e3,
-    K: 1e3,
-    M: 1e6,
-    G: 1e9,
-    T: 1e12,
-    P: 1e15,
-    E: 1e18,
-    Ki: 1024,
-    Mi: 1024 ** 2,
-    Gi: 1024 ** 3,
-    Ti: 1024 ** 4,
-    Pi: 1024 ** 5,
-    Ei: 1024 ** 6,
-};
-// A quantity of Kubernetes (`250m`, `1.5`, `512Mi`, `12345678n`, `1e3`), as a number.
-function parseQuantity(value) {
-    if (value === undefined) {
-        return undefined;
-    }
-    if (typeof value === 'number') {
-        return value;
-    }
-    const match = /^([+-]?[0-9.]+(?:[eE][+-]?\d+)?)([a-zA-Z]*)$/.exec(value.trim());
-    if (!match || !(match[2] in UNITS)) {
-        return undefined;
-    }
-    const number = Number(match[1]);
-    return Number.isFinite(number) ? number * UNITS[match[2]] : undefined;
-}
-exports.parseQuantity = parseQuantity;
+const units_1 = require("../utils/units");
+Object.defineProperty(exports, "parseQuantity", { enumerable: true, get: function () { return units_1.parseQuantity; } });
 function add(into, cpu, memory) {
     if (cpu !== undefined) {
         into.cpu = (into.cpu ?? 0) + cpu;
@@ -182,8 +149,8 @@ class KubernetesCapacity {
                     use.pods++;
                     for (const container of pod.spec?.containers ?? []) {
                         const requests = container.resources?.requests ?? {};
-                        add(group.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
-                        add(use.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
+                        add(group.requested, (0, units_1.parseQuantity)(requests.cpu) ?? 0, (0, units_1.parseQuantity)(requests.memory) ?? 0);
+                        add(use.requested, (0, units_1.parseQuantity)(requests.cpu) ?? 0, (0, units_1.parseQuantity)(requests.memory) ?? 0);
                     }
                 }
             }
@@ -196,8 +163,8 @@ class KubernetesCapacity {
                     namespace,
                 });
                 const hard = quota.spec?.hard ?? {};
-                const cpu = parseQuantity(hard['requests.cpu'] ?? hard.cpu);
-                const memory = parseQuantity(hard['requests.memory'] ?? hard.memory);
+                const cpu = (0, units_1.parseQuantity)(hard['requests.cpu'] ?? hard.cpu);
+                const memory = (0, units_1.parseQuantity)(hard['requests.memory'] ?? hard.memory);
                 if (cpu !== undefined || memory !== undefined) {
                     group.quota = { ...(cpu !== undefined ? { cpu } : {}), ...(memory !== undefined ? { memory } : {}) };
                 }
@@ -215,10 +182,10 @@ class KubernetesCapacity {
                     for (const pod of metrics.items) {
                         const use = usesOfPods.get(pod.metadata?.name ?? '');
                         for (const container of pod.containers) {
-                            add(used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                            add(used, (0, units_1.parseQuantity)(container.usage.cpu), (0, units_1.parseQuantity)(container.usage.memory));
                             if (use) {
                                 use.used ?? (use.used = { cpu: 0, memory: 0 });
-                                add(use.used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                                add(use.used, (0, units_1.parseQuantity)(container.usage.cpu), (0, units_1.parseQuantity)(container.usage.memory));
                             }
                         }
                     }
@@ -248,7 +215,7 @@ class KubernetesCapacity {
             const allocatable = {};
             for (const node of nodes.items) {
                 const of = node.status?.allocatable ?? {};
-                add(allocatable, parseQuantity(of.cpu), parseQuantity(of.memory));
+                add(allocatable, (0, units_1.parseQuantity)(of.cpu), (0, units_1.parseQuantity)(of.memory));
             }
             machines = { count: nodes.items.length, allocatable };
             if (this.instance.metrics) {
@@ -256,7 +223,7 @@ class KubernetesCapacity {
                     const metrics = await this.instance.metrics.getNodeMetrics();
                     const nodesUsed = {};
                     for (const node of metrics.items) {
-                        add(nodesUsed, parseQuantity(node.usage.cpu), parseQuantity(node.usage.memory));
+                        add(nodesUsed, (0, units_1.parseQuantity)(node.usage.cpu), (0, units_1.parseQuantity)(node.usage.memory));
                     }
                     machines.used = nodesUsed;
                 }

package/dist/utils/units.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseCpus = exports.formatSize = exports.parseSize = exports.formatDuration = exports.parseDuration = void 0;
+exports.parseQuantity = exports.parseCpus = exports.formatSize = exports.parseSize = exports.formatDuration = exports.parseDuration = void 0;
 const DURATION_UNITS = {
     ms: 1,
     s: 1000,
@@ -97,4 +97,39 @@ function parseCpus(value) {
     return millicores / 1000;
 }
 exports.parseCpus = parseCpus;
+const QUANTITY_UNITS = {
+    n: 1e-9,
+    u: 1e-6,
+    m: 1e-3,
+    '': 1,
+    k: 1e3,
+    K: 1e3,
+    M: 1e6,
+    G: 1e9,
+    T: 1e12,
+    P: 1e15,
+    E: 1e18,
+    Ki: 1024,
+    Mi: 1024 ** 2,
+    Gi: 1024 ** 3,
+    Ti: 1024 ** 4,
+    Pi: 1024 ** 5,
+    Ei: 1024 ** 6,
+};
+// A quantity of Kubernetes (`250m`, `1.5`, `512Mi`, `12345678n`, `1e3`), as a number.
+function parseQuantity(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value === 'number') {
+        return value;
+    }
+    const match = /^([+-]?[0-9.]+(?:[eE][+-]?\d+)?)([a-zA-Z]*)$/.exec(value.trim());
+    if (!match || !(match[2] in QUANTITY_UNITS)) {
+        return undefined;
+    }
+    const number = Number(match[1]);
+    return Number.isFinite(number) ? number * QUANTITY_UNITS[match[2]] : undefined;
+}
+exports.parseQuantity = parseQuantity;
 //# sourceMappingURL=units.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "2.0.0-next.6",
+  "version": "2.0.0-next.8",
   "keywords": [
     "build",
     "build-tool",