hammerkit 2.0.0-next.5 → 2.0.0-next.6

package/dist/kubernetes/follow-pod-logs.js

@@ -6,6 +6,11 @@ const readline_1 = require("readline");
 const log_1 = require("../log");
 // what is read of an instance of a container that already ended
 const MAX_ENDED_LOG_BYTES = 1024 * 1024;
+// how often in a row the log of one instance is read again after its reading broke off at once
+const MAX_READS = 20;
+// `timestamps: true` puts the time a line was written before it (RFC3339 with up to nine
+// digits of a second), which says where a log that broke off got to
+const TIMESTAMPED = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z(?: (.*))?$/;
 /**
  * Follows the log of a container in the pods of a selector, as `docker attach` does for a
  * service on Docker: the instance that runs as it writes, and an instance that ended before
@@ -14,7 +19,12 @@ const MAX_ENDED_LOG_BYTES = 1024 * 1024;
  */
 function followPodLogs(instance, namespace, labelSelector, container, onLine, options = {}) {
     const seen = new Set();
-    const following = [];
+    const positions = new Map();
+    // the logs followed now, resolved when they end
+    const streams = new Map();
+    const following = new Map();
+    // the instances that ended, by the last look
+    const ended = new Set();
     const reading = new Set();
     let stopped = false;
     let failed = false;
@@ -34,28 +44,91 @@ function followPodLogs(instance, namespace, labelSelector, container, onLine, op
         const tracked = promise.catch(fail).finally(() => reading.delete(tracked));
         reading.add(tracked);
     };
-    const readEnded = async (pod, previous) => {
+    const positionOf = (id) => {
+        let position = positions.get(id);
+        if (!position) {
+            position = { at: null, since: null, count: 0, reads: 0 };
+            positions.set(id, position);
+        }
+        return position;
+    };
+    // a reader of the log of one instance from where it got to: the lines read before, again
+    // at the start of a log read since their second, are passed over
+    const reader = (position) => {
+        const from = position.at;
+        const skip = position.count;
+        let atFrom = 0;
+        return (raw) => {
+            const match = TIMESTAMPED.exec(raw);
+            if (!match) {
+                onLine(raw);
+                return;
+            }
+            const at = `${match[1]}.${(match[2] ?? '').padEnd(9, '0')}`;
+            if (from !== null && at <= from) {
+                if (at < from || ++atFrom <= skip) {
+                    return;
+                }
+            }
+            if (at === position.at) {
+                position.count++;
+            }
+            else {
+                position.at = at;
+                position.since = `${match[1]}Z`;
+                position.count = 1;
+            }
+            onLine(match[3] ?? '');
+        };
+    };
+    const readEnded = async (pod, id, previous) => {
+        const position = positionOf(id);
+        position.reads++;
         const log = await instance.coreApi.readNamespacedPodLog({
             name: pod,
             namespace,
             container,
             previous,
+            timestamps: true,
             limitBytes: MAX_ENDED_LOG_BYTES,
+            // this call takes no sinceTime: a minute more than since then, for clocks that differ
+            ...(position.since
+                ? { sinceSeconds: Math.max(1, Math.ceil((Date.now() - Date.parse(position.since)) / 1000) + 60) }
+                : {}),
         });
+        const read = reader(position);
         for (const line of String(log ?? '').split(/\r?\n/)) {
             if (line.length > 0) {
-                onLine(line);
+                read(line);
             }
         }
     };
-    const follow = async (pod) => {
+    const follow = async (pod, id) => {
+        const position = positionOf(id);
+        position.reads++;
         const output = new stream_1.PassThrough();
-        let ended = false;
-        // the instance ended or the request broke: a restarted one is looked for at once
+        const started = Date.now();
+        let over = false;
+        let done = () => undefined;
+        const stream = new Promise((resolve) => (done = resolve));
+        streams.set(id, stream);
+        // the instance ended or the request broke: it is looked at again at once, a restarted
+        // one followed, and what this one wrote after the break read from where it got to
         const end = () => {
-            if (!ended) {
-                ended = true;
+            if (!over) {
+                over = true;
                 streaming--;
+                // a follow that lasted is cut now and then (a quiet stretch, a proxy): only reads
+                // that break off at once count against the limit
+                if (Date.now() - started >= 10000) {
+                    position.reads = 0;
+                }
+                streams.delete(id);
+                following.delete(id);
+                done();
+                if (!stopped && position.reads < MAX_READS) {
+                    seen.delete(id);
+                }
                 wake();
             }
         };
@@ -64,11 +137,15 @@ function followPodLogs(instance, namespace, labelSelector, container, onLine, op
             end();
         }));
         output.on('end', end);
-        (0, readline_1.createInterface)({ input: output }).on('line', (line) => onLine(line));
+        (0, readline_1.createInterface)({ input: output }).on('line', reader(position));
         streaming++;
         let controller;
         try {
-            controller = await instance.log.log(namespace, pod, container, output, { follow: true });
+            controller = await instance.log.log(namespace, pod, container, output, {
+                follow: true,
+                timestamps: true,
+                ...(position.since ? { sinceTime: position.since } : {}),
+            });
         }
         catch (e) {
             end();
@@ -78,7 +155,7 @@ function followPodLogs(instance, namespace, labelSelector, container, onLine, op
             controller.abort();
         }
         else {
-            following.push(controller);
+            following.set(id, controller);
         }
     };
     const look = (pod) => {
@@ -91,22 +168,28 @@ function followPodLogs(instance, namespace, labelSelector, container, onLine, op
         // an instance that ended before this one, not seen while it ran: its lines go first
         let before = Promise.resolve();
         const last = status.lastState?.terminated?.containerID;
+        const current = status.containerID;
+        if (last) {
+            ended.add(last);
+        }
+        if (current && status.state?.terminated) {
+            ended.add(current);
+        }
         if (last && !seen.has(last)) {
             seen.add(last);
-            before = readEnded(name, true).catch(fail);
+            before = readEnded(name, last, true).catch(fail);
             track(before);
         }
-        const current = status.containerID;
         if (!current || seen.has(current)) {
             return;
         }
         if (status.state?.running) {
             seen.add(current);
-            track(before.then(() => follow(name)));
+            track(before.then(() => follow(name, current)));
         }
         else if (status.state?.terminated) {
             seen.add(current);
-            track(before.then(() => readEnded(name, false)));
+            track(before.then(() => readEnded(name, current, false)));
         }
     };
     const poll = async () => {
@@ -157,26 +240,40 @@ function followPodLogs(instance, namespace, labelSelector, container, onLine, op
         async settle() {
             await polling;
             await poll();
-            // what follows a running instance never ends by itself
-            const ended = [...reading];
-            let wait;
-            await Promise.race([
-                Promise.all(ended),
-                new Promise((resolve) => {
-                    wait = setTimeout(resolve, 5000);
-                    wait.unref?.();
-                }),
-            ]);
-            clearTimeout(wait);
+            // what follows an instance that ended ends by itself once it has read all there is
+            const endedStreams = () => [...streams].filter(([id]) => ended.has(id));
+            await within(Promise.all([...reading, ...endedStreams().map(([, stream]) => stream)]), 5000);
+            // one still going after that is cut, and read from where it got to
+            const cut = endedStreams();
+            if (cut.length > 0) {
+                for (const [id] of cut) {
+                    following.get(id)?.abort();
+                }
+                await within(Promise.all(cut.map(([, stream]) => stream)), 1000);
+                await poll();
+                await within(Promise.all([...reading]), 5000);
+            }
         },
         stop() {
             stopped = true;
             clearTimeout(timer);
-            for (const controller of following.splice(0)) {
+            for (const controller of following.values()) {
                 controller.abort();
             }
+            following.clear();
         },
     };
 }
 exports.followPodLogs = followPodLogs;
+async function within(promise, ms) {
+    let wait;
+    await Promise.race([
+        promise,
+        new Promise((resolve) => {
+            wait = setTimeout(resolve, ms);
+            wait.unref?.();
+        }),
+    ]);
+    clearTimeout(wait);
+}
 //# sourceMappingURL=follow-pod-logs.js.map

package/dist/kubernetes/network-policy.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeSandboxProxies = exports.getProxyReadinessScript = exports.removeAllKubernetesItemNetworks = exports.releaseKubernetesItemNetwork = exports.awaitPodsEnded = exports.removeKubernetesItemNetwork = exports.ensureKubernetesItemNetwork = exports.buildProxyPolicy = exports.buildNetworkPolicy = exports.getNetworkPolicyName = exports.getSandboxProxy = exports.getProxyName = exports.getNetworkLabels = exports.NETWORK_PROXY = exports.NETWORK_LIMITED = exports.NETWORK_LABEL = exports.RUN_LABEL = void 0;
+exports.removeSandboxProxies = exports.getProxyReadinessScript = exports.removeAllKubernetesItemNetworks = exports.releaseKubernetesItemNetwork = exports.awaitPodsEnded = exports.removeKubernetesItemNetwork = exports.ensureKubernetesItemNetwork = exports.buildProxyPolicy = exports.buildNetworkPolicy = exports.getNetworkPolicyName = exports.getSandboxProxy = exports.getProxyName = exports.getNetworkLabels = exports.NETWORK_PROXY = exports.NETWORK_LIMITED = exports.NETWORK_LABEL = exports.PROXY_OF_SANDBOX_LABEL = exports.RUN_LABEL = void 0;
 const crypto_1 = require("crypto");
 const stream_1 = require("stream");
 const readline_1 = require("readline");
@@ -21,7 +21,7 @@ const resources_1 = require("./resources");
 const PROXY_LABEL = 'hammerkit.dev/proxy-for';
 const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
 // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
-const PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
+exports.PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
 const PROXY_READY_TIMEOUT_MS = 60000;
 // the token of the endpoints of secret providers, in a Secret named like the proxy
 const SECRET_TOKEN_KEY = 'token';
@@ -77,7 +77,7 @@ exports.getSandboxProxy = getSandboxProxy;
 function getProxyLabels(env, item, proxy) {
     return {
         [NETWORK_OF_LABEL]: item.id(),
-        ...(proxy.sandbox ? { [PROXY_OF_SANDBOX_LABEL]: proxy.sandbox.label } : (0, job_placement_1.getRunLabels)(env)),
+        ...(proxy.sandbox ? { [exports.PROXY_OF_SANDBOX_LABEL]: proxy.sandbox.label } : (0, job_placement_1.getRunLabels)(env)),
     };
 }
 function getNetworkPolicyName(item, scope) {
@@ -543,7 +543,7 @@ async function removeSandboxProxies(instance, namespace, of, keep = () => false)
     const deployments = await instance.appsApi.listNamespacedDeployment({
         namespace,
         labelSelector: [
-            `${PROXY_OF_SANDBOX_LABEL}=${of.sandbox}`,
+            `${exports.PROXY_OF_SANDBOX_LABEL}=${of.sandbox}`,
             ...(of.item ? [`${NETWORK_OF_LABEL}=${of.item.id()}`] : []),
         ].join(','),
     });

package/dist/kubernetes/restore-kubernetes-data.js

@@ -22,7 +22,9 @@ async function restoreKubernetesData(work, kubernetes, instance, environment, pa
 options = {}) {
     const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, options.stateKey);
     const stateDir = kubernetes.server?.serviceRun ? (0, volumes_1.getItemFilesDir)(work, options.stateKey) : undefined;
-    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', async (name) => {
+    // a run stopped while the pod waits for room under the quota
+    const { signal } = environment.abortCtrl;
+    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', signal, async (name) => {
         if (options.reset) {
             await (0, ensure_persistent_data_1.clearClaim)(instance, kubernetes, work, name);
         }

package/dist/kubernetes/store-kubernetes-data.js

@@ -10,7 +10,9 @@ async function storeKubernetesData(work, kubernetes, instance, environment, path
 // getStateFilesDir)
 stateKey) {
     const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, stateKey);
-    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'read', async (name) => {
+    // a run stopped while the pod waits for room under the quota
+    const { signal } = environment.abortCtrl;
+    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'read', signal, async (name) => {
         for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
             const writerStream = environment.file.createWriteStream(generatedArchive.filename);
             const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['tar', 'zcf', '-', '-C', generatedArchive.path, '.'], writerStream, process.stderr, null, false, (status) => {

package/dist/kubernetes/volumes.js

@@ -68,15 +68,14 @@ stateKey) {
         }
         appendVolume(work, persistence, mount.containerPath, mount.localPath, mount.isFile, true, kubernetes, mounted, own);
     }
+    // its deps' src too, uploaded into its own claim: a container task sees them as it does on
+    // docker (a tsc -b of a project that references another reads the other's tsconfig)
     for (const src of work.data.src) {
-        if (src.inherited) {
-            continue;
-        }
         if (workspace &&
             appendWorkspaceMount(workspace, persistence, src.absolutePath, src.absolutePath, src.isFile, false)) {
             continue;
         }
-        appendVolume(work, persistence, src.absolutePath, src.absolutePath, src.isFile, true, kubernetes, mounted, own); // TODO matcher
+        appendVolume(work, persistence, src.absolutePath, src.absolutePath, src.isFile, true, kubernetes, mounted, own, src.inherited ? (0, claim_layout_1.getInheritedSourceSubPath)(work, src.absolutePath) : undefined); // TODO matcher
     }
     if (work.data.type === 'container-task') {
         for (const generate of work.data.generates) {
@@ -98,9 +97,10 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
 stateDir, 
 // the folder its upload goes to, when it is not the one the path is mounted from (a
 // service that watches its files, see syncLiveFiles)
-uploadDir = stateDir) {
+uploadDir = stateDir, 
+// where it goes in the claim, when not where getClaimSubPath puts it
+claimPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath)) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
-    const claimPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const subPath = (stateDir ? `${stateDir}/` : '') + claimPath;
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {

package/dist/optimizer/calculate-checksum.js

@@ -1,17 +1,60 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.calculateChecksum = void 0;
+exports.calculateChecksum = exports.racyWindow = void 0;
 const crypto_1 = require("crypto");
+// The checksums a process computed, by the file context and path: every task of a run hashes
+// the sources of the tasks it depends on too, so most files are asked for many times
+const known = new WeakMap();
+// A file written again soon after it was hashed, with the same size, keeps its times on a
+// file system whose timestamps are coarser than that: a checksum of a file changed within
+// the window before it was hashed is not kept. Whole seconds are taken for a file system
+// that has no more (FAT, ext3, HFS+), else the window is short.
+function racyWindow(stats) {
+    return stats.lastModified % 1000 === 0 && stats.identity.changed % 1000 === 0 ? 2000 : 50;
+}
+exports.racyWindow = racyWindow;
 // Hashes the raw bytes: decoding to text first would map every invalid UTF-8
 // sequence to U+FFFD, so two different binary files could share a checksum.
 // For UTF-8 text the result is unchanged. sha256, which nobody can make two files share:
 // a cache hit is also what a verified check counts (check-key.ts).
-async function calculateChecksum(environment, path) {
+//
+// With the `stats` of the file, the checksum is reused while the file stays the same file:
+// the same modification and change time, size and inode. A write changes its times, a
+// replacement (a checkout, a rename over it) its inode.
+async function calculateChecksum(environment, path, stats) {
+    const identity = stats?.identity;
+    if (!identity) {
+        return hashFile(environment, path);
+    }
+    let files = known.get(environment.file);
+    if (!files) {
+        files = new Map();
+        known.set(environment.file, files);
+    }
+    const cached = files.get(path);
+    if (cached &&
+        cached.lastModified === stats.lastModified &&
+        cached.size === identity.size &&
+        cached.changed === identity.changed &&
+        cached.inode === identity.inode) {
+        return cached.checksum;
+    }
+    const hashedAt = Date.now();
+    const checksum = await hashFile(environment, path);
+    if (hashedAt - Math.max(stats.lastModified, identity.changed) > racyWindow({ ...stats, identity })) {
+        files.set(path, { lastModified: stats.lastModified, ...identity, checksum });
+    }
+    else {
+        files.delete(path);
+    }
+    return checksum;
+}
+exports.calculateChecksum = calculateChecksum;
+async function hashFile(environment, path) {
     const hash = (0, crypto_1.createHash)('sha256');
     for await (const chunk of environment.file.readStream(path)) {
         hash.update(chunk);
     }
     return hash.digest('hex');
 }
-exports.calculateChecksum = calculateChecksum;
 //# sourceMappingURL=calculate-checksum.js.map

package/dist/optimizer/get-work-cache-stats.js

@@ -5,55 +5,67 @@ const calculate_checksum_1 = require("./calculate-checksum");
 const path_1 = require("path");
 const crypto_1 = require("crypto");
 const promises_1 = require("fs/promises");
+const limit_1 = require("../utils/limit");
 function isExcluded(path, excluded) {
     return excluded.some((excludedPath) => path === excludedPath || path.startsWith(excludedPath + path_1.sep));
 }
-async function addWorkCacheStats(result, cwd, path, matcher, context, root, excluded) {
+// What the walk does at once: a stat, a listing or a checksum of a file each. Awaited one
+// after the other, a walk of a thousand files spent most of its time waiting for the disk.
+const WALK_CONCURRENCY = 32;
+async function addWorkCacheStats(result, cwd, path, matcher, context, root, excluded, limit = (0, limit_1.createLimit)(WALK_CONCURRENCY)) {
     if (isExcluded(path, excluded)) {
         return 0;
     }
+    // the slot is held for what this path needs of the disk, not while its children are walked
+    const visited = await limit(() => visitPath(cwd, path, matcher, context, root));
+    if (visited.type === 'file') {
+        // '/'-separated on every OS, so a Windows host keys a container task like Linux
+        result.files[visited.key] = { lastModified: visited.lastModified, checksum: visited.checksum };
+        return 1;
+    }
+    if (visited.type === 'directory') {
+        const counts = await Promise.all(visited.files.map((file) => addWorkCacheStats(result, cwd, (0, path_1.join)(path, file), matcher, context, false, excluded, limit)));
+        return counts.reduce((sum, count) => sum + count, 0);
+    }
+    return 0;
+}
+async function visitPath(cwd, path, matcher, context, root) {
+    const key = (0, path_1.relative)(cwd, path).split(path_1.sep).join(path_1.posix.sep);
     // On a server the files were written by a commit or a task: a link is a source as it is,
     // its target, and never followed. A `**` would walk `x -> /` through the server's disk.
     if (context.hosted) {
         const link = await (0, promises_1.lstat)(path).catch(() => undefined);
         if (link?.isSymbolicLink()) {
             if (!matcher(path)) {
-                return 0;
+                return { type: 'none' };
             }
             const checksum = (0, crypto_1.createHash)('sha256')
                 .update(`link:${await (0, promises_1.readlink)(path)}`)
                 .digest('hex');
-            result.files[(0, path_1.relative)(cwd, path).split(path_1.sep).join(path_1.posix.sep)] = { lastModified: link.mtimeMs, checksum };
-            return 1;
+            return { type: 'file', key, lastModified: link.mtimeMs, checksum };
         }
     }
     const exists = await context.file.exists(path);
     if (!exists) {
-        return 0;
+        return { type: 'none' };
     }
     const stats = await context.file.stats(path);
     if (stats.type === 'file') {
         if (!matcher(path)) {
-            return 0;
+            return { type: 'none' };
         }
-        const checksum = await (0, calculate_checksum_1.calculateChecksum)(context, path);
-        // '/'-separated on every OS, so a Windows host keys a container task like Linux
-        result.files[(0, path_1.relative)(cwd, path).split(path_1.sep).join(path_1.posix.sep)] = { lastModified: stats.lastModified, checksum };
-        return 1;
+        const checksum = await (0, calculate_checksum_1.calculateChecksum)(context, path, stats);
+        return { type: 'file', key, lastModified: stats.lastModified, checksum };
     }
     else if (stats.type === 'directory') {
         // A glob such as src/**/*.ts never matches a directory itself, so a
         // directory is walked when files inside it could match.
         if (!root && !matcher(path, true)) {
-            return 0;
-        }
-        let matched = 0;
-        for (const file of await context.file.listFiles(path)) {
-            matched += await addWorkCacheStats(result, cwd, (0, path_1.join)(path, file), matcher, context, false, excluded);
+            return { type: 'none' };
         }
-        return matched;
+        return { type: 'directory', files: await context.file.listFiles(path) };
     }
-    return 0;
+    return { type: 'none' };
 }
 async function getWorkCacheStats(work, environment) {
     return (await collectSourceStats(work, environment, [])).stats;
@@ -66,14 +78,14 @@ async function collectSourceStats(work, environment, excluded) {
         created: new Date(),
         files: {},
     };
-    const unmatched = [];
-    for (const src of work.src) {
+    // one limit for all its srcs, which are walked at once: an inherited src is often a large
+    // part of what a task hashes
+    const limit = (0, limit_1.createLimit)(WALK_CONCURRENCY);
+    const matches = await Promise.all(work.src.map((src) => {
         const matcher = (file, partial) => src.matcher(file, work.cwd, partial);
-        const matched = await addWorkCacheStats(result, work.cwd, src.absolutePath, matcher, environment, true, excluded);
-        if (matched === 0 && !src.inherited && !isExcluded(src.absolutePath, excluded)) {
-            unmatched.push(src);
-        }
-    }
+        return addWorkCacheStats(result, work.cwd, src.absolutePath, matcher, environment, true, excluded, limit);
+    }));
+    const unmatched = work.src.filter((src, index) => matches[index] === 0 && !src.inherited && !isExcluded(src.absolutePath, excluded));
     return { stats: result, unmatched };
 }
 // The source stats of a task or service, leaving out everything its (transitive)

package/dist/planner/work-item-executions.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readExecutions = exports.clearExecutions = exports.recordExecution = void 0;
+// kept by the task, not its item: the runtime and the scheduler hold copies of the item
+const executions = new WeakMap();
+function recordExecution(task, startedAt, finishedAt) {
+    if (Number.isNaN(startedAt.getTime()) || Number.isNaN(finishedAt.getTime()) || finishedAt < startedAt) {
+        return;
+    }
+    let list = executions.get(task);
+    if (!list) {
+        list = [];
+        executions.set(task, list);
+    }
+    list.push({ startedAt, finishedAt });
+}
+exports.recordExecution = recordExecution;
+// a task that runs again (watch) starts with none
+function clearExecutions(task) {
+    executions.delete(task);
+}
+exports.clearExecutions = clearExecutions;
+function readExecutions(task) {
+    return executions.get(task) ?? [];
+}
+exports.readExecutions = readExecutions;
+//# sourceMappingURL=work-item-executions.js.map

package/dist/planner/work-runtime-kubernetes.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.kubernetesForwardRuntime = exports.isRolledOut = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.getTaskJobContainers = exports.describeFailedJob = void 0;
+exports.kubernetesForwardRuntime = exports.isRolledOut = exports.kubernetesServiceRuntime = exports.kubernetesTaskRuntime = exports.getTaskJobContainers = exports.describeFailedJob = exports.recordJobExecution = void 0;
 const abort_1 = require("../executer/abort");
 const kubernetes_service_1 = require("../executer/kubernetes-service");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
@@ -12,6 +12,7 @@ const apply_1 = require("../kubernetes/apply");
 const ensure_kubernetes_service_exists_1 = require("../kubernetes/ensure-kubernetes-service-exists");
 const ensure_kubernetes_deployment_exists_1 = require("../kubernetes/ensure-kubernetes-deployment-exists");
 const follow_pod_logs_1 = require("../kubernetes/follow-pod-logs");
+const work_item_executions_1 = require("./work-item-executions");
 const service_definition_1 = require("./service-definition");
 const ensure_namespace_1 = require("../kubernetes/ensure-namespace");
 const ensure_persistent_data_1 = require("../kubernetes/ensure-persistent-data");
@@ -62,32 +63,17 @@ async function getJobPod(instance, kubernetes, job) {
         .sort((a, b) => (a.metadata?.creationTimestamp?.toISOString() ?? '').localeCompare(b.metadata?.creationTimestamp?.toISOString() ?? ''))
         .pop();
 }
-// a task's log is kept to this, the run's own cap is smaller
-const MAX_JOB_LOG_BYTES = 4 * 1024 * 1024;
-// What the command printed, as the task's output: read from the cluster once the Job
-// has ended. Without a pod (it never started) there is nothing to say.
-async function writeJobLogs(instance, kubernetes, task, job, container) {
-    try {
-        const pod = await getJobPod(instance, kubernetes, job);
-        if (!pod?.metadata?.name) {
-            return;
-        }
-        const log = await instance.coreApi.readNamespacedPodLog({
-            name: pod.metadata.name,
-            namespace: kubernetes.namespace,
-            container,
-            limitBytes: MAX_JOB_LOG_BYTES,
-        });
-        for (const line of String(log ?? '').split(/\r?\n/)) {
-            if (line.length > 0) {
-                task.status.console('stdout', line);
-            }
-        }
-    }
-    catch (e) {
-        task.status.write('debug', `the log of ${job} could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+// When the command of a Job ran, as its container says: what the task spent before it (the
+// pod scheduled, the image pulled, the init containers) and after it is not the command's.
+async function recordJobExecution(instance, kubernetes, task, job, container) {
+    const pod = await getJobPod(instance, kubernetes, job).catch(() => undefined);
+    const status = pod?.status?.containerStatuses?.find((s) => s.name === container);
+    const terminated = status?.state?.terminated ?? status?.lastState?.terminated;
+    if (terminated?.startedAt && terminated.finishedAt) {
+        (0, work_item_executions_1.recordExecution)(task, new Date(terminated.startedAt), new Date(terminated.finishedAt));
     }
 }
+exports.recordJobExecution = recordJobExecution;
 // the command of a Job exited with this code, as a crash on docker or a host
 class JobCrash extends Error {
     constructor(exitCode) {
@@ -394,24 +380,33 @@ function kubernetesTaskRuntime(task, kubernetes) {
                     };
                     try {
                         const pod = await (0, apply_1.apply)(instance, spec);
-                        if (!pod.status?.succeeded) {
-                            try {
-                                await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
-                            }
-                            catch (e) {
-                                if (!(e instanceof abort_1.AbortError)) {
-                                    await writeJobLogs(instance, kubernetes, task, podName, container);
-                                    const failure = await describeFailedJob(instance, kubernetes, task.name, podName, container, inJob);
-                                    if (failure?.type === 'crash') {
-                                        throw new JobCrash(failure.exitCode);
+                        // what the command prints, while it prints it; the last lines once the pod ended
+                        const logs = (0, follow_pod_logs_1.followPodLogs)(instance, kubernetes.namespace, `job-name=${podName}`, container, (line) => task.status.console('stdout', line), { onError: (text) => task.status.write('debug', text) });
+                        try {
+                            if (!pod.status?.succeeded) {
+                                try {
+                                    await (0, await_running_state_1.awaitJobCompletion)(instance, kubernetes, spec.metadata.name, options.abort);
+                                }
+                                catch (e) {
+                                    if (!(e instanceof abort_1.AbortError)) {
+                                        await logs.settle();
+                                        await recordJobExecution(instance, kubernetes, task.data, podName, container);
+                                        const failure = await describeFailedJob(instance, kubernetes, task.name, podName, container, inJob);
+                                        if (failure?.type === 'crash') {
+                                            throw new JobCrash(failure.exitCode);
+                                        }
+                                        throw new Error(failure?.message ?? (0, log_1.getErrorMessage)(e));
                                     }
-                                    throw new Error(failure?.message ?? (0, log_1.getErrorMessage)(e));
+                                    throw e;
                                 }
-                                throw e;
                             }
+                            await logs.settle();
+                        }
+                        finally {
+                            logs.stop();
                         }
-                        await writeJobLogs(instance, kubernetes, task, podName, container);
                         task.status.write('debug', 'pod completed');
+                        await recordJobExecution(instance, kubernetes, task.data, podName, container);
                         // its init containers are done with the credentials
                         await (0, job_secrets_1.removeJobCredentials)(instance, kubernetes, task);
                         if (i < task.data.cmds.length) {