hammerkit 2.0.0-next.4 → 2.0.0-next.6

package/dist/dashboard/index.html

@@ -4,10 +4,10 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="robots" content="noindex" />
-    <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
+    <link rel="icon" href="/logo.png" type="image/png" />
     <title>hammerkit</title>
-    <script type="module" crossorigin src="/assets/index-zPXa4klZ.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-CjnKm5-3.css">
+    <script type="module" crossorigin src="/assets/index-vE22S5E_.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-gIBDC_m8.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/executer/docker-task.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dockerTask = exports.getCapabilityHostConfig = exports.buildCreateOptions = void 0;
+const work_item_executions_1 = require("../planner/work-item-executions");
 const run_labels_1 = require("../docker/run-labels");
 const execute_docker_1 = require("./execute-docker");
 const abort_1 = require("./abort");
@@ -106,7 +107,9 @@ async function dockerTask(docker, item, environment, options) {
             for (const cmd of item.data.cmds) {
                 (0, abort_1.checkForAbort)(options.abort);
                 item.status.write('info', `execute cmd "${cmd.cmd}" in container`);
+                const commandStarted = new Date();
                 const result = await (0, execute_docker_1.execCommand)(item.status, environment, container, (0, execute_docker_1.convertToPosixPath)(cmd.cwd), [item.data.shell, '-c', cmd.cmd], item.data.user, undefined, options.abort);
+                (0, work_item_executions_1.recordExecution)(item.data, commandStarted, new Date());
                 if (result.type === 'timeout') {
                     throw new Error(`command ${cmd.cmd} timed out`);
                 }

package/dist/executer/execute-work-task.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeWorkTask = void 0;
+const work_item_executions_1 = require("../planner/work-item-executions");
 const work_task_1 = require("../planner/work-task");
 const states_1 = require("./states");
 const abort_1 = require("./abort");
@@ -83,7 +84,7 @@ dependents = null) {
             // build summary can show the cause column. Reporting only.
             let missCauses;
             if (options.explain) {
-                const explanation = await (0, explain_1.explainTask)(work, options.cacheDefault, environment);
+                const explanation = await (0, explain_1.explainTask)(work, options.cacheDefault, environment, undefined, undefined, (0, watch_loop_1.getSharedStateKeys)(work, options));
                 missCauses = explanation.causes.map(explain_1.describeCause);
                 for (const cause of missCauses) {
                     work.status.write('info', `cache miss: ${cause}`);
@@ -103,6 +104,7 @@ dependents = null) {
                 });
                 const timeout = (0, deadline_1.getTaskTimeout)(work.data.timeout, options);
                 const deadline = (0, deadline_1.withDeadline)(abort, timeout);
+                (0, work_item_executions_1.clearExecutions)(work.data);
                 try {
                     await work.runtime.execute(environment, {
                         cache: cacheState,

package/dist/executer/local-task.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.localTask = void 0;
+const work_item_executions_1 = require("../planner/work-item-executions");
 const abort_1 = require("./abort");
 const execute_command_1 = require("./execute-command");
 const log_1 = require("../log");
@@ -18,7 +19,9 @@ async function localTask(item, environment, options) {
         for (const cmd of item.data.cmds) {
             (0, abort_1.checkForAbort)(options.abort);
             item.status.write('info', `execute cmd "${cmd.cmd}" locally`);
+            const commandStarted = new Date();
             const exitCode = await (0, execute_command_1.executeCommand)(item.status, options.abort, cmd.cwd, cmd.cmd, envs, environment);
+            (0, work_item_executions_1.recordExecution)(item.data, commandStarted, new Date());
             if (exitCode !== 0) {
                 options.state.set({
                     type: 'crash',

package/dist/executer/scheduler/enqueue-next.js

@@ -18,6 +18,7 @@ async function checkCacheState(item, defaultCacheMethod, environment,
 // the cache is off
 // restore: false only looks a backend hit up, its outputs are downloaded once
 // something that runs depends on them (`run --no-restore`, see deferred-restore)
+// memo: the keys of the execution so far, see getExecutionStateKeys
 options = {}) {
     (0, deferred_restore_1.clearDeferredRestore)(item);
     // A task whose cache is off is never looked up, so hashing its sources buys
@@ -28,7 +29,11 @@ options = {}) {
         item.status.write('debug', `${item.name} is skipping cache check, because caching is disabled`);
         return { cached: false, stateKey: exports.UNCACHED_STATE_KEY, resolved: declared, provable: false };
     }
-    const { stateKey, stats: currentStats, resolved, provable, unmatched, } = await (0, state_key_1.computeStateKey)(item, defaultCacheMethod, environment);
+    const { stateKey, stats: currentStats, resolved, provable, unmatched, } = await (0, state_key_1.computeStateKey)(item, defaultCacheMethod, environment, options.memo);
+    // what depends on it takes its key from here
+    if (options.memo && !options.memo.has(item.id())) {
+        options.memo.set(item.id(), Promise.resolve({ stateKey, provable }));
+    }
     for (const src of unmatched) {
         item.status.write('warn', `src "${src.source}" matches no files`);
     }

package/dist/executer/scheduler/state-key.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.computeStateKey = exports.combineStateKeys = exports.isOwnProvable = exports.resolveEffective = void 0;
+exports.computeStateKey = exports.combineStateKeys = exports.getExecutionStateKeys = exports.isOwnProvable = exports.resolveEffective = void 0;
 const get_work_cache_stats_1 = require("../../optimizer/get-work-cache-stats");
 const work_task_1 = require("../../planner/work-task");
 // The state-key engine is shared by the scheduler's hit/miss decision
@@ -29,6 +29,43 @@ function isOwnProvable(item, unmatched) {
     return unmatched.length < item.data.src.filter((src) => !src.inherited).length;
 }
 exports.isOwnProvable = isOwnProvable;
+// What an item on its own way down stands for. Items that depend on or need each other in a
+// loop fail to plan (checkForLoop) and never run: this only has to end. The loop is told by
+// the path of one computation, never by the shared memo, which tasks checked at the same time
+// read, so a placeholder can never end up in another task's key.
+const LOOP = { stateKey: 'loop', provable: true };
+// Keep a computation in the memo, and take it out again if it fails, so a later check
+// computes it anew instead of failing the same way.
+function memoized(memo, key, compute) {
+    const existing = memo.get(key);
+    if (existing) {
+        return existing;
+    }
+    const promise = compute();
+    memo.set(key, promise);
+    promise.catch(() => {
+        if (memo.get(key) === promise) {
+            memo.delete(key);
+        }
+    });
+    return promise;
+}
+const executionKeys = new WeakMap();
+// The keys of one execution (the options of `exec`, which all its items share): its items
+// are checked once, before they run, from the files as they are then. Not for watch mode,
+// where a key is computed again because files changed.
+// The items it holds keys of must have no loop: checkForLoop fails the tree before it runs.
+// Two tasks checked at once that enter a loop of needs from opposite ends would each wait for
+// the other's computation.
+function getExecutionStateKeys(options) {
+    let memo = executionKeys.get(options);
+    if (!memo) {
+        memo = new Map();
+        executionKeys.set(options, memo);
+    }
+    return memo;
+}
+exports.getExecutionStateKeys = getExecutionStateKeys;
 // Fold a task's own state key together with the state keys of its dependencies.
 // A change anywhere in the dependency subtree must change the resulting key, so
 // that a downstream task is invalidated when an upstream source changes — even
@@ -51,34 +88,36 @@ function concreteMethod(resolved, defaultCacheMethod) {
 // state keys of its dependencies, recursively. This is derived purely from the
 // work tree and the filesystem, so it does not depend on scheduler/run state —
 // important because it runs before dependencies are awaited.
-async function resolveEffectiveStateKey(item, defaultCacheMethod, environment, memo) {
-    const existing = memo.get(item.id());
-    if (existing !== undefined) {
-        return existing;
+function resolveEffectiveStateKey(item, defaultCacheMethod, environment, memo, path) {
+    if (path.has(item.id())) {
+        return Promise.resolve(LOOP);
     }
-    const { stateKey, provable } = await computeStateKey(item, defaultCacheMethod, environment, memo);
-    const effective = { stateKey, provable };
-    memo.set(item.id(), effective);
-    return effective;
+    return memoized(memo, item.id(), async () => {
+        const { stateKey, provable } = await computeStateKey(item, defaultCacheMethod, environment, memo, path);
+        return { stateKey, provable };
+    });
 }
 // Compute the combined state key for a single item together with the source
 // stats it was derived from, sharing one filesystem walk with callers that also
 // need the stats (the scheduler, the explain engine).
-async function computeStateKey(item, defaultCacheMethod, environment, memo = new Map()) {
+async function computeStateKey(item, defaultCacheMethod, environment, memo = new Map(), 
+// the items being computed on the way here, see LOOP
+parentPath = new Set()) {
+    const path = new Set(parentPath).add(item.id());
     const resolved = resolveEffective(item, defaultCacheMethod);
     const { stats, unmatched } = await (0, get_work_cache_stats_1.getWorkItemCacheStats)(item, environment);
     const ownKey = (0, get_work_cache_stats_1.getStateKey)(stats, concreteMethod(resolved, defaultCacheMethod));
     const depKeys = [];
     let provable = isOwnProvable(item, unmatched);
     for (const dep of item.deps) {
-        const effective = await resolveEffectiveStateKey(dep, defaultCacheMethod, environment, memo);
+        const effective = await resolveEffectiveStateKey(dep, defaultCacheMethod, environment, memo, path);
         depKeys.push(effective.stateKey);
         provable = provable && effective.provable;
     }
     // what a task that needs a service passed against: the sources of the service and of the
     // tasks it depends on (its definition is in the task id, getWorkServiceCacheIdentity)
     for (const need of item.needs) {
-        const effective = await resolveNeedStateKey(need.service, defaultCacheMethod, environment, memo);
+        const effective = await resolveNeedStateKey(need.service, defaultCacheMethod, environment, memo, path);
         depKeys.push(`${need.name}=${effective.stateKey}`);
     }
     return { stateKey: combineStateKeys(ownKey, depKeys), stats, resolved, provable, unmatched };
@@ -87,29 +126,27 @@ exports.computeStateKey = computeStateKey;
 // A needed service's sources folded with the state keys of its dependencies and needs. It
 // does not make the task that needs it unprovable: a dependency without `src` (resetting the
 // service's data, say) is skipped with the service when the task is a cache hit.
-async function resolveNeedStateKey(service, defaultCacheMethod, environment, memo) {
+function resolveNeedStateKey(service, defaultCacheMethod, environment, memo, parentPath) {
     const memoKey = `need\0${service.name}`;
-    const existing = memo.get(memoKey);
-    if (existing !== undefined) {
-        return existing;
-    }
-    // services that need each other in a loop fail to plan (checkForLoop): this only has to end
-    memo.set(memoKey, { stateKey: 'loop', provable: true });
-    const resolved = resolveEffective(service, defaultCacheMethod);
-    const { stats } = await (0, get_work_cache_stats_1.getWorkItemCacheStats)(service, environment);
-    const keys = [];
-    for (const dep of service.deps) {
-        keys.push((await resolveEffectiveStateKey(dep, defaultCacheMethod, environment, memo)).stateKey);
-    }
-    for (const need of service.needs) {
-        const effective = await resolveNeedStateKey(need.service, defaultCacheMethod, environment, memo);
-        keys.push(`${need.name}=${effective.stateKey}`);
+    if (parentPath.has(memoKey)) {
+        return Promise.resolve(LOOP);
     }
-    const effective = {
-        stateKey: combineStateKeys((0, get_work_cache_stats_1.getStateKey)(stats, concreteMethod(resolved, defaultCacheMethod)), keys),
-        provable: true,
-    };
-    memo.set(memoKey, effective);
-    return effective;
+    const path = new Set(parentPath).add(memoKey);
+    return memoized(memo, memoKey, async () => {
+        const resolved = resolveEffective(service, defaultCacheMethod);
+        const { stats } = await (0, get_work_cache_stats_1.getWorkItemCacheStats)(service, environment);
+        const keys = [];
+        for (const dep of service.deps) {
+            keys.push((await resolveEffectiveStateKey(dep, defaultCacheMethod, environment, memo, path)).stateKey);
+        }
+        for (const need of service.needs) {
+            const effective = await resolveNeedStateKey(need.service, defaultCacheMethod, environment, memo, path);
+            keys.push(`${need.name}=${effective.stateKey}`);
+        }
+        return {
+            stateKey: combineStateKeys((0, get_work_cache_stats_1.getStateKey)(stats, concreteMethod(resolved, defaultCacheMethod)), keys),
+            provable: true,
+        };
+    });
 }
 //# sourceMappingURL=state-key.js.map

package/dist/executer/watch-loop.js

@@ -1,20 +1,33 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.watchLoop = void 0;
+exports.watchLoop = exports.getSharedStateKeys = exports.isWatchMode = void 0;
 const watch_state_key_1 = require("./watch-state-key");
 const state_1 = require("./state");
 const abort_event_1 = require("../utils/abort-event");
 const state_resolver_1 = require("./state-resolver");
 const abort_1 = require("./abort");
 const enqueue_next_1 = require("./scheduler/enqueue-next");
-async function watchLoop(work, environment, options, factory) {
-    // a container service that watches its own files (`continuous`); a kubernetes-service and a
-    // task have no such field
+const state_key_1 = require("./scheduler/state-key");
+// Whether an item is rechecked when its files change: not in daemon mode, and not for a
+// container service that watches its own files (`continuous`); a kubernetes-service and a
+// task have no such field
+function isWatchMode(work, options) {
     const continuous = 'continuous' in work.data ? work.data.continuous : false;
-    const watchMode = options.watch && !options.daemon && !continuous;
+    return !!options.watch && !options.daemon && !continuous;
+}
+exports.isWatchMode = isWatchMode;
+// The keys of the execution to share, see getExecutionStateKeys: none in watch mode, where a
+// key is computed again because files changed
+function getSharedStateKeys(work, options) {
+    return isWatchMode(work, options) ? undefined : (0, state_key_1.getExecutionStateKeys)(options);
+}
+exports.getSharedStateKeys = getSharedStateKeys;
+async function watchLoop(work, environment, options, factory) {
+    const watchMode = isWatchMode(work, options);
     const currentStats = await (0, enqueue_next_1.checkCacheState)(work, options.cacheDefault, environment, {
         watch: watchMode,
         restore: options.restore,
+        memo: getSharedStateKeys(work, options),
     });
     (0, abort_1.checkForAbort)(environment.abortCtrl.signal);
     const watchState = watchMode

package/dist/file/get-file-context.js

@@ -39,7 +39,11 @@ function getFileContext(cwd) {
                             resolve({ type: 'directory' });
                         }
                         else if (stats.isFile()) {
-                            resolve({ type: 'file', lastModified: stats.mtimeMs });
+                            resolve({
+                                type: 'file',
+                                lastModified: stats.mtimeMs,
+                                identity: { size: stats.size, changed: stats.ctimeMs, inode: stats.ino },
+                            });
                         }
                         else {
                             resolve({ type: 'other' });

package/dist/kubernetes/claim-layout.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getClaimSubPath = void 0;
+exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getInheritedSourceSubPath = exports.getClaimSubPath = void 0;
 const crypto_1 = require("crypto");
 const path_1 = require("path");
 const portable_path_1 = require("../planner/utils/portable-path");
@@ -45,6 +45,15 @@ function getClaimSubPath(work, path) {
     return `${name}-${(0, crypto_1.createHash)('sha256').update(own).digest('hex').slice(0, 12)}`;
 }
 exports.getClaimSubPath = getClaimSubPath;
+// Where a src of one of its deps is in the claim of `work`: always with the hash of its
+// path, so it is apart from the paths of the task itself, which getClaimSubPath names
+// without looking at them (they are not part of the task's id, see claimLayout), and from
+// the other srcs of its deps (`package.json` and `core/package.json` of an `npm ci`)
+function getInheritedSourceSubPath(work, path) {
+    const name = (0, path_1.basename)(path) || 'root';
+    return `${name}-${(0, crypto_1.createHash)('sha256').update((0, portable_path_1.portablePath)(work.data.projectRoot, path)).digest('hex').slice(0, 12)}`;
+}
+exports.getInheritedSourceSubPath = getInheritedSourceSubPath;
 function byDepth(a, b) {
     return a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0);
 }

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.packSource = exports.clearStateFiles = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearStateFiles = exports.restoreLiveFiles = exports.syncLiveFiles = exports.SYNC_FILES = exports.getStateDirsInUse = exports.REMOVE_OTHER_STATES = exports.execInPod = exports.clearClaim = exports.getPodForPersistence = exports.applyWithinQuota = exports.QUOTA_TIMEOUT = exports.TRANSFER_POD_RESOURCES = void 0;
 const volumes_1 = require("./volumes");
 const apply_1 = require("./apply");
 const path_1 = require("path");
@@ -8,6 +8,7 @@ const tar_1 = require("../utils/tar");
 const resources_1 = require("./resources");
 const await_running_state_1 = require("./await-running-state");
 const sleep_1 = require("../utils/sleep");
+const abort_1 = require("../executer/abort");
 const version_1 = require("../version");
 const log_1 = require("../log");
 const ensure_kubernetes_persistent_volume_claim_exists_1 = require("./ensure-kubernetes-persistent-volume-claim-exists");
@@ -42,7 +43,44 @@ async function removeStalePod(instance, namespace, name) {
         await (0, sleep_1.sleep)(500);
     }
 }
-async function getPodForPersistence(instance, env, service, persistence, type, fn) {
+// What the pod that copies files into and out of a claim gets: tar and find, through
+// exec. Without its own, the default limit of the namespace is taken, which is sized for
+// tasks (4Gi on a server), and a run's quota then has no room for the pods of its tasks.
+exports.TRANSFER_POD_RESOURCES = {
+    requests: { cpu: '50m', memory: '64Mi' },
+    limits: { memory: '512Mi' },
+};
+// how long a transfer pod waits for room under the quota of its namespace, such as the
+// pods of other tasks of the run ending, before its task fails
+exports.QUOTA_TIMEOUT = 10 * 60 * 1000;
+// A Job waits for room under a full quota, a pod is refused: so a transfer pod is
+// created again until the pods of the tasks it runs beside are done and make room
+async function applyWithinQuota(instance, podSpec, service, timeout = exports.QUOTA_TIMEOUT, pollInterval = 2000, 
+// a run stopped while it waits: it stops waiting
+signal) {
+    const deadline = Date.now() + timeout;
+    for (let waiting = false;; waiting = true) {
+        if (signal) {
+            (0, abort_1.checkForAbort)(signal);
+        }
+        try {
+            return await (0, apply_1.apply)(instance, podSpec);
+        }
+        catch (e) {
+            if ((0, apply_1.statusCodeOf)(e) !== 403 || !/exceeded quota/i.test((0, log_1.getErrorMessage)(e)) || Date.now() >= deadline) {
+                throw e;
+            }
+            if (!waiting) {
+                service.status.write('info', `wait for room under the quota of ${podSpec.metadata.namespace}`);
+            }
+            await (0, sleep_1.sleep)(pollInterval);
+        }
+    }
+}
+exports.applyWithinQuota = applyWithinQuota;
+async function getPodForPersistence(instance, env, service, persistence, type, 
+// stops a wait for room under the quota
+signal, fn) {
     if (persistence.volumes.length === 0) {
         return;
     }
@@ -102,6 +140,7 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
                     command: ['sh'],
                     tty: true,
                     volumeMounts: mounts,
+                    resources: exports.TRANSFER_POD_RESOURCES,
                     ...(0, job_placement_1.getContainerSecurity)(env),
                 },
             ],
@@ -110,7 +149,7 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     };
     service.status.write('debug', `create pod ${name}`);
     await removeStalePod(instance, env.namespace, name);
-    const pod = await (0, apply_1.apply)(instance, podSpec);
+    const pod = await applyWithinQuota(instance, podSpec, service, exports.QUOTA_TIMEOUT, 2000, signal);
     if (pod.status?.phase !== 'Running') {
         service.status.write('debug', `await running state ${name}`);
         await (0, await_running_state_1.awaitRunningState)(instance, env, name, 'Running');
@@ -273,7 +312,7 @@ exports.syncLiveFiles = syncLiveFiles;
 // its files mounts them from: a move that did not get up leaves the service at the commit
 // it was at
 async function restoreLiveFiles(instance, env, work, persistence, from) {
-    await getPodForPersistence(instance, env, work, persistence, 'write', (pod) => syncLiveFiles(instance, env, work, persistence, pod, from));
+    await getPodForPersistence(instance, env, work, persistence, 'write', undefined, (pod) => syncLiveFiles(instance, env, work, persistence, pod, from));
 }
 exports.restoreLiveFiles = restoreLiveFiles;
 // Prepares the folder of a state in the claim of a work item of a service run before its
@@ -367,7 +406,8 @@ options = {}) {
         inUse.add(dir);
     }
     service.status.write('debug', 'start upload container');
-    await getPodForPersistence(instance, env, service, persistence, 'write', async (name) => {
+    const { signal } = environment.abortCtrl;
+    await getPodForPersistence(instance, env, service, persistence, 'write', signal, async (name) => {
         if (options.reset) {
             await clearClaim(instance, env, service, name);
         }