hammerkit 1.27.0-next.9 → 2.0.0-next.1

package/dist/kubernetes/service-rollouts.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serviceRollouts = exports.ServiceRollouts = void 0;
+const apply_1 = require("./apply");
+const log_1 = require("../log");
+/**
+ * The Deployments an execution of a service run replaced or made (rollOut). A move whose
+ * services do not all get up leaves the run where it was: the execution puts every one of
+ * them back as it found it and removes the ones it made, so no service serves the commit the
+ * run did not get to, and nothing of it takes the quota of the namespace.
+ */
+class ServiceRollouts {
+    constructor() {
+        this.rolled = new Map();
+    }
+    // before the Deployment is replaced or made; the first time counts
+    add(namespace, name, previous, restore) {
+        const key = `${namespace}/${name}`;
+        if (!this.rolled.has(key)) {
+            this.rolled.set(key, {
+                namespace,
+                name,
+                ...(previous?.spec ? { previous } : {}),
+                ...(restore ? { restore } : {}),
+            });
+        }
+    }
+    get size() {
+        return this.rolled.size;
+    }
+    async undo(instance, say) {
+        const rolled = [...this.rolled.values()];
+        this.rolled.clear();
+        for (const { namespace, name, previous, restore } of rolled) {
+            try {
+                if (previous?.spec && restore) {
+                    say(`putting the files of ${name} back as they were`);
+                    await restore();
+                }
+                if (previous?.spec) {
+                    say(`putting ${name} back as it was`);
+                    await (0, apply_1.replaceDeployment)(instance, {
+                        kind: 'Deployment',
+                        apiVersion: 'apps/v1',
+                        metadata: {
+                            namespace,
+                            name,
+                            labels: previous.metadata?.labels,
+                            annotations: previous.metadata?.annotations,
+                        },
+                        spec: previous.spec,
+                    });
+                }
+                else {
+                    say(`removing ${name}, which it made`);
+                    await instance.appsApi
+                        .deleteNamespacedDeployment({ name, namespace, propagationPolicy: 'Background' })
+                        .catch((e) => {
+                        if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                            throw e;
+                        }
+                    });
+                }
+            }
+            catch (e) {
+                say(`${name} could not be put back: ${(0, log_1.getErrorMessage)(e)}`);
+            }
+        }
+    }
+}
+exports.ServiceRollouts = ServiceRollouts;
+// the rollouts of this process: a worker executes one run
+exports.serviceRollouts = new ServiceRollouts();
+//# sourceMappingURL=service-rollouts.js.map

package/dist/kubernetes/store-kubernetes-data.js

@@ -5,8 +5,11 @@ const log_1 = require("../log");
 const event_cache_1 = require("../executer/event-cache");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
-async function storeKubernetesData(work, kubernetes, instance, environment, path) {
-    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes);
+async function storeKubernetesData(work, kubernetes, instance, environment, path, 
+// the state the outputs are of: on a service run, they are in its folder (see
+// getStateFilesDir)
+stateKey) {
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, stateKey);
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'read', async (name) => {
         for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
             const writerStream = environment.file.createWriteStream(generatedArchive.filename);

package/dist/kubernetes/volumes.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec = exports.appendVolume = exports.getKubernetesPersistence = exports.getVolumeName = void 0;
+exports.getItemFilesDir = exports.isLiveService = exports.LIVE_FILES_DIR = exports.getStateFilesDir = exports.STATE_FILES_PREFIX = exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getClaimMountLabel = exports.getClaimPodSpec = exports.getWorkspacePodSpec = exports.appendVolume = exports.getKubernetesPersistence = exports.getVolumeName = void 0;
 const path_1 = require("path");
 const fs_1 = require("fs");
+const crypto_1 = require("crypto");
 const kept_claims_1 = require("./kept-claims");
 const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
@@ -41,7 +42,15 @@ function appendWorkspaceMount(workspace, persistence, containerPath, localPath,
     });
     return true;
 }
-async function getKubernetesPersistence(work, kubernetes) {
+async function getKubernetesPersistence(work, kubernetes, 
+// the state the work runs at, on a service run (see getStateFilesDir); what it inherits is
+// at the state its own work item completed at
+stateKey) {
+    const dirOf = (item) => kubernetes.server?.serviceRun ? getItemFilesDir(item, item === work ? stateKey : undefined) : undefined;
+    const own = dirOf(work);
+    // a service that watches its files mounts them from the folder they are synced into, and
+    // takes them into the folder of their state (see syncLiveFiles)
+    const mounted = own && isLiveService(work, kubernetes) ? exports.LIVE_FILES_DIR : own;
     const persistence = {
         volumes: [],
         sizes: {},
@@ -57,7 +66,7 @@ async function getKubernetesPersistence(work, kubernetes) {
             appendWorkspaceMount(workspace, persistence, mount.containerPath, mount.localPath, mount.isFile, mount.readOnly)) {
             continue;
         }
-        appendVolume(work, persistence, mount.containerPath, mount.localPath, mount.isFile, true, kubernetes);
+        appendVolume(work, persistence, mount.containerPath, mount.localPath, mount.isFile, true, kubernetes, mounted, own);
     }
     for (const src of work.data.src) {
         if (src.inherited) {
@@ -67,24 +76,32 @@ async function getKubernetesPersistence(work, kubernetes) {
             appendWorkspaceMount(workspace, persistence, src.absolutePath, src.absolutePath, src.isFile, false)) {
             continue;
         }
-        appendVolume(work, persistence, src.absolutePath, src.absolutePath, src.isFile, true, kubernetes); // TODO matcher
+        appendVolume(work, persistence, src.absolutePath, src.absolutePath, src.isFile, true, kubernetes, mounted, own); // TODO matcher
     }
     if (work.data.type === 'container-task') {
         for (const generate of work.data.generates) {
-            appendVolume(generate.inherited ?? work, persistence, generate.path, null, false, !!generate.inherited, kubernetes);
+            appendVolume(generate.inherited ?? work, persistence, generate.path, null, false, !!generate.inherited, kubernetes, dirOf(generate.inherited ?? work));
         }
     }
     else {
         for (const volume of work.data.volumes) {
-            appendVolume(volume.inherited ?? work, persistence, volume.containerPath, null, false, !!volume.inherited || volume.readOnly, kubernetes);
+            appendVolume(volume.inherited ?? work, persistence, volume.containerPath, null, false, !!volume.inherited || volume.readOnly, kubernetes, 
+            // the data of a service stays where it is, the outputs of its deps are by their state
+            volume.inherited ? dirOf(volume.inherited) : undefined);
         }
     }
     return persistence;
 }
 exports.getKubernetesPersistence = getKubernetesPersistence;
-function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
+function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes, 
+// the folder of the state of `sourceWork` the path is in, see getStateFilesDir
+stateDir, 
+// the folder its upload goes to, when it is not the one the path is mounted from (a
+// service that watches its files, see syncLiveFiles)
+uploadDir = stateDir) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
-    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
+    const claimPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
+    const subPath = (stateDir ? `${stateDir}/` : '') + claimPath;
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -113,7 +130,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
-            subPath,
+            subPath: (uploadDir ? `${uploadDir}/` : '') + claimPath,
             stateKey: '', // TODO
         });
     }
@@ -128,6 +145,9 @@ exports.appendVolume = appendVolume;
  */
 function getWorkspacePodSpec(kubernetes, persistence, task) {
     const workspace = kubernetes.server?.workspace;
+    if (kubernetes.server?.serviceRun && !workspace) {
+        return getClaimPodSpec(persistence.volumes ?? []);
+    }
     if (!workspace || persistence.workspace.length === 0) {
         return { labels: {} };
     }
@@ -141,6 +161,41 @@ function getWorkspacePodSpec(kubernetes, persistence, task) {
     };
 }
 exports.getWorkspacePodSpec = getWorkspacePodSpec;
+/**
+ * A pod of a service run that mounts claims: labelled by each of them, and made to go where a
+ * pod runs that mounts all of them. A run that moves to another commit starts a new pod of a
+ * service, a Job of a changed task or an upload while the pods of the commit before still
+ * mount the same claims, which a cloud disk attaches to one node at a time: a pod on another
+ * node would wait for the disk until its deadline. Required, as one term: while no pod mounts
+ * all of them (the first one, or after the run slept), the scheduler lets the pod go anywhere,
+ * because it matches the term itself, so a Deployment that keeps the term is never held back
+ * by it. Mounting all of them is what holds the disks on one node.
+ */
+function getClaimPodSpec(volumes) {
+    const keys = [
+        ...new Set(volumes.flatMap((volume) => volume.persistentVolumeClaim ? [getClaimMountLabel(volume.persistentVolumeClaim.claimName)] : [])),
+    ].sort();
+    if (keys.length === 0) {
+        return { labels: {} };
+    }
+    const labels = Object.fromEntries(keys.map((key) => [key, 'mounted']));
+    return {
+        labels,
+        affinity: {
+            podAffinity: {
+                requiredDuringSchedulingIgnoredDuringExecution: [
+                    { labelSelector: { matchLabels: labels }, topologyKey: 'kubernetes.io/hostname' },
+                ],
+            },
+        },
+    };
+}
+exports.getClaimPodSpec = getClaimPodSpec;
+// a claim name is longer than a label name may be: its hash
+function getClaimMountLabel(claimName) {
+    return `claim.hammerkit.dev/${(0, crypto_1.createHash)('sha256').update(claimName).digest('hex').slice(0, 16)}`;
+}
+exports.getClaimMountLabel = getClaimMountLabel;
 // the label of the pods that mount the claim of a sandbox
 exports.SANDBOX_LABEL = 'hammerkit.dev/sandbox';
 function getSandboxAffinity(label) {
@@ -153,4 +208,34 @@ function getSandboxAffinity(label) {
     };
 }
 exports.getSandboxAffinity = getSandboxAffinity;
+// The folder of a service run's claim that holds the files of one state of a work item: the
+// src and mounts it uploads, and the outputs of a task. A run that moves to another commit
+// uploads and builds the files of the new state into a folder of their own, so the pods of
+// the commit before keep the files and outputs they started with until the new ones are up,
+// and a Deployment that is put back finds them again. The volumes of a service hold its data,
+// which stays where it is.
+exports.STATE_FILES_PREFIX = '.hammerkit-files-';
+function getStateFilesDir(stateKey) {
+    return `${exports.STATE_FILES_PREFIX}${(0, crypto_1.createHash)('sha256').update(stateKey).digest('hex').slice(0, 16)}`;
+}
+exports.getStateFilesDir = getStateFilesDir;
+// The folder of the claim of a service of a service run that watches its own files
+// (`continuous`), which its pod mounts its `src` and `mounts` from. The files of each state
+// are uploaded into the folder of that state, and synced into this one from there: the pod
+// stays, and the service sees the files of the new commit (syncLiveFiles).
+exports.LIVE_FILES_DIR = `${exports.STATE_FILES_PREFIX}live`;
+function isLiveService(work, kubernetes) {
+    return (!!kubernetes.server?.serviceRun &&
+        !kubernetes.server.workspace &&
+        work.data.type === 'container-service' &&
+        work.data.continuous === true);
+}
+exports.isLiveService = isLiveService;
+// The folder of the state of `item`: `stateKey`, or the one it completed at in this run
+function getItemFilesDir(item, stateKey) {
+    const key = stateKey ??
+        item.state?.current.stateKey;
+    return key ? getStateFilesDir(key) : undefined;
+}
+exports.getItemFilesDir = getItemFilesDir;
 //# sourceMappingURL=volumes.js.map

package/dist/main.js

@@ -8,6 +8,7 @@ const redact_1 = require("./utils/redact");
 const empty_writable_1 = require("./utils/empty-writable");
 const run_program_1 = require("./run-program");
 const abort_on_signals_1 = require("./utils/abort-on-signals");
+const terminal_prompt_1 = require("./utils/terminal-prompt");
 const abortCtrl = new AbortController();
 (0, abort_on_signals_1.abortOnSignals)(process, abortCtrl);
 let settled = false;
@@ -32,6 +33,7 @@ const secrets = (0, redact_1.createSecretRegistry)();
     stdout: process.stdout,
     stderr: process.stderr,
     stdoutColumns: process.stdout.columns,
+    prompt: (0, terminal_prompt_1.getTerminalPrompt)(process.env, abortCtrl.signal),
 }, process.argv, false).then(() => {
     settled = true;
 }, () => {

package/dist/optimizer/calculate-checksum.js

@@ -4,9 +4,10 @@ exports.calculateChecksum = void 0;
 const crypto_1 = require("crypto");
 // Hashes the raw bytes: decoding to text first would map every invalid UTF-8
 // sequence to U+FFFD, so two different binary files could share a checksum.
-// For UTF-8 text the result is unchanged.
+// For UTF-8 text the result is unchanged. sha256, which nobody can make two files share:
+// a cache hit is also what a verified check counts (check-key.ts).
 async function calculateChecksum(environment, path) {
-    const hash = (0, crypto_1.createHash)('sha1');
+    const hash = (0, crypto_1.createHash)('sha256');
     for await (const chunk of environment.file.readStream(path)) {
         hash.update(chunk);
     }

package/dist/optimizer/check-key.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCheckKey = void 0;
+const crypto_1 = require("crypto");
+const state_key_1 = require("../executer/scheduler/state-key");
+const get_work_cache_stats_1 = require("./get-work-cache-stats");
+const work_task_cache_description_1 = require("./work-task-cache-description");
+/**
+ * What a passed check is recorded under, and what `remote verify` looks it up by: the
+ * inputs of the task's cache key, so a cached result counts for the same key it was
+ * stored under. Its definition (the services it needs included), the files of its `src`,
+ * the keys of its dependencies, and the sources of the services it needs and of the tasks
+ * they depend on.
+ *
+ * It is computed on a checkout before anything ran in it, on both ends, so it also covers
+ * the files a commit holds where a dependency generates (the cache leaves those out, as
+ * they change once the dependency ran), and it leaves out `.git`, which is no file of a
+ * commit. Two checkouts of the same files give the same key wherever they are.
+ * Null for a task that can not be proven up to date (no `src` of its own, or none that
+ * matches a file, here or in a dependency): its result says nothing about any files.
+ */
+async function getCheckKey(item, environment, memo = new Map(), path = new Set()) {
+    return memoized(memo, path, `task\0${item.id()}`, (inner) => computeCheckKey(item, environment, memo, inner));
+}
+exports.getCheckKey = getCheckKey;
+async function computeCheckKey(item, environment, memo, path) {
+    const { stats, unmatched } = await (0, get_work_cache_stats_1.getWorkItemCheckoutStats)(item, environment);
+    if (!(0, state_key_1.isOwnProvable)(item, unmatched)) {
+        return null;
+    }
+    const inputs = await getInputKeys(item, environment, memo, path);
+    if (inputs === null) {
+        return null;
+    }
+    return hash({ task: (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item), files: commitFiles(stats), ...inputs });
+}
+// The sources of a needed service, with the keys of what it depends on and needs; its
+// definition is in the description of the task that needs it.
+function getNeedKey(item, environment, memo, path) {
+    return memoized(memo, path, `need\0${item.name}`, async (inner) => {
+        const inputs = await getInputKeys(item, environment, memo, inner);
+        if (inputs === null) {
+            return null;
+        }
+        return hash({ files: commitFiles(await (0, get_work_cache_stats_1.getWorkCacheStats)(item.data, environment)), ...inputs });
+    });
+}
+async function getInputKeys(item, environment, memo, path) {
+    const deps = [];
+    for (const dep of item.deps) {
+        const key = await getCheckKey(dep, environment, memo, path);
+        if (key === null) {
+            return null;
+        }
+        deps.push(key);
+    }
+    const needs = [];
+    for (const need of item.needs) {
+        const key = await getNeedKey(need.service, environment, memo, path);
+        if (key === null) {
+            return null;
+        }
+        needs.push(`${need.name}=${key}`);
+    }
+    return { deps: deps.sort(), needs: needs.sort() };
+}
+// a commit never holds `.git`: what a checkout keeps there differs between a CI job's clone,
+// the server's fetch and a sandbox's copy, which has none
+function commitFiles(stats) {
+    return Object.keys(stats.files)
+        .filter((path) => !path.split('/').includes('.git'))
+        .sort()
+        .map((path) => [path, stats.files[path].checksum]);
+}
+function hash(value) {
+    return (0, crypto_1.createHash)('sha256').update(JSON.stringify(value)).digest('hex');
+}
+// `path` holds the keys being computed on the way here: one coming back is a loop (services
+// that need each other), which can not be proven and fails to plan anyway. Waiting on its
+// pending promise would never end. The keys are computed one after another, so every pending
+// promise in the memo is on the path.
+function memoized(memo, path, key, compute) {
+    if (path.has(key)) {
+        return Promise.resolve(null);
+    }
+    const known = memo.get(key);
+    if (known) {
+        return known;
+    }
+    const computed = compute(new Set([...path, key]));
+    memo.set(key, computed);
+    return computed;
+}
+//# sourceMappingURL=check-key.js.map

package/dist/optimizer/check-keys.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCheckKeys = void 0;
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const reference_parser_1 = require("../schema/reference-parser");
+const schema_parser_1 = require("../schema/schema-parser");
+const work_scope_parser_1 = require("../schema/work-scope-parser");
+const log_1 = require("../log");
+const check_key_1 = require("./check-key");
+/**
+ * The check keys of the tasks of a scope, as both ends compute them: the process of a run on
+ * the server before it executes anything, and `remote verify` in a CI job's checkout. So the
+ * build file is read the same way on both, as a server reads it (no `.env`, links are files)
+ * but placed nowhere: where a server runs a task (a pool, its tool image, the arch of its
+ * nodes) is the server's, and a result counts for the server it passed on. The secrets'
+ * providers and accounts are the server's too, and no part of a key, so any is taken here.
+ */
+async function getCheckKeys(entry, environment, scope) {
+    const keyed = { ...environment, hosted: { catalog: anyCatalog(), localTasksInContainers: true } };
+    const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(entry, keyed);
+    const workTree = (0, work_scope_parser_1.getWorkContext)(await (0, reference_parser_1.parseReferences)(ctx, parsed, keyed), scope, keyed);
+    const memo = new Map();
+    const keys = {};
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        try {
+            const key = await (0, check_key_1.getCheckKey)(task, keyed, memo);
+            keys[task.name] = key
+                ? { key }
+                : {
+                    key: null,
+                    reason: 'it, or a task it or a service it needs depends on, declares no src that matches a file',
+                };
+        }
+        catch (e) {
+            // a secret whose value is part of the key (`cache: true`) is not known here
+            keys[task.name] = { key: null, reason: (0, log_1.getErrorMessage)(e) };
+        }
+    }
+    return keys;
+}
+exports.getCheckKeys = getCheckKeys;
+// Every provider and account name resolves, to a definition nothing runs.
+function anyCatalog() {
+    const provider = (name) => ({
+        name,
+        type: 'command',
+        command: [],
+        env: {},
+        timeoutMs: 0,
+        declaredIn: 'the server',
+    });
+    const account = (name) => ({
+        name,
+        default: false,
+        providers: new Proxy({}, { get: () => ({ env: {} }) }),
+        declaredIn: 'the server',
+    });
+    return {
+        providers: new Proxy({}, { get: (_target, name) => (typeof name === 'string' ? provider(name) : undefined) }),
+        accounts: new Proxy({}, { get: (_target, name) => (typeof name === 'string' ? account(name) : undefined) }),
+        explicitAccounts: true,
+    };
+}
+//# sourceMappingURL=check-keys.js.map

package/dist/optimizer/get-work-cache-stats.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getStateKey = exports.getWorkItemCacheStats = exports.getWorkCacheStats = void 0;
+exports.stateKeyDigest = exports.getStateKey = exports.getWorkItemCheckoutStats = exports.getWorkItemCacheStats = exports.getWorkCacheStats = void 0;
 const calculate_checksum_1 = require("./calculate-checksum");
 const path_1 = require("path");
 const crypto_1 = require("crypto");
@@ -20,7 +20,7 @@ async function addWorkCacheStats(result, cwd, path, matcher, context, root, excl
             if (!matcher(path)) {
                 return 0;
             }
-            const checksum = (0, crypto_1.createHash)('sha1')
+            const checksum = (0, crypto_1.createHash)('sha256')
                 .update(`link:${await (0, promises_1.readlink)(path)}`)
                 .digest('hex');
             result.files[(0, path_1.relative)(cwd, path).split(path_1.sep).join(path_1.posix.sep)] = { lastModified: link.mtimeMs, checksum };
@@ -85,6 +85,13 @@ function getWorkItemCacheStats(item, environment) {
     return collectSourceStats(item.data, environment, dependencyOutputs(item));
 }
 exports.getWorkItemCacheStats = getWorkItemCacheStats;
+// The source stats of a task with what its dependencies generate: on a checkout nothing ran
+// in, those are the files a commit holds there, see check-key.ts.
+async function getWorkItemCheckoutStats(item, environment) {
+    const { stats } = await collectSourceStats(item.data, environment, []);
+    return { stats, unmatched: (await getWorkItemCacheStats(item, environment)).unmatched };
+}
+exports.getWorkItemCheckoutStats = getWorkItemCheckoutStats;
 function dependencyOutputs(item) {
     const outputs = new Set();
     const visited = new Set();
@@ -113,7 +120,14 @@ function getStateKey(stats, cacheMethod) {
             contents.push(`${key}:${stats.files[key].lastModified}`);
         }
     }
-    return (0, crypto_1.createHash)('md5').update(contents.join(',')).digest('hex');
+    return stateKeyDigest(contents.join(','));
 }
 exports.getStateKey = getStateKey;
+// A state key is also the value of the `hammerkit.dev/state` label on Kubernetes, which
+// takes at most 63 characters: the sha256 in base 36 has 50.
+function stateKeyDigest(content) {
+    const hex = (0, crypto_1.createHash)('sha256').update(content).digest('hex');
+    return BigInt(`0x${hex}`).toString(36).padStart(50, '0');
+}
+exports.stateKeyDigest = stateKeyDigest;
 //# sourceMappingURL=get-work-cache-stats.js.map

package/dist/optimizer/work-task-cache-description.js

@@ -1,12 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storedCacheDescription = exports.getWorkTaskCacheDescription = void 0;
+exports.storedCacheDescription = exports.getWorkServiceCacheIdentity = exports.getWorkTaskCacheDescription = void 0;
 const os_1 = require("os");
 const crypto_1 = require("crypto");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const portable_path_1 = require("../planner/utils/portable-path");
 const work_secret_1 = require("../planner/work-secret");
 const claim_layout_1 = require("../kubernetes/claim-layout");
+const work_service_1 = require("../planner/work-service");
+const work_service_id_1 = require("../planner/work-service-id");
+const work_cache_id_1 = require("../planner/work-cache-id");
 // Every path is made project-relative (portablePath) so the description, and the
 // task id hashed from it, is identical wherever the project is checked out.
 //
@@ -42,9 +45,54 @@ function getWorkTaskCacheDescription(item) {
         deps: item.deps.length > 0 ? item.deps.map((dep) => dep.id()).sort() : undefined,
         secrets: cachedSecrets.length > 0 ? cachedSecrets : undefined,
         claimLayout: (0, claim_layout_1.hasPathsEndingAlike)(item) || (0, claim_layout_1.hasOutputsFlatteningAlike)(item) ? 2 : undefined,
+        needs: item.needs.length > 0 ? getNeedsIdentities(item) : undefined,
     };
 }
 exports.getWorkTaskCacheDescription = getWorkTaskCacheDescription;
+// `visiting` holds the services on the way here: services that need each other in a loop
+// fail to plan (checkForLoop), the identity only has to come back to say so
+function getNeedsIdentities(item, visiting = new Set()) {
+    return (item.needs
+        // an init task needs the service it initializes, whose identity holds the init's id
+        .filter((need) => !isInitOf(item, need.service))
+        .map((need) => visiting.has(need.service)
+        ? `${need.name}=loop`
+        : `${need.name}=${getWorkServiceCacheIdentity(need.service, visiting)}`)
+        .sort());
+}
+function isInitOf(item, service) {
+    return (0, work_service_1.isContainerWorkService)(service.data) && service.data.init?.task.data === item.data;
+}
+// Everything that makes a service what a task needing it talks to: its definition, the ids of
+// the tasks it depends on and of its init task, and the services it needs in turn. The source
+// of those tasks and of the service is in the state key (computeStateKey), like a dependency's.
+// Not the service id, which leaves out what a service runs (getWorkServiceId).
+function getWorkServiceCacheIdentity(item, visiting = new Set()) {
+    const service = item.data;
+    const definition = (0, work_service_1.isContainerWorkService)(service)
+        ? {
+            cmd: service.cmd?.cmd ?? null,
+            envs: (0, replace_env_variables_1.getEnvironmentVariables)(service.envs),
+            healthcheck: service.healthcheck?.cmd.cmd ?? null,
+            secrets: service.secrets
+                .filter((secret) => secret.cache)
+                .map((secret) => `${(0, work_secret_1.getPortableSecretName)(service.projectRoot, secret)}=${secret.digest()}`)
+                .sort(),
+            // as declared: arguments after -- of a run of the init leave the service as it is
+            init: service.init ? (0, work_cache_id_1.getDeclaredWorkTaskId)(service.init.task) : null,
+        }
+        : { namespace: service.namespace };
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify({
+        id: (0, work_service_id_1.getWorkServiceId)(service),
+        ...definition,
+        ports: service.ports.map((port) => `${port.hostPort ?? ''}:${port.containerPort}`).sort(),
+        deps: item.deps.map((dep) => dep.id()).sort(),
+        needs: getNeedsIdentities(item, new Set([...visiting, item])),
+    }))
+        .digest('hex');
+}
+exports.getWorkServiceCacheIdentity = getWorkServiceCacheIdentity;
 // The form written to disk: description.json in a cache entry (pushed to remote
 // backends, readable by everyone who can pull) and the local explain record.
 // Env is where tokens get passed, so each value is stored as a digest; the task

package/dist/planner/service-definition.js

@@ -6,6 +6,7 @@ const work_secret_1 = require("./work-secret");
 const work_service_1 = require("./work-service");
 const work_network_1 = require("./work-network");
 const work_cache_id_1 = require("./work-cache-id");
+const portable_path_1 = require("./utils/portable-path");
 // Everything a running container of the service was created from, plus the
 // definitions of the services it needs: a dependent links to the container it
 // started with, so a recreated need means a recreated dependent too. Stored as
@@ -14,19 +15,23 @@ const work_cache_id_1 = require("./work-cache-id");
 function getServiceDefinitionHash(item) {
     const service = item.data;
     const network = (0, work_network_1.getEffectiveNetwork)(service);
+    const portable = (path) => (0, portable_path_1.portablePath)(service.projectRoot, path);
     const definition = {
         image: service.image,
         cmd: service.cmd?.cmd ?? null,
         shell: service.shell,
-        workdir: service.workdir,
+        // where the checkout is does not count: a service run that moves to another commit runs
+        // from another directory beside the first, and keeps what did not change
+        workdir: service.workdir ? portable(service.workdir) : null,
         envs: {
             variables: service.envs.variables,
             replacements: service.envs.replacements.map((r) => [r.key, r.name, r.value]),
         },
         ports: service.ports.map((p) => `${p.hostPort ?? ''}:${p.containerPort}`),
-        mounts: service.mounts.map((m) => `${m.localPath}:${m.containerPath}:${m.readOnly ? 'ro' : 'rw'}`),
-        volumes: service.volumes.map((v) => `${v.name}:${v.containerPath}:${v.readOnly ? 'ro' : 'rw'}`),
-        src: service.src.map((s) => s.absolutePath),
+        mounts: service.mounts.map((m) => `${portable(m.localPath)}:${m.containerPath}:${m.readOnly ? 'ro' : 'rw'}`),
+        // the volume of a dep's output is named by where it is, which its path says already
+        volumes: service.volumes.map((v) => `${v.inherited ? '' : v.name}:${portable(v.containerPath)}:${v.readOnly ? 'ro' : 'rw'}`),
+        src: service.src.map((s) => portable(s.absolutePath)),
         healthcheck: service.healthcheck?.cmd.cmd ?? null,
         // where secrets go, never their values: a rotated value needs a `down`
         secrets: service.secrets.map((s) => s.source.type === 'provider'

package/dist/planner/utils/append-work-dependencies.js

@@ -17,8 +17,10 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                 }
             }
         }
-        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
-            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+        // an aggregate hands on what its dependencies generate, as if what depends on it
+        // depended on them
+        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data) || (0, work_task_1.isAggregateTask)(depNode.data)) &&
+            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data) || (0, work_task_1.isAggregateTask)(item.data))) {
             // the outputs of the dep's own deps too, which stay those of the task that made them
             // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
@@ -39,7 +41,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                         });
                     }
                 }
-                else {
+                else if ((0, work_task_1.isWorkTask)(item.data)) {
                     if (!item.data.generates.some((v) => v.volumeName === generate.volumeName)) {
                         item.data.generates.push({
                             volumeName: generate.volumeName,