hammerkit 1.27.0-next.2 → 1.27.0-next.20

package/dist/kubernetes/volumes.js

@@ -4,6 +4,7 @@ exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec
 const path_1 = require("path");
 const fs_1 = require("fs");
 const kept_claims_1 = require("./kept-claims");
+const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
     return (0, kept_claims_1.getClaimName)(service, kubernetes);
 }
@@ -50,7 +51,7 @@ async function getKubernetesPersistence(work, kubernetes) {
         mounts: [],
         initCommands: [],
     };
-    const workspace = kubernetes?.server?.workspace;
+    const workspace = kubernetes.server?.workspace;
     for (const mount of work.data.mounts) {
         if (workspace &&
             appendWorkspaceMount(workspace, persistence, mount.containerPath, mount.localPath, mount.isFile, mount.readOnly)) {
@@ -83,6 +84,7 @@ async function getKubernetesPersistence(work, kubernetes) {
 exports.getKubernetesPersistence = getKubernetesPersistence;
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
+    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -92,7 +94,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             },
         });
         persistence.sizes[volumeName] = 'volumeSize' in sourceWork.data ? sourceWork.data.volumeSize : null;
-        if (kubernetes && (0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
+        if ((0, kept_claims_1.isKeptClaim)(sourceWork, kubernetes)) {
             persistence.kept.push(volumeName);
         }
     }
@@ -101,7 +103,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             readOnly,
             mountPath: containerPath,
             name: volumeName,
-            subPath: (0, path_1.basename)(containerPath),
+            subPath,
         },
         uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
         cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
@@ -111,6 +113,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
+            subPath,
             stateKey: '', // TODO
         });
     }

package/dist/network/egress-proxy.js

@@ -5,10 +5,12 @@
 // `CONNECT` tunnels to an allowed host pass, everything else gets a 403 and a
 // `refused` event on stdout. For each `host:port` entry it also listens on that
 // port and forwards raw TCP to that host, which the item reaches under the
-// host's name. The script is passed to `node -e`, so the proxy
+// host's name. On a server's cluster it also lets the init containers that read
+// the task's secrets reach the endpoints of their providers: a tunnel to port 443
+// that carries the proxy's token, which those containers get and the task not. The script is passed to `node -e`, so the proxy
 // needs no image of its own, only a stock Node image.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
+exports.formatEgressProxyEvent = exports.formatRefusedHost = exports.parseEgressProxyEvent = exports.getProxylessEnvironment = exports.getProxyClientEnvironment = exports.getEgressProxyPorts = exports.getItemProxyEnvironment = exports.getSecretProxyEnvironment = exports.getEgressProxyEnvironment = exports.getEgressProxyCommand = exports.EGRESS_PROXY_SCRIPT = exports.EGRESS_PROXY_HOST = exports.EGRESS_PROXY_PORT = exports.EGRESS_PROXY_IMAGE = void 0;
 const work_network_1 = require("../planner/work-network");
 const network_schema_1 = require("../schema/network-schema");
 // Pinned, so the code behind the boundary is the same on every host. To bump
@@ -25,6 +27,7 @@ exports.EGRESS_PROXY_SCRIPT = `
 const http = require('http')
 const net = require('net')
 const dns = require('dns')
+const crypto = require('crypto')
 const entries = JSON.parse(process.env.HAMMERKIT_ALLOW || '[]').map((h) => h.toLowerCase())
 // names and patterns for HTTP(S)
 const allow = entries.filter((e) => !e.includes(':'))
@@ -38,6 +41,9 @@ const publicOnly = JSON.parse(process.env.HAMMERKIT_PUBLIC_ONLY || '[]').map((h)
 // the names of the needed services, for tools that ignore NO_PROXY: exact
 // names only, never patterns
 const needs = JSON.parse(process.env.HAMMERKIT_NEEDS || '[]').map((h) => h.toLowerCase())
+// the endpoints of secret providers, for a request with the token only
+const secretAllow = JSON.parse(process.env.HAMMERKIT_SECRET_ALLOW || '[]').map((h) => h.toLowerCase())
+const secretToken = process.env.HAMMERKIT_SECRET_TOKEN || ''
 const port = parseInt(process.env.HAMMERKIT_PROXY_PORT || '${exports.EGRESS_PROXY_PORT}', 10)
 // the name whose address the proxy listens on, every address when unset
 const bind = process.env.HAMMERKIT_PROXY_BIND
@@ -64,6 +70,31 @@ function allowedBy(host, port) {
   }
   return allow.some((rule) => rule.startsWith('*.') && name.endsWith(rule.substring(1))) ? 'wildcard' : null
 }
+// Whether the request carries the token as the password of Proxy-Authorization (Basic).
+function hasSecretToken(req) {
+  if (secretToken.length === 0) {
+    return false
+  }
+  const match = /^basic\\s+(\\S+)$/i.exec(String(req.headers['proxy-authorization'] || ''))
+  if (!match) {
+    return false
+  }
+  const credentials = Buffer.from(match[1], 'base64').toString('utf8')
+  const colon = credentials.indexOf(':')
+  if (colon < 0) {
+    return false
+  }
+  const given = Buffer.from(credentials.substring(colon + 1), 'utf8')
+  const expected = Buffer.from(secretToken, 'utf8')
+  return given.length === expected.length && crypto.timingSafeEqual(given, expected)
+}
+// the rule of a tunnel to an endpoint of a secret provider: like a wildcard, it reaches
+// no private address
+function allowedForSecrets(req, host, port) {
+  const name = host.toLowerCase().replace(/\\.$/, '')
+  const listed = secretAllow.some((rule) => (rule.startsWith('*.') ? name.endsWith(rule.substring(1)) : name === rule))
+  return listed && port === 443 && hasSecretToken(req) ? 'wildcard' : null
+}
 function ipv4(address) {
   const mapped = /^::ffff:(\\d+\\.\\d+\\.\\d+\\.\\d+)$/i.exec(address)
   const value = mapped ? mapped[1] : address
@@ -197,7 +228,7 @@ const server = http.createServer((req, res) => {
 server.on('connect', (req, socket, head) => {
   const target = parseAuthority(req.url || '')
   socket.on('error', () => socket.destroy())
-  const rule = target ? allowedBy(target.host, target.port) : null
+  const rule = target ? allowedBy(target.host, target.port) || allowedForSecrets(req, target.host, target.port) : null
   if (!rule) {
     refuse(target ? target.host : String(req.url))
     socket.end('HTTP/1.1 403 Forbidden\\r\\ncontent-type: text/plain\\r\\n\\r\\nhammerkit: network access is not allowed\\n')
@@ -290,7 +321,8 @@ exports.getEgressProxyCommand = getEgressProxyCommand;
 // `needs` are the names of the needed services, allowed as exactly those
 // names. `bind` is the name the proxy listens on alone, so that nothing on its
 // other networks reaches it. `publicOnly` are the entries of `hosts` that reach
-// no private address (see WorkNetwork).
+// no private address (see WorkNetwork). The endpoints of secret providers and
+// the token they need are set by the caller (getSecretProxyEnvironment).
 function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
     return {
         HAMMERKIT_ALLOW: JSON.stringify(hosts),
@@ -301,6 +333,12 @@ function getEgressProxyEnvironment(hosts, needs, bind, publicOnly = []) {
     };
 }
 exports.getEgressProxyEnvironment = getEgressProxyEnvironment;
+// The endpoints of secret providers the proxy lets a tunnel with its token reach; the
+// token itself is set where the proxy runs, see HAMMERKIT_SECRET_TOKEN.
+function getSecretProxyEnvironment(hosts) {
+    return hosts.length > 0 ? { HAMMERKIT_SECRET_ALLOW: JSON.stringify(hosts) } : {};
+}
+exports.getSecretProxyEnvironment = getSecretProxyEnvironment;
 // The proxy of a task or service: the hosts of its network in this run (under
 // the run's ceiling), and the names of the services it needs, on every runtime.
 function getItemProxyEnvironment(item, bind) {
@@ -315,8 +353,9 @@ function getEgressProxyPorts(network) {
     return [exports.EGRESS_PROXY_PORT, ...(0, work_network_1.getTcpForwards)(network).map((forward) => forward.port)];
 }
 exports.getEgressProxyPorts = getEgressProxyPorts;
-function getProxyClientEnvironment(directHosts) {
-    const proxy = `http://${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
+// `token` is sent as the password, for the endpoints of secret providers.
+function getProxyClientEnvironment(directHosts, token) {
+    const proxy = `http://${token !== undefined ? `hammerkit:${token}@` : ''}${exports.EGRESS_PROXY_HOST}:${exports.EGRESS_PROXY_PORT}`;
     const noProxy = ['localhost', '127.0.0.1', ...directHosts].join(',');
     return {
         HTTP_PROXY: proxy,

package/dist/optimizer/calculate-checksum.js

@@ -4,9 +4,10 @@ exports.calculateChecksum = void 0;
 const crypto_1 = require("crypto");
 // Hashes the raw bytes: decoding to text first would map every invalid UTF-8
 // sequence to U+FFFD, so two different binary files could share a checksum.
-// For UTF-8 text the result is unchanged.
+// For UTF-8 text the result is unchanged. sha256, which nobody can make two files share:
+// a cache hit is also what a verified check counts (check-key.ts).
 async function calculateChecksum(environment, path) {
-    const hash = (0, crypto_1.createHash)('sha1');
+    const hash = (0, crypto_1.createHash)('sha256');
     for await (const chunk of environment.file.readStream(path)) {
         hash.update(chunk);
     }

package/dist/optimizer/check-key.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCheckKey = void 0;
+const crypto_1 = require("crypto");
+const state_key_1 = require("../executer/scheduler/state-key");
+const get_work_cache_stats_1 = require("./get-work-cache-stats");
+const work_task_cache_description_1 = require("./work-task-cache-description");
+/**
+ * What a passed check is recorded under, and what `remote verify` looks it up by: the
+ * inputs of the task's cache key, so a cached result counts for the same key it was
+ * stored under. Its definition (the services it needs included), the files of its `src`,
+ * the keys of its dependencies, and the sources of the services it needs and of the tasks
+ * they depend on.
+ *
+ * It is computed on a checkout before anything ran in it, on both ends, so it also covers
+ * the files a commit holds where a dependency generates (the cache leaves those out, as
+ * they change once the dependency ran), and it leaves out `.git`, which is no file of a
+ * commit. Two checkouts of the same files give the same key wherever they are.
+ * Null for a task that can not be proven up to date (no `src` of its own, or none that
+ * matches a file, here or in a dependency): its result says nothing about any files.
+ */
+async function getCheckKey(item, environment, memo = new Map(), path = new Set()) {
+    return memoized(memo, path, `task\0${item.id()}`, (inner) => computeCheckKey(item, environment, memo, inner));
+}
+exports.getCheckKey = getCheckKey;
+async function computeCheckKey(item, environment, memo, path) {
+    const { stats, unmatched } = await (0, get_work_cache_stats_1.getWorkItemCheckoutStats)(item, environment);
+    if (!(0, state_key_1.isOwnProvable)(item, unmatched)) {
+        return null;
+    }
+    const inputs = await getInputKeys(item, environment, memo, path);
+    if (inputs === null) {
+        return null;
+    }
+    return hash({ task: (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item), files: commitFiles(stats), ...inputs });
+}
+// The sources of a needed service, with the keys of what it depends on and needs; its
+// definition is in the description of the task that needs it.
+function getNeedKey(item, environment, memo, path) {
+    return memoized(memo, path, `need\0${item.name}`, async (inner) => {
+        const inputs = await getInputKeys(item, environment, memo, inner);
+        if (inputs === null) {
+            return null;
+        }
+        return hash({ files: commitFiles(await (0, get_work_cache_stats_1.getWorkCacheStats)(item.data, environment)), ...inputs });
+    });
+}
+async function getInputKeys(item, environment, memo, path) {
+    const deps = [];
+    for (const dep of item.deps) {
+        const key = await getCheckKey(dep, environment, memo, path);
+        if (key === null) {
+            return null;
+        }
+        deps.push(key);
+    }
+    const needs = [];
+    for (const need of item.needs) {
+        const key = await getNeedKey(need.service, environment, memo, path);
+        if (key === null) {
+            return null;
+        }
+        needs.push(`${need.name}=${key}`);
+    }
+    return { deps: deps.sort(), needs: needs.sort() };
+}
+// a commit never holds `.git`: what a checkout keeps there differs between a CI job's clone,
+// the server's fetch and a sandbox's copy, which has none
+function commitFiles(stats) {
+    return Object.keys(stats.files)
+        .filter((path) => !path.split('/').includes('.git'))
+        .sort()
+        .map((path) => [path, stats.files[path].checksum]);
+}
+function hash(value) {
+    return (0, crypto_1.createHash)('sha256').update(JSON.stringify(value)).digest('hex');
+}
+// `path` holds the keys being computed on the way here: one coming back is a loop (services
+// that need each other), which can not be proven and fails to plan anyway. Waiting on its
+// pending promise would never end. The keys are computed one after another, so every pending
+// promise in the memo is on the path.
+function memoized(memo, path, key, compute) {
+    if (path.has(key)) {
+        return Promise.resolve(null);
+    }
+    const known = memo.get(key);
+    if (known) {
+        return known;
+    }
+    const computed = compute(new Set([...path, key]));
+    memo.set(key, computed);
+    return computed;
+}
+//# sourceMappingURL=check-key.js.map

package/dist/optimizer/check-keys.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCheckKeys = void 0;
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const reference_parser_1 = require("../schema/reference-parser");
+const schema_parser_1 = require("../schema/schema-parser");
+const work_scope_parser_1 = require("../schema/work-scope-parser");
+const log_1 = require("../log");
+const check_key_1 = require("./check-key");
+/**
+ * The check keys of the tasks of a scope, as both ends compute them: the process of a run on
+ * the server before it executes anything, and `remote verify` in a CI job's checkout. So the
+ * build file is read the same way on both, as a server reads it (no `.env`, links are files)
+ * but placed nowhere: where a server runs a task (a pool, its tool image, the arch of its
+ * nodes) is the server's, and a result counts for the server it passed on. The secrets'
+ * providers and accounts are the server's too, and no part of a key, so any is taken here.
+ */
+async function getCheckKeys(entry, environment, scope) {
+    const keyed = { ...environment, hosted: { catalog: anyCatalog(), localTasksInContainers: true } };
+    const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(entry, keyed);
+    const workTree = (0, work_scope_parser_1.getWorkContext)(await (0, reference_parser_1.parseReferences)(ctx, parsed, keyed), scope, keyed);
+    const memo = new Map();
+    const keys = {};
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        try {
+            const key = await (0, check_key_1.getCheckKey)(task, keyed, memo);
+            keys[task.name] = key
+                ? { key }
+                : {
+                    key: null,
+                    reason: 'it, or a task it or a service it needs depends on, declares no src that matches a file',
+                };
+        }
+        catch (e) {
+            // a secret whose value is part of the key (`cache: true`) is not known here
+            keys[task.name] = { key: null, reason: (0, log_1.getErrorMessage)(e) };
+        }
+    }
+    return keys;
+}
+exports.getCheckKeys = getCheckKeys;
+// Every provider and account name resolves, to a definition nothing runs.
+function anyCatalog() {
+    const provider = (name) => ({
+        name,
+        type: 'command',
+        command: [],
+        env: {},
+        timeoutMs: 0,
+        declaredIn: 'the server',
+    });
+    const account = (name) => ({
+        name,
+        default: false,
+        providers: new Proxy({}, { get: () => ({ env: {} }) }),
+        declaredIn: 'the server',
+    });
+    return {
+        providers: new Proxy({}, { get: (_target, name) => (typeof name === 'string' ? provider(name) : undefined) }),
+        accounts: new Proxy({}, { get: (_target, name) => (typeof name === 'string' ? account(name) : undefined) }),
+        explicitAccounts: true,
+    };
+}
+//# sourceMappingURL=check-keys.js.map

package/dist/optimizer/get-work-cache-stats.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getStateKey = exports.getWorkItemCacheStats = exports.getWorkCacheStats = void 0;
+exports.stateKeyDigest = exports.getStateKey = exports.getWorkItemCheckoutStats = exports.getWorkItemCacheStats = exports.getWorkCacheStats = void 0;
 const calculate_checksum_1 = require("./calculate-checksum");
 const path_1 = require("path");
 const crypto_1 = require("crypto");
@@ -20,7 +20,7 @@ async function addWorkCacheStats(result, cwd, path, matcher, context, root, excl
             if (!matcher(path)) {
                 return 0;
             }
-            const checksum = (0, crypto_1.createHash)('sha1')
+            const checksum = (0, crypto_1.createHash)('sha256')
                 .update(`link:${await (0, promises_1.readlink)(path)}`)
                 .digest('hex');
             result.files[(0, path_1.relative)(cwd, path).split(path_1.sep).join(path_1.posix.sep)] = { lastModified: link.mtimeMs, checksum };
@@ -85,6 +85,13 @@ function getWorkItemCacheStats(item, environment) {
     return collectSourceStats(item.data, environment, dependencyOutputs(item));
 }
 exports.getWorkItemCacheStats = getWorkItemCacheStats;
+// The source stats of a task with what its dependencies generate: on a checkout nothing ran
+// in, those are the files a commit holds there, see check-key.ts.
+async function getWorkItemCheckoutStats(item, environment) {
+    const { stats } = await collectSourceStats(item.data, environment, []);
+    return { stats, unmatched: (await getWorkItemCacheStats(item, environment)).unmatched };
+}
+exports.getWorkItemCheckoutStats = getWorkItemCheckoutStats;
 function dependencyOutputs(item) {
     const outputs = new Set();
     const visited = new Set();
@@ -113,7 +120,14 @@ function getStateKey(stats, cacheMethod) {
             contents.push(`${key}:${stats.files[key].lastModified}`);
         }
     }
-    return (0, crypto_1.createHash)('md5').update(contents.join(',')).digest('hex');
+    return stateKeyDigest(contents.join(','));
 }
 exports.getStateKey = getStateKey;
+// A state key is also the value of the `hammerkit.dev/state` label on Kubernetes, which
+// takes at most 63 characters: the sha256 in base 36 has 50.
+function stateKeyDigest(content) {
+    const hex = (0, crypto_1.createHash)('sha256').update(content).digest('hex');
+    return BigInt(`0x${hex}`).toString(36).padStart(50, '0');
+}
+exports.stateKeyDigest = stateKeyDigest;
 //# sourceMappingURL=get-work-cache-stats.js.map

package/dist/optimizer/work-task-cache-description.js

@@ -1,11 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storedCacheDescription = exports.getWorkTaskCacheDescription = void 0;
+exports.storedCacheDescription = exports.getWorkServiceCacheIdentity = exports.getWorkTaskCacheDescription = void 0;
 const os_1 = require("os");
 const crypto_1 = require("crypto");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const portable_path_1 = require("../planner/utils/portable-path");
 const work_secret_1 = require("../planner/work-secret");
+const claim_layout_1 = require("../kubernetes/claim-layout");
+const work_service_1 = require("../planner/work-service");
+const work_service_id_1 = require("../planner/work-service-id");
+const work_cache_id_1 = require("../planner/work-cache-id");
 // Every path is made project-relative (portablePath) so the description, and the
 // task id hashed from it, is identical wherever the project is checked out.
 //
@@ -40,9 +44,55 @@ function getWorkTaskCacheDescription(item) {
         cwd: portable(task.cwd),
         deps: item.deps.length > 0 ? item.deps.map((dep) => dep.id()).sort() : undefined,
         secrets: cachedSecrets.length > 0 ? cachedSecrets : undefined,
+        claimLayout: (0, claim_layout_1.hasPathsEndingAlike)(item) || (0, claim_layout_1.hasOutputsFlatteningAlike)(item) ? 2 : undefined,
+        needs: item.needs.length > 0 ? getNeedsIdentities(item) : undefined,
     };
 }
 exports.getWorkTaskCacheDescription = getWorkTaskCacheDescription;
+// `visiting` holds the services on the way here: services that need each other in a loop
+// fail to plan (checkForLoop), the identity only has to come back to say so
+function getNeedsIdentities(item, visiting = new Set()) {
+    return (item.needs
+        // an init task needs the service it initializes, whose identity holds the init's id
+        .filter((need) => !isInitOf(item, need.service))
+        .map((need) => visiting.has(need.service)
+        ? `${need.name}=loop`
+        : `${need.name}=${getWorkServiceCacheIdentity(need.service, visiting)}`)
+        .sort());
+}
+function isInitOf(item, service) {
+    return (0, work_service_1.isContainerWorkService)(service.data) && service.data.init?.task.data === item.data;
+}
+// Everything that makes a service what a task needing it talks to: its definition, the ids of
+// the tasks it depends on and of its init task, and the services it needs in turn. The source
+// of those tasks and of the service is in the state key (computeStateKey), like a dependency's.
+// Not the service id, which leaves out what a service runs (getWorkServiceId).
+function getWorkServiceCacheIdentity(item, visiting = new Set()) {
+    const service = item.data;
+    const definition = (0, work_service_1.isContainerWorkService)(service)
+        ? {
+            cmd: service.cmd?.cmd ?? null,
+            envs: (0, replace_env_variables_1.getEnvironmentVariables)(service.envs),
+            healthcheck: service.healthcheck?.cmd.cmd ?? null,
+            secrets: service.secrets
+                .filter((secret) => secret.cache)
+                .map((secret) => `${(0, work_secret_1.getPortableSecretName)(service.projectRoot, secret)}=${secret.digest()}`)
+                .sort(),
+            // as declared: arguments after -- of a run of the init leave the service as it is
+            init: service.init ? (0, work_cache_id_1.getDeclaredWorkTaskId)(service.init.task) : null,
+        }
+        : { namespace: service.namespace };
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify({
+        id: (0, work_service_id_1.getWorkServiceId)(service),
+        ...definition,
+        ports: service.ports.map((port) => `${port.hostPort ?? ''}:${port.containerPort}`).sort(),
+        deps: item.deps.map((dep) => dep.id()).sort(),
+        needs: getNeedsIdentities(item, new Set([...visiting, item])),
+    }))
+        .digest('hex');
+}
+exports.getWorkServiceCacheIdentity = getWorkServiceCacheIdentity;
 // The form written to disk: description.json in a cache entry (pushed to remote
 // backends, readable by everyone who can pull) and the local explain record.
 // Env is where tokens get passed, so each value is stored as a digest; the task

package/dist/planner/utils/append-work-dependencies.js

@@ -17,9 +17,14 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                 }
             }
         }
-        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
-            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+        // an aggregate hands on what its dependencies generate, as if what depends on it
+        // depended on them
+        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data) || (0, work_task_1.isAggregateTask)(depNode.data)) &&
+            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data) || (0, work_task_1.isAggregateTask)(item.data))) {
+            // the outputs of the dep's own deps too, which stay those of the task that made them
+            // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
+                const inherited = generate.inherited ?? depNode;
                 if (generate.isFile) {
                     // TODO check if thats correct
                     continue;
@@ -31,12 +36,12 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             export: false,
                             resetOnChange: false,
                             containerPath: generate.path,
-                            inherited: depNode,
+                            inherited,
                             readOnly: false,
                         });
                     }
                 }
-                else {
+                else if ((0, work_task_1.isWorkTask)(item.data)) {
                     if (!item.data.generates.some((v) => v.volumeName === generate.volumeName)) {
                         item.data.generates.push({
                             volumeName: generate.volumeName,
@@ -44,7 +49,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             exportAlways: false,
                             resetOnChange: false,
                             path: generate.path,
-                            inherited: depNode,
+                            inherited,
                             isFile: false,
                         });
                     }

package/dist/planner/utils/append-work-service.js

@@ -120,6 +120,7 @@ function parseService(workTree, service, environment, context) {
             volumeSize: service.schema.volumeSize ?? null,
             network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
             networkCeiling: null,
+            args: service.schema.args ?? null,
         };
     }
 }

package/dist/planner/utils/append-work-task.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendWorkTask = void 0;
 const work_secret_1 = require("../work-secret");
+const work_task_1 = require("../work-task");
 const parse_work_generate_1 = require("./parse-work-generate");
 const template_value_1 = require("./template-value");
 const parse_work_mounts_1 = require("./parse-work-mounts");
@@ -25,7 +26,8 @@ function appendWorkTask(workTree, cwd, referenceTask, environment, context) {
     const parsed = parseTask(cwd, referenceTask, environment, context);
     // on a server's cluster a local task runs in a tool image like a container task
     const server = workTree.environment.type === 'kubernetes' ? workTree.environment.server : undefined;
-    const task = !server
+    // an aggregate runs nothing, so it needs no pod either
+    const task = !server || (0, work_task_1.isAggregateTask)(parsed)
         ? parsed
         : parsed.type === 'local-task'
             ? (0, job_placement_1.runInToolImage)(parsed, server, referenceTask.schema.platform)
@@ -83,6 +85,7 @@ function parseTask(cwd, task, environment, context) {
         resources: (0, work_resources_1.parseWorkResources)(task.schema.resources),
         network: (0, work_network_1.parseWorkNetwork)(task.schema.network),
         networkCeiling: null,
+        args: task.schema.args ?? null,
         runner: task.schema.runner ?? task.scope.schema.runner ?? null,
     };
     if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
@@ -100,13 +103,23 @@ function parseTask(cwd, task, environment, context) {
         if (fileSecret) {
             throw new Error(`task ${task.relativeName}: a local task takes secrets as env only (${(0, work_secret_1.getSecretName)(fileSecret)})`);
         }
-        if (baseWorkNode.network.type !== 'open') {
+        // on a server's cluster it runs in a pod of a pool's tool image (runInToolImage)
+        const contained = !!environment.hosted?.kubernetes || !!environment.hosted?.localTasksInContainers;
+        if (baseWorkNode.network.type !== 'open' && !contained) {
             throw new Error(`task ${task.relativeName}: network ${(0, work_network_1.formatNetwork)(baseWorkNode.network)} needs a container, ` +
                 `hammerkit can't limit the network of a local task`);
         }
+        const aggregate = task.deps.length > 0 &&
+            task.needs.length === 0 &&
+            baseWorkNode.src.length === 0 &&
+            !baseWorkNode.continuous &&
+            baseWorkNode.cmds.length === 0 &&
+            baseWorkNode.generates.length === 0 &&
+            baseWorkNode.secrets.length === 0;
         return {
             ...baseWorkNode,
             type: 'local-task',
+            ...(aggregate ? { aggregate } : {}),
         };
     }
 }

package/dist/planner/utils/get-work-runtime.js

@@ -4,6 +4,8 @@ exports.getWorkTaskRuntime = exports.getWorkServiceRuntime = void 0;
 const work_runtime_docker_1 = require("../work-runtime-docker");
 const work_runtime_kubernetes_1 = require("../work-runtime-kubernetes");
 const work_runtime_local_1 = require("../work-runtime-local");
+const work_runtime_aggregate_1 = require("../work-runtime-aggregate");
+const work_task_1 = require("../work-task");
 const work_item_1 = require("../work-item");
 function getWorkServiceRuntime(workTree, service) {
     if ((0, work_item_1.isContainerWorkServiceItem)(service)) {
@@ -24,7 +26,10 @@ function getWorkServiceRuntime(workTree, service) {
 }
 exports.getWorkServiceRuntime = getWorkServiceRuntime;
 function getWorkTaskRuntime(workTree, task) {
-    if ((0, work_item_1.isLocalWorkTaskItem)(task)) {
+    if ((0, work_task_1.isAggregateTask)(task.data)) {
+        return (0, work_runtime_aggregate_1.aggregateWorkRuntime)();
+    }
+    else if ((0, work_item_1.isLocalWorkTaskItem)(task)) {
         return (0, work_runtime_local_1.getLocalWorkRuntime)(task);
     }
     else if ((0, work_item_1.isContainerWorkTaskItem)(task)) {

package/dist/planner/utils/normalize-path.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizePath = void 0;
+exports.normalizeHostPath = exports.getHomeReference = exports.normalizePath = void 0;
 const path_1 = require("path");
 function normalizePath(cwd, pwd, path) {
     if (path.startsWith('/')) {
@@ -14,4 +14,22 @@ function normalizePath(cwd, pwd, path) {
     }
 }
 exports.normalizePath = normalizePath;
+// What a path of the host starts with to name the home directory: `~` and `$HOME`, and
+// `$PWD`, which in a mount has always meant the home directory, not the project it reads like
+function getHomeReference(path) {
+    for (const reference of ['~', '$HOME']) {
+        if (path === reference || path.startsWith(`${reference}/`) || path.startsWith(`${reference}\\`)) {
+            return reference;
+        }
+    }
+    return path.startsWith('$PWD') ? '$PWD' : null;
+}
+exports.getHomeReference = getHomeReference;
+// A path of the host in a mount: in the home directory when it starts with one of the
+// references above, else in the directory of the build file unless it is absolute
+function normalizeHostPath(cwd, home, path) {
+    const reference = getHomeReference(path);
+    return reference ? (0, path_1.join)(home, path.substring(reference.length)) : normalizePath(cwd, home, path);
+}
+exports.normalizeHostPath = normalizeHostPath;
 //# sourceMappingURL=normalize-path.js.map

package/dist/planner/utils/parse-work-mount.js

@@ -7,6 +7,12 @@ const path_1 = require("path");
 function parseWorkMount(cwd, dir) {
     const parts = dir.split(':');
     if (parts.length === 1) {
+        const home = (0, normalize_path_1.getHomeReference)(dir);
+        if (home === '~' || home === '$HOME') {
+            // the container has no such directory: the same path on both sides would be one of the
+            // project there (`$PWD` has always been, so it stays)
+            throw new Error(`mount ${dir} is in the home directory, name where it goes in the container (${dir}:/path)`);
+        }
         return parseLocalMount(cwd, dir, dir, dir, false);
     }
     else if (parts.length === 2) {
@@ -22,7 +28,7 @@ function parseWorkMount(cwd, dir) {
 exports.parseWorkMount = parseWorkMount;
 function parseLocalMount(cwd, localPath, containerPath, mount, readOnly) {
     return {
-        localPath: (0, normalize_path_1.normalizePath)(cwd, (0, os_1.homedir)(), localPath),
+        localPath: (0, normalize_path_1.normalizeHostPath)(cwd, (0, os_1.homedir)(), localPath),
         containerPath: (0, normalize_path_1.normalizePath)(cwd, cwd, containerPath),
         isFile: (0, path_1.extname)(localPath).length > 1,
         mount,

package/dist/planner/utils/parse-work-port.js

@@ -8,7 +8,10 @@ function parseWorkPort(port) {
         return { containerPort: port, hostPort: port };
     }
     else if (parts.length === 2) {
-        return { containerPort: parsePort(parts[1], false), hostPort: parsePort(parts[0], true) };
+        // `0:80` is not published, as an empty host port: 0 (a free port) is for the endpoints
+        // of a server only
+        const hostPort = parsePort(parts[0], true);
+        return { containerPort: parsePort(parts[1], false), hostPort: hostPort === 0 ? null : hostPort };
     }
     else {
         throw new Error(`invalid port ${port}`);

package/dist/planner/utils/task-arguments.js

@@ -101,6 +101,9 @@ function applyArguments(selection, args, keepCache = false) {
     if (item.type === 'kubernetes-service') {
         throw new Error(`service ${item.name} is a kubernetes service and has no cmd for the arguments after --`);
     }
+    if (item.args === false) {
+        throw new Error(`${item.type === 'container-service' ? 'service' : 'task'} ${item.name} takes no arguments after -- (args: false)`);
+    }
     const target = getArgumentsTarget(item);
     if (!target) {
         throw new Error(item.type === 'container-service'