hammerkit 1.27.0-next.18 → 1.27.0-next.19

package/dist/cache/explain.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.explainWorkTree = exports.explainTask = exports.describeCause = void 0;
+const work_task_1 = require("../planner/work-task");
 const state_key_1 = require("../executer/scheduler/state-key");
 const work_task_cache_description_1 = require("../optimizer/work-task-cache-description");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
@@ -157,6 +158,22 @@ async function explainTask(item, defaultCacheMethod, environment, memo = new Map
         visiting.delete(item.id());
         return explanation;
     }
+    // Mirrors executeWorkTask: an aggregate is never looked up itself, and is reported
+    // cached when all its dependencies were. What changed is in its dependencies.
+    if ((0, work_task_1.isAggregateTask)(item.data)) {
+        const causes = [];
+        for (const dep of item.deps) {
+            const depExplanation = await explainTask(dep, defaultCacheMethod, environment, memo, visiting);
+            if (depExplanation.status !== 'hit') {
+                causes.push({ kind: 'dependency-changed', identifier: dep.name });
+            }
+        }
+        const status = causes.length === 0 ? 'hit' : 'miss';
+        const explanation = { taskId: item.id(), taskName: item.name, status, causes };
+        memo.set(item.id(), explanation);
+        visiting.delete(item.id());
+        return explanation;
+    }
     const runtimeStateKey = await safeCurrentStateKey(item, environment);
     const hasEntry = await safeHas(item, stateKey, environment);
     const predictedHit = runtimeStateKey === stateKey || hasEntry;

package/dist/executer/execute-work-task.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeWorkTask = void 0;
+const work_task_1 = require("../planner/work-task");
 const states_1 = require("./states");
 const abort_1 = require("./abort");
 const log_1 = require("../log");
@@ -12,7 +13,8 @@ const archive_task_entry_1 = require("./archive-task-entry");
 const units_1 = require("../utils/units");
 async function pushToBackend(work, environment, cacheState, options) {
     const { resolved, stateKey, provable } = cacheState;
-    if (resolved.method === 'none' || !provable) {
+    // an aggregate has nothing to store, see checkCacheState
+    if (resolved.method === 'none' || !provable || (0, work_task_1.isAggregateTask)(work.data)) {
         return;
     }
     if (options.cacheReadOnly) {
@@ -29,7 +31,7 @@ async function pushToBackend(work, environment, cacheState, options) {
     }
 }
 // A task without cmds only groups its dependencies (`ci: { deps: [test, build] }`).
-// Having no src of its own it is never a cache hit, so its dependencies are
+// It is never a cache hit itself (see checkCacheState), so its dependencies are
 // always checked and their outputs restored. When every one of them was a hit,
 // nothing ran: report the group as cached too. Reporting only, the cache
 // decision is untouched.

package/dist/executer/scheduler/enqueue-next.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkCacheState = exports.UNCACHED_STATE_KEY = void 0;
 const work_item_1 = require("../../planner/work-item");
+const work_task_1 = require("../../planner/work-task");
 const get_cache_directory_1 = require("../../optimizer/get-cache-directory");
 const cache_metadata_1 = require("../cache-metadata");
 const work_task_cache_description_1 = require("../../optimizer/work-task-cache-description");
@@ -35,6 +36,12 @@ options = {}) {
         item.status.write('debug', `${item.name} always runs, it or one of its dependencies has no src files`);
         return { cached: false, stateKey, resolved, provable };
     }
+    // An aggregate runs nothing, so running it costs nothing, and it is never a hit itself:
+    // asked for, it still has its dependencies checked and their outputs restored. Its key
+    // counts for what depends on it, and it is reported cached when they all were.
+    if ((0, work_item_1.isWorkTaskItem)(item) && (0, work_task_1.isAggregateTask)(item.data)) {
+        return { cached: false, stateKey, resolved, provable };
+    }
     if ((0, work_item_1.isWorkTaskItem)(item)) {
         const runtimeStateKey = await item.runtime.currentStateKey(environment);
         if (runtimeStateKey === stateKey) {

package/dist/executer/scheduler/state-key.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.computeStateKey = exports.combineStateKeys = exports.isOwnProvable = exports.resolveEffective = void 0;
 const get_work_cache_stats_1 = require("../../optimizer/get-work-cache-stats");
+const work_task_1 = require("../../planner/work-task");
 // The state-key engine is shared by the scheduler's hit/miss decision
 // (`checkCacheState`) and the read-only cache-explain engine, so a prediction
 // can never diverge from the decision it predicts — there is one implementation.
@@ -20,6 +21,10 @@ exports.resolveEffective = resolveEffective;
 // whose `src` entries all match no file (a typo, a moved path). Services are not
 // affected.
 function isOwnProvable(item, unmatched) {
+    if ((0, work_task_1.isAggregateTask)(item.data)) {
+        // it runs nothing and reads nothing: what its dependencies prove is all there is
+        return true;
+    }
     // unmatched ⊆ declared, so this is also false when nothing is declared
     return unmatched.length < item.data.src.filter((src) => !src.inherited).length;
 }

package/dist/planner/utils/append-work-dependencies.js

@@ -17,8 +17,10 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                 }
             }
         }
-        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
-            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+        // an aggregate hands on what its dependencies generate, as if what depends on it
+        // depended on them
+        if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data) || (0, work_task_1.isAggregateTask)(depNode.data)) &&
+            ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data) || (0, work_task_1.isAggregateTask)(item.data))) {
             // the outputs of the dep's own deps too, which stay those of the task that made them
             // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
@@ -39,7 +41,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                         });
                     }
                 }
-                else {
+                else if ((0, work_task_1.isWorkTask)(item.data)) {
                     if (!item.data.generates.some((v) => v.volumeName === generate.volumeName)) {
                         item.data.generates.push({
                             volumeName: generate.volumeName,

package/dist/planner/utils/append-work-task.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendWorkTask = void 0;
 const work_secret_1 = require("../work-secret");
+const work_task_1 = require("../work-task");
 const parse_work_generate_1 = require("./parse-work-generate");
 const template_value_1 = require("./template-value");
 const parse_work_mounts_1 = require("./parse-work-mounts");
@@ -25,7 +26,8 @@ function appendWorkTask(workTree, cwd, referenceTask, environment, context) {
     const parsed = parseTask(cwd, referenceTask, environment, context);
     // on a server's cluster a local task runs in a tool image like a container task
     const server = workTree.environment.type === 'kubernetes' ? workTree.environment.server : undefined;
-    const task = !server
+    // an aggregate runs nothing, so it needs no pod either
+    const task = !server || (0, work_task_1.isAggregateTask)(parsed)
         ? parsed
         : parsed.type === 'local-task'
             ? (0, job_placement_1.runInToolImage)(parsed, server, referenceTask.schema.platform)
@@ -107,9 +109,17 @@ function parseTask(cwd, task, environment, context) {
             throw new Error(`task ${task.relativeName}: network ${(0, work_network_1.formatNetwork)(baseWorkNode.network)} needs a container, ` +
                 `hammerkit can't limit the network of a local task`);
         }
+        const aggregate = task.deps.length > 0 &&
+            task.needs.length === 0 &&
+            baseWorkNode.src.length === 0 &&
+            !baseWorkNode.continuous &&
+            baseWorkNode.cmds.length === 0 &&
+            baseWorkNode.generates.length === 0 &&
+            baseWorkNode.secrets.length === 0;
         return {
             ...baseWorkNode,
             type: 'local-task',
+            ...(aggregate ? { aggregate } : {}),
         };
     }
 }

package/dist/planner/utils/get-work-runtime.js

@@ -4,6 +4,8 @@ exports.getWorkTaskRuntime = exports.getWorkServiceRuntime = void 0;
 const work_runtime_docker_1 = require("../work-runtime-docker");
 const work_runtime_kubernetes_1 = require("../work-runtime-kubernetes");
 const work_runtime_local_1 = require("../work-runtime-local");
+const work_runtime_aggregate_1 = require("../work-runtime-aggregate");
+const work_task_1 = require("../work-task");
 const work_item_1 = require("../work-item");
 function getWorkServiceRuntime(workTree, service) {
     if ((0, work_item_1.isContainerWorkServiceItem)(service)) {
@@ -24,7 +26,10 @@ function getWorkServiceRuntime(workTree, service) {
 }
 exports.getWorkServiceRuntime = getWorkServiceRuntime;
 function getWorkTaskRuntime(workTree, task) {
-    if ((0, work_item_1.isLocalWorkTaskItem)(task)) {
+    if ((0, work_task_1.isAggregateTask)(task.data)) {
+        return (0, work_runtime_aggregate_1.aggregateWorkRuntime)();
+    }
+    else if ((0, work_item_1.isLocalWorkTaskItem)(task)) {
         return (0, work_runtime_local_1.getLocalWorkRuntime)(task);
     }
     else if ((0, work_item_1.isContainerWorkTaskItem)(task)) {

package/dist/planner/work-network.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.applyNetworkCeiling = exports.formatNetwork = exports.getTcpForwards = exports.matchesHost = exports.isLimitedNetwork = exports.toNetworkSchema = exports.exceedsNetwork = exports.getEffectiveNetwork = exports.limitNetwork = exports.parseNetworkOption = exports.parseWorkNetwork = exports.OPEN_NETWORK = void 0;
 const network_schema_1 = require("../schema/network-schema");
+const work_task_1 = require("./work-task");
 exports.OPEN_NETWORK = { type: 'open' };
 function parseWorkNetwork(schema) {
     if (!schema || schema === 'open') {
@@ -153,7 +154,8 @@ exports.formatNetwork = formatNetwork;
 function applyNetworkCeiling(workTree, ceiling) {
     const limit = ceiling && isLimitedNetwork(ceiling) ? ceiling : null;
     if (limit) {
-        const local = Object.values(workTree.tasks).find((task) => task.data.type === 'local-task');
+        // an aggregate runs nothing, so there is nothing of it to limit
+        const local = Object.values(workTree.tasks).find((task) => task.data.type === 'local-task' && !(0, work_task_1.isAggregateTask)(task.data));
         if (local) {
             throw new Error(`--network ${formatNetwork(limit)} needs every task in a container, ` +
                 `hammerkit can't limit the network of the local task ${local.name}`);

package/dist/planner/work-runtime-aggregate.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.aggregateWorkRuntime = void 0;
+// An aggregate runs nothing, so it touches nothing: no pid or state file in its
+// directory (on a server that is a checkout it doesn't trust), no outputs.
+function aggregateWorkRuntime() {
+    return {
+        async initialize() {
+            // nothing to prepare
+        },
+        async remove() {
+            // it has no outputs of its own
+        },
+        async execute() {
+            // its state stays running, and executeWorkTask completes it
+        },
+        async stop() {
+            // nothing is running
+        },
+        async archive() {
+            // never pushed, see pushToBackend
+        },
+        async restore() {
+            // never a hit, see checkCacheState
+        },
+        async currentStateKey() {
+            return null;
+        },
+    };
+}
+exports.aggregateWorkRuntime = aggregateWorkRuntime;
+//# sourceMappingURL=work-runtime-aggregate.js.map

package/dist/planner/work-task.js

@@ -1,8 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWorkTask = exports.isContainerWorkTask = void 0;
+exports.isAggregateTask = exports.isWorkTask = exports.isContainerWorkTask = void 0;
 const isContainerWorkTask = (val) => val.type === 'container-task';
 exports.isContainerWorkTask = isContainerWorkTask;
 const isWorkTask = (val) => val.type === 'container-task' || val.type === 'local-task';
 exports.isWorkTask = isWorkTask;
+// A task that only groups its dependencies: `deps`, and no `image`, `cmds`, `src`,
+// `generates`, `secrets`, `needs` or `continuous`. It needs no runtime, hands on what its dependencies generate to what
+// depends on it, and is up to date (or verified) when all its dependencies are.
+const isAggregateTask = (val) => val.type === 'local-task' && val.aggregate === true;
+exports.isAggregateTask = isAggregateTask;
 //# sourceMappingURL=work-task.js.map

package/dist/server/plan.js

@@ -188,7 +188,8 @@ function findPolicyProblems(items, policy, root) {
 function findItemPolicyProblems(item, policy, root) {
     const problems = [];
     const runtime = item.data.type === 'local-task' ? 'local' : 'docker';
-    if (policy.runtimes && !policy.runtimes.includes(runtime)) {
+    // an aggregate runs nothing, on no runtime
+    if (policy.runtimes && !(0, work_task_1.isAggregateTask)(item.data) && !policy.runtimes.includes(runtime)) {
         problems.push(`${item.name}: runs on the ${runtime} runtime, which this server does not allow (runtimes: ${policy.runtimes.join(', ')})`);
     }
     const data = item.data;
@@ -221,7 +222,8 @@ function findItemPolicyProblems(item, policy, root) {
             reason: ` to what reads ${reads.join(', ')}`,
         });
     }
-    for (const { ceiling, setting, reason } of ceilings) {
+    // an aggregate reaches nothing: it runs no command
+    for (const { ceiling, setting, reason } of (0, work_task_1.isAggregateTask)(data) ? [] : ceilings) {
         // on a cluster a local task is a pod of a pool's tool image, limited like any other
         if (data.type === 'local-task' && policy.backend !== 'kubernetes') {
             problems.push(`${item.name}: is a local task, which can't be limited to the network this server allows${reason} ` +
@@ -297,7 +299,7 @@ function toPlan(root, item) {
         : [];
     const ports = 'ports' in data ? data.ports.map((port) => `${port.hostPort ?? ''}:${port.containerPort}`) : [];
     const flags = [];
-    if (data.type === 'local-task') {
+    if (data.type === 'local-task' && !(0, work_task_1.isAggregateTask)(data)) {
         flags.push('local-task');
     }
     if ('ports' in data && data.ports.some((port) => port.hostPort !== null)) {
@@ -310,7 +312,7 @@ function toPlan(root, item) {
     return {
         kind: isTask ? 'task' : 'service',
         ...(data.description ? { description: data.description } : {}),
-        runtime: data.type === 'local-task' ? 'local' : 'docker',
+        runtime: (0, work_task_1.isAggregateTask)(data) ? 'none' : data.type === 'local-task' ? 'local' : 'docker',
         ...((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data) ? { image: data.image } : {}),
         cmds: 'cmds' in data ? data.cmds.map((cmd) => cmd.cmd) : 'cmd' in data && data.cmd ? [data.cmd.cmd] : [],
         command: (0, task_arguments_1.getArgumentsTarget)(data)?.cmd ?? null,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.18",
+  "version": "1.27.0-next.19",
   "keywords": [
     "build",
     "build-tool",