hammerkit 1.27.0-next.23 → 1.27.0-next.25

package/build.schema.json

@@ -459,6 +459,91 @@
                 },
                 "additionalProperties": false,
                 "description": "local task"
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "deps": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/deps"
+                  },
+                  "needs": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/needs"
+                  },
+                  "description": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/description"
+                  },
+                  "extend": {
+                    "type": "string"
+                  },
+                  "reset": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/reset"
+                  },
+                  "envs": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/envs"
+                  },
+                  "labels": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/labels"
+                  },
+                  "cache": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cache"
+                  },
+                  "cmds": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/cmds"
+                  },
+                  "src": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/src"
+                  },
+                  "generates": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/generates"
+                  },
+                  "shell": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/shell"
+                  },
+                  "continuous": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/continuous"
+                  },
+                  "timeout": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
+                  },
+                  "secrets": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
+                  },
+                  "account": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/account"
+                  },
+                  "resources": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
+                  },
+                  "network": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
+                  },
+                  "args": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/args"
+                  },
+                  "runner": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
+                  },
+                  "publish": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/publish"
+                  },
+                  "image": {
+                    "type": "string"
+                  },
+                  "mounts": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/mounts"
+                  },
+                  "volumeSize": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize"
+                  },
+                  "capAdd": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/capAdd"
+                  }
+                },
+                "required": [
+                  "extend"
+                ],
+                "additionalProperties": false,
+                "description": "container task taking its image from the task it extends"
               }
             ]
           }

package/dist/cache/backends/local-cache-backend.js

@@ -80,6 +80,12 @@ function createLocalCacheBackend(spec) {
                 return false;
             }
         },
+        async touch(taskId, stateKey, environment) {
+            const remoteDir = (0, path_1.join)(root, taskId, stateKey);
+            if (await environment.file.exists((0, path_1.join)(remoteDir, 'stats.json'))) {
+                await (0, promises_1.writeFile)((0, path_1.join)(remoteDir, LAST_ACCESS_FILE), `${Date.now()}`);
+            }
+        },
         async push(taskId, stateKey, from, environment) {
             const remoteDir = (0, path_1.join)(root, taskId, stateKey);
             await environment.file.createDirectory(remoteDir);

package/dist/cli.js

@@ -81,6 +81,7 @@ class Cli {
                         timeout: options?.timeout ?? null,
                         maxTimeout: options?.maxTimeout ?? null,
                         skipDeps: options?.skipDeps ?? true,
+                        restore: options?.restore ?? true,
                         wait: options?.wait ?? 'ready',
                         resourceCheck: options?.resourceCheck ?? true,
                         network: options?.network ?? null,

package/dist/executer/deferred-restore.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.restoreDeferred = exports.clearDeferredRestore = exports.deferRestore = void 0;
+const abort_1 = require("./abort");
+// Keyed by the item's state, which every run replaces (resetWorkTree): a hit
+// deferred in one run of a Cli is never restored in a later one, over outputs
+// that run built.
+const deferred = new WeakMap();
+function deferRestore(item, run) {
+    deferred.set(item.state, { run, promise: null });
+}
+exports.deferRestore = deferRestore;
+// Every cache check starts from scratch: a key that changed since a hit was
+// deferred must not restore the old entry.
+function clearDeferredRestore(item) {
+    deferred.delete(item.state);
+}
+exports.clearDeferredRestore = clearDeferredRestore;
+// Restore every given item that was a hit without its outputs, in parallel,
+// each at most once however many items wait on it.
+async function restoreDeferred(work, items, abort) {
+    await Promise.all(items.map((item) => {
+        const entry = deferred.get(item.state);
+        if (!entry) {
+            return Promise.resolve();
+        }
+        if (!entry.promise) {
+            work.status.write('debug', `restore ${item.name} from cache, ${work.name} needs its outputs`);
+            entry.promise = entry.run();
+        }
+        return entry.promise;
+    }));
+    (0, abort_1.checkForAbort)(abort);
+}
+exports.restoreDeferred = restoreDeferred;
+//# sourceMappingURL=deferred-restore.js.map

package/dist/executer/execute-work-service.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeWorkService = exports.stopService = void 0;
 const work_item_1 = require("../planner/work-item");
 const await_completed_dependencies_1 = require("./await-completed-dependencies");
+const deferred_restore_1 = require("./deferred-restore");
 const abort_1 = require("./abort");
 const log_1 = require("../log");
 const watch_loop_1 = require("./watch-loop");
@@ -35,6 +36,7 @@ async function executeWorkService(work, environment, options) {
         });
         await (0, watch_loop_1.watchLoop)(work, environment, options, async (cacheState, abort, stop) => {
             await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
+            await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
             work.state.set({
                 type: 'ready',
                 stateKey: cacheState.stateKey,

package/dist/executer/execute-work-task.js

@@ -11,6 +11,7 @@ const deadline_1 = require("./deadline");
 const explain_1 = require("../cache/explain");
 const archive_task_entry_1 = require("./archive-task-entry");
 const units_1 = require("../utils/units");
+const deferred_restore_1 = require("./deferred-restore");
 async function pushToBackend(work, environment, cacheState, options) {
     const { resolved, stateKey, provable } = cacheState;
     // an aggregate has nothing to store, see checkCacheState
@@ -89,6 +90,10 @@ dependents = null) {
                 }
             }
             await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
+            // an aggregate runs nothing, so it needs no outputs on disk
+            if (!(0, work_task_1.isAggregateTask)(work.data)) {
+                await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
+            }
             await (0, await_completed_dependencies_1.awaitRunningNeeds)(work, work.needs.map((n) => n.service), abort);
             await options.processManager.task(work, async () => {
                 work.state.set({

package/dist/executer/scheduler/enqueue-next.js

@@ -9,13 +9,17 @@ const work_task_cache_description_1 = require("../../optimizer/work-task-cache-d
 const log_1 = require("../../log");
 const state_key_1 = require("./state-key");
 const work_instance_id_1 = require("../../planner/work-instance-id");
+const deferred_restore_1 = require("../deferred-restore");
 // The key a task whose cache is off runs and records its outputs under. It is
 // no hash, so a later run with its cache on never takes those outputs as its own.
 exports.UNCACHED_STATE_KEY = 'uncached';
 async function checkCacheState(item, defaultCacheMethod, environment, 
 // watch mode compares keys to rerun on a change, so it needs one even when
 // the cache is off
+// restore: false only looks a backend hit up, its outputs are downloaded once
+// something that runs depends on them (`run --no-restore`, see deferred-restore)
 options = {}) {
+    (0, deferred_restore_1.clearDeferredRestore)(item);
     // A task whose cache is off is never looked up, so hashing its sources buys
     // nothing, and a `**` over a node_modules takes most of a minute (every run
     // with arguments). Its dependents still hash its sources for their own key.
@@ -47,13 +51,30 @@ options = {}) {
         if (runtimeStateKey === stateKey) {
             return { cached: true, stateKey, resolved, provable };
         }
+        if (options.restore === false && options.watch) {
+            item.status.write('debug', `${item.name} restores cache hits, --no-restore has no effect in watch mode`);
+        }
+        if (options.restore === false && !options.watch) {
+            try {
+                if (await resolved.backend.has(item.id(), stateKey, environment)) {
+                    item.status.write('info', `${item.name} found in cache "${resolved.name}" (${resolved.backend.type}), not restored`);
+                    await touchEntry(item, resolved, stateKey, environment);
+                    (0, deferred_restore_1.deferRestore)(item, async () => {
+                        if (!(await restoreFromBackend(item, resolved, stateKey, currentStats, environment))) {
+                            throw new Error(`${item.name} is no longer in cache "${resolved.name}", its outputs can not be restored. ` +
+                                'Run again without --no-restore.');
+                        }
+                    });
+                    return { cached: true, stateKey, resolved, provable };
+                }
+            }
+            catch (e) {
+                item.status.write('warn', `${item.name} failed to look up cache "${resolved.name}": ${(0, log_1.getErrorMessage)(e)} — continuing with local execution`);
+            }
+            return { cached: false, stateKey, resolved, provable };
+        }
         try {
-            const cacheDir = (0, get_cache_directory_1.getCacheDirectory)((0, work_instance_id_1.getWorkInstanceId)(item));
-            const pulled = await resolved.backend.pull(item.id(), stateKey, cacheDir, environment);
-            if (pulled) {
-                item.status.write('info', `${item.name} pulled from cache "${resolved.name}" (${resolved.backend.type})`);
-                await item.runtime.restore(environment, cacheDir, stateKey);
-                await (0, cache_metadata_1.writeCacheMetadata)(environment, item, currentStats, (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item));
+            if (await restoreFromBackend(item, resolved, stateKey, currentStats, environment)) {
                 return { cached: true, stateKey, resolved, provable };
             }
         }
@@ -64,4 +85,27 @@ options = {}) {
     return { cached: false, stateKey, resolved, provable };
 }
 exports.checkCacheState = checkCacheState;
+// Pull a task's entry and restore its outputs into the checkout. False when the
+// backend has no such entry.
+async function restoreFromBackend(item, resolved, stateKey, currentStats, environment) {
+    const cacheDir = (0, get_cache_directory_1.getCacheDirectory)((0, work_instance_id_1.getWorkInstanceId)(item));
+    const pulled = await resolved.backend.pull(item.id(), stateKey, cacheDir, environment);
+    if (!pulled) {
+        return false;
+    }
+    item.status.write('info', `${item.name} pulled from cache "${resolved.name}" (${resolved.backend.type})`);
+    await item.runtime.restore(environment, cacheDir, stateKey);
+    await (0, cache_metadata_1.writeCacheMetadata)(environment, item, currentStats, (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item));
+    return true;
+}
+// A hit that is only looked up still counts as used, so retention keeps it.
+// Best effort: the hit stands whether or not the marker could be written.
+async function touchEntry(item, resolved, stateKey, environment) {
+    try {
+        await resolved.backend.touch?.(item.id(), stateKey, environment);
+    }
+    catch (e) {
+        item.status.write('debug', `${item.name} could not mark its cache entry as used: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+}
 //# sourceMappingURL=enqueue-next.js.map

package/dist/executer/watch-loop.js

@@ -12,7 +12,10 @@ async function watchLoop(work, environment, options, factory) {
     // kubernetes-service has no such field. Read it wherever it exists.
     const continuous = 'continuous' in work.data ? work.data.continuous : false;
     const watchMode = options.watch && !options.daemon && !continuous;
-    const currentStats = await (0, enqueue_next_1.checkCacheState)(work, options.cacheDefault, environment, { watch: watchMode });
+    const currentStats = await (0, enqueue_next_1.checkCacheState)(work, options.cacheDefault, environment, {
+        watch: watchMode,
+        restore: options.restore,
+    });
     (0, abort_1.checkForAbort)(environment.abortCtrl.signal);
     const watchState = watchMode
         ? (0, watch_state_key_1.watchStateKey)(work, currentStats, environment, options)

package/dist/planner/utils/append-work-service.js

@@ -143,7 +143,11 @@ function appendServiceInit(workTree, service, item, environment, context) {
         task.needs[selfNeed] = { name: task.needs[selfNeed].name, service: view };
     }
     else {
-        task.needs.push({ name: item.name, service: view });
+        const name = getInitNeedName(service);
+        if (task.needs.some((need) => need.name === name)) {
+            throw new Error(`${task.name} needs two services as ${name}, give one another name with name:`);
+        }
+        task.needs.push({ name, service: view });
     }
     removeRequirer(item, task.name);
     // explicit, so neither the default method nor `--cache` turns it back on
@@ -153,6 +157,18 @@ function appendServiceInit(workTree, service, item, environment, context) {
     }
     item.data.init = { task, state };
 }
+// The host name the init reaches its service by: the name its task declares the
+// need by, else the service's name in its own build file. Planning the task
+// skips that need (it is the init's), and the full name of a service of an
+// include, `services:db`, is no host name: Docker and DNS take no colon.
+function getInitNeedName(service) {
+    const declared = service.init?.task.needs.find((need) => need.service.relativeName === service.relativeName);
+    if (declared) {
+        return declared.relativeName;
+    }
+    const prefix = service.scope.namePrefix;
+    return prefix.length > 0 ? service.relativeName.slice(prefix.length + 1) : service.relativeName;
+}
 const initViews = new WeakMap();
 // The service itself for the view its init task needs, else the item.
 function getPlannedService(item) {

package/dist/program.js

@@ -705,6 +705,7 @@ async function getProgram(environment, argv) {
             .addOption(new commander_1.Option('--max-timeout <duration>', 'fail any task after this long, also one with a longer timeout of its own (e.g. 2m)').argParser(units_1.parseDuration))
             .addOption(new commander_1.Option('--network <level>', 'the most any task or service reaches besides its needs: none, open or allow:<host>,<host>').argParser(parseNetworkArgument))
             .addOption(new commander_1.Option('--no-skip-deps', 'run dependencies even when nothing needing them has to run'))
+            .addOption(new commander_1.Option('--no-restore', 'only look cache hits up, download their outputs only when a task that runs needs them'))
             .action(async (task, options, command) => {
             try {
                 // For machine-readable JSON, suppress the human progress logger (which
@@ -740,6 +741,7 @@ async function getProgram(environment, argv) {
                     maxTimeout: options.maxTimeout ?? null,
                     network: options.network ?? null,
                     skipDeps: options.skipDeps !== false,
+                    restore: options.restore !== false,
                 });
                 // Reporting only: the summary never changes the exit code or behavior.
                 if (!options.watch) {

package/dist/schema/build-file-container-task-schema.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildFileContainerTaskSchema = void 0;
+exports.CONTAINER_TASK_PROPERTIES = exports.buildFileExtendingContainerTaskSchema = exports.buildFileContainerTaskSchema = void 0;
 const build_file_base_task_schema_1 = require("./build-file-base-task-schema");
 const zod_1 = require("zod");
 const size_schema_1 = require("./size-schema");
@@ -17,4 +17,15 @@ exports.buildFileContainerTaskSchema = build_file_base_task_schema_1.buildFileBa
 })
     .strict()
     .describe('container task');
+// A task that extends another takes the image of its base, so it may set the
+// container properties without an image of its own; once extended, a task whose
+// base has no image either is refused them (applySchemaExtension).
+exports.buildFileExtendingContainerTaskSchema = exports.buildFileContainerTaskSchema
+    .extend({
+    image: (0, zod_1.string)().optional(),
+    extend: (0, zod_1.string)(),
+})
+    .strict()
+    .describe('container task taking its image from the task it extends');
+exports.CONTAINER_TASK_PROPERTIES = ['mounts', 'volumeSize', 'capAdd'];
 //# sourceMappingURL=build-file-container-task-schema.js.map

package/dist/schema/build-file-task-schema.js

@@ -4,7 +4,11 @@ exports.isBuildFileLocalTaskSchema = exports.isBuildFileContainerTaskSchema = ex
 const zod_1 = require("zod");
 const build_file_container_task_schema_1 = require("./build-file-container-task-schema");
 const build_file_local_task_schema_1 = require("./build-file-local-task-schema");
-exports.buildFileTaskSchema = (0, zod_1.union)([build_file_container_task_schema_1.buildFileContainerTaskSchema, build_file_local_task_schema_1.buildFileLocalTaskSchema]);
+exports.buildFileTaskSchema = (0, zod_1.union)([
+    build_file_container_task_schema_1.buildFileContainerTaskSchema,
+    build_file_local_task_schema_1.buildFileLocalTaskSchema,
+    build_file_container_task_schema_1.buildFileExtendingContainerTaskSchema,
+]);
 const isBuildFileContainerTaskSchema = (val) => 'image' in val;
 exports.isBuildFileContainerTaskSchema = isBuildFileContainerTaskSchema;
 const isBuildFileLocalTaskSchema = (val) => !('image' in val);

package/dist/schema/reference-parser.js

@@ -7,6 +7,7 @@ const split_name_1 = require("../planner/utils/split-name");
 const read_env_file_1 = require("../parser/read-env-file");
 const label_values_1 = require("../executer/label-values");
 const merge_environment_variables_1 = require("../environment/merge-environment-variables");
+const build_file_container_task_schema_1 = require("./build-file-container-task-schema");
 const resolve_cache_1 = require("../cache/resolve-cache");
 const project_root_1 = require("../planner/project-root");
 const secret_catalog_1 = require("../secrets/secret-catalog");
@@ -222,6 +223,7 @@ function applySchemaExtension(reference, task) {
         if (!reset.has('mounts'))
             taskSchema.mounts = extendArray(taskSchema.mounts, extend.schema.mounts);
     }
+    rejectContainerPropertiesWithoutImage(task);
     // deps and needs resolve in the file that declares them, so every base up the
     // chain contributes its own, not only the nearest; what a task resets is not
     // passed on to the tasks extending it either
@@ -236,6 +238,16 @@ function applySchemaExtension(reference, task) {
     }
     task.extension = 'applied';
 }
+function rejectContainerPropertiesWithoutImage(task) {
+    if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
+        return;
+    }
+    const schema = task.schema;
+    const property = build_file_container_task_schema_1.CONTAINER_TASK_PROPERTIES.find((key) => schema[key] !== undefined);
+    if (property) {
+        throw new Error(`task ${task.relativeName} sets ${property}, but neither it nor its base has an image`);
+    }
+}
 function extendArray(base, extend) {
     if (!extend) {
         return base;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.23",
+  "version": "1.27.0-next.25",
   "keywords": [
     "build",
     "build-tool",