hammerkit 1.27.0-next.23 → 1.27.0-next.24

package/dist/cache/backends/local-cache-backend.js

@@ -80,6 +80,12 @@ function createLocalCacheBackend(spec) {
                 return false;
             }
         },
+        async touch(taskId, stateKey, environment) {
+            const remoteDir = (0, path_1.join)(root, taskId, stateKey);
+            if (await environment.file.exists((0, path_1.join)(remoteDir, 'stats.json'))) {
+                await (0, promises_1.writeFile)((0, path_1.join)(remoteDir, LAST_ACCESS_FILE), `${Date.now()}`);
+            }
+        },
         async push(taskId, stateKey, from, environment) {
             const remoteDir = (0, path_1.join)(root, taskId, stateKey);
             await environment.file.createDirectory(remoteDir);

package/dist/cli.js

@@ -81,6 +81,7 @@ class Cli {
                         timeout: options?.timeout ?? null,
                         maxTimeout: options?.maxTimeout ?? null,
                         skipDeps: options?.skipDeps ?? true,
+                        restore: options?.restore ?? true,
                         wait: options?.wait ?? 'ready',
                         resourceCheck: options?.resourceCheck ?? true,
                         network: options?.network ?? null,

package/dist/executer/deferred-restore.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.restoreDeferred = exports.clearDeferredRestore = exports.deferRestore = void 0;
+const abort_1 = require("./abort");
+// Keyed by the item's state, which every run replaces (resetWorkTree): a hit
+// deferred in one run of a Cli is never restored in a later one, over outputs
+// that run built.
+const deferred = new WeakMap();
+function deferRestore(item, run) {
+    deferred.set(item.state, { run, promise: null });
+}
+exports.deferRestore = deferRestore;
+// Every cache check starts from scratch: a key that changed since a hit was
+// deferred must not restore the old entry.
+function clearDeferredRestore(item) {
+    deferred.delete(item.state);
+}
+exports.clearDeferredRestore = clearDeferredRestore;
+// Restore every given item that was a hit without its outputs, in parallel,
+// each at most once however many items wait on it.
+async function restoreDeferred(work, items, abort) {
+    await Promise.all(items.map((item) => {
+        const entry = deferred.get(item.state);
+        if (!entry) {
+            return Promise.resolve();
+        }
+        if (!entry.promise) {
+            work.status.write('debug', `restore ${item.name} from cache, ${work.name} needs its outputs`);
+            entry.promise = entry.run();
+        }
+        return entry.promise;
+    }));
+    (0, abort_1.checkForAbort)(abort);
+}
+exports.restoreDeferred = restoreDeferred;
+//# sourceMappingURL=deferred-restore.js.map

package/dist/executer/execute-work-service.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeWorkService = exports.stopService = void 0;
 const work_item_1 = require("../planner/work-item");
 const await_completed_dependencies_1 = require("./await-completed-dependencies");
+const deferred_restore_1 = require("./deferred-restore");
 const abort_1 = require("./abort");
 const log_1 = require("../log");
 const watch_loop_1 = require("./watch-loop");
@@ -35,6 +36,7 @@ async function executeWorkService(work, environment, options) {
         });
         await (0, watch_loop_1.watchLoop)(work, environment, options, async (cacheState, abort, stop) => {
             await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
+            await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
             work.state.set({
                 type: 'ready',
                 stateKey: cacheState.stateKey,

package/dist/executer/execute-work-task.js

@@ -11,6 +11,7 @@ const deadline_1 = require("./deadline");
 const explain_1 = require("../cache/explain");
 const archive_task_entry_1 = require("./archive-task-entry");
 const units_1 = require("../utils/units");
+const deferred_restore_1 = require("./deferred-restore");
 async function pushToBackend(work, environment, cacheState, options) {
     const { resolved, stateKey, provable } = cacheState;
     // an aggregate has nothing to store, see checkCacheState
@@ -89,6 +90,10 @@ dependents = null) {
                 }
             }
             await (0, await_completed_dependencies_1.awaitCompletedDependencies)(work, work.deps, abort);
+            // an aggregate runs nothing, so it needs no outputs on disk
+            if (!(0, work_task_1.isAggregateTask)(work.data)) {
+                await (0, deferred_restore_1.restoreDeferred)(work, (0, await_completed_dependencies_1.transitiveDeps)(work.deps), abort);
+            }
             await (0, await_completed_dependencies_1.awaitRunningNeeds)(work, work.needs.map((n) => n.service), abort);
             await options.processManager.task(work, async () => {
                 work.state.set({

package/dist/executer/scheduler/enqueue-next.js

@@ -9,13 +9,17 @@ const work_task_cache_description_1 = require("../../optimizer/work-task-cache-d
 const log_1 = require("../../log");
 const state_key_1 = require("./state-key");
 const work_instance_id_1 = require("../../planner/work-instance-id");
+const deferred_restore_1 = require("../deferred-restore");
 // The key a task whose cache is off runs and records its outputs under. It is
 // no hash, so a later run with its cache on never takes those outputs as its own.
 exports.UNCACHED_STATE_KEY = 'uncached';
 async function checkCacheState(item, defaultCacheMethod, environment, 
 // watch mode compares keys to rerun on a change, so it needs one even when
 // the cache is off
+// restore: false only looks a backend hit up, its outputs are downloaded once
+// something that runs depends on them (`run --no-restore`, see deferred-restore)
 options = {}) {
+    (0, deferred_restore_1.clearDeferredRestore)(item);
     // A task whose cache is off is never looked up, so hashing its sources buys
     // nothing, and a `**` over a node_modules takes most of a minute (every run
     // with arguments). Its dependents still hash its sources for their own key.
@@ -47,13 +51,30 @@ options = {}) {
         if (runtimeStateKey === stateKey) {
             return { cached: true, stateKey, resolved, provable };
         }
+        if (options.restore === false && options.watch) {
+            item.status.write('debug', `${item.name} restores cache hits, --no-restore has no effect in watch mode`);
+        }
+        if (options.restore === false && !options.watch) {
+            try {
+                if (await resolved.backend.has(item.id(), stateKey, environment)) {
+                    item.status.write('info', `${item.name} found in cache "${resolved.name}" (${resolved.backend.type}), not restored`);
+                    await touchEntry(item, resolved, stateKey, environment);
+                    (0, deferred_restore_1.deferRestore)(item, async () => {
+                        if (!(await restoreFromBackend(item, resolved, stateKey, currentStats, environment))) {
+                            throw new Error(`${item.name} is no longer in cache "${resolved.name}", its outputs can not be restored. ` +
+                                'Run again without --no-restore.');
+                        }
+                    });
+                    return { cached: true, stateKey, resolved, provable };
+                }
+            }
+            catch (e) {
+                item.status.write('warn', `${item.name} failed to look up cache "${resolved.name}": ${(0, log_1.getErrorMessage)(e)} — continuing with local execution`);
+            }
+            return { cached: false, stateKey, resolved, provable };
+        }
         try {
-            const cacheDir = (0, get_cache_directory_1.getCacheDirectory)((0, work_instance_id_1.getWorkInstanceId)(item));
-            const pulled = await resolved.backend.pull(item.id(), stateKey, cacheDir, environment);
-            if (pulled) {
-                item.status.write('info', `${item.name} pulled from cache "${resolved.name}" (${resolved.backend.type})`);
-                await item.runtime.restore(environment, cacheDir, stateKey);
-                await (0, cache_metadata_1.writeCacheMetadata)(environment, item, currentStats, (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item));
+            if (await restoreFromBackend(item, resolved, stateKey, currentStats, environment)) {
                 return { cached: true, stateKey, resolved, provable };
             }
         }
@@ -64,4 +85,27 @@ options = {}) {
     return { cached: false, stateKey, resolved, provable };
 }
 exports.checkCacheState = checkCacheState;
+// Pull a task's entry and restore its outputs into the checkout. False when the
+// backend has no such entry.
+async function restoreFromBackend(item, resolved, stateKey, currentStats, environment) {
+    const cacheDir = (0, get_cache_directory_1.getCacheDirectory)((0, work_instance_id_1.getWorkInstanceId)(item));
+    const pulled = await resolved.backend.pull(item.id(), stateKey, cacheDir, environment);
+    if (!pulled) {
+        return false;
+    }
+    item.status.write('info', `${item.name} pulled from cache "${resolved.name}" (${resolved.backend.type})`);
+    await item.runtime.restore(environment, cacheDir, stateKey);
+    await (0, cache_metadata_1.writeCacheMetadata)(environment, item, currentStats, (0, work_task_cache_description_1.getWorkTaskCacheDescription)(item));
+    return true;
+}
+// A hit that is only looked up still counts as used, so retention keeps it.
+// Best effort: the hit stands whether or not the marker could be written.
+async function touchEntry(item, resolved, stateKey, environment) {
+    try {
+        await resolved.backend.touch?.(item.id(), stateKey, environment);
+    }
+    catch (e) {
+        item.status.write('debug', `${item.name} could not mark its cache entry as used: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+}
 //# sourceMappingURL=enqueue-next.js.map

package/dist/executer/watch-loop.js

@@ -12,7 +12,10 @@ async function watchLoop(work, environment, options, factory) {
     // kubernetes-service has no such field. Read it wherever it exists.
     const continuous = 'continuous' in work.data ? work.data.continuous : false;
     const watchMode = options.watch && !options.daemon && !continuous;
-    const currentStats = await (0, enqueue_next_1.checkCacheState)(work, options.cacheDefault, environment, { watch: watchMode });
+    const currentStats = await (0, enqueue_next_1.checkCacheState)(work, options.cacheDefault, environment, {
+        watch: watchMode,
+        restore: options.restore,
+    });
     (0, abort_1.checkForAbort)(environment.abortCtrl.signal);
     const watchState = watchMode
         ? (0, watch_state_key_1.watchStateKey)(work, currentStats, environment, options)

package/dist/program.js

@@ -705,6 +705,7 @@ async function getProgram(environment, argv) {
             .addOption(new commander_1.Option('--max-timeout <duration>', 'fail any task after this long, also one with a longer timeout of its own (e.g. 2m)').argParser(units_1.parseDuration))
             .addOption(new commander_1.Option('--network <level>', 'the most any task or service reaches besides its needs: none, open or allow:<host>,<host>').argParser(parseNetworkArgument))
             .addOption(new commander_1.Option('--no-skip-deps', 'run dependencies even when nothing needing them has to run'))
+            .addOption(new commander_1.Option('--no-restore', 'only look cache hits up, download their outputs only when a task that runs needs them'))
             .action(async (task, options, command) => {
             try {
                 // For machine-readable JSON, suppress the human progress logger (which
@@ -740,6 +741,7 @@ async function getProgram(environment, argv) {
                     maxTimeout: options.maxTimeout ?? null,
                     network: options.network ?? null,
                     skipDeps: options.skipDeps !== false,
+                    restore: options.restore !== false,
                 });
                 // Reporting only: the summary never changes the exit code or behavior.
                 if (!options.watch) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.23",
+  "version": "1.27.0-next.24",
   "keywords": [
     "build",
     "build-tool",