hammerkit 1.6.0 → 1.7.0

package/dist/cache/backends/registry-cache-backend.js

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRegistryCacheBackend = exports.entryTag = void 0;
+const crypto_1 = require("crypto");
+const os_1 = require("os");
+const path_1 = require("path");
+const tar_1 = require("tar");
+const registry_reference_1 = require("./registry/registry-reference");
+const registry_client_1 = require("./registry/registry-client");
+const MAX_TAG_LENGTH = 128;
+// The deterministic tag for one entry. A task id (sha1) plus a state key (md5)
+// fits comfortably; anything longer is hashed rather than rejected.
+function entryTag(taskId, stateKey) {
+    const tag = `${taskId}-${stateKey}`;
+    if (tag.length <= MAX_TAG_LENGTH && /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(tag)) {
+        return tag;
+    }
+    return `${taskId.substring(0, 40)}-${(0, crypto_1.createHash)('sha256').update(stateKey).digest('hex').substring(0, 64)}`;
+}
+exports.entryTag = entryTag;
+async function withTempDir(environment, fn) {
+    const dir = (0, path_1.join)((0, os_1.tmpdir)(), `hammerkit-registry-${(0, crypto_1.randomUUID)()}`);
+    await environment.file.createDirectory(dir);
+    try {
+        return await fn(dir);
+    }
+    finally {
+        await environment.file.remove(dir);
+    }
+}
+// Stores each cache entry as a standard single-layer OCI image (ADR-0005): the
+// layer is an uncompressed tar of the entry directory (whose archives are
+// already gzipped), tagged `<taskId>-<stateKey>`. The manifest is written only
+// after both blobs are committed, so a partial upload is never visible to
+// has()/pull(); a re-push of the same content is an identical overwrite.
+function createRegistryCacheBackend(spec) {
+    const reference = (0, registry_reference_1.parseRegistryReference)(spec.repository, spec.insecure);
+    const clients = new WeakMap();
+    const client = (environment) => {
+        let instance = clients.get(environment);
+        if (!instance) {
+            instance = new registry_client_1.RegistryClient(reference, environment);
+            clients.set(environment, instance);
+        }
+        return instance;
+    };
+    return {
+        type: 'registry',
+        async has(taskId, stateKey, environment) {
+            return client(environment).manifestExists(entryTag(taskId, stateKey));
+        },
+        async pull(taskId, stateKey, into, environment) {
+            const registry = client(environment);
+            const manifest = await registry.getManifest(entryTag(taskId, stateKey));
+            if (!manifest) {
+                return false;
+            }
+            if (manifest.layers.length !== 1) {
+                throw new Error(`cache entry ${entryTag(taskId, stateKey)} is not a single-layer image`);
+            }
+            return withTempDir(environment, async (dir) => {
+                const layerFile = (0, path_1.join)(dir, 'layer.tar');
+                await registry.downloadBlob(manifest.layers[0], layerFile);
+                await environment.file.createDirectory(into);
+                // node-tar strips absolute paths and `..` segments by default, so a
+                // crafted layer cannot write outside `into`
+                await (0, tar_1.extract)({ file: layerFile, cwd: into });
+                return true;
+            });
+        },
+        async push(taskId, stateKey, from, environment) {
+            const registry = client(environment);
+            await withTempDir(environment, async (dir) => {
+                const layerFile = (0, path_1.join)(dir, 'layer.tar');
+                const files = (await environment.file.listFiles(from)).sort();
+                // portable + mtime-free so identical entries produce identical digests
+                // `noMtime` is supported by tar 6 but missing from its type definitions
+                await (0, tar_1.create)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
+                const layer = await (0, registry_client_1.digestFile)(layerFile);
+                const config = Buffer.from(JSON.stringify({
+                    architecture: 'amd64',
+                    os: 'linux',
+                    created: new Date().toISOString(),
+                    rootfs: { type: 'layers', diff_ids: [layer.digest] },
+                    config: { Labels: { 'dev.hammerkit.task-id': taskId, 'dev.hammerkit.state-key': stateKey } },
+                }));
+                const configDigest = (0, registry_client_1.digestBuffer)(config);
+                await registry.uploadBlob(layer.digest, layer.size, { file: layerFile });
+                await registry.uploadBlob(configDigest.digest, configDigest.size, { buffer: config });
+                const manifest = {
+                    schemaVersion: 2,
+                    mediaType: registry_client_1.OCI_MANIFEST,
+                    config: { mediaType: registry_client_1.OCI_CONFIG, ...configDigest },
+                    layers: [{ mediaType: registry_client_1.OCI_LAYER_TAR, ...layer }],
+                };
+                await registry.putManifest(entryTag(taskId, stateKey), manifest);
+            });
+        },
+        async clear(taskId, environment) {
+            const registry = client(environment);
+            const tags = (await registry.listTags()).filter((tag) => tag.startsWith(`${taskId}-`));
+            for (const tag of tags) {
+                const digest = await registry.manifestDigest(tag);
+                if (digest) {
+                    await registry.deleteManifest(digest);
+                }
+            }
+        },
+        // Entries from the repository's tags: size from the manifest, creation time
+        // from the image config. Registries keep no last-use marker, so retention
+        // works from age and size.
+        async list(environment) {
+            const registry = client(environment);
+            const entries = [];
+            for (const tag of await registry.listTags()) {
+                const match = /^([0-9a-f]{40})-(.+)$/.exec(tag);
+                if (!match) {
+                    continue;
+                }
+                const manifest = await registry.getManifest(tag);
+                if (!manifest) {
+                    continue;
+                }
+                const config = await registry.getBlobJson(manifest.config.digest);
+                entries.push({
+                    taskId: match[1],
+                    stateKey: match[2],
+                    size: manifest.config.size + manifest.layers.reduce((sum, layer) => sum + layer.size, 0),
+                    createdAt: config.created ? Date.parse(config.created) : null,
+                    lastAccessedAt: null,
+                });
+            }
+            return entries;
+        },
+        async remove(taskId, stateKey, environment) {
+            const registry = client(environment);
+            const digest = await registry.manifestDigest(entryTag(taskId, stateKey));
+            if (digest) {
+                await registry.deleteManifest(digest);
+            }
+        },
+    };
+}
+exports.createRegistryCacheBackend = createRegistryCacheBackend;
+//# sourceMappingURL=registry-cache-backend.js.map

package/dist/cache/backends/s3-cache-backend.js

@@ -14,6 +14,14 @@ function isWithin(into, filename) {
     const target = (0, path_1.resolve)(base, filename);
     return target === base || target.startsWith(base + path_1.sep);
 }
+// Only a missing object is a cache miss. Anything else (unreachable endpoint,
+// denied credentials) is rethrown: the inline cache lookup catches it and
+// degrades to a miss with a warning, while an explicit `cache pull`/`push`
+// reports it and fails.
+function isNotFound(e) {
+    const err = e;
+    return err?.name === 'NotFound' || err?.name === 'NoSuchKey' || err?.$metadata?.httpStatusCode === 404;
+}
 function createS3CacheBackend(spec) {
     const client = new client_s3_1.S3Client({
         region: spec.region,
@@ -30,6 +38,28 @@ function createS3CacheBackend(spec) {
     function taskPrefix(taskId) {
         return [prefix, taskId].filter((p) => p.length > 0).join('/') + '/';
     }
+    async function listObjects(listPrefix) {
+        const objects = [];
+        let token;
+        do {
+            const listed = await client.send(new client_s3_1.ListObjectsV2Command({ Bucket: spec.bucket, Prefix: listPrefix, ContinuationToken: token }));
+            for (const obj of listed.Contents ?? []) {
+                if (obj.Key) {
+                    objects.push({ Key: obj.Key, Size: obj.Size ?? 0, LastModified: obj.LastModified });
+                }
+            }
+            token = listed.IsTruncated ? listed.NextContinuationToken : undefined;
+        } while (token);
+        return objects;
+    }
+    async function deleteObjects(keys) {
+        for (let i = 0; i < keys.length; i += 1000) {
+            await client.send(new client_s3_1.DeleteObjectsCommand({
+                Bucket: spec.bucket,
+                Delete: { Objects: keys.slice(i, i + 1000).map((Key) => ({ Key })) },
+            }));
+        }
+    }
     return {
         type: 's3',
         async has(taskId, stateKey) {
@@ -40,49 +70,50 @@ function createS3CacheBackend(spec) {
                 }));
                 return true;
             }
-            catch {
-                return false;
+            catch (e) {
+                if (isNotFound(e)) {
+                    return false;
+                }
+                throw e;
             }
         },
         async pull(taskId, stateKey, into, environment) {
             try {
-                const head = await client
-                    .send(new client_s3_1.HeadObjectCommand({
+                await client.send(new client_s3_1.HeadObjectCommand({
                     Bucket: spec.bucket,
                     Key: keyFor(taskId, stateKey, 'stats.json'),
-                }))
-                    .catch(() => null);
-                if (!head) {
+                }));
+            }
+            catch (e) {
+                if (isNotFound(e)) {
                     return false;
                 }
-                await environment.file.createDirectory(into);
-                const listed = await client.send(new client_s3_1.ListObjectsV2Command({
+                throw e;
+            }
+            await environment.file.createDirectory(into);
+            const listed = await client.send(new client_s3_1.ListObjectsV2Command({
+                Bucket: spec.bucket,
+                Prefix: dirPrefix(taskId, stateKey),
+            }));
+            for (const obj of listed.Contents ?? []) {
+                if (!obj.Key)
+                    continue;
+                const filename = obj.Key.substring(dirPrefix(taskId, stateKey).length);
+                if (!filename)
+                    continue;
+                if (!isWithin(into, filename)) {
+                    environment.console.warn(`skipping cache object outside restore dir: ${obj.Key}`);
+                    continue;
+                }
+                const body = await client.send(new client_s3_1.GetObjectCommand({
                     Bucket: spec.bucket,
-                    Prefix: dirPrefix(taskId, stateKey),
+                    Key: obj.Key,
                 }));
-                for (const obj of listed.Contents ?? []) {
-                    if (!obj.Key)
-                        continue;
-                    const filename = obj.Key.substring(dirPrefix(taskId, stateKey).length);
-                    if (!filename)
-                        continue;
-                    if (!isWithin(into, filename)) {
-                        environment.console.warn(`skipping cache object outside restore dir: ${obj.Key}`);
-                        continue;
-                    }
-                    const body = await client.send(new client_s3_1.GetObjectCommand({
-                        Bucket: spec.bucket,
-                        Key: obj.Key,
-                    }));
-                    if (body.Body instanceof stream_1.Readable) {
-                        await environment.file.writeStream((0, path_1.join)(into, filename), body.Body);
-                    }
+                if (body.Body instanceof stream_1.Readable) {
+                    await environment.file.writeStream((0, path_1.join)(into, filename), body.Body);
                 }
-                return true;
-            }
-            catch {
-                return false;
             }
+            return true;
         },
         async push(taskId, stateKey, from, environment) {
             const files = await environment.file.listFiles(from);
@@ -101,22 +132,44 @@ function createS3CacheBackend(spec) {
             }
         },
         async clear(taskId) {
-            let token;
-            do {
-                const listed = await client.send(new client_s3_1.ListObjectsV2Command({
-                    Bucket: spec.bucket,
-                    Prefix: taskPrefix(taskId),
-                    ContinuationToken: token,
-                }));
-                const objects = (listed.Contents ?? []).filter((o) => o.Key).map((o) => ({ Key: o.Key }));
-                if (objects.length > 0) {
-                    await client.send(new client_s3_1.DeleteObjectsCommand({
-                        Bucket: spec.bucket,
-                        Delete: { Objects: objects },
-                    }));
+            await deleteObjects((await listObjects(taskPrefix(taskId))).map((o) => o.Key));
+        },
+        // Entries from object metadata: size is the sum of the entry's objects,
+        // created is when stats.json (written last) landed. S3 keeps no last-use
+        // marker, so retention on S3 works from age and size.
+        async list() {
+            const root = prefix.length > 0 ? `${prefix}/` : '';
+            const entries = new Map();
+            for (const obj of await listObjects(root)) {
+                const [taskId, stateKey, ...rest] = obj.Key.substring(root.length).split('/');
+                if (!taskId || !stateKey || rest.length === 0) {
+                    continue;
+                }
+                const id = `${taskId}/${stateKey}`;
+                const entry = entries.get(id) ?? {
+                    taskId,
+                    stateKey,
+                    size: 0,
+                    createdAt: null,
+                    lastAccessedAt: null,
+                    complete: false,
+                };
+                entry.size = (entry.size ?? 0) + obj.Size;
+                if (rest.join('/') === 'stats.json') {
+                    entry.complete = true;
+                    entry.createdAt = obj.LastModified?.getTime() ?? null;
                 }
-                token = listed.IsTruncated ? listed.NextContinuationToken : undefined;
-            } while (token);
+                entries.set(id, entry);
+            }
+            return [...entries.values()].filter((entry) => entry.complete).map(({ complete, ...entry }) => entry); // eslint-disable-line @typescript-eslint/no-unused-vars
+        },
+        async remove(taskId, stateKey) {
+            // stats.json first, so a concurrent reader sees the entry as missing
+            // rather than half-deleted
+            const objects = (await listObjects(dirPrefix(taskId, stateKey))).map((o) => o.Key);
+            const stats = objects.filter((key) => key.endsWith('/stats.json'));
+            await deleteObjects(stats);
+            await deleteObjects(objects.filter((key) => !stats.includes(key)));
         },
     };
 }

package/dist/cache/cache-backend-registry.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createCacheBackend = exports.registerCacheBackend = void 0;
 const local_cache_backend_1 = require("./backends/local-cache-backend");
 const s3_cache_backend_1 = require("./backends/s3-cache-backend");
+const registry_cache_backend_1 = require("./backends/registry-cache-backend");
 const factories = new Map();
 function registerCacheBackend(type, factory) {
     factories.set(type, factory);
@@ -18,4 +19,5 @@ function createCacheBackend(spec) {
 exports.createCacheBackend = createCacheBackend;
 registerCacheBackend('local', (spec) => (0, local_cache_backend_1.createLocalCacheBackend)(spec));
 registerCacheBackend('s3', (spec) => (0, s3_cache_backend_1.createS3CacheBackend)(spec));
+registerCacheBackend('registry', (spec) => (0, registry_cache_backend_1.createRegistryCacheBackend)(spec));
 //# sourceMappingURL=cache-backend-registry.js.map

package/dist/cache/cache-inventory.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.autoPrune = exports.pruneCache = exports.hasPolicy = exports.retentionPolicyOf = exports.listCache = void 0;
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const cache_sync_1 = require("./cache-sync");
+const retention_1 = require("./retention");
+const resolve_cache_1 = require("./resolve-cache");
+const units_1 = require("../utils/units");
+function catalogOf(workTree) {
+    return workTree.caches ?? (0, resolve_cache_1.withBuiltinCaches)(undefined);
+}
+function taskNames(workTree) {
+    const names = new Map();
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        names.set(task.id(), task.name);
+    }
+    return names;
+}
+// List the entries a named cache (default: the machine-local `default` cache)
+// holds, labelled with the task names of this build file where they match.
+async function listCache(workTree, cacheName, environment) {
+    const name = cacheName ?? resolve_cache_1.DEFAULT_CACHE_NAME;
+    const backend = (0, cache_sync_1.resolveRemoteBackend)(workTree, name);
+    if (!backend.list) {
+        throw new Error(`the ${backend.type} backend of cache "${name}" cannot list its entries`);
+    }
+    const names = taskNames(workTree);
+    return (await backend.list(environment)).map((entry) => ({ ...entry, taskName: names.get(entry.taskId) ?? null }));
+}
+exports.listCache = listCache;
+// The retention policy of a cache: its declared `retention` block, with any
+// command-line values taking precedence.
+function retentionPolicyOf(workTree, cacheName, overrides) {
+    const declared = catalogOf(workTree)[cacheName ?? resolve_cache_1.DEFAULT_CACHE_NAME]?.retention;
+    return {
+        maxAge: overrides.maxAge ?? (declared?.maxAge ? (0, units_1.parseDuration)(declared.maxAge) : undefined),
+        maxSize: overrides.maxSize ?? (declared?.maxSize ? (0, units_1.parseSize)(declared.maxSize) : undefined),
+        keepPerTask: overrides.keepPerTask ?? declared?.keepPerTask,
+    };
+}
+exports.retentionPolicyOf = retentionPolicyOf;
+function hasPolicy(policy) {
+    return policy.maxAge !== undefined || policy.maxSize !== undefined || policy.keepPerTask !== undefined;
+}
+exports.hasPolicy = hasPolicy;
+// Apply a retention policy to a cache. Only ever touches the cache it is given:
+// a remote is pruned only when named explicitly (ADR-0001).
+async function pruneCache(workTree, cacheName, policy, options, environment) {
+    const name = cacheName ?? resolve_cache_1.DEFAULT_CACHE_NAME;
+    const backend = (0, cache_sync_1.resolveRemoteBackend)(workTree, name);
+    const entries = await listCache(workTree, name, environment);
+    const plan = (0, retention_1.planRetention)(entries, policy, Date.now());
+    if (!options.dryRun && plan.evict.length > 0) {
+        if (!backend.remove) {
+            throw new Error(`the ${backend.type} backend of cache "${name}" cannot remove entries`);
+        }
+        for (const entry of plan.evict) {
+            await backend.remove(entry.taskId, entry.stateKey, environment);
+        }
+    }
+    return plan;
+}
+exports.pruneCache = pruneCache;
+// After a successful run: prune every machine-local cache the run used that
+// declares a retention policy. Remote caches are never pruned implicitly.
+async function autoPrune(workTree, environment) {
+    const catalog = catalogOf(workTree);
+    const used = new Set();
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        used.add(task.data.caching.name);
+    }
+    const results = [];
+    for (const name of used) {
+        const spec = catalog[name];
+        if (!spec?.retention || spec.backend.type !== 'local') {
+            continue;
+        }
+        const policy = retentionPolicyOf(workTree, name, {});
+        results.push({ cacheName: name, plan: await pruneCache(workTree, name, policy, { dryRun: false }, environment) });
+    }
+    return results;
+}
+exports.autoPrune = autoPrune;
+//# sourceMappingURL=cache-inventory.js.map

package/dist/cache/cache-sync.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.syncCache = exports.resolveRemoteBackend = void 0;
+const crypto_1 = require("crypto");
+const os_1 = require("os");
+const path_1 = require("path");
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const state_key_1 = require("../executer/scheduler/state-key");
+const resolve_cache_1 = require("./resolve-cache");
+const archive_task_entry_1 = require("../executer/archive-task-entry");
+function resolveRemoteBackend(workTree, remote) {
+    // resolveCache throws a clear "cache ... is not defined" error for unknown names
+    return (0, resolve_cache_1.resolveCache)({ name: remote }, workTree.caches ?? (0, resolve_cache_1.withBuiltinCaches)(undefined), `--remote ${remote}`).backend;
+}
+exports.resolveRemoteBackend = resolveRemoteBackend;
+// Move cache entries between each in-scope task's own cache and a named remote,
+// executing nothing (ADR-0001). The entries moved are those for the tasks'
+// *current* state keys, computed by the same engine the scheduler uses, so a
+// pull pre-warms exactly what the next build looks up. The work tree already
+// contains the transitive dependencies of the selected scope.
+//
+// Source absence is best-effort ('missing'); any backend error propagates, so
+// an explicit pull/push reports an unreachable remote instead of degrading to a
+// miss like the inline cache lookup does.
+async function syncCache(workTree, options, environment) {
+    const remote = resolveRemoteBackend(workTree, options.remote);
+    const results = [];
+    for (const item of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        const { stateKey, resolved, provable } = await (0, state_key_1.computeStateKey)(item, options.cacheDefault, environment);
+        const local = resolved.backend;
+        const result = (status) => ({
+            taskId: item.id(),
+            taskName: item.name,
+            stateKey,
+            status,
+        });
+        if (resolved.method === 'none' || local === remote || !provable) {
+            results.push(result('skipped'));
+            continue;
+        }
+        const [source, destination] = options.direction === 'pull' ? [remote, local] : [local, remote];
+        if (await destination.has(item.id(), stateKey, environment)) {
+            results.push(result('present'));
+            continue;
+        }
+        // Outputs that are up to date in this checkout but were never stored in a
+        // backend (e.g. restored by the runtime's own state) are packaged directly.
+        if (options.direction === 'push' && !(await local.has(item.id(), stateKey, environment))) {
+            if ((await item.runtime.currentStateKey(environment)) !== stateKey) {
+                results.push(result('missing'));
+                continue;
+            }
+            await remote.push(item.id(), stateKey, await (0, archive_task_entry_1.archiveTaskEntry)(item, environment), environment);
+            results.push(result('transferred'));
+            continue;
+        }
+        // Stage through a private directory: both backends write their completion
+        // marker (stats.json) last, so an interrupted transfer is never visible as a
+        // complete entry on the destination.
+        const staging = (0, path_1.join)((0, os_1.tmpdir)(), `hammerkit-cache-sync-${(0, crypto_1.randomUUID)()}`);
+        try {
+            const pulled = await source.pull(item.id(), stateKey, staging, environment);
+            if (!pulled) {
+                results.push(result('missing'));
+                continue;
+            }
+            await destination.push(item.id(), stateKey, staging, environment);
+            results.push(result('transferred'));
+        }
+        finally {
+            await environment.file.remove(staging);
+        }
+    }
+    return results;
+}
+exports.syncCache = syncCache;
+//# sourceMappingURL=cache-sync.js.map