hammerkit 1.27.0-next.5 → 1.27.0-next.7

package/dist/cache/backends/registry-cache-backend.js

@@ -5,6 +5,7 @@ const crypto_1 = require("crypto");
 const os_1 = require("os");
 const path_1 = require("path");
 const tar_1 = require("tar");
+const tar_2 = require("../../utils/tar");
 const registry_reference_1 = require("./registry/registry-reference");
 const registry_client_1 = require("./registry/registry-client");
 const MAX_TAG_LENGTH = 128;
@@ -74,8 +75,7 @@ function createRegistryCacheBackend(spec) {
                 const layerFile = (0, path_1.join)(dir, 'layer.tar');
                 const files = (await environment.file.listFiles(from)).sort();
                 // portable + mtime-free so identical entries produce identical digests
-                // `noMtime` is supported by tar 6 but missing from its type definitions
-                await (0, tar_1.create)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
+                await (0, tar_2.packArchive)({ file: layerFile, cwd: from, portable: true, noMtime: true }, files);
                 const layer = await (0, registry_client_1.digestFile)(layerFile);
                 const config = Buffer.from(JSON.stringify({
                     architecture: 'amd64',

package/dist/executer/event-cache.js

@@ -8,6 +8,8 @@ const execute_docker_1 = require("./execute-docker");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
+const crypto_1 = require("crypto");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 const move_files_1 = require("../file/move-files");
 const work_instance_id_1 = require("../planner/work-instance-id");
 async function restoreCache(environment, path, workTree) {
@@ -43,8 +45,24 @@ function* getArchivePaths(task, path) {
         }
     }
     else {
-        for (const generate of task.generates.filter((g) => !g.inherited)) {
-            const generateId = (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join('-');
+        // `a-b` and `a/b` flatten alike: of those, the least deep (then the first in sorted
+        // order) keeps the name, the others' names carry a hash of their relative path, so a
+        // store and a restore in another directory or order find the same archive
+        const generates = task.generates
+            .filter((g) => !g.inherited)
+            .map((generate) => ({ generate, rel: (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join(path_1.posix.sep) }));
+        const holders = new Map();
+        for (const { rel } of [...generates].sort((a, b) => (0, claim_layout_1.byDepth)(a.rel, b.rel))) {
+            const id = rel.split(path_1.posix.sep).join('-');
+            if (!holders.has(id)) {
+                holders.set(id, rel);
+            }
+        }
+        for (const { generate, rel } of generates) {
+            let generateId = rel.split(path_1.posix.sep).join('-');
+            if (holders.get(generateId) !== rel) {
+                generateId += '-' + (0, crypto_1.createHash)('sha256').update(rel).digest('hex').slice(0, 12);
+            }
             yield {
                 filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
                 path: generate.path,

package/dist/kubernetes/claim-layout.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getClaimSubPath = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const portable_path_1 = require("../planner/utils/portable-path");
+// The paths a work item keeps in its claim. Its consumers mount its outputs by the same
+// paths, so they find the same folders.
+function getClaimPaths(work) {
+    const data = work.data;
+    const paths = [];
+    if ('mounts' in data) {
+        paths.push(...data.mounts.map((mount) => mount.containerPath));
+    }
+    if ('src' in data) {
+        paths.push(...data.src.filter((src) => !src.inherited).map((src) => src.absolutePath));
+    }
+    if ('generates' in data) {
+        paths.push(...data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    }
+    if ('volumes' in data) {
+        paths.push(...data.volumes.filter((volume) => !volume.inherited).map((volume) => volume.containerPath));
+    }
+    return paths;
+}
+// Where `path` is in the claim of `work`, relative to its root. A path is named after its
+// last segment, so claims without two such paths are laid out as they always were (a
+// service's data among them). Of the paths with the same last segment, the least deep one
+// (then the first in sorted order) keeps the name, and the others also carry a hash of
+// their path. Both go by the path
+// relative to the project, as the task's id does, so a run in another checkout directory
+// (each run of a server has its own) or with its `generates` in another order finds the
+// same folders in a kept claim.
+function getClaimSubPath(work, path) {
+    const name = (0, path_1.basename)(path) || 'root';
+    const alike = getClaimPaths(work).filter((other) => other !== path && (0, path_1.basename)(other) === (0, path_1.basename)(path));
+    if (alike.length === 0) {
+        return name;
+    }
+    const own = (0, portable_path_1.portablePath)(work.data.projectRoot, path);
+    const first = [own, ...alike.map((other) => (0, portable_path_1.portablePath)(work.data.projectRoot, other))].sort(byDepth)[0];
+    if (first === own) {
+        return name;
+    }
+    return `${name}-${(0, crypto_1.createHash)('sha256').update(own).digest('hex').slice(0, 12)}`;
+}
+exports.getClaimSubPath = getClaimSubPath;
+function byDepth(a, b) {
+    return a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0);
+}
+exports.byDepth = byDepth;
+// Whether two different paths of the claim of `work` end alike. Before getClaimSubPath
+// named them apart, they shared one folder, and outputs made then are mixed.
+function hasPathsEndingAlike(work) {
+    const paths = new Set(getClaimPaths(work));
+    return new Set([...paths].map((path) => (0, path_1.basename)(path))).size < paths.size;
+}
+exports.hasPathsEndingAlike = hasPathsEndingAlike;
+// Whether two outputs of `work` flatten to the same name of a cache archive (`a-b` and
+// `a/b`, see getArchivePaths). Before they were named apart, one archive held both.
+function hasOutputsFlatteningAlike(work) {
+    const data = work.data;
+    if (!('generates' in data) || !('cwd' in data)) {
+        return false;
+    }
+    const paths = new Set(data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    const names = new Set([...paths].map((path) => (0, path_1.relative)(data.cwd, path).split(path_1.sep).join('-')));
+    return names.size < paths.size;
+}
+exports.hasOutputsFlatteningAlike = hasOutputsFlatteningAlike;
+//# sourceMappingURL=claim-layout.js.map

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
-const tar_1 = require("tar");
+const tar_1 = require("../utils/tar");
 const resources_1 = require("./resources");
 const await_running_state_1 = require("./await-running-state");
 const sleep_1 = require("../utils/sleep");
@@ -137,6 +137,25 @@ async function clearClaim(instance, env, work, pod) {
     });
 }
 exports.clearClaim = clearClaim;
+// The archive that uploads a source into the claim: its file or directory, at its place in
+// the claim (see getClaimSubPath) instead of under its own name
+function packSource(source) {
+    const fileItem = (0, path_1.basename)(source.localPath);
+    return stream_1.Readable.from((0, tar_1.packArchive)({
+        cwd: (0, path_1.dirname)(source.localPath),
+        filter(path) {
+            return source.matcher(path, source.localPath);
+        },
+        onWriteEntry(entry) {
+            entry.path = source.subPath + entry.path.slice(fileItem.length);
+            // the second of two hard-linked files links to the first by its path in the archive
+            if (entry.type === 'Link' && entry.linkpath?.startsWith(fileItem)) {
+                entry.linkpath = source.subPath + entry.linkpath.slice(fileItem.length);
+            }
+        },
+    }, [fileItem]));
+}
+exports.packSource = packSource;
 async function ensurePersistentData(instance, env, environment, service, persistence, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -163,13 +182,7 @@ options = {}) {
             if (!(await environment.file.exists(source.localPath))) {
                 continue;
             }
-            const fileItem = (0, path_1.basename)(source.localPath);
-            const data = stream_1.Readable.from((0, tar_1.create)({
-                cwd: (0, path_1.dirname)(source.localPath),
-                filter(path) {
-                    return source.matcher(path, source.localPath);
-                },
-            }, [fileItem]));
+            const data = packSource(source);
             service.status.console('stdout', `upload ${source.localPath}`);
             const targetPath = `/dev/hammerkit/${service.id()}`;
             const res = await instance.exec.exec(env.namespace, name, 'volume', ['tar', 'xf', '-', '-C', targetPath], process.stdout, process.stderr, data, false, (status) => {

package/dist/kubernetes/restore-kubernetes-data.js

@@ -1,11 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreKubernetesData = void 0;
+exports.restoreKubernetesData = exports.getRestoreTarget = void 0;
 const event_cache_1 = require("../executer/event-cache");
 const log_1 = require("../log");
-const path_1 = require("path");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
+const claim_layout_1 = require("./claim-layout");
+// Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
+// archive of an output: the output's place in the claim, which its mount reads
+function getRestoreTarget(work, path) {
+    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+}
+exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -18,7 +24,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
+            const targetPath = getRestoreTarget(work, generatedArchive.path);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;

package/dist/kubernetes/volumes.js

@@ -4,6 +4,7 @@ exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec
 const path_1 = require("path");
 const fs_1 = require("fs");
 const kept_claims_1 = require("./kept-claims");
+const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
     return (0, kept_claims_1.getClaimName)(service, kubernetes);
 }
@@ -83,6 +84,7 @@ async function getKubernetesPersistence(work, kubernetes) {
 exports.getKubernetesPersistence = getKubernetesPersistence;
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
+    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -101,7 +103,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             readOnly,
             mountPath: containerPath,
             name: volumeName,
-            subPath: (0, path_1.basename)(containerPath),
+            subPath,
         },
         uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
         cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
@@ -111,6 +113,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
+            subPath,
             stateKey: '', // TODO
         });
     }

package/dist/optimizer/work-task-cache-description.js

@@ -6,6 +6,7 @@ const crypto_1 = require("crypto");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const portable_path_1 = require("../planner/utils/portable-path");
 const work_secret_1 = require("../planner/work-secret");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 // Every path is made project-relative (portablePath) so the description, and the
 // task id hashed from it, is identical wherever the project is checked out.
 //
@@ -40,6 +41,7 @@ function getWorkTaskCacheDescription(item) {
         cwd: portable(task.cwd),
         deps: item.deps.length > 0 ? item.deps.map((dep) => dep.id()).sort() : undefined,
         secrets: cachedSecrets.length > 0 ? cachedSecrets : undefined,
+        claimLayout: (0, claim_layout_1.hasPathsEndingAlike)(item) || (0, claim_layout_1.hasOutputsFlatteningAlike)(item) ? 2 : undefined,
     };
 }
 exports.getWorkTaskCacheDescription = getWorkTaskCacheDescription;

package/dist/planner/utils/append-work-dependencies.js

@@ -19,7 +19,10 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
         }
         if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
             ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+            // the outputs of the dep's own deps too, which stay those of the task that made them
+            // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
+                const inherited = generate.inherited ?? depNode;
                 if (generate.isFile) {
                     // TODO check if thats correct
                     continue;
@@ -31,7 +34,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             export: false,
                             resetOnChange: false,
                             containerPath: generate.path,
-                            inherited: depNode,
+                            inherited,
                             readOnly: false,
                         });
                     }
@@ -44,7 +47,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             exportAlways: false,
                             resetOnChange: false,
                             path: generate.path,
-                            inherited: depNode,
+                            inherited,
                             isFile: false,
                         });
                     }

package/dist/planner/work-runtime-docker.js

@@ -12,6 +12,7 @@ const event_cache_1 = require("../executer/event-cache");
 const get_docker_executor_1 = require("../executer/get-docker-executor");
 const path_1 = require("path");
 const tar_1 = require("tar");
+const tar_2 = require("../utils/tar");
 const plan_work_volume_1 = require("./utils/plan-work-volume");
 const work_instance_id_1 = require("./work-instance-id");
 const stream_1 = require("stream");
@@ -302,7 +303,7 @@ async function archiveContainer(docker, environment, item, path) {
         for (const generatedArchive of (0, event_cache_1.getArchivePaths)(item.data, path)) {
             // a file output is a bind mount of a host file, so archive it from the host
             const readable = hostFiles.has(generatedArchive.path)
-                ? stream_1.Readable.from((0, tar_1.create)({ cwd: (0, path_1.dirname)(generatedArchive.path) }, [(0, path_1.basename)(generatedArchive.path)]))
+                ? stream_1.Readable.from((0, tar_2.packArchive)({ cwd: (0, path_1.dirname)(generatedArchive.path) }, [(0, path_1.basename)(generatedArchive.path)]))
                 : await container.getArchive({
                     path: generatedArchive.path,
                 });

package/dist/planner/work-runtime-local.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLocalWorkRuntime = exports.getStateFilename = void 0;
 const tar_1 = require("tar");
+const tar_2 = require("../utils/tar");
 const path_1 = require("path");
 const local_task_1 = require("../executer/local-task");
 const event_cache_1 = require("../executer/event-cache");
@@ -121,7 +122,7 @@ async function archiveLocal(environment, task, path) {
     for (const generatedArchive of (0, event_cache_1.getArchivePaths)(task.data, path)) {
         await environment.file.writeStream(generatedArchive.filename, 
         // tar's stream is a Minipass stream; Readable.from adapts it to a Node one
-        stream_1.Readable.from((0, tar_1.create)({ cwd: task.data.cwd, gzip: true }, [(0, path_1.relative)(task.data.cwd, generatedArchive.path)])));
+        stream_1.Readable.from((0, tar_2.packArchive)({ cwd: task.data.cwd, gzip: true }, [(0, path_1.relative)(task.data.cwd, generatedArchive.path)])));
     }
 }
 async function restoreLocal(environment, task, path) {

package/dist/server/kubernetes-sandbox-storage.js

@@ -7,6 +7,7 @@ const path_1 = require("path");
 const stream_1 = require("stream");
 const promises_2 = require("stream/promises");
 const tar_1 = require("tar");
+const tar_2 = require("../utils/tar");
 const apply_1 = require("../kubernetes/apply");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const network_policy_1 = require("../kubernetes/network-policy");
@@ -463,7 +464,7 @@ class KubernetesSandboxStorage {
             await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'fetch', '--quiet', '--depth', '1', '--', fetch.url, fetch.ref], fetch.env, fetch.signal, what);
             await (0, git_checkout_1.git)([`--git-dir=${repository}`, 'update-ref', `refs/heads/${sandbox_pod_1.INCOMING_REF}`, 'FETCH_HEAD'], fetch.env, fetch.signal, what, op);
             const incoming = `${CLAIM_PATH}/${sandbox_pod_1.INCOMING}`;
-            await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_1.create)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
+            await this.withPod(key, (pod) => this.exec(pod, ['sh', '-c', 'rm -rf "$1" && mkdir -p "$1" && tar xf - -C "$1"', 'sh', incoming], stream_1.Readable.from((0, tar_2.packArchive)({ cwd: repository, portable: true }, ['.'])), null, fetch.signal));
             return (await this.request(key, { op, id: key.id }, fetch.signal)).result;
         }
         finally {

package/dist/utils/tar.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.packArchive = void 0;
+const tar_1 = require("tar");
+function packArchive(options, files) {
+    return (0, tar_1.create)({ ...options, jobs: 1 }, files);
+}
+exports.packArchive = packArchive;
+//# sourceMappingURL=tar.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.5",
+  "version": "1.27.0-next.7",
   "keywords": [
     "build",
     "build-tool",