hammerkit 1.27.0-next.5 → 1.27.0-next.6

package/dist/executer/event-cache.js

@@ -8,6 +8,8 @@ const execute_docker_1 = require("./execute-docker");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
+const crypto_1 = require("crypto");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 const move_files_1 = require("../file/move-files");
 const work_instance_id_1 = require("../planner/work-instance-id");
 async function restoreCache(environment, path, workTree) {
@@ -43,8 +45,24 @@ function* getArchivePaths(task, path) {
         }
     }
     else {
-        for (const generate of task.generates.filter((g) => !g.inherited)) {
-            const generateId = (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join('-');
+        // `a-b` and `a/b` flatten alike: of those, the least deep (then the first in sorted
+        // order) keeps the name, the others' names carry a hash of their relative path, so a
+        // store and a restore in another directory or order find the same archive
+        const generates = task.generates
+            .filter((g) => !g.inherited)
+            .map((generate) => ({ generate, rel: (0, path_1.relative)(task.cwd, generate.path).split(path_1.sep).join(path_1.posix.sep) }));
+        const holders = new Map();
+        for (const { rel } of [...generates].sort((a, b) => (0, claim_layout_1.byDepth)(a.rel, b.rel))) {
+            const id = rel.split(path_1.posix.sep).join('-');
+            if (!holders.has(id)) {
+                holders.set(id, rel);
+            }
+        }
+        for (const { generate, rel } of generates) {
+            let generateId = rel.split(path_1.posix.sep).join('-');
+            if (holders.get(generateId) !== rel) {
+                generateId += '-' + (0, crypto_1.createHash)('sha256').update(rel).digest('hex').slice(0, 12);
+            }
             yield {
                 filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
                 path: generate.path,

package/dist/kubernetes/claim-layout.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasOutputsFlatteningAlike = exports.hasPathsEndingAlike = exports.byDepth = exports.getClaimSubPath = void 0;
+const crypto_1 = require("crypto");
+const path_1 = require("path");
+const portable_path_1 = require("../planner/utils/portable-path");
+// The paths a work item keeps in its claim. Its consumers mount its outputs by the same
+// paths, so they find the same folders.
+function getClaimPaths(work) {
+    const data = work.data;
+    const paths = [];
+    if ('mounts' in data) {
+        paths.push(...data.mounts.map((mount) => mount.containerPath));
+    }
+    if ('src' in data) {
+        paths.push(...data.src.filter((src) => !src.inherited).map((src) => src.absolutePath));
+    }
+    if ('generates' in data) {
+        paths.push(...data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    }
+    if ('volumes' in data) {
+        paths.push(...data.volumes.filter((volume) => !volume.inherited).map((volume) => volume.containerPath));
+    }
+    return paths;
+}
+// Where `path` is in the claim of `work`, relative to its root. A path is named after its
+// last segment, so claims without two such paths are laid out as they always were (a
+// service's data among them). Of the paths with the same last segment, the least deep one
+// (then the first in sorted order) keeps the name, and the others also carry a hash of
+// their path. Both go by the path
+// relative to the project, as the task's id does, so a run in another checkout directory
+// (each run of a server has its own) or with its `generates` in another order finds the
+// same folders in a kept claim.
+function getClaimSubPath(work, path) {
+    const name = (0, path_1.basename)(path) || 'root';
+    const alike = getClaimPaths(work).filter((other) => other !== path && (0, path_1.basename)(other) === (0, path_1.basename)(path));
+    if (alike.length === 0) {
+        return name;
+    }
+    const own = (0, portable_path_1.portablePath)(work.data.projectRoot, path);
+    const first = [own, ...alike.map((other) => (0, portable_path_1.portablePath)(work.data.projectRoot, other))].sort(byDepth)[0];
+    if (first === own) {
+        return name;
+    }
+    return `${name}-${(0, crypto_1.createHash)('sha256').update(own).digest('hex').slice(0, 12)}`;
+}
+exports.getClaimSubPath = getClaimSubPath;
+function byDepth(a, b) {
+    return a.split('/').length - b.split('/').length || (a < b ? -1 : a > b ? 1 : 0);
+}
+exports.byDepth = byDepth;
+// Whether two different paths of the claim of `work` end alike. Before getClaimSubPath
+// named them apart, they shared one folder, and outputs made then are mixed.
+function hasPathsEndingAlike(work) {
+    const paths = new Set(getClaimPaths(work));
+    return new Set([...paths].map((path) => (0, path_1.basename)(path))).size < paths.size;
+}
+exports.hasPathsEndingAlike = hasPathsEndingAlike;
+// Whether two outputs of `work` flatten to the same name of a cache archive (`a-b` and
+// `a/b`, see getArchivePaths). Before they were named apart, one archive held both.
+function hasOutputsFlatteningAlike(work) {
+    const data = work.data;
+    if (!('generates' in data) || !('cwd' in data)) {
+        return false;
+    }
+    const paths = new Set(data.generates.filter((generate) => !generate.inherited).map((generate) => generate.path));
+    const names = new Set([...paths].map((path) => (0, path_1.relative)(data.cwd, path).split(path_1.sep).join('-')));
+    return names.size < paths.size;
+}
+exports.hasOutputsFlatteningAlike = hasOutputsFlatteningAlike;
+//# sourceMappingURL=claim-layout.js.map

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
 const tar_1 = require("tar");
@@ -137,6 +137,25 @@ async function clearClaim(instance, env, work, pod) {
     });
 }
 exports.clearClaim = clearClaim;
+// The archive that uploads a source into the claim: its file or directory, at its place in
+// the claim (see getClaimSubPath) instead of under its own name
+function packSource(source) {
+    const fileItem = (0, path_1.basename)(source.localPath);
+    return stream_1.Readable.from((0, tar_1.create)({
+        cwd: (0, path_1.dirname)(source.localPath),
+        filter(path) {
+            return source.matcher(path, source.localPath);
+        },
+        onWriteEntry(entry) {
+            entry.path = source.subPath + entry.path.slice(fileItem.length);
+            // the second of two hard-linked files links to the first by its path in the archive
+            if (entry.type === 'Link' && entry.linkpath?.startsWith(fileItem)) {
+                entry.linkpath = source.subPath + entry.linkpath.slice(fileItem.length);
+            }
+        },
+    }, [fileItem]));
+}
+exports.packSource = packSource;
 async function ensurePersistentData(instance, env, environment, service, persistence, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -163,13 +182,7 @@ options = {}) {
             if (!(await environment.file.exists(source.localPath))) {
                 continue;
             }
-            const fileItem = (0, path_1.basename)(source.localPath);
-            const data = stream_1.Readable.from((0, tar_1.create)({
-                cwd: (0, path_1.dirname)(source.localPath),
-                filter(path) {
-                    return source.matcher(path, source.localPath);
-                },
-            }, [fileItem]));
+            const data = packSource(source);
             service.status.console('stdout', `upload ${source.localPath}`);
             const targetPath = `/dev/hammerkit/${service.id()}`;
             const res = await instance.exec.exec(env.namespace, name, 'volume', ['tar', 'xf', '-', '-C', targetPath], process.stdout, process.stderr, data, false, (status) => {

package/dist/kubernetes/restore-kubernetes-data.js

@@ -1,11 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreKubernetesData = void 0;
+exports.restoreKubernetesData = exports.getRestoreTarget = void 0;
 const event_cache_1 = require("../executer/event-cache");
 const log_1 = require("../log");
-const path_1 = require("path");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
+const claim_layout_1 = require("./claim-layout");
+// Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
+// archive of an output: the output's place in the claim, which its mount reads
+function getRestoreTarget(work, path) {
+    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+}
+exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -18,7 +24,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
+            const targetPath = getRestoreTarget(work, generatedArchive.path);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;

package/dist/kubernetes/volumes.js

@@ -4,6 +4,7 @@ exports.getSandboxAffinity = exports.SANDBOX_LABEL = exports.getWorkspacePodSpec
 const path_1 = require("path");
 const fs_1 = require("fs");
 const kept_claims_1 = require("./kept-claims");
+const claim_layout_1 = require("./claim-layout");
 function getVolumeName(service, kubernetes) {
     return (0, kept_claims_1.getClaimName)(service, kubernetes);
 }
@@ -83,6 +84,7 @@ async function getKubernetesPersistence(work, kubernetes) {
 exports.getKubernetesPersistence = getKubernetesPersistence;
 function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly, kubernetes) {
     const volumeName = getVolumeName(sourceWork, kubernetes);
+    const subPath = (0, claim_layout_1.getClaimSubPath)(sourceWork, containerPath);
     const existVolume = persistence.volumes.some((v) => v.name === volumeName);
     if (!existVolume) {
         persistence.volumes.push({
@@ -101,7 +103,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             readOnly,
             mountPath: containerPath,
             name: volumeName,
-            subPath: (0, path_1.basename)(containerPath),
+            subPath,
         },
         uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
         cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
@@ -111,6 +113,7 @@ function appendVolume(sourceWork, persistence, containerPath, localPath, isFile,
             localPath: localPath,
             matcher: () => true, // TODO
             containerPath,
+            subPath,
             stateKey: '', // TODO
         });
     }

package/dist/optimizer/work-task-cache-description.js

@@ -6,6 +6,7 @@ const crypto_1 = require("crypto");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const portable_path_1 = require("../planner/utils/portable-path");
 const work_secret_1 = require("../planner/work-secret");
+const claim_layout_1 = require("../kubernetes/claim-layout");
 // Every path is made project-relative (portablePath) so the description, and the
 // task id hashed from it, is identical wherever the project is checked out.
 //
@@ -40,6 +41,7 @@ function getWorkTaskCacheDescription(item) {
         cwd: portable(task.cwd),
         deps: item.deps.length > 0 ? item.deps.map((dep) => dep.id()).sort() : undefined,
         secrets: cachedSecrets.length > 0 ? cachedSecrets : undefined,
+        claimLayout: (0, claim_layout_1.hasPathsEndingAlike)(item) || (0, claim_layout_1.hasOutputsFlatteningAlike)(item) ? 2 : undefined,
     };
 }
 exports.getWorkTaskCacheDescription = getWorkTaskCacheDescription;

package/dist/planner/utils/append-work-dependencies.js

@@ -19,7 +19,10 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
         }
         if (((0, work_task_1.isContainerWorkTask)(depNode.data) || (0, work_service_1.isContainerWorkService)(depNode.data)) &&
             ((0, work_task_1.isContainerWorkTask)(item.data) || (0, work_service_1.isContainerWorkService)(item.data))) {
+            // the outputs of the dep's own deps too, which stay those of the task that made them
+            // (on Kubernetes, they are in its claim)
             for (const generate of depNode.data.generates) {
+                const inherited = generate.inherited ?? depNode;
                 if (generate.isFile) {
                     // TODO check if thats correct
                     continue;
@@ -31,7 +34,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             export: false,
                             resetOnChange: false,
                             containerPath: generate.path,
-                            inherited: depNode,
+                            inherited,
                             readOnly: false,
                         });
                     }
@@ -44,7 +47,7 @@ function appendWorkDependencies(workTree, referenced, item, environment, context
                             exportAlways: false,
                             resetOnChange: false,
                             path: generate.path,
-                            inherited: depNode,
+                            inherited,
                             isFile: false,
                         });
                     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.27.0-next.5",
+  "version": "1.27.0-next.6",
   "keywords": [
     "build",
     "build-tool",