@catladder/pipeline 0.0.0-next-e3ccf7d4 → 0.0.0-next-08efdbb7

package/dist/backends/github/createGithubJobs.js

@@ -157,24 +157,33 @@ var createCacheSteps = function (context, job) {
   }
   var lowered = (_b = (0, createJobCache_1.createJobCacheFromCacheConfigs)(context, job.caches)) !== null && _b !== void 0 ? _b : [];
   return (0, utils_1.ensureArray)(lowered).map(function (cache) {
-    var _a, _b, _c, _d, _e, _f;
+    var _a, _b, _c, _d, _e, _f, _g;
     var paths = (0, utils_1.ensureArray)((_a = cache.paths) !== null && _a !== void 0 ? _a : []);
     var key = cache.key;
-    var _g = typeof key === "string" || key === undefined ? {
+    var _h = typeof key === "string" || key === undefined ? ((_b = cache.keyFiles) === null || _b === void 0 ? void 0 : _b.length) ? {
+        // content-keyed: an unchanged hash is an exact hit and the
+        // cache is never re-saved (no per-run copies, no races
+        // between parallel jobs saving the same key)
+        cacheKey: "".concat(key !== null && key !== void 0 ? key : "cache", "-${{ hashFiles(").concat(cache.keyFiles.map(function (f) {
+          return "'".concat(f, "'");
+        }).join(", "), ") }}"),
+        restoreKeyPrefix: "".concat(key !== null && key !== void 0 ? key : "cache", "-"),
+        label: key !== null && key !== void 0 ? key : "cache"
+      } : {
         // unique key per run so the cache is re-saved, restored via prefix
         cacheKey: "".concat(key !== null && key !== void 0 ? key : "cache", "-${{ github.run_id }}"),
         restoreKeyPrefix: "".concat(key !== null && key !== void 0 ? key : "cache", "-"),
         label: key !== null && key !== void 0 ? key : "cache"
       } : {
-        cacheKey: "".concat((_b = key.prefix) !== null && _b !== void 0 ? _b : "files", "-${{ hashFiles(").concat(((_c = key.files) !== null && _c !== void 0 ? _c : []).map(function (f) {
+        cacheKey: "".concat((_c = key.prefix) !== null && _c !== void 0 ? _c : "files", "-${{ hashFiles(").concat(((_d = key.files) !== null && _d !== void 0 ? _d : []).map(function (f) {
           return "'".concat(f, "'");
         }).join(", "), ") }}"),
-        restoreKeyPrefix: "".concat((_d = key.prefix) !== null && _d !== void 0 ? _d : "files", "-"),
-        label: (_e = key.prefix) !== null && _e !== void 0 ? _e : ((_f = key.files) !== null && _f !== void 0 ? _f : []).join(",")
+        restoreKeyPrefix: "".concat((_e = key.prefix) !== null && _e !== void 0 ? _e : "files", "-"),
+        label: (_f = key.prefix) !== null && _f !== void 0 ? _f : ((_g = key.files) !== null && _g !== void 0 ? _g : []).join(",")
       },
-      cacheKey = _g.cacheKey,
-      restoreKeyPrefix = _g.restoreKeyPrefix,
-      label = _g.label;
+      cacheKey = _h.cacheKey,
+      restoreKeyPrefix = _h.restoreKeyPrefix,
+      label = _h.label;
     return {
       name: "Cache ".concat(label),
       // pull-only caches are restored but never saved

package/dist/backends/gitlab/createGitlabJobs.js

@@ -260,8 +260,21 @@ var makeGitlabJob = function (context, job, allJobs, images, baseRules) {
   var when = jobWhen !== null && jobWhen !== void 0 ? jobWhen : gate ? gate === "manual" ? "manual" : "on_success" : undefined;
   var allow_failure = jobAllowFailure !== null && jobAllowFailure !== void 0 ? jobAllowFailure : gate ? gate === "manual" : undefined;
   // the neutral `caches` declarations resolve to gitlab cache configs;
-  // an explicitly set gitlab `cache` takes precedence
-  var cache = jobCache !== null && jobCache !== void 0 ? jobCache : caches && context.type !== "agent" ? (0, createJobCache_1.createJobCacheFromCacheConfigs)(context, caches) : undefined;
+  // an explicitly set gitlab `cache` takes precedence. `keyFiles` is a
+  // github-only content-key hint and must not leak into the gitlab yaml
+  var stripKeyFiles = function (resolved) {
+    return Array.isArray(resolved) ? resolved.map(function (_a) {
+      var _keyFiles = _a.keyFiles,
+        cacheRest = __rest(_a, ["keyFiles"]);
+      return cacheRest;
+    }) : function (_a) {
+      var _keyFiles = _a.keyFiles,
+        cacheRest = __rest(_a, ["keyFiles"]);
+      return cacheRest;
+    }(resolved);
+  };
+  var resolvedCaches = caches && context.type !== "agent" ? (0, createJobCache_1.createJobCacheFromCacheConfigs)(context, caches) : undefined;
+  var cache = jobCache !== null && jobCache !== void 0 ? jobCache : resolvedCaches && stripKeyFiles(resolvedCaches);
   var stage = envMode === "stagePerEnv" && context.type !== "agent" ? "".concat(job.stage, " ").concat(context.env) : job.stage;
   var deduplicatedGitlabNeeds = getGitlabNeeds(context, job, allJobs, imageNeeds);
   var fullJobName = getFullJobName({

package/dist/build/cache/createJobCache.js

@@ -88,7 +88,8 @@ var createJobCacheFromCacheConfigs = function (context, caches) {
       scope = _a.scope,
       pathMode = _a.pathMode,
       buildDir = _a.buildDir,
-      rest = __rest(_a, ["key", "paths", "policy", "scope", "pathMode", "buildDir"]);
+      keyFiles = _a.keyFiles,
+      rest = __rest(_a, ["key", "paths", "policy", "scope", "pathMode", "buildDir", "keyFiles"]);
     var baseDir = pathMode === "absolute" ? "" : buildDir !== null && buildDir !== void 0 ? buildDir : context.build.dir;
     var transformedKey = scope === "global" ? key : typeof key === "string" ? (scope === "buildDir" // really edge case...
     ? (0, slugify_1.default)(baseDir) : context.name) + "-" + key : __assign(__assign({}, key), {
@@ -96,13 +97,17 @@ var createJobCacheFromCacheConfigs = function (context, caches) {
         return pathMode === "absolute" ? f : (0, path_1.join)(baseDir, f);
       })
     });
-    return __assign({
+    return __assign(__assign({
       key: transformedKey,
       policy: policy !== null && policy !== void 0 ? policy : "pull-push",
       paths: pathMode === "absolute" ? paths : paths === null || paths === void 0 ? void 0 : paths.map(function (p) {
         return (0, path_1.join)(baseDir, p);
       })
-    }, rest);
+    }, (keyFiles === null || keyFiles === void 0 ? void 0 : keyFiles.length) ? {
+      keyFiles: keyFiles.map(function (f) {
+        return pathMode === "absolute" ? f : (0, path_1.join)(baseDir, f);
+      })
+    } : {}), rest);
   });
   return __spreadArray(__spreadArray([], __read(advancedCaches), false), __read(simpleCaches), false);
 };

package/dist/build/node/cache.js

@@ -180,7 +180,10 @@ var getYarnCache = function (context_1) {
             pathMode: componentIsInWorkspace ? "absolute" : "relative",
             key: "yarn",
             policy: policy,
-            paths: [".yarn"]
+            paths: [".yarn"],
+            // content-key for immutable-cache backends (github); the lockfile
+            // decides whether the cached content could have changed
+            keyFiles: ["yarn.lock"]
           }]];
       }
     });
@@ -214,6 +217,10 @@ var getNodeModulesCache = function (context_1) {
             // if component is in a shared workspace, use workspace cache. use individual cache else
             key: componentIsInWorkspace ? "node-modules-workspace" : (0, slugify_1.default)(context.build.dir) + "-node-modules",
             // we use the dirname, not the component name, because in certain cases we have two apps in the same directory and want to share the cache, e.g. when having storybook in the same package.json
+            // content-key for immutable-cache backends (github); the lockfile
+            // decides whether the cached content could have changed (absolute
+            // pathMode: resolve the standalone component's lockfile explicitly)
+            keyFiles: [componentIsInWorkspace ? "yarn.lock" : (0, path_1.join)(context.build.dir, "yarn.lock")],
             policy: policy,
             paths: __spreadArray([], __read(componentIsInWorkspace ? (0, lodash_es_1.uniq)(__spreadArray(__spreadArray(["node_modules"], __read((_a = workspaces.map(function (w) {
               return (0, path_1.join)(w.location, "node_modules");

package/dist/build/types.d.ts

@@ -14,6 +14,17 @@ export type CacheConfigAdvanced = Omit<CatladderJobCache, "paths" | "key"> & {
    * defaults to context
    */
   scope?: "context" | "buildDir" | "global";
+  /**
+   * files whose content determines whether the cached content changed
+   * (e.g. the lockfile for a node_modules cache).
+   *
+   * Backends with immutable caches (github) key the cache by the hash
+   * of these files: an unchanged hash is an exact hit and the cache is
+   * never re-saved (without this, every run saves a fresh multi-GB
+   * cache under a run-unique key, and parallel jobs race on saving
+   * it). Gitlab ignores this and keeps its stable, mutable key.
+   */
+  keyFiles?: string[];
 } & CacheConfigSimple;
 export type CacheConfigSimple = {
   /**

package/dist/constants.js

@@ -4,5 +4,5 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.DOCKER_REGISTRY = exports.PIPELINE_IMAGE_TAG = void 0;
-exports.PIPELINE_IMAGE_TAG = "next-e3ccf7d4" || "latest";
+exports.PIPELINE_IMAGE_TAG = "next-08efdbb7" || "latest";
 exports.DOCKER_REGISTRY = "git.panter.ch:5001/catladder/catladder" || "git.panter.ch:5001/catladder/catladder";