@invarn/cibuild 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;AA6HF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YA0HzF,iBAAiB;CA0MhC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YAsLzF,iBAAiB;CA0JhC"}
|
|
@@ -87,28 +87,64 @@ function resolvePresetChain(technology) {
|
|
|
87
87
|
return chain;
|
|
88
88
|
}
|
|
89
89
|
/**
|
|
90
|
-
* Emits the
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* hit to ~/cache, and streams it back — so we pipe straight to extraction
|
|
94
|
-
* without re-writing it on the guest. The optional per-build bearer token
|
|
95
|
-
* ($CIBUILD_CACHE_TOKEN) is sent only when set. Inert when the env var is
|
|
96
|
-
* empty, so the same script runs with or without a daemon.
|
|
90
|
+
* Emits the `curl` invocation the cache steps talk to the daemon with. The
|
|
91
|
+
* bearer token is per-build and only present on the Invarn runner, so it is
|
|
92
|
+
* expanded conditionally and the same script works without one.
|
|
97
93
|
*/
|
|
98
|
-
function
|
|
94
|
+
function daemonCurl(args) {
|
|
95
|
+
return `curl -fsS \${CIBUILD_CACHE_TOKEN:+-H "Authorization: Bearer $CIBUILD_CACHE_TOKEN"} ${args}`;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Restores a cache through the daemon rather than a shared directory.
|
|
99
|
+
*
|
|
100
|
+
* Two requests, mirroring the directory-based flow it replaces: the exact key
|
|
101
|
+
* first, then — on a miss — the scope's newest tarball. The daemon resolves
|
|
102
|
+
* each from its own disk or a tailnet peer, so a hit here covers what used to
|
|
103
|
+
* be separate "local" and "peer" branches; which one it was is recorded in the
|
|
104
|
+
* daemon's own host-side log, where the fleet-wide view already lives.
|
|
105
|
+
*
|
|
106
|
+
* The 30-day age cap is preserved: the daemon dates the fallback with
|
|
107
|
+
* `mtimeMs`, standing in for the `stat` the guest can no longer perform.
|
|
108
|
+
*/
|
|
109
|
+
function emitDaemonPullCommands() {
|
|
99
110
|
return [
|
|
100
|
-
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
|
|
104
|
-
// status is preserved (pipefail), so the fall-through still works.
|
|
105
|
-
'elif [ -n "$CIBUILD_CACHE_DAEMON" ] && { curl -fsS ${CIBUILD_CACHE_TOKEN:+-H "Authorization: Bearer $CIBUILD_CACHE_TOKEN"} "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst" | zstd -dc | tar -xf - -C /; } 2>/dev/null; then',
|
|
111
|
+
'echo "Cache daemon: $CIBUILD_CACHE_DAEMON"',
|
|
112
|
+
// Exact key. stderr silenced — a miss is the normal case and curl/zstd
|
|
113
|
+
// both editorialise about it; pipefail still carries the exit status.
|
|
114
|
+
`if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`,
|
|
106
115
|
' echo "Cache found (daemon), extracting..."',
|
|
107
|
-
// LRU bump the tarball the daemon just persisted to ~/cache (same dir as
|
|
108
|
-
// the --dir mount); tolerate its absence in case the daemon served it
|
|
109
|
-
// without a local copy.
|
|
110
|
-
' touch "$CACHE_FILE" 2>/dev/null || true',
|
|
111
116
|
' echo "CACHE_SOURCE=daemon CACHE_KEY=$CACHE_KEY"',
|
|
117
|
+
'else',
|
|
118
|
+
// Scope fallback: the newest tarball for this <keyPrefix>-<projectId>.
|
|
119
|
+
// Gradle/Xcode re-hash their inputs on top of the warm directory, hitting
|
|
120
|
+
// the unchanged work and rebuilding only what actually moved.
|
|
121
|
+
' __ci_fb_scope="${CACHE_KEY%-*}"',
|
|
122
|
+
` __ci_fb_json=$(${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/scope/$__ci_fb_scope/latest"')} 2>/dev/null || true)`,
|
|
123
|
+
// Parse without a JSON tool: the guest image is not guaranteed to ship jq.
|
|
124
|
+
` __ci_fb_key=$(printf '%s' "$__ci_fb_json" | sed -n 's/.*"key"[[:space:]]*:[[:space:]]*"\\([^"]*\\)".*/\\1/p')`,
|
|
125
|
+
` __ci_fb_mtime=$(printf '%s' "$__ci_fb_json" | sed -n 's/.*"mtimeMs"[[:space:]]*:[[:space:]]*\\([0-9]*\\).*/\\1/p')`,
|
|
126
|
+
' if [ -n "$__ci_fb_key" ] && [ -n "$__ci_fb_mtime" ]; then',
|
|
127
|
+
' __ci_fb_age_days=$(( ($(date +%s) - __ci_fb_mtime / 1000) / 86400 ))',
|
|
128
|
+
// Age cap: a fallback older than 30d likely carries obsolete toolchain
|
|
129
|
+
// installs or cache entries from a pre-wrapper-bump world. Go cold rather
|
|
130
|
+
// than seed from ancient state. Exact-key hits are unaffected.
|
|
131
|
+
' if [ "$__ci_fb_age_days" -le 30 ]; then',
|
|
132
|
+
' echo "Cache fallback (daemon): using prior tarball $__ci_fb_key (${__ci_fb_age_days}d old)"',
|
|
133
|
+
` if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$__ci_fb_key.tar.zst"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`,
|
|
134
|
+
' echo "CACHE_SOURCE=fallback_daemon CACHE_KEY=$CACHE_KEY FALLBACK_KEY=$__ci_fb_key"',
|
|
135
|
+
' else',
|
|
136
|
+
' echo "Cache fallback $__ci_fb_key could not be fetched — going cold"',
|
|
137
|
+
' echo "CACHE_SOURCE=cold CACHE_KEY=$CACHE_KEY"',
|
|
138
|
+
' fi',
|
|
139
|
+
' else',
|
|
140
|
+
' echo "Cache fallback candidate $__ci_fb_key is ${__ci_fb_age_days}d old, exceeds 30d cap — going cold"',
|
|
141
|
+
' echo "CACHE_SOURCE=cold CACHE_KEY=$CACHE_KEY FALLBACK_REJECTED=$__ci_fb_key"',
|
|
142
|
+
' fi',
|
|
143
|
+
' else',
|
|
144
|
+
' echo "No cache found for key: $CACHE_KEY"',
|
|
145
|
+
' echo "CACHE_SOURCE=cold CACHE_KEY=$CACHE_KEY"',
|
|
146
|
+
' fi',
|
|
147
|
+
'fi',
|
|
112
148
|
];
|
|
113
149
|
}
|
|
114
150
|
/**
|
|
@@ -154,7 +190,11 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
154
190
|
commands.push('');
|
|
155
191
|
commands.push('# Ensure cache directory exists');
|
|
156
192
|
commands.push(`CACHE_DIR="${this.escapeBash(config.paths.cacheDir)}"`);
|
|
157
|
-
|
|
193
|
+
// Only the filesystem transport needs the directory to exist. Over the
|
|
194
|
+
// daemon the cache lives on the host, and on a runner this path is no
|
|
195
|
+
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
196
|
+
// whole step under errexit.
|
|
197
|
+
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
158
198
|
// Generate cache file name based on cache key
|
|
159
199
|
commands.push('');
|
|
160
200
|
commands.push('# Generate cache file path');
|
|
@@ -174,6 +214,17 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
174
214
|
// Check if cache file exists
|
|
175
215
|
commands.push('');
|
|
176
216
|
commands.push('# Check if cache exists');
|
|
217
|
+
// Transport split — see the preset path. An explicit-key step has no scope
|
|
218
|
+
// to fall back to, so the daemon side is the exact key or nothing.
|
|
219
|
+
commands.push('if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
|
|
220
|
+
commands.push(` if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`);
|
|
221
|
+
commands.push(' echo "Cache found (daemon), extracting..."');
|
|
222
|
+
commands.push(' echo "CACHE_SOURCE=daemon"');
|
|
223
|
+
commands.push(' else');
|
|
224
|
+
commands.push(` echo "No cache found for key: ${this.escapeBash(cacheKey)}"`);
|
|
225
|
+
commands.push(' echo "CACHE_SOURCE=cold"');
|
|
226
|
+
commands.push(' fi');
|
|
227
|
+
commands.push('else');
|
|
177
228
|
commands.push('if [ -f "$CACHE_FILE" ]; then');
|
|
178
229
|
commands.push(' echo "Cache found (local), extracting..."');
|
|
179
230
|
if (isDebugMode) {
|
|
@@ -181,8 +232,6 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
181
232
|
}
|
|
182
233
|
commands.push(' zstd -dc "$CACHE_FILE" | tar -xf - -C /');
|
|
183
234
|
commands.push(' echo "CACHE_SOURCE=local"');
|
|
184
|
-
// Daemon hit (ADR 0001) — see emitDaemonPullBranch.
|
|
185
|
-
commands.push(...emitDaemonPullBranch());
|
|
186
235
|
// Peer fallback
|
|
187
236
|
if (peerCacheDir) {
|
|
188
237
|
commands.push('elif [ -f "$PEER_CACHE_FILE" ] 2>/dev/null; then');
|
|
@@ -195,6 +244,8 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
195
244
|
commands.push(` echo "No cache found for key: ${this.escapeBash(cacheKey)}"`);
|
|
196
245
|
commands.push(' echo "CACHE_SOURCE=cold"');
|
|
197
246
|
commands.push('fi');
|
|
247
|
+
// Closes the daemon/filesystem transport split.
|
|
248
|
+
commands.push('fi');
|
|
198
249
|
const script = this.createBashScriptFromCommands(commands, stepName);
|
|
199
250
|
return this.createScriptStep(script, stepName);
|
|
200
251
|
}
|
|
@@ -213,7 +264,11 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
213
264
|
// Compute cache key from lockfile checksum at runtime
|
|
214
265
|
// If technology has a fallback chain, try each lockfile in order
|
|
215
266
|
commands.push(`CACHE_DIR="${this.escapeBash(config.paths.cacheDir)}"`);
|
|
216
|
-
|
|
267
|
+
// Only the filesystem transport needs the directory to exist. Over the
|
|
268
|
+
// daemon the cache lives on the host, and on a runner this path is no
|
|
269
|
+
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
270
|
+
// whole step under errexit.
|
|
271
|
+
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
217
272
|
commands.push('');
|
|
218
273
|
// Helper: resolve lockfile path — checks exact path first, then searches
|
|
219
274
|
// recursively (handles SPM Package.resolved inside .xcodeproj/.xcworkspace)
|
|
@@ -273,6 +328,12 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
273
328
|
}
|
|
274
329
|
commands.push('CACHE_FILE="$CACHE_DIR/$CACHE_KEY.tar.zst"');
|
|
275
330
|
commands.push('');
|
|
331
|
+
// Transport split. On a runner the cache lives on the host and is reached
|
|
332
|
+
// over HTTP; standalone, it is a directory on this machine. The daemon URL
|
|
333
|
+
// is the switch, so the same generated script serves both.
|
|
334
|
+
commands.push('if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
|
|
335
|
+
commands.push(...emitDaemonPullCommands());
|
|
336
|
+
commands.push('else');
|
|
276
337
|
// Peer cache directory — set from config, empty string disables peer lookup
|
|
277
338
|
const peerCacheDir = config.paths.peerCacheDir || '';
|
|
278
339
|
if (peerCacheDir) {
|
|
@@ -292,13 +353,6 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
292
353
|
// hit daily would still be reaped on its 31st day from creation.
|
|
293
354
|
commands.push(' touch "$CACHE_FILE"');
|
|
294
355
|
commands.push(' echo "CACHE_SOURCE=local CACHE_KEY=$CACHE_KEY"');
|
|
295
|
-
// 1b. Daemon hit (ADR 0001) — on a local miss, ask this runner's cache
|
|
296
|
-
// daemon for the exact key. The daemon resolves it from a tailnet peer
|
|
297
|
-
// and atomically persists it to ~/cache (the same dir as the --dir mount),
|
|
298
|
-
// so we stream the response straight to extraction with no redundant
|
|
299
|
-
// guest-side write. Gated at runtime on the env var the runner injects, so
|
|
300
|
-
// the generated script is inert when no daemon is configured.
|
|
301
|
-
commands.push(...emitDaemonPullBranch());
|
|
302
356
|
// 2. Peer hit — tee to warm local cache while extracting in one pass
|
|
303
357
|
if (peerCacheDir) {
|
|
304
358
|
commands.push('elif [ -f "$PEER_CACHE_FILE" ] 2>/dev/null; then');
|
|
@@ -362,6 +416,8 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
362
416
|
commands.push(' echo "CACHE_SOURCE=cold CACHE_KEY=$CACHE_KEY"');
|
|
363
417
|
commands.push(' fi');
|
|
364
418
|
commands.push('fi');
|
|
419
|
+
// Closes the daemon/filesystem transport split.
|
|
420
|
+
commands.push('fi');
|
|
365
421
|
if (isDebugMode) {
|
|
366
422
|
commands.push('echo "Restored paths:"');
|
|
367
423
|
const allPaths = [...new Set(chain.flatMap(p => p.paths))];
|
|
@@ -427,7 +483,11 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
427
483
|
commands.push('');
|
|
428
484
|
commands.push('# Ensure cache directory exists');
|
|
429
485
|
commands.push(`CACHE_DIR="${this.escapeBash(config.paths.cacheDir)}"`);
|
|
430
|
-
|
|
486
|
+
// Only the filesystem transport needs the directory to exist. Over the
|
|
487
|
+
// daemon the cache lives on the host, and on a runner this path is no
|
|
488
|
+
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
489
|
+
// whole step under errexit.
|
|
490
|
+
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
431
491
|
// Generate cache file name based on cache key
|
|
432
492
|
commands.push('');
|
|
433
493
|
commands.push('# Generate cache file path');
|
|
@@ -488,7 +548,10 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
488
548
|
// Create cache if we have paths
|
|
489
549
|
commands.push('');
|
|
490
550
|
commands.push('# Skip if cache already exists for this key');
|
|
491
|
-
|
|
551
|
+
// Over the daemon there is no local file to test, so the "already up to
|
|
552
|
+
// date" shortcut can't apply — the daemon's PUT is an idempotent overwrite
|
|
553
|
+
// under the same key, which gets the same outcome.
|
|
554
|
+
commands.push('if [ -z "$CIBUILD_CACHE_DAEMON" ] && [ -f "$CACHE_FILE" ]; then');
|
|
492
555
|
commands.push(' echo "Cache already up to date for key: $CACHE_KEY"');
|
|
493
556
|
commands.push('elif [ ${#PATHS_TO_CACHE[@]} -gt 0 ]; then');
|
|
494
557
|
commands.push(' echo "Caching ${#PATHS_TO_CACHE[@]} path(s)..."');
|
|
@@ -496,6 +559,14 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
496
559
|
commands.push(' echo "Paths to cache:"');
|
|
497
560
|
commands.push(' printf " %s\\n" "${PATHS_TO_CACHE[@]}"');
|
|
498
561
|
}
|
|
562
|
+
// Transport split — see the preset path.
|
|
563
|
+
commands.push(' if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
|
|
564
|
+
commands.push(` if tar -cf - "\${PATHS_TO_CACHE[@]}" 2>/dev/null | zstd -3 | ${daemonCurl('-T - "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} > /dev/null; then`);
|
|
565
|
+
commands.push(' echo "Cache created successfully"');
|
|
566
|
+
commands.push(' else');
|
|
567
|
+
commands.push(' echo "Warning: failed to upload cache to the daemon"');
|
|
568
|
+
commands.push(' fi');
|
|
569
|
+
commands.push(' else');
|
|
499
570
|
// Atomic write into a per-process-unique temp (CBR-12). The "runs serialized
|
|
500
571
|
// per runner" assumption behind a fixed "$CACHE_FILE.tmp" is false with two VM
|
|
501
572
|
// slots and collides with the host cache daemon mutating this VirtioFS-shared
|
|
@@ -511,6 +582,8 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
511
582
|
commands.push(' else');
|
|
512
583
|
commands.push(' echo "Warning: Failed to create cache file"');
|
|
513
584
|
commands.push(' fi');
|
|
585
|
+
// Closes the daemon/filesystem transport split.
|
|
586
|
+
commands.push(' fi');
|
|
514
587
|
commands.push('else');
|
|
515
588
|
commands.push(' echo "No paths to cache"');
|
|
516
589
|
commands.push('fi');
|
|
@@ -531,7 +604,11 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
531
604
|
commands.push('');
|
|
532
605
|
// Compute cache key from lockfile checksum at runtime
|
|
533
606
|
commands.push(`CACHE_DIR="${this.escapeBash(config.paths.cacheDir)}"`);
|
|
534
|
-
|
|
607
|
+
// Only the filesystem transport needs the directory to exist. Over the
|
|
608
|
+
// daemon the cache lives on the host, and on a runner this path is no
|
|
609
|
+
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
610
|
+
// whole step under errexit.
|
|
611
|
+
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
535
612
|
commands.push('');
|
|
536
613
|
commands.push('__ci_find_lockfile() {');
|
|
537
614
|
commands.push(' if [ -f "$1" ]; then echo "$1"; return; fi');
|
|
@@ -610,6 +687,19 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
610
687
|
// build. Atomic .tmp + mv overwrites any prior tarball under this key.
|
|
611
688
|
commands.push('if [ ${#PATHS_TO_CACHE[@]} -gt 0 ]; then');
|
|
612
689
|
commands.push(' echo "Caching ${#PATHS_TO_CACHE[@]} path(s)..."');
|
|
690
|
+
// Transport split, mirroring cache-pull. Over the daemon the tarball is
|
|
691
|
+
// streamed straight out as the PUT body — nothing is staged on the guest,
|
|
692
|
+
// so there is no temp file to collide or orphan, and the daemon does the
|
|
693
|
+
// atomic temp+rename on the host side where the cache actually lives.
|
|
694
|
+
commands.push(' if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
|
|
695
|
+
commands.push(` if tar -cf - "\${PATHS_TO_CACHE[@]}" 2>/dev/null | zstd -3 | ${daemonCurl('-T - "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} > /dev/null; then`);
|
|
696
|
+
commands.push(' echo "Cache created successfully"');
|
|
697
|
+
commands.push(' else');
|
|
698
|
+
// Never fatal: a build that produced good output should not be failed by a
|
|
699
|
+
// cache upload, and the next build simply misses.
|
|
700
|
+
commands.push(' echo "Warning: failed to upload cache to the daemon"');
|
|
701
|
+
commands.push(' fi');
|
|
702
|
+
commands.push(' else');
|
|
613
703
|
// Per-process-unique temp (CBR-12). The old fixed "$CACHE_FILE.tmp" assumed
|
|
614
704
|
// a single serialized writer — false with two VM slots, and it collides with
|
|
615
705
|
// the host cache daemon mutating this VirtioFS-shared dir, which (a) made the
|
|
@@ -632,6 +722,8 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
632
722
|
commands.push(' else');
|
|
633
723
|
commands.push(' echo "Warning: Failed to create cache file"');
|
|
634
724
|
commands.push(' fi');
|
|
725
|
+
// Closes the daemon/filesystem transport split.
|
|
726
|
+
commands.push(' fi');
|
|
635
727
|
commands.push('else');
|
|
636
728
|
commands.push(' echo "No paths to cache"');
|
|
637
729
|
commands.push('fi');
|
|
@@ -326,10 +326,26 @@ describe('Step Implementations', () => {
|
|
|
326
326
|
expect(result.script).toContain('ls -t "$CACHE_DIR"/"$__ci_fb_scope"-*.tar.zst');
|
|
327
327
|
expect(result.script).toContain('CACHE_SOURCE=fallback_');
|
|
328
328
|
expect(result.script).toContain('FALLBACK_KEY=');
|
|
329
|
-
// Fallback must NOT warm $CACHE_FILE (otherwise cache-push would skip)
|
|
330
|
-
|
|
329
|
+
// Fallback must NOT warm $CACHE_FILE (otherwise cache-push would skip).
|
|
330
|
+
// Both transports declare a scope, so anchor on the last one — the
|
|
331
|
+
// filesystem branch, which is the only one with a $CACHE_FILE to warm.
|
|
332
|
+
const sections = result.script.split('__ci_fb_scope=');
|
|
333
|
+
const fallbackSection = sections[sections.length - 1] ?? '';
|
|
331
334
|
expect(fallbackSection).not.toContain('mv "$CACHE_FILE.tmp" "$CACHE_FILE"');
|
|
332
335
|
});
|
|
336
|
+
test('asks the daemon for the scope fallback, dated, when there is no shared dir', async () => {
|
|
337
|
+
const executor = new CachePullStepExecutor();
|
|
338
|
+
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
339
|
+
// The daemon answers the same two questions the directory listing did:
|
|
340
|
+
// which key, and how old.
|
|
341
|
+
expect(result.script).toContain('"$CIBUILD_CACHE_DAEMON/cache/scope/$__ci_fb_scope/latest"');
|
|
342
|
+
expect(result.script).toContain('CACHE_SOURCE=fallback_daemon');
|
|
343
|
+
// The 30-day cap has to survive losing `stat`, so it reads the daemon's
|
|
344
|
+
// mtime instead. Without this the cap silently disappears.
|
|
345
|
+
expect(result.script).toContain('__ci_fb_mtime');
|
|
346
|
+
expect(result.script).toContain('-le 30');
|
|
347
|
+
expect(result.script).toContain('FALLBACK_REJECTED=');
|
|
348
|
+
});
|
|
333
349
|
test('should try peer dir for fallback when local has no prior tarball', async () => {
|
|
334
350
|
const executor = new CachePullStepExecutor();
|
|
335
351
|
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
@@ -386,22 +402,36 @@ describe('Step Implementations', () => {
|
|
|
386
402
|
// unauthenticated daemon isn't sent a bogus "Bearer " header.
|
|
387
403
|
expect(result.script).toContain('${CIBUILD_CACHE_TOKEN:+-H "Authorization: Bearer $CIBUILD_CACHE_TOKEN"}');
|
|
388
404
|
});
|
|
389
|
-
test('
|
|
405
|
+
test('picks its transport up front — daemon when configured, directory otherwise', async () => {
|
|
390
406
|
const executor = new CachePullStepExecutor();
|
|
391
407
|
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
392
|
-
|
|
408
|
+
// The daemon is no longer a branch *within* the directory search; it is
|
|
409
|
+
// the alternative to it, chosen before any lookup happens. A runner has
|
|
410
|
+
// no cache directory to search once the mount is gone, and standalone
|
|
411
|
+
// cibuild has no daemon — the same script has to serve both.
|
|
412
|
+
const splitAt = result.script.indexOf('if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
|
|
393
413
|
const daemonAt = result.script.indexOf('CACHE_SOURCE=daemon');
|
|
394
|
-
const
|
|
395
|
-
expect(
|
|
396
|
-
expect(daemonAt).toBeGreaterThan(
|
|
397
|
-
expect(
|
|
414
|
+
const localAt = result.script.indexOf('CACHE_SOURCE=local');
|
|
415
|
+
expect(splitAt).toBeGreaterThanOrEqual(0);
|
|
416
|
+
expect(daemonAt).toBeGreaterThan(splitAt);
|
|
417
|
+
expect(localAt).toBeGreaterThan(daemonAt);
|
|
418
|
+
});
|
|
419
|
+
test('does not require the cache directory to exist when using the daemon', async () => {
|
|
420
|
+
const executor = new CachePullStepExecutor();
|
|
421
|
+
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
422
|
+
// An unconditional mkdir aborts the step under errexit once the mount is
|
|
423
|
+
// gone, which is how this fails in production rather than in a test.
|
|
424
|
+
expect(result.script).toContain('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
425
|
+
expect(result.script).not.toContain('\nmkdir -p "$CACHE_DIR"');
|
|
398
426
|
});
|
|
399
427
|
test('routes a manual cache_key pull miss through the cache daemon', async () => {
|
|
400
428
|
const executor = new CachePullStepExecutor();
|
|
401
429
|
const result = await executor.execute({ cache_key: 'my-key' }, {}, testConfig);
|
|
402
430
|
expect(result.script).toContain('[ -n "$CIBUILD_CACHE_DAEMON" ]');
|
|
403
431
|
expect(result.script).toContain('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"');
|
|
404
|
-
|
|
432
|
+
// An explicit-key step reports the bare source, matching its sibling
|
|
433
|
+
// markers; the key is already fixed and echoed by the step header.
|
|
434
|
+
expect(result.script).toContain('CACHE_SOURCE=daemon');
|
|
405
435
|
});
|
|
406
436
|
});
|
|
407
437
|
describe('CachePushStepExecutor', () => {
|
|
@@ -418,15 +448,33 @@ describe('Step Implementations', () => {
|
|
|
418
448
|
expect(result.script).toContain('node-modules-v1');
|
|
419
449
|
expect(result.script).toContain('.ci-cache');
|
|
420
450
|
});
|
|
421
|
-
test('
|
|
451
|
+
test('uploads to the daemon when one is configured, on both step forms', async () => {
|
|
422
452
|
const executor = new CachePushStepExecutor();
|
|
423
453
|
const manual = await executor.execute({ cache_key: 'my-key', cache_paths: ['build'] }, {}, testConfig);
|
|
424
454
|
const preset = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
455
|
+
// This is the half that used to be missing: push wrote only to the
|
|
456
|
+
// shared directory, so removing the mount would have silently dropped
|
|
457
|
+
// every cache while still reporting success.
|
|
425
458
|
for (const result of [manual, preset]) {
|
|
426
|
-
expect(result.script).
|
|
427
|
-
expect(result.script).
|
|
459
|
+
expect(result.script).toContain('-T - "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"');
|
|
460
|
+
expect(result.script).toContain('${CIBUILD_CACHE_TOKEN:+-H "Authorization: Bearer $CIBUILD_CACHE_TOKEN"}');
|
|
461
|
+
// Streamed straight out — nothing is staged on the guest, so there is
|
|
462
|
+
// no temp file to collide with a concurrent slot or orphan on failure.
|
|
463
|
+
expect(result.script).toContain('| zstd -3 | curl');
|
|
428
464
|
}
|
|
429
465
|
});
|
|
466
|
+
test('a failed upload warns instead of failing the build', async () => {
|
|
467
|
+
const executor = new CachePushStepExecutor();
|
|
468
|
+
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
469
|
+
expect(result.script).toContain('Warning: failed to upload cache to the daemon');
|
|
470
|
+
});
|
|
471
|
+
test('still writes to the cache directory when no daemon is configured', async () => {
|
|
472
|
+
const executor = new CachePushStepExecutor();
|
|
473
|
+
const result = await executor.execute({ technology: 'kmm' }, {}, testConfig);
|
|
474
|
+
// Standalone cibuild has no daemon and must keep working unchanged.
|
|
475
|
+
expect(result.script).toContain('CACHE_TMP="$CACHE_FILE.$$.$RANDOM.tmp"');
|
|
476
|
+
expect(result.script).toContain('mv "$CACHE_TMP" "$CACHE_FILE"');
|
|
477
|
+
});
|
|
430
478
|
test('should skip gracefully when cache_key is missing', async () => {
|
|
431
479
|
const executor = new CachePushStepExecutor();
|
|
432
480
|
const inputs = {
|
|
@@ -72,7 +72,7 @@ describe('deriveStructuralFacts — unclippedBounds path (real clamping renders)
|
|
|
72
72
|
canvas: { left: 0, top: 0, right: 360, bottom: 720 },
|
|
73
73
|
nodes: [
|
|
74
74
|
{
|
|
75
|
-
id: '
|
|
75
|
+
id: 'HomeOfferBanner',
|
|
76
76
|
bounds: { left: 16, top: 100, right: 344, bottom: 262 },
|
|
77
77
|
unclippedBounds: { left: 16, top: 100, right: 344, bottom: 262 },
|
|
78
78
|
parentId: null,
|
|
@@ -85,7 +85,7 @@ describe('deriveStructuralFacts — unclippedBounds path (real clamping renders)
|
|
|
85
85
|
id: 'voucherTag',
|
|
86
86
|
bounds: { left: 250, top: 210, right: 330, bottom: 262 },
|
|
87
87
|
unclippedBounds: { left: 250, top: 210, right: 330, bottom: 280 },
|
|
88
|
-
parentId: '
|
|
88
|
+
parentId: 'HomeOfferBanner',
|
|
89
89
|
clipsChildren: false,
|
|
90
90
|
},
|
|
91
91
|
],
|
|
@@ -95,7 +95,7 @@ describe('deriveStructuralFacts — unclippedBounds path (real clamping renders)
|
|
|
95
95
|
expect(facts[0]).toMatchObject({
|
|
96
96
|
kind: 'clipped_by_parent',
|
|
97
97
|
nodeId: 'voucherTag',
|
|
98
|
-
clipperId: '
|
|
98
|
+
clipperId: 'HomeOfferBanner',
|
|
99
99
|
edges: ['bottom'],
|
|
100
100
|
clippedPx: { bottom: 18 },
|
|
101
101
|
});
|
|
@@ -307,7 +307,7 @@ describe('embedded render-side stage parity', () => {
|
|
|
307
307
|
}
|
|
308
308
|
});
|
|
309
309
|
describe('deriveStructuralFacts — motivating regression (voucher tag bottom-clip)', () => {
|
|
310
|
-
// The render-side layout dump for the
|
|
310
|
+
// The render-side layout dump for the HomeOfferBanner-style case: a card
|
|
311
311
|
// clips children, and the voucher tag (with its euro coin) overflows the
|
|
312
312
|
// card's bottom. The deriver MUST surface the bottom clip so the verdict
|
|
313
313
|
// can compare its extent to the design and never miss it by eye again.
|
|
@@ -315,10 +315,10 @@ describe('deriveStructuralFacts — motivating regression (voucher tag bottom-cl
|
|
|
315
315
|
const dump = {
|
|
316
316
|
canvas: { left: 0, top: 0, right: 360, bottom: 720 },
|
|
317
317
|
nodes: [
|
|
318
|
-
{ id: '
|
|
319
|
-
{ id: 'promoText', bounds: { left: 32, top: 116, right: 240, bottom: 180 }, parentId: '
|
|
318
|
+
{ id: 'HomeOfferBanner', bounds: { left: 16, top: 100, right: 344, bottom: 262 }, parentId: null, clipsChildren: true },
|
|
319
|
+
{ id: 'promoText', bounds: { left: 32, top: 116, right: 240, bottom: 180 }, parentId: 'HomeOfferBanner', clipsChildren: false },
|
|
320
320
|
// voucher tag overflows the card bottom (262) down to 280 → 18px clipped
|
|
321
|
-
{ id: 'voucherTag', bounds: { left: 250, top: 210, right: 330, bottom: 280 }, parentId: '
|
|
321
|
+
{ id: 'voucherTag', bounds: { left: 250, top: 210, right: 330, bottom: 280 }, parentId: 'HomeOfferBanner', clipsChildren: false },
|
|
322
322
|
],
|
|
323
323
|
};
|
|
324
324
|
const facts = deriveStructuralFacts(dump);
|
|
@@ -326,7 +326,7 @@ describe('deriveStructuralFacts — motivating regression (voucher tag bottom-cl
|
|
|
326
326
|
expect(clip).toMatchObject({
|
|
327
327
|
kind: 'clipped_by_parent',
|
|
328
328
|
nodeId: 'voucherTag',
|
|
329
|
-
clipperId: '
|
|
329
|
+
clipperId: 'HomeOfferBanner',
|
|
330
330
|
edges: ['bottom'],
|
|
331
331
|
clippedPx: { bottom: 18 },
|
|
332
332
|
});
|