@invarn/cibuild 2.6.0 → 2.6.2

package/dist/src/yaml/steps/cache.js

@@ -86,6 +86,126 @@ function resolvePresetChain(technology) {
     }
     return chain;
 }
+/**
+ * Resolves the `identifier:pattern` shorthand a `cache_paths` entry may use
+ * (`derived_data:MyApp-*`, `builds:nightly`) into a real path. Anything else is
+ * returned unchanged.
+ *
+ * Shared because the two sides used to disagree: `cache-push` resolved these
+ * and `cache-pull` did not, so the classification below would have put a
+ * `derived_data:` entry in the project-relative class on the way out and the
+ * absolute class on the way in.
+ */
+function resolveCachePathIdentifier(path, config) {
+    if (!path.includes(':'))
+        return path;
+    const [identifier, pattern] = path.split(':', 2);
+    switch (identifier) {
+        case 'derived_data':
+            return `${config.paths.derivedDataDir}/${pattern}`;
+        case 'builds':
+            return `${config.paths.buildsDir}/${pattern}`;
+        default:
+            return path;
+    }
+}
+/**
+ * Splits `cache_paths` into the two classes that decide where a member has to
+ * be extracted to.
+ *
+ * `cache-push` archives whatever `PATHS_TO_CACHE` holds. A `~`- or `/`-rooted
+ * entry is expanded to an absolute path and tar stores it with the leading `/`
+ * stripped (`/home/builder/.gradle/caches` -> `home/builder/.gradle/caches`),
+ * which round-trips through `-C /`. A project-relative entry (`Pods`,
+ * `.gradle`, `node_modules`) is stored as-is, relative to the step's working
+ * directory — and `-C /` then tries to create it at the root of the
+ * filesystem, which the build user cannot do.
+ *
+ * So the class is decided by the declared form, not by anything in the tarball,
+ * and both steps can compute it from the same input.
+ */
+function classifyCachePaths(paths, config) {
+    const absolute = [];
+    const relative = [];
+    for (const raw of paths) {
+        const path = resolveCachePathIdentifier(raw, config);
+        if (path.startsWith('~') || path.startsWith('/')) {
+            absolute.push(path);
+        }
+        else {
+            relative.push(path);
+        }
+    }
+    return { absolute, relative };
+}
+/**
+ * Emits `__ci_cache_extract`, which every restore path pipes its tar stream
+ * into, plus the two member lists it reads.
+ *
+ * **The defect it closes.** Every restore used one `tar -xf - -C /`. For a
+ * `cache_paths` that mixes the two classes — which every Gradle/Android
+ * pipeline does, including the one the README ships — the absolute half landed
+ * (that *was* the warmth) and the relative half could not, so tar exited
+ * non-zero, the compound condition went false, and a build that had just been
+ * restored reported `CACHE_SOURCE=cold`. Worse, a pipeline whose paths are
+ * *entirely* project-relative (`Pods`, `Podfile.lock`, `node_modules`) had
+ * every member fail: those caches were written on every build and had never
+ * restored anything. Measured on GNU tar 1.35 and bsdtar 3.5.3 alike.
+ *
+ * On the filesystem transport the same failure was not a mis-report at all —
+ * the extraction is a bare command there, so under `set -e -o pipefail` it
+ * ended the step.
+ *
+ * **Cost.** A restore with no project-relative paths keeps today's single
+ * streaming pass and stages nothing, so the shape that already worked pays
+ * nothing. The mixed and all-relative shapes stage the tar and read it twice,
+ * which is the price of extracting two roots from one stream.
+ *
+ * Written for bash 3.2 — macOS ships it, and the generated script says
+ * `#!/bin/bash`.
+ */
+function emitCacheExtractHelper(paths, config, indent = '') {
+    const { absolute, relative } = classifyCachePaths(paths, config);
+    const lit = (s) => `'${s.replace(/'/g, "'\\''")}'`;
+    const out = [
+        // Captured before anything can cd: project-relative members are recorded
+        // against the step's working directory, so that is where they go back.
+        `${indent}__ci_cache_root="$PWD"`,
+        `${indent}__ci_cache_abs=()`,
+        `${indent}__ci_cache_rel=()`,
+    ];
+    for (const p of absolute) {
+        // Expanded at runtime — `~` means ${CIBUILD_USER_HOME:-$HOME}, which the
+        // generator cannot know. The leading `/` is stripped to match how tar
+        // recorded the member.
+        out.push(`${indent}__ci_p=${lit(p)}`);
+        out.push(`${indent}__ci_p="\${__ci_p/#~/\${CIBUILD_USER_HOME:-$HOME}}"`);
+        out.push(`${indent}__ci_cache_abs+=("\${__ci_p#/}")`);
+    }
+    for (const p of relative) {
+        out.push(`${indent}__ci_cache_rel+=(${lit(p)})`);
+    }
+    out.push(`${indent}__ci_cache_extract() {`, 
+    // Nothing project-relative: the archive is entirely absolute-derived and
+    // one streaming pass is both correct and cheapest.
+    `${indent}  if [ \${#__ci_cache_rel[@]} -eq 0 ]; then`, `${indent}    tar -xf - -C /`, `${indent}    return`, `${indent}  fi`, `${indent}  local __t="\${TMPDIR:-/tmp}/cibuild-cache-extract.$$.tar"`, `${indent}  local __rc=0 __top __c __is_rel __p`, `${indent}  local __abs_ops __rel_ops`, `${indent}  cat > "$__t" || { rm -f "$__t"; return 1; }`, 
+    // The split is decided on the archive's own top-level names, not by
+    // `--exclude`. tar's exclude patterns are unanchored: `--exclude=.gradle`
+    // also matches `home/builder/.gradle`, so excluding the project-relative
+    // half from the pass that targets `/` silently dropped the absolute half
+    // too whenever the two shared a component name — which the canonical
+    // Gradle shape (`~/.gradle/caches` plus `.gradle`) always does.
+    `${indent}  __top=$(tar -tf "$__t" 2>/dev/null | sed -e 's|^\\./||' -e 's|/.*||' | grep -v '^$' | sort -u) || __rc=$?`, `${indent}  if [ "$__rc" -ne 0 ]; then rm -f "$__t"; return "$__rc"; fi`, `${indent}  __abs_ops=()`, `${indent}  __rel_ops=()`, 
+    // Read rather than word-split: a cached path may contain spaces
+    // (`derived_data:My App-*`). Process substitution keeps the loop in this
+    // shell, so the arrays it fills survive it.
+    `${indent}  while IFS= read -r __c; do`, `${indent}    __is_rel=0`, `${indent}    for __p in "\${__ci_cache_rel[@]}"; do`, `${indent}      case "$__p" in "$__c"|"$__c"/*) __is_rel=1 ;; esac`, `${indent}    done`, 
+    // A name claimed by both classes resolves to `/`, which is what it did
+    // before this existed. It needs a project directory named after a
+    // filesystem root while also caching an absolute path under it.
+    `${indent}    if [ "$__is_rel" -eq 1 ]; then __rel_ops+=("$__c"); else __abs_ops+=("$__c"); fi`, `${indent}  done < <(printf '%s\\n' "$__top")`, `${indent}  if [ \${#__abs_ops[@]} -gt 0 ]; then`, `${indent}    tar -xf "$__t" -C / "\${__abs_ops[@]}" || __rc=$?`, `${indent}  fi`, `${indent}  if [ "$__rc" -eq 0 ] && [ \${#__rel_ops[@]} -gt 0 ]; then`, `${indent}    tar -xf "$__t" -C "$__ci_cache_root" "\${__rel_ops[@]}" || __rc=$?`, `${indent}  fi`, `${indent}  rm -f "$__t"`, `${indent}  return "$__rc"`, `${indent}}`);
+    return out;
+}
 /** The token `%{http_code}` is reported under; see `daemonCurl`. */
 const DAEMON_STATUS_TOKEN = 'ci_http_status';
 /**
@@ -182,16 +302,53 @@ function daemonCurlKeepingErrorBody(args) {
  * reported success. Every curated Invarn template uses the explicit-key form,
  * so nothing was ever stored by any of them. One definition cannot diverge
  * from itself.
+ *
+ * The failure message names the stage that failed, which took a second fix.
+ * `$( … 2>&1 )` captured curl's stdout, and `--fail-with-body` writes the
+ * daemon's response body there on success as well as on refusal — so when the
+ * *middle* stage died (no `zstd` on the image), pipefail took the else branch
+ * and printed the daemon's **success** body as the reason the upload failed:
+ *
+ *     Warning: failed to upload cache to the daemon — {"ok":true}
+ *
+ * No stream separates those two cases, because the body is on stdout either
+ * way; only the stage's exit status does. Hence `PIPESTATUS` rather than the
+ * pull side's diag-file split — the file is still here, but it is the daemon's
+ * answer, and it is quoted only when curl is what refused. A rejection's
+ * reason still reaches the log, which `cache-wire-test.sh` asserts.
  */
 function emitDaemonPushCommands() {
     return [
-        `    if __ci_push_err=$(tar -cf - "\${PATHS_TO_CACHE[@]}" 2>/dev/null | zstd -3 | ${daemonCurlKeepingErrorBody('-T - "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} 2>&1); then`,
+        // curl's own stdout and stderr, kept out of the pipeline's status. Mirrors
+        // the pull side's `$__ci_cache_diag`.
+        '    __ci_push_out="${TMPDIR:-/tmp}/cibuild-cache-push.$$.out"',
+        `    if tar -cf - "\${PATHS_TO_CACHE[@]}" 2>/dev/null | zstd -3 | ${daemonCurlKeepingErrorBody('-T - "$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst"')} >"$__ci_push_out" 2>&1; then`,
         '      echo "Cache created successfully"',
         '    else',
+        // Read first, before anything else in this branch: PIPESTATUS holds the
+        // last pipeline's statuses and the next command replaces it.
+        '      __ci_push_rc=("${PIPESTATUS[@]}")',
         // Never fatal: a build that produced good output should not be failed by
         // a cache upload, and the next build simply misses. Saying *why* costs
         // nothing and is the difference between a warning and a diagnosis.
-        "      __ci_push_msg=$(printf '%s' \"$__ci_push_err\" | tr '\\n' ' ')",
+        '      __ci_push_msg=""',
+        '      if [ "${__ci_push_rc[2]}" != 0 ]; then',
+        // The daemon refused, so the daemon's own answer is the reason —
+        // `invalid_cache_key`, `cache_busy`, `empty_body`, an auth failure.
+        // `[ -s ]` guards the substitution: an assignment whose command
+        // substitution fails is a non-zero statement, and under errexit that ends
+        // the step.
+        '        if [ -s "$__ci_push_out" ]; then',
+        "          __ci_push_msg=$(tr '\\n' ' ' < \"$__ci_push_out\")",
+        '        fi',
+        '      elif [ "${__ci_push_rc[1]}" = 127 ]; then',
+        // curl was happy — it uploaded whatever reached it, which was nothing.
+        // 127 is the shell failing to find the command, and zstd itself never
+        // exits 127, so this is unambiguous and worth saying outright.
+        '        __ci_push_msg="zstd is not installed on this image, so nothing was uploaded"',
+        '      else',
+        '        __ci_push_msg="nothing was uploaded — tar exited ${__ci_push_rc[0]}, zstd exited ${__ci_push_rc[1]}"',
+        '      fi',
         '      __ci_push_msg=${__ci_push_msg:0:300}',
         '      if [ -n "$__ci_push_msg" ]; then',
         '        echo "Warning: failed to upload cache to the daemon — $__ci_push_msg"',
@@ -199,6 +356,7 @@ function emitDaemonPushCommands() {
         '        echo "Warning: failed to upload cache to the daemon"',
         '      fi',
         '    fi',
+        '    rm -f "$__ci_push_out"',
     ];
 }
 /**
@@ -221,7 +379,7 @@ function emitDaemonPullCommands() {
         // case and they both editorialise about it; pipefail still carries the
         // exit status. curl's own stderr is kept, in the file the diagnostics
         // above read: it is the only thing that knows *why*.
-        `if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`,
+        `if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | __ci_cache_extract; } 2>/dev/null; then`,
         '  echo "Cache found (daemon), extracting..."',
         '  echo "CACHE_SOURCE=daemon CACHE_KEY=$CACHE_KEY"',
         'else',
@@ -243,7 +401,7 @@ function emitDaemonPullCommands() {
         // than seed from ancient state. Exact-key hits are unaffected.
         '    if [ "$__ci_fb_age_days" -le 30 ]; then',
         '      echo "Cache fallback (daemon): using prior tarball $__ci_fb_key (${__ci_fb_age_days}d old)"',
-        `      if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$__ci_fb_key.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`,
+        `      if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$__ci_fb_key.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | __ci_cache_extract; } 2>/dev/null; then`,
         '        echo "CACHE_SOURCE=fallback_daemon CACHE_KEY=$CACHE_KEY FALLBACK_KEY=$__ci_fb_key"',
         '      else',
         '        echo "Cache fallback $__ci_fb_key could not be fetched — going cold"',
@@ -326,6 +484,11 @@ export class CachePullStepExecutor extends BaseStepExecutor {
             commands.push(`PEER_CACHE_DIR="${this.escapeBash(peerCacheDir)}"`);
             commands.push(`PEER_CACHE_FILE="$PEER_CACHE_DIR/${this.escapeBash(cacheKey)}.tar.zst"`);
         }
+        // Where each class of member is restored to. Emitted before every
+        // transport, because all of them extract.
+        commands.push('');
+        commands.push('# Restore roots');
+        commands.push(...emitCacheExtractHelper(cachePaths, config));
         // Check if cache file exists
         commands.push('');
         commands.push('# Check if cache exists');
@@ -333,7 +496,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
         // to fall back to, so the daemon side is the exact key or nothing.
         commands.push('if [ -n "$CIBUILD_CACHE_DAEMON" ]; then');
         commands.push(...emitDaemonPullDiagnostics('  '));
-        commands.push(`  if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | tar -xf - -C /; } 2>/dev/null; then`);
+        commands.push(`  if { ${daemonCurl('"$CIBUILD_CACHE_DAEMON/cache/$CACHE_KEY.tar.zst" 2>"$__ci_cache_diag"')} | zstd -dc | __ci_cache_extract; } 2>/dev/null; then`);
         commands.push('    echo "Cache found (daemon), extracting..."');
         commands.push('    echo "CACHE_SOURCE=daemon"');
         commands.push('  else');
@@ -348,13 +511,13 @@ export class CachePullStepExecutor extends BaseStepExecutor {
         if (isDebugMode) {
             commands.push('  echo "Cache file size: $(du -h "$CACHE_FILE" | cut -f1)"');
         }
-        commands.push('  zstd -dc "$CACHE_FILE" | tar -xf - -C /');
+        commands.push('  zstd -dc "$CACHE_FILE" | __ci_cache_extract');
         commands.push('  echo "CACHE_SOURCE=local"');
         // Peer fallback
         if (peerCacheDir) {
             commands.push('elif [ -f "$PEER_CACHE_FILE" ] 2>/dev/null; then');
             commands.push('  echo "Cache found (peer), extracting..."');
-            commands.push('  tee "$CACHE_FILE.tmp" < "$PEER_CACHE_FILE" | zstd -dc | tar -xf - -C /');
+            commands.push('  tee "$CACHE_FILE.tmp" < "$PEER_CACHE_FILE" | zstd -dc | __ci_cache_extract');
             commands.push('  mv "$CACHE_FILE.tmp" "$CACHE_FILE"');
             commands.push('  echo "CACHE_SOURCE=peer"');
         }
@@ -446,6 +609,13 @@ export class CachePullStepExecutor extends BaseStepExecutor {
         }
         commands.push('CACHE_FILE="$CACHE_DIR/$CACHE_KEY.tar.zst"');
         commands.push('');
+        // Where each class of member is restored to. Every path below extracts, so
+        // this is emitted once, above the transport split. `allPaths` is the same
+        // union cache-push archives from, which is what keeps the two sides
+        // agreeing about which class a path is in.
+        commands.push('# Restore roots');
+        commands.push(...emitCacheExtractHelper([...new Set(chain.flatMap(p => p.paths))], config));
+        commands.push('');
         // Transport split. On a runner the cache lives on the host and is reached
         // over HTTP; standalone, it is a directory on this machine. The daemon URL
         // is the switch, so the same generated script serves both.
@@ -465,7 +635,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
         if (isDebugMode) {
             commands.push('  echo "Cache file size: $(du -h "$CACHE_FILE" | cut -f1)"');
         }
-        commands.push('  zstd -dc "$CACHE_FILE" | tar -xf - -C /');
+        commands.push('  zstd -dc "$CACHE_FILE" | __ci_cache_extract');
         // LRU bump: lift this tarball to the top of `ls -t` so retention's age
         // cap counts last-used time, not last-written. Without this, a tarball
         // hit daily would still be reaped on its 31st day from creation.
@@ -479,7 +649,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
                 commands.push('  echo "Peer cache file size: $(du -h "$PEER_CACHE_FILE" | cut -f1)"');
             }
             // Single NFS read: tee forks the stream to local cache file AND decompression
-            commands.push('  tee "$CACHE_FILE.tmp" < "$PEER_CACHE_FILE" | zstd -dc | tar -xf - -C /');
+            commands.push('  tee "$CACHE_FILE.tmp" < "$PEER_CACHE_FILE" | zstd -dc | __ci_cache_extract');
             commands.push('  mv "$CACHE_FILE.tmp" "$CACHE_FILE"');
             commands.push('  echo "CACHE_SOURCE=peer CACHE_KEY=$CACHE_KEY"');
         }
@@ -518,7 +688,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
         if (isDebugMode) {
             commands.push('      echo "Fallback size: $(du -h "$__ci_fb" | cut -f1)"');
         }
-        commands.push('      zstd -dc "$__ci_fb" | tar -xf - -C /');
+        commands.push('      zstd -dc "$__ci_fb" | __ci_cache_extract');
         // LRU bump for the fallback tarball — we restored from it, so it earns
         // its place at the top of the retention sort even if the exact-key
         // tarball is the one that gets created in this run's cache-push.
@@ -623,23 +793,11 @@ export class CachePushStepExecutor extends BaseStepExecutor {
         commands.push('# Check which paths exist');
         commands.push('PATHS_TO_CACHE=()');
         for (const path of cachePaths) {
-            // Resolve path identifiers (e.g., "derived_data:pattern" -> "~/Library/.../DerivedData/pattern")
-            let resolvedPath = path;
-            // Check if path uses identifier format (identifier:pattern)
-            if (path.includes(':')) {
-                const [identifier, pattern] = path.split(':', 2);
-                switch (identifier) {
-                    case 'derived_data':
-                        resolvedPath = `${config.paths.derivedDataDir}/${pattern}`;
-                        break;
-                    case 'builds':
-                        resolvedPath = `${config.paths.buildsDir}/${pattern}`;
-                        break;
-                    default:
-                        // Unknown identifier, use as-is
-                        resolvedPath = path;
-                }
-            }
+            // Resolve path identifiers (e.g., "derived_data:pattern" -> "~/Library/.../DerivedData/pattern").
+            // Shared with cache-pull's member classification: if only one side
+            // resolved these, the two would disagree about which restore root the
+            // path belongs to.
+            const resolvedPath = resolveCachePathIdentifier(path, config);
             // Expand tilde by using parameter expansion
             commands.push(`EXPANDED_PATH="${this.escapeBash(resolvedPath)}"`);
             commands.push('EXPANDED_PATH="${EXPANDED_PATH/#~/${CIBUILD_USER_HOME:-$HOME}}"');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@invarn/cibuild",
-  "version": "2.6.0",
+  "version": "2.6.2",
   "description": "CI Build CLI — local pipeline orchestration and validation",
   "type": "module",
   "main": "dist/cli.cjs",