@invarn/cibuild 2.5.9 → 2.6.1

package/dist/src/yaml/steps/cache-pull-restore-roots.test.js

@@ -0,0 +1,200 @@
+/**
+ * `cache-pull` restores every cached path to the root it was recorded against.
+ *
+ * `cache-push` archives two kinds of entry in one tarball. A `~`- or `/`-rooted
+ * path is expanded and tar stores it with the leading `/` stripped
+ * (`/home/builder/.gradle/caches` -> `home/builder/.gradle/caches`). A
+ * project-relative path (`Pods`, `.gradle`, `node_modules`) is stored as-is,
+ * relative to the step's working directory. Every restore then used a single
+ * `tar -xf - -C /`, which is right for the first kind and impossible for the
+ * second: the build user cannot create `/Pods`.
+ *
+ * Three shapes, three different failures, all measured before this was fixed:
+ *
+ * | `cache_paths`                            | before                          |
+ * |------------------------------------------|---------------------------------|
+ * | all `~`-rooted                           | correct                         |
+ * | mixed (every Gradle/Android pipeline)    | `~` half landed, relative half never did, tar exited non-zero, reported **cold** on a build that was warm |
+ * | all project-relative (Pods, node_modules)| **nothing** landed — those caches were written on every build and had never restored |
+ *
+ * So the mixed shape lied and the all-relative shape silently did nothing. Both
+ * are asserted here, and the all-`~` shape is asserted too because it already
+ * worked and the fix must not cost it anything.
+ *
+ * ### Why this runs the real `tar`
+ *
+ * The sibling daemon suite stubs `tar` to `cat > /dev/null`, because the
+ * generated script extracts with `-C /` and nothing there is worth writing to
+ * the root of the machine running the suite. This suite cannot: *where the
+ * bytes land* is the entire question, and a stub answers it by construction.
+ *
+ * It stays out of the real root by pointing `CIBUILD_USER_HOME` at a temporary
+ * directory. `~/.gradle/caches` then expands to an absolute path *inside* that
+ * directory, so `-C /` reassembles it exactly where it belongs and writes
+ * nothing outside the temp tree. The path is resolved through `realpath` first:
+ * on macOS `/var` is a symlink to `/private/var`, and tar refuses to extract
+ * through a symlink.
+ *
+ * `zstd` is stubbed to an identity copy, as in the sibling suite — it is not on
+ * every host, and compression is not what is under test.
+ */
+import { describe, test, expect } from '@jest/globals';
+import { execFile } from 'child_process';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, realpathSync, rmSync, chmodSync, } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { promisify } from 'util';
+import { CachePullStepExecutor } from './cache.js';
+import { testConfigNoPeer } from './test-config.js';
+const run = promisify(execFile);
+const CACHE_KEY = 'gradle-abc123def456-fingerprint';
+async function restore(shape) {
+    // Resolved: see the header — tar will not extract through a symlink, and on
+    // macOS the temp root sits behind one.
+    const root = realpathSync(mkdtempSync(join(tmpdir(), 'cibuild-restore-roots-')));
+    const home = join(root, 'home');
+    const checkout = join(root, 'checkout');
+    const bin = join(root, 'bin');
+    mkdirSync(home, { recursive: true });
+    mkdirSync(checkout, { recursive: true });
+    mkdirSync(bin, { recursive: true });
+    try {
+        // Identity stand-in for zstd: copies a named file, or stdin when the
+        // daemon path pipes into it.
+        writeFileSync(join(bin, 'zstd'), '#!/bin/bash\nfor a in "$@"; do case "$a" in -*) ;; *) exec cat "$a";; esac; done\nexec cat\n', 'utf-8');
+        chmodSync(join(bin, 'zstd'), 0o755);
+        // Create the content, remembering where each piece has to return to.
+        const landings = [];
+        const operands = [];
+        for (const f of shape.files) {
+            const [where, rel] = f.at.split(':', 2);
+            const base = where === 'home' ? home : checkout;
+            const abs = join(base, rel);
+            mkdirSync(join(abs, '..'), { recursive: true });
+            writeFileSync(abs, f.body, 'utf-8');
+            landings.push({ path: abs, body: f.body });
+        }
+        // The tar operands are what cache-push would have built: `~` paths
+        // expanded to absolute, project-relative paths left alone.
+        for (const p of shape.cachePaths) {
+            operands.push(p.startsWith('~') ? join(home, p.slice(2)) : p);
+        }
+        // One archive, mixed operands, created from the checkout — exactly what
+        // `tar -cf - "${PATHS_TO_CACHE[@]}"` produces.
+        const cacheDir = join(checkout, '.ci-cache');
+        mkdirSync(cacheDir, { recursive: true });
+        const tarball = join(cacheDir, `${CACHE_KEY}.tar.zst`);
+        await run('tar', ['-cf', tarball, ...operands], { cwd: checkout });
+        // Remove the originals: "it landed" is only meaningful if it was gone.
+        for (const l of landings)
+            rmSync(l.path, { force: true });
+        const { script } = await new CachePullStepExecutor().execute({ cache_key: CACHE_KEY, cache_paths: shape.cachePaths }, {}, testConfigNoPeer);
+        const scriptPath = join(root, '__cache_pull.sh');
+        writeFileSync(scriptPath, script, 'utf-8');
+        const env = {
+            ...process.env,
+            PATH: `${bin}:${process.env.PATH}`,
+            CIBUILD_USER_HOME: home,
+        };
+        // The filesystem transport is the one under test here; the daemon URL
+        // would switch the script to the HTTP path.
+        delete env.CIBUILD_CACHE_DAEMON;
+        let stdout = '';
+        let exitCode = 0;
+        try {
+            const r = await run('bash', [scriptPath], {
+                cwd: checkout,
+                encoding: 'utf-8',
+                env,
+                timeout: 20000,
+            });
+            stdout = String(r.stdout) + String(r.stderr);
+        }
+        catch (e) {
+            const err = e;
+            stdout = String(err.stdout ?? '') + String(err.stderr ?? '');
+            exitCode = err.code ?? 1;
+        }
+        // Read the landings back while the tree is still alive.
+        const observed = landings.map((l) => ({
+            path: l.path,
+            body: existsSync(l.path) ? readFileSync(l.path, 'utf-8') : '',
+        }));
+        return { stdout, exitCode, landings: observed };
+    }
+    finally {
+        rmSync(root, { recursive: true, force: true });
+    }
+}
+describe('cache-pull restores each class of path to its own root', () => {
+    test('all `~`-rooted: keeps working, and still streams', async () => {
+        // The shape that was already correct. It is here so the fix cannot buy the
+        // other two at its expense.
+        const { stdout, exitCode, landings } = await restore({
+            cachePaths: ['~/.gradle/caches'],
+            files: [{ at: 'home:.gradle/caches/modules/marker', body: 'gradle-cache\n' }],
+        });
+        expect(exitCode).toBe(0);
+        expect(stdout).toContain('CACHE_SOURCE=local');
+        expect(stdout).not.toContain('CACHE_SOURCE=cold');
+        expect(landings).toEqual([
+            { path: landings[0].path, body: 'gradle-cache\n' },
+        ]);
+    });
+    test('mixed: reports warm AND the project-relative half lands in the checkout', async () => {
+        // Every Gradle/Android pipeline, including the one the README ships. Before
+        // the fix: the `~` half restored (that was the real warmth), `.gradle`
+        // could not become `/.gradle`, tar exited non-zero, and the step reported
+        // CACHE_SOURCE=cold — on the filesystem transport it ended the step outright.
+        const { stdout, exitCode, landings } = await restore({
+            cachePaths: ['~/.gradle/caches', '.gradle'],
+            files: [
+                { at: 'home:.gradle/caches/modules/marker', body: 'gradle-cache\n' },
+                { at: 'checkout:.gradle/configuration-cache/marker', body: 'project-gradle\n' },
+            ],
+        });
+        expect(exitCode).toBe(0);
+        expect(stdout).toContain('CACHE_SOURCE=local');
+        expect(stdout).not.toContain('CACHE_SOURCE=cold');
+        // Both halves, at their own roots.
+        expect(landings.map((l) => l.body)).toEqual(['gradle-cache\n', 'project-gradle\n']);
+    });
+    test('all project-relative: restores into the checkout instead of nothing', async () => {
+        // The iOS/Pods and node shape. Before the fix every member was extracted to
+        // `/` and every one failed, so these caches were pushed on every build and
+        // had never restored anything — a silent miss that costs a `pod install`
+        // rather than failing a build, which is why it went unnoticed.
+        const { stdout, exitCode, landings } = await restore({
+            cachePaths: ['Pods', 'Podfile.lock'],
+            files: [
+                { at: 'checkout:Pods/Alamofire/marker', body: 'pods\n' },
+                { at: 'checkout:Podfile.lock', body: 'lock\n' },
+            ],
+        });
+        expect(exitCode).toBe(0);
+        expect(stdout).toContain('CACHE_SOURCE=local');
+        expect(stdout).not.toContain('CACHE_SOURCE=cold');
+        expect(landings.map((l) => l.body)).toEqual(['pods\n', 'lock\n']);
+    });
+    test('a project-relative path is never written to the filesystem root', async () => {
+        // The direct statement of the bug: `Pods` must not be attempted at `/Pods`.
+        // Asserted on the generated script rather than by running it, because the
+        // only honest runtime check would be whether `/Pods` appeared on the
+        // machine running the suite.
+        const { script } = await new CachePullStepExecutor().execute({ cache_key: CACHE_KEY, cache_paths: ['~/.gradle/caches', 'Pods'] }, {}, testConfigNoPeer);
+        // No restore pipes into `tar -C /` any more; they all go through the
+        // helper, which picks the root per class. The helper keeps a single
+        // streaming `tar -xf - -C /` for the all-absolute shape, so the assertion
+        // is about the call sites, not about the string appearing anywhere.
+        expect(script).not.toContain('| tar -xf - -C /');
+        expect(script).toContain('| __ci_cache_extract');
+        // `Pods` is declared project-relative, so it can only be extracted
+        // relative to the checkout ...
+        expect(script).toContain('__ci_cache_rel+=(\'Pods\')');
+        // ... and the tilde path is expanded at runtime, then matched against the
+        // archive's own top-level names.
+        expect(script).toContain('__ci_cache_abs+=("${__ci_p#/}")');
+        expect(script).toContain('tar -xf "$__t" -C "$__ci_cache_root"');
+    });
+});
+//# sourceMappingURL=cache-pull-restore-roots.test.js.map

package/dist/src/yaml/steps/cache-push-daemon.test.js

@@ -37,8 +37,11 @@ const CACHE_KEY = 'pods-abc123def456-main';
  *
  * @param curlExit  exit status the stub returns, and `curlBody` is what it
  *                  prints — so a rejection can be exercised without a daemon.
+ * @param zstdExit  exit status the `zstd` stub returns, which is how the
+ *                  *middle* stage is made to fail. 127 is what a shell that
+ *                  cannot find the binary produces, i.e. an image without it.
  */
-async function runPush({ curlExit = 0, curlBody = '' } = {}) {
+async function runPush({ curlExit = 0, curlBody = '', zstdExit = 0, } = {}) {
     const dir = mkdtempSync(join(tmpdir(), 'cibuild-push-'));
     try {
         // Something real to cache, so the step does not take its empty branch.
@@ -63,7 +66,11 @@ async function runPush({ curlExit = 0, curlBody = '' } = {}) {
         chmodSync(join(bin, 'curl'), 0o755);
         // zstd is not on every host, and without it the pipeline fails before
         // curl is ever reached — which would make these tests pass vacuously.
-        writeFileSync(join(bin, 'zstd'), '#!/bin/bash\ncat\n', 'utf-8');
+        // A failing stub drains stdin before exiting, so tar completes normally
+        // and the only non-zero status in the pipeline is the one under test.
+        writeFileSync(join(bin, 'zstd'), zstdExit === 0
+            ? '#!/bin/bash\ncat\n'
+            : `#!/bin/bash\ncat > /dev/null\nexit ${zstdExit}\n`, 'utf-8');
         chmodSync(join(bin, 'zstd'), 0o755);
         const { script } = await new CachePushStepExecutor().execute({ cache_key: CACHE_KEY, cache_paths: ['Pods'] }, {}, testConfigNoPeer);
         const scriptPath = join(dir, '__cache_push.sh');
@@ -128,5 +135,28 @@ describe('cache-push with an explicit key, over the daemon', () => {
         });
         expect(stdout).toContain('invalid_cache_key');
     });
+    test('a failed compressor is not reported as the daemon saying yes', async () => {
+        // What a worker image with no zstd binary actually logged:
+        //
+        //     Warning: failed to upload cache to the daemon — {"ok":true}
+        //
+        // `--fail-with-body` writes the daemon's response body to curl's *stdout*
+        // on success as well as on refusal, and the old code captured that as the
+        // failure reason. curl is perfectly happy to upload nothing, so the
+        // success body is precisely what a broken middle stage produces — the
+        // warning blamed the one stage that worked.
+        const { stdout } = await runPush({ zstdExit: 127, curlBody: '{"ok":true}' });
+        expect(stdout).toContain('Warning: failed to upload cache to the daemon');
+        expect(stdout).not.toContain('{"ok":true}');
+        expect(stdout).toContain('zstd is not installed');
+    });
+    test('a middle stage failing for any other reason still names the stage', async () => {
+        // 127 is the case that bit us, but the fix is not about 127: the reason is
+        // only the daemon's answer when the daemon is what refused.
+        const { stdout } = await runPush({ zstdExit: 1, curlBody: '{"ok":true}' });
+        expect(stdout).toContain('Warning: failed to upload cache to the daemon');
+        expect(stdout).not.toContain('{"ok":true}');
+        expect(stdout).toContain('zstd exited 1');
+    });
 });
 //# sourceMappingURL=cache-push-daemon.test.js.map

package/dist/src/yaml/steps/cache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;AAgLF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YA0HzF,iBAAiB;CA0MhC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YAkLzF,iBAAiB;CAgJhC"}
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;AAubF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YAmIzF,iBAAiB;CAkNhC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YAqKzF,iBAAiB;CAgJhC"}