@invarn/cibuild 2.6.3 → 2.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +10 -10
- package/dist/src/yaml/steps/cache-pull-daemon.test.js +11 -2
- package/dist/src/yaml/steps/cache-pull-restore-roots.test.js +10 -1
- package/dist/src/yaml/steps/cache-push-daemon.test.js +10 -1
- package/dist/src/yaml/steps/cache.d.ts.map +1 -1
- package/dist/src/yaml/steps/cache.js +31 -8
- package/package.json +1 -1
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import { describe, test, expect } from '@jest/globals';
|
|
27
27
|
import { execFile } from 'child_process';
|
|
28
|
+
import { createHash } from 'crypto';
|
|
28
29
|
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, chmodSync } from 'fs';
|
|
29
30
|
import { createServer } from 'http';
|
|
30
31
|
import { join } from 'path';
|
|
@@ -40,6 +41,14 @@ import { testConfigNoPeer } from './test-config.js';
|
|
|
40
41
|
*/
|
|
41
42
|
const run = promisify(execFile);
|
|
42
43
|
const CACHE_KEY = 'pods-abc123def456-release-1.4';
|
|
44
|
+
/**
|
|
45
|
+
* An explicit `cache_key` is prefixed with a project scope so two repositories
|
|
46
|
+
* writing the same key don't share a tarball. These fixtures run outside a git
|
|
47
|
+
* remote, so the step's shell hashes the literal `"no-remote"` — derived here
|
|
48
|
+
* rather than pasted, so it tracks the step.
|
|
49
|
+
*/
|
|
50
|
+
const PROJECT_SCOPE = createHash('sha256').update('no-remote', 'utf8').digest('hex').slice(0, 8);
|
|
51
|
+
const SCOPED_CACHE_KEY = `${PROJECT_SCOPE}-${CACHE_KEY}`;
|
|
43
52
|
async function listen(handler, paths) {
|
|
44
53
|
const server = createServer((req, res) => {
|
|
45
54
|
const path = req.url ?? '/';
|
|
@@ -145,8 +154,8 @@ describe('cache-pull with an explicit key, over the daemon', () => {
|
|
|
145
154
|
// 404 is the daemon's answer for `cache_miss`. The normal case on any
|
|
146
155
|
// first build: it must not start printing warnings.
|
|
147
156
|
const { stdout, paths } = await runPull({ keyStatus: 404 });
|
|
148
|
-
expect(paths).toEqual([`/cache/${
|
|
149
|
-
expect(stdout).toContain(`No cache found for key: ${
|
|
157
|
+
expect(paths).toEqual([`/cache/${SCOPED_CACHE_KEY}.tar.zst`]);
|
|
158
|
+
expect(stdout).toContain(`No cache found for key: ${SCOPED_CACHE_KEY}`);
|
|
150
159
|
expect(stdout).toContain('CACHE_SOURCE=cold');
|
|
151
160
|
expect(stdout).not.toContain('Warning');
|
|
152
161
|
});
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
*/
|
|
41
41
|
import { describe, test, expect } from '@jest/globals';
|
|
42
42
|
import { execFile } from 'child_process';
|
|
43
|
+
import { createHash } from 'crypto';
|
|
43
44
|
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, realpathSync, rmSync, chmodSync, } from 'fs';
|
|
44
45
|
import { join } from 'path';
|
|
45
46
|
import { tmpdir } from 'os';
|
|
@@ -48,6 +49,14 @@ import { CachePullStepExecutor } from './cache.js';
|
|
|
48
49
|
import { testConfigNoPeer } from './test-config.js';
|
|
49
50
|
const run = promisify(execFile);
|
|
50
51
|
const CACHE_KEY = 'gradle-abc123def456-fingerprint';
|
|
52
|
+
/**
|
|
53
|
+
* An explicit `cache_key` is prefixed with a project scope so two repositories
|
|
54
|
+
* that write the same key don't share one tarball. These fixtures run outside a
|
|
55
|
+
* git remote, so the step's shell falls back to hashing the literal
|
|
56
|
+
* `"no-remote"` — derived here rather than pasted, so it tracks the step.
|
|
57
|
+
*/
|
|
58
|
+
const PROJECT_SCOPE = createHash('sha256').update('no-remote', 'utf8').digest('hex').slice(0, 8);
|
|
59
|
+
const SCOPED_CACHE_KEY = `${PROJECT_SCOPE}-${CACHE_KEY}`;
|
|
51
60
|
async function restore(shape) {
|
|
52
61
|
// Resolved: see the header — tar will not extract through a symlink, and on
|
|
53
62
|
// macOS the temp root sits behind one.
|
|
@@ -83,7 +92,7 @@ async function restore(shape) {
|
|
|
83
92
|
// `tar -cf - "${PATHS_TO_CACHE[@]}"` produces.
|
|
84
93
|
const cacheDir = join(checkout, '.ci-cache');
|
|
85
94
|
mkdirSync(cacheDir, { recursive: true });
|
|
86
|
-
const tarball = join(cacheDir, `${
|
|
95
|
+
const tarball = join(cacheDir, `${SCOPED_CACHE_KEY}.tar.zst`);
|
|
87
96
|
await run('tar', ['-cf', tarball, ...operands], { cwd: checkout });
|
|
88
97
|
// Remove the originals: "it landed" is only meaningful if it was gone.
|
|
89
98
|
for (const l of landings)
|
|
@@ -25,12 +25,21 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import { describe, test, expect } from '@jest/globals';
|
|
27
27
|
import { execFileSync } from 'child_process';
|
|
28
|
+
import { createHash } from 'crypto';
|
|
28
29
|
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, chmodSync } from 'fs';
|
|
29
30
|
import { join } from 'path';
|
|
30
31
|
import { tmpdir } from 'os';
|
|
31
32
|
import { CachePushStepExecutor } from './cache.js';
|
|
32
33
|
import { testConfigNoPeer } from './test-config.js';
|
|
33
34
|
const CACHE_KEY = 'pods-abc123def456-main';
|
|
35
|
+
/**
|
|
36
|
+
* An explicit `cache_key` is prefixed with a project scope so two repositories
|
|
37
|
+
* writing the same key don't share a tarball. These fixtures run outside a git
|
|
38
|
+
* remote, so the step's shell hashes the literal `"no-remote"` — derived here
|
|
39
|
+
* rather than pasted, so it tracks the step.
|
|
40
|
+
*/
|
|
41
|
+
const PROJECT_SCOPE = createHash('sha256').update('no-remote', 'utf8').digest('hex').slice(0, 8);
|
|
42
|
+
const SCOPED_CACHE_KEY = `${PROJECT_SCOPE}-${CACHE_KEY}`;
|
|
34
43
|
/**
|
|
35
44
|
* Generates the explicit-key push step, runs it, and reports what `curl` was
|
|
36
45
|
* actually asked to do.
|
|
@@ -107,7 +116,7 @@ describe('cache-push with an explicit key, over the daemon', () => {
|
|
|
107
116
|
test('uploads under the key it was given', async () => {
|
|
108
117
|
const { urls } = await runPush();
|
|
109
118
|
expect(urls).toHaveLength(1);
|
|
110
|
-
expect(urls[0]).toBe(`http://daemon.invalid/cache/${
|
|
119
|
+
expect(urls[0]).toBe(`http://daemon.invalid/cache/${SCOPED_CACHE_KEY}.tar.zst`);
|
|
111
120
|
});
|
|
112
121
|
test('never uploads to an empty key', async () => {
|
|
113
122
|
// The defect, named. `<daemon>/cache/.tar.zst` is what an unassigned
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../../src/yaml/steps/cache.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBxD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,4HAA4H;IAC5H,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAUrD,CAAC;AA4bF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YAiJzF,iBAAiB;CAkNhC;AAED;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,gBAAgB;IACnD,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;YA2KzF,iBAAiB;CAgJhC"}
|
|
@@ -36,7 +36,12 @@ export const CACHE_PRESETS = {
|
|
|
36
36
|
function emitProjectScopeCommands() {
|
|
37
37
|
return [
|
|
38
38
|
'__ci_git_remote=$(git config --get remote.origin.url 2>/dev/null | sed -E "s#(://)[^@/]+@#\\1#" || echo "no-remote")',
|
|
39
|
-
|
|
39
|
+
// `shasum` is Perl-provided and is not guaranteed on a slim Linux image;
|
|
40
|
+
// `sha256sum` is coreutils and is absent from macOS. Both print
|
|
41
|
+
// `<hex> -`, so either yields the same 8-char project id — which is what
|
|
42
|
+
// keeps this byte-identical to `deriveProjectId` in the runner.
|
|
43
|
+
'__ci_sha256() { if command -v shasum >/dev/null 2>&1; then shasum -a 256; else sha256sum; fi; }',
|
|
44
|
+
'__ci_proj=$(printf "%s" "${__ci_git_remote:-no-remote}" | __ci_sha256 | cut -c1-8)',
|
|
40
45
|
];
|
|
41
46
|
}
|
|
42
47
|
/**
|
|
@@ -458,7 +463,6 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
458
463
|
const isDebugMode = this.getInput(inputs, 'is_debug_mode', false);
|
|
459
464
|
const commands = [];
|
|
460
465
|
commands.push('# Cache Pull');
|
|
461
|
-
commands.push(`echo "Cache key: ${this.escapeBash(cacheKey)}"`);
|
|
462
466
|
// Create cache directory if it doesn't exist
|
|
463
467
|
commands.push('');
|
|
464
468
|
commands.push('# Ensure cache directory exists');
|
|
@@ -468,11 +472,25 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
468
472
|
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
469
473
|
// whole step under errexit.
|
|
470
474
|
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
475
|
+
// Project scope: an explicit `cache_key` is author-written and carries no
|
|
476
|
+
// repo identity, so two projects that happen to write the same key (e.g.
|
|
477
|
+
// `gradle-{{checksum "gradle/wrapper/gradle-wrapper.properties"}}`, which
|
|
478
|
+
// depends only on the Gradle version) would share one tarball on a shared
|
|
479
|
+
// runner. Prefixing the project id keeps each repo's cache its own.
|
|
480
|
+
//
|
|
481
|
+
// The id goes at the FRONT, deliberately. The preset path spells its key
|
|
482
|
+
// `<keyPrefix>-<projectId>-<checksum>`, and that middle-segment shape is
|
|
483
|
+
// load-bearing elsewhere: it is how a scope is recovered from a tarball
|
|
484
|
+
// filename. An explicit key is author-written and must stay outside the
|
|
485
|
+
// scope model, so it must not be able to imitate that shape by accident.
|
|
486
|
+
commands.push('');
|
|
487
|
+
commands.push(...emitProjectScopeCommands());
|
|
471
488
|
// Generate cache file name based on cache key
|
|
472
489
|
commands.push('');
|
|
473
490
|
commands.push('# Generate cache file path');
|
|
474
|
-
commands.push(`CACHE_KEY="
|
|
491
|
+
commands.push(`CACHE_KEY="\${__ci_proj}-${this.escapeBash(cacheKey)}"`);
|
|
475
492
|
commands.push('CACHE_FILE="$CACHE_DIR/$CACHE_KEY.tar.zst"');
|
|
493
|
+
commands.push('echo "Cache key: $CACHE_KEY"');
|
|
476
494
|
if (isDebugMode) {
|
|
477
495
|
commands.push('echo "Debug mode enabled"');
|
|
478
496
|
commands.push('echo "Cache file: $CACHE_FILE"');
|
|
@@ -482,7 +500,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
482
500
|
if (peerCacheDir) {
|
|
483
501
|
commands.push('');
|
|
484
502
|
commands.push(`PEER_CACHE_DIR="${this.escapeBash(peerCacheDir)}"`);
|
|
485
|
-
commands.push(
|
|
503
|
+
commands.push('PEER_CACHE_FILE="$PEER_CACHE_DIR/$CACHE_KEY.tar.zst"');
|
|
486
504
|
}
|
|
487
505
|
// Where each class of member is restored to. Emitted before every
|
|
488
506
|
// transport, because all of them extract.
|
|
@@ -501,7 +519,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
501
519
|
commands.push(' echo "CACHE_SOURCE=daemon"');
|
|
502
520
|
commands.push(' else');
|
|
503
521
|
commands.push(...emitDaemonPullFailureDiagnosis(' ', '$CACHE_KEY'));
|
|
504
|
-
commands.push(
|
|
522
|
+
commands.push(' echo "No cache found for key: $CACHE_KEY"');
|
|
505
523
|
commands.push(' echo "CACHE_SOURCE=cold"');
|
|
506
524
|
commands.push(' fi');
|
|
507
525
|
commands.push(' rm -f "$__ci_cache_diag"');
|
|
@@ -522,7 +540,7 @@ export class CachePullStepExecutor extends BaseStepExecutor {
|
|
|
522
540
|
commands.push(' echo "CACHE_SOURCE=peer"');
|
|
523
541
|
}
|
|
524
542
|
commands.push('else');
|
|
525
|
-
commands.push(
|
|
543
|
+
commands.push(' echo "No cache found for key: $CACHE_KEY"');
|
|
526
544
|
commands.push(' echo "CACHE_SOURCE=cold"');
|
|
527
545
|
commands.push('fi');
|
|
528
546
|
// Closes the daemon/filesystem transport split.
|
|
@@ -766,7 +784,6 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
766
784
|
const ignoreCheckOnPaths = this.getInput(inputs, 'ignore_check_on_paths', false);
|
|
767
785
|
const commands = [];
|
|
768
786
|
commands.push('# Cache Push');
|
|
769
|
-
commands.push(`echo "Cache key: ${this.escapeBash(cacheKey)}"`);
|
|
770
787
|
// Create cache directory if it doesn't exist
|
|
771
788
|
commands.push('');
|
|
772
789
|
commands.push('# Ensure cache directory exists');
|
|
@@ -776,14 +793,20 @@ export class CachePushStepExecutor extends BaseStepExecutor {
|
|
|
776
793
|
// longer mounted into the VM — an unconditional mkdir would abort the
|
|
777
794
|
// whole step under errexit.
|
|
778
795
|
commands.push('[ -n "$CIBUILD_CACHE_DAEMON" ] || mkdir -p "$CACHE_DIR"');
|
|
796
|
+
// Project scope — must match the pull side exactly, or a push writes a
|
|
797
|
+
// tarball no pull can ever find. See the note there for why an explicit
|
|
798
|
+
// key needs the repo identity prefixed.
|
|
799
|
+
commands.push('');
|
|
800
|
+
commands.push(...emitProjectScopeCommands());
|
|
779
801
|
// Generate cache file name based on cache key
|
|
780
802
|
commands.push('');
|
|
781
803
|
commands.push('# Generate cache file path');
|
|
782
804
|
// CACHE_KEY first, CACHE_FILE derived from it — exactly as the pull side
|
|
783
805
|
// does. This step used to set only CACHE_FILE while the daemon URL below
|
|
784
806
|
// interpolated `$CACHE_KEY`; see emitDaemonPushCommands.
|
|
785
|
-
commands.push(`CACHE_KEY="
|
|
807
|
+
commands.push(`CACHE_KEY="\${__ci_proj}-${this.escapeBash(cacheKey)}"`);
|
|
786
808
|
commands.push('CACHE_FILE="$CACHE_DIR/$CACHE_KEY.tar.zst"');
|
|
809
|
+
commands.push('echo "Cache key: $CACHE_KEY"');
|
|
787
810
|
if (isDebugMode) {
|
|
788
811
|
commands.push('echo "Debug mode enabled"');
|
|
789
812
|
commands.push('echo "Cache file: $CACHE_FILE"');
|