@xemahq/repo-build-tooling 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +7 -5
- package/src/composed-image-coordinate.mjs +25 -11
- package/src/composed-image-coordinate.test.mjs +17 -2
- package/src/composed-image-dockerfile.mjs +28 -18
- package/src/composed-image-dockerfile.test.mjs +28 -1
- package/src/composed-image.mjs +9 -4
- package/src/composed-image.test.mjs +9 -0
- package/src/ghcr-manifest.mjs +4 -2
- package/src/registry-fetch.mjs +82 -0
- package/src/registry-fetch.test.mjs +151 -0
- package/src/service-bootstrap-concurrency.mjs +49 -0
- package/src/service-bootstrap-protocols.mjs +108 -0
- package/src/service-bootstrap-protocols.test.mjs +70 -0
- package/src/service-bootstrap.mjs +491 -0
- package/src/service-bootstrap.test.mjs +304 -0
- package/src/worker-descriptor-generation.mjs +77 -0
package/README.md
CHANGED
|
@@ -108,6 +108,25 @@ build carries every composable platform service of the repository at one commit.
|
|
|
108
108
|
Every step refuses rather than skips. The tool has no dependencies: the host and
|
|
109
109
|
the kernel schema are resolved from the members being built.
|
|
110
110
|
|
|
111
|
+
### `xema-service-bootstrap [--check] [--filter <key>] [--root <dir>]`
|
|
112
|
+
|
|
113
|
+
Generates the descriptors `@xemahq/xema-service-nest` boots from, out of the
|
|
114
|
+
repository's own `xema-biome.json` manifests. For every `kind: service` component
|
|
115
|
+
it writes `src/generated/<name>.bootstrap.generated.ts` (a `BiomeServiceDescriptor`),
|
|
116
|
+
and for every `kind: worker` component `src/generated/<key>.worker-descriptor.generated.ts`
|
|
117
|
+
(a `XemaWorkerDescriptor`; a `shared-oci-image` worker lands in the package of the
|
|
118
|
+
component whose image it shares). Only packages that declare
|
|
119
|
+
`@xemahq/xema-service-nest` are written to.
|
|
120
|
+
|
|
121
|
+
`requirements.scaling.concurrency.maximumPerInstance` becomes
|
|
122
|
+
`maximumConcurrentRequests` on a service and `maximumConcurrentActivities` on a
|
|
123
|
+
worker. A component without a positive integer there is refused, naming the
|
|
124
|
+
manifest and the component.
|
|
125
|
+
|
|
126
|
+
`--check` writes nothing and fails on any drift. A run that finds no manifest, or no
|
|
127
|
+
service or worker to generate, fails rather than reporting success. Run it from the
|
|
128
|
+
repository root, or pass `--root`.
|
|
129
|
+
|
|
111
130
|
## Scope boundary
|
|
112
131
|
|
|
113
132
|
Dev-time tooling that **more than one repository** runs. Two rules keep it from
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xemahq/repo-build-tooling",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Dev-time build tooling shared by every Xema repository. Ships as plain ESM with zero dependencies so the published artifact is the reviewed source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Neuralchowder Inc. <developer@xema.dev> (https://xema.dev)",
|
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
"xema-check-workspace-ranges": "src/check-workspace-range-matches-local.mjs",
|
|
26
26
|
"xema-composed-image": "src/composed-image.mjs",
|
|
27
27
|
"xema-readiness": "src/readiness.mjs",
|
|
28
|
-
"xema-scrub-swagger-paths": "src/scrub-swagger-plugin-paths.mjs"
|
|
28
|
+
"xema-scrub-swagger-paths": "src/scrub-swagger-plugin-paths.mjs",
|
|
29
|
+
"xema-service-bootstrap": "src/service-bootstrap.mjs"
|
|
29
30
|
},
|
|
30
31
|
"exports": {
|
|
31
32
|
"./check-no-vendored-deps": "./src/check-no-vendored-deps.mjs",
|
|
@@ -34,12 +35,13 @@
|
|
|
34
35
|
"./ghcr-manifest": "./src/ghcr-manifest.mjs",
|
|
35
36
|
"./package.json": "./package.json",
|
|
36
37
|
"./readiness": "./src/readiness.mjs",
|
|
37
|
-
"./scrub-swagger-plugin-paths": "./src/scrub-swagger-plugin-paths.mjs"
|
|
38
|
+
"./scrub-swagger-plugin-paths": "./src/scrub-swagger-plugin-paths.mjs",
|
|
39
|
+
"./service-bootstrap": "./src/service-bootstrap.mjs"
|
|
38
40
|
},
|
|
39
41
|
"scripts": {
|
|
40
|
-
"test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/ghcr-manifest.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs"
|
|
42
|
+
"test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/ghcr-manifest.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/registry-fetch.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs"
|
|
41
43
|
},
|
|
42
44
|
"xemaRelease": {
|
|
43
|
-
"sourceDigest": "sha256:
|
|
45
|
+
"sourceDigest": "sha256:525d85f0cc251fa763bc18261159ae689b59c3465d9da0d3c9f6037445d68713"
|
|
44
46
|
}
|
|
45
47
|
}
|
|
@@ -32,19 +32,33 @@ function required(env, name) {
|
|
|
32
32
|
return value;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* The consumer's own `package.json` of the hash package, found through Node's own
|
|
37
|
+
* `node_modules` lookup paths for the consumer, NOT through `require.resolve`: the
|
|
38
|
+
* package's `exports` map need not expose `./package.json`. Only an absent file
|
|
39
|
+
* means "not installed"; any other read error is rethrown as it is.
|
|
40
|
+
*/
|
|
41
|
+
function findHashManifest(root) {
|
|
42
|
+
for (const modulesDir of createRequire(join(root, 'package.json')).resolve.paths(HASH_PACKAGE) ?? []) {
|
|
43
|
+
const candidate = join(modulesDir, HASH_PACKAGE, 'package.json');
|
|
44
|
+
try {
|
|
45
|
+
readFileSync(candidate, 'utf8');
|
|
46
|
+
return candidate;
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
|
|
35
54
|
/** The consumer's own `xema-compute-image-tag` bin path; refuses when it is not installed there. */
|
|
36
55
|
export function resolveHashBin(root) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
`${HASH_PACKAGE} is not installed at ${root} — the content tag is computed only by its ${HASH_BIN} bin; ` +
|
|
44
|
-
'add it as a devDependency of this repository and install',
|
|
45
|
-
);
|
|
46
|
-
}
|
|
47
|
-
throw error;
|
|
56
|
+
const manifestPath = findHashManifest(root);
|
|
57
|
+
if (manifestPath === undefined) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
`${HASH_PACKAGE} is not installed at ${root} — the content tag is computed only by its ${HASH_BIN} bin; ` +
|
|
60
|
+
'add it as a devDependency of this repository and install',
|
|
61
|
+
);
|
|
48
62
|
}
|
|
49
63
|
const bin = JSON.parse(readFileSync(manifestPath, 'utf8')).bin?.[HASH_BIN];
|
|
50
64
|
if (typeof bin !== 'string') throw new Error(`${HASH_PACKAGE} at ${manifestPath} ships no "${HASH_BIN}" bin`);
|
|
@@ -26,7 +26,7 @@ function registry(manifestStatus) {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/** A consumer root whose own install carries a stand-in `xema-compute-image-tag` that echoes its arguments into a file. */
|
|
29
|
-
function consumer({ withHash = true } = {}) {
|
|
29
|
+
function consumer({ withHash = true, exportsMap } = {}) {
|
|
30
30
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'xema-coordinate-'));
|
|
31
31
|
fs.writeFileSync(path.join(root, 'package.json'), '{"name":"consumer"}');
|
|
32
32
|
if (withHash) {
|
|
@@ -34,7 +34,7 @@ function consumer({ withHash = true } = {}) {
|
|
|
34
34
|
fs.mkdirSync(path.join(pkg, 'dist/bin'), { recursive: true });
|
|
35
35
|
fs.writeFileSync(
|
|
36
36
|
path.join(pkg, 'package.json'),
|
|
37
|
-
JSON.stringify({ name: '@xemahq/distribution-source-hash', version: '1.0.0', bin: { 'xema-compute-image-tag': 'dist/bin/compute-image-tag.js' } }),
|
|
37
|
+
JSON.stringify({ name: '@xemahq/distribution-source-hash', version: '1.0.0', ...(exportsMap ? { exports: exportsMap } : {}), bin: { 'xema-compute-image-tag': 'dist/bin/compute-image-tag.js' } }),
|
|
38
38
|
);
|
|
39
39
|
fs.writeFileSync(
|
|
40
40
|
path.join(pkg, 'dist/bin/compute-image-tag.js'),
|
|
@@ -106,3 +106,18 @@ test('emitCoordinate prints every output and appends them to GITHUB_OUTPUT', ()
|
|
|
106
106
|
assert.equal(fs.readFileSync(file, 'utf8'), `image=i\ncontent_tag=${HASH}\nexists=false\nentry={}\n`);
|
|
107
107
|
assert.match(lines.at(-1), /is missing — build/);
|
|
108
108
|
});
|
|
109
|
+
|
|
110
|
+
test('a package whose exports map lacks ./package.json still resolves, as published 0.2.2 does', () => {
|
|
111
|
+
const root = consumer({ exportsMap: { '.': './dist/index.js' } });
|
|
112
|
+
assert.equal(
|
|
113
|
+
resolveHashBin(root),
|
|
114
|
+
path.join(root, 'node_modules/@xemahq/distribution-source-hash', 'dist/bin/compute-image-tag.js'),
|
|
115
|
+
);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test('a read error other than absence is rethrown with its real message, not mapped to not installed', () => {
|
|
119
|
+
const root = consumer({ withHash: false });
|
|
120
|
+
// package.json is a directory: EISDIR, which is not "absent"
|
|
121
|
+
fs.mkdirSync(path.join(root, 'node_modules/@xemahq/distribution-source-hash/package.json'), { recursive: true });
|
|
122
|
+
assert.throws(() => resolveHashBin(root), (error) => error.code === 'EISDIR' && !/not installed/.test(error.message));
|
|
123
|
+
});
|
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
// builder as the `node` user with NODE_ENV=production, after the production
|
|
14
14
|
// install, which is the user and environment the image runs with.
|
|
15
15
|
|
|
16
|
-
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
17
|
-
import { join, posix } from 'node:path';
|
|
16
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
17
|
+
import { dirname, join, posix } from 'node:path';
|
|
18
18
|
|
|
19
19
|
import { COMPOSED_IMAGE_DIR } from './composed-image-paths.mjs';
|
|
20
20
|
|
|
21
|
+
const TOKEN_REFERENCE = /_authToken\b|\$\{(?:NPM_TOKEN|GH_PACKAGES_TOKEN)\}/;
|
|
21
22
|
const VERSION = /^\d+\.\d+\.\d+$/;
|
|
22
23
|
const DIRECTORY = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*\/?$/;
|
|
23
24
|
const CACHE_ID = /^[a-z0-9][a-z0-9-]*$/;
|
|
@@ -39,8 +40,14 @@ function validate({ toolingVersion, copy, cacheId, buildSource }) {
|
|
|
39
40
|
if (!REPOSITORY.test(buildSource ?? '')) throw new Error(`--build-source must name a repository, got "${buildSource}"`);
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
/**
|
|
43
|
-
export function
|
|
43
|
+
/** Whether the repository's `.npmrc` authenticates with a token (`_authToken`, `${NPM_TOKEN}`); a public-registry `.npmrc` needs none. */
|
|
44
|
+
export function npmrcUsesToken(npmrcText) {
|
|
45
|
+
return TOKEN_REFERENCE.test(npmrcText);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The Dockerfile text. `usesToken` mounts the `npm_token` build secret; derive it with `npmrcUsesToken`. `imageDir` is where the repository keeps the image (and so where this is written). */
|
|
49
|
+
export function renderComposedDockerfile({ toolingVersion, copy, cacheId, buildSource, usesToken, imageDir = COMPOSED_IMAGE_DIR }) {
|
|
50
|
+
if (typeof usesToken !== 'boolean') throw new Error('usesToken must be a boolean derived from .npmrc');
|
|
44
51
|
validate({ toolingVersion, copy, cacheId, buildSource });
|
|
45
52
|
const dirFlag = imageDir === COMPOSED_IMAGE_DIR ? '' : ` --image-dir ${imageDir}`;
|
|
46
53
|
const tool = (command) => `xema-composed-image ${command}${dirFlag}`;
|
|
@@ -48,6 +55,11 @@ export function renderComposedDockerfile({ toolingVersion, copy, cacheId, buildS
|
|
|
48
55
|
.map((d) => `COPY ${d} ./${d}`)
|
|
49
56
|
.join('\n');
|
|
50
57
|
const store = `--mount=type=cache,id=pnpm-store-${cacheId},target=/root/.pnpm-store`;
|
|
58
|
+
const mount = usesToken ? ' --mount=type=secret,id=npm_token,required=true \\\\\n' : '';
|
|
59
|
+
const tokenEnv = usesToken ? 'NPM_TOKEN="$(cat /run/secrets/npm_token)" GH_PACKAGES_TOKEN="$NPM_TOKEN" \\\\\n ' : '';
|
|
60
|
+
const tokenExport = usesToken
|
|
61
|
+
? 'export NPM_TOKEN="$(cat /run/secrets/npm_token)" && export GH_PACKAGES_TOKEN="$NPM_TOKEN" && \\\\\n '
|
|
62
|
+
: '';
|
|
51
63
|
const metadata = `--mount=type=cache,id=pnpm-metadata-${cacheId},target=/root/.cache/pnpm`;
|
|
52
64
|
return `# @generated by \`xema-composed-image dockerfile --write\` (@xemahq/repo-build-tooling@${toolingVersion}) — DO NOT EDIT.
|
|
53
65
|
# Re-render with the same options; \`--check\` refuses a stale file.
|
|
@@ -90,13 +102,9 @@ RUN ${tool('filters')} > /tmp/member-filters && \\
|
|
|
90
102
|
test -s /tmp/member-filters
|
|
91
103
|
|
|
92
104
|
RUN ${store} \\
|
|
93
|
-
--
|
|
94
|
-
NPM_TOKEN="$(cat /run/secrets/npm_token)" GH_PACKAGES_TOKEN="$NPM_TOKEN" \\
|
|
95
|
-
pnpm fetch $(cat /tmp/member-filters) --frozen-lockfile
|
|
105
|
+
${mount} ${tokenEnv}pnpm fetch $(cat /tmp/member-filters) --frozen-lockfile
|
|
96
106
|
RUN ${store} \\
|
|
97
|
-
--
|
|
98
|
-
NPM_TOKEN="$(cat /run/secrets/npm_token)" GH_PACKAGES_TOKEN="$NPM_TOKEN" \\
|
|
99
|
-
pnpm install --prefer-offline $(cat /tmp/member-filters) --frozen-lockfile
|
|
107
|
+
${mount} ${tokenEnv}pnpm install --prefer-offline $(cat /tmp/member-filters) --frozen-lockfile
|
|
100
108
|
|
|
101
109
|
# Every member that owns a Prisma schema generates its OWN client into its own
|
|
102
110
|
# \`node_modules/.prisma/<service>\` (the schema's \`output\`), before it builds.
|
|
@@ -106,9 +114,7 @@ RUN for schema in $(${tool('dirs')}); do \\
|
|
|
106
114
|
fi; \\
|
|
107
115
|
done
|
|
108
116
|
|
|
109
|
-
RUN
|
|
110
|
-
NPM_TOKEN="$(cat /run/secrets/npm_token)" GH_PACKAGES_TOKEN="$NPM_TOKEN" \\
|
|
111
|
-
pnpm $(cat /tmp/member-filters) run build
|
|
117
|
+
RUN ${mount ? `${mount.trimStart()} ` : ''}${tokenEnv}pnpm $(cat /tmp/member-filters) run build
|
|
112
118
|
|
|
113
119
|
# Production install, with the members' WORKSPACE packages INJECTED — the same
|
|
114
120
|
# resolution the per-service image gets from \`pnpm deploy\` (legacy deploy
|
|
@@ -134,9 +140,7 @@ RUN --mount=type=secret,id=npm_token,required=true \\
|
|
|
134
140
|
# install built, and the members' \`postinstall\` is the Prisma loop below.
|
|
135
141
|
RUN ${store} \\
|
|
136
142
|
${metadata} \\
|
|
137
|
-
--
|
|
138
|
-
export NPM_TOKEN="$(cat /run/secrets/npm_token)" && export GH_PACKAGES_TOKEN="$NPM_TOKEN" && \\
|
|
139
|
-
INJECT="--config.inject-workspace-packages=true --config.dedupe-injected-deps=false" && \\
|
|
143
|
+
${mount} ${tokenExport}INJECT="--config.inject-workspace-packages=true --config.dedupe-injected-deps=false" && \\
|
|
140
144
|
cp pnpm-lock.yaml /tmp/pnpm-lock.committed.yaml && \\
|
|
141
145
|
pnpm install --lockfile-only --prefer-offline \${INJECT} $(cat /tmp/member-filters) && \\
|
|
142
146
|
${tool('pins')} /tmp/pnpm-lock.committed.yaml && \\
|
|
@@ -170,10 +174,16 @@ CMD ["node", "/monorepo/xema-composed-process"]
|
|
|
170
174
|
/** `--write` writes the rendered Dockerfile; `--check` throws when the committed one differs. */
|
|
171
175
|
export function syncComposedDockerfile({ mode, root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, ...options }) {
|
|
172
176
|
if (mode !== 'write' && mode !== 'check') throw new Error(`mode must be write or check, got "${mode}"`);
|
|
173
|
-
|
|
177
|
+
validate(options);
|
|
178
|
+
const npmrc = join(root, '.npmrc');
|
|
179
|
+
if (!existsSync(npmrc)) throw new Error(`${npmrc} is absent — the Dockerfile copies it and its token use decides the build secret`);
|
|
180
|
+
const rendered = renderComposedDockerfile({ ...options, usesToken: npmrcUsesToken(readFileSync(npmrc, 'utf8')), imageDir });
|
|
174
181
|
const relativeTarget = posix.join(imageDir, 'Dockerfile');
|
|
175
182
|
const target = join(root, relativeTarget);
|
|
176
|
-
if (mode === 'write')
|
|
183
|
+
if (mode === 'write') {
|
|
184
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
185
|
+
writeFileSync(target, rendered);
|
|
186
|
+
}
|
|
177
187
|
else if ((existsSync(target) ? readFileSync(target, 'utf8') : '') !== rendered) {
|
|
178
188
|
throw new Error(`${relativeTarget} is stale. Run: xema-composed-image dockerfile --write (with the same options)`);
|
|
179
189
|
}
|
|
@@ -4,13 +4,14 @@ import os from 'node:os';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import test from 'node:test';
|
|
6
6
|
|
|
7
|
-
import { renderComposedDockerfile, syncComposedDockerfile } from './composed-image-dockerfile.mjs';
|
|
7
|
+
import { npmrcUsesToken, renderComposedDockerfile, syncComposedDockerfile } from './composed-image-dockerfile.mjs';
|
|
8
8
|
|
|
9
9
|
const OPTIONS = {
|
|
10
10
|
toolingVersion: '0.9.0',
|
|
11
11
|
copy: ['packages', 'tooling/dev/', 'biomes'],
|
|
12
12
|
cacheId: 'alpha-composed-process',
|
|
13
13
|
buildSource: 'alpha',
|
|
14
|
+
usesToken: true,
|
|
14
15
|
};
|
|
15
16
|
|
|
16
17
|
test('the template carries every parameter and no per-repository literal', () => {
|
|
@@ -51,9 +52,35 @@ test('the tool version, the directories and the ids are validated', () => {
|
|
|
51
52
|
test('--write renders and --check refuses a stale or missing file', () => {
|
|
52
53
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'xema-dockerfile-'));
|
|
53
54
|
fs.mkdirSync(path.join(root, 'infra/composed-process'), { recursive: true });
|
|
55
|
+
fs.writeFileSync(path.join(root, '.npmrc'), '//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n');
|
|
54
56
|
assert.throws(() => syncComposedDockerfile({ mode: 'check', root, ...OPTIONS }), /is stale/);
|
|
55
57
|
assert.equal(syncComposedDockerfile({ mode: 'write', root, ...OPTIONS }), 'infra/composed-process/Dockerfile');
|
|
56
58
|
syncComposedDockerfile({ mode: 'check', root, ...OPTIONS });
|
|
57
59
|
assert.throws(() => syncComposedDockerfile({ mode: 'check', root, ...OPTIONS, toolingVersion: '0.9.1' }), /is stale/);
|
|
58
60
|
assert.throws(() => syncComposedDockerfile({ mode: 'sideways', root, ...OPTIONS }), /write or check/);
|
|
59
61
|
});
|
|
62
|
+
|
|
63
|
+
test('the npm_token secret is mounted only when .npmrc references a token', () => {
|
|
64
|
+
const withToken = renderComposedDockerfile(OPTIONS);
|
|
65
|
+
assert.equal(withToken.match(/type=secret,id=npm_token,required=true/g).length, 4);
|
|
66
|
+
assert.equal(withToken.match(/NPM_TOKEN="\$\(cat \/run\/secrets\/npm_token\)"/g).length, 4);
|
|
67
|
+
const without = renderComposedDockerfile({ ...OPTIONS, usesToken: false });
|
|
68
|
+
assert.doesNotMatch(without, /npm_token|NPM_TOKEN|GH_PACKAGES_TOKEN|type=secret/);
|
|
69
|
+
assert.match(without, /RUN --mount=type=cache,id=pnpm-store-alpha-composed-process,target=\/root\/.pnpm-store \\\n {2}pnpm fetch/);
|
|
70
|
+
assert.match(without, /RUN pnpm \$\(cat \/tmp\/member-filters\) run build/);
|
|
71
|
+
assert.throws(() => renderComposedDockerfile({ ...OPTIONS, usesToken: undefined }), /usesToken/);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('the token use is derived from .npmrc, deterministically', () => {
|
|
75
|
+
assert.equal(npmrcUsesToken('//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n'), true);
|
|
76
|
+
assert.equal(npmrcUsesToken('@x:registry=https://npm.pkg.github.com\n//npm.pkg.github.com/:_authToken=abc'), true);
|
|
77
|
+
assert.equal(npmrcUsesToken('registry=https://registry.npmjs.org/\nauto-install-peers=true\n'), false);
|
|
78
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'xema-dockerfile-npmrc-'));
|
|
79
|
+
assert.throws(() => syncComposedDockerfile({ mode: 'write', root, ...OPTIONS }), /\.npmrc is absent/);
|
|
80
|
+
fs.writeFileSync(path.join(root, '.npmrc'), 'registry=https://registry.npmjs.org/\n');
|
|
81
|
+
syncComposedDockerfile({ mode: 'write', root, ...OPTIONS });
|
|
82
|
+
const text = fs.readFileSync(path.join(root, 'infra/composed-process/Dockerfile'), 'utf8');
|
|
83
|
+
assert.doesNotMatch(text, /npm_token/);
|
|
84
|
+
fs.writeFileSync(path.join(root, '.npmrc'), '//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n');
|
|
85
|
+
assert.throws(() => syncComposedDockerfile({ mode: 'check', root, ...OPTIONS }), /is stale/);
|
|
86
|
+
});
|
package/src/composed-image.mjs
CHANGED
|
@@ -70,13 +70,14 @@
|
|
|
70
70
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
71
71
|
|
|
72
72
|
import { execFileSync } from 'node:child_process';
|
|
73
|
-
import { existsSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
73
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
74
74
|
import { createRequire } from 'node:module';
|
|
75
75
|
import { dirname, join, posix, relative, resolve } from 'node:path';
|
|
76
76
|
import { pathToFileURL } from 'node:url';
|
|
77
77
|
|
|
78
78
|
import { COMPOSED_IMAGE_DIR } from './composed-image-paths.mjs';
|
|
79
79
|
import { composedCoordinate, emitCoordinate } from './composed-image-coordinate.mjs';
|
|
80
|
+
import { describeErrorChain } from './registry-fetch.mjs';
|
|
80
81
|
import { syncComposedDockerfile } from './composed-image-dockerfile.mjs';
|
|
81
82
|
import { runImageLoad } from './load-image.mjs';
|
|
82
83
|
import { runRefusalSmoke } from './smoke-refusal.mjs';
|
|
@@ -263,8 +264,10 @@ export function syncImageBuildInputs({ mode, root = process.cwd(), imageDir = CO
|
|
|
263
264
|
const rendered = renderImageBuildInputs(derived);
|
|
264
265
|
const relativeTarget = posix.join(imageDir, IMAGE_BUILD_INPUTS_FILE);
|
|
265
266
|
const target = join(root, relativeTarget);
|
|
266
|
-
if (mode === 'write')
|
|
267
|
-
|
|
267
|
+
if (mode === 'write') {
|
|
268
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
269
|
+
writeFileSync(target, rendered);
|
|
270
|
+
} else if ((existsSync(target) ? readFileSync(target, 'utf8') : '') !== rendered) {
|
|
268
271
|
throw new Error(
|
|
269
272
|
`${relativeTarget} is stale — the composed image would not carry exactly the composable services. ` +
|
|
270
273
|
'Run: xema-composed-image members --write (with the same options)',
|
|
@@ -551,7 +554,9 @@ if (invokedPath !== undefined && import.meta.url === pathToFileURL(invokedPath).
|
|
|
551
554
|
// the verdict is in, so the step ends here rather than on their timers.
|
|
552
555
|
process.exit(0);
|
|
553
556
|
} catch (error) {
|
|
554
|
-
|
|
557
|
+
const [head, ...causes] = describeErrorChain(error);
|
|
558
|
+
console.error(`xema-composed-image: ${error?.message ?? head}`);
|
|
559
|
+
for (const cause of causes) console.error(` caused by ${cause}`);
|
|
555
560
|
process.exit(1);
|
|
556
561
|
}
|
|
557
562
|
}
|
|
@@ -247,3 +247,12 @@ test('the new commands parse their options; coordinate is asynchronous and refus
|
|
|
247
247
|
assert.throws(() => main(['dockerfile', '--write'], { log() {} }), /exact x\.y\.z/);
|
|
248
248
|
assert.throws(() => main(['smoke-refusal'], { log() {} }), /usage/);
|
|
249
249
|
});
|
|
250
|
+
|
|
251
|
+
test('members --write on a first run creates the missing image directory', () => {
|
|
252
|
+
const root = fixture();
|
|
253
|
+
assert.equal(fs.existsSync(path.join(root, 'infra/composed-process')), false);
|
|
254
|
+
main(['members', '--root', root, '--manifests', 'products/**/xema-biome.json', '--write'], { log() {} });
|
|
255
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(root, 'infra/composed-process/image-build-inputs.json'), 'utf8')), {
|
|
256
|
+
workspacePackages: ['products/pai/core/pai-core/api/pai-core-api'],
|
|
257
|
+
});
|
|
258
|
+
});
|
package/src/ghcr-manifest.mjs
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
// Only 200 and 404 are answers; anything else throws, so an unreachable or
|
|
5
5
|
// unauthorised registry can never read as "missing" and trigger a rebuild.
|
|
6
6
|
|
|
7
|
+
import { registryFetch } from './registry-fetch.mjs';
|
|
8
|
+
|
|
7
9
|
const MANIFEST_ACCEPT = [
|
|
8
10
|
'application/vnd.oci.image.index.v1+json',
|
|
9
11
|
'application/vnd.oci.image.manifest.v1+json',
|
|
@@ -38,7 +40,7 @@ export async function acquireGhcrPullToken({ registry, repository, user, token,
|
|
|
38
40
|
const url = new URL(`https://${registry}/token`);
|
|
39
41
|
url.searchParams.set('service', registry);
|
|
40
42
|
url.searchParams.set('scope', `repository:${repository}:pull`);
|
|
41
|
-
const response = await fetchImpl
|
|
43
|
+
const response = await registryFetch(fetchImpl, url, {
|
|
42
44
|
method: 'GET',
|
|
43
45
|
headers: { Authorization: basicAuthorization(user, token), Accept: 'application/json' },
|
|
44
46
|
});
|
|
@@ -60,7 +62,7 @@ export async function acquireGhcrPullToken({ registry, repository, user, token,
|
|
|
60
62
|
/** Returns `true` only for a proven existing manifest and `false` only for 404. */
|
|
61
63
|
export async function ghcrManifestExists({ registry, repository, contentHash, user, token, fetchImpl = fetch }) {
|
|
62
64
|
const bearer = await acquireGhcrPullToken({ registry, repository, user, token, fetchImpl });
|
|
63
|
-
const response = await fetchImpl
|
|
65
|
+
const response = await registryFetch(fetchImpl, `https://${registry}/v2/${repository}/manifests/${contentHash}`, {
|
|
64
66
|
method: 'HEAD',
|
|
65
67
|
headers: { Authorization: `Bearer ${bearer}`, Accept: MANIFEST_ACCEPT },
|
|
66
68
|
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// The one place a registry request is sent and a network failure is named.
|
|
2
|
+
//
|
|
3
|
+
// Node's fetch rejects with `TypeError: fetch failed` and keeps the real reason
|
|
4
|
+
// (ENOTFOUND, ECONNREFUSED, a TLS error, a timeout) in `error.cause`. Letting that
|
|
5
|
+
// escape as a bare message hides which request failed and why.
|
|
6
|
+
|
|
7
|
+
/** The URL with credentials and every query value removed; only the query keys remain. */
|
|
8
|
+
export function redactUrl(input) {
|
|
9
|
+
let url;
|
|
10
|
+
try {
|
|
11
|
+
url = new URL(String(input));
|
|
12
|
+
} catch {
|
|
13
|
+
return '<unparseable url>';
|
|
14
|
+
}
|
|
15
|
+
url.username = '';
|
|
16
|
+
url.password = '';
|
|
17
|
+
const keys = [...new Set(url.searchParams.keys())];
|
|
18
|
+
url.search = '';
|
|
19
|
+
url.hash = '';
|
|
20
|
+
return keys.length > 0 ? `${url.href}?${keys.map((key) => `${key}=<redacted>`).join('&')}` : url.href;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function describeOne(error) {
|
|
24
|
+
if (!(error instanceof Error)) return String(error);
|
|
25
|
+
const details = ['code', 'errno', 'syscall', 'hostname', 'address', 'port']
|
|
26
|
+
.filter((key) => error[key] !== undefined)
|
|
27
|
+
.map((key) => `${key}=${error[key]}`);
|
|
28
|
+
const head = `${error.name}: ${error.message}`;
|
|
29
|
+
return details.length > 0 ? `${head} (${details.join(', ')})` : head;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One line per link of the `cause` chain, outermost first. Bounded against cycles. */
|
|
33
|
+
export function describeErrorChain(error) {
|
|
34
|
+
const lines = [];
|
|
35
|
+
const seen = new Set();
|
|
36
|
+
for (let current = error; current !== undefined && !seen.has(current) && lines.length < 10; ) {
|
|
37
|
+
lines.push(describeOne(current));
|
|
38
|
+
if (!(current instanceof Error)) break;
|
|
39
|
+
seen.add(current);
|
|
40
|
+
current = current.cause;
|
|
41
|
+
}
|
|
42
|
+
return lines;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Total attempts per request, the first included. */
|
|
46
|
+
export const MAX_ATTEMPTS = 3;
|
|
47
|
+
/** Fixed wait in milliseconds after failed attempt n (1-based); deterministic, no jitter. */
|
|
48
|
+
export const RETRY_BACKOFF_MS = [1000, 3000];
|
|
49
|
+
|
|
50
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
51
|
+
|
|
52
|
+
function causeCode(error) {
|
|
53
|
+
for (let current = error, depth = 0; current instanceof Error && depth < 10; current = current.cause, depth += 1) {
|
|
54
|
+
if (current.code !== undefined) return String(current.code);
|
|
55
|
+
}
|
|
56
|
+
return 'unknown';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Send one registry request. Only a transport failure (the fetch promise rejects) is
|
|
61
|
+
* retried (an abort, `AbortError` or an aborted `init.signal`, is rethrown at once), up to MAX_ATTEMPTS with RETRY_BACKOFF_MS between attempts; any HTTP response,
|
|
62
|
+
* whatever its status, is returned as is. The final failure is thrown naming the method,
|
|
63
|
+
* the redacted URL and the attempt count, with the last error as `cause`. Headers are
|
|
64
|
+
* never reported. `options.sleep` replaces the real wait (for tests).
|
|
65
|
+
*/
|
|
66
|
+
export async function registryFetch(fetchImpl, url, init, { sleep = realSleep } = {}) {
|
|
67
|
+
const method = init?.method ?? 'GET';
|
|
68
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
69
|
+
try {
|
|
70
|
+
return await fetchImpl(url, init);
|
|
71
|
+
} catch (error) {
|
|
72
|
+
if (error?.name === 'AbortError' || init?.signal?.aborted === true) throw error;
|
|
73
|
+
if (attempt >= MAX_ATTEMPTS) {
|
|
74
|
+
throw new Error(`${method} ${redactUrl(url)} failed after ${attempt} attempts`, { cause: error });
|
|
75
|
+
}
|
|
76
|
+
process.stderr.write(
|
|
77
|
+
`registry request failed (attempt ${attempt} of ${MAX_ATTEMPTS}): ${method} ${redactUrl(url)} cause=${causeCode(error)}; retrying\n`,
|
|
78
|
+
);
|
|
79
|
+
await sleep(RETRY_BACKOFF_MS[attempt - 1]);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import net from 'node:net';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import test from 'node:test';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
9
|
+
|
|
10
|
+
import { MAX_ATTEMPTS, RETRY_BACKOFF_MS, describeErrorChain, redactUrl, registryFetch } from './registry-fetch.mjs';
|
|
11
|
+
|
|
12
|
+
const BIN = path.join(path.dirname(fileURLToPath(import.meta.url)), 'composed-image.mjs');
|
|
13
|
+
|
|
14
|
+
async function closedPort() {
|
|
15
|
+
const server = net.createServer();
|
|
16
|
+
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
|
|
17
|
+
const { port } = server.address();
|
|
18
|
+
await new Promise((resolve) => server.close(resolve));
|
|
19
|
+
return port;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
test('redactUrl drops credentials, query values and the fragment', () => {
|
|
23
|
+
const out = redactUrl('https://user:pw@ghcr.io/token?service=ghcr.io&scope=secret-scope#frag');
|
|
24
|
+
assert.equal(out, 'https://ghcr.io/token?service=<redacted>&scope=<redacted>');
|
|
25
|
+
assert.equal(redactUrl('not a url'), '<unparseable url>');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('a refused connection names the method, the URL and the cause code, never the headers', async () => {
|
|
29
|
+
const port = await closedPort();
|
|
30
|
+
const url = `http://127.0.0.1:${port}/v2/org/img/manifests/abc?access_token=SECRETQ`;
|
|
31
|
+
const error = await registryFetch(fetch, url, { method: 'HEAD', headers: { Authorization: 'Bearer SECRETH' } }, { sleep: async () => {} }).then(
|
|
32
|
+
() => assert.fail('expected the request to fail'),
|
|
33
|
+
(caught) => caught,
|
|
34
|
+
);
|
|
35
|
+
const text = [error.message, ...describeErrorChain(error)].join('\n');
|
|
36
|
+
assert.match(error.message, new RegExp(`^HEAD http://127\\.0\\.0\\.1:${port}/v2/org/img/manifests/abc\\?access_token=<redacted> failed after 3 attempts$`));
|
|
37
|
+
assert.match(text, /ECONNREFUSED/);
|
|
38
|
+
assert.doesNotMatch(text, /SECRETQ|SECRETH/);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test('describeErrorChain walks the cause chain and survives a cycle', () => {
|
|
42
|
+
const inner = Object.assign(new Error('getaddrinfo ENOTFOUND ghcr.io'), { code: 'ENOTFOUND', hostname: 'ghcr.io' });
|
|
43
|
+
const outer = new TypeError('fetch failed', { cause: inner });
|
|
44
|
+
inner.cause = outer;
|
|
45
|
+
const lines = describeErrorChain(outer);
|
|
46
|
+
assert.equal(lines.length, 2);
|
|
47
|
+
assert.match(lines[1], /code=ENOTFOUND, hostname=ghcr\.io/);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('the CLI prints the failing request and the cause chain to stderr and exits 1', () => {
|
|
51
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-cause-'));
|
|
52
|
+
try {
|
|
53
|
+
const hashDir = path.join(root, 'node_modules', '@xemahq', 'distribution-source-hash');
|
|
54
|
+
fs.mkdirSync(hashDir, { recursive: true });
|
|
55
|
+
fs.writeFileSync(
|
|
56
|
+
path.join(hashDir, 'package.json'),
|
|
57
|
+
JSON.stringify({ name: '@xemahq/distribution-source-hash', bin: { 'xema-compute-image-tag': 'bin.js' } }),
|
|
58
|
+
);
|
|
59
|
+
fs.writeFileSync(path.join(hashDir, 'bin.js'), `console.log('${'a'.repeat(64)}');\n`);
|
|
60
|
+
fs.writeFileSync(path.join(root, 'package.json'), '{}');
|
|
61
|
+
const preload = path.join(root, 'preload.mjs');
|
|
62
|
+
fs.writeFileSync(
|
|
63
|
+
preload,
|
|
64
|
+
`globalThis.fetch = async () => { throw new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo ENOTFOUND ghcr.io'), { code: 'ENOTFOUND', hostname: 'ghcr.io' }) }); };\n`,
|
|
65
|
+
);
|
|
66
|
+
const result = spawnSync(process.execPath, ['--import', preload, BIN, 'coordinate', '--image-name', 'img'], {
|
|
67
|
+
cwd: root,
|
|
68
|
+
encoding: 'utf8',
|
|
69
|
+
env: {
|
|
70
|
+
PATH: process.env.PATH,
|
|
71
|
+
SHA: 'b'.repeat(40),
|
|
72
|
+
REGISTRY: 'ghcr.io',
|
|
73
|
+
REGISTRY_ORG: 'org',
|
|
74
|
+
REGISTRY_USER: 'u',
|
|
75
|
+
REGISTRY_TOKEN: 'TOPSECRET',
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
assert.equal(result.status, 1);
|
|
79
|
+
assert.match(result.stderr, /GET https:\/\/ghcr\.io\/token\?service=<redacted>&scope=<redacted> failed after 3 attempts/);
|
|
80
|
+
assert.match(result.stderr, /caused by TypeError: fetch failed/);
|
|
81
|
+
assert.match(result.stderr, /caused by Error: getaddrinfo ENOTFOUND ghcr\.io \(code=ENOTFOUND, hostname=ghcr\.io\)/);
|
|
82
|
+
assert.doesNotMatch(result.stderr, /TOPSECRET/);
|
|
83
|
+
} finally {
|
|
84
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
const transportError = () =>
|
|
89
|
+
new TypeError('fetch failed', { cause: Object.assign(new Error('getaddrinfo EAI_AGAIN ghcr.io'), { code: 'EAI_AGAIN' }) });
|
|
90
|
+
|
|
91
|
+
test('a transient transport failure is retried once and then returns the response', async () => {
|
|
92
|
+
let calls = 0;
|
|
93
|
+
const waits = [];
|
|
94
|
+
const response = new Response('ok', { status: 200 });
|
|
95
|
+
const fetchImpl = async () => {
|
|
96
|
+
calls += 1;
|
|
97
|
+
if (calls === 1) throw transportError();
|
|
98
|
+
return response;
|
|
99
|
+
};
|
|
100
|
+
const result = await registryFetch(fetchImpl, 'https://ghcr.io/token?scope=x', { method: 'GET' }, { sleep: async (ms) => waits.push(ms) });
|
|
101
|
+
assert.equal(calls, 2);
|
|
102
|
+
assert.equal(result, response);
|
|
103
|
+
assert.deepEqual(waits, [RETRY_BACKOFF_MS[0]]);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('three transport failures throw naming 3 attempts and keep the cause chain', async () => {
|
|
107
|
+
let calls = 0;
|
|
108
|
+
const waits = [];
|
|
109
|
+
const fetchImpl = async () => {
|
|
110
|
+
calls += 1;
|
|
111
|
+
throw transportError();
|
|
112
|
+
};
|
|
113
|
+
const error = await registryFetch(fetchImpl, 'https://ghcr.io/v2/o/i/manifests/a', { method: 'HEAD' }, { sleep: async (ms) => waits.push(ms) }).then(
|
|
114
|
+
() => assert.fail('expected failure'),
|
|
115
|
+
(caught) => caught,
|
|
116
|
+
);
|
|
117
|
+
assert.equal(calls, MAX_ATTEMPTS);
|
|
118
|
+
assert.deepEqual(waits, [1000, 3000]);
|
|
119
|
+
assert.equal(error.message, 'HEAD https://ghcr.io/v2/o/i/manifests/a failed after 3 attempts');
|
|
120
|
+
assert.match(describeErrorChain(error).join('\n'), /fetch failed[\s\S]*code=EAI_AGAIN/);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test('an HTTP error status is returned after one call, never retried', async () => {
|
|
124
|
+
let calls = 0;
|
|
125
|
+
const fetchImpl = async () => {
|
|
126
|
+
calls += 1;
|
|
127
|
+
return new Response('down', { status: 503 });
|
|
128
|
+
};
|
|
129
|
+
const result = await registryFetch(fetchImpl, 'https://ghcr.io/token', { method: 'GET' }, { sleep: async () => assert.fail('must not sleep') });
|
|
130
|
+
assert.equal(calls, 1);
|
|
131
|
+
assert.equal(result.status, 503);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test('an abort is rethrown at once, never retried', async () => {
|
|
135
|
+
let calls = 0;
|
|
136
|
+
const controller = new AbortController();
|
|
137
|
+
controller.abort();
|
|
138
|
+
const fetchImpl = async () => {
|
|
139
|
+
calls += 1;
|
|
140
|
+
throw new Error('request failed');
|
|
141
|
+
};
|
|
142
|
+
const error = await registryFetch(fetchImpl, 'https://ghcr.io/token', { method: 'GET', signal: controller.signal }, { sleep: async () => assert.fail('must not sleep') }).then(
|
|
143
|
+
() => assert.fail('expected failure'),
|
|
144
|
+
(caught) => caught,
|
|
145
|
+
);
|
|
146
|
+
assert.equal(calls, 1);
|
|
147
|
+
assert.equal(error.message, 'request failed');
|
|
148
|
+
let named = 0;
|
|
149
|
+
await registryFetch(async () => { named += 1; throw new DOMException('aborted', 'AbortError'); }, 'https://ghcr.io/token', {}, { sleep: async () => assert.fail('must not sleep') }).catch(() => {});
|
|
150
|
+
assert.equal(named, 1);
|
|
151
|
+
});
|