@xemahq/repo-build-tooling 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +4 -4
- package/src/composed-image-class.mjs +60 -0
- package/src/composed-image-class.test.mjs +218 -0
- package/src/composed-image-committed-class.mjs +19 -0
- package/src/composed-image-coordinate-cli.test.mjs +2 -2
- package/src/composed-image-fixture.test.mjs +73 -1
- package/src/composed-image-in-image-closure.test.mjs +2 -0
- package/src/composed-image-manifest-components.mjs +28 -0
- package/src/composed-image-members.mjs +6 -6
- package/src/composed-image-paths.mjs +4 -2
- package/src/composed-image-prebuilt-dockerfile.mjs +59 -15
- package/src/composed-image-prebuilt-dockerfile.test.mjs +8 -5
- package/src/composed-image-stage-tree.mjs +42 -13
- package/src/composed-image-stage-tree.test.mjs +64 -5
- package/src/composed-image.mjs +104 -32
- package/src/composed-image.test.mjs +15 -15
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
7
7
|
import { dirname, join, posix } from 'node:path';
|
|
8
8
|
|
|
9
|
-
import { COMPOSED_IMAGE_DIR, COMPOSED_MANIFEST_FILE, IN_IMAGE_ENTRY, NODE_MAJOR_LABEL, PROOF_DIRECTORY,
|
|
9
|
+
import { COMPOSED_IMAGE_DIR, COMPOSED_MANIFEST_FILE, IN_IMAGE_ENTRY, NODE_MAJOR_LABEL, PROOF_DIRECTORY, STAGE_DIST_LAYER, STAGE_PROOF_DIRECTORY, STAGE_STORE_LAYER, STAGE_WORKSPACE_LAYER, VERIFIED_IMAGE_FILE } from './composed-image-paths.mjs';
|
|
10
10
|
import { toolVersion } from './composed-image-stage.mjs';
|
|
11
11
|
import { IGNORED_DIRECTORIES, IGNORED_FILE_PATTERNS } from './composed-image-package-files.mjs';
|
|
12
|
+
import { BaseKind, CLASS_RECIPE, TOOLSET_FILES, assertImageClass } from './composed-image-class.mjs';
|
|
12
13
|
import { memberFilters } from './composed-image-members.mjs';
|
|
13
14
|
import { CONTEXT_DIRECTORY, rootBuildInputNames } from './staged-image-proof.mjs';
|
|
14
15
|
|
|
@@ -23,6 +24,32 @@ const COPIED_DIRECTORY = `${PROOF_DIRECTORY}/copied`;
|
|
|
23
24
|
|
|
24
25
|
const NODE_MAJOR = /^[1-9][0-9]*$/;
|
|
25
26
|
|
|
27
|
+
/** The image label that carries the class the image was rendered for. */
|
|
28
|
+
export const IMAGE_CLASS_LABEL = 'dev.xema.composed-image.class';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The class's recipe against what the caller passed: the base the class is built `FROM` (node-runtime, or the
|
|
32
|
+
* Chromium base for web-access) and the toolsets it carries, each pinned by digest. A flag the class does not use is
|
|
33
|
+
* refused, never ignored.
|
|
34
|
+
*/
|
|
35
|
+
function resolveRecipe({ imageClass, base, chromiumBase, toolsets }) {
|
|
36
|
+
const recipe = CLASS_RECIPE[assertImageClass(imageClass)];
|
|
37
|
+
const given = Object.keys(toolsets ?? {}).sort();
|
|
38
|
+
const wanted = [...recipe.toolsets].sort();
|
|
39
|
+
if (given.join() !== wanted.join()) {
|
|
40
|
+
throw new Error(`the ${imageClass} image carries the toolset(s) ${wanted.join(', ') || 'none'}; --toolset gave ${given.join(', ') || 'none'}`);
|
|
41
|
+
}
|
|
42
|
+
for (const [name, reference] of Object.entries(toolsets ?? {})) {
|
|
43
|
+
if (!DIGEST_REFERENCE.test(reference)) throw new Error(`--toolset ${name} must be an image reference pinned by digest, got "${reference}"`);
|
|
44
|
+
}
|
|
45
|
+
if (recipe.base === BaseKind.Chromium) {
|
|
46
|
+
if (!DIGEST_REFERENCE.test(chromiumBase ?? '')) throw new Error(`the ${imageClass} image is built on the Chromium base: --chromium-base must be pinned by digest, got "${chromiumBase}"`);
|
|
47
|
+
} else if (chromiumBase !== undefined) {
|
|
48
|
+
throw new Error(`--chromium-base belongs to the web-access class, not ${imageClass}`);
|
|
49
|
+
}
|
|
50
|
+
return { recipe, from: recipe.base === BaseKind.Chromium ? chromiumBase : base };
|
|
51
|
+
}
|
|
52
|
+
|
|
26
53
|
function validate({ copy, buildSource, stageDir, base, nodeMajor }) {
|
|
27
54
|
if (!NODE_MAJOR.test(nodeMajor ?? '')) throw new Error(`--node-major must be the Node major of the --base image (e.g. 22), got "${nodeMajor}"`);
|
|
28
55
|
if (!DIGEST_REFERENCE.test(base ?? '')) throw new Error(`--base must be an image reference pinned by digest (<ref>@sha256:<64 hex>), got "${base}"`);
|
|
@@ -56,8 +83,10 @@ export function parseRenderedDockerfile(dockerfileText) {
|
|
|
56
83
|
}
|
|
57
84
|
|
|
58
85
|
/** The prebuilt Dockerfile text. The members' literal filters and the root build inputs are derived from `root`. */
|
|
59
|
-
export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, copy = [], buildSource, stageDir = DEFAULT_STAGE_DIR, base, nodeMajor }) {
|
|
86
|
+
export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, imageClass, copy = [], buildSource, stageDir = DEFAULT_STAGE_DIR, base, chromiumBase, toolsets = {}, nodeMajor }) {
|
|
87
|
+
assertImageClass(imageClass);
|
|
60
88
|
validate({ copy, buildSource, stageDir, base, nodeMajor });
|
|
89
|
+
const { recipe, from } = resolveRecipe({ imageClass, base, chromiumBase, toolsets });
|
|
61
90
|
const toolingVersion = toolVersion();
|
|
62
91
|
const filters = memberFilters({ root, imageDir }).join(' ');
|
|
63
92
|
const rootInputs = rootBuildInputNames(root);
|
|
@@ -66,20 +95,27 @@ export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMP
|
|
|
66
95
|
const toolMount = `--mount=type=bind,source=${stageDir}/tool,target=${toolDir}`;
|
|
67
96
|
const node = `${toolDir}/${IN_IMAGE_ENTRY}`;
|
|
68
97
|
const copies = normalizedCopy(copy).map(copyLine).join('\n');
|
|
69
|
-
const
|
|
98
|
+
const stageLayer = (layer) => `COPY ${stageDir}/${layer}/ /`;
|
|
99
|
+
const allLayers = [STAGE_STORE_LAYER, STAGE_WORKSPACE_LAYER, STAGE_DIST_LAYER].map(stageLayer).join('\n');
|
|
100
|
+
const toolsetStages = recipe.toolsets.map((name) => `FROM ${toolsets[name]} AS ${name}\n`).join('');
|
|
101
|
+
const toolsetCopies = recipe.toolsets.map((name) => `COPY --from=${name} ${TOOLSET_FILES[name].from} ${TOOLSET_FILES[name].to}\n`).join('');
|
|
70
102
|
const dockerfile = `# @generated by \`xema-composed-image dockerfile --prebuilt --write\` (@xemahq/repo-build-tooling@${toolingVersion}) — DO NOT EDIT.
|
|
71
103
|
# Re-render with the same options; \`--check\` refuses a stale file (and a stale Dockerfile.dockerignore).
|
|
72
104
|
#
|
|
73
|
-
# The COMPOSED-PROCESS image of this repository, COPIED from a tree the CI runner staged:
|
|
74
|
-
# image per build (repository at commit), every member of \`image-build-inputs.json\` in ONE module
|
|
75
|
-
# graph behind the SDK's \`xema-composed-process\` host. Docker installs nothing, compiles nothing
|
|
105
|
+
# The ${imageClass.toUpperCase()} COMPOSED-PROCESS image of this repository, COPIED from a tree the CI runner staged: one
|
|
106
|
+
# image per class per build (repository at commit), every member of \`image-build-inputs.json\` in ONE module
|
|
107
|
+
# graph (ONE node process, never a child per member) behind the SDK's \`xema-composed-process\` host. Docker installs nothing, compiles nothing
|
|
76
108
|
# and has no network. The runner built the members (\`turbo run build prisma:generate\`) and
|
|
77
109
|
# \`xema-composed-image stage ${stageDir} --platform linux/<arch>\` produced, from the job's own
|
|
78
110
|
# pnpm store, the production tree of the members' closures for that platform (the install, the
|
|
79
111
|
# injected workspace packages, the member index, the Prisma clients and the platform's schema
|
|
80
|
-
# engine) and wrote it to \`${stageDir}/${STAGE_STORE_LAYER}/\` (the \`.pnpm\` packages,
|
|
81
|
-
# \`${stageDir}/${
|
|
82
|
-
#
|
|
112
|
+
# engine) and wrote it as three layers, least to most often changed: \`${stageDir}/${STAGE_STORE_LAYER}/\` (the \`.pnpm\` packages,
|
|
113
|
+
# stable until the lockfile changes), \`${stageDir}/${STAGE_WORKSPACE_LAYER}/\` (the injected workspace packages, Prisma clients and
|
|
114
|
+
# package files) and \`${stageDir}/${STAGE_DIST_LAYER}/\` (the members' dist and the member index, last).
|
|
115
|
+
# BUILD WITH \`SOURCE_DATE_EPOCH=0\` (build-arg and environment) and \`--output type=image,rewrite-timestamp=true\`: the staged files keep
|
|
116
|
+
# their real mtimes (BuildKit skips re-sending a file whose size and mtime are unchanged, so epoch mtimes would ship stale
|
|
117
|
+
# bytes) and the exporter, not the context, makes a layer's digest depend on its content alone.
|
|
118
|
+
# \`${stageDir}/tool/\` is this tool, bind-mounted into the steps below and never copied into a layer.
|
|
83
119
|
#
|
|
84
120
|
# Stages:
|
|
85
121
|
# proof The staged tree and the repository's inputs meet. The root build inputs that decide the
|
|
@@ -88,18 +124,19 @@ export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMP
|
|
|
88
124
|
# stage's manifest names BUILD_SHA and those bytes, this tool and this platform, and
|
|
89
125
|
# the literal \`--filter\` words below are the ones it staged. Then the Prisma clients run and every
|
|
90
126
|
# member loads ALONE as \`node\`, production, on the target arch.
|
|
91
|
-
#
|
|
127
|
+
# tail The dist layer plus the proof's verdict as ONE layer, so a commit that changes no dependency pushes exactly one layer.
|
|
128
|
+
# (final) base@digest${recipe.toolsets.length > 0 ? `, the ${recipe.toolsets.join(' and ')} toolset` : ''}, the store layer, the workspace layer, the tail.
|
|
92
129
|
#
|
|
93
130
|
# The \`--filter <package>...\` words below are LITERAL: the content hash of this image reads
|
|
94
131
|
# them to narrow the lockfile to the members' closures.
|
|
95
|
-
FROM ${
|
|
132
|
+
${toolsetStages}FROM ${from} AS base
|
|
96
133
|
|
|
97
134
|
FROM base AS proof
|
|
98
135
|
WORKDIR /${PROOF_DIRECTORY}
|
|
99
136
|
ARG BUILD_SHA
|
|
100
137
|
COPY ${stageDir}/${STAGE_PROOF_DIRECTORY}/${COMPOSED_MANIFEST_FILE} /${PROOF_DIRECTORY}/
|
|
101
138
|
COPY ${rootInputs.join(' ')} /${PROOF_DIRECTORY}/${CONTEXT_DIRECTORY}/
|
|
102
|
-
${copies === '' ? '' : `${copies}\n`}${
|
|
139
|
+
${copies === '' ? '' : `${copies}\n`}${allLayers}
|
|
103
140
|
RUN --network=none ${toolMount} \\
|
|
104
141
|
: "\${BUILD_SHA:?BUILD_SHA build-arg is required}" && \\
|
|
105
142
|
T="node ${node}" && \\
|
|
@@ -112,11 +149,18 @@ USER node
|
|
|
112
149
|
RUN --network=none ${toolMount} \\
|
|
113
150
|
NODE_ENV=production node ${node} load --each --root /monorepo${dirFlag}
|
|
114
151
|
|
|
152
|
+
FROM scratch AS tail
|
|
153
|
+
${stageLayer(STAGE_DIST_LAYER)}
|
|
154
|
+
COPY --from=proof /${PROOF_DIRECTORY}/${VERIFIED_IMAGE_FILE} /monorepo/${VERIFIED_IMAGE_FILE}
|
|
155
|
+
|
|
115
156
|
FROM base
|
|
116
|
-
WORKDIR /monorepo
|
|
117
157
|
LABEL ${NODE_MAJOR_LABEL}="${nodeMajor}"
|
|
118
|
-
${
|
|
119
|
-
|
|
158
|
+
${toolsetCopies}${stageLayer(STAGE_STORE_LAYER)}
|
|
159
|
+
# After the store layer, which creates /monorepo: WORKDIR then adds no layer of its own.
|
|
160
|
+
WORKDIR /monorepo
|
|
161
|
+
${stageLayer(STAGE_WORKSPACE_LAYER)}
|
|
162
|
+
COPY --from=tail / /
|
|
163
|
+
LABEL ${IMAGE_CLASS_LABEL}="${imageClass}"
|
|
120
164
|
LABEL ${BUILD_SOURCE_LABEL}="${buildSource}"
|
|
121
165
|
ENV NODE_ENV=production
|
|
122
166
|
# Set by the IMAGE, never by a deployment (kernel \`COMPOSED_PROCESS_ENV.MemberIndex\`).
|
|
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
|
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import test from 'node:test';
|
|
5
|
+
import { ComposedImageClass } from './composed-image-class.mjs';
|
|
5
6
|
|
|
6
7
|
import { parseRenderedDockerfile, renderPrebuiltDockerfile, renderPrebuiltDockerignore, syncPrebuiltDockerfile } from './composed-image-prebuilt-dockerfile.mjs';
|
|
7
8
|
import { parseArguments } from './composed-image.mjs';
|
|
@@ -11,7 +12,7 @@ function options(root, extra = {}) {
|
|
|
11
12
|
write(root, 'pnpm-lock.yaml', "lockfileVersion: '9.0'\n");
|
|
12
13
|
write(root, 'package.json', { name: 'r' });
|
|
13
14
|
write(root, 'tooling/dev/x.txt', 'x');
|
|
14
|
-
return { root, copy: ['tooling/dev'], buildSource: 'alpha', base: BASE, nodeMajor: '22', ...extra };
|
|
15
|
+
return { root, imageClass: ComposedImageClass.Common, copy: ['tooling/dev'], buildSource: 'alpha', base: BASE, nodeMajor: '22', ...extra };
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
test('the Dockerfile installs nothing: no pnpm, no corepack, no network step, no secret', () => {
|
|
@@ -21,13 +22,13 @@ test('the Dockerfile installs nothing: no pnpm, no corepack, no network step, no
|
|
|
21
22
|
for (const run of dockerfile.split(/^RUN /m).slice(1)) assert.match(run, /^--network=none /);
|
|
22
23
|
});
|
|
23
24
|
|
|
24
|
-
test('the proof stage COPYs the root build inputs flat, the --copy directories, the manifest and the
|
|
25
|
+
test('the proof stage COPYs the root build inputs flat, the --copy directories, the manifest and the three layers; the filters are literal', () => {
|
|
25
26
|
const { dockerfile } = renderPrebuiltDockerfile(options(workspace()));
|
|
26
27
|
const proof = dockerfile.split('FROM base AS proof\n')[1].split('\nFROM base\n')[0];
|
|
27
28
|
assert.match(proof, /^COPY \.xema-stage\/proof\/xema-composed-image\.json \/proof\/$/m);
|
|
28
29
|
assert.match(proof, /^COPY \.npmrc package\.json pnpm-lock\.yaml pnpm-workspace\.yaml \/proof\/context\/$/m);
|
|
29
30
|
assert.match(proof, /^COPY tooling\/dev\/ \/proof\/copied\/tooling\/dev\/$/m);
|
|
30
|
-
assert.match(proof, /^COPY \.xema-stage\/layers\/store\/ \/\nCOPY \.xema-stage\/layers\/
|
|
31
|
+
assert.match(proof, /^COPY \.xema-stage\/layers\/store\/ \/\nCOPY \.xema-stage\/layers\/workspace\/ \/\nCOPY \.xema-stage\/layers\/dist\/ \/$/m);
|
|
31
32
|
assert.match(proof, /prove \/proof --root \/monorepo --commit "\$\{BUILD_SHA\}" --filter @t\/alpha-api\.\.\. --filter @t\/alpha-db-api\.\.\./);
|
|
32
33
|
assert.match(proof, /prisma-check --root \/monorepo/);
|
|
33
34
|
assert.match(proof, /USER node\nRUN --network=none \S+ \\\n {2}NODE_ENV=production node \S+ load --each --root \/monorepo/);
|
|
@@ -46,7 +47,9 @@ test('the final stage only COPYs the staged layers and the proof verdict; the to
|
|
|
46
47
|
const { dockerfile } = renderPrebuiltDockerfile(options(workspace()));
|
|
47
48
|
const final = dockerfile.split('\nFROM base\n')[1];
|
|
48
49
|
const copies = final.split('\n').filter((line) => line.startsWith('COPY '));
|
|
49
|
-
assert.deepEqual(copies, ['COPY .xema-stage/layers/store/ /', 'COPY .xema-stage/layers/
|
|
50
|
+
assert.deepEqual(copies, ['COPY .xema-stage/layers/store/ /', 'COPY .xema-stage/layers/workspace/ /', 'COPY --from=tail / /']);
|
|
51
|
+
const tail = dockerfile.split('FROM scratch AS tail\n')[1].split('\nFROM base\n')[0];
|
|
52
|
+
assert.equal(tail, 'COPY .xema-stage/layers/dist/ /\nCOPY --from=proof /proof/xema-composed-verified.json /monorepo/xema-composed-verified.json\n');
|
|
50
53
|
assert.doesNotMatch(final, /xema-composed-image|repo-build-tooling|\/opt\//);
|
|
51
54
|
assert.doesNotMatch(dockerfile, /COPY \.xema-stage\/tool/);
|
|
52
55
|
assert.equal(dockerfile.match(/--mount=type=bind,source=\.xema-stage\/tool,target=\/opt\/xema-composed-image/g).length, 2);
|
|
@@ -59,7 +62,7 @@ test('a custom stage dir and image dir are threaded; a bad one is refused', () =
|
|
|
59
62
|
const root = workspace();
|
|
60
63
|
const custom = renderPrebuiltDockerfile(options(root, { stageDir: 'build/stage' })).dockerfile;
|
|
61
64
|
assert.match(custom, /source=build\/stage\/tool/);
|
|
62
|
-
assert.match(custom, /COPY build\/stage\/layers\/
|
|
65
|
+
assert.match(custom, /COPY build\/stage\/layers\/workspace\/ \/\n/);
|
|
63
66
|
assert.throws(() => renderPrebuiltDockerfile(options(root, { stageDir: '../x' })), /repository-relative/);
|
|
64
67
|
assert.throws(() => renderPrebuiltDockerfile(options(root, { base: 'node:22' })), /pinned by digest/);
|
|
65
68
|
assert.throws(() => renderPrebuiltDockerfile(options(root, { nodeMajor: undefined })), /--node-major/);
|
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
// The runner half of the staged composed image, after `composed-image-stage.mjs` has checked and copied the
|
|
2
2
|
// members' build outputs: it turns them into the production tree the Dockerfile COPYs, for ONE platform.
|
|
3
3
|
//
|
|
4
|
-
// <stage>/layers/app/monorepo/ everything of /monorepo but the registry packages: root manifests, the closure's
|
|
5
|
-
// package.json + declared `files` (the built outputs among them), the injected workspace
|
|
6
|
-
// copies, the member index and host link, the migrate script
|
|
7
4
|
// <stage>/layers/store/monorepo/node_modules/.pnpm/ the registry packages (stable until the lockfile changes)
|
|
5
|
+
// <stage>/layers/workspace/monorepo/ root manifests, the closure's package.json + declared `files` (the
|
|
6
|
+
// non-member built outputs among them), the injected workspace copies, the Prisma
|
|
7
|
+
// clients and engine, the migrate script
|
|
8
|
+
// <stage>/layers/dist/monorepo/ the members' build outputs (default `dist`), the member index and the host link: the
|
|
9
|
+
// only layer a commit that changes no dependency changes
|
|
10
|
+
//
|
|
11
|
+
// The files keep their REAL mtimes: BuildKit's context transfer skips a file whose size and mtime are unchanged, so an
|
|
12
|
+
// epoch mtime would let a persistent builder ship stale bytes under a new content key. A layer's digest does not depend
|
|
13
|
+
// on them because the build rewrites timestamps at export (`SOURCE_DATE_EPOCH=0` and `rewrite-timestamp=true` on the
|
|
14
|
+
// image exporter; see the Dockerfile header), so an unchanged store or workspace layer is the same digest every build.
|
|
8
15
|
// <stage>/proof/xema-composed-image.json the manifest the in-image `prove` binds to BUILD_SHA and the context
|
|
9
16
|
//
|
|
10
17
|
// The install is the runner's OFFLINE production install of the members' closures from the pnpm store the job's own
|
|
@@ -15,16 +22,16 @@ import { createRequire } from 'node:module';
|
|
|
15
22
|
import { basename, dirname, join, posix } from 'node:path';
|
|
16
23
|
import { fileURLToPath } from 'node:url';
|
|
17
24
|
|
|
18
|
-
import { COMPOSED_IMAGE_DIR, COMPOSED_MANIFEST_FILE, COMPOSED_MANIFEST_SCHEMA, IMAGE_ROOT_DIRECTORY,
|
|
25
|
+
import { COMPOSED_IMAGE_DIR, COMPOSED_MANIFEST_FILE, COMPOSED_MANIFEST_SCHEMA, IMAGE_ROOT_DIRECTORY, STAGE_DIST_LAYER, STAGE_PROOF_DIRECTORY, STAGE_STORE_LAYER, STAGE_WORKSPACE_LAYER } from './composed-image-paths.mjs';
|
|
19
26
|
import { installClosure } from './composed-image-install.mjs';
|
|
20
|
-
import { writeMemberIndex } from './composed-image-in-image.mjs';
|
|
27
|
+
import { HOST_LINK, MEMBER_INDEX_FILE, writeMemberIndex } from './composed-image-in-image.mjs';
|
|
21
28
|
import { IMAGE_BUILD_INPUTS_FILE, memberFilters, memberManifests, readMembers } from './composed-image-members.mjs';
|
|
22
29
|
import { packageSources } from './composed-image-package-files.mjs';
|
|
23
30
|
import { needsPrismaSchemaEngine, prismaEnginesDirectories } from './composed-image-prisma-engine.mjs';
|
|
24
31
|
import { parseRenderedDockerfile } from './composed-image-prebuilt-dockerfile.mjs';
|
|
25
32
|
import { onlyBuiltDependencies, REBUILD_EXCLUDED, rebuildList } from './composed-image-rebuild.mjs';
|
|
26
33
|
import { syncInjected } from './composed-image-sync-injected.mjs';
|
|
27
|
-
import { copyPlannedOutputs, copyTool, planBuildOutputs, renderedDockerfile, toolVersion } from './composed-image-stage.mjs';
|
|
34
|
+
import { DEFAULT_OUTPUTS, copyPlannedOutputs, copyTool, planBuildOutputs, renderedDockerfile, toolVersion } from './composed-image-stage.mjs';
|
|
28
35
|
import { buildInputs, deployedPackages, downloadSchemaEngine, runGit } from './service-image-stage.mjs';
|
|
29
36
|
import { IMAGE_PLATFORMS, platformArch, scanNatives } from './service-image-platform.mjs';
|
|
30
37
|
import { TOOL_NAME, hashDirectory, isIgnoredEntry, sha256File } from './staged-image-proof.mjs';
|
|
@@ -64,10 +71,10 @@ function copyPackageFiles({ root, treeRoot, closure, problems }) {
|
|
|
64
71
|
/** `node_modules/.pnpm` entries that are registry packages: directories without an injected (`@file+`) workspace copy, and not pnpm's hoist directory. */
|
|
65
72
|
const isStoreEntry = (entry) => entry.isDirectory() && !entry.name.includes('@file+') && entry.name !== 'node_modules';
|
|
66
73
|
|
|
67
|
-
/** Move the registry packages out of the
|
|
74
|
+
/** Move the registry packages out of the workspace layer into the store layer; the overlay of both is the tree the install made. */
|
|
68
75
|
function splitLayers(stageRoot) {
|
|
69
76
|
const pnpm = posix.join('node_modules', '.pnpm');
|
|
70
|
-
const from = join(stageRoot,
|
|
77
|
+
const from = join(stageRoot, STAGE_WORKSPACE_LAYER, IMAGE_ROOT_DIRECTORY, pnpm);
|
|
71
78
|
const to = join(stageRoot, STAGE_STORE_LAYER, IMAGE_ROOT_DIRECTORY, pnpm);
|
|
72
79
|
mkdirSync(to, { recursive: true });
|
|
73
80
|
let moved = 0;
|
|
@@ -78,9 +85,28 @@ function splitLayers(stageRoot) {
|
|
|
78
85
|
return moved;
|
|
79
86
|
}
|
|
80
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Move what changes on every commit out of the workspace layer into the dist layer: each member's `outputs` directories,
|
|
90
|
+
* the member index and the host link. All of them must exist, or the image would lack a member. The union of the three
|
|
91
|
+
* layers is the tree the install and the index made.
|
|
92
|
+
*/
|
|
93
|
+
function splitDistLayer({ stageRoot, members, outputs }) {
|
|
94
|
+
const from = join(stageRoot, STAGE_WORKSPACE_LAYER, IMAGE_ROOT_DIRECTORY);
|
|
95
|
+
const to = join(stageRoot, STAGE_DIST_LAYER, IMAGE_ROOT_DIRECTORY);
|
|
96
|
+
const moves = [MEMBER_INDEX_FILE, HOST_LINK, ...members.flatMap((directory) => outputs.map((output) => posix.join(directory, output)))];
|
|
97
|
+
for (const rel of moves) {
|
|
98
|
+
if (lstatSync(join(from, rel), { throwIfNoEntry: false }) === undefined) {
|
|
99
|
+
throw new Error(`${rel} is not in the workspace layer — the dist layer cannot be split from it`);
|
|
100
|
+
}
|
|
101
|
+
mkdirSync(dirname(join(to, rel)), { recursive: true });
|
|
102
|
+
renameSync(join(from, rel), join(to, rel));
|
|
103
|
+
}
|
|
104
|
+
return moves.length;
|
|
105
|
+
}
|
|
106
|
+
|
|
81
107
|
/**
|
|
82
108
|
* Build the production tree for `platform` under `stageRoot` (which already holds the members' build outputs under
|
|
83
|
-
* `layers/
|
|
109
|
+
* `layers/workspace/monorepo` and the tool), prove it and write the manifest. Throws once, naming every offender.
|
|
84
110
|
* `install`, `writeIndex`, `fetchEngine` and `git` are injectable for tests.
|
|
85
111
|
*/
|
|
86
112
|
export async function stageComposedTree({
|
|
@@ -91,6 +117,7 @@ export async function stageComposedTree({
|
|
|
91
117
|
platform,
|
|
92
118
|
copy,
|
|
93
119
|
buildSource,
|
|
120
|
+
outputs = DEFAULT_OUTPUTS,
|
|
94
121
|
install = installClosure,
|
|
95
122
|
writeIndex = writeMemberIndex,
|
|
96
123
|
fetchEngine = downloadSchemaEngine,
|
|
@@ -98,7 +125,7 @@ export async function stageComposedTree({
|
|
|
98
125
|
log = console.log,
|
|
99
126
|
}) {
|
|
100
127
|
const arch = platformArch(platform);
|
|
101
|
-
const treeRoot = join(stageRoot,
|
|
128
|
+
const treeRoot = join(stageRoot, STAGE_WORKSPACE_LAYER, IMAGE_ROOT_DIRECTORY);
|
|
102
129
|
const problems = [];
|
|
103
130
|
const { commit, inputs, problems: inputProblems } = buildInputs(root, git);
|
|
104
131
|
problems.push(...inputProblems);
|
|
@@ -156,6 +183,8 @@ export async function stageComposedTree({
|
|
|
156
183
|
const index = writeIndex({ root: treeRoot, imageDir, commit, buildSourceRepository: buildSource });
|
|
157
184
|
copyFileSync(join(dirname(fileURLToPath(import.meta.url)), MIGRATE_SCRIPT), join(treeRoot, MIGRATE_TARGET));
|
|
158
185
|
const moved = splitLayers(stageRoot);
|
|
186
|
+
const members = readMembers({ root, imageDir }).map(([, directory]) => directory);
|
|
187
|
+
splitDistLayer({ stageRoot, members, outputs });
|
|
159
188
|
|
|
160
189
|
const manifest = {
|
|
161
190
|
schema: COMPOSED_MANIFEST_SCHEMA,
|
|
@@ -164,7 +193,7 @@ export async function stageComposedTree({
|
|
|
164
193
|
inputs,
|
|
165
194
|
copied: Object.fromEntries(copy.map((directory) => [directory.replace(/\/$/, ''), hashDirectory(join(root, directory))])),
|
|
166
195
|
filters,
|
|
167
|
-
members
|
|
196
|
+
members,
|
|
168
197
|
nodeAbi: process.versions.modules,
|
|
169
198
|
natives: byPackage,
|
|
170
199
|
foreignNatives: foreign.length,
|
|
@@ -187,12 +216,12 @@ export async function stageComposedTree({
|
|
|
187
216
|
export async function stageComposedImage({ root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, dir, platform, outputs, exec, formatSpawn, concurrency, nodeMajor, ...treeOptions }) {
|
|
188
217
|
platformArch(platform);
|
|
189
218
|
const { copy, buildSource } = parseRenderedDockerfile(renderedDockerfile(root, imageDir));
|
|
190
|
-
const planned = await planBuildOutputs({ root, imageDir, dir, outputs, exec, formatSpawn, concurrency, nodeMajor, prefix: posix.join(
|
|
219
|
+
const planned = await planBuildOutputs({ root, imageDir, dir, outputs, exec, formatSpawn, concurrency, nodeMajor, prefix: posix.join(STAGE_WORKSPACE_LAYER, IMAGE_ROOT_DIRECTORY) });
|
|
191
220
|
mkdirSync(planned.target, { recursive: true });
|
|
192
221
|
try {
|
|
193
222
|
copyPlannedOutputs(planned);
|
|
194
223
|
copyTool(join(planned.target, 'tool'));
|
|
195
|
-
return await stageComposedTree({ root, imageDir, stageRoot: planned.target, closure: planned.closure, platform, copy, buildSource, ...treeOptions });
|
|
224
|
+
return await stageComposedTree({ root, imageDir, stageRoot: planned.target, closure: planned.closure, platform, copy, buildSource, ...(outputs === undefined ? {} : { outputs }), ...treeOptions });
|
|
196
225
|
} catch (error) {
|
|
197
226
|
// `planBuildOutputs` proved the directory was absent or empty: a refused stage leaves nothing half-built behind.
|
|
198
227
|
rmSync(planned.target, { recursive: true, force: true });
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import fs from 'node:fs';
|
|
3
4
|
import path from 'node:path';
|
|
4
5
|
import test from 'node:test';
|
|
6
|
+
import { ComposedImageClass } from './composed-image-class.mjs';
|
|
5
7
|
|
|
6
8
|
import { renderPrebuiltDockerfile } from './composed-image-prebuilt-dockerfile.mjs';
|
|
7
9
|
import { stageComposedImage } from './composed-image-stage-tree.mjs';
|
|
@@ -27,7 +29,7 @@ function ready() {
|
|
|
27
29
|
write(root, 'tooling/dev/node_modules/ignored.js', 'x');
|
|
28
30
|
write(root, 'packages/sdks/lib-a/assets/data.json', '{}');
|
|
29
31
|
write(root, 'biomes/alpha/api/alpha-api/README.md', 'ignored');
|
|
30
|
-
write(root, 'infra/composed-process/Dockerfile', renderPrebuiltDockerfile({ root, copy: ['tooling/dev'], buildSource: 'alpha', base: BASE, nodeMajor: String(process.versions.node.split('.')[0]) }).dockerfile);
|
|
32
|
+
write(root, 'infra/composed-process/Dockerfile', renderPrebuiltDockerfile({ root, imageClass: ComposedImageClass.Common, copy: ['tooling/dev'], buildSource: 'alpha', base: BASE, nodeMajor: String(process.versions.node.split('.')[0]) }).dockerfile);
|
|
31
33
|
return root;
|
|
32
34
|
}
|
|
33
35
|
|
|
@@ -47,6 +49,7 @@ function fakeInstall(calls = []) {
|
|
|
47
49
|
|
|
48
50
|
const writeIndex = ({ root, imageDir, commit, buildSourceRepository }) => {
|
|
49
51
|
write(root, 'xema-composed-member-index.json', { commit, buildSourceRepository, imageDir });
|
|
52
|
+
fs.symlinkSync('node_modules/@xemahq/xema-service-nest/bin/host.js', path.join(root, 'xema-composed-process'));
|
|
50
53
|
return { members: 2, host: path.join(root, 'node_modules/@xemahq/xema-service-nest/bin/host.js') };
|
|
51
54
|
};
|
|
52
55
|
|
|
@@ -62,8 +65,7 @@ test('stages the layers, the manifest and the tool; only package.json + declared
|
|
|
62
65
|
assert.equal(calls.length, 1);
|
|
63
66
|
assert.equal(calls[0].arch, 'amd64');
|
|
64
67
|
assert.deepEqual(calls[0].filters, ['--filter', '@t/alpha-api...', '--filter', '@t/alpha-db-api...']);
|
|
65
|
-
const
|
|
66
|
-
const appFiles = files(app);
|
|
68
|
+
const appFiles = [...files(path.join(root, '.xema-stage/layers/workspace/monorepo')), ...files(path.join(root, '.xema-stage/layers/dist/monorepo'))];
|
|
67
69
|
for (const expected of ['package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', 'tooling/dev/patch.txt', 'infra/composed-process/image-build-inputs.json', 'biomes/alpha/xema-biome.json',
|
|
68
70
|
'biomes/alpha/api/alpha-api/package.json', 'biomes/alpha/api/alpha-api/dist/main.js', 'biomes/alpha/api/alpha-db-api/prisma/schema.prisma', 'biomes/alpha/api/alpha-db-api/node_modules/.prisma/alpha-db/index.js',
|
|
69
71
|
'packages/sdks/lib-a/assets/data.json', 'packages/sdks/lib-a/dist/index.js', 'xema-composed-migrate.mjs', 'xema-composed-member-index.json']) {
|
|
@@ -84,12 +86,17 @@ test('stages the layers, the manifest and the tool; only package.json + declared
|
|
|
84
86
|
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(root, '.xema-stage/proof/xema-composed-image.json'), 'utf8')), manifest);
|
|
85
87
|
});
|
|
86
88
|
|
|
87
|
-
test('the registry packages are the store layer; the injected workspace copy and
|
|
89
|
+
test('the registry packages are the store layer; the injected workspace copy and the libraries stay in the workspace layer; the members\' dist, the index and the host link are the dist layer', async () => {
|
|
88
90
|
const root = ready();
|
|
89
91
|
await run(root);
|
|
90
92
|
const store = files(path.join(root, '.xema-stage/layers/store/monorepo'));
|
|
91
93
|
assert.deepEqual(store.filter((f) => f.endsWith('package.json')), ['node_modules/.pnpm/left-pad@1.0.0/node_modules/left-pad/package.json']);
|
|
92
|
-
const app = files(path.join(root, '.xema-stage/layers/
|
|
94
|
+
const app = files(path.join(root, '.xema-stage/layers/workspace/monorepo'));
|
|
95
|
+
assert.deepEqual(files(path.join(root, '.xema-stage/layers/dist/monorepo')).filter((f) => !fs.lstatSync(path.join(root, '.xema-stage/layers/dist/monorepo', f)).isDirectory()),
|
|
96
|
+
['biomes/alpha/api/alpha-api/dist/main.js', 'biomes/alpha/api/alpha-db-api/dist/main.js', 'xema-composed-member-index.json', 'xema-composed-process']);
|
|
97
|
+
assert.ok(fs.lstatSync(path.join(root, '.xema-stage/layers/dist/monorepo/xema-composed-process')).isSymbolicLink());
|
|
98
|
+
assert.ok(!app.some((f) => /alpha-(db-)?api\/dist/.test(f)), 'no member output stays in the workspace layer');
|
|
99
|
+
assert.ok(app.includes('packages/sdks/lib-a/dist/index.js'), 'a library output is a workspace-layer file');
|
|
93
100
|
assert.ok(app.includes('node_modules/.pnpm/@t+lib-a@file+packages+sdks+lib-a/node_modules/@t/lib-a/dist/index.js'), 'the injected copy is synced with the package files');
|
|
94
101
|
assert.ok(app.includes('node_modules/.pnpm/lock.yaml') && app.includes('node_modules/.pnpm/node_modules/left-pad'));
|
|
95
102
|
assert.ok(!app.includes('node_modules/.pnpm/left-pad@1.0.0/node_modules/left-pad/package.json'));
|
|
@@ -131,3 +138,55 @@ test('a tree whose native code is not for the platform is refused', async () =>
|
|
|
131
138
|
const arm = ready();
|
|
132
139
|
await assert.rejects(run(arm, { platform: 'linux/arm64', install }), /left-pad carries native code and none of it is for linux\/arm64/);
|
|
133
140
|
});
|
|
141
|
+
|
|
142
|
+
/** What a layer's digest depends on once the build rewrites timestamps at export: every entry's path, kind, mode and bytes (or link target), in path order. */
|
|
143
|
+
function layerDigest(directory) {
|
|
144
|
+
const hash = createHash('sha256');
|
|
145
|
+
const visit = (dir) => {
|
|
146
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) {
|
|
147
|
+
const full = path.join(dir, entry.name);
|
|
148
|
+
const stat = fs.lstatSync(full);
|
|
149
|
+
const body = stat.isSymbolicLink() ? fs.readlinkSync(full) : stat.isFile() ? fs.readFileSync(full) : '';
|
|
150
|
+
hash.update(`${path.relative(directory, full)}\0${stat.mode}\0`).update(body).update('\0');
|
|
151
|
+
if (stat.isDirectory()) visit(full);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
visit(directory);
|
|
155
|
+
return hash.digest('hex');
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const LAYERS = ['store', 'workspace', 'dist'];
|
|
159
|
+
const layerDigests = (root) => Object.fromEntries(LAYERS.map((layer) => [layer, layerDigest(path.join(root, '.xema-stage/layers', layer))]));
|
|
160
|
+
|
|
161
|
+
test('a commit that changes only a member\'s dist rebuilds exactly one layer: store and workspace are byte-identical, dist differs', async () => {
|
|
162
|
+
const first = ready();
|
|
163
|
+
await run(first);
|
|
164
|
+
const before = layerDigests(first);
|
|
165
|
+
// Later, with another runner clock: the sources are written afresh (new, real mtimes, kept in the context) and one member's built output changed.
|
|
166
|
+
const second = ready();
|
|
167
|
+
write(second, 'biomes/alpha/api/alpha-api/dist/main.js', '// main, edited\n');
|
|
168
|
+
const old = new Date(2001, 0, 1);
|
|
169
|
+
fs.utimesSync(path.join(second, 'biomes/alpha/api/alpha-db-api/dist/main.js'), old, old);
|
|
170
|
+
fs.utimesSync(path.join(second, 'packages/sdks/lib-a/dist/index.js'), old, old);
|
|
171
|
+
await run(second);
|
|
172
|
+
const after = layerDigests(second);
|
|
173
|
+
assert.equal(after.store, before.store);
|
|
174
|
+
assert.equal(after.workspace, before.workspace);
|
|
175
|
+
assert.notEqual(after.dist, before.dist);
|
|
176
|
+
assert.equal(LAYERS.filter((layer) => after[layer] !== before[layer]).length, 1);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test('a change to a library (workspace package) changes the workspace layer and the dist layer is not the cause; a lockfile-only registry change changes the store layer', async () => {
|
|
180
|
+
const base = ready();
|
|
181
|
+
await run(base);
|
|
182
|
+
const before = layerDigests(base);
|
|
183
|
+
const lib = ready();
|
|
184
|
+
write(lib, 'packages/sdks/lib-a/dist/index.js', '// lib, edited\n');
|
|
185
|
+
await run(lib);
|
|
186
|
+
assert.equal(layerDigests(lib).store, before.store);
|
|
187
|
+
assert.notEqual(layerDigests(lib).workspace, before.workspace);
|
|
188
|
+
assert.equal(layerDigests(lib).dist, before.dist);
|
|
189
|
+
const dep = ready();
|
|
190
|
+
await run(dep, { install: (args) => { fakeInstall()(args); write(args.treeRoot, 'node_modules/.pnpm/left-pad@1.0.1/node_modules/left-pad/package.json', { name: 'left-pad' }); } });
|
|
191
|
+
assert.notEqual(layerDigests(dep).store, before.store);
|
|
192
|
+
});
|