@xemahq/repo-build-tooling 0.11.0 → 0.12.0

package/README.md

@@ -33,7 +33,7 @@ pnpm exec xema-scrub-swagger-paths dist
 | `xema-check-no-vendored-deps` | Fails on a `workspace:` range for a package the repository does not own |
 | `xema-check-workspace-ranges` | Fails on a `workspace:` range the local package does not satisfy |
 
-`xema-composed-image members --write|--check` accepts `--manifests <glob>` (repeatable, default `**/xema-biome.json`), `--canonical-dockerfile <path>` and `--cross-repo-manifest <path>`. `xema-composed-image dockerfile` requires `--tooling-version`, `--copy`, `--cache-id` and `--build-source`. `dockerfile --prebuilt` renders the Dockerfile and `Dockerfile.dockerignore` of an image assembled from `stage <dir> [--output <rel>]…` output (requires `--tooling-version`, `--cache-id`, `--build-source`; `--copy` and `--stage-dir` optional); `load --each` loads every member in its own process. Run `xema-composed-image` with no arguments for the full usage.
+`xema-composed-image members --write|--check` accepts `--manifests <glob>` (repeatable, default `**/xema-biome.json`), `--canonical-dockerfile <path>` and `--cross-repo-manifest <path>`. `xema-composed-image dockerfile` requires `--tooling-version`, `--copy`, `--cache-id` and `--build-source`. `dockerfile --prebuilt` renders the Dockerfile and `Dockerfile.dockerignore` of an image assembled from `stage <dir> [--output <rel>]… [--also <pkg>]…` output (requires `--tooling-version`, `--cache-id`, `--build-source`; `--copy` and `--stage-dir` optional); `load --each` loads every member in its own process. `stage` and `filters` accept `--also <workspace package name>` (repeatable): the package's closure is built and staged (built outputs only, no Prisma checks) alongside the members; an unknown name is refused. Run `xema-composed-image` with no arguments for the full usage.
 
 Every step refuses rather than skips. The package has no dependencies.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xemahq/repo-build-tooling",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Dev-time build tooling shared by every Xema repository. Ships as plain ESM with zero dependencies so the published artifact is the reviewed source.",
   "license": "Apache-2.0",
   "author": "Neuralchowder Inc. <developer@xema.dev> (https://xema.dev)",
@@ -42,6 +42,6 @@
     "test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/ghcr-manifest.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/registry-fetch.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs && node --test src/composed-image-prebuilt-dockerfile.test.mjs && node --test src/composed-image-stage.test.mjs && node --test src/composed-image-sync-injected.test.mjs && node --test src/composed-image-prisma-check.test.mjs && node --test src/composed-migrate.test.mjs"
   },
   "xemaRelease": {
-    "sourceDigest": "sha256:8a0729af873e6dee7bc38a38408c3f0f57447884e6461ad36e06e8165cf28f4b"
+    "sourceDigest": "sha256:65f155b64c9951c7c47a7d2a2b9cf65e08615471f0d5c49097a18460cbed260d"
   }
 }

package/src/composed-image-members.mjs

@@ -43,10 +43,15 @@ function packageName(root, directory) {
   return manifest.name;
 }
 
-/** The pnpm `--filter <package>...` arguments covering every member's closure. */
+/**
+ * The pnpm `--filter <package>...` arguments covering every member's closure, then
+ * each `also` package's (a workspace package name); a name a member already
+ * selected is listed once.
+ */
 export function memberFilters(options = {}) {
   const root = options.root ?? process.cwd();
-  return readMembers(options).map(([, directory]) => `--filter ${packageName(root, directory)}...`);
+  const names = readMembers(options).map(([, directory]) => packageName(root, directory));
+  return [...new Set([...names, ...(options.also ?? [])])].map((name) => `--filter ${name}...`);
 }
 
 
@@ -66,21 +71,30 @@ export function memberManifests({ root = process.cwd(), imageDir = COMPOSED_IMAG
 
 const defaultExec = (file, args, { cwd }) => execFileSync(file, args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
 
-/**
- * The workspace packages the members' filters select (the members and everything
- * they depend on inside the workspace): repository-relative directories, sorted.
- * `exec(file, args, { cwd })` returns stdout; injectable for tests.
- */
-export function memberClosure({ root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, exec = defaultExec } = {}) {
-  const words = memberFilters({ root, imageDir }).flatMap((filter) => filter.split(' '));
-  const listing = exec('pnpm', ['ls', '-r', '--depth', '-1', '--json', ...words], { cwd: root });
+function parseListing(listing) {
   let packages;
   try {
     packages = JSON.parse(listing);
   } catch (error) {
     throw new Error(`pnpm ls did not print JSON: ${error.message}`);
   }
-  if (!Array.isArray(packages) || packages.length === 0) throw new Error('pnpm ls selected no workspace package');
+  if (!Array.isArray(packages)) throw new Error('pnpm ls did not print a JSON array');
+  return packages;
+}
+
+/**
+ * The workspace packages the members' filters select (the members and everything
+ * they depend on inside the workspace): repository-relative directories, sorted.
+ * `exec(file, args, { cwd })` returns stdout; injectable for tests.
+ */
+export function memberClosure({ root = process.cwd(), imageDir = COMPOSED_IMAGE_DIR, exec = defaultExec, also = [] } = {}) {
+  for (const name of also) {
+    const found = parseListing(exec('pnpm', ['ls', '-r', '--depth', '-1', '--json', '--filter', name], { cwd: root }));
+    if (!found.some((pkg) => pkg?.name === name)) throw new Error(`--also ${name}: no workspace package has that name`);
+  }
+  const words = memberFilters({ root, imageDir, also }).flatMap((filter) => filter.split(' '));
+  const packages = parseListing(exec('pnpm', ['ls', '-r', '--depth', '-1', '--json', ...words], { cwd: root }));
+  if (packages.length === 0) throw new Error('pnpm ls selected no workspace package');
   const real = realpathSync(root);
   const directories = new Set();
   for (const pkg of packages) {

package/src/composed-image-prebuilt-dockerfile.mjs

@@ -123,7 +123,9 @@ export function renderPrebuiltDockerfile({
 #   assemble  source of the closure + the staged outputs, the injected copies synced to
 #             their \`files\`, then the proofs on the target arch: Prisma clients run, the
 #             member index is written, every member loads ALONE as \`node\`, production.
-#   (final)   \`/monorepo\` from assemble. Nothing of the tool ships.
+#   (final)   \`/monorepo\` of assemble, shipped as THREE layers so a commit that changes only
+#             members' code re-pushes only the layers that changed (see the final stage).
+#             Nothing of the tool ships.
 #
 # The \`--filter <package>...\` words below are LITERAL: the content hash of this image reads
 # them to narrow the lockfile to the members' closures.
@@ -168,9 +170,22 @@ USER node
 RUN ${toolMount} \\
   NODE_ENV=production node ${node} load --each${dirFlag}
 
+# The final stage is layered by how often each part changes, so a registry and a node
+# already holding the stable layers fetch only the rest. The three copies overlay to
+# exactly assemble's /monorepo (proven by a tree diff of both images).
+#   1 deps       root manifests, the external .pnpm store, and the injected stubs that hold
+#                only package.json. Stable while the lockfile and the closure's package.json
+#                files are unchanged.
+#   2 assemble   members, dist, Prisma clients, member index, migrate script: everything
+#                but the .pnpm store, whose packages layer 1 already carries.
+#   3 injected   the workspace copies \`node_modules/.pnpm/*@file+*\`: \`sync-injected\` fills
+#                them from the members' files, so they change with every commit. \`--parents\`
+#                keeps their paths under /.
 FROM base
 WORKDIR /monorepo
-COPY --from=assemble /monorepo /monorepo
+COPY --from=deps /monorepo /monorepo
+COPY --from=assemble --exclude=node_modules/.pnpm /monorepo /monorepo
+COPY --from=assemble --parents /monorepo/node_modules/.pnpm/*@file+* /
 ENV NODE_ENV=production
 # Set by the IMAGE, never by a deployment (kernel \`COMPOSED_PROCESS_ENV.MemberIndex\`).
 ENV XEMA_COMPOSED_MEMBER_INDEX=/monorepo/xema-composed-member-index.json

package/src/composed-image-prebuilt-dockerfile.test.mjs

@@ -39,7 +39,7 @@ test('the tool is only bind-mounted, and the final stage carries nothing of it',
   assert.doesNotMatch(dockerfile, /\bnpm install|COPY \.xema-stage\/tool/);
   const final = dockerfile.split('\nFROM base\n')[1];
   assert.doesNotMatch(final, /xema-composed-image|repo-build-tooling|npm install|\/opt\//);
-  assert.match(final, /COPY --from=assemble \/monorepo \/monorepo/);
+  assert.doesNotMatch(final, /COPY --from=assemble \/monorepo \/monorepo/);
   assert.match(dockerfile, /USER node\nRUN --mount=\S+ \\\n {2}NODE_ENV=production node \/opt\/xema-composed-image\/src\/composed-image\.mjs load --each\n/);
   assert.match(dockerfile, /cp \/opt\/xema-composed-image\/src\/composed-migrate\.mjs \/monorepo\/xema-composed-migrate\.mjs/);
 });
@@ -101,3 +101,16 @@ test('a token-using repository gets single-backslash line continuations (a doubl
   assert.match(dockerfile, /--mount=type=secret,id=npm_token,required=true \\\n/);
   for (const line of dockerfile.split('\n')) assert.doesNotMatch(line, /\\\\\s*$/, `line ends with a doubled backslash: ${line}`);
 });
+
+test('the final stage is layered: deps, then assemble without the .pnpm store, then the injected copies', () => {
+  const { dockerfile } = renderPrebuiltDockerfile(options(workspace()));
+  const final = dockerfile.split('\nFROM base\n')[1];
+  const copies = final.split('\n').filter((line) => line.startsWith('COPY '));
+  assert.deepEqual(copies, [
+    'COPY --from=deps /monorepo /monorepo',
+    'COPY --from=assemble --exclude=node_modules/.pnpm /monorepo /monorepo',
+    'COPY --from=assemble --parents /monorepo/node_modules/.pnpm/*@file+* /',
+  ]);
+  assert.ok(final.indexOf('ENV NODE_ENV=production') > final.lastIndexOf('COPY --from=assemble'));
+  assert.match(final, /USER node\nCMD \["node", "\/monorepo\/xema-composed-process"\]\n$/);
+});

package/src/composed-image-stage.mjs

@@ -3,7 +3,8 @@
 // directory the `--prebuilt` Dockerfile copies from. Docker compiles nothing; this
 // is the only place outputs enter an image, so it is also where they are refused.
 //
-//   <dir>/out/<package dir>/<output>/               every closure package's outputs (default: dist)
+//   <dir>/out/<package dir>/<output>/               every closure package's outputs (default: dist);
+//                                                   `--also <pkg>` adds that package's closure (no member checks)
 //   <dir>/out/<member>/node_modules/.prisma/        every member's generated Prisma clients
 //   <dir>/tool/                                     this package (package.json + src, no tests):
 //                                                   bind-mounted into the build, never copied into a layer
@@ -65,6 +66,7 @@ export async function stageBuildOutputs({
   imageDir = COMPOSED_IMAGE_DIR,
   dir,
   outputs = DEFAULT_OUTPUTS,
+  also = [],
   exec,
   nodeMajor = process.versions.node.split('.')[0],
   formatSpawn,
@@ -90,7 +92,7 @@ export async function stageBuildOutputs({
 
   const members = readMembers({ root, imageDir }).map(([key, directory]) => ({ key, directory }));
   const memberDirectories = new Set(members.map((m) => m.directory));
-  const closure = memberClosure({ root, imageDir, exec });
+  const closure = memberClosure({ root, imageDir, exec, also });
   const schemaJobs = [];
   for (const { key, directory } of members) {
     if (!closure.includes(directory)) problems.push(`${key}: ${directory} is not in the closure pnpm selected`);

package/src/composed-image-stage.test.mjs

@@ -100,3 +100,54 @@ test('refuses a client generated from another schema than the member carries; th
   await assert.rejects(stage(stale), /alpha-db-api: .*differs from the member's prisma\/schema\.prisma .*stale/);
   assert.equal((await stage(ready())).packages, 3);
 });
+
+/** An `exec` that answers per-name probes (`--filter <name>`) and the closure listing (`--filter <name>...`). */
+function alsoPnpm(root, known) {
+  return (_file, args) => {
+    const filters = args.flatMap((a, i) => (args[i - 1] === '--filter' ? [a] : []));
+    const names = filters.filter((f) => !f.endsWith('...'));
+    if (names.length > 0) return JSON.stringify(known.filter((p) => p.name === names[0]).map((p) => ({ name: p.name, path: path.join(root, p.dir) })));
+    const wanted = new Set(filters.map((f) => f.slice(0, -3)));
+    const dirs = new Set(['biomes/alpha/api/alpha-api', 'biomes/alpha/api/alpha-db-api', 'packages/sdks/lib-a']);
+    for (const p of known) if (wanted.has(p.name)) for (const d of p.closure) dirs.add(d);
+    return JSON.stringify([...dirs].map((dir) => ({ name: dir, path: path.join(root, dir) })));
+  };
+}
+const EXTRA = [{ name: '@t/extra', dir: 'packages/sdks/extra', closure: ['packages/sdks/extra', 'packages/sdks/extra-dep'] },
+  { name: '@t/lib-a', dir: 'packages/sdks/lib-a', closure: ['packages/sdks/lib-a'] }];
+
+function withExtra() {
+  const root = ready();
+  write(root, 'packages/sdks/extra/package.json', { name: '@t/extra' });
+  write(root, 'packages/sdks/extra/dist/index.js', '// extra\n');
+  write(root, 'packages/sdks/extra-dep/dist/index.js', '// dep\n');
+  return root;
+}
+
+test('--also stages the named package closure without member checks, and a member named again is idempotent', async () => {
+  const root = withExtra();
+  const result = await stage(root, { exec: alsoPnpm(root, EXTRA), also: ['@t/extra', '@t/lib-a'] });
+  assert.equal(result.packages, 5);
+  const staged = files(path.join(root, '.xema-stage/out'));
+  for (const expected of ['packages/sdks/extra/dist/index.js', 'packages/sdks/extra-dep/dist/index.js', 'packages/sdks/lib-a/dist/index.js']) {
+    assert.ok(staged.includes(expected), expected);
+  }
+  assert.ok(!staged.some((f) => f.startsWith('packages/sdks/extra/node_modules')));
+});
+
+test('--also refuses an unknown package and a native addon in the extra closure', async () => {
+  const root = withExtra();
+  await assert.rejects(stage(root, { exec: alsoPnpm(root, EXTRA), also: ['@t/nope'] }), /--also @t\/nope: no workspace package/);
+  assert.equal(fs.existsSync(path.join(root, '.xema-stage')), false);
+  write(root, 'packages/sdks/extra-dep/dist/a.node', 'x');
+  await assert.rejects(stage(root, { exec: alsoPnpm(root, EXTRA), also: ['@t/extra'] }), /extra-dep\/dist\/a\.node/);
+});
+
+test('filters and the argument parser carry --also', async () => {
+  const root = ready();
+  const { memberFilters } = await import('./composed-image-members.mjs');
+  const { parseArguments } = await import('./composed-image.mjs');
+  assert.deepEqual(memberFilters({ root, also: ['@t/extra', '@t/lib-a', '@t/extra'] }).slice(-2), ['--filter @t/extra...', '--filter @t/lib-a...']);
+  assert.equal(memberFilters({ root, also: ['@t/alpha-api'] }).filter((f) => f === '--filter @t/alpha-api...').length, 1);
+  assert.deepEqual(parseArguments(['stage', 'd', '--also', 'a', '--also', 'b']).options.also, ['a', 'b']);
+});

package/src/composed-image.mjs

@@ -24,7 +24,8 @@
 //            derive the member set and write / verify
 //            `<image-dir>/image-build-inputs.json` (its existence at a commit IS
 //            the repository's composed-process declaration).
-//   filters  print `--filter <package>...` for every member, so install and
+//   filters [--also <pkg>]…
+//            print `--filter <package>...` for every member (and each --also package), so install and
 //            build cover exactly the members' closures.
 //   dirs     print every member's directory, one per line.
 //   pins <committed-lockfile>
@@ -38,10 +39,11 @@
 //   load --each
 //            the same check, one `node` process PER member, so a member that only
 //            loads because another loaded first is caught (the prebuilt image's gate).
-//   stage <dir> [--output <rel>]…
+//   stage <dir> [--output <rel>]… [--also <pkg>]…
 //            gather the runner's build outputs (default `dist`) and the members'
 //            generated Prisma clients into a build-step directory, with this tool,
-//            for the `--prebuilt` Dockerfile (`composed-image-stage.mjs`).
+//            for the `--prebuilt` Dockerfile (`composed-image-stage.mjs`). Each `--also`
+//            workspace package adds its closure's built outputs (no Prisma checks).
 //   sync-injected | prisma-check
 //            IN-IMAGE proofs of the prebuilt build: the injected workspace copies hold
 //            their packages' `files`; every Prisma client runs on the target arch.
@@ -454,18 +456,18 @@ export function loadMembersEach({ root = process.cwd(), imageDir = COMPOSED_IMAG
 const USAGE =
   'usage: xema-composed-image <command> [--root <dir>] [--image-dir <dir>]\n' +
   '  members --write|--check [--manifests <glob>]… [--canonical-dockerfile <path>] [--cross-repo-manifest <path>]\n' +
-  '  filters | dirs | pins <committed-lockfile> | index <commit> --build-source <repo> | load [--each]\n' +
-  '  stage <dir> [--output <rel>]…   | sync-injected | prisma-check | migrate <member> [--root <dir>]\n' +
+  '  filters [--also <pkg>]… | dirs | pins <committed-lockfile> | index <commit> --build-source <repo> | load [--each]\n' +
+  '  stage <dir> [--output <rel>]… [--also <pkg>]…   | sync-injected | prisma-check | migrate <member> [--root <dir>]\n' +
   '  coordinate --image-name <name>   (env: SHA REGISTRY REGISTRY_ORG REGISTRY_USER REGISTRY_TOKEN [TAG_ARGS_FILE])\n' +
   '  load-image <image>\n' +
   '  smoke-refusal <image> [--member <biomeId>/<componentKey>]x2\n' +
   '  dockerfile --write|--check --tooling-version <x.y.z> --copy <dir>… --cache-id <id> --build-source <repo>\n' +
   '  dockerfile --prebuilt --write|--check --tooling-version <x.y.z> --cache-id <id> --build-source <repo> [--copy <dir>]… [--stage-dir <dir>]';
 
-const REPEATABLE = new Set(['manifests', 'copy', 'member', 'output']);
+const REPEATABLE = new Set(['manifests', 'copy', 'member', 'output', 'also']);
 const BOOLEAN = new Set(['prebuilt', 'each']);
 const VALUED = new Set(['root', 'image-dir', 'manifests', 'canonical-dockerfile', 'cross-repo-manifest', 'build-source',
-  'image-name', 'tooling-version', 'cache-id', 'copy', 'member', 'stage-dir', 'output',
+  'image-name', 'tooling-version', 'cache-id', 'copy', 'member', 'stage-dir', 'output', 'also',
 ]);
 
 export function parseArguments(argv) {
@@ -518,7 +520,7 @@ export function main(argv, { log = console.log, write = (text) => process.stdout
       return;
     }
     case 'filters':
-      write(`${memberFilters({ root, imageDir }).join(' ')}\n`);
+      write(`${memberFilters({ root, imageDir, also: options.also }).join(' ')}\n`);
       return;
     case 'dirs':
       write(`${readMembers({ root, imageDir }).map(([, directory]) => directory).join('\n')}\n`);
@@ -541,7 +543,7 @@ export function main(argv, { log = console.log, write = (text) => process.stdout
       log(`composed members: all ${loadMembers({ root, imageDir })} module graph(s) load`);
       return;
     case 'stage': {
-      return stageBuildOutputs({ root, imageDir, dir: argument, outputs: options.output }).then((staged) => {
+      return stageBuildOutputs({ root, imageDir, dir: argument, outputs: options.output, also: options.also }).then((staged) => {
         log(`composed stage ${argument}: ${staged.packages} package(s), ${staged.outputs} output dir(s), ${staged.files} file(s), tool ${staged.toolFiles} file(s)`);
       });
     }