@xemahq/repo-build-tooling 0.31.1 → 0.32.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xemahq/repo-build-tooling",
-  "version": "0.31.1",
+  "version": "0.32.0",
   "description": "Dev-time build tooling shared by every Xema repository. Ships as plain ESM, depending only on @xemahq/release-policy for registry access, so the published artifact is the reviewed source.",
   "license": "Apache-2.0",
   "author": "Neuralchowder Inc. <developer@xema.dev> (https://xema.dev)",
@@ -53,6 +53,6 @@
     "test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/composed-image-class.test.mjs && node --test src/composed-image-coordinate-cli.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs && node --test src/composed-image-prebuilt-dockerfile.test.mjs && node --test src/composed-image-in-image-closure.test.mjs && node --test src/composed-image-stage.test.mjs && node --test src/composed-image-stage-tree.test.mjs && node --test src/composed-image-install.test.mjs && node --test src/composed-image-proof.test.mjs && node --test src/composed-image-pnpm-context.test.mjs && node --test src/composed-image-sync-injected.test.mjs && node --test src/composed-image-prisma-check.test.mjs && node --test src/composed-image-prisma-engine.test.mjs && node --test src/composed-migrate.test.mjs && node --test src/promote-image.test.mjs && node --test src/arm64-supplement-needed.test.mjs && node --test src/service-image.test.mjs && node --test src/image-cpu-arch.test.mjs && node --test src/staged-image-proof.test.mjs && node --test src/effective-input.test.mjs"
   },
   "xemaRelease": {
-    "sourceDigest": "sha256:a1e1068eca88fad502417b887f16c732ecf0c9ed0ca339849880ea780aa894d4"
+    "sourceDigest": "sha256:cda113cf9ccac7c7af99832d6d0ad68bd3b043b0e99a6e590df9d12cf1a36881"
   }
 }

package/src/composed-image-prebuilt-dockerfile.mjs

@@ -96,7 +96,7 @@ export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMP
   const node = `${toolDir}/${IN_IMAGE_ENTRY}`;
   const copies = normalizedCopy(copy).map(copyLine).join('\n');
   const stageLayer = (layer) => `COPY ${stageDir}/${layer}/ /`;
-  const allLayers = [STAGE_STORE_LAYER, STAGE_WORKSPACE_LAYER, STAGE_DIST_LAYER].map(stageLayer).join('\n');
+  const stableLayers = [STAGE_STORE_LAYER, STAGE_WORKSPACE_LAYER].map(stageLayer).join('\n');
   const toolsetStages = recipe.toolsets.map((name) => `FROM ${toolsets[name]} AS ${name}\n`).join('');
   const toolsetCopies = recipe.toolsets.map((name) => `COPY --from=${name} ${TOOLSET_FILES[name].from} ${TOOLSET_FILES[name].to}\n`).join('');
   const dockerfile = `# @generated by \`xema-composed-image dockerfile --prebuilt --write\` (@xemahq/repo-build-tooling@${toolingVersion}) — DO NOT EDIT.
@@ -118,7 +118,8 @@ export function renderPrebuiltDockerfile({ root = process.cwd(), imageDir = COMP
 # \`${stageDir}/tool/\` is this tool, bind-mounted into the steps below and never copied into a layer.
 #
 # Stages:
-#   proof  The staged tree and the repository's inputs meet. The root build inputs that decide the
+#   proof  (COPYs run least to most often changed: store, workspace, root inputs, \`--copy\` directories, dist, manifest, so a source change keeps the store layer cached.)
+#          The staged tree and the repository's inputs meet. The root build inputs that decide the
 #          staged tree are COPIED into \`/${PROOF_DIRECTORY}/${CONTEXT_DIRECTORY}/\` and the \`--copy\` directories into
 #          \`/${COPIED_DIRECTORY}/\`, which puts them in the image's content tag. \`prove\` refuses the build unless the
 #          stage's manifest names BUILD_SHA and those bytes, this tool and this platform, and
@@ -134,9 +135,10 @@ ${toolsetStages}FROM ${from} AS base
 FROM base AS proof
 WORKDIR /${PROOF_DIRECTORY}
 ARG BUILD_SHA
-COPY ${stageDir}/${STAGE_PROOF_DIRECTORY}/${COMPOSED_MANIFEST_FILE} /${PROOF_DIRECTORY}/
+${stableLayers}
 COPY ${rootInputs.join(' ')} /${PROOF_DIRECTORY}/${CONTEXT_DIRECTORY}/
-${copies === '' ? '' : `${copies}\n`}${allLayers}
+${copies === '' ? '' : `${copies}\n`}${stageLayer(STAGE_DIST_LAYER)}
+COPY ${stageDir}/${STAGE_PROOF_DIRECTORY}/${COMPOSED_MANIFEST_FILE} /${PROOF_DIRECTORY}/
 RUN --network=none ${toolMount} \\
   : "\${BUILD_SHA:?BUILD_SHA build-arg is required}" && \\
   T="node ${node}" && \\

package/src/composed-image-prebuilt-dockerfile.test.mjs

@@ -28,12 +28,24 @@ test('the proof stage COPYs the root build inputs flat, the --copy directories,
   assert.match(proof, /^COPY \.xema-stage\/proof\/xema-composed-image\.json \/proof\/$/m);
   assert.match(proof, /^COPY \.npmrc package\.json pnpm-lock\.yaml pnpm-workspace\.yaml \/proof\/context\/$/m);
   assert.match(proof, /^COPY tooling\/dev\/ \/proof\/copied\/tooling\/dev\/$/m);
-  assert.match(proof, /^COPY \.xema-stage\/layers\/store\/ \/\nCOPY \.xema-stage\/layers\/workspace\/ \/\nCOPY \.xema-stage\/layers\/dist\/ \/$/m);
+  assert.match(proof, /^COPY \.xema-stage\/layers\/store\/ \/\nCOPY \.xema-stage\/layers\/workspace\/ \/$/m);
+  assert.match(proof, /^COPY \.xema-stage\/layers\/dist\/ \/$/m);
   assert.match(proof, /prove \/proof --root \/monorepo --commit "\$\{BUILD_SHA\}" --filter @t\/alpha-api\.\.\. --filter @t\/alpha-db-api\.\.\./);
   assert.match(proof, /prisma-check --root \/monorepo/);
   assert.match(proof, /USER node\nRUN --network=none \S+ \\\n {2}NODE_ENV=production node \S+ load --each --root \/monorepo/);
 });
 
+test('the proof stage COPYs in stability order: store, workspace, root inputs, --copy trees, dist, manifest', () => {
+  const { dockerfile } = renderPrebuiltDockerfile(options(workspace()));
+  const proof = dockerfile.split('FROM base AS proof\n')[1].split('\nFROM base\n')[0];
+  const copies = proof.split('\n').filter((line) => line.startsWith('COPY '));
+  const at = (re) => copies.findIndex((line) => re.test(line));
+  const order = [/layers\/store\//, /layers\/workspace\//, /\/proof\/context\/$/, /\/proof\/copied\//, /layers\/dist\//, /xema-composed-image\.json/].map(at);
+  assert.ok(order.every((i) => i >= 0), `every COPY is present: ${order}`);
+  assert.deepEqual(order, [...order].sort((a, b) => a - b));
+  assert.equal(order[0], 0, 'the store COPY precedes every source-tree COPY');
+});
+
 test('the root inputs the Dockerfile COPYs are exactly the ROOT_BUILD_INPUT files present', () => {
   const root = workspace();
   const opts = options(root);

package/src/service-image-stage.mjs

@@ -310,3 +310,16 @@ export async function stageService({
     `prisma ${prisma === null ? 'none' : `${prisma.enginesVersion} ${prisma.binaryTarget}`}, tool ${toolFiles} files`);
   return manifest;
 }
+
+/**
+ * Stage several services for one platform in this process, one after the other (each into its own
+ * stage directory, with the outputs `stageService` gives a single service). The process-wide memo of
+ * verified schema-engine downloads is shared, so services on the same engines commit and target
+ * download it once. `services` is `[{ service, path, stageDir }]`; the first refusal stops the batch.
+ */
+export async function stageServices({ services, ...shared }) {
+  if (!Array.isArray(services) || services.length === 0) throw new Error('stage-batch needs at least one --service <pkg>=<dir>=<stage-dir>');
+  const manifests = [];
+  for (const { service, path, stageDir } of services) manifests.push(await stageService({ ...shared, service, path, stageDir }));
+  return manifests;
+}

package/src/service-image.mjs

@@ -14,13 +14,18 @@
 //            manifest (commit, root build inputs, tool version) and the tool the Dockerfile's
 //            proof stage runs (`service-image-stage.mjs`).
 //
+//   xema-service-image stage-batch --platform linux/<amd64|arm64> --service <pkg>=<dir>=<stage-dir> [--service ...]
+//            `stage` for N services in ONE process, each into its own stage dir with the same outputs;
+//            the verified schema-engine download is reused across them, so a shared Prisma engine is fetched once.
+//
 // The Dockerfile then runs only `prove` (`service-image-in-image.mjs`, on the target, offline)
 // and COPYs `app/`: no install, no build, no registry token in the image build.
 // ═══════════════════════════════════════════════════════════════════════════
 import { realpathSync } from 'node:fs';
+import { isAbsolute, relative, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { DEFAULT_STAGE_DIR, stageService } from './service-image-stage.mjs';
+import { DEFAULT_STAGE_DIR, stageService, stageServices } from './service-image-stage.mjs';
 
 const USAGE = 'usage: xema-service-image stage --service <pkg> --path <dir> --platform linux/<amd64|arm64> [--stage-dir <dir>]';
 const OPTIONS = new Set(['service', 'path', 'platform', 'stage-dir']);
@@ -42,10 +47,44 @@ export function parseStageArgs(argv) {
   return { service: options.service, path: options.path, platform: options.platform, stageDir: options['stage-dir'] ?? DEFAULT_STAGE_DIR };
 }
 
+const BATCH_USAGE = 'usage: xema-service-image stage-batch --platform linux/<amd64|arm64> --service <pkg>=<dir>=<stage-dir> [--service ...]';
+
+/** True when `outer` is `inner` or a directory above it, after resolving both. */
+function containsOrEquals(outer, inner) {
+  const rel = relative(resolve(outer), resolve(inner));
+  return rel === '' || (rel !== '..' && !rel.startsWith('../') && !isAbsolute(rel));
+}
+
+export function parseBatchArgs(argv) {
+  let platform;
+  const services = [];
+  for (let i = 0; i < argv.length; i += 2) {
+    const flag = argv[i];
+    const value = argv[i + 1];
+    if (flag !== '--platform' && flag !== '--service') throw new Error(`unknown argument '${flag}'\n${BATCH_USAGE}`);
+    if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value\n${BATCH_USAGE}`);
+    if (flag === '--platform') {
+      if (platform !== undefined) throw new Error('--platform is given twice');
+      platform = value;
+      continue;
+    }
+    const parts = value.split('=');
+    if (parts.length !== 3 || parts.some((part) => part === '')) throw new Error(`--service '${value}' is not <pkg>=<dir>=<stage-dir>\n${BATCH_USAGE}`);
+    const [service, path, stageDir] = parts;
+    const nested = services.find((entry) => containsOrEquals(entry.stageDir, stageDir) || containsOrEquals(stageDir, entry.stageDir));
+    if (nested !== undefined) throw new Error(`stage dirs '${nested.stageDir}' and '${stageDir}' are the same or one contains the other`);
+    services.push({ service, path, stageDir });
+  }
+  if (platform === undefined) throw new Error(`--platform is required\n${BATCH_USAGE}`);
+  if (services.length === 0) throw new Error(`--service is required\n${BATCH_USAGE}`);
+  return { platform, services };
+}
+
 async function main(argv) {
   const [verb, ...rest] = argv;
-  if (verb !== 'stage') throw new Error(USAGE);
-  await stageService(parseStageArgs(rest));
+  if (verb === 'stage') await stageService(parseStageArgs(rest));
+  else if (verb === 'stage-batch') await stageServices(parseBatchArgs(rest));
+  else throw new Error(`${USAGE}\n${BATCH_USAGE}`);
 }
 
 // realpath both sides: Node resolves the entry module's symlinks (a pnpm bin shim runs this file

package/src/service-image.test.mjs

@@ -10,10 +10,10 @@ import { fileURLToPath } from 'node:url';
 import { gzipSync } from 'node:zlib';
 
 import { copyTool } from './composed-image-stage.mjs';
-import { parseStageArgs } from './service-image.mjs';
+import { parseBatchArgs, parseStageArgs } from './service-image.mjs';
 import { parseProveArgs, prove } from './service-image-in-image.mjs';
 import { IMAGE_PLATFORMS, currentPlatform, nativeFormat, platformArch, scanNatives } from './service-image-platform.mjs';
-import { PRISMA_ENGINES_URL, SCHEMA_ENGINE_SHA256, deployArgs, downloadSchemaEngine, pnpmEntryName, stageService } from './service-image-stage.mjs';
+import { PRISMA_ENGINES_URL, SCHEMA_ENGINE_SHA256, deployArgs, downloadSchemaEngine, pnpmEntryName, stageService, stageServices } from './service-image-stage.mjs';
 
 const temps = [];
 after(() => {
@@ -311,6 +311,53 @@ test('downloadSchemaEngine downloads once per engines commit and target, however
   for (const dir of dirs) assert.deepEqual(fs.readFileSync(path.join(dir, `schema-engine-${target}`)), engine);
 });
 
+test('parseBatchArgs reads one platform and repeated <pkg>=<dir>=<stage-dir> services', () => {
+  assert.deepEqual(parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=apps/a=.s/a', '--service', 'b=apps/b=.s/b']), {
+    platform: 'linux/amd64', services: [{ service: 'a', path: 'apps/a', stageDir: '.s/a' }, { service: 'b', path: 'apps/b', stageDir: '.s/b' }],
+  });
+  assert.throws(() => parseBatchArgs(['--service', 'a=b=c']), /--platform is required/);
+  assert.throws(() => parseBatchArgs(['--platform', 'linux/amd64']), /--service is required/);
+  assert.throws(() => parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=b']), /not <pkg>=<dir>=<stage-dir>/);
+  assert.throws(() => parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=b=c', '--service', 'd=e=c']), /same or one contains the other/);
+  assert.throws(() => parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=b=.s', '--service', 'd=e=.s/./d']), /same or one contains/);
+  assert.throws(() => parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=b=.s/x/..', '--service', 'd=e=.s/d']), /same or one contains/);
+  assert.doesNotThrow(() => parseBatchArgs(['--platform', 'linux/amd64', '--service', 'a=b=.s/a', '--service', 'd=e=.s/ab']));
+});
+
+test('stage-batch stages every service into its own stage dir with one engine fetch for a shared engine', async (t) => {
+  const engine = elf(183);
+  const version = '9'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  const fetches = [];
+  t.mock.method(globalThis, 'fetch', async (asked) => {
+    fetches.push(asked);
+    return new Response(gzipSync(engine));
+  });
+  const root = fixture({ prisma: true });
+  const manifests = await stageServices({
+    root,
+    platform: 'linux/arm64',
+    services: [
+      { service: 'svc', path: 'apps/svc', stageDir: '.stage-a' },
+      { service: 'svc', path: 'apps/svc', stageDir: '.stage-b' },
+    ],
+    log: () => {},
+    deploy: fakeDeploy({ prisma: true }),
+    git: fakeGit(),
+    fetchEngine: (args) => downloadSchemaEngine({ ...args, enginesVersion: version, checksums }),
+  });
+  assert.equal(fetches.length, 1);
+  assert.equal(manifests.length, 2);
+  const single = await stage(fixture({ prisma: true }), { deploy: fakeDeploy({ prisma: true }), fetchEngine: async ({ enginesDir, binaryTarget }) => write(path.join(enginesDir, `schema-engine-${binaryTarget}`), engine) });
+  for (const dir of ['.stage-a', '.stage-b']) {
+    const manifest = JSON.parse(fs.readFileSync(path.join(root, dir, 'service/xema-service-image.json'), 'utf8'));
+    assert.deepEqual({ ...manifest, prisma: null }, { ...single, prisma: null });
+    assert.deepEqual(manifest.prisma, { enginesVersion: ENGINES_VERSION, binaryTarget: target, sha256: sha256(engine) });
+  }
+  await assert.rejects(stageServices({ root, platform: 'linux/arm64', services: [] }), /at least one --service/);
+});
+
 test('downloadSchemaEngine names the engine, URL, commit, target, bytes and cause when the body read fails', async (t) => {
   const engine = elf(186);
   const version = 'f'.repeat(40);