@xemahq/repo-build-tooling 0.26.2 → 0.26.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xemahq/repo-build-tooling",
-  "version": "0.26.2",
+  "version": "0.26.4",
   "description": "Dev-time build tooling shared by every Xema repository. Ships as plain ESM, depending only on @xemahq/release-policy for registry access, so the published artifact is the reviewed source.",
   "license": "Apache-2.0",
   "author": "Neuralchowder Inc. <developer@xema.dev> (https://xema.dev)",
@@ -51,9 +51,9 @@
     "@xemahq/release-policy": "^16.0.0"
   },
   "scripts": {
-    "test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/composed-image-coordinate-cli.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs && node --test src/composed-image-prebuilt-dockerfile.test.mjs && node --test src/composed-image-in-image-closure.test.mjs && node --test src/composed-image-stage.test.mjs && node --test src/composed-image-stage-tree.test.mjs && node --test src/composed-image-install.test.mjs && node --test src/composed-image-proof.test.mjs && node --test src/composed-image-pnpm-context.test.mjs && node --test src/composed-image-sync-injected.test.mjs && node --test src/composed-image-prisma-check.test.mjs && node --test src/composed-image-prisma-engine.test.mjs && node --test src/composed-migrate.test.mjs && node --test src/promote-image.test.mjs && node --test src/arm64-supplement-needed.test.mjs && node --test src/prepush-sanity.test.mjs && node --test src/service-image.test.mjs && node --test src/install-git-hooks.test.mjs"
+    "test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/composed-image-coordinate-cli.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs && node --test src/composed-image-prebuilt-dockerfile.test.mjs && node --test src/composed-image-in-image-closure.test.mjs && node --test src/composed-image-stage.test.mjs && node --test src/composed-image-stage-tree.test.mjs && node --test src/composed-image-install.test.mjs && node --test src/composed-image-proof.test.mjs && node --test src/composed-image-pnpm-context.test.mjs && node --test src/composed-image-sync-injected.test.mjs && node --test src/composed-image-prisma-check.test.mjs && node --test src/composed-image-prisma-engine.test.mjs && node --test src/composed-migrate.test.mjs && node --test src/promote-image.test.mjs && node --test src/arm64-supplement-needed.test.mjs && node --test src/prepush-sanity.test.mjs && node --test src/service-image.test.mjs && node --test src/install-git-hooks.test.mjs && node --test src/image-cpu-arch.test.mjs"
   },
   "xemaRelease": {
-    "sourceDigest": "sha256:3968389805a3386f7699a5663964650254125b4277fe1c89a740296b1a66ebef"
+    "sourceDigest": "sha256:c6b72c65876d81f6a2548bb6da641b908eccb57c9a6ecaa435e31da9b237d75e"
   }
 }

package/src/composed-image-install.mjs

@@ -10,6 +10,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
 import { assertPinsUnchanged } from './composed-image-in-image.mjs';
+import { nodeArch } from './service-image-platform.mjs';
 
 /** The install-flag words that make pnpm copy workspace packages into the store layout. */
 export const injectFlags = () => ['--config.inject-workspace-packages=true', '--config.dedupe-injected-deps=false'];
@@ -26,7 +27,7 @@ export const lockfileOnlyCommand = (filters) => ['pnpm', 'install', '--lockfile-
 /** The production install for `filters` on `linux/<arch>` as `[file, ...args]`. */
 export const productionInstallCommand = (filters, arch) => [
   'pnpm', 'install', '--prod', '--offline', '--ignore-scripts', ...injectFlags(), ...filters, '--frozen-lockfile',
-  '--os=linux', `--cpu=${arch}`, '--libc=glibc',
+  '--os=linux', `--cpu=${nodeArch(arch)}`, '--libc=glibc',
 ];
 
 const defaultExec = (file, args, { cwd }) => execFileSync(file, args, { cwd, stdio: 'inherit' });

package/src/image-cpu-arch.test.mjs

@@ -0,0 +1,67 @@
+// pnpm's `--cpu` takes Node `process.arch` names. The OCI name `amd64` is unknown to it and silently skips every
+// optional x64 binary, so an amd64 image lost its @swc/core binding.
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import test from 'node:test';
+import { injectFlags, productionInstallCommand } from './composed-image-install.mjs';
+import { deployArgs } from './service-image-stage.mjs';
+import { nodeArch } from './service-image-platform.mjs';
+
+test('nodeArch maps amd64 to x64 and arm64 to arm64, and refuses an arch with no row', () => {
+  assert.equal(nodeArch('amd64'), 'x64');
+  assert.equal(nodeArch('arm64'), 'arm64');
+  assert.throws(() => nodeArch('riscv64'), /no Node arch for image arch 'riscv64'/);
+});
+
+test('both install commands pass the Node arch to --cpu and throw for an unknown arch', () => {
+  assert.ok(productionInstallCommand([], 'amd64').includes('--cpu=x64'));
+  assert.ok(productionInstallCommand([], 'arm64').includes('--cpu=arm64'));
+  assert.ok(deployArgs({ service: 's', arch: 'amd64', destination: '/d' }).includes('--cpu=x64'));
+  assert.ok(deployArgs({ service: 's', arch: 'arm64', destination: '/d' }).includes('--cpu=arm64'));
+  assert.throws(() => productionInstallCommand([], 'riscv64'), /no Node arch/);
+  assert.throws(() => deployArgs({ service: 's', arch: 'riscv64', destination: '/d' }), /no Node arch/);
+});
+
+// A fixture shaped like @swc/core: a package whose optional dependencies are one binding per platform.
+function fixturePackage(dir, name, manifest) {
+  const pkg = join(dir, name.replace(/\W/g, '_'));
+  mkdirSync(pkg, { recursive: true });
+  writeFileSync(join(pkg, 'package.json'), JSON.stringify({ name, version: '1.0.0', ...manifest }));
+  return pkg;
+}
+
+test('an amd64 stage installs @swc/core-linux-x64-gnu and no arm64 binding; arm64 the reverse', (t) => {
+  const dir = mkdtempSync(join(tmpdir(), 'xema-cpu-arch-'));
+  t.after(() => rmSync(dir, { recursive: true, force: true }));
+  const x64 = fixturePackage(dir, '@swc/core-linux-x64-gnu', { os: ['linux'], cpu: ['x64'], libc: ['glibc'] });
+  const arm64 = fixturePackage(dir, '@swc/core-linux-arm64-gnu', { os: ['linux'], cpu: ['arm64'], libc: ['glibc'] });
+  const core = fixturePackage(dir, '@swc/core', {
+    optionalDependencies: { '@swc/core-linux-x64-gnu': `file:${x64}`, '@swc/core-linux-arm64-gnu': `file:${arm64}` },
+  });
+  const store = join(dir, 'store');
+  const env = { ...process.env, npm_config_store_dir: store };
+  const run = (cwd, args) => {
+    try {
+      return execFileSync('pnpm', args, { cwd, env, stdio: 'pipe' });
+    } catch (error) {
+      throw new Error(`pnpm ${args.join(' ')}: ${error.stdout}${error.stderr}`);
+    }
+  };
+  for (const arch of ['amd64', 'arm64']) {
+    const app = join(dir, `app-${arch}`);
+    mkdirSync(app);
+    writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'app', version: '1.0.0', dependencies: { '@swc/core': `file:${core}` } }));
+    // Resolve and fetch every platform's tarball into the store, as the real stage's committed lockfile and primed store do.
+    run(app, ['install', '--ignore-workspace', '--ignore-scripts', ...injectFlags(), '--force', '--config.supported-architectures.cpu=*', '--config.supported-architectures.os=*', '--config.supported-architectures.libc=*']);
+    rmSync(join(app, 'node_modules'), { recursive: true, force: true });
+    const [file, ...args] = productionInstallCommand(['--ignore-workspace'], arch);
+    run(app, args);
+    assert.equal(file, 'pnpm');
+    const present = (binding) => existsSync(join(app, 'node_modules', '.pnpm', 'node_modules', '@swc', binding));
+    assert.equal(present('core-linux-x64-gnu'), arch === 'amd64', `${arch}: x64 binding`);
+    assert.equal(present('core-linux-arm64-gnu'), arch === 'arm64', `${arch}: arm64 binding`);
+  }
+});

package/src/service-image-platform.mjs

@@ -17,6 +17,14 @@ export const IMAGE_PLATFORMS = Object.freeze({
   arm64: Object.freeze({ node: 'arm64', elfMachine: 183, prismaBinaryTarget: 'linux-arm64-openssl-3.0.x' }),
 });
 
+/** The Node `process.arch` name of an image arch (`amd64` → `x64`): what pnpm's `--cpu` takes. Refuses an arch with no row. */
+export function nodeArch(arch) {
+  if (!Object.hasOwn(IMAGE_PLATFORMS, arch)) {
+    throw new Error(`no Node arch for image arch '${arch}'; known: ${Object.keys(IMAGE_PLATFORMS).join(', ')}`);
+  }
+  return IMAGE_PLATFORMS[arch].node;
+}
+
 /** `linux/<arch>` → `<arch>`; refuses any platform this recipe does not build. */
 export function platformArch(platform) {
   const match = /^linux\/([a-z0-9]+)$/.exec(String(platform ?? ''));

package/src/service-image-stage.mjs

@@ -31,7 +31,7 @@ import { prismaEnginesDirectories } from './composed-image-prisma-engine.mjs';
 import { copyTool, toolVersion } from './composed-image-stage.mjs';
 import { MANIFEST_FILE, MANIFEST_SCHEMA } from './service-image-in-image.mjs';
 import { LOCKFILE, TOOL_NAME, rootBuildInputNames, sha256File } from './staged-image-proof.mjs';
-import { IMAGE_PLATFORMS, platformArch, scanNatives } from './service-image-platform.mjs';
+import { IMAGE_PLATFORMS, nodeArch, platformArch, scanNatives } from './service-image-platform.mjs';
 
 
 export const DEFAULT_STAGE_DIR = '.xema-stage';
@@ -48,7 +48,7 @@ export const deployArgs = ({ service, arch, destination }) => [
   '--offline',
   '--ignore-scripts',
   '--os=linux',
-  `--cpu=${arch}`,
+  `--cpu=${nodeArch(arch)}`,
   '--libc=glibc',
   '--config.force-legacy-deploy=false',
   '--config.inject-workspace-packages=true',
@@ -85,10 +85,31 @@ export function deployedPackages(appDir) {
 
 const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
 
-async function get(url) {
-  const response = await fetch(url);
-  if (!response.ok) throw new Error(`GET ${url}: ${response.status} ${response.statusText}`);
-  return Buffer.from(await response.arrayBuffer());
+/** What names a failed engine download: the engine, the URL, the engines commit and the target. */
+const engineFailure = ({ url, enginesVersion, binaryTarget }, detail) =>
+  `Prisma schema-engine download failed: ${detail} (url ${url}, engines commit ${enginesVersion}, target ${binaryTarget})`;
+
+/** GET `url` as bytes. A transport failure or a non-2xx status names the engine, URL, commit, target and bytes received. */
+async function getEngine(url, identity) {
+  const named = { url, ...identity };
+  let response;
+  try {
+    response = await fetch(url);
+  } catch (cause) {
+    throw new Error(engineFailure(named, `request failed after 0 bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  if (!response.ok) throw new Error(engineFailure(named, `HTTP ${response.status} ${response.statusText}`));
+  const chunks = [];
+  let received = 0;
+  try {
+    for await (const chunk of response.body) {
+      chunks.push(chunk);
+      received += chunk.length;
+    }
+  } catch (cause) {
+    throw new Error(engineFailure(named, `body read failed after ${received} bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  return Buffer.concat(chunks);
 }
 
 /**
@@ -105,12 +126,32 @@ export const SCHEMA_ENGINE_SHA256 = {
   },
 };
 
+/** In-process memo of the verified engine bytes, by `${enginesVersion}:${binaryTarget}`; holds the promise so concurrent stagers share one download. */
+const verifiedEngines = new Map();
+
+async function fetchVerifiedEngine({ binaryTarget, enginesVersion, expected }) {
+  const url = `${PRISMA_ENGINES_URL}/${enginesVersion}/${binaryTarget}/schema-engine`;
+  const identity = { enginesVersion, binaryTarget };
+  const compressed = await getEngine(`${url}.gz`, identity);
+  let engine;
+  try {
+    engine = gunzipSync(compressed);
+  } catch (cause) {
+    throw new Error(engineFailure({ url: `${url}.gz`, ...identity }, `gunzip failed on ${compressed.length} bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  const actual = sha256(engine);
+  if (actual !== expected) {
+    throw new Error(`Prisma schema-engine ${url}.gz (engines commit ${enginesVersion}, target ${binaryTarget}) unzips to sha256 ${actual}, the committed sha256 is ${expected}`);
+  }
+  return engine;
+}
+
 /**
  * The schema engine at `enginesVersion` (the engines commit the lockfile pins, through the
- * deploy's @prisma/engines-version) for `binaryTarget`, fetched from Prisma for every stage and
+ * deploy's @prisma/engines-version) for `binaryTarget`, fetched from Prisma once per process (the verified bytes are memoised by commit and target) and
  * written to `enginesDir`. The unzipped engine must have the sha256 `checksums` commits for
  * (enginesVersion, binaryTarget); a pair it does not list is refused before any download. There is
- * no cache: Prisma's own downloader keeps one across runs in the home directory, honours mirror
+ * no cross-run cache: Prisma's own downloader keeps one across runs in the home directory, honours mirror
  * variables and retries, so it is not used.
  */
 export async function downloadSchemaEngine({ enginesDir, binaryTarget, enginesVersion, checksums = SCHEMA_ENGINE_SHA256 }) {
@@ -119,9 +160,17 @@ export async function downloadSchemaEngine({ enginesDir, binaryTarget, enginesVe
   if (expected === undefined) {
     throw new Error(`no committed sha256 for the Prisma schema engine at engines commit ${enginesVersion} for ${binaryTarget}: add it to SCHEMA_ENGINE_SHA256 in service-image-stage.mjs`);
   }
-  const url = `${PRISMA_ENGINES_URL}/${enginesVersion}/${binaryTarget}/schema-engine`;
-  const engine = gunzipSync(await get(`${url}.gz`));
-  if (sha256(engine) !== expected) throw new Error(`${url}.gz unzips to sha256 ${sha256(engine)}, the committed sha256 is ${expected}`);
+  const key = `${enginesVersion}:${binaryTarget}`;
+  let verified = verifiedEngines.get(key);
+  if (verified === undefined) {
+    verified = fetchVerifiedEngine({ binaryTarget, enginesVersion, expected });
+    verifiedEngines.set(key, verified);
+    // A failed download is not remembered: the next call fetches again.
+    verified.catch(() => {
+      if (verifiedEngines.get(key) === verified) verifiedEngines.delete(key);
+    });
+  }
+  const engine = await verified;
   const file = join(enginesDir, `schema-engine-${binaryTarget}`);
   writeFileSync(file, engine);
   chmodSync(file, 0o755);

package/src/service-image.test.mjs

@@ -282,8 +282,10 @@ test('downloadSchemaEngine fetches the pinned engine, checks it against the COMM
 
   // A binary whose published .sha256 agrees with it is still refused when it is not the committed one.
   const other = elf(184);
-  serve({ [`${url}.gz`]: gzipSync(other), [`${url}.sha256`]: `${sha256(other)}  x\n` });
-  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }), /unzips to sha256 .*, the committed sha256 is /);
+  const mismatchChecksums = { ['d'.repeat(40)]: { [target]: sha256(engine) } };
+  const mismatchUrl = `${PRISMA_ENGINES_URL}/${'d'.repeat(40)}/${target}/schema-engine`;
+  serve({ [`${mismatchUrl}.gz`]: gzipSync(other), [`${mismatchUrl}.sha256`]: `${sha256(other)}  x\n` });
+  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: 'd'.repeat(40), checksums: mismatchChecksums }), new RegExp(`unzips to sha256 ${sha256(other)}, the committed sha256 is ${sha256(engine)}`));
   t.mock.restoreAll();
   const unfetched = serve({});
   await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: 'c'.repeat(40), checksums }), /no committed sha256 .* engines commit c{40} for linux-arm64/);
@@ -292,6 +294,94 @@ test('downloadSchemaEngine fetches the pinned engine, checks it against the COMM
   await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: 'x', enginesVersion: '1.2.3' }), /is not an engines commit/);
 });
 
+test('downloadSchemaEngine downloads once per engines commit and target, however many services stage', async (t) => {
+  const engine = elf(185);
+  const version = 'e'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  const fetches = [];
+  t.mock.method(globalThis, 'fetch', async (asked) => {
+    fetches.push(asked);
+    return new Response(gzipSync(engine));
+  });
+  const dirs = [temp(), temp(), temp()];
+  await Promise.all(dirs.slice(0, 2).map((enginesDir) => downloadSchemaEngine({ enginesDir, binaryTarget: target, enginesVersion: version, checksums })));
+  await downloadSchemaEngine({ enginesDir: dirs[2], binaryTarget: target, enginesVersion: version, checksums });
+  assert.equal(fetches.length, 1);
+  for (const dir of dirs) assert.deepEqual(fs.readFileSync(path.join(dir, `schema-engine-${target}`)), engine);
+});
+
+test('downloadSchemaEngine names the engine, URL, commit, target, bytes and cause when the body read fails', async (t) => {
+  const engine = elf(186);
+  const version = 'f'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => {
+    let sent = false;
+    const body = new ReadableStream({
+      pull(controller) {
+        if (sent) return controller.error(new TypeError('terminated'));
+        sent = true;
+        controller.enqueue(new Uint8Array(7));
+      },
+    });
+    return new Response(body);
+  });
+  const error = await downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }).then(
+    () => assert.fail('expected a rejection'),
+    (e) => e,
+  );
+  for (const part of ['schema-engine', url, version, target, '7 bytes', 'terminated']) assert.ok(error.message.includes(part), `${part} in: ${error.message}`);
+  assert.equal(error.cause.message, 'terminated');
+});
+
+test('downloadSchemaEngine names the engine, URL, commit and target on a non-2xx status', async (t) => {
+  const engine = elf(187);
+  const version = '1'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => new Response('', { status: 503, statusText: 'Service Unavailable' }));
+  await assert.rejects(
+    downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }),
+    (error) => [`schema-engine`, url, version, target, 'HTTP 503 Service Unavailable'].every((part) => error.message.includes(part)),
+  );
+});
+
+test('downloadSchemaEngine does not remember a failed download', async (t) => {
+  const engine = elf(188);
+  const version = '2'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return calls === 1 ? new Response('', { status: 502, statusText: 'Bad Gateway' }) : new Response(gzipSync(engine));
+  });
+  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }), /HTTP 502/);
+  const enginesDir = temp();
+  await downloadSchemaEngine({ enginesDir, binaryTarget: target, enginesVersion: version, checksums });
+  assert.equal(calls, 2);
+  assert.deepEqual(fs.readFileSync(path.join(enginesDir, `schema-engine-${target}`)), engine);
+});
+
+test('downloadSchemaEngine names the engine, URL, commit, target and cause when the gzip is corrupt', async (t) => {
+  const engine = elf(189);
+  const version = '3'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => new Response(Buffer.from('not gzip at all')));
+  const error = await downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }).then(
+    () => assert.fail('expected a rejection'),
+    (e) => e,
+  );
+  for (const part of ['schema-engine', url, version, target, 'gunzip failed']) assert.ok(error.message.includes(part), `${part} in: ${error.message}`);
+  assert.ok(error.cause instanceof Error);
+  assert.ok(error.message.includes(error.cause.message));
+});
+
 test('the committed engine table has the amd64 and arm64 rows for the engines commit xema-base pins', () => {
   const rows = SCHEMA_ENGINE_SHA256['3c6e192761c0362d496ed980de936e2f3cebcd3a'];
   assert.deepEqual(Object.keys(rows).sort(), ['debian-openssl-3.0.x', 'linux-arm64-openssl-3.0.x']);