@xemahq/repo-build-tooling 0.26.2 → 0.26.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xemahq/repo-build-tooling",
-  "version": "0.26.2",
+  "version": "0.26.3",
   "description": "Dev-time build tooling shared by every Xema repository. Ships as plain ESM, depending only on @xemahq/release-policy for registry access, so the published artifact is the reviewed source.",
   "license": "Apache-2.0",
   "author": "Neuralchowder Inc. <developer@xema.dev> (https://xema.dev)",
@@ -54,6 +54,6 @@
     "test": "node src/scrub-swagger-plugin-paths.mjs --self-test && node --test src/check-workspace-range-matches-local.test.mjs && node --test src/check-no-vendored-deps.test.mjs && node --test src/readiness.test.mjs && node --test src/composed-image.test.mjs && node --test src/composed-image-coordinate-cli.test.mjs && node --test src/declaration-refusal.test.mjs && node --test src/smoke-refusal.test.mjs && node --test src/load-image.test.mjs && node --test src/composed-image-coordinate.test.mjs && node --test src/composed-image-dockerfile.test.mjs && node --test src/service-bootstrap.test.mjs && node --test src/service-bootstrap-protocols.test.mjs && node --test src/composed-image-prebuilt-dockerfile.test.mjs && node --test src/composed-image-in-image-closure.test.mjs && node --test src/composed-image-stage.test.mjs && node --test src/composed-image-stage-tree.test.mjs && node --test src/composed-image-install.test.mjs && node --test src/composed-image-proof.test.mjs && node --test src/composed-image-pnpm-context.test.mjs && node --test src/composed-image-sync-injected.test.mjs && node --test src/composed-image-prisma-check.test.mjs && node --test src/composed-image-prisma-engine.test.mjs && node --test src/composed-migrate.test.mjs && node --test src/promote-image.test.mjs && node --test src/arm64-supplement-needed.test.mjs && node --test src/prepush-sanity.test.mjs && node --test src/service-image.test.mjs && node --test src/install-git-hooks.test.mjs"
   },
   "xemaRelease": {
-    "sourceDigest": "sha256:3968389805a3386f7699a5663964650254125b4277fe1c89a740296b1a66ebef"
+    "sourceDigest": "sha256:e36e1ebf9b6c0b4f12640d1f24212c43bb61d9a5a532279fb4f12a20dde70519"
   }
 }

package/src/service-image-stage.mjs

@@ -85,10 +85,31 @@ export function deployedPackages(appDir) {
 
 const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
 
-async function get(url) {
-  const response = await fetch(url);
-  if (!response.ok) throw new Error(`GET ${url}: ${response.status} ${response.statusText}`);
-  return Buffer.from(await response.arrayBuffer());
+/** What names a failed engine download: the engine, the URL, the engines commit and the target. */
+const engineFailure = ({ url, enginesVersion, binaryTarget }, detail) =>
+  `Prisma schema-engine download failed: ${detail} (url ${url}, engines commit ${enginesVersion}, target ${binaryTarget})`;
+
+/** GET `url` as bytes. A transport failure or a non-2xx status names the engine, URL, commit, target and bytes received. */
+async function getEngine(url, identity) {
+  const named = { url, ...identity };
+  let response;
+  try {
+    response = await fetch(url);
+  } catch (cause) {
+    throw new Error(engineFailure(named, `request failed after 0 bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  if (!response.ok) throw new Error(engineFailure(named, `HTTP ${response.status} ${response.statusText}`));
+  const chunks = [];
+  let received = 0;
+  try {
+    for await (const chunk of response.body) {
+      chunks.push(chunk);
+      received += chunk.length;
+    }
+  } catch (cause) {
+    throw new Error(engineFailure(named, `body read failed after ${received} bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  return Buffer.concat(chunks);
 }
 
 /**
@@ -105,12 +126,32 @@ export const SCHEMA_ENGINE_SHA256 = {
   },
 };
 
+/** In-process memo of the verified engine bytes, by `${enginesVersion}:${binaryTarget}`; holds the promise so concurrent stagers share one download. */
+const verifiedEngines = new Map();
+
+async function fetchVerifiedEngine({ binaryTarget, enginesVersion, expected }) {
+  const url = `${PRISMA_ENGINES_URL}/${enginesVersion}/${binaryTarget}/schema-engine`;
+  const identity = { enginesVersion, binaryTarget };
+  const compressed = await getEngine(`${url}.gz`, identity);
+  let engine;
+  try {
+    engine = gunzipSync(compressed);
+  } catch (cause) {
+    throw new Error(engineFailure({ url: `${url}.gz`, ...identity }, `gunzip failed on ${compressed.length} bytes: ${cause?.message ?? cause}`), { cause });
+  }
+  const actual = sha256(engine);
+  if (actual !== expected) {
+    throw new Error(`Prisma schema-engine ${url}.gz (engines commit ${enginesVersion}, target ${binaryTarget}) unzips to sha256 ${actual}, the committed sha256 is ${expected}`);
+  }
+  return engine;
+}
+
 /**
  * The schema engine at `enginesVersion` (the engines commit the lockfile pins, through the
- * deploy's @prisma/engines-version) for `binaryTarget`, fetched from Prisma for every stage and
+ * deploy's @prisma/engines-version) for `binaryTarget`, fetched from Prisma once per process (the verified bytes are memoised by commit and target) and
  * written to `enginesDir`. The unzipped engine must have the sha256 `checksums` commits for
  * (enginesVersion, binaryTarget); a pair it does not list is refused before any download. There is
- * no cache: Prisma's own downloader keeps one across runs in the home directory, honours mirror
+ * no cross-run cache: Prisma's own downloader keeps one across runs in the home directory, honours mirror
  * variables and retries, so it is not used.
  */
 export async function downloadSchemaEngine({ enginesDir, binaryTarget, enginesVersion, checksums = SCHEMA_ENGINE_SHA256 }) {
@@ -119,9 +160,17 @@ export async function downloadSchemaEngine({ enginesDir, binaryTarget, enginesVe
   if (expected === undefined) {
     throw new Error(`no committed sha256 for the Prisma schema engine at engines commit ${enginesVersion} for ${binaryTarget}: add it to SCHEMA_ENGINE_SHA256 in service-image-stage.mjs`);
   }
-  const url = `${PRISMA_ENGINES_URL}/${enginesVersion}/${binaryTarget}/schema-engine`;
-  const engine = gunzipSync(await get(`${url}.gz`));
-  if (sha256(engine) !== expected) throw new Error(`${url}.gz unzips to sha256 ${sha256(engine)}, the committed sha256 is ${expected}`);
+  const key = `${enginesVersion}:${binaryTarget}`;
+  let verified = verifiedEngines.get(key);
+  if (verified === undefined) {
+    verified = fetchVerifiedEngine({ binaryTarget, enginesVersion, expected });
+    verifiedEngines.set(key, verified);
+    // A failed download is not remembered: the next call fetches again.
+    verified.catch(() => {
+      if (verifiedEngines.get(key) === verified) verifiedEngines.delete(key);
+    });
+  }
+  const engine = await verified;
   const file = join(enginesDir, `schema-engine-${binaryTarget}`);
   writeFileSync(file, engine);
   chmodSync(file, 0o755);

package/src/service-image.test.mjs

@@ -282,8 +282,10 @@ test('downloadSchemaEngine fetches the pinned engine, checks it against the COMM
 
   // A binary whose published .sha256 agrees with it is still refused when it is not the committed one.
   const other = elf(184);
-  serve({ [`${url}.gz`]: gzipSync(other), [`${url}.sha256`]: `${sha256(other)}  x\n` });
-  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }), /unzips to sha256 .*, the committed sha256 is /);
+  const mismatchChecksums = { ['d'.repeat(40)]: { [target]: sha256(engine) } };
+  const mismatchUrl = `${PRISMA_ENGINES_URL}/${'d'.repeat(40)}/${target}/schema-engine`;
+  serve({ [`${mismatchUrl}.gz`]: gzipSync(other), [`${mismatchUrl}.sha256`]: `${sha256(other)}  x\n` });
+  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: 'd'.repeat(40), checksums: mismatchChecksums }), new RegExp(`unzips to sha256 ${sha256(other)}, the committed sha256 is ${sha256(engine)}`));
   t.mock.restoreAll();
   const unfetched = serve({});
   await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: 'c'.repeat(40), checksums }), /no committed sha256 .* engines commit c{40} for linux-arm64/);
@@ -292,6 +294,94 @@ test('downloadSchemaEngine fetches the pinned engine, checks it against the COMM
   await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: 'x', enginesVersion: '1.2.3' }), /is not an engines commit/);
 });
 
+test('downloadSchemaEngine downloads once per engines commit and target, however many services stage', async (t) => {
+  const engine = elf(185);
+  const version = 'e'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  const fetches = [];
+  t.mock.method(globalThis, 'fetch', async (asked) => {
+    fetches.push(asked);
+    return new Response(gzipSync(engine));
+  });
+  const dirs = [temp(), temp(), temp()];
+  await Promise.all(dirs.slice(0, 2).map((enginesDir) => downloadSchemaEngine({ enginesDir, binaryTarget: target, enginesVersion: version, checksums })));
+  await downloadSchemaEngine({ enginesDir: dirs[2], binaryTarget: target, enginesVersion: version, checksums });
+  assert.equal(fetches.length, 1);
+  for (const dir of dirs) assert.deepEqual(fs.readFileSync(path.join(dir, `schema-engine-${target}`)), engine);
+});
+
+test('downloadSchemaEngine names the engine, URL, commit, target, bytes and cause when the body read fails', async (t) => {
+  const engine = elf(186);
+  const version = 'f'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => {
+    let sent = false;
+    const body = new ReadableStream({
+      pull(controller) {
+        if (sent) return controller.error(new TypeError('terminated'));
+        sent = true;
+        controller.enqueue(new Uint8Array(7));
+      },
+    });
+    return new Response(body);
+  });
+  const error = await downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }).then(
+    () => assert.fail('expected a rejection'),
+    (e) => e,
+  );
+  for (const part of ['schema-engine', url, version, target, '7 bytes', 'terminated']) assert.ok(error.message.includes(part), `${part} in: ${error.message}`);
+  assert.equal(error.cause.message, 'terminated');
+});
+
+test('downloadSchemaEngine names the engine, URL, commit and target on a non-2xx status', async (t) => {
+  const engine = elf(187);
+  const version = '1'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => new Response('', { status: 503, statusText: 'Service Unavailable' }));
+  await assert.rejects(
+    downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }),
+    (error) => [`schema-engine`, url, version, target, 'HTTP 503 Service Unavailable'].every((part) => error.message.includes(part)),
+  );
+});
+
+test('downloadSchemaEngine does not remember a failed download', async (t) => {
+  const engine = elf(188);
+  const version = '2'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return calls === 1 ? new Response('', { status: 502, statusText: 'Bad Gateway' }) : new Response(gzipSync(engine));
+  });
+  await assert.rejects(downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }), /HTTP 502/);
+  const enginesDir = temp();
+  await downloadSchemaEngine({ enginesDir, binaryTarget: target, enginesVersion: version, checksums });
+  assert.equal(calls, 2);
+  assert.deepEqual(fs.readFileSync(path.join(enginesDir, `schema-engine-${target}`)), engine);
+});
+
+test('downloadSchemaEngine names the engine, URL, commit, target and cause when the gzip is corrupt', async (t) => {
+  const engine = elf(189);
+  const version = '3'.repeat(40);
+  const target = 'linux-arm64-openssl-3.0.x';
+  const url = `${PRISMA_ENGINES_URL}/${version}/${target}/schema-engine.gz`;
+  const checksums = { [version]: { [target]: sha256(engine) } };
+  t.mock.method(globalThis, 'fetch', async () => new Response(Buffer.from('not gzip at all')));
+  const error = await downloadSchemaEngine({ enginesDir: temp(), binaryTarget: target, enginesVersion: version, checksums }).then(
+    () => assert.fail('expected a rejection'),
+    (e) => e,
+  );
+  for (const part of ['schema-engine', url, version, target, 'gunzip failed']) assert.ok(error.message.includes(part), `${part} in: ${error.message}`);
+  assert.ok(error.cause instanceof Error);
+  assert.ok(error.message.includes(error.cause.message));
+});
+
 test('the committed engine table has the amd64 and arm64 rows for the engines commit xema-base pins', () => {
   const rows = SCHEMA_ENGINE_SHA256['3c6e192761c0362d496ed980de936e2f3cebcd3a'];
   assert.deepEqual(Object.keys(rows).sort(), ['debian-openssl-3.0.x', 'linux-arm64-openssl-3.0.x']);