@xemahq/xema 0.38.0 → 0.40.0

package/edition/resolve-images.mjs

@@ -35,6 +35,7 @@
 import { execFileSync } from 'node:child_process';
 import { writeFileSync } from 'node:fs';
 import { join, relative, sep } from 'node:path';
+import { setImmediate as yieldToEventLoop } from 'node:timers/promises';
 
 import { assertBuildLedgerComplete } from './build-ledger-completeness.mjs';
 import { BUILD_LEDGER_FILE, readBuildLedger } from './build-ledger.mjs';
@@ -45,7 +46,7 @@ import {
   distributionOutputPath,
   readBoundDistributionFile,
 } from './distribution-paths.mjs';
-import { editionWorkspaceConfig } from './edition-config.mjs';
+import { editionWorkspaceConfig, workspaceSection } from './edition-config.mjs';
 import { editionPinnedInputs } from './edition-pinned-inputs.mjs';
 import { dataRoot, editionsRoot, WorkspaceRoot, configureEditionWorkspaceFromProcess, isEntryModule, sourcePath, sourcesRoot } from './edition-workspace.mjs';
 import {
@@ -398,6 +399,14 @@ function missingArtifactDependencyMessage(lock, serviceName, binding, biomeIndex
  * `dispatchesRuntimeComponents`: the runtime components that biome launches,
  * whose images enter this roster as runtime-component rows although their
  * biomes are not rostered (see `dispatched-runtime-components.mjs`).
+ *
+ * `options.supplementalSources` is the edition workspace's
+ * `runtimeImageSources.supplementalSources`: the declarations the source
+ * index's supplemental subjects were built from, carrying each one's
+ * `launchedBy`. A subject launched by a biome ships, owned by that biome, in
+ * every edition whose roster contains it; one a platform service binds ships
+ * with that service. A declared subject with neither is refused by name: it
+ * could never enter any edition, and it used to be dropped silently.
  */
 export function resolveImageStructure(
   lock,
@@ -405,7 +414,7 @@ export function resolveImageStructure(
   enums,
   requiredPlatforms,
   declaredSources,
-  { biomeIndex } = {},
+  { biomeIndex, supplementalSources } = {},
 ) {
   if (!Array.isArray(requiredPlatforms) || requiredPlatforms.length === 0) {
     throw new Error(
@@ -420,6 +429,12 @@ export function resolveImageStructure(
         'the runtime components a rostered biome dispatches, so a roster without it can omit images',
     );
   }
+  if (!Array.isArray(supplementalSources)) {
+    throw new Error(
+      `distribution "${lock.distributionId}" resolved its images without the supplemental source declarations — ` +
+        'they say which biome launches a supplemental subject, so a roster without them can omit images',
+    );
+  }
   const subjects = new Map();
   const webBiomes = [];
   const platformServiceBindings = [];
@@ -436,6 +451,28 @@ export function resolveImageStructure(
     }
     return pin;
   };
+  // A supplemental subject of the runtime-image source index, built at its
+  // repository pin, owned by whatever brings it into this edition.
+  const addSupplementalSubject = (supplemental, owners) => {
+    const repository = requireRepositoryPin(supplemental.repo);
+    if (supplemental.sourceCommit !== repository.sourceCommit) {
+      throw new Error(
+        `runtime subject "${supplemental.kind}:${supplemental.id}" source commit does not match its repository pin`,
+      );
+    }
+    addSubject(subjects, {
+      kind: supplemental.kind,
+      id: supplemental.id,
+      imageRef: supplemental.imageRef,
+      repo: supplemental.repo,
+      path: supplemental.path,
+      sourceCommit: repository.sourceCommit,
+      sourceManifestHash: supplemental.sourceManifestHash,
+      sourceManifestPath: supplemental.sourceManifestPath,
+      requiredPlatforms,
+      owners,
+    });
+  };
 
   for (const biome of lock.biomes) {
     // WHERE THIS BIOME'S IMAGES LIVE, and WHICH COMMIT THEY ARE BUILT FROM.
@@ -674,6 +711,50 @@ export function resolveImageStructure(
     ]),
   );
 
+  // Supplemental subjects a BIOME launches (`launchedBy`): sealed, owned by
+  // that biome, in every edition whose roster contains it. The catalogue
+  // checks run for every edition, rostered launcher or not.
+  const boundSubjects = new Set(
+    runtimeSourceIndex.platformServiceBindings.map((binding) => bindingKey(binding)),
+  );
+  const launchers = new Map(
+    supplementalSources
+      .filter((source) => source.launchedBy !== undefined)
+      .map((source) => [subjectKey(source), source.launchedBy]),
+  );
+  for (const key of launchers.keys()) {
+    if (!supplementalSubjects.has(key)) {
+      throw new Error(
+        `supplemental source "${key.replace('\0', ':')}" declares a launcher, but the runtime image source index ` +
+          'has no such supplemental subject — rebuild the index from the current declarations',
+      );
+    }
+  }
+  for (const [key, supplemental] of supplementalSubjects) {
+    const launcher = launchers.get(key);
+    if (launcher === undefined) {
+      if (!boundSubjects.has(key)) {
+        throw new Error(
+          `supplemental subject "${supplemental.kind}:${supplemental.id}" has neither a launchedBy nor a ` +
+            'platform-service binding, so no edition could ever carry it — declare what launches it ' +
+            '(runtimeImageSources.supplementalSources[].launchedBy) or bind it to a platform service',
+        );
+      }
+      continue;
+    }
+    if (!biomeIndex.some((entry) => entry.id === launcher.biomeId && entry.source === undefined)) {
+      throw new Error(
+        `supplemental subject "${supplemental.kind}:${supplemental.id}" is launched by biome ` +
+          `"${launcher.biomeId}", which the first-party biome index does not carry`,
+      );
+    }
+    if (lock.biomes.some((biome) => biome.id === launcher.biomeId)) {
+      addSupplementalSubject(supplemental, [
+        { kind: enums.RuntimeImageOwnerKind.Biome, biomeId: launcher.biomeId },
+      ]);
+    }
+  }
+
   for (const service of lock.platformServices ?? []) {
     let binding;
     // The discriminator is the DECLARED runtime binding, never "does the lock
@@ -726,29 +807,12 @@ export function resolveImageStructure(
       const key = bindingKey(binding);
       const supplemental = supplementalSubjects.get(key);
       if (supplemental) {
-        const repository = requireRepositoryPin(supplemental.repo);
-        if (supplemental.sourceCommit !== repository.sourceCommit) {
-          throw new Error(
-            `runtime subject "${supplemental.kind}:${supplemental.id}" source commit does not match its repository pin`,
-          );
-        }
-        addSubject(subjects, {
-          kind: supplemental.kind,
-          id: supplemental.id,
-          imageRef: supplemental.imageRef,
-          repo: supplemental.repo,
-          path: supplemental.path,
-          sourceCommit: repository.sourceCommit,
-          sourceManifestHash: supplemental.sourceManifestHash,
-          sourceManifestPath: supplemental.sourceManifestPath,
-          requiredPlatforms,
-          owners: [
-            {
-              kind: enums.RuntimeImageOwnerKind.PlatformService,
-              platformService: service.name,
-            },
-          ],
-        });
+        addSupplementalSubject(supplemental, [
+          {
+            kind: enums.RuntimeImageOwnerKind.PlatformService,
+            platformService: service.name,
+          },
+        ]);
       } else {
         const target = subjects.get(key);
         if (!target) {
@@ -1523,6 +1587,18 @@ function snapshotImageInputs(schemas, { inputs, lock }) {
   };
 }
 
+/**
+ * The supplemental source declarations an image resolve reads beside
+ * `runtimeSourceIndex`. An index with no supplemental subjects (a customer
+ * edition's own rows) needs none, and its workspace need not declare the
+ * section; any other reads the edition workspace's.
+ */
+export function supplementalSourcesFor(runtimeSourceIndex) {
+  return runtimeSourceIndex.supplementalSubjects.length === 0
+    ? []
+    : workspaceSection('runtimeImageSources', 'resolving supplemental image subjects').supplementalSources;
+}
+
 /**
  * The image ROWS of one edition: its subjects (hashed at the commits the lock
  * pins), its platform-service bindings and its bundled web biomes, after every
@@ -1531,7 +1607,7 @@ function snapshotImageInputs(schemas, { inputs, lock }) {
  * of one (a customer's own rows, merged with the catalogue's) parses the lock
  * it completes. `resolveImages` is these rows parsed as the whole lock.
  */
-export async function resolveImageRows(id, { inputs, lock } = {}) {
+export async function resolveImageRows(id, { inputs, lock, onSubjectHashed } = {}) {
   assertCanonicalDistributionId(id);
   if ((inputs === undefined) !== (lock === undefined)) {
     throw new Error('resolveImages takes the snapshot `inputs` and its distribution `lock` together, or neither');
@@ -1555,7 +1631,7 @@ export async function resolveImageRows(id, { inputs, lock } = {}) {
       schemas,
       requiredPlatforms,
       declaredSources,
-      { biomeIndex },
+      { biomeIndex, supplementalSources: supplementalSourcesFor(runtimeSourceIndex) },
     ),
     schemas,
     requiredPlatforms,
@@ -1587,8 +1663,8 @@ export async function resolveImageRows(id, { inputs, lock } = {}) {
   // made that edition-independent quantity depend on the edition, which failed
   // in both directions at once:
   //   • it is UNCOMPUTABLE for an edition that does not itself resolve the
-  //     source repo — `appliance-lean` and `minimal` do not ship the xema-store
-  //     biome, so re-resolving either one threw here and their locks could not
+  //     source repo — `appliance-lean` does not ship the xema-store
+  //     biome, so re-resolving it threw here and their locks could not
   //     be regenerated at all, even though the image they name is the very one
   //     the other six editions resolve fine;
   //   • had it not thrown it would have been WORSE, silently minting a second
@@ -1629,14 +1705,15 @@ export async function resolveImageRows(id, { inputs, lock } = {}) {
           ...(crossRepoInputsByConsumer.get(`${subject.repo}\u0000`) ?? []),
           ...(crossRepoInputsByConsumer.get(`${subject.repo}\u0000${subject.id}`) ?? []),
         ];
-  const subjects = structure.subjects.map((subject) => {
+  const subjects = [];
+  for (const subject of structure.subjects) {
     const sourceCommit = commitsBefore.get(subject.repo);
     const repoDirectory = locate(subject.repo).dir;
     const servicePath = subjectServicePath(subject, sourceCommit, locate);
     const { sourceManifestPath: ignoredSourceManifestPath, ...lockedSubject } =
       subject;
     void ignoredSourceManifestPath;
-    return {
+    const hashed = {
       ...lockedSubject,
       sourceCommit,
       contentHash: exactImageBuildInputHash(
@@ -1652,7 +1729,15 @@ export async function resolveImageRows(id, { inputs, lock } = {}) {
         imageInputSourceHashAlgorithm(),
       ),
     };
-  });
+    subjects.push(hashed);
+    // The hash is the last thing a registry probe waits for: hand the subject
+    // over now, and let the event loop run so the probes of the subjects
+    // already hashed progress while the rest hash (hashing is synchronous).
+    if (onSubjectHashed !== undefined) {
+      onSubjectHashed(hashed);
+      await yieldToEventLoop();
+    }
+  }
   // Release lane only: a develop snapshot or customer overlay is not covered by
   // the committed ledger.
   if (inputs === undefined) {
@@ -1690,8 +1775,8 @@ export async function resolveImageRows(id, { inputs, lock } = {}) {
 }
 
 /** One edition's COMPLETE image lock: its rows, parsed and validated as the kernel's image lock. */
-export async function resolveImages(id, { inputs, lock } = {}) {
-  const { subjects, platformServiceBindings, webBiomes, contractAuthority, commits } = await resolveImageRows(id, { inputs, lock });
+export async function resolveImages(id, { inputs, lock, onSubjectHashed } = {}) {
+  const { subjects, platformServiceBindings, webBiomes, contractAuthority, commits } = await resolveImageRows(id, { inputs, lock, onSubjectHashed });
   const schemas = await loadKernelDistributionSchemas();
   const imageLock = validateImmutableImageLock(
     schemas.parseImageLock({

package/edition/verify-image-platforms.mjs

@@ -46,6 +46,7 @@ import {
   loadKernelDistributionSchemas,
   validateImmutableImageLock,
 } from './lib.mjs';
+import { mapWithConcurrency } from './map-with-concurrency.mjs';
 import { OrasRegistry } from './oras.mjs';
 import {
   REGISTRY_CREDENTIALS_FILE,
@@ -284,18 +285,20 @@ function missingPlatformMessage({ distributionId, subject, coordinate, missing,
   );
 }
 
-async function mapWithConcurrency(items, limit, worker) {
-  const results = new Array(items.length);
-  let cursor = 0;
-  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
-    while (cursor < items.length) {
-      const index = cursor;
-      cursor += 1;
-      results[index] = await worker(items[index], index);
-    }
-  });
-  await Promise.all(runners);
-  return results;
+/**
+ * One `OrasRegistry` per registry authority of `subjects`, every credential
+ * resolved BEFORE the first request: a run that would die on subject 300 for a
+ * missing token must die on subject 0 instead.
+ */
+export function registryClientsFor(subjects, { declarations, env, fetchImpl, run }) {
+  const clients = new Map();
+  for (const subject of subjects) {
+    const { authority } = splitImageRef(subject.imageRef);
+    if (clients.has(authority)) {continue;}
+    const credential = resolveRegistryCredential({ authority, declarations, env });
+    clients.set(authority, new OrasRegistry({ credential, fetchImpl, ...(run ? { run } : {}) }));
+  }
+  return clients;
 }
 
 /**
@@ -335,30 +338,29 @@ export async function verifyDistributionPlatforms({
   // successes, so a caller can name every missing coordinate at once. It never
   // swallows one: the caller decides what each failure means.
   collectFailures = false,
+  // `(line) => void`: one line per verified subject, with its elapsed ms; null is silent.
+  progress = null,
 }) {
   const declarations = loadRegistryCredentialDeclarations(distributionsRoot);
 
-  // Resolve every credential BEFORE the first request. A run that would die on
-  // subject 300 for a missing token must die on subject 0 instead.
-  const clients = new Map();
-  for (const subject of imageLock.subjects) {
-    const { authority } = splitImageRef(subject.imageRef);
-    if (clients.has(authority)) {continue;}
-    const credential = resolveRegistryCredential({ authority, declarations, env });
-    clients.set(
-      authority,
-      new OrasRegistry({ credential, fetchImpl, ...(run ? { run } : {}) }),
-    );
-  }
+  const clients = registryClientsFor(imageLock.subjects, { declarations, env, fetchImpl, run });
 
   const failures = [];
   const verified = await mapWithConcurrency(
     imageLock.subjects,
     concurrency,
-    async (subject) => {
+    async (subject, index) => {
+      const startedAt = Date.now();
       try {
-        return await verifySubject(subject);
+        const verifiedSubject = await verifySubject(subject);
+        progress?.(
+          `[seal] ${index + 1}/${imageLock.subjects.length} ${subject.kind}:${subject.id} ok ${Date.now() - startedAt}ms`,
+        );
+        return verifiedSubject;
       } catch (error) {
+        progress?.(
+          `[seal] ${index + 1}/${imageLock.subjects.length} ${subject.kind}:${subject.id} FAILED ${Date.now() - startedAt}ms`,
+        );
         if (!collectFailures) {throw error;}
         failures.push({ subject, error });
         return undefined;

package/edition/workspace-sections.mjs

@@ -73,6 +73,24 @@ function subjectKind(value, where) {
   return value;
 }
 
+/**
+ * What launches a supplemental subject that no platform service binds: the
+ * owner its image-lock row is sealed under. Closed, one member until a second
+ * real case exists: a BIOME, which launches the subject's image itself, so
+ * the subject ships in every edition whose roster contains that biome.
+ */
+export const SupplementalLauncherKind = Object.freeze({
+  Biome: 'biome',
+});
+
+const LAUNCHER_KINDS = Object.values(SupplementalLauncherKind);
+
+function supplementalLauncher(value, where) {
+  exactObject(value, where, ['kind', 'biomeId']);
+  if (!LAUNCHER_KINDS.includes(value.kind)) {fail(`${where}.kind`, `must be one of ${LAUNCHER_KINDS.join(', ')}`);}
+  return Object.freeze({ kind: value.kind, biomeId: text(value.biomeId, `${where}.biomeId`) });
+}
+
 export function parseFirstPartySources(value) {
   const where = '"firstPartySources"';
   if (!isObject(value) || Object.keys(value).length === 0) {fail(where, 'must be a non-empty {repo handle: source id} object');}
@@ -138,13 +156,16 @@ export function parseRuntimeImageSources(value) {
   const where = '"runtimeImageSources"';
   exactObject(value, where, ['supplementalSources', 'crossRepoBuildInputs', 'buildStepContextDirectories']);
   const supplementalSources = list(value.supplementalSources, `${where}.supplementalSources`, (item, at) => {
-    exactObject(item, at, ['kind', 'id', 'repo', 'path', 'sourceManifestPath']);
+    exactObject(item, at, ['kind', 'id', 'repo', 'path', 'sourceManifestPath'], ['launchedBy']);
     return {
       kind: subjectKind(item.kind, `${at}.kind`),
       id: text(item.id, `${at}.id`),
       repo: repoHandle(item.repo, `${at}.repo`),
       path: text(item.path, `${at}.path`),
       sourceManifestPath: text(item.sourceManifestPath, `${at}.sourceManifestPath`),
+      ...(Object.hasOwn(item, 'launchedBy')
+        ? { launchedBy: supplementalLauncher(item.launchedBy, `${at}.launchedBy`) }
+        : {}),
     };
   });
   uniqueBy(supplementalSources, `${where}.supplementalSources`, (item) => `${item.kind}/${item.id}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xemahq/xema",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/xema-dev/xema-cli.git",
@@ -92,6 +92,6 @@
     "typecheck": "tsc --noEmit"
   },
   "xemaRelease": {
-    "sourceDigest": "sha256:6ceee7eaad77b410aeb26998f61349944a56481549d146a3bc42a7a325ab3890"
+    "sourceDigest": "sha256:e3d77e6ac0d871437aec5ec5749446091e4a65c86ce04df7900b22954ae71308"
   }
 }