@xemahq/xema 0.38.0 → 0.40.0

package/edition/dispatched-runtime-components.mjs

@@ -25,7 +25,7 @@ import { compareStrings } from './string-order.mjs';
  * rostered biomes that dispatch it. Every unresolvable edge is refused, naming
  * the consumer and the target.
  *
- * @param {string} context  the edition the refusal names, e.g. `distribution "core"`
+ * @param {string} context  the edition the refusal names, e.g. `distribution "appliance-base"`
  * @param {object[]} consumers  the rostered index entries
  * @param {object[]} index  the index the targets are resolved against
  */

package/edition/dist-check.mjs

@@ -73,6 +73,7 @@ import {
   resolveComposedProcessStructure,
   resolveImageStructure,
   sourceTreeLocator,
+  supplementalSourcesFor,
 } from './resolve-images.mjs';
 import { withoutSealedDigests } from './seal-image-digests.mjs';
 import { createPinnedSourceAuthority } from './source-authority.mjs';
@@ -301,7 +302,7 @@ export async function checkImagesLock(id, checkSourceEvidence) {
         schemas,
         requiredPlatforms,
         resolveDistributionDeclaredSources(schemas, id),
-        { biomeIndex: pinned.biomeIndex },
+        { biomeIndex: pinned.biomeIndex, supplementalSources: supplementalSourcesFor(runtimeSourceIndex) },
       ),
       schemas,
       requiredPlatforms,
@@ -523,7 +524,7 @@ function discoverDistributionIds() {
  *
  *   Ownership is a property of the platform, not of what an edition preloads.
  *   An earlier design put the claim on each `LockedBiome`, which made a trimmed
- *   edition (`minimal`, `appliance-lean`) leave `kb`/`agent`/… unowned — so an
+ *   edition (`appliance-lean`) leave `kb`/`agent`/… unowned — so an
  *   org admin could install the owning biome from the store and THEN bind
  *   `kb:page.read@1` to a URL they control, silently shadowing the platform
  *   data plane. The table is now built from the FULL biome index and reserves

package/edition/lib.mjs

@@ -2562,7 +2562,7 @@ export function assertDeclaredDatabasesUnowned(entries) {
  *
  *   Ownership is a property of the PLATFORM, not of what an edition happens to
  *   preload. If the table were built from the selected set, a trimmed edition
- *   (`minimal`, `appliance-lean`) that ships without `knowledge-base` would
+ *   (`appliance-lean`) that ships without `knowledge-base` would
  *   leave `kb` unowned — and an org admin could then install `knowledge-base`
  *   from the store and bind `kb:page.read@1` to a URL they control, silently
  *   shadowing the platform data plane. Being AUTHORED by a first-party biome is

package/edition/map-with-concurrency.mjs

@@ -0,0 +1,19 @@
+// map-with-concurrency — run an async worker over a list with a bounded number in flight.
+
+/**
+ * `worker(item, index)` for every item, at most `limit` at a time; the results
+ * keep the items' order. The first rejection rejects the whole call.
+ */
+export async function mapWithConcurrency(items, limit, worker) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (cursor < items.length) {
+      const index = cursor;
+      cursor += 1;
+      results[index] = await worker(items[index], index);
+    }
+  });
+  await Promise.all(runners);
+  return results;
+}

package/edition/oras-process.mjs

@@ -0,0 +1,35 @@
+// oras-process — run the oras binary WITHOUT blocking the event loop.
+//
+// A blocking `execFileSync` turns every caller's worker pool into a queue of
+// one: six "concurrent" subjects still ran their registry calls one after the
+// other. This runner spawns the child and resolves on its exit, so the pool's
+// width is the real concurrency.
+
+import { execFile } from 'node:child_process';
+
+/**
+ * Run `file` with `args`; resolve with stdout bytes. A failure rejects with the
+ * child's stderr text on `error.stderr` (a string), so the caller reads the
+ * registry status from what oras itself wrote.
+ */
+export function runExecutable(file, args, { input, env, timeoutMs, maxBufferBytes }) {
+  return new Promise((resolve, reject) => {
+    const child = execFile(
+      file,
+      args,
+      { env, timeout: timeoutMs, maxBuffer: maxBufferBytes, encoding: 'buffer' },
+      (error, stdout, stderr) => {
+        if (error) {
+          error.stderr = stderr.toString('utf8');
+          reject(error);
+          return;
+        }
+        resolve(stdout);
+      },
+    );
+    // A child that exits without reading its stdin must not crash the process
+    // with EPIPE; the exit status already carries the failure.
+    child.stdin.on('error', () => {});
+    child.stdin.end(input);
+  });
+}

package/edition/oras.mjs

@@ -13,30 +13,36 @@
 //     those bytes; its version must be the one this module's parsing is written
 //     against;
 //   * oras runs under a scrubbed environment (no ambient proxy, config or
-//     credential variable reaches it) and a fresh EMPTY registry config, so
+//     credential variable reaches it) and an EMPTY registry config, so
 //     ambient docker/oras credentials are never used and an authenticated
 //     registry never degrades into an anonymous read;
 //   * the token travels on stdin only (`--password-stdin`), never in argv;
 //   * before oras ever receives a credential, the registry's own `/v2/`
 //     challenge is probed and a token realm on another origin is refused;
-//   * a digest-addressed read whose bytes do not hash to the requested digest
-//     throws; a tag read and a digest read are both resolved to the registry's
-//     descriptor first, and the manifest media type is the descriptor's, never
-//     the manifest body's optional `mediaType` (Helm's has none);
-//   * descriptor sizes are checked BEFORE the fetch, actual lengths after it;
+//   * a manifest or blob is fetched in ONE call (`--descriptor --output <file>`:
+//     the descriptor on stdout, the raw content in a scratch file). Its bytes
+//     must hash to the digest the registry described (and, for a digest read,
+//     to the requested digest), or the read throws; the manifest media type is
+//     the descriptor's, never the manifest body's optional `mediaType` (Helm's
+//     has none);
+//   * declared size, on-disk size and read length are all held to the cap
+//     BEFORE the content is read into memory;
+//   * oras runs asynchronously, so a caller's worker pool really runs in parallel;
+//   * every call of one registry shares one empty registry config for the run;
 //   * a registry status is read only from oras's own error forms; any other
 //     failure carries no status and is thrown as undecided;
 //   * `--plain-http` and `--insecure` are never passed.
 
-import { execFileSync } from 'node:child_process';
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import {
   accessSync,
   constants,
   mkdtempSync,
+  readFileSync,
   realpathSync,
   rmSync,
   statSync,
+  unlinkSync,
   writeFileSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
@@ -46,6 +52,7 @@ import { editionWorkspaceConfig } from './edition-config.mjs';
 import { dataRoot as configuredDataRoot } from './edition-workspace.mjs';
 import { SHA256_DIGEST_PATTERN } from './evidence-grammars.mjs';
 import { readEvidenceTrustPin } from './evidence-trust-pin.mjs';
+import { runExecutable } from './oras-process.mjs';
 import {
   OCI_INDEX_MEDIA_TYPES,
   REGISTRY_CREDENTIALS_FILE,
@@ -248,18 +255,17 @@ export function prepareTrustedOras({ env = process.env, trustedDigests, notice =
 /** The default runner: the trusted oras of the configured data root, under the scrubbed environment. */
 function defaultRunner(trust = configuredOrasTrust) {
   let prepared;
-  return (args, { stdin } = {}) => {
+  return async (args, { stdin } = {}) => {
     if (prepared === undefined) {
       prepared = prepareTrustedOras({ trustedDigests: trust() });
       process.once('exit', prepared.dispose);
     }
     try {
-      return execFileSync(prepared.path(), args, {
+      return await runExecutable(prepared.path(), args, {
         input: stdin,
         env: orasEnvironment(prepared.home),
-        timeout: ORAS_RUN_TIMEOUT_MS,
-        maxBuffer: ORAS_MAX_BUFFER_BYTES,
-        stdio: ['pipe', 'pipe', 'pipe'],
+        timeoutMs: ORAS_RUN_TIMEOUT_MS,
+        maxBufferBytes: ORAS_MAX_BUFFER_BYTES,
       });
     } catch (error) {
       const stderr = error?.stderr ? error.stderr.toString('utf8') : '';
@@ -325,6 +331,38 @@ export function statusOfOrasFailure(text, { ref = '' } = {}) {
   return null;
 }
 
+/**
+ * ONE oras working directory per registry authority per process: the (empty)
+ * registry config every call of that registry points at, and the scratch
+ * files a fetch writes its content to. The config never holds a credential
+ * (that goes to oras on stdin, per call), so sharing it carries no secret; it
+ * replaces a temp directory made and removed around every single call.
+ */
+const registrySessions = new Map();
+
+function registrySession(authority) {
+  let session = registrySessions.get(authority);
+  if (session === undefined) {
+    const directory = mkdtempSync(join(tmpdir(), 'xema-oras-'));
+    const configPath = join(directory, 'registry-config.json');
+    writeFileSync(configPath, '');
+    session = { directory, configPath };
+    registrySessions.set(authority, session);
+    if (registrySessions.size === 1) {
+      process.once('exit', disposeRegistrySessions);
+    }
+  }
+  return session;
+}
+
+/** Remove every registry session directory; called at process exit, and by tests. */
+export function disposeRegistrySessions() {
+  for (const { directory } of registrySessions.values()) {
+    rmSync(directory, { recursive: true, force: true });
+  }
+  registrySessions.clear();
+}
+
 /**
  * A read-only OCI registry client bound to ONE registry authority and ONE
  * resolved credential, doing every operation with the pinned oras.
@@ -417,28 +455,58 @@ export class OrasRegistry {
     return this.#token ? text.split(this.#token).join('***') : text;
   }
 
-  /** One oras invocation with an empty registry config; returns stdout bytes. */
+  /** One oras invocation under this registry's shared empty config; returns stdout bytes. */
   async #oras(subcommand, flags, ref) {
     await this.#ensureRealmIsSameOrigin();
-    const configDirectory = mkdtempSync(join(tmpdir(), 'xema-oras-'));
-    const configPath = join(configDirectory, 'registry-config.json');
+    const { configPath } = registrySession(this.authority);
+    const args = [...subcommand, '--registry-config', configPath];
+    let stdin;
+    if (this.#credential.access === 'authenticated') {
+      args.push('--username', this.#username, '--password-stdin');
+      stdin = this.#token;
+    }
+    args.push(...flags, ref);
+    try {
+      const out = await this.#run(args, { stdin });
+      return Buffer.isBuffer(out) ? out : Buffer.from(out ?? '');
+    } catch (error) {
+      throw this.#failure(ref, error);
+    }
+  }
+
+  /**
+   * ONE oras call that returns the descriptor (stdout) AND the raw content
+   * (written to a scratch file by `--output`): the call that used to be a
+   * descriptor round trip followed by a content round trip. The content is
+   * read into memory only after both its declared and its on-disk size are
+   * inside `maxBytes`.
+   */
+  async #fetchContent(subcommand, ref, maxBytes, acceptDescriptor) {
+    const { directory } = registrySession(this.authority);
+    const outputPath = join(directory, `content-${randomUUID()}`);
     try {
-      writeFileSync(configPath, '');
-      const args = [...subcommand, '--registry-config', configPath];
-      let stdin;
-      if (this.#credential.access === 'authenticated') {
-        args.push('--username', this.#username, '--password-stdin');
-        stdin = this.#token;
+      const described = await this.#oras(subcommand, ['--descriptor', '--output', outputPath], ref);
+      const descriptor = this.#parseDescriptor(described, ref);
+      acceptDescriptor(descriptor);
+      if (descriptor.size > maxBytes) {
+        throw new Error(`${ref} declares ${descriptor.size} bytes, exceeding ${maxBytes}`);
       }
-      args.push(...flags, ref);
-      try {
-        const out = await this.#run(args, { stdin });
-        return Buffer.isBuffer(out) ? out : Buffer.from(out ?? '');
-      } catch (error) {
-        throw this.#failure(ref, error);
+      const onDisk = statSync(outputPath).size;
+      if (onDisk > maxBytes) {
+        throw new Error(`${ref} returned more than ${maxBytes} bytes`);
+      }
+      const bytes = readFileSync(outputPath);
+      const actual = sha256DigestOf(bytes);
+      if (actual !== descriptor.digest) {
+        throw new Error(`${ref} returned content addressed ${actual}, not the digest ${descriptor.digest} it described`);
       }
+      return { descriptor, bytes };
     } finally {
-      rmSync(configDirectory, { recursive: true, force: true });
+      try {
+        unlinkSync(outputPath);
+      } catch {
+        // No content file exists when the fetch itself failed; the failure is already thrown.
+      }
     }
   }
 
@@ -555,88 +623,49 @@ export class OrasRegistry {
   }
 
   /**
-   * The descriptor the registry reports for a manifest reference, refused when
-   * its media type is not an accepted manifest type or it declares more than
-   * the manifest size cap.
+   * Fetch a manifest in ONE call and hold it to everything a separate
+   * descriptor call used to: its media type is an accepted manifest type, its
+   * size is inside the cap, and its bytes hash to the digest the registry
+   * described. `expectedDigest`, when the caller asked by digest, must be that
+   * digest. The media type is the descriptor's, never the body's optional
+   * `mediaType` (Helm's has none).
    */
-  async #manifestDescriptor(ref) {
-    const out = await this.#oras(['manifest', 'fetch'], ['--descriptor'], ref);
-    const descriptor = this.#parseDescriptor(out, ref);
-    if (!ACCEPTED_MANIFEST_MEDIA_TYPES.includes(descriptor.mediaType)) {
-      throw new Error(
-        `${ref} returned unsupported manifest media type "${descriptor.mediaType}"`,
-      );
-    }
-    if (descriptor.size > MAX_MANIFEST_BYTES) {
-      throw new Error(`${ref} declares ${descriptor.size} bytes, exceeding ${MAX_MANIFEST_BYTES}`);
-    }
-    return descriptor;
+  async #manifest(ref, expectedDigest) {
+    const { descriptor, bytes } = await this.#fetchContent(['manifest', 'fetch'], ref, MAX_MANIFEST_BYTES, (described) => {
+      if (!ACCEPTED_MANIFEST_MEDIA_TYPES.includes(described.mediaType)) {
+        throw new Error(`${ref} returned unsupported manifest media type "${described.mediaType}"`);
+      }
+      if (expectedDigest !== undefined && described.digest !== expectedDigest) {
+        throw new Error(`${ref} described ${described.digest}, not the requested ${expectedDigest}`);
+      }
+    });
+    return { ...decodeManifest(bytes, descriptor.mediaType, ref), digest: descriptor.digest, bytes };
   }
 
-  /**
-   * Fetch a manifest by TAG: resolve the descriptor, then read by digest, so the
-   * bytes are always hash-checked against what the registry said the tag is.
-   */
+  /** Fetch a manifest by TAG: one call, the content verified against the digest the registry described. */
   async getManifestByTag(repository, tag) {
-    const descriptor = await this.#manifestDescriptor(`${this.#ref(repository)}:${tag}`);
-    return this.#manifestByDigest(repository, descriptor.digest, descriptor.mediaType);
+    return this.#manifest(`${this.#ref(repository)}:${tag}`);
   }
 
-  /**
-   * Fetch a manifest by DIGEST: the media type is the one the registry reports
-   * for that digest (a manifest body's own `mediaType` is optional, and Helm's
-   * has none), and the bytes must hash to exactly that digest.
-   */
+  /** Fetch a manifest by DIGEST: one call, the bytes must hash to exactly that digest. */
   async getManifestByDigest(repository, digest) {
     if (!SHA256_DIGEST_PATTERN.test(digest)) {
       throw new Error(`"${digest}" is not a canonical lowercase sha256 digest`);
     }
-    const ref = `${this.#ref(repository)}@${digest}`;
-    const descriptor = await this.#manifestDescriptor(ref);
-    if (descriptor.digest !== digest) {
-      throw new Error(`${ref} described ${descriptor.digest}, not the requested ${digest}`);
-    }
-    return this.#manifestByDigest(repository, digest, descriptor.mediaType);
-  }
-
-  async #manifestByDigest(repository, digest, expectedMediaType) {
-    if (!SHA256_DIGEST_PATTERN.test(digest)) {
-      throw new Error(`"${digest}" is not a canonical lowercase sha256 digest`);
-    }
-    const ref = `${this.#ref(repository)}@${digest}`;
-    const bytes = await this.#oras(['manifest', 'fetch'], [], ref);
-    if (bytes.length > MAX_MANIFEST_BYTES) {
-      throw new Error(`${ref} returned more than ${MAX_MANIFEST_BYTES} bytes`);
-    }
-    const actual = sha256DigestOf(bytes);
-    if (actual !== digest) {
-      throw new Error(`${ref} returned content addressed ${actual}, not the requested ${digest}`);
-    }
-    return { ...decodeManifest(bytes, expectedMediaType, ref), digest, bytes };
+    return this.#manifest(`${this.#ref(repository)}@${digest}`, digest);
   }
 
-  /** Fetch a blob by DIGEST; the descriptor size is checked before the fetch, the bytes after. */
+  /** Fetch a blob by DIGEST in one call; its declared and actual size are bounded by `maxBytes`, its bytes must hash to that digest. */
   async getBlob(repository, digest, maxBytes = MAX_CONFIG_BLOB_BYTES) {
     if (!SHA256_DIGEST_PATTERN.test(digest)) {
       throw new Error(`"${digest}" is not a canonical lowercase sha256 digest`);
     }
     const ref = `${this.#ref(repository)}@${digest}`;
-    const described = await this.#oras(['blob', 'fetch'], ['--descriptor'], ref);
-    const descriptor = this.#parseDescriptor(described, ref);
-    if (descriptor.digest !== digest) {
-      throw new Error(`${ref} described ${descriptor.digest}, not the requested ${digest}`);
-    }
-    if (descriptor.size > maxBytes) {
-      throw new Error(`${ref} declares ${descriptor.size} bytes, exceeding ${maxBytes}`);
-    }
-    const bytes = await this.#oras(['blob', 'fetch'], ['--output', '-'], ref);
-    if (bytes.length > maxBytes) {
-      throw new Error(`${ref} returned more than ${maxBytes} bytes`);
-    }
-    const actual = sha256DigestOf(bytes);
-    if (actual !== digest) {
-      throw new Error(`${ref} returned content addressed ${actual}, not the requested ${digest}`);
-    }
+    const { bytes } = await this.#fetchContent(['blob', 'fetch'], ref, maxBytes, (described) => {
+      if (described.digest !== digest) {
+        throw new Error(`${ref} described ${described.digest}, not the requested ${digest}`);
+      }
+    });
     return bytes;
   }
 }

package/edition/resolve-chart-set.mjs

@@ -69,6 +69,7 @@ import {
 import { vendorChartSetConfig } from './edition-config.mjs';
 import { WorkspaceRoot, configureEditionWorkspaceFromProcess, dataRoot, editionsRoot, isEntryModule } from './edition-workspace.mjs';
 import { serializeJson } from './lib.mjs';
+import { mapWithConcurrency } from './map-with-concurrency.mjs';
 import { OrasRegistry } from './oras.mjs';
 import {
   loadRegistryCredentialDeclarations,
@@ -141,6 +142,9 @@ export function selectNewestIndex(candidates) {
 
 const DIGEST_PATTERN = OCI_DIGEST_PATTERN;
 
+/** Registry reads in flight at once while reading a chart set. */
+const CHART_FETCH_CONCURRENCY = 6;
+
 /** The index chart annotation listing the charts the publisher closed the set with. */
 export const INDEX_ROSTER_ANNOTATION = 'xema.dev/chart-set-charts';
 
@@ -226,8 +230,10 @@ export function parseIndexRoster(annotation, label) {
  */
 export async function sealRoster(roster, lookup, label) {
   const sealed = {};
-  for (const [name, member] of Object.entries(roster)) {
-    const registryDigest = await lookup(name, member.version);
+  const members = Object.entries(roster);
+  const registryDigests = await mapWithConcurrency(members, CHART_FETCH_CONCURRENCY, ([name, member]) => lookup(name, member.version));
+  for (const [position, [name, member]] of members.entries()) {
+    const registryDigest = registryDigests[position];
     if (!DIGEST_PATTERN.test(String(registryDigest))) {
       throw new Error(`${label}: registry returned no sha256 manifest digest for ${name}:${member.version}`);
     }
@@ -290,16 +296,14 @@ export function readLock(path = chartSetLockPath()) {
 async function indexCandidates(client, indexName) {
   const repository = registryRepository(indexName);
   const tags = (await client.listTags(repository)).filter(isVersionTag);
-  const candidates = [];
-  for (const tag of tags) {
+  return mapWithConcurrency(tags, CHART_FETCH_CONCURRENCY, async (tag) => {
     const manifest = await client.getManifestByTag(repository, tag);
     const document = JSON.parse(manifest.bytes.toString('utf8'));
-    candidates.push({
+    return {
       tag,
       createdAt: document?.annotations?.['org.opencontainers.image.created'] ?? null,
-    });
-  }
-  return candidates;
+    };
+  });
 }
 
 /** Read every publisher's newest index through `client`: its version, its roster, and the manifest digest the registry served for it. */