@xemahq/xema 0.26.0 → 0.28.0

package/edition/verify-image-platforms.mjs

@@ -0,0 +1,495 @@
+#!/usr/bin/env node
+// verify-image-platforms — delivery-time platform verification (plan decision D4).
+//
+//   node repos-infra/xema-distributions/verify-image-platforms.mjs <distributionId>
+//
+// `images.lock.json` DECLARES `requiredPlatforms` per subject. Resolve time
+// cannot assert those platforms exist: arm64 lands ASYNCHRONOUSLY in a separate
+// `release-arm64.yaml` triggered by `workflow_run` after the amd64 build
+// succeeds, and only then does `docker buildx imagetools create` merge it into a
+// multi-arch index on the canonical `:<contentHash>` tag. So the manifest a lock
+// points at legitimately changes shape minutes-to-hours after the lock resolved.
+//
+// DELIVERY is therefore the only place the claim can be checked, and it is
+// checked against the REAL registry, fail-closed. A distribution whose arm64
+// supplement has not landed is simply not deliverable yet. That is what makes
+// standing the arm64 lane down (`SKIP_SUPPLEMENTARY_ARCHES`) SAFE: it surfaces
+// as a loudly-undeliverable distribution rather than a silently amd64-only one.
+//
+// This is NOT a substitute for the offline signed-evidence verifier, and the
+// offline verifier is not a substitute for this. The offline verifier proves the
+// PRODUCER SIGNED a claim about a platform set; this proves the REGISTRY IS
+// SERVING those platforms at the coordinate the lock names. Neither implies the
+// other: evidence can be signed for an image the arm64 merge never reached, and
+// a registry can serve an index nobody attested.
+//
+// What is proven, per required platform, is a fully content-addressed chain:
+//
+//   tag  ->  index bytes (hash-checked against Docker-Content-Digest)
+//        ->  child manifest fetched BY DIGEST (bytes must hash to that digest)
+//        ->  image config fetched BY DIGEST (bytes must hash to that digest)
+//        ->  the config's own os/architecture/variant
+//
+// The last hop is load-bearing, not belt-and-braces: D3 merges a separately
+// built `:<contentHash>-arm64` tag into the index, so a mislabelled or
+// mis-sourced merge is the realistic failure. An index descriptor that CLAIMS
+// linux/arm64 while the image config underneath says amd64 is exactly the bug
+// that would otherwise ship an unbootable appliance. We do not take the
+// descriptor's word for it.
+
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { assertCanonicalDistributionId } from './distribution-paths.mjs';
+import { WorkspaceRoot, configureEditionWorkspaceFromProcess, editionsRoot, isEntryModule } from './edition-workspace.mjs';
+import {
+  loadKernelDistributionSchemas,
+  validateImmutableImageLock,
+} from './lib.mjs';
+import { OrasRegistry } from './oras.mjs';
+import {
+  REGISTRY_CREDENTIALS_FILE,
+  loadRegistryCredentialDeclarations,
+  resolveRegistryCredential,
+  splitImageRef,
+} from './registry-credentials.mjs';
+import { assertReleaseContent, assertReleaseEditionTree } from './release-channel-guard.mjs';
+
+if (isEntryModule(import.meta.url)) {
+  configureEditionWorkspaceFromProcess([WorkspaceRoot.Data, WorkspaceRoot.Editions, WorkspaceRoot.Kernel]);
+}
+
+/**
+ * BuildKit publishes SBOM/provenance attestations as extra children of the
+ * image index, carrying the placeholder platform `unknown/unknown` and the
+ * `vnd.docker.reference.type=attestation-manifest` annotation. They are not
+ * runnable platforms and must never be counted as one — nor may they be
+ * silently ignored under any OTHER shape, which is why an unrecognised child is
+ * a hard failure rather than a skip.
+ */
+const ATTESTATION_REFERENCE_TYPE_ANNOTATION = 'vnd.docker.reference.type';
+const ATTESTATION_REFERENCE_TYPE = 'attestation-manifest';
+const UNKNOWN_PLATFORM_TOKEN = 'unknown';
+
+const DEFAULT_CONCURRENCY = 6;
+
+/**
+ * `linux/arm64` and `linux/arm64/v8` are the SAME platform: v8 is arm64's
+ * implied variant, and containerd's platform matcher normalizes it away. The
+ * three sources compared here disagree about spelling it — a lock declares
+ * `linux/arm64`, an index descriptor written by buildx often carries
+ * `variant: "v8"`, and the image config may carry either — so without this the
+ * verifier would report a missing arm64 that is plainly present.
+ *
+ * This is the ONLY normalization. It is exact and closed: any other variant
+ * difference (e.g. `arm/v7` vs `arm/v6`) stays a real mismatch and fails.
+ */
+const IMPLIED_VARIANTS = new Map([['arm64', 'v8']]);
+
+function normalizedVariant(architecture, variant) {
+  if (!variant) {return undefined;}
+  return IMPLIED_VARIANTS.get(architecture) === variant ? undefined : variant;
+}
+
+/**
+ * One spelling of a platform identity, used for the lock's declaration, the
+ * index descriptor and the image config alike — the comparison is only sound if
+ * all three are normalized the same way.
+ */
+export function platformIdentity(platform) {
+  const variant = normalizedVariant(platform.architecture, platform.variant);
+  return variant
+    ? `${platform.os}/${platform.architecture}/${variant}`
+    : `${platform.os}/${platform.architecture}`;
+}
+
+function isAttestationChild(child) {
+  if (
+    child.annotations?.[ATTESTATION_REFERENCE_TYPE_ANNOTATION] ===
+    ATTESTATION_REFERENCE_TYPE
+  ) {
+    return true;
+  }
+  return (
+    child.platform?.os === UNKNOWN_PLATFORM_TOKEN &&
+    child.platform?.architecture === UNKNOWN_PLATFORM_TOKEN
+  );
+}
+
+function assertChildDescriptor(child, index, label) {
+  if (typeof child !== 'object' || child === null || Array.isArray(child)) {
+    throw new Error(`${label} manifests[${index}] is not an object`);
+  }
+  if (typeof child.digest !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(child.digest)) {
+    throw new Error(
+      `${label} manifests[${index}] has no canonical sha256 digest`,
+    );
+  }
+  if (
+    typeof child.platform !== 'object' ||
+    child.platform === null ||
+    typeof child.platform.os !== 'string' ||
+    typeof child.platform.architecture !== 'string' ||
+    (child.platform.variant !== undefined &&
+      typeof child.platform.variant !== 'string')
+  ) {
+    throw new Error(
+      `${label} manifests[${index}] (${child.digest}) declares no usable platform — ` +
+        'a child that is neither a real platform nor a BuildKit attestation makes the index unreadable',
+    );
+  }
+}
+
+/**
+ * Read the image config a manifest points at and return the platform IT
+ * declares. The config blob is fetched by digest and hash-verified, so this is
+ * the authoritative platform of the actual image content.
+ */
+async function readManifestConfigPlatform(client, repository, manifest, label) {
+  const config = manifest.document.config;
+  if (
+    typeof config !== 'object' ||
+    config === null ||
+    typeof config.digest !== 'string'
+  ) {
+    throw new Error(`${label} has no image config descriptor`);
+  }
+  const bytes = await client.getBlob(repository, config.digest);
+  let document;
+  try {
+    document = JSON.parse(bytes.toString('utf8'));
+  } catch (error) {
+    throw new Error(`${label} image config is not valid JSON: ${error.message}`);
+  }
+  if (
+    typeof document !== 'object' ||
+    document === null ||
+    typeof document.os !== 'string' ||
+    typeof document.architecture !== 'string' ||
+    (document.variant !== undefined && typeof document.variant !== 'string')
+  ) {
+    throw new Error(`${label} image config declares no os/architecture`);
+  }
+  return {
+    os: document.os,
+    architecture: document.architecture,
+    ...(document.variant === undefined ? {} : { variant: document.variant }),
+  };
+}
+
+/**
+ * Resolve the set of platforms a published coordinate ACTUALLY serves.
+ * Every returned platform is proven by a hash-verified image config, never by
+ * an index descriptor's self-declaration alone.
+ */
+export async function resolvePublishedPlatforms({ client, repository, tag, coordinate }) {
+  const top = await client.getManifestByTag(repository, tag);
+  const platforms = [];
+
+  if (!top.isIndex) {
+    // A single-arch manifest. Legitimate before the arm64 supplement merges;
+    // never sufficient for a multi-platform requirement.
+    const platform = await readManifestConfigPlatform(
+      client,
+      repository,
+      top,
+      `${coordinate} manifest ${top.digest}`,
+    );
+    platforms.push({
+      platform,
+      manifestDigest: top.digest,
+      declaredBy: 'image-config',
+    });
+    return { indexDigest: top.digest, isIndex: false, platforms };
+  }
+
+  const children = top.document.manifests;
+  if (!Array.isArray(children) || children.length === 0) {
+    throw new Error(`${coordinate} index ${top.digest} declares no child manifests`);
+  }
+  const label = `${coordinate} index ${top.digest}`;
+  const runnable = [];
+  for (const [index, child] of children.entries()) {
+    assertChildDescriptor(child, index, label);
+    if (isAttestationChild(child)) {continue;}
+    if (
+      child.platform.os === UNKNOWN_PLATFORM_TOKEN ||
+      child.platform.architecture === UNKNOWN_PLATFORM_TOKEN
+    ) {
+      throw new Error(
+        `${label} manifests[${index}] (${child.digest}) is half-unknown ` +
+          `("${platformIdentity(child.platform)}") and carries no attestation annotation`,
+      );
+    }
+    runnable.push(child);
+  }
+  if (runnable.length === 0) {
+    throw new Error(
+      `${label} contains only attestation manifests and no runnable platform`,
+    );
+  }
+
+  for (const child of runnable) {
+    const childManifest = await client.getManifestByDigest(repository, child.digest);
+    if (childManifest.isIndex) {
+      throw new Error(
+        `${label} child ${child.digest} is itself an index; nested indexes are refused`,
+      );
+    }
+    const actual = await readManifestConfigPlatform(
+      client,
+      repository,
+      childManifest,
+      `${coordinate} child ${child.digest}`,
+    );
+    const declared = platformIdentity(child.platform);
+    const proven = platformIdentity(actual);
+    if (declared !== proven) {
+      throw new Error(
+        `${label} child ${child.digest} is described as "${declared}" but its image ` +
+          `config declares "${proven}" — the index misdescribes its own content`,
+      );
+    }
+    platforms.push({
+      platform: actual,
+      manifestDigest: child.digest,
+      declaredBy: 'image-config',
+    });
+  }
+
+  const seen = new Set();
+  for (const entry of platforms) {
+    const identity = platformIdentity(entry.platform);
+    if (seen.has(identity)) {
+      throw new Error(`${label} serves duplicate platform "${identity}"`);
+    }
+    seen.add(identity);
+  }
+  return { indexDigest: top.digest, isIndex: true, platforms };
+}
+
+function missingPlatformMessage({ distributionId, subject, coordinate, missing, present }) {
+  return (
+    `distribution "${distributionId}" is NOT deliverable: subject ` +
+    `${subject.kind}:${subject.id} requires platform${missing.length > 1 ? 's' : ''} ` +
+    `${missing.join(', ')}, but the published manifest at ${coordinate} serves only ` +
+    `${present.length > 0 ? present.join(', ') : '(nothing runnable)'}. ` +
+    'The most likely cause is that the supplementary-architecture lane has not run ' +
+    'or has not finished for this commit: arm64 builds in a separate release-arm64 ' +
+    'workflow triggered by workflow_run AFTER the amd64 build succeeds, and only then ' +
+    'is it merged into the canonical tag as a multi-arch index. Check that ' +
+    "release-arm64 ran green for this repository at commit " +
+    `${subject.sourceCommit}, and that SKIP_SUPPLEMENTARY_ARCHES is not standing the ` +
+    'lane down.'
+  );
+}
+
+async function mapWithConcurrency(items, limit, worker) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (cursor < items.length) {
+      const index = cursor;
+      cursor += 1;
+      results[index] = await worker(items[index], index);
+    }
+  });
+  await Promise.all(runners);
+  return results;
+}
+
+/**
+ * Verify that EVERY subject's published manifest carries EVERY platform its
+ * `requiredPlatforms` declares. Throws on the first uncertainty of any kind.
+ *
+ * Returns a per-subject report suitable for embedding in a delivery manifest.
+ */
+export async function verifyDistributionPlatforms({
+  imageLock,
+  // ONE ROOT FOR BOTH READS IN THIS FILE. This defaulted to
+  // `dataRoot()` while its sibling `loadImageLockForVerification`
+  // ninety lines below defaulted to `editionsRoot()` — two defaults for one
+  // fact, in one file, and they DIFFER the moment a root is configured. The
+  // lock then came from the configured root and the registry CREDENTIALS from
+  // this repository.
+  //
+  // The two halves fail with opposite loudness, which is why only the quiet
+  // one survived: a lock read against the wrong root is an ENOENT, and a
+  // credential read against the wrong root simply authenticates against a
+  // different declaration file and proceeds. `seal-image-digests.mjs` already
+  // compensated at ITS call site (`distributionsRoot: root`) and pinned it
+  // behaviourally in `seal-image-digests.test.mjs`; the other two call sites —
+  // `deliver-images.mjs` and this file's own `main()` — did not, so the
+  // compensation covered one caller rather than the seam.
+  //
+  // With no root configured `editionsRoot() === dataRoot()`, so this is
+  // byte-identical for every unconfigured run.
+  distributionsRoot = editionsRoot(),
+  env = process.env,
+  fetchImpl = globalThis.fetch,
+  // The oras runner `(args, { stdin }) => Buffer`; null runs the pinned oras.
+  run = null,
+  concurrency = DEFAULT_CONCURRENCY,
+  // false (the release): the first subject that fails aborts the run. true:
+  // every subject is verified and the failures are RETURNED beside the
+  // successes, so a caller can name every missing coordinate at once. It never
+  // swallows one: the caller decides what each failure means.
+  collectFailures = false,
+}) {
+  const declarations = loadRegistryCredentialDeclarations(distributionsRoot);
+
+  // Resolve every credential BEFORE the first request. A run that would die on
+  // subject 300 for a missing token must die on subject 0 instead.
+  const clients = new Map();
+  for (const subject of imageLock.subjects) {
+    const { authority } = splitImageRef(subject.imageRef);
+    if (clients.has(authority)) {continue;}
+    const credential = resolveRegistryCredential({ authority, declarations, env });
+    clients.set(
+      authority,
+      new OrasRegistry({ credential, fetchImpl, ...(run ? { run } : {}) }),
+    );
+  }
+
+  const failures = [];
+  const verified = await mapWithConcurrency(
+    imageLock.subjects,
+    concurrency,
+    async (subject) => {
+      try {
+        return await verifySubject(subject);
+      } catch (error) {
+        if (!collectFailures) {throw error;}
+        failures.push({ subject, error });
+        return undefined;
+      }
+    },
+  );
+  const subjects = verified.filter((entry) => entry !== undefined);
+
+  async function verifySubject(subject) {
+    const { authority, repository } = splitImageRef(subject.imageRef);
+    const coordinate = `${subject.imageRef}:${subject.contentHash}`;
+    // A subject that requires no platform cannot be verified — every check
+    // would pass vacuously and the image would ship unexamined.
+    if (
+      !Array.isArray(subject.requiredPlatforms) ||
+      subject.requiredPlatforms.length === 0
+    ) {
+      throw new Error(
+        `subject ${subject.kind}:${subject.id} declares no requiredPlatforms; ` +
+          'there is nothing to verify and a vacuous pass is not a verification',
+      );
+    }
+    const client = clients.get(authority);
+    const published = await resolvePublishedPlatforms({
+      client,
+      repository,
+      tag: subject.contentHash,
+      coordinate,
+    });
+    const present = published.platforms.map((entry) =>
+      platformIdentity(entry.platform),
+    );
+    const presentSet = new Set(present);
+    const required = subject.requiredPlatforms.map(platformIdentity);
+    const missing = required.filter((identity) => !presentSet.has(identity));
+    if (missing.length > 0) {
+      throw new Error(
+        missingPlatformMessage({
+          distributionId: imageLock.distributionId,
+          subject,
+          coordinate,
+          missing,
+          present,
+        }),
+      );
+    }
+    return {
+      subjectKind: subject.kind,
+      subjectId: subject.id,
+      coordinate,
+      manifestDigest: published.indexDigest,
+      multiArchIndex: published.isIndex,
+      requiredPlatforms: required,
+      publishedPlatforms: present.sort(),
+      platformManifestDigests: published.platforms
+        .map((entry) => ({
+          platform: platformIdentity(entry.platform),
+          manifestDigest: entry.manifestDigest,
+        }))
+        .sort((left, right) =>
+          left.platform < right.platform ? -1 : left.platform > right.platform ? 1 : 0,
+        ),
+    };
+  }
+
+  return {
+    distributionId: imageLock.distributionId,
+    verifiedAt: new Date().toISOString(),
+    subjectCount: subjects.length,
+    registries: [...clients.keys()].sort(),
+    subjects,
+    ...(collectFailures ? { failures } : {}),
+  };
+}
+
+export async function loadImageLockForVerification(
+  distributionId,
+  distributionsRoot = editionsRoot(),
+) {
+  assertCanonicalDistributionId(distributionId);
+  const schemas = await loadKernelDistributionSchemas();
+  const editionDir = join(distributionsRoot, distributionId);
+  assertReleaseEditionTree(editionDir);
+  const raw = JSON.parse(readFileSync(join(editionDir, 'images.lock.json'), 'utf8'));
+  assertReleaseContent(raw, `${editionDir}/images.lock.json`);
+  return validateImmutableImageLock(schemas.parseImageLock(raw), schemas);
+}
+
+async function main() {
+  // `--editions-root <dir>` is consumed by the ONE shared resolver in
+  // `lib.mjs` (off `process.argv`), so it must be stripped before the
+  // positional parse below. It was NOT, and the arity check rejected it: the
+  // flag this whole plane is built around could never be passed to this
+  // command at all, while `editionsRoot()` had already honoured it.
+  const [distributionId, ...rest] = process.argv.slice(2);
+  if (!distributionId || rest.length > 0) {
+    throw new Error(
+      'usage: xema edition verify-platforms ' +
+        `<distributionId>`,
+    );
+  }
+  // ONE root, resolved once, passed to BOTH reads explicitly. The defaults
+  // would now agree, but a caller that states the root cannot be made wrong by
+  // a later default drifting — which is exactly how the two in this file came
+  // to disagree.
+  const root = editionsRoot();
+  const imageLock = await loadImageLockForVerification(distributionId, root);
+  const report = await verifyDistributionPlatforms({
+    imageLock,
+    distributionsRoot: root,
+  });
+  const indexes = report.subjects.filter((subject) => subject.multiArchIndex).length;
+  console.log(
+    `[dist:verify-platforms] distribution "${report.distributionId}": ${report.subjectCount} ` +
+      `subjects verified against ${report.registries.join(', ')} ` +
+      `(${indexes} multi-arch indexes, ${report.subjectCount - indexes} single-manifest).`,
+  );
+  for (const subject of report.subjects) {
+    console.log(
+      `    ok ${subject.subjectKind}:${subject.subjectId} -> ${subject.publishedPlatforms.join(', ')}`,
+    );
+  }
+}
+
+if (isEntryModule(import.meta.url)) {
+  main().catch((error) => {
+    console.error(`[dist:verify-platforms] ${error.message}`);
+    console.error(
+      `[dist:verify-platforms] platform verification is fail-closed: nothing above was assumed. ` +
+        `Registry credentials are declared in ${REGISTRY_CREDENTIALS_FILE}.`,
+    );
+    process.exit(1);
+  });
+}

package/edition/why.mjs

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+// why — explain why a biome or platform service is in an edition.
+//
+// Usage:
+//   node why.mjs <edition-id> <biome-id | platform-service-name>
+//
+// Reads ONLY the committed `<edition>/distribution.lock.json`, through the
+// kernel's tolerant reader, and renders its positive-closure `resolution`
+// section: every root class the entity was named in, and the one chain of
+// REQUIRED edges (dependency or resource) that pulled it in from a root. The
+// output is deterministic — it is a rendering of committed bytes, never a
+// re-resolution — so two runs over one lock print the same text.
+//
+// Exit status: 0 when the entity is in the edition and explained, 1 when it is
+// not in the edition or the edition records no provenance (a selector-model
+// edition), 2 on a usage or read error.
+
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { WorkspaceRoot, configureEditionWorkspaceFromProcess, editionsRoot, isEntryModule } from './edition-workspace.mjs';
+import { loadKernelDistributionSchemas } from './lib.mjs';
+import { explainInclusion } from './positive-closure.mjs';
+
+if (isEntryModule(import.meta.url)) {
+  configureEditionWorkspaceFromProcess([WorkspaceRoot.Editions, WorkspaceRoot.Kernel]);
+}
+
+async function main() {
+  const [editionId, name, ...rest] = process.argv.slice(2);
+  if (!editionId || !name || rest.length > 0) {
+    console.error(
+      'usage: node why.mjs <edition-id> <biome-id | platform-service-name>',
+    );
+    return 2;
+  }
+  const schemas = await loadKernelDistributionSchemas();
+  const lock = schemas.parseDistributionLockForReading(
+    JSON.parse(
+      readFileSync(join(editionsRoot(), editionId, 'distribution.lock.json'), 'utf8'),
+    ),
+  );
+  const { found, lines } = explainInclusion(schemas, lock, name);
+  for (const line of lines) {console.log(line);}
+  return found ? 0 : 1;
+}
+
+main().then(
+  (code) => {
+    process.exitCode = code;
+  },
+  (error) => {
+    console.error(`[why] ${error.message}`);
+    process.exitCode = 2;
+  },
+);