@xemahq/xema 0.25.1 → 0.27.0

package/edition/no-follow-file.mjs

@@ -0,0 +1,94 @@
+// Read one file under a root as exact bytes: contained in the root after
+// realpath, opened no-follow and non-blocking (a FIFO cannot stall the open),
+// in a parent directory that is the one the containment check resolved, a
+// regular file within a size bound, and unchanged while it was read. Shared by
+// every reader of operator-stated or evidence inputs; it imports nothing from
+// the library, so any module may use it.
+
+import {
+  closeSync,
+  constants as fsConstants,
+  fstatSync,
+  openSync,
+  readSync,
+  realpathSync,
+  statSync,
+} from 'node:fs';
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+
+export function readBoundedNoFollowFile(root, candidate, label, maxBytes) {
+  if (
+    typeof candidate !== 'string' ||
+    candidate.length === 0 ||
+    isAbsolute(candidate)
+  ) {
+    throw new Error(`${label} must be a repository-relative path`);
+  }
+  const rootReal = realpathSync(root);
+  const lexicalPath = resolve(rootReal, candidate);
+  const directoryReal = realpathSync(dirname(lexicalPath));
+  const rel = relative(rootReal, directoryReal);
+  if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
+    throw new Error(`${label} escapes its evidence root`);
+  }
+  const path = join(directoryReal, basename(lexicalPath));
+  const directoryChecked = statSync(directoryReal);
+  let descriptor;
+  try {
+    descriptor = openSync(
+      path,
+      fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW | fsConstants.O_NONBLOCK,
+    );
+  } catch (error) {
+    if (error?.code === 'ELOOP') {
+      throw new Error(
+        `${label} escapes its evidence root or is not a no-follow file`,
+      );
+    }
+    throw error;
+  }
+  try {
+    // The containment check resolved `directoryReal`; a directory swapped in
+    // under that path between the check and the open is not the one checked.
+    const directoryOpened = statSync(directoryReal);
+    if (
+      directoryOpened.dev !== directoryChecked.dev ||
+      directoryOpened.ino !== directoryChecked.ino ||
+      realpathSync(directoryReal) !== directoryReal
+    ) {
+      throw new Error(`${label}'s directory changed while it was opened`);
+    }
+    const before = fstatSync(descriptor);
+    if (!before.isFile()) {
+      throw new Error(`${label} must be a no-follow regular file`);
+    }
+    if (before.size > maxBytes) {
+      throw new Error(`${label} exceeds ${maxBytes} bytes`);
+    }
+    const bytes = Buffer.alloc(before.size);
+    let offset = 0;
+    while (offset < bytes.length) {
+      const count = readSync(
+        descriptor,
+        bytes,
+        offset,
+        bytes.length - offset,
+        offset,
+      );
+      if (count === 0) {throw new Error(`${label} was truncated while reading`);}
+      offset += count;
+    }
+    if (readSync(descriptor, Buffer.alloc(1), 0, 1, offset) !== 0) {
+      throw new Error(`${label} grew while reading`);
+    }
+    const after = fstatSync(descriptor);
+    for (const field of ['dev', 'ino', 'size', 'mtimeMs', 'ctimeMs']) {
+      if (after[field] !== before[field]) {
+        throw new Error(`${label} changed while reading`);
+      }
+    }
+    return { path, bytes };
+  } finally {
+    closeSync(descriptor);
+  }
+}

package/edition/oci-evidence-verbs.mjs

@@ -0,0 +1,392 @@
+#!/usr/bin/env node
+// `xema edition evidence oci <attach-plan|validate|backfill-plan>`: the OCI
+// form of image source evidence, for a caller that runs `oras` itself.
+//
+//   attach-plan   --evidence-directory <abs dir> [--input <file>]
+//   validate      --evidence-directory <abs dir> --manifest <file> --subject-digest <digest>
+//                 [--input <file> --cosign <abs path> [--now <epoch-ms>]]
+//   backfill-plan --distribution <id>… --producer-repository-root <abs dir>…
+//
+// The library reads and decides; it never writes to a registry. `attach-plan`
+// validates a produced evidence directory and prints the exact `oras attach`
+// operands; `validate` holds a pulled artifact to its manifest (and, with
+// `--input`, to the producer input, the signer and a remaining lifetime);
+// `backfill-plan` joins the committed evidence of the producing repositories
+// to the locked subjects and to what each registry already serves.
+
+import { readFileSync } from 'node:fs';
+import { isAbsolute, join, resolve } from 'node:path';
+
+import { MIN_REMAINING_TTL_SECONDS } from './check-committed-image-evidence.mjs';
+import {
+  expectedProducerSource,
+  indexCommittedProducerEvidence,
+  parseEvidenceCliFlags,
+} from './distribution-evidence-set.mjs';
+import { assertCanonicalDistributionId } from './distribution-paths.mjs';
+import {
+  WorkspaceRoot,
+  configureEditionWorkspaceFromArgv,
+  dataRoot,
+  editionsRoot,
+  isEntryModule,
+} from './edition-workspace.mjs';
+import { SHA256_DIGEST_PATTERN } from './evidence-grammars.mjs';
+import { canonicalJson, loadValidatedLocks } from './image-evidence.mjs';
+import { compareStrings, loadKernelDistributionSchemas, serializeJson } from './lib.mjs';
+import {
+  IMAGE_SOURCE_EVIDENCE_ARTIFACT_TYPE,
+  createRegistryClientResolver,
+  resolveImageSourceEvidenceReferrer,
+  validateMaterializedImageSourceEvidence,
+} from './oci-image-source-evidence.mjs';
+import { REFERRERS_DISTRIBUTION_SPEC } from './oras.mjs';
+import { assertProducerInput, buildImageSourceStatements, cosignVerifyAttestationArgs, createCosignVerifier, readPinnedContractAuthority } from './produce-image-source-evidence.mjs';
+import {
+  producerEvidenceLayerFiles,
+  producerStatementMatchesInput,
+  readProducerEvidenceDirectory,
+} from './producer-evidence-bundle.mjs';
+import { splitImageRef } from './registry-credentials.mjs';
+
+const OCI_CREATED_ANNOTATION = 'org.opencontainers.image.created';
+
+/** What `backfill-plan` reports for one locked subject. */
+export const OciBackfillStatus = Object.freeze({
+  MissingGit: 'missing-git',
+  Present: 'present',
+  Missing: 'missing',
+});
+
+const USAGE = 'usage: xema edition evidence oci <attach-plan|validate|backfill-plan> [options]';
+
+function assertAbsolute(value, label) {
+  if (typeof value !== 'string' || !isAbsolute(value)) {
+    throw new Error(`${label} must be an absolute path`);
+  }
+  return value;
+}
+
+/** One evidence artifact describes one image: one repository, one manifest digest. */
+function inputSubject(input, expectedContractAuthority) {
+  assertProducerInput(input, expectedContractAuthority);
+  const repositories = new Set(input.subjects.map((subject) => subject.imageRepository));
+  const digests = new Set(input.subjects.map((subject) => subject.manifestDigest));
+  if (repositories.size !== 1 || digests.size !== 1) {
+    throw new Error('one OCI evidence artifact requires exactly one image repository and manifest digest');
+  }
+  return { imageRepository: [...repositories][0], subjectDigest: [...digests][0] };
+}
+
+function expectedStatements(input, schemas, expectedContractAuthority) {
+  return new Map(
+    buildImageSourceStatements(input, schemas, expectedContractAuthority).map((statement) => [
+      `${statement.predicate.source.kind}:${statement.predicate.source.id}`,
+      statement,
+    ]),
+  );
+}
+
+/**
+ * The `oras attach` operands of one produced evidence directory. With `input`
+ * every record must be the statement that input builds; without it the image
+ * is the one every record names.
+ */
+export function planOciEvidenceAttach({ evidenceDirectory, input, schemas, expectedContractAuthority }) {
+  assertAbsolute(evidenceDirectory, 'the evidence directory');
+  const evidence = readProducerEvidenceDirectory({ directory: evidenceDirectory, schemas });
+  if (evidence.records.length === 0) {
+    throw new Error(`${evidenceDirectory} holds no producer evidence`);
+  }
+  let image;
+  if (input === undefined) {
+    const [{ artifact }] = evidence.records.map((record) => record.statement.predicate);
+    image = { imageRepository: artifact.imageRepository, subjectDigest: artifact.manifestDigest };
+  } else {
+    image = inputSubject(input, expectedContractAuthority);
+    const expected = expectedStatements(input, schemas, expectedContractAuthority);
+    for (const record of evidence.records) {
+      const expectedStatement = expected.get(record.key);
+      if (expectedStatement === undefined || !producerStatementMatchesInput(record.statement, expectedStatement)) {
+        throw new Error(`${record.key} produced evidence is not bound to the input`);
+      }
+    }
+  }
+  for (const record of evidence.records) {
+    const { artifact } = record.statement.predicate;
+    if (artifact.imageRepository !== image.imageRepository || artifact.manifestDigest !== image.subjectDigest) {
+      throw new Error(`${record.key} evidence is not bound to ${image.imageRepository}@${image.subjectDigest}`);
+    }
+  }
+  const created = evidence.records[0].statement.predicate.producerRun.issuedAt;
+  if (evidence.records.some((record) => record.statement.predicate.producerRun.issuedAt !== created)) {
+    throw new Error('one OCI evidence artifact cannot carry multiple issuance times');
+  }
+  return {
+    schemaVersion: 1,
+    kind: 'oci-attach-plan',
+    evidenceDirectory,
+    ...image,
+    subject: `${image.imageRepository}@${image.subjectDigest}`,
+    artifactType: IMAGE_SOURCE_EVIDENCE_ARTIFACT_TYPE,
+    distributionSpec: REFERRERS_DISTRIBUTION_SPEC,
+    annotations: { [OCI_CREATED_ANNOTATION]: created },
+    files: producerEvidenceLayerFiles(evidence.records),
+  };
+}
+
+/**
+ * Hold a pulled evidence artifact to its manifest. With `input` it also
+ * answers whether the artifact can be reused instead of producing again: the
+ * statements are the ones the input builds, every bundle verifies against the
+ * declared signer, and enough lifetime remains.
+ */
+export function assessMaterializedOciEvidence({
+  evidenceDirectory,
+  manifest,
+  subjectDigest,
+  schemas,
+  input,
+  expectedContractAuthority,
+  runCosign,
+  now = Date.now(),
+}) {
+  assertAbsolute(evidenceDirectory, 'the evidence directory');
+  if (!SHA256_DIGEST_PATTERN.test(subjectDigest ?? '')) {
+    throw new Error('the subject digest must be sha256:<64 hex>');
+  }
+  const evidence = validateMaterializedImageSourceEvidence({
+    directory: evidenceDirectory,
+    manifest,
+    subjectDigest,
+    schemas,
+  });
+  const base = {
+    schemaVersion: 1,
+    kind: 'oci-evidence',
+    subjectDigest,
+    created: evidence.created,
+    subjects: evidence.records.map((record) => record.key),
+  };
+  if (input === undefined) {
+    return { ...base, matchesInput: null, remainingSeconds: null, reusable: null };
+  }
+  if (typeof runCosign !== 'function') {
+    throw new Error('a cosign verifier is required to assess evidence against an input');
+  }
+  const image = inputSubject(input, expectedContractAuthority);
+  if (image.subjectDigest !== subjectDigest) {
+    throw new Error(`the input describes ${image.subjectDigest}, not ${subjectDigest}`);
+  }
+  const expected = expectedStatements(input, schemas, expectedContractAuthority);
+  let matchesInput = evidence.records.length === expected.size;
+  let oldestExpiry = Number.POSITIVE_INFINITY;
+  for (const record of evidence.records) {
+    const expectedStatement = expected.get(record.key);
+    matchesInput &&=
+      expectedStatement !== undefined && producerStatementMatchesInput(record.statement, expectedStatement);
+    oldestExpiry = Math.min(oldestExpiry, Date.parse(record.statement.predicate.producerRun.expiresAt));
+    // A bundle that does not verify is a refusal, never "not reusable".
+    runCosign(
+      cosignVerifyAttestationArgs({
+        bundlePath: join(evidenceDirectory, record.descriptor.bundlePath),
+        signer: input.signer,
+        certificateWorkflowSha: record.statement.predicate.producerRun.producerWorkflowSha,
+        predicateType: record.statement.predicateType,
+      }),
+    );
+  }
+  const remainingSeconds = Math.floor((oldestExpiry - now) / 1000);
+  return {
+    ...base,
+    matchesInput,
+    remainingSeconds,
+    reusable: matchesInput && remainingSeconds >= MIN_REMAINING_TTL_SECONDS,
+  };
+}
+
+/**
+ * For every locked subject of the named editions: its committed evidence and
+ * whether the registry already serves it as a referrer. A `missing` item
+ * carries the attach plan the caller runs.
+ */
+export async function planOciEvidenceBackfill({
+  distributionIds,
+  dataRoot: data,
+  editionsRoot: editions,
+  producerRepositoryRoots,
+  schemas,
+  registryClientFactory,
+}) {
+  const gitIndex = indexCommittedProducerEvidence({
+    producerRepositoryRoots,
+    schemas,
+    onDefect: ({ directory, reason }) => {
+      throw new Error(`Git evidence at ${directory} is unusable: ${reason}`);
+    },
+  });
+  const clientFor = createRegistryClientResolver({ dataRoot: data, factory: registryClientFactory });
+  const work = new Map();
+  const missingGit = new Map();
+  for (const distributionId of distributionIds) {
+    assertCanonicalDistributionId(distributionId);
+    const { imageLock, repositoryPins } = await loadValidatedLocks(distributionId, data, schemas, {
+      editionsRoot: editions,
+    });
+    for (const subject of imageLock.subjects) {
+      const source = expectedProducerSource({
+        subject,
+        repositoryPin: repositoryPins.get(subject.repo),
+        sourceHashAlgorithm: imageLock.sourceHashAlgorithm,
+      });
+      const matches = gitIndex.get(canonicalJson(source)) ?? [];
+      if (matches.length > 1) {
+        throw new Error(`current ${subject.kind}:${subject.id} requires exactly one Git bundle; found ${matches.length}`);
+      }
+      if (matches.length === 0) {
+        missingGit.set(`${subject.kind}:${subject.id}\0${subject.sourceCommit}`, {
+          status: OciBackfillStatus.MissingGit,
+          subjectKind: subject.kind,
+          subjectId: subject.id,
+          imageRepository: subject.imageRef,
+          subjectDigest: null,
+          artifactDigest: null,
+        });
+        continue;
+      }
+      const [match] = matches;
+      const subjectDigest = match.statement.predicate.artifact.manifestDigest;
+      const key = `${subject.imageRef}@${subjectDigest}`;
+      const incumbent = work.get(key);
+      if (incumbent && incumbent.directory !== match.directory) {
+        throw new Error(`${key} resolves to multiple Git evidence directories`);
+      }
+      work.set(key, { subject, directory: match.directory, imageRepository: subject.imageRef, subjectDigest });
+    }
+  }
+  const items = [...missingGit.values()];
+  for (const item of [...work.values()].sort((left, right) =>
+    compareStrings(left.imageRepository, right.imageRepository),
+  )) {
+    const { authority, repository } = splitImageRef(item.imageRepository);
+    const client = clientFor(authority);
+    const served = await client.getManifestByTag(repository, item.subject.contentHash);
+    if (served.digest !== item.subjectDigest) {
+      throw new Error(
+        `${item.imageRepository}:${item.subject.contentHash} serves ${served.digest}, not signed ${item.subjectDigest}`,
+      );
+    }
+    const existing = await resolveImageSourceEvidenceReferrer({
+      client,
+      imageRepository: item.imageRepository,
+      subjectDigest: item.subjectDigest,
+      schemas,
+    });
+    const reported = {
+      subjectKind: item.subject.kind,
+      subjectId: item.subject.id,
+      imageRepository: item.imageRepository,
+      subjectDigest: item.subjectDigest,
+    };
+    items.push(
+      existing === null
+        ? {
+            status: OciBackfillStatus.Missing,
+            ...reported,
+            artifactDigest: null,
+            attach: planOciEvidenceAttach({ evidenceDirectory: item.directory, schemas }),
+          }
+        : { status: OciBackfillStatus.Present, ...reported, artifactDigest: existing.artifactDigest },
+    );
+  }
+  return { schemaVersion: 1, kind: 'oci-backfill-plan', items };
+}
+
+const readJsonFile = (path, label) => {
+  try {
+    return JSON.parse(readFileSync(resolve(path), 'utf8'));
+  } catch (error) {
+    throw new Error(`${label} ${path} is not readable JSON: ${error.message}`);
+  }
+};
+
+async function main() {
+  const [operation, ...argv] = process.argv.slice(2);
+  if (operation === 'attach-plan') {
+    const rest = configureEditionWorkspaceFromArgv(argv, [WorkspaceRoot.Data, WorkspaceRoot.Kernel]);
+    const flags = parseEvidenceCliFlags(rest, { single: ['evidence-directory', 'input'] });
+    const schemas = await loadKernelDistributionSchemas();
+    const input = flags.optional('input');
+    process.stdout.write(
+      serializeJson(
+        planOciEvidenceAttach({
+          evidenceDirectory: flags.require('evidence-directory'),
+          input: input === undefined ? undefined : readJsonFile(input, 'producer input'),
+          schemas,
+          expectedContractAuthority: readPinnedContractAuthority({ dataRoot: dataRoot(), schemas }),
+        }),
+      ),
+    );
+    return;
+  }
+  if (operation === 'validate') {
+    const rest = configureEditionWorkspaceFromArgv(argv, [WorkspaceRoot.Data, WorkspaceRoot.Kernel]);
+    const flags = parseEvidenceCliFlags(rest, {
+      single: ['evidence-directory', 'manifest', 'subject-digest', 'input', 'cosign', 'now'],
+    });
+    const schemas = await loadKernelDistributionSchemas();
+    const input = flags.optional('input');
+    const cosign = flags.optional('cosign');
+    const now = flags.optional('now');
+    if ((input === undefined) !== (cosign === undefined)) {
+      throw new Error('--input and --cosign are given together or not at all');
+    }
+    if (now !== undefined && !/^\d+$/.test(now)) {
+      throw new Error('--now must be epoch milliseconds');
+    }
+    process.stdout.write(
+      serializeJson(
+        assessMaterializedOciEvidence({
+          evidenceDirectory: flags.require('evidence-directory'),
+          manifest: readJsonFile(flags.require('manifest'), 'OCI manifest'),
+          subjectDigest: flags.require('subject-digest'),
+          schemas,
+          input: input === undefined ? undefined : readJsonFile(input, 'producer input'),
+          expectedContractAuthority: readPinnedContractAuthority({ dataRoot: dataRoot(), schemas }),
+          runCosign: cosign === undefined ? undefined : createCosignVerifier(cosign).runCosign,
+          now: now === undefined ? Date.now() : Number(now),
+        }),
+      ),
+    );
+    return;
+  }
+  if (operation === 'backfill-plan') {
+    const rest = configureEditionWorkspaceFromArgv(argv, [
+      WorkspaceRoot.Data,
+      WorkspaceRoot.Editions,
+      WorkspaceRoot.Kernel,
+    ]);
+    const flags = parseEvidenceCliFlags(rest, { repeated: ['distribution', 'producer-repository-root'] });
+    const plan = await planOciEvidenceBackfill({
+      distributionIds: flags.requireMany('distribution'),
+      dataRoot: dataRoot(),
+      editionsRoot: editionsRoot(),
+      producerRepositoryRoots: flags.requireMany('producer-repository-root').map((root) => resolve(root)),
+      schemas: await loadKernelDistributionSchemas(),
+    });
+    process.stdout.write(serializeJson(plan));
+    // 2: something is still to be produced or attached.
+    if (plan.items.some((item) => item.status !== OciBackfillStatus.Present)) {
+      process.exitCode = 2;
+    }
+    return;
+  }
+  throw new Error(USAGE);
+}
+
+if (isEntryModule(import.meta.url)) {
+  main().catch((error) => {
+    console.error(`[oci-evidence] ${error.message}`);
+    process.exitCode = 1;
+  });
+}