@prismatic-io/lux 0.0.1

package/src/orchestrator/candidates.ts

@@ -0,0 +1,683 @@
+import { randomUUID } from "node:crypto";
+import { cp, lstat, mkdir, open, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from "node:path";
+import { temporaryDirectory } from "tempy";
+import { glob } from "tinyglobby";
+import { z } from "zod";
+import { exec } from "../core/exec.js";
+import {
+  type Candidate,
+  type CandidateProposal,
+  CandidateProposalSchema,
+  contentHash,
+  type FileSetSubject,
+  matchesAnyGlob,
+  stableJson,
+  valueHash,
+} from "../core/index.js";
+import { assertCandidateEnvelopeIntegrity } from "./candidate-integrity.js";
+import { claimLockForRecovery, publishOwnedLock, releaseOwnedLock } from "./owned-lock.js";
+
+const normalizedRelative = (path: string): string => {
+  if (isAbsolute(path)) throw new Error(`candidate component must be relative: ${path}`);
+  const normalized = normalize(path).replaceAll("\\", "/").replace(/^\.\//, "");
+  if (!normalized || normalized === ".." || normalized.startsWith("../")) {
+    throw new Error(`candidate component escapes the subject root: ${path}`);
+  }
+  return normalized;
+};
+
+const allowedComponent = (path: string, subject: FileSetSubject): string => {
+  const normalized = normalizedRelative(path);
+  if (!matchesAnyGlob(normalized, subject.mutable) || matchesAnyGlob(normalized, subject.exclude)) {
+    throw new Error(`candidate component is outside the mutable file set: ${normalized}`);
+  }
+  return normalized;
+};
+
+const readTextFile = async (path: string): Promise<string> => {
+  const bytes = await readFile(path);
+  if (bytes.includes(0)) throw new Error(`mutable candidate component is not text: ${path}`);
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    throw new Error(`mutable candidate component is not valid UTF-8: ${path}`);
+  }
+};
+
+export const hashSubjectTree = async (
+  subjectRoot: string,
+  subject: Pick<FileSetSubject, "exclude">,
+): Promise<string> => {
+  const entries: [path: string, type: "directory" | "file", mode: number, hash?: string][] = [];
+  const rootMetadata = await lstat(subjectRoot);
+  if (!rootMetadata.isDirectory() || rootMetadata.isSymbolicLink()) {
+    throw new Error("experiment subject root must be a real directory");
+  }
+  entries.push([".", "directory", rootMetadata.mode]);
+  const inspectDirectory = async (directory: string, prefix = ""): Promise<void> => {
+    const children = await readdir(directory, { withFileTypes: true });
+    for (const entry of children.sort((left, right) => left.name.localeCompare(right.name))) {
+      const path = prefix ? `${prefix}/${entry.name}` : entry.name;
+      if (
+        matchesAnyGlob(path, subject.exclude) ||
+        matchesAnyGlob(`${path}/__descendant__`, subject.exclude)
+      ) {
+        continue;
+      }
+      const fullPath = join(directory, entry.name);
+      const metadata = await lstat(fullPath);
+      if (metadata.isSymbolicLink()) {
+        throw new Error(`experiment subjects cannot contain symbolic links: ${path}`);
+      }
+      if (metadata.isDirectory()) {
+        entries.push([path, "directory", metadata.mode]);
+        await inspectDirectory(fullPath, path);
+        continue;
+      }
+      if (!metadata.isFile()) {
+        throw new Error(`experiment subjects cannot contain special filesystem entries: ${path}`);
+      }
+      entries.push([path, "file", metadata.mode, contentHash(await readFile(fullPath))]);
+    }
+  };
+  await inspectDirectory(subjectRoot);
+  return contentHash(stableJson(entries));
+};
+
+export const captureSeedCandidate = async (
+  subjectRoot: string,
+  subject: FileSetSubject,
+  now: Date = new Date(),
+): Promise<Candidate> => {
+  const files = await glob(subject.mutable, {
+    cwd: subjectRoot,
+    dot: true,
+    onlyFiles: true,
+    followSymbolicLinks: false,
+    ignore: subject.exclude,
+  });
+  if (files.length === 0) {
+    throw new Error(`subject mutable patterns matched no files under ${subjectRoot}`);
+  }
+  const components: Record<string, string> = Object.fromEntries(
+    await Promise.all(
+      files
+        .sort()
+        .map(
+          async (path) =>
+            [normalizedRelative(path), await readTextFile(join(subjectRoot, path))] as const,
+        ),
+    ),
+  );
+  return {
+    schemaVersion: 1,
+    id: "baseline",
+    parentIds: [],
+    createdAt: now.toISOString(),
+    proposer: "source",
+    baseSourceHash: await hashSubjectTree(subjectRoot, subject),
+    candidateHash: valueHash(components),
+    components,
+    sourceBytes: Object.values(components).reduce(
+      (total, content) => total + Buffer.byteLength(content),
+      0,
+    ),
+  };
+};
+
+export const createCandidate = (
+  parent: Candidate,
+  proposal: CandidateProposal,
+  subject: FileSetSubject,
+  options: { id?: string; proposer: string; now?: Date },
+): Candidate => {
+  const parsedProposal = CandidateProposalSchema.parse(proposal);
+  const updates = Object.fromEntries(
+    Object.entries(parsedProposal.updates).map(([path, content]) => {
+      const normalized = allowedComponent(path, subject);
+      if (!Object.hasOwn(parent.components, normalized)) {
+        throw new Error(`candidate proposal cannot add a new component: ${normalized}`);
+      }
+      if (content.includes("\0")) {
+        throw new Error(`candidate proposal contains NUL bytes: ${normalized}`);
+      }
+      return [normalized, content];
+    }),
+  );
+  if (Object.keys(updates).length === 0) {
+    throw new Error("candidate proposal contains no component updates");
+  }
+  const components = { ...parent.components, ...updates };
+  const candidateHash = valueHash(components);
+  return {
+    schemaVersion: 1,
+    id: options.id ?? `candidate-${candidateHash.slice(0, 12)}`,
+    parentIds: [parent.id],
+    createdAt: (options.now ?? new Date()).toISOString(),
+    proposer: options.proposer,
+    baseSourceHash: parent.baseSourceHash,
+    candidateHash,
+    components,
+    sourceBytes: Object.values(components).reduce(
+      (total, content) => total + Buffer.byteLength(content),
+      0,
+    ),
+    ...(parsedProposal.rationale ? { rationale: parsedProposal.rationale } : {}),
+    ...(parsedProposal.usage ? { proposalUsage: parsedProposal.usage } : {}),
+  };
+};
+
+const excludedFromCopy = (root: string, path: string, excludes: readonly string[]): boolean => {
+  const rel = relative(root, path).split(sep).join("/");
+  if (!rel) return false;
+  return matchesAnyGlob(rel, excludes);
+};
+
+export const materializeCandidate = async (
+  sourceRoot: string,
+  subject: FileSetSubject,
+  candidate: Candidate,
+): Promise<string> => {
+  const verified = assertCandidateEnvelopeIntegrity(candidate);
+  const currentHash = await hashSubjectTree(sourceRoot, subject);
+  if (currentHash !== verified.baseSourceHash) {
+    throw new Error("experiment subject source changed before candidate materialization");
+  }
+  const target = temporaryDirectory({ prefix: `lux-candidate-${verified.id}-` });
+  try {
+    await cp(sourceRoot, target, {
+      recursive: true,
+      filter: (source) => !excludedFromCopy(sourceRoot, source, subject.exclude),
+    });
+    for (const [path, content] of Object.entries(verified.components)) {
+      const relativePath = allowedComponent(path, subject);
+      const destination = resolve(target, relativePath);
+      await mkdir(resolve(destination, ".."), { recursive: true });
+      await writeFile(destination, content);
+    }
+    return target;
+  } catch (error) {
+    await rm(target, { recursive: true, force: true });
+    throw error;
+  }
+};
+
+export const validateCandidate = async (
+  materializedRoot: string,
+  subject: FileSetSubject,
+  signal?: AbortSignal,
+  verifyExecutionIdentity?: () => Promise<void>,
+): Promise<void> => {
+  const expectedTreeHash = await hashSubjectTree(materializedRoot, { exclude: [] });
+  for (const validation of subject.validate) {
+    assertValidationArgumentsBound(materializedRoot, validation.args);
+    const validationHome = temporaryDirectory({ prefix: "lux-validation-home-" });
+    const env = Object.fromEntries(
+      ["PATH", "LANG", "LC_ALL", "TERM", "TMPDIR", "CI", "NO_COLOR"]
+        .flatMap((name) =>
+          process.env[name] === undefined ? [] : [[name, process.env[name] as string]],
+        )
+        .concat([["HOME", validationHome]]),
+    );
+    const result = await (async () => {
+      await verifyExecutionIdentity?.();
+      try {
+        return await exec(validation.command, validation.args, {
+          cwd: materializedRoot,
+          timeoutMs: validation.timeoutMs,
+          env,
+          ...(signal ? { signal } : {}),
+        });
+      } finally {
+        await verifyExecutionIdentity?.();
+      }
+    })().finally(() => rm(validationHome, { recursive: true, force: true }));
+    if (result.timedOut || result.exitCode !== 0) {
+      const reason = result.timedOut
+        ? `timed out after ${validation.timeoutMs}ms`
+        : `exited ${result.exitCode}`;
+      throw new Error(
+        `candidate validation '${validation.command}' ${reason}: ${result.stderr.slice(0, 500)}`,
+      );
+    }
+    let currentTreeHash: string;
+    try {
+      currentTreeHash = await hashSubjectTree(materializedRoot, { exclude: [] });
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        `candidate validation '${validation.command}' modified the materialized candidate; ` +
+          `validators must be observational/read-only: ${reason}`,
+      );
+    }
+    if (currentTreeHash !== expectedTreeHash) {
+      throw new Error(
+        `candidate validation '${validation.command}' modified the materialized candidate; ` +
+          "validators must be observational/read-only",
+      );
+    }
+  }
+};
+
+const validationArgumentValues = (argument: string): string[] => {
+  const values = [argument];
+  const equals = argument.indexOf("=");
+  if (equals >= 0) values.push(argument.slice(equals + 1));
+  const longOption = /^--[^:=]+:(.+)$/.exec(argument)?.[1];
+  if (longOption) values.push(longOption);
+  const shortOption = /^-[A-Za-z](.+)$/.exec(argument)?.[1];
+  if (shortOption) values.push(shortOption);
+  if (argument.startsWith("@") && argument.length > 1) values.push(argument.slice(1));
+  return [...new Set(values)];
+};
+
+export const assertValidationArgumentsBound = (
+  subjectRoot: string,
+  args: readonly string[],
+): void => {
+  for (const argument of args) {
+    if (/(?:file:|[a-z][a-z0-9+.-]*:\/\/)/i.test(argument)) {
+      throw new Error(`candidate validation argument references external behavior: ${argument}`);
+    }
+    if (argument.startsWith("-") && /[\\/]/.test(argument)) {
+      throw new Error(
+        `candidate validation path values must be standalone relative arguments: ${argument}`,
+      );
+    }
+    for (const value of validationArgumentValues(argument)) {
+      if (!value || value.startsWith("-")) continue;
+      if (
+        isAbsolute(value) ||
+        /^[A-Za-z]:[\\/]/.test(value) ||
+        /^\\\\/.test(value) ||
+        /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value)
+      ) {
+        throw new Error(`candidate validation argument references external behavior: ${argument}`);
+      }
+      const path = resolve(subjectRoot, value);
+      const fromSubject = relative(resolve(subjectRoot), path);
+      if (fromSubject === ".." || fromSubject.startsWith(`..${sep}`) || isAbsolute(fromSubject)) {
+        throw new Error(
+          `candidate validation argument escapes the identity-bound subject: ${argument}`,
+        );
+      }
+    }
+  }
+};
+
+const ApplyJournalEntrySchema = z
+  .object({
+    destination: z.string(),
+    temporary: z.string(),
+    backup: z.string(),
+    mode: z.number().int().nonnegative(),
+    originalHash: z.string(),
+  })
+  .strict();
+
+const ApplyJournalSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    subjectRoot: z.string(),
+    candidateHash: z.string(),
+    status: z.enum(["staging", "committing", "committed"]),
+    entries: z.array(ApplyJournalEntrySchema),
+  })
+  .strict();
+type ApplyJournal = z.infer<typeof ApplyJournalSchema>;
+type ApplyJournalEntry = z.infer<typeof ApplyJournalEntrySchema>;
+
+const ApplyLockSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    pid: z.number().int().positive(),
+    hostname: z.string(),
+    acquiredAt: z.string(),
+    subjectRoot: z.string(),
+    candidateHash: z.string(),
+    stageRoot: z.string(),
+    lockToken: z.exactOptional(z.string()),
+    status: z.enum(["running", "committed"]).default("running"),
+  })
+  .strict();
+type ApplyLock = z.infer<typeof ApplyLockSchema>;
+type OwnedApplyLock = ApplyLock & { lockToken: string };
+
+const applyPaths = (subjectRoot: string): { root: string; parent: string; lockPath: string } => {
+  const root = resolve(subjectRoot);
+  const parent = dirname(root);
+  return {
+    root,
+    parent,
+    lockPath: join(parent, `.lux-apply-${basename(root)}.lock`),
+  };
+};
+
+const syncDirectory = async (path: string): Promise<void> => {
+  let handle: Awaited<ReturnType<typeof open>> | undefined;
+  try {
+    handle = await open(path, "r");
+    await handle.sync();
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    if (process.platform === "win32" && code === "EPERM") return;
+    if (code !== "EINVAL" && code !== "ENOTSUP" && code !== "EISDIR") throw error;
+  } finally {
+    await handle?.close();
+  }
+};
+
+const writeDurableExclusive = async (
+  path: string,
+  contents: string | Uint8Array,
+  mode?: number,
+): Promise<void> => {
+  const handle = await open(path, "wx", mode);
+  try {
+    await handle.writeFile(contents);
+    if (mode !== undefined) await handle.chmod(mode);
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+};
+
+const writeJsonDurableAtomic = async (path: string, value: unknown): Promise<void> => {
+  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  await writeDurableExclusive(temporary, `${JSON.stringify(value, null, 2)}\n`);
+  try {
+    await rename(temporary, path);
+    await syncDirectory(dirname(path));
+  } finally {
+    await rm(temporary, { force: true });
+  }
+};
+
+const assertRecoveryPath = (
+  path: string,
+  parent: string,
+  expectedPrefix: string,
+  description: string,
+): void => {
+  const fromParent = relative(parent, path);
+  if (
+    fromParent === "" ||
+    fromParent === ".." ||
+    fromParent.startsWith(`..${sep}`) ||
+    isAbsolute(fromParent) ||
+    dirname(fromParent) !== "." ||
+    !basename(fromParent).startsWith(expectedPrefix)
+  ) {
+    throw new Error(`interrupted candidate ${description} is outside its expected location`);
+  }
+};
+
+const assertJournalContained = (
+  journal: ApplyJournal,
+  expectedRoot: string,
+  stageRoot: string,
+): void => {
+  if (resolve(journal.subjectRoot) !== expectedRoot) {
+    throw new Error("interrupted candidate journal targets a different subject root");
+  }
+  for (const entry of journal.entries) {
+    const fromRoot = relative(expectedRoot, resolve(entry.destination));
+    if (
+      fromRoot === "" ||
+      fromRoot === ".." ||
+      fromRoot.startsWith(`..${sep}`) ||
+      isAbsolute(fromRoot)
+    ) {
+      throw new Error("interrupted candidate destination escapes the subject root");
+    }
+    for (const [kind, path] of [
+      ["backup", entry.backup],
+      ["temporary", entry.temporary],
+    ] as const) {
+      const fromStage = relative(stageRoot, resolve(path));
+      if (
+        fromStage === "" ||
+        fromStage === ".." ||
+        fromStage.startsWith(`..${sep}`) ||
+        isAbsolute(fromStage) ||
+        dirname(fromStage) !== "."
+      ) {
+        throw new Error(`interrupted candidate ${kind} escapes its staging location`);
+      }
+    }
+  }
+};
+
+const restoreJournalEntry = async (entry: ApplyJournalEntry): Promise<void> => {
+  const backup = await readFile(entry.backup);
+  if (contentHash(backup) !== entry.originalHash) {
+    throw new Error(`candidate rollback backup failed its integrity check: ${entry.backup}`);
+  }
+  const temporary = join(
+    dirname(entry.destination),
+    `.lux-apply-restore-${basename(entry.destination)}-${randomUUID()}.tmp`,
+  );
+  await writeDurableExclusive(temporary, backup, entry.mode);
+  try {
+    await rename(temporary, entry.destination);
+    await syncDirectory(dirname(entry.destination));
+  } finally {
+    await rm(temporary, { force: true });
+  }
+};
+
+const ownerIsAlive = (owner: ApplyLock): boolean => {
+  if (owner.hostname !== hostname()) return true;
+  try {
+    process.kill(owner.pid, 0);
+    return true;
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    if (code === "ESRCH") return false;
+    if (code === "EPERM") return true;
+    throw error;
+  }
+};
+
+const recoverInterruptedApply = async (
+  lockPath: string,
+  expectedRoot: string,
+): Promise<{ committedCandidateHash?: string }> => {
+  const claim = await claimLockForRecovery(lockPath);
+  if (!claim) return {};
+  let owner: ApplyLock;
+  try {
+    try {
+      owner = ApplyLockSchema.parse(JSON.parse(await readFile(claim.path, "utf8")));
+    } catch {
+      throw new Error(`candidate application lock is present but unreadable: ${lockPath}`);
+    }
+    if (resolve(owner.subjectRoot) !== expectedRoot) {
+      throw new Error("candidate application lock targets a different subject root");
+    }
+    if (ownerIsAlive(owner)) {
+      throw new Error(`candidate application is already owned by process ${owner.pid}`);
+    }
+    const { parent } = applyPaths(expectedRoot);
+    const stageRoot = resolve(owner.stageRoot);
+    assertRecoveryPath(stageRoot, parent, `.lux-apply-stage-${basename(expectedRoot)}-`, "stage");
+    let journal: ApplyJournal | undefined;
+    try {
+      journal = ApplyJournalSchema.parse(
+        JSON.parse(await readFile(join(stageRoot, "transaction.json"), "utf8")),
+      );
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw new Error(`candidate application journal is unreadable: ${stageRoot}`);
+      }
+    }
+    if (journal) {
+      assertJournalContained(journal, expectedRoot, stageRoot);
+      if (journal.candidateHash !== owner.candidateHash) {
+        throw new Error("candidate application lock and journal disagree");
+      }
+      if (owner.status === "committed" && journal.status !== "committed") {
+        throw new Error("candidate application lock claims an incomplete journal is committed");
+      }
+      if (journal.status === "committing") {
+        for (const entry of [...journal.entries].reverse()) await restoreJournalEntry(entry);
+      }
+    }
+    await rm(stageRoot, { recursive: true, force: true });
+    await claim.removeLock();
+    await syncDirectory(parent);
+    return journal?.status === "committed" || owner.status === "committed"
+      ? { committedCandidateHash: owner.candidateHash }
+      : {};
+  } finally {
+    await claim.release();
+  }
+};
+
+const acquireApply = async (
+  subjectRoot: string,
+  candidateHash: string,
+): Promise<
+  | { alreadyCommitted: true }
+  | { alreadyCommitted: false; lockPath: string; stageRoot: string; owner: OwnedApplyLock }
+> => {
+  const { root, parent, lockPath } = applyPaths(subjectRoot);
+  const stageRoot = join(
+    parent,
+    `.lux-apply-stage-${basename(root)}-${process.pid}-${randomUUID()}`,
+  );
+  let owner: OwnedApplyLock;
+  try {
+    owner = await publishOwnedLock(lockPath, {
+      schemaVersion: 1,
+      pid: process.pid,
+      hostname: hostname(),
+      acquiredAt: new Date().toISOString(),
+      subjectRoot: root,
+      candidateHash,
+      stageRoot,
+      status: "running",
+    });
+    await syncDirectory(parent);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    const recovered = await recoverInterruptedApply(lockPath, root);
+    if (recovered.committedCandidateHash === candidateHash) {
+      return { alreadyCommitted: true };
+    }
+    return acquireApply(subjectRoot, candidateHash);
+  }
+  try {
+    await mkdir(stageRoot);
+    return { alreadyCommitted: false, lockPath, stageRoot, owner };
+  } catch (error) {
+    await releaseOwnedLock(lockPath, owner.lockToken);
+    throw error;
+  }
+};
+
+export const applyCandidate = async (
+  subjectRoot: string,
+  subject: FileSetSubject,
+  candidate: Candidate,
+): Promise<void> => {
+  assertCandidateEnvelopeIntegrity(candidate);
+  const rootMetadata = await lstat(resolve(subjectRoot));
+  if (!rootMetadata.isDirectory() || rootMetadata.isSymbolicLink()) {
+    throw new Error("candidate subject root must be a real directory");
+  }
+  const ownership = await acquireApply(subjectRoot, candidate.candidateHash);
+  if (ownership.alreadyCommitted) {
+    for (const [path, content] of Object.entries(candidate.components)) {
+      const destination = resolve(subjectRoot, allowedComponent(path, subject));
+      if ((await readTextFile(destination)) !== content) {
+        throw new Error(
+          "recovered candidate transaction was marked committed but its files do not match",
+        );
+      }
+    }
+    return;
+  }
+  const { lockPath, stageRoot, owner } = ownership;
+  const journalPath = join(stageRoot, "transaction.json");
+  let journal: ApplyJournal | undefined;
+  let preserveRecovery = false;
+  try {
+    const currentHash = await hashSubjectTree(subjectRoot, subject);
+    if (currentHash !== candidate.baseSourceHash) {
+      throw new Error(
+        `subject source changed since candidate creation (${candidate.baseSourceHash} != ${currentHash})`,
+      );
+    }
+    const entries: ApplyJournalEntry[] = [];
+    for (const [path, content] of Object.entries(candidate.components)) {
+      const destination = resolve(subjectRoot, allowedComponent(path, subject));
+      const metadata = await lstat(destination);
+      if (!metadata.isFile() || metadata.isSymbolicLink()) {
+        throw new Error(`candidate destination must be a regular file: ${path}`);
+      }
+      const index = entries.length;
+      const backup = join(stageRoot, `${index}.backup`);
+      const temporary = join(stageRoot, `${index}.next`);
+      const original = await readFile(destination);
+      await writeDurableExclusive(backup, original, metadata.mode);
+      await writeDurableExclusive(temporary, content, metadata.mode);
+      entries.push({
+        destination,
+        temporary,
+        backup,
+        mode: metadata.mode,
+        originalHash: contentHash(original),
+      });
+    }
+    journal = {
+      schemaVersion: 1,
+      subjectRoot: resolve(subjectRoot),
+      candidateHash: candidate.candidateHash,
+      status: "staging",
+      entries,
+    };
+    await writeJsonDurableAtomic(journalPath, journal);
+    const beforeCommitHash = await hashSubjectTree(subjectRoot, subject);
+    if (beforeCommitHash !== candidate.baseSourceHash) {
+      throw new Error("subject source changed while candidate application was being staged");
+    }
+    journal.status = "committing";
+    await writeJsonDurableAtomic(journalPath, journal);
+    for (const entry of journal.entries) {
+      await rename(entry.temporary, entry.destination);
+      await syncDirectory(dirname(entry.destination));
+    }
+    journal.status = "committed";
+    await writeJsonDurableAtomic(journalPath, journal);
+    owner.status = "committed";
+    await writeJsonDurableAtomic(lockPath, owner);
+  } catch (error) {
+    const rollbackErrors: unknown[] = [];
+    if (journal?.status === "committing") {
+      for (const entry of [...journal.entries].reverse()) {
+        try {
+          await restoreJournalEntry(entry);
+        } catch (rollbackError) {
+          rollbackErrors.push(rollbackError);
+        }
+      }
+    }
+    if (rollbackErrors.length > 0) {
+      preserveRecovery = true;
+      throw new AggregateError(
+        [error, ...rollbackErrors],
+        `candidate application failed and rollback was incomplete; recoverable backups remain at ${stageRoot}`,
+      );
+    }
+    throw error;
+  } finally {
+    if (!preserveRecovery) {
+      await rm(stageRoot, { recursive: true, force: true });
+      await releaseOwnedLock(lockPath, owner.lockToken);
+      await syncDirectory(dirname(lockPath));
+    }
+  }
+};