@runuai/host 0.9.148 → 0.9.149

package/lib/secrets-proxy/runtime.ts

@@ -0,0 +1,582 @@
+/**
+ * The secrets proxy as the runner runs it (ADR-153 §3): the persistent CA,
+ * the leaf cache, per-task placeholders, the engine and its audit log,
+ * composed once per runner process.
+ *
+ * What a task may use comes from outside, through TaskProxyPolicySource:
+ * the declarations work resolves each task's effective policy (binding ∩
+ * schema observation) and the current real values. This module only turns
+ * that into what the engine consults per request, so a policy change (a key
+ * withheld, a domain dropped) applies to the next request with nothing to
+ * invalidate here.
+ */
+
+import { AsyncLocalStorage } from "node:async_hooks";
+import { promises as dns } from "node:dns";
+import { readFileSync } from "node:fs";
+import { BlockList, isIP } from "node:net";
+import { networkInterfaces } from "node:os";
+import { join } from "node:path";
+
+import { createAuditWriter } from "./audit";
+import { createLeafCertificateCache, type LeafCertificateCache } from "./ca";
+import { loadOrCreateCertificateAuthorities, type CertificateAuthorities, type SealedText } from "./ca-store";
+import { createSecretsProxy, type ProxyKeyRef, type ProxyTaskState, type SecretsProxy } from "./engine";
+import { isProxiableDomain } from "./hostname";
+import type { TaskEnvHead } from "../project-env-task-policy";
+import { createPlaceholderStore, type PlaceholderShape } from "./placeholders";
+
+const HOUR_MS = 60 * 60 * 1_000;
+/** How often the runner reviews its CA dates and retries a failed bundle
+ *  delivery, so an idle runner mints the successor (and gets it to running
+ *  tasks) when its window opens, not at the next handshake or task start,
+ *  which it may not see for months. */
+const TRUST_REVIEW_EVERY_MS = HOUR_MS;
+
+/** One key a task currently has in `proxy` mode (effective policy). */
+export interface ProxiedKeyPolicy {
+  projectId: string;
+  key: string;
+  domains: readonly string[];
+  /** The repo's `@placeholder` shape, if it declared one. */
+  placeholder?: PlaceholderShape | null;
+  /** The head the key's value must have (ADR-153 §4). A source that knows
+   *  heads gives the value only at it, so a value read after the head moved
+   *  is withheld until the engine learns the new one. */
+  head?: TaskEnvHead;
+}
+
+export interface TaskProxyPolicySource {
+  /** The task's keys in proxy mode right now, or null when the task is not
+   *  (or no longer) served — an unknown, stopped or revoked generation. */
+  proxiedKeys(taskId: string): readonly ProxiedKeyPolicy[] | null;
+  /** The key's current real value, read per request; null withholds. */
+  realValue(taskId: string, ref: { projectId: string; key: string; head?: TaskEnvHead }): string | null;
+  /** Every name the task does not get as its value (proxied or withheld),
+   *  when the source knows more than its proxied keys. */
+  notDelivered?(taskId: string): readonly string[] | null;
+}
+
+/** ADR-153's kill switch: `UAI_SECRETS_PROXY=off` stops the proxy, and the
+ *  policy then withholds proxied keys (never delivers them). */
+export function secretsProxyEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  return env.UAI_SECRETS_PROXY?.trim().toLowerCase() !== "off";
+}
+
+/** Upstream resolution for a runner whose /etc/hosts the task does NOT share
+ *  (Docker and Apple hosts): the OS resolver, which also honours the
+ *  platform's scoped and VPN resolvers. */
+export async function resolveWithSystem(hostname: string): Promise<string[]> {
+  const answers = await dns.lookup(hostname, { all: true, verbatim: true });
+  return answers.map((answer) => answer.address);
+}
+
+/** Where a machine's real (uplink) DNS servers are listed: systemd-resolved's
+ *  own upstream list first, then the classic file. */
+export const UPLINK_RESOLV_FILES: readonly string[] = ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"];
+
+/** This host's own interface addresses: a resolver listening on one of them
+ *  (dnsmasq on the private address) is as local as one on loopback. */
+function interfaceAddresses(): string[] {
+  return Object.values(networkInterfaces())
+    .flat()
+    .flatMap((entry) => (entry === undefined ? [] : [entry.address]));
+}
+
+/** Matches a server on this host in any textual form — `0:0:0:0:0:0:0:1`,
+ *  `::ffff:7f00:1` — since BlockList compares parsed addresses. IPv4 and
+ *  IPv6 rules stay in separate lists (see docs/agent-notes.md). */
+function localServerMatcher(localAddresses: readonly string[]): (address: string) => boolean {
+  const v4 = new BlockList();
+  v4.addSubnet("127.0.0.0", 8, "ipv4");
+  v4.addSubnet("0.0.0.0", 8, "ipv4");
+  const v6 = new BlockList();
+  // ::, ::1 and the IPv4-compatible forms of the above.
+  v6.addSubnet("::", 96, "ipv6");
+  v6.addSubnet("::ffff:127.0.0.0", 104, "ipv6");
+  v6.addSubnet("::ffff:0.0.0.0", 104, "ipv6");
+  for (const raw of localAddresses) {
+    const address = raw.split("%")[0] ?? "";
+    const family = isIP(address);
+    if (family === 4) {
+      v4.addAddress(address, "ipv4");
+      v6.addAddress(`::ffff:${address}`, "ipv6");
+    } else if (family === 6) {
+      v6.addAddress(address, "ipv6");
+    }
+  }
+  return (raw) => {
+    const address = raw.split("%")[0] ?? "";
+    const family = isIP(address);
+    // Anything unparsable counts as local: it is never queried.
+    if (family === 4) return v4.check(address, "ipv4");
+    if (family === 6) return v6.check(address, "ipv6");
+    return true;
+  };
+}
+
+/**
+ * The machine's uplink DNS servers: the `nameserver`s of the first file in
+ * `files` that lists any server not on this host. A server on this host never
+ * counts — systemd-resolved's 127.0.0.53 stub, or dnsmasq, answers from
+ * /etc/hosts, which on a machine carries the task's pins. Empty when there is
+ * no uplink server at all.
+ */
+export function uplinkDnsServers(
+  files: readonly string[] = UPLINK_RESOLV_FILES,
+  localAddresses: readonly string[] = interfaceAddresses(),
+): string[] {
+  const isLocalServer = localServerMatcher(localAddresses);
+  for (const file of files) {
+    let text: string;
+    try {
+      text = readFileSync(file, "utf8");
+    } catch {
+      continue;
+    }
+    const servers: string[] = [];
+    for (const line of text.split("\n")) {
+      const [keyword, address] = line.trim().split(/\s+/);
+      if (keyword !== "nameserver" || address === undefined || isIP(address) === 0) continue;
+      if (!isLocalServer(address) && !servers.includes(address)) servers.push(address);
+    }
+    if (servers.length > 0) return servers;
+  }
+  return [];
+}
+
+/** Upstream resolution for a runner that shares /etc/hosts with the task (a
+ *  machine's sidecar): DNS queried directly, so the task's pins — which
+ *  point proxied names at loopback — can never answer for the upstream.
+ *  The servers are re-read per lookup (DHCP can change them); none, which
+ *  would leave only a stub that reads /etc/hosts, fails the lookup closed. */
+export function resolverIgnoringHostsFile(
+  servers: () => readonly string[] = () => uplinkDnsServers(),
+): (hostname: string) => Promise<string[]> {
+  return async (hostname) => {
+    const list = servers();
+    if (list.length === 0) throw new Error("no uplink DNS server: refusing to resolve through a local stub");
+    const resolver = new dns.Resolver();
+    resolver.setServers([...list]);
+    const [v4, v6] = await Promise.allSettled([resolver.resolve4(hostname), resolver.resolve6(hostname)]);
+    const addresses = [
+      ...(v4.status === "fulfilled" ? v4.value : []),
+      ...(v6.status === "fulfilled" ? v6.value : []),
+    ];
+    if (addresses.length === 0) throw new Error("no A or AAAA records");
+    return addresses;
+  };
+}
+
+export interface SecretsProxyRuntimeDeps {
+  /** The runner's data dir: the CA and placeholders live under
+   *  `secrets-proxy/`, the audit log under `logs/`. */
+  dataDir: string;
+  policy: TaskProxyPolicySource;
+  seal(plaintext: string): SealedText;
+  open(ct: Buffer, nonce: Buffer): string;
+  /** Required, because the right answer depends on where the runner sits:
+   *  `resolveWithSystem` on a Docker or Apple host, `resolverIgnoringHostsFile()`
+   *  in a machine's sidecar, whose /etc/hosts carries the task's pins. */
+  resolveHost: (hostname: string) => Promise<string[]>;
+  now?: () => number;
+  log?: Pick<Console, "warn">;
+}
+
+export interface SecretsProxyRuntime {
+  readonly proxy: SecretsProxy;
+  /** What every container or VM start installs into the task's trust store:
+   *  the signing CA and, inside its last 90 days, its successor (PEM). */
+  caBundlePem(): string;
+  /** The runner's hook for keeping running tasks' trust current, level-
+   *  triggered: the listener installs the bundle it is given into every
+   *  running task and resolves once they all have it. It is called at
+   *  subscription — so a successor minted while the runner was down, during
+   *  its start, or just before a crash still reaches running tasks — and
+   *  whenever the bundle changes; a delivery that throws or rejects is
+   *  logged and retried at the next hourly review until one succeeds.
+   *
+   *  Each delivery gets its own signal, aborted if the watcher is
+   *  unsubscribed or the runtime closes while that delivery runs. A finished
+   *  delivery's signal never aborts: nothing of it is left to stop, and a
+   *  handler left on it must not run later, outside the delivery that
+   *  registered it.
+   *
+   *  Teardown is quiescent: the returned unsubscribe, and close(), abort the
+   *  delivery under way and resolve only once it has ended, so an old
+   *  runtime can never install its bundle over a replacement's. A listener
+   *  must stop promptly when its signal aborts.
+   *  Once close() has begun, watchTrust delivers nothing. Unsubscribe and
+   *  close() are always quiescent, so neither may be called from inside a
+   *  listener (or from work it started while its delivery lasts): there it
+   *  would wait for itself, or for a delivery waiting on it. Such a call is
+   *  rejected and changes nothing. */
+  watchTrust(listener: (bundlePem: string, signal: AbortSignal) => void | Promise<void>): () => Promise<void>;
+  /** Delivers the current bundle to every watcher again, though they all
+   *  acknowledged it: a task's own install failed, or it was still starting,
+   *  so the bundle is not everywhere after all. */
+  requestTrustDelivery(): void;
+  /** The placeholder the /env renderer hands a task for a key the policy
+   *  serves it in proxy mode; throws for any other key or task. */
+  placeholderFor(taskId: string, key: ProxiedKeyPolicy): string;
+  /** The canonical domains a task's proxied keys may reach — what its
+   *  /etc/hosts block and redirect chain are reconciled to. */
+  domainsFor(taskId: string): string[];
+  /** The task's proxy-mode keys, each with the canonical, proxiable domains
+   *  it may reach (what its agents are told). */
+  proxiedKeysFor(taskId: string): { projectId: string; key: string; domains: string[] }[];
+  /** Stop (or any revocation): end the task's connections. Its placeholders
+   *  stay — they are the task's for its whole life, across stops, resumes and
+   *  runner restarts — and while the policy source does not serve the task,
+   *  nothing is served or minted for it. */
+  stopTask(taskId: string): void;
+  /** Deletion: stopTask, then forget the task's placeholders. Call it once
+   *  the policy source no longer serves the task, so nothing mints them
+   *  again. */
+  dropTask(taskId: string): void;
+  close(): Promise<void>;
+}
+
+function proxiableDomains(keys: readonly ProxiedKeyPolicy[]): Set<string> {
+  const domains = new Set<string>();
+  for (const key of keys) {
+    // isProxiableDomain also requires the name to be canonical.
+    for (const domain of key.domains) if (isProxiableDomain(domain)) domains.add(domain);
+  }
+  return domains;
+}
+
+/** One string per policy snapshot, so derived state is rebuilt only when the
+ *  policy actually changes. */
+function policySignature(keys: readonly ProxiedKeyPolicy[]): string {
+  return JSON.stringify(keys.map((key) => [key.projectId, key.key, [...key.domains], key.placeholder ?? null, key.head ?? null]));
+}
+
+/** One trust listener call, while its delivery lasts, linked to the
+ *  delivery it started inside, if any: a delivery begun from within another
+ *  replaces the async context, and its end must not make its children look
+ *  outside the one still running around it. */
+interface DeliveryScope {
+  active: boolean;
+  readonly parent: DeliveryScope | null;
+}
+
+/** The async context of trust listener calls — of ANY runtime, so a listener
+ *  of one runtime cannot tear down another whose listener is tearing down
+ *  the first (during a replacement two are alive). It grants nothing: it only
+ *  lets such a call be refused instead of deadlocking. Work a listener leaves
+ *  behind that runs after every enclosing delivery ended is an ordinary
+ *  caller again. */
+const insideDelivery = new AsyncLocalStorage<DeliveryScope>();
+
+function refuseInsideDelivery(call: string): void {
+  for (let scope = insideDelivery.getStore() ?? null; scope !== null; scope = scope.parent) {
+    if (scope.active) {
+      throw new Error(`${call} cannot be called from inside a trust listener: it would wait for a delivery`);
+    }
+  }
+}
+
+export function createSecretsProxyRuntime(deps: SecretsProxyRuntimeDeps): SecretsProxyRuntime {
+  const dir = join(deps.dataDir, "secrets-proxy");
+  const now = deps.now ?? Date.now;
+  const log = deps.log ?? console;
+  /** A logger that throws, or a reason that cannot be formatted, must never
+   *  break a handshake or skip a delivery's cleanup. */
+  const warn = (message: () => string): void => {
+    try {
+      log.warn(message());
+    } catch {
+      // Nothing more can be done with a report that cannot be made.
+    }
+  };
+  const reasonOf = (error: unknown): string => {
+    try {
+      return error instanceof Error ? error.message : String(error);
+    } catch {
+      return "an error that could not be formatted";
+    }
+  };
+  const loadAuthorities = (): CertificateAuthorities =>
+    loadOrCreateCertificateAuthorities({
+      dir,
+      seal: deps.seal,
+      open: deps.open,
+      now,
+      log: { warn: (message: string) => warn(() => message) },
+    });
+  let authorities = loadAuthorities();
+  let leaves = createLeafCertificateCache({ ca: authorities.signer, now });
+  const bundleOf = (set: CertificateAuthorities): string =>
+    set.trusted.map(({ certificatePem }) => (certificatePem.endsWith("\n") ? certificatePem : `${certificatePem}\n`)).join("");
+
+  interface TrustWatcher {
+    readonly listener: (bundlePem: string, signal: AbortSignal) => void | Promise<void>;
+    /** Aborted at unsubscribe or close(); nothing starts after it. Internal:
+     *  the listener never sees it, so nothing is registered on it. */
+    readonly stop: AbortController;
+    /** The last bundle the listener acknowledged. */
+    delivered: string | null;
+    /** A delivery was asked for (requestTrustDelivery) after the one under
+     *  way began: that one may already be past the task that needs it, so
+     *  another follows even if it is acknowledged. */
+    redeliver: boolean;
+    /** The delivery under way, if any. Never rejects. */
+    inFlight: Promise<void> | null;
+    /** That delivery's own signal and scope, so its abort is dispatched
+     *  inside it. Cleared when it ends: its signal then never aborts. */
+    current: { readonly controller: AbortController; readonly scope: DeliveryScope } | null;
+  }
+  const watchers = new Set<TrustWatcher>();
+  /** Every delivery under way, including an unsubscribed watcher's, so
+   *  close() can wait for all of them. */
+  const deliveries = new Set<Promise<void>>();
+  /** Set as close() begins: no delivery starts after it. */
+  let closed = false;
+
+  /** The engine's close, shared by every close() call. */
+  let engineClosed: Promise<void> | null = null;
+  /** Hands the current bundle to a watcher that does not have it yet, never
+   *  inside a TLS callback and never twice at once. A success that turns out
+   *  stale (the bundle changed meanwhile) delivers again at once; a failure
+   *  waits for the next review. */
+  const deliver = (watcher: TrustWatcher): void => {
+    const bundlePem = bundleOf(authorities);
+    const { signal } = watcher.stop;
+    if (closed || signal.aborted || watcher.inFlight !== null) return;
+    if (watcher.delivered === bundlePem && !watcher.redeliver) return;
+    watcher.redeliver = false;
+    let acknowledged = false;
+    const scope: DeliveryScope = { active: true, parent: insideDelivery.getStore() ?? null };
+    const controller = new AbortController();
+    const delivery: Promise<void> = Promise.resolve()
+      // Checked again here: an unsubscribe or close() may have come between.
+      .then(() =>
+        signal.aborted ? undefined : insideDelivery.run(scope, () => watcher.listener(bundlePem, controller.signal)),
+      )
+      .then(
+        () => {
+          acknowledged = !signal.aborted;
+        },
+        (error: unknown) => {
+          if (!signal.aborted) {
+            warn(
+              () =>
+                "[secrets-proxy] the CA bundle did not reach every running task; retrying at the next review: " +
+                reasonOf(error),
+            );
+          }
+        },
+      )
+      // Unconditional, and the delivery never rejects: close() waits on it.
+      .finally(() => {
+        scope.active = false;
+        watcher.current = null;
+        watcher.inFlight = null;
+        deliveries.delete(delivery);
+      })
+      .then(() => {
+        if (!acknowledged) return;
+        watcher.delivered = bundlePem;
+        deliver(watcher);
+      })
+      .catch(() => undefined);
+    watcher.inFlight = delivery;
+    watcher.current = { controller, scope };
+    deliveries.add(delivery);
+  };
+  /** Stops a watcher, and aborts its delivery under way, if any. AbortSignal
+   *  runs its handlers in the context of whoever calls abort(), not the one
+   *  that registered them: the dispatch happens inside that delivery's scope
+   *  (active, with its ancestry), so a handler is fenced like the listener
+   *  that registered it. */
+  const abortWatcher = (watcher: TrustWatcher): void => {
+    watcher.stop.abort();
+    const current = watcher.current;
+    if (current !== null) insideDelivery.run(current.scope, () => current.controller.abort());
+  };
+
+  /** The CA set, reviewed when its dates call for it: a successor joins the
+   *  bundle inside the signer's last 90 days, and the signer changes only on
+   *  its last day, to a CA every task started since has installed. */
+  const currentAuthorities = (): CertificateAuthorities => {
+    // A closed runtime never writes the store again: its replacement, on the
+    // same data dir, may already be reviewing it.
+    if (closed || now() < authorities.reviewAt) return authorities;
+    let next: CertificateAuthorities;
+    try {
+      next = loadAuthorities();
+    } catch (error) {
+      // The signer has at least its last day left: keep it, and retry in an
+      // hour rather than failing every handshake on a store error.
+      warn(() => `[secrets-proxy] could not review the proxy CA: ${reasonOf(error)}`);
+      authorities = { ...authorities, reviewAt: now() + HOUR_MS };
+      return authorities;
+    }
+    const before = bundleOf(authorities);
+    if (next.signer.certificatePem !== authorities.signer.certificatePem) {
+      leaves = createLeafCertificateCache({ ca: next.signer, now });
+    }
+    authorities = next;
+    const after = bundleOf(next);
+    if (after !== before) for (const watcher of watchers) deliver(watcher);
+    return authorities;
+  };
+  const reviewTimer = setInterval(() => {
+    currentAuthorities();
+    // Also retries any delivery that failed.
+    for (const watcher of watchers) deliver(watcher);
+  }, TRUST_REVIEW_EVERY_MS);
+  reviewTimer.unref();
+  const signerLeaves: LeafCertificateCache = {
+    get authority() {
+      return currentAuthorities().signer;
+    },
+    leafFor(hostname) {
+      currentAuthorities();
+      return leaves.leafFor(hostname);
+    },
+  };
+  const placeholders = createPlaceholderStore({ dir });
+
+  const mint = (taskId: string, key: ProxiedKeyPolicy): string => {
+    // The placeholder store belongs to the replacement once close() began.
+    if (closed) throw new Error("the secrets proxy runtime is closed");
+    return placeholders.placeholderFor({
+      taskId,
+      projectId: key.projectId,
+      key: key.key,
+      shape: key.placeholder ?? null,
+    });
+  };
+
+  /** The engine asks for a task's state on every accept, server name and
+   *  request; it is derived once per policy snapshot. */
+  const derived = new Map<string, { signature: string; state: ProxyTaskState }>();
+  const taskState = (taskId: string): ProxyTaskState | null => {
+    const keys = deps.policy.proxiedKeys(taskId);
+    if (keys === null) {
+      derived.delete(taskId);
+      return null;
+    }
+    const signature = policySignature(keys);
+    const cached = derived.get(taskId);
+    if (cached !== undefined && cached.signature === signature) return cached.state;
+    const byPlaceholder = new Map<string, ProxyKeyRef>();
+    for (const key of keys) {
+      byPlaceholder.set(mint(taskId, key), {
+        projectId: key.projectId,
+        key: key.key,
+        domains: [...key.domains],
+        ...(key.head !== undefined ? { head: key.head } : {}),
+      });
+    }
+    const domains = proxiableDomains(keys);
+    const placeholderList = [...byPlaceholder.keys()];
+    const state: ProxyTaskState = {
+      domains: () => domains,
+      lookup: (placeholder) => byPlaceholder.get(placeholder) ?? null,
+      placeholders: () => placeholderList,
+      // Values are never cached: a rotation applies to the next request.
+      realValue: (ref) =>
+        deps.policy.realValue(taskId, {
+          projectId: ref.projectId,
+          key: ref.key,
+          ...(ref.head !== undefined ? { head: ref.head } : {}),
+        }),
+    };
+    derived.set(taskId, { signature, state });
+    return state;
+  };
+
+  const proxy = createSecretsProxy({
+    getTaskState: taskState,
+    ca: signerLeaves,
+    resolveHost: deps.resolveHost,
+    audit: createAuditWriter(join(deps.dataDir, "logs")),
+    now,
+  });
+
+  const stopTask = (taskId: string): void => {
+    if (closed) return;
+    derived.delete(taskId);
+    proxy.closeTask(taskId);
+  };
+
+  return {
+    proxy,
+    caBundlePem: () => bundleOf(currentAuthorities()),
+    watchTrust(listener) {
+      // A closing or closed runtime has nothing left to deliver, and a
+      // delivery started now would outlive the close() that is waiting.
+      if (closed) return async () => undefined;
+      // Reviewed first, so a change this makes is delivered once, not twice.
+      currentAuthorities();
+      const watcher: TrustWatcher = {
+        listener,
+        stop: new AbortController(),
+        delivered: null,
+        redeliver: false,
+        inFlight: null,
+        current: null,
+      };
+      watchers.add(watcher);
+      deliver(watcher);
+      return async () => {
+        refuseInsideDelivery("unsubscribe");
+        abortWatcher(watcher);
+        watchers.delete(watcher);
+        if (watcher.inFlight !== null) await watcher.inFlight;
+      };
+    },
+    requestTrustDelivery() {
+      if (closed) return;
+      for (const watcher of watchers) {
+        // Starts one now, or right after the one under way.
+        watcher.redeliver = true;
+        deliver(watcher);
+      }
+    },
+    placeholderFor(taskId, key) {
+      // Only for a key the policy serves the task right now, checked in the
+      // same synchronous step as the mint: a request racing a stop or a
+      // deletion cannot persist placeholders for a task that is gone.
+      const served = deps.policy
+        .proxiedKeys(taskId)
+        ?.find((candidate) => candidate.projectId === key.projectId && candidate.key === key.key);
+      if (served === undefined) throw new Error("this key is not proxied for this task");
+      return mint(taskId, served);
+    },
+    domainsFor: (taskId) => [...proxiableDomains(deps.policy.proxiedKeys(taskId) ?? [])].sort(),
+    proxiedKeysFor: (taskId) =>
+      (deps.policy.proxiedKeys(taskId) ?? [])
+        .map((key) => ({
+          projectId: key.projectId,
+          key: key.key,
+          domains: [...proxiableDomains([key])].sort(),
+        }))
+        .filter((key) => key.domains.length > 0),
+    stopTask,
+    dropTask(taskId) {
+      // A closed runtime never deletes what its replacement may have written.
+      if (closed) return;
+      stopTask(taskId);
+      placeholders.dropTask(taskId);
+    },
+    close: async () => {
+      refuseInsideDelivery("close()");
+      // Synchronous, before any await: nothing is delivered, queued, retried
+      // or written from here on.
+      closed = true;
+      clearInterval(reviewTimer);
+      for (const watcher of watchers) abortWatcher(watcher);
+      watchers.clear();
+      engineClosed ??= proxy.close();
+      // Returns only once the deliveries under way have ended, so the caller
+      // can start a replacement safely. No delivery can start after the
+      // fence, so this snapshot is all of them.
+      await Promise.all([...deliveries]);
+      await engineClosed;
+    },
+  };
+}