@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/diagnostics.ts

@@ -42,7 +42,8 @@ const EXPECTED: Record<string, string> = {
  */
 export function diagnostic(node: string, check: Check): Diagnostic {
   const phase = check.id.split(':')[0]!;
-  const failed = check.status !== 'pass';
+  // A check that does not apply to this machine asks nothing of anyone.
+  const failed = check.status !== 'pass' && check.status !== 'not-applicable';
   const code = /^[A-Z][A-Z0-9_]{2,100}$/.test(check.summary)
     ? check.summary
     : `${check.id.replaceAll(/[^a-z0-9]/g, '_').toUpperCase()}_${failed ? 'UNVERIFIED' : 'VERIFIED'}`;
@@ -59,7 +60,9 @@ export function diagnostic(node: string, check: Check): Diagnostic {
     observed: check.summary,
     scope: scope(check.id),
     durationMs: check.phaseDurationMs ?? null,
-    retryable: failed && /OFFLINE|UNREACHABLE|METRICS_STALE|TIMEOUT/.test(code),
+    retryable:
+      failed &&
+      /OFFLINE|UNREACHABLE|METRICS_STALE|TIMEOUT|FIRST_BOOT_PENDING/.test(code),
     actionRequired: failed,
     remedy: {
       owner: owner(phase),

package/src/exposure.ts

@@ -0,0 +1,102 @@
+import type { FleetDefinition } from './definition';
+import { record } from './config';
+
+/** What a worker opens beyond loopback and the tailnet (docs/ubuntu-sync.md,
+ * Exposure). Declared fleet-wide as `worker.exposure` and per worker as
+ * `exposure`; a field a worker names replaces the fleet-wide one. Applied by
+ * ansible/ubuntu/tasks/firewall.yml and ssh.yml, measured as worker:exposure
+ * and worker:public-reachability.
+ */
+export interface ExposurePolicy {
+  /** Break-glass: the SSH port stays open on every interface. */
+  publicSsh: boolean;
+  /** Opened on every interface; a listener on one is allowed. */
+  publicPorts: PublicPort[];
+  /** A listener outside the allow-list that no fleet service owns, such as
+   * an agent's development server: reported as a warning or a failure. The
+   * firewall closes it either way.
+   */
+  otherListeners: 'warn' | 'fail';
+}
+export interface PublicPort {
+  protocol: 'tcp' | 'udp';
+  port: number;
+}
+export type ExposureDeclaration = Partial<ExposurePolicy>;
+
+/** Nothing public: inbound only over loopback and the tailnet. */
+export const DEFAULT_EXPOSURE: ExposurePolicy = {
+  publicSsh: false,
+  publicPorts: [],
+  otherListeners: 'warn',
+};
+
+export const EXPOSURE_EXPECTED =
+  'publicSsh (true or false), publicPorts (a list of {protocol: tcp or udp, port}, each once) and otherListeners ("warn" or "fail")';
+
+function publicPort(value: unknown): PublicPort | null {
+  if (!record(value)) return null;
+  const { protocol, port } = value;
+  if (Object.keys(value).some((key) => key !== 'protocol' && key !== 'port'))
+    return null;
+  if (protocol !== 'tcp' && protocol !== 'udp') return null;
+  if (!Number.isInteger(port) || Number(port) < 1 || Number(port) > 65535)
+    return null;
+  return { protocol, port: port as number };
+}
+
+/** A list of ports, each once; null when it is not one. */
+function publicPorts(value: unknown): PublicPort[] | null {
+  if (!Array.isArray(value) || value.length > 32) return null;
+  const parsed = value.map(publicPort);
+  if (parsed.some((entry) => entry === null)) return null;
+  const ports = parsed as PublicPort[];
+  const keys = ports.map((entry) => `${entry.protocol}/${entry.port}`);
+  return new Set(keys).size === keys.length ? ports : null;
+}
+
+const FIELDS: Record<string, (value: unknown) => boolean> = {
+  publicSsh: (value) => typeof value === 'boolean',
+  publicPorts: (value) => publicPorts(value) !== null,
+  otherListeners: (value) => value === 'warn' || value === 'fail',
+};
+
+/** An exposure declaration, or null when it is not one. */
+export function parseExposure(value: unknown): ExposureDeclaration | null {
+  if (!record(value)) return null;
+  const valid = Object.entries(value).every(
+    ([key, field]) => Object.hasOwn(FIELDS, key) && FIELDS[key]!(field)
+  );
+  return valid ? ({ ...value } as ExposureDeclaration) : null;
+}
+
+/** The policy one worker runs under: the default, then the fleet-wide
+ * declaration, then the worker's own.
+ */
+export function effectiveExposure(
+  fleetWide: ExposureDeclaration | undefined,
+  own: ExposureDeclaration | undefined
+): ExposurePolicy {
+  return { ...DEFAULT_EXPOSURE, ...fleetWide, ...own };
+}
+
+/** What a worker opens and listens on, from the fleet configuration. */
+export interface WorkerExposure {
+  policy: ExposurePolicy;
+  /** The port sshd listens on (network.sshPort; 22 when undeclared). */
+  sshPort: number;
+  /** The port T3 serves on loopback (network.t3Port; 3773). */
+  t3Port: number;
+}
+
+export function workerExposure(
+  definition: Pick<FleetDefinition, 'worker' | 'workers'>,
+  node: string
+): WorkerExposure {
+  const machine = definition.workers.find((entry) => entry.id === node);
+  return {
+    policy: effectiveExposure(definition.worker?.exposure, machine?.exposure),
+    sshPort: machine?.network?.sshPort ?? 22,
+    t3Port: machine?.network?.t3Port ?? 3773,
+  };
+}

package/src/fleet-config.ts

@@ -24,6 +24,7 @@ import {
 import { isOperator, stateDefinition } from './state/operator';
 import type { BucketLocation } from './state/bucket';
 import { definitionPath, stateRoot } from './state/paths';
+import { expandRecords } from './worker-records';
 
 /** Where every machine keeps the fleet configuration fleet-sync approved.
  * Launchers and scheduled jobs get no environment and read it from here.
@@ -96,7 +97,8 @@ function read(path: string): FleetDefinition {
       `The fleet configuration at ${path} is not readable JSON.`
     );
   }
-  return parseDefinition(value);
+  // A project's fleet.json may name generated worker records beside it.
+  return parseDefinition(expandRecords(value, dirname(path)));
 }
 
 function load(): FleetDefinition {

package/src/git-identity.ts

@@ -25,6 +25,7 @@ export async function convergeGitIdentity(
         git_name: git.name,
         git_email: git.email,
         git_home: home,
+        git_signing: git.signing ?? '',
       }),
     }
   );
@@ -33,7 +34,9 @@ export async function convergeGitIdentity(
       id: 'git:configuration',
       status: result.ok ? 'pass' : 'fail',
       summary: result.ok
-        ? 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
+        ? git.signing
+          ? "The declared Git identity, the GitHub credential helper and commit signing with this node's own SSH key are written in ~/.config/git/config."
+          : 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
         : 'GIT_CONFIGURATION_INCOMPLETE',
       ...(result.ok
         ? {}
@@ -152,9 +155,79 @@ export async function gitPushAuthCheck(
   };
 }
 
+/** The node's own commit signing key, made on the node by
+ * `ansible/github/git.yml`: an interface, like every path written to a
+ * machine.
+ */
+export function signingKey(home: string = homedir()): string {
+  return resolve(home, '.ssh/dev-fleet-git-signing');
+}
+
+/** What the signing probe signs: a fixed text, so nothing of the node's is. */
+const SIGNING_PROBE = 'dev-fleet signing probe\n';
+
+/** With `git.signing` declared, every commit and tag is signed with the
+ * node's own key: Git is configured to, and the key really signs. Without it
+ * there is no check.
+ */
+export async function gitSigningCheck(
+  git: GitIdentity | undefined,
+  run: Runner,
+  home: string = homedir()
+): Promise<Check | null> {
+  if (git?.signing !== 'ssh') return null;
+  const id = 'worker:git-signing';
+  const key = signingKey(home);
+  const format = await effective(run, 'gpg.format');
+  const signer = await effective(run, 'user.signingkey');
+  const always = await effective(run, 'commit.gpgsign');
+  if (format !== 'ssh' || signer !== `${key}.pub` || always !== 'true')
+    return {
+      id,
+      status: 'fail',
+      summary:
+        !format && !signer && !always
+          ? 'GIT_SIGNING_MISSING'
+          : 'GIT_SIGNING_OVERRIDDEN',
+      remedy: {
+        action:
+          'Rerun fleet-sync: it makes the node its own signing key and writes the signing settings in ~/.config/git/config. A ~/.gitconfig that sets gpg.format, user.signingkey or commit.gpgsign wins over it; remove those lines.',
+      },
+    };
+  const signed = await run(
+    ['ssh-keygen', '-Y', 'sign', '-n', 'git', '-f', key],
+    {
+      input: SIGNING_PROBE,
+      timeoutMs: 10_000,
+    }
+  );
+  if (!signed.ok || !signed.stdout.includes('-----BEGIN SSH SIGNATURE-----'))
+    return {
+      id,
+      status: 'fail',
+      summary: 'GIT_SIGNING_FAILED',
+      remedy: {
+        action:
+          "The node's signing key (~/.ssh/dev-fleet-git-signing) is missing or cannot sign without a prompt. Remove both halves of it and rerun fleet-sync, which makes a new one; then add the new public key to the GitHub account as a signing key.",
+      },
+    };
+  return {
+    id,
+    status: 'pass',
+    summary:
+      'Commits and tags on this node are signed with its own SSH key. GitHub shows them as verified once ~/.ssh/dev-fleet-git-signing.pub is a signing key of the account.',
+  };
+}
+
 export async function gitChecks(
   git: GitIdentity | undefined,
-  run: Runner
+  run: Runner,
+  home: string = homedir()
 ): Promise<Check[]> {
-  return [await gitIdentityCheck(git, run), await gitPushAuthCheck(git, run)];
+  const signing = await gitSigningCheck(git, run, home);
+  return [
+    await gitIdentityCheck(git, run),
+    await gitPushAuthCheck(git, run),
+    ...(signing ? [signing] : []),
+  ];
 }

package/src/ide/ide.ts

@@ -40,4 +40,9 @@ export interface WorkerContext {
    * time it is seen through a verified link.
    */
   remember?: (node: string, identity: string) => Promise<void>;
+  /** Pairs the manager's IDE with the worker's when the manager cannot
+   * reach it and a pairing was asked for (a replaced worker): the check
+   * that says how it went, or nothing when no pairing was asked for.
+   */
+  pair?: (node: FleetNode) => Promise<Check | undefined>;
 }

package/src/ide/t3.ts

@@ -78,7 +78,7 @@ export function t3Ide(inputs: T3Inputs = {}): Ide {
         await managerVersion(home(), context.workspace),
         await updatesCheck(
           inputs.policyPath ?? MACOS_POLICY,
-          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`
+          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml`
         ),
       ];
     },
@@ -89,9 +89,19 @@ export function t3Ide(inputs: T3Inputs = {}): Ide {
       const restarted = context.earlier.some(
         (check) => check.summary === T3_RESTARTED_SUMMARY
       );
-      const linked = await verifyT3Linked(node, context.run, {
+      let linked = await verifyT3Linked(node, context.run, {
         reconnectMs: restarted ? 180_000 : 0,
       });
+      // A replaced worker: the manager pairs with its new environment.
+      const paired =
+        linked.status === 'fail' && context.pair
+          ? await context.pair(node)
+          : undefined;
+      if (paired) {
+        checks.push(paired);
+        if (paired.status === 'pass')
+          linked = await verifyT3Linked(node, context.run);
+      }
       checks.push(linked);
       if (
         linked.status !== 'fail' &&

package/src/input-hashes.ts

@@ -22,10 +22,16 @@ export async function fileHash(path: string): Promise<string | null> {
   }
 }
 
+/** A repository without a packages directory (dev-fleet itself) has an
+ * empty set of workspace manifests, which is known, not missing.
+ */
 export async function manifestHash(root: string): Promise<string | null> {
   try {
     const entries = await readdir(join(root, 'packages'), {
       withFileTypes: true,
+    }).catch((error: NodeJS.ErrnoException) => {
+      if (error.code === 'ENOENT') return [];
+      throw error;
     });
     const hashes: string[] = [];
     for (const entry of entries

package/src/machine/machine-class.ts

@@ -0,0 +1,59 @@
+import type { Runner } from '../types';
+
+/** What kind of machine a worker is. A virtual machine has no lid, no
+ * battery and no firmware its guest can read, and restarting it after a
+ * power loss is its provider's job: checks of those are not applicable to
+ * it. Containers and WSL2 are not workers at all (docs/support-matrix.md).
+ */
+export type MachineClass = 'physical' | 'virtual' | 'unsupported';
+
+/** What systemd-detect-virt prints for a container or WSL rather than a
+ * hypervisor. */
+const CONTAINERS = [
+  'container-other',
+  'docker',
+  'lxc',
+  'lxc-libvirt',
+  'openvz',
+  'podman',
+  'pouch',
+  'proot',
+  'rkt',
+  'systemd-nspawn',
+  'wsl',
+];
+
+/** The class of a machine from the probe's environment word (`machine`,
+ * `container` or `wsl`) and what systemd-detect-virt printed: `none` is bare
+ * metal, a hypervisor (`kvm`, `qemu`, `amazon`, `microsoft`, ...) a virtual
+ * machine. When it cannot tell (`unknown`, or nothing), the machine is
+ * taken as physical, so no check that applies to one is left out.
+ */
+export function machineClass(
+  environment: string,
+  virtualization: string
+): MachineClass {
+  if (environment !== 'machine' || CONTAINERS.includes(virtualization))
+    return 'unsupported';
+  return ['', 'none', 'unknown'].includes(virtualization)
+    ? 'physical'
+    : 'virtual';
+}
+
+/** The class of the machine this runs on, and the hypervisor, measured by
+ * systemd-detect-virt. The worker's own checks use it; the manager's probe
+ * records it for each worker before anything is installed. */
+export async function detectMachineClass(
+  run: Runner
+): Promise<{ machine: MachineClass; virtualization: string }> {
+  const detected = await run(['systemd-detect-virt']);
+  const word = detected.stdout.trim();
+  const virtualization = /^[a-z0-9-]{1,32}$/.test(word) ? word : 'unknown';
+  return {
+    machine: machineClass(
+      CONTAINERS.includes(virtualization) ? 'container' : 'machine',
+      virtualization
+    ),
+    virtualization,
+  };
+}

package/src/machine/platform.ts

@@ -1,10 +1,16 @@
 import platforms from '../../ansible/platforms.json';
-import { kernelArchitectures, ubuntuRelease } from '../platform/ubuntu-release';
+import { UBUNTU_WORKER, workerPlatform } from '../platform/ubuntu-worker';
+import { belowMinimum, MINIMUM_WORKER } from '../worker-sizing';
+import { machineClass } from './machine-class';
 
-/** One POSIX probe for every worker, four lines: family, release, CPU and
- * the environment it runs in (`wsl`, `container` or `machine`, a physical
- * or virtual machine of its own). Read-only, and it needs nothing installed.
- * `root` moves the files it reads, for a stand-in; workers use `/`.
+/** One POSIX probe for every worker, nine lines: family, release, CPU, the
+ * environment it runs in (`wsl`, `container` or `machine`, a physical or
+ * virtual machine of its own), the hypervisor systemd-detect-virt names
+ * (`none` on bare metal, `unknown` when it cannot tell), the CPU count, all
+ * memory in MiB, the size of the root filesystem in MiB, and its clock in
+ * seconds since the epoch, which the manager compares with its own.
+ * Read-only, and it needs nothing installed. `root` moves the files it
+ * reads, for a stand-in; workers use `/`.
  */
 export function platformProbe(root = ''): string {
   return `sh -c '${[
@@ -13,31 +19,54 @@ export function platformProbe(root = ''): string {
     `if grep -qiE "microsoft|wsl" ${root}/proc/sys/kernel/osrelease 2>/dev/null; then echo wsl`,
     `elif [ -e ${root}/.dockerenv ] || [ -e ${root}/run/.containerenv ] || systemd-detect-virt --container --quiet 2>/dev/null; then echo container`,
     'else echo machine; fi',
+    'v=$(systemd-detect-virt --vm 2>/dev/null)',
+    'echo "${v:-unknown}"',
+    'nproc 2>/dev/null || echo',
+    // Each size prints its own line, empty when unread, so the clock after
+    // them stays on the ninth.
+    `m=$(awk "/^MemTotal:/{print int(\\$2/1024)}" ${root}/proc/meminfo 2>/dev/null)`,
+    'echo "$m"',
+    `d=$(df -Pk ${root}/ 2>/dev/null | awk "NR==2{print int(\\$2/1024)}")`,
+    'echo "$d"',
+    'date -u +%s.%N',
   ].join('; ')}'`;
 }
 export const PLATFORM_PROBE = platformProbe();
 
 export interface PlatformBaseline {
-  id: 'ubuntu:release' | 'platform:environment';
+  id: 'ubuntu:release' | 'platform:environment' | 'platform:worker-size';
   supported: boolean;
   summary: string;
 }
 
-/** A worker is an accepted Ubuntu release on one of that release's CPUs. */
+/** A worker is an accepted release of a worker platform on one of that
+ * release's CPUs. The check keeps its ID and its refusal: Ubuntu is the only
+ * worker platform, and anything else is refused as not that baseline. */
 export function evaluatePlatform(probe: string): PlatformBaseline {
   const [family, version, cpu] = probe.split('\n').map((line) => line.trim());
-  const release = family === 'ubuntu' ? ubuntuRelease(version) : undefined;
+  const platform = workerPlatform(family);
+  const release = platform?.release(version);
   const supported =
-    release !== undefined && kernelArchitectures(release).includes(cpu ?? '');
+    platform !== undefined &&
+    release !== undefined &&
+    platform.kernelArchitectures(release).includes(cpu ?? '');
   return {
     id: 'ubuntu:release',
     supported,
     summary: supported
-      ? `Ubuntu ${release.release} and supported CPU architecture observed.`
-      : 'UBUNTU_BASELINE_MISMATCH',
+      ? `${platform.label} ${release.release} and supported CPU architecture observed.`
+      : (platform ?? UBUNTU_WORKER).mismatch,
   };
 }
 
+/** The clock the probe read, in seconds since the epoch, or undefined. */
+export function probedClock(probe: string): number | undefined {
+  const line = probe.split('\n')[8]?.trim() ?? '';
+  // GNU date prints nanoseconds; another date prints a literal N.
+  const match = /^(\d{9,})(\.\d+)?/.exec(line);
+  return match ? Number(match[1]! + (match[2] ?? '')) : undefined;
+}
+
 /** Environments the support matrix (ansible/platforms.json) refuses, by the
  * word the probe prints for them. */
 const REFUSED = new Map(
@@ -62,3 +91,53 @@ export function evaluateEnvironment(probe: string): PlatformBaseline {
       : `WORKER_ENVIRONMENT_UNSUPPORTED: ${refused ?? 'unknown environment'}`,
   };
 }
+
+const line = (probe: string, index: number): string =>
+  probe.split('\n')[index]?.trim() ?? '';
+
+/** The class of machine the probe found: physical, virtual or unsupported
+ * (a container or WSL2), recorded for every worker. A worker that cannot
+ * say which hypervisor it runs on is taken as physical, so nothing that
+ * applies to a physical machine is left out. */
+export function evaluateMachineClass(probe: string): {
+  id: 'platform:machine-class';
+  summary: string;
+} {
+  const environment = line(probe, 3);
+  const virtualization = line(probe, 4) || 'unknown';
+  const found = machineClass(environment, virtualization);
+  return {
+    id: 'platform:machine-class',
+    summary:
+      found === 'virtual'
+        ? `Virtual machine (${virtualization}): checks of firmware power recovery and of a lid do not apply.`
+        : found === 'physical'
+          ? `Physical machine (systemd-detect-virt reported ${virtualization}).`
+          : `Unsupported: ${environment || 'unknown environment'}.`,
+  };
+}
+
+const BELOW_MINIMUM = 'WORKER_SIZE_BELOW_MINIMUM';
+
+/** The machine against the stated minimum worker size (`sizing` in the
+ * Ubuntu policy), before anything is installed; null when the probe did not
+ * report a size. */
+export function evaluateSize(probe: string): PlatformBaseline | null {
+  const [cpus, memoryMiB, diskMiB] = [5, 6, 7].map((index) =>
+    Number(line(probe, index) || NaN)
+  );
+  if (![cpus, memoryMiB, diskMiB].every((value) => Number.isInteger(value)))
+    return null;
+  const lacks = belowMinimum({
+    cpus: cpus!,
+    memoryMiB: memoryMiB!,
+    diskMiB: diskMiB!,
+  });
+  return {
+    id: 'platform:worker-size',
+    supported: lacks.length === 0,
+    summary: lacks.length
+      ? `${BELOW_MINIMUM}: ${lacks.join(', ')}`
+      : `${cpus} CPUs, ${memoryMiB} MiB of memory and a ${diskMiB} MiB root filesystem: at least the minimum worker (${MINIMUM_WORKER.cpus} CPUs, ${MINIMUM_WORKER.memoryMiB} MiB, ${MINIMUM_WORKER.diskMiB} MiB).`,
+  };
+}

package/src/machine/state.ts

@@ -20,6 +20,12 @@ export interface Enrollment {
   approvedRevision: string;
   appliedRevision?: string;
   previousRevision?: string;
+  /** The revision each worker last converged to, by fleet id. A worker
+   * keeps its progress when the fleet as a whole is not ready.
+   */
+  workers?: Record<string, string>;
+  /** Tombstones: when each retired worker left the fleet, by fleet id. */
+  retired?: Record<string, string>;
 }
 
 export const isRevision = (value: unknown): value is string =>
@@ -80,20 +86,39 @@ async function lockAbandoned(lock: string): Promise<boolean> {
   }
 }
 
-/** Removes an abandoned lock. The rename is atomic, so when several managers
- * race to recover it exactly one wins; the others simply retry the lock. */
-async function reclaimAbandonedLock(lock: string): Promise<boolean> {
+/** Removes an abandoned lock. Judging a lock abandoned and renaming it aside
+ * are two steps: between them another reclaimer can have replaced it with a
+ * live lock of its own. So the lock is judged again where it was put aside,
+ * where nobody else can change it, and put back when it turns out to be held.
+ * `judged` runs between the two steps; only the features pause there. */
+async function reclaimAbandonedLock(
+  lock: string,
+  judged: () => Promise<void>
+): Promise<boolean> {
   if (!(await lockAbandoned(lock))) return false;
+  await judged();
   const aside = `${lock}.abandoned-${randomUUID()}`;
   try {
     await rename(lock, aside);
   } catch {
     return true; // Someone else already recovered it; retry.
   }
+  if (!(await lockAbandoned(aside))) {
+    await rename(aside, lock).catch(() => undefined);
+    return false;
+  }
   await rm(aside, { recursive: true, force: true });
   return true;
 }
 
+/** Removes the lock only while it is still the one this run wrote. */
+async function releaseLock(lock: string, owner: string): Promise<void> {
+  const current = await readFile(join(lock, 'owner.json'), 'utf8').catch(
+    () => null
+  );
+  if (current === owner) await rm(lock, { recursive: true, force: true });
+}
+
 /** Bundles are written next to the lock and removed in a `finally`, which a
  * killed run never reaches. Holding the lock means none is in use. */
 async function sweepTransferBundles(root: string): Promise<void> {
@@ -107,7 +132,8 @@ async function sweepTransferBundles(root: string): Promise<void> {
  * cleanup. Never steals a lock from a still-running installer. */
 export async function withMachineLock<T>(
   root: string,
-  operation: () => Promise<T>
+  operation: () => Promise<T>,
+  judged: () => Promise<void> = async () => {}
 ): Promise<T> {
   await mkdir(root, { recursive: true, mode: 0o700 });
   const lock = join(root, 'sync.lock');
@@ -120,26 +146,28 @@ export async function withMachineLock<T>(
         throw new FleetUsageError(
           'Cannot create manager lock; inspect private state-directory permissions and free disk space.'
         );
-      if (attempt < 3 && (await reclaimAbandonedLock(lock))) continue;
+      if (attempt < 3 && (await reclaimAbandonedLock(lock, judged))) continue;
       throw new FleetLockError(
         'Fleet sync is locked. Inspect running installers before removing an abandoned sync.lock directory, then rerun.'
       );
     }
   }
+  const owner =
+    JSON.stringify({
+      version: 1,
+      pid: process.pid,
+      startedAt: new Date().toISOString(),
+      run: randomUUID(),
+    }) + '\n';
   try {
-    await writeFile(
-      join(lock, 'owner.json'),
-      JSON.stringify({
-        version: 1,
-        pid: process.pid,
-        startedAt: new Date().toISOString(),
-      }) + '\n',
-      { flag: 'wx', mode: 0o600 }
-    );
+    await writeFile(join(lock, 'owner.json'), owner, {
+      flag: 'wx',
+      mode: 0o600,
+    });
     await sweepTransferBundles(root);
     return await operation();
   } finally {
-    await rm(lock, { recursive: true });
+    await releaseLock(lock, owner);
   }
 }
 
@@ -156,6 +184,30 @@ export async function ensurePrivateRoot(root: string): Promise<void> {
   }
 }
 
+function validWorkers(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, revision]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) && isRevision(revision)
+      ))
+  );
+}
+
+function validRetired(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, at]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) &&
+          typeof at === 'string' &&
+          !Number.isNaN(Date.parse(at))
+      ))
+  );
+}
+
 /** A valid enrollment record, or null. */
 export function parseEnrollment(data: unknown): Enrollment | null {
   if (!record(data) || data['version'] !== 1) return null;
@@ -167,7 +219,9 @@ export function parseEnrollment(data: unknown): Enrollment | null {
   return isRevision(data['approvedRevision']) &&
     ['appliedRevision', 'previousRevision'].every(
       (key) => data[key] === undefined || isRevision(data[key])
-    )
+    ) &&
+    validWorkers(data['workers']) &&
+    validRetired(data['retired'])
     ? (data as unknown as Enrollment)
     : null;
 }