@crvouga/dev-fleet 0.6.0 → 0.8.0

package/src/cli/fleet-restore.ts

@@ -0,0 +1,205 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { parseDefinition, type FleetDefinition } from '../definition';
+import { definitionSource } from '../fleet-config';
+import { managerHost } from '../platform/manager-host';
+import { makeReport, renderReport, reportExitCode } from '../report';
+import { runCommand } from '../runner';
+import { connection } from '../state/connection';
+import { managerHolder } from '../state/holder';
+import { stateRoot } from '../state/paths';
+import { pull, standing, type StateContext } from '../state/sync';
+import type { Check, Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetRestoreOptions {
+  dryRun?: boolean;
+  json?: boolean;
+}
+
+export interface FleetRestoreServices {
+  run: Runner;
+  env: NodeJS.ProcessEnv;
+  home: string;
+  holder?: string | null;
+}
+
+/** What a replacement manager gets back, in the order it is restored. */
+export const RESTORES = [
+  {
+    what: 'Fleet state (roster, approved revision, configuration last applied)',
+    how: 'Downloaded from the bucket by this command.',
+  },
+  {
+    what: 'Machine policy, toolchain and checkouts',
+    how: 'Rebuilt from Git by devfleet fleet-sync.',
+  },
+  {
+    what: 'CLI proxy state (provider logins, keys, Tailscale identity)',
+    how: 'Restored from its encrypted snapshots by devfleet cliproxy install (vault secrets provider only).',
+  },
+  {
+    what: 'Monitoring hub',
+    how: 'Recreated by devfleet fleet-sync with its accounts and systems; metric history is lost.',
+  },
+  {
+    what: 'T3 ownership record',
+    how: 'Adopted again with devfleet t3-apply --import ID.',
+  },
+];
+
+const check = (
+  id: string,
+  ok: boolean,
+  [pass, failure, action, missing = 'fail']: [
+    string,
+    string,
+    string,
+    ('fail' | 'warn')?,
+  ]
+): Check =>
+  ok
+    ? { id, status: 'pass', summary: pass }
+    : { id, status: missing, summary: failure, remedy: { action } };
+
+function definition(services: FleetRestoreServices): FleetDefinition | null {
+  const source = definitionSource(services.env, services.home);
+  if (source.kind !== 'file' || !existsSync(source.path)) return null;
+  try {
+    return parseDefinition(JSON.parse(readFileSync(source.path, 'utf8')));
+  } catch {
+    return null;
+  }
+}
+
+/** Every prerequisite of a manager restore, read-only. */
+async function prerequisites(
+  services: FleetRestoreServices,
+  context: StateContext
+): Promise<Check[]> {
+  const fleet = definition(services);
+  const reached = connection(services.env, services.home);
+  const checks: Check[] = [
+    check('restore:configuration', fleet !== null, [
+      'The project’s fleet.json is found and valid.',
+      'No valid fleet.json was found.',
+      'Run this from the project that holds your fleet.json, after git clone and npm ci.',
+    ]),
+    check('restore:bucket', reached.bucket !== null, [
+      'The bucket is named, with credentials for it.',
+      'No bucket is configured.',
+      'Name it under "state" in fleet.json and pass DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY.',
+    ]),
+  ];
+  if (reached.bucket) {
+    const stands = await standing(context);
+    checks.push(
+      check(
+        'restore:state',
+        stands === 'bucket-ahead' || stands === 'in-sync',
+        [
+          'The bucket answers and holds the fleet state.',
+          `The fleet state cannot be restored from the bucket: it stands "${stands}".`,
+          'See the state:replica remedy of devfleet fleet-state; a corrupt replica is restored with --restore --timestamp.',
+        ]
+      )
+    );
+  }
+  if (fleet) {
+    const host = managerHost(fleet.manager.os);
+    const name = await host.localHostName(services.run);
+    checks.push(
+      check('restore:host-name', name === fleet.manager.id, [
+        `This machine’s local host name is the manager’s id (${fleet.manager.id}).`,
+        `This machine’s local host name is not the manager’s id (${fleet.manager.id}).`,
+        host.platform === 'macOS'
+          ? `Set it: sudo scutil --set LocalHostName ${fleet.manager.id}`
+          : `Set it: sudo hostnamectl set-hostname ${fleet.manager.id}`,
+      ])
+    );
+  }
+  const git = await services.run(['git', '--version']);
+  checks.push(
+    check('restore:git', git.ok, [
+      'Git is installed.',
+      'Git is not installed.',
+      'Install Git; fleet-sync installs everything else.',
+    ])
+  );
+  const tailscale = await services.run(['tailscale', 'status', '--json']);
+  checks.push(
+    check(
+      'restore:tailscale',
+      tailscale.ok && /"BackendState":\s*"Running"/.test(tailscale.stdout),
+      [
+        'This machine is signed in to the tailnet.',
+        'This machine is not signed in to the tailnet.',
+        'Sign in to Tailscale and approve the device: that is a person’s.',
+        'warn',
+      ]
+    )
+  );
+  const docker = await services.run(['docker', 'version']);
+  checks.push(
+    check('restore:container-engine', docker.ok, [
+      'The container engine answers.',
+      'The container engine does not answer.',
+      'Install and start Docker Desktop; the proxy and the hub run in it.',
+      'warn',
+    ])
+  );
+  return checks;
+}
+
+/** Brings a replacement manager back from the bucket: `--dry-run` verifies
+ * every prerequisite and lists what would be restored, changing nothing.
+ */
+export async function fleetRestore(
+  options: FleetRestoreOptions,
+  services: FleetRestoreServices = {
+    run: runCommand,
+    env: process.env,
+    home: homedir(),
+    holder: managerHolder(process.env),
+  }
+): Promise<CommandOutcome> {
+  const reached = connection(services.env, services.home);
+  const context: StateContext = {
+    root: stateRoot(services.env, services.home),
+    bucket: reached.bucket,
+    run: services.run,
+    holder: services.holder ?? null,
+  };
+  const checks = await prerequisites(services, context);
+  const ready = !checks.some((entry) => entry.status === 'fail');
+  const done: string[] = [];
+  if (ready && !options.dryRun) {
+    const after = await pull(context);
+    checks.push(
+      check('restore:applied', after === 'in-sync', [
+        'This machine holds the fleet state from the bucket. Next: devfleet fleet-sync (it may need --take-over while the old manager’s lease runs), then devfleet cliproxy install.',
+        `The fleet state was not restored: it stands "${after}".`,
+        'Run devfleet fleet-state for the remedy.',
+      ])
+    );
+    done.push('Restored the fleet state from the bucket.');
+  }
+  const data = {
+    ...makeReport('fleet-restore', [{ node: 'manager', checks }]),
+    dryRun: options.dryRun === true,
+    restores: RESTORES,
+  };
+  return {
+    exitCode: reportExitCode(data, false),
+    data,
+    text: [
+      ...done,
+      renderReport(data),
+      '',
+      options.dryRun ? 'Would restore, in this order:' : 'Restored in order:',
+      ...RESTORES.map(
+        (entry, index) => `${index + 1}. ${entry.what}: ${entry.how}`
+      ),
+    ].join('\n'),
+  };
+}

package/src/cli/fleet-state.ts

@@ -25,6 +25,7 @@ import {
   type Connection,
 } from '../state/connection';
 import { adoptEarlier, legacyPath, readLegacy } from '../state/legacy';
+import { managerHolder } from '../state/holder';
 import { bucketPath, databasePath, stateRoot } from '../state/paths';
 import {
   DOCUMENTS,
@@ -36,7 +37,11 @@ import {
 import {
   mutateState,
   pending,
+  leaseConflict,
+  leaseHeldMessage,
   pull,
+  pullAt,
+  refresh,
   push,
   standing,
   type Standing,
@@ -58,6 +63,8 @@ export interface FleetStateOptions {
   push?: boolean;
   restore?: boolean;
   yes?: boolean;
+  timestamp?: string;
+  takeOver?: boolean;
   json?: boolean;
   nonInteractive?: boolean;
 }
@@ -68,6 +75,10 @@ export interface FleetStateServices {
   home: string;
   /** Asks for one hidden value on the terminal; null when there is none. */
   ask: ((question: string) => Promise<string>) | null;
+  /** This machine, as the manager lease names it; without one the lease is
+   * neither checked nor taken.
+   */
+  holder?: string | null;
 }
 
 function terminalAsk(
@@ -84,11 +95,16 @@ function terminalAsk(
 const reach = (services: FleetStateServices): Connection =>
   connection(services.env, services.home);
 
-function context(services: FleetStateServices): StateContext {
+function context(
+  services: FleetStateServices,
+  options: FleetStateOptions = {}
+): StateContext {
   return {
     root: stateRoot(services.env, services.home),
     bucket: reach(services).bucket,
     run: services.run,
+    holder: services.holder ?? null,
+    takeOver: options.takeOver === true,
   };
 }
 
@@ -109,6 +125,8 @@ function action(options: FleetStateOptions): Action | 'status' {
       'Choose one of --init, --import, --export, --example, --push or --restore.'
     );
   const settings = ['bucket', 'endpoint', 'region', 'prefix', 'directory'];
+  if (options.timestamp !== undefined && chosen[0] !== 'restore')
+    throw new FleetUsageError('--timestamp belongs to --restore.');
   if (
     chosen[0] !== 'init' &&
     settings.some((name) => options[name as keyof FleetStateOptions])
@@ -367,8 +385,22 @@ async function init(
       `Saved the bucket settings to ${bucketPath(services.env, services.home)}.`
     );
   }
-  const target = context(services);
+  const target = context(services, options);
   const had = listDocuments(databasePath(target.root)).length > 0;
+  // A replacement connects to restore. While another manager holds the
+  // lease this one only takes the bucket's version and writes nothing.
+  const fresh = await refresh(target);
+  const held = leaseConflict(target);
+  if (held) {
+    if (!had && fresh.replaced)
+      done.push('Restored the fleet state from the bucket.');
+    done.push(leaseHeldMessage(held));
+    return report(services, {
+      standing: fresh.standing,
+      done,
+      missing: 'warn',
+    });
+  }
   const { value, standing: after } = await mutateState(target, (before) => {
     // A state the bucket already holds is the fleet's; this machine's
     // earlier files only ever seed an empty one.
@@ -388,7 +420,8 @@ async function init(
 
 async function importDefinition(
   path: string,
-  services: FleetStateServices
+  services: FleetStateServices,
+  options: FleetStateOptions
 ): Promise<CommandOutcome> {
   let value: unknown;
   try {
@@ -399,7 +432,7 @@ async function importDefinition(
     );
   }
   const body = JSON.stringify(parseDefinition(value), null, 2) + '\n';
-  const target = context(services);
+  const target = context(services, options);
   const { standing: after } = await mutateState(target, () => {
     adoptEarlier(services.env, services.home);
     return writeDocument(databasePath(target.root), 'definition', body);
@@ -458,6 +491,23 @@ async function restore(
     throw new FleetUsageError(
       'No bucket is configured, so there is nothing to restore from. Name it under "state" in fleet.json, or run devfleet fleet-state --init --bucket NAME --endpoint https://HOST.'
     );
+  if (options.timestamp !== undefined) {
+    // The bucket's current version is replaced too: --yes is the consent.
+    if (!options.yes)
+      throw new FleetUsageError(
+        '--restore --timestamp replaces the fleet state in the bucket and on this machine with an earlier version. Repeat with --yes.'
+      );
+    const after = await pullAt(target, options.timestamp);
+    return report(services, {
+      standing: after,
+      done:
+        after === 'in-sync'
+          ? [
+              `This machine and the bucket now hold the fleet state as it was at ${options.timestamp}.`,
+            ]
+          : [],
+    });
+  }
   if (pending(target) && !options.yes)
     throw new FleetUsageError(
       'This machine has fleet state the bucket does not hold, and --restore discards it. Upload it with --push, or repeat with --yes to discard it.'
@@ -477,6 +527,7 @@ const DEFAULTS = (options: FleetStateOptions): FleetStateServices => ({
   env: process.env,
   home: homedir(),
   ask: terminalAsk(options),
+  holder: managerHolder(process.env),
 });
 
 /** `fleet-state` alone is read-only. Each flag is one explicit action on the
@@ -488,7 +539,8 @@ export async function fleetState(
 ): Promise<CommandOutcome> {
   const chosen = action(options);
   if (chosen === 'init') return init(options, services);
-  if (chosen === 'import') return importDefinition(options.import!, services);
+  if (chosen === 'import')
+    return importDefinition(options.import!, services, options);
   if (chosen === 'export') return exportDefinition(services);
   if (chosen === 'example') return exampleDefinition();
   if (chosen === 'restore') return restore(options, services);

package/src/cli/fleet-sync-forward.ts

@@ -104,9 +104,11 @@ export function forwardedFlags(
 ): string[] {
   const flags = revision ? ['--revision', revision] : [];
   for (const node of options.node ?? []) flags.push('--node', node);
+  if (options.canary) flags.push('--canary', options.canary);
   if (options.schedule) flags.push('--schedule', options.schedule);
   const switches: [boolean | undefined, string][] = [
     [options.all, '--all'],
+    [options.vaultPasswordLogin, '--vault-password-login'],
     [options.dryRun, '--dry-run'],
     [options.apply, '--apply'],
     [options.yes, '--yes'],

package/src/cli/fleet-sync-manager.ts

@@ -1,7 +1,6 @@
 import { blocking, type Check, type Runner } from '../types';
 import type { FleetDefinition } from '../definition';
 import type { ManagerNode } from '../manager-sync';
-import { hostId } from '../topology';
 import { convergeProxy } from '../proxy-converge';
 import { attempt } from '../node-converge-shared';
 import { quotaLimited, verifyFleetProviders } from '../agents/fleet-verify';
@@ -14,12 +13,17 @@ import { IDES } from '../ide';
 import { enabled, managerChecks } from '../ide/orchestrate';
 import { idesOf } from '../ide/definition';
 import { convergeGitIdentity, gitChecks } from '../git-identity';
-import macos from '../../ansible/macos/policy.json';
-import { ansibleCommand } from '../platform/ansible';
+import { ansibleCommand, runAnsible } from '../platform/ansible';
 import { convergeGithub } from '../github/converge';
 import { PACKAGE_ROOT } from '../package-info';
 import { managerHealth } from '../manager-health';
 import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import {
+  MACOS_MANAGER,
+  managerHost,
+  type ManagerHost,
+} from '../platform/manager-host';
 import { join } from 'node:path';
 
 /** The manager converges itself locally (never over SSH) before any worker:
@@ -37,12 +41,18 @@ export async function convergeManager(
 ): Promise<ManagerNode> {
   const started = Date.now();
   const id = definition.manager.id;
-  const identity = await managerIdentity(id, run);
+  const host = managerHost(definition.manager.os);
+  const identity = await managerIdentity(id, run, host);
   const checks: Check[] = [identity];
   if (identity.status === 'pass') {
     // Launchers and scheduled jobs on the manager read the installed copy.
     if (input.apply) installConfig();
-    checks.push(await managerPower(run, input.workspace));
+    checks.push(
+      input.apply
+        ? await holdManagerPower(run, input.workspace, host)
+        : await managerPower(run, input.workspace, host)
+    );
+    checks.push(await managerBootRecovery(run, input.workspace, host));
     checks.push(await managerDocker(run));
     // Each enabled IDE: its settings, pinned version and policy.
     checks.push(
@@ -193,9 +203,13 @@ export async function managerAgentProfiles(
 }
 
 /** fleet-sync runs only on the declared manager. */
-export async function managerIdentity(id: string, run: Runner): Promise<Check> {
-  const result = await run(['scutil', '--get', 'LocalHostName']);
-  const local = result.ok ? hostId(result.stdout.trim()) : '';
+export async function managerIdentity(
+  id: string,
+  run: Runner,
+  host: ManagerHost = MACOS_MANAGER,
+  platform: NodeJS.Platform = process.platform
+): Promise<Check> {
+  const local = (await host.localHostName(run)) ?? '';
   return local === id
     ? {
         id: 'manager:identity',
@@ -207,7 +221,10 @@ export async function managerIdentity(id: string, run: Runner): Promise<Check> {
         status: 'fail',
         summary: 'FLEET_NOT_MANAGER',
         remedy: {
-          action: `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`,
+          action:
+            platform === 'darwin'
+              ? `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`
+              : `This machine runs ${platform}, and the manager is always a macOS machine. Run fleet-sync on the manager (${id}), or from a worker that can SSH to it.`,
         },
       };
 }
@@ -218,15 +235,10 @@ export async function managerIdentity(id: string, run: Runner): Promise<Check> {
  */
 export async function managerPower(
   run: Runner,
-  workspace = '.'
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER
 ): Promise<Check> {
-  const result = await run(['pmset', '-g', 'custom']);
-  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
-  const observed = (key: string) =>
-    new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
-  const drifted = Object.entries(macos.pmset)
-    .filter(([key, value]) => observed(key) !== value)
-    .map(([key]) => key);
+  const drifted = await host.powerDrift(run);
   return drifted.length === 0
     ? {
         id: 'manager:power',
@@ -240,11 +252,82 @@ export async function managerPower(
           ? 'MANAGER_SLEEPS_ON_POWER'
           : `MANAGER_POWER_POLICY_DRIFT ${drifted.join(' ')}`,
         remedy: {
-          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once, as an administrator: cd ${workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`,
+          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Grant the manager its power settings once, as an administrator; every fleet-sync then applies the policy with no prompt: cd ${workspace} && ${ansibleCommand().join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml`,
         },
       };
 }
 
+/** After a power cut the manager comes back by itself: the machine starts
+ * again, a user session exists for its per-user services, and the container
+ * engine that hosts the proxy and the hub starts with it. A FileVault Mac
+ * waits for a person; that is reported as a warning with its consequence.
+ */
+export async function managerBootRecovery(
+  run: Runner,
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER,
+  home = homedir()
+): Promise<Check> {
+  const items = await host.bootRecovery(run, home);
+  const failing = items.filter((entry) => entry.status === 'fail');
+  const warning = items.filter((entry) => entry.status === 'warn');
+  if (!failing.length && !warning.length)
+    return {
+      id: 'manager:boot-recovery',
+      status: 'pass',
+      summary: 'The manager comes back by itself after a power cut or reboot.',
+    };
+  const named = failing.length ? failing : warning;
+  return {
+    id: 'manager:boot-recovery',
+    status: failing.length ? 'fail' : 'warn',
+    summary: `MANAGER_BOOT_RECOVERY_INCOMPLETE ${named.map((entry) => entry.item).join(' ')}`,
+    remedy: {
+      action: [
+        ...[...failing, ...warning].map((entry) => entry.detail),
+        host.bootRemedy(workspace),
+      ].join(' '),
+    },
+  };
+}
+
+/** Whether the one grant manager-privilege.yml installs is in place: this
+ * user may run `pmset -c` without a password. Lists, never runs.
+ */
+async function powerGranted(run: Runner): Promise<boolean> {
+  return (
+    await run(['sudo', '-n', '-l', '/usr/bin/pmset', '-c', 'sleep', '0'], {
+      timeoutMs: 10_000,
+    })
+  ).ok;
+}
+
+/** On every applying sync: a drifted power policy is applied with
+ * manager.yml when the grant is in place, then measured again. Without the
+ * grant the check says how to give it, once.
+ */
+export async function holdManagerPower(
+  run: Runner,
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER,
+  { root = PACKAGE_ROOT, scratch }: { root?: string; scratch?: string } = {}
+): Promise<Check> {
+  const before = await managerPower(run, workspace, host);
+  if (
+    before.status === 'pass' ||
+    host.platform !== 'macOS' ||
+    !(await powerGranted(run))
+  )
+    return before;
+  await runAnsible(
+    run,
+    join(root, 'ansible/macos/manager.yml'),
+    { input: '{}', timeoutMs: 120_000 },
+    scratch
+  );
+  return managerPower(run, workspace, host);
+}
+
 /** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
 export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
   const { vars } = Bun.TOML.parse(

package/src/cli/fleet-sync-remote.ts

@@ -103,16 +103,26 @@ export async function selected(
   return outcome(report, planFingerprint, started);
 }
 
-/** A revision new to the fleet is proven on the first declared worker before
- * the rest; repeating one that already converged everywhere needs no canary,
- * so every worker converges at once.
+/** A revision new to the fleet is proven on one worker before the rest: the
+ * one `--canary` or fleet.json names, else the first declared worker, and
+ * the next reachable one when that cannot be reached. Repeating a revision
+ * that already converged everywhere needs no canary, so every worker
+ * converges at once.
  */
 function canary(
   canonical: FleetDefinition,
   options: FleetSyncOptions
 ): { canary?: string } {
-  const first = canonical.workers[0];
-  return first && !options.rolledOut ? { canary: first.id } : {};
+  if (
+    options.canary &&
+    !canonical.workers.some((worker) => worker.id === options.canary)
+  )
+    throw new FleetUsageError(
+      '--canary names a worker fleet.json does not declare.'
+    );
+  const preferred =
+    options.canary ?? canonical.canary ?? canonical.workers[0]?.id;
+  return preferred && !options.rolledOut ? { canary: preferred } : {};
 }
 
 /** Saves the full JSON report, then renders the short human summary. */
@@ -165,6 +175,7 @@ function deployWithMonitoring(
     /** The manager's checkout, and the roster an identity is recorded in. */
     workspace: string;
     roster: string;
+    passwordLogin: boolean;
   }
 ): DeployNode {
   return async (node, revision) => {
@@ -189,7 +200,10 @@ function deployWithMonitoring(
       const checks = await authorizePrivilege(node, payload.authorizeSudo, run);
       if (checks.length) return { node: node.id, checks };
     }
-    const vault = { interactive: ui.interactiveAuthorization };
+    const vault = {
+      interactive: ui.interactiveAuthorization,
+      passwordLogin: ui.passwordLogin,
+    };
     if (ui.interactiveAuthorization) ui.progress.stop();
     const before = await authorizeVault(node, vault, run);
     ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
@@ -242,6 +256,7 @@ function scopeDefinition(canonical: FleetDefinition, ids?: string[]) {
 function interfaceOptions(options: FleetSyncOptions) {
   return {
     progress: syncProgress(!options.json),
+    passwordLogin: Boolean(options.vaultPasswordLogin),
     interactiveAuthorization: Boolean(
       options.interactiveAuthorization &&
       !options.json &&

package/src/cli/fleet-sync-state.ts

@@ -4,7 +4,14 @@ import { storeEnrollment } from '../state/enrollment';
 import { adoptEarlier } from '../state/legacy';
 import { databasePath, stateRoot } from '../state/paths';
 import { writeDocument } from '../state/store';
-import { pending, push, type Standing, type StateContext } from '../state/sync';
+import {
+  claimLease,
+  refuseHeldLease,
+  pending,
+  push,
+  type Standing,
+  type StateContext,
+} from '../state/sync';
 import type { Check } from '../types';
 
 /** What one fleet-sync run records in the fleet state: the revision it was
@@ -14,8 +21,11 @@ import type { Check } from '../types';
 export interface RunRecord {
   /** Before any machine changes: this revision is the approved one. */
   approve(): void;
-  /** Once the result is known. The check says whether the bucket holds it. */
-  settle(converged: boolean): Promise<Check>;
+  /** Once the result is known. The check says whether the bucket holds it.
+   * `applied` is the revision each worker that converged now holds: it is
+   * kept even when the fleet as a whole is not ready.
+   */
+  settle(converged: boolean, applied?: Record<string, string>): Promise<Check>;
   /** For a run that stopped before its result: its approval is still uploaded. */
   abandon(): Promise<void>;
 }
@@ -51,21 +61,30 @@ export function runRecord(
       : Promise.resolve(state.standing);
   return {
     approve: () => {
+      refuseHeldLease(state.context);
       storeEnrollment(approval, env);
-      if (state.definition === undefined) return;
-      adoptEarlier(env);
-      writeDocument(
-        databasePath(stateRoot(env)),
-        'definition',
-        state.definition
-      );
+      if (state.definition !== undefined) {
+        adoptEarlier(env);
+        writeDocument(
+          databasePath(stateRoot(env)),
+          'definition',
+          state.definition
+        );
+      }
+      claimLease(state.context);
     },
-    async settle(converged) {
+    async settle(converged, applied = {}) {
       settled = true;
+      const workers = Object.keys(applied).length
+        ? { workers: { ...previous?.workers, ...applied } }
+        : {};
+      if (!converged && workers.workers)
+        storeEnrollment({ ...approval, ...workers }, env);
       if (converged)
         storeEnrollment(
           {
             ...approval,
+            ...workers,
             appliedRevision: revision,
             ...(previous?.appliedRevision &&
             previous.appliedRevision !== revision