@crvouga/dev-fleet 0.3.0 → 0.5.0

package/src/platform/debian-version.ts

@@ -0,0 +1,62 @@
+/** Debian version ordering, as `dpkg --compare-versions` applies it:
+ * epoch, then upstream version, then revision. Negative when `a` sorts
+ * before `b`, zero when they are the same version, positive after. */
+export function compareDebianVersions(a: string, b: string): number {
+  const [epochA, upstreamA, revisionA] = split(a);
+  const [epochB, upstreamB, revisionB] = split(b);
+  return (
+    epochA - epochB ||
+    comparePart(upstreamA, upstreamB) ||
+    comparePart(revisionA, revisionB)
+  );
+}
+
+function split(version: string): [number, string, string] {
+  const colon = version.indexOf(':');
+  const epoch = colon > 0 ? Number(version.slice(0, colon)) || 0 : 0;
+  const rest = colon > 0 ? version.slice(colon + 1) : version;
+  const dash = rest.lastIndexOf('-');
+  return dash > 0
+    ? [epoch, rest.slice(0, dash), rest.slice(dash + 1)]
+    : [epoch, rest, '0'];
+}
+
+/** `~` sorts before everything, even the end of the string; letters sort
+ * before every other character. */
+function order(char: string | undefined): number {
+  if (char === undefined) return 0;
+  if (char === '~') return -1;
+  if (/[A-Za-z]/.test(char)) return char.charCodeAt(0);
+  return char.charCodeAt(0) + 256;
+}
+
+/** Alternating runs: text, then digits, then text again, starting with text. */
+function runs(part: string): string[] {
+  return part
+    .match(/\D*\d*/g)!
+    .filter(Boolean)
+    .flatMap((run) => {
+      const digits = /\d*$/.exec(run)![0];
+      return [run.slice(0, run.length - digits.length), digits];
+    });
+}
+
+function compareText(a: string, b: string): number {
+  for (let i = 0; i < Math.max(a.length, b.length); i += 1) {
+    const difference = order(a[i]) - order(b[i]);
+    if (difference) return difference;
+  }
+  return 0;
+}
+
+function comparePart(a: string, b: string): number {
+  const left = runs(a);
+  const right = runs(b);
+  for (let i = 0; i < Math.max(left.length, right.length); i += 2) {
+    const difference =
+      compareText(left[i] ?? '', right[i] ?? '') ||
+      Number(left[i + 1] || 0) - Number(right[i + 1] || 0);
+    if (difference) return difference;
+  }
+  return 0;
+}

package/src/platform/ubuntu-audit.ts

@@ -1,5 +1,5 @@
-import policy from '../../ansible/ubuntu/policy.json';
 import type { Runner } from '../types';
+import { packagePins, UBUNTU_RELEASES, ubuntuRelease } from './ubuntu-release';
 
 export interface UbuntuAudit {
   supported: boolean;
@@ -11,7 +11,13 @@ export interface UbuntuAudit {
   passwordlessSudo: boolean;
   rebootPending: boolean;
 }
-const PACKAGES = [...Object.keys(policy.packages), 'tailscale'];
+// Every package any accepted release pins, so one audit serves them all.
+const PACKAGES = [
+  ...new Set([
+    ...UBUNTU_RELEASES.flatMap((release) => Object.keys(packagePins(release))),
+    'tailscale',
+  ]),
+];
 
 /** Read-only audit: never emit package sources, environment, accounts or raw errors. */
 export async function auditUbuntu(run: Runner): Promise<UbuntuAudit> {
@@ -32,14 +38,15 @@ export async function auditUbuntu(run: Runner): Promise<UbuntuAudit> {
   const version =
     /^VERSION_ID="?([0-9]+\.[0-9]+)"?$/m.exec(release.stdout)?.[1] ?? null;
   const reported = arch.stdout.trim();
-  const architecture = policy.architectures.includes(reported)
+  const accepted = ubuntuRelease(version);
+  const architecture = accepted?.architectures.includes(reported)
     ? reported
     : null;
   return {
     supported:
       release.ok &&
       /^ID=ubuntu$/m.test(release.stdout) &&
-      version === policy.release &&
+      accepted !== undefined &&
       architecture !== null,
     release: version,
     architecture,

package/src/platform/ubuntu-release.ts

@@ -0,0 +1,41 @@
+import policy from '../../ansible/ubuntu/policy.json';
+
+/** One Ubuntu release workers may run. The accepted releases are named once,
+ * in ansible/ubuntu/policy.json; nothing else spells a release or codename. */
+export interface UbuntuRelease {
+  release: string;
+  codename: string;
+  /** Debian architecture names, as `dpkg --print-architecture` reports them. */
+  architectures: string[];
+  /** Pins of this release that differ from the policy's `packages`. */
+  packages?: Record<string, string>;
+}
+
+export const UBUNTU_RELEASES: readonly UbuntuRelease[] = policy.releases;
+
+/** The package pins of one release: the policy's, with the release's own
+ * pins over them (as tasks/release.yml combines them for Ansible). */
+export function packagePins(
+  release: UbuntuRelease | undefined
+): Record<string, string> {
+  return { ...policy.packages, ...release?.packages };
+}
+
+/** `uname -m` for each Debian architecture name. */
+const KERNEL_ARCHITECTURES: Record<string, string> = {
+  amd64: 'x86_64',
+  arm64: 'aarch64',
+};
+
+export function ubuntuRelease(
+  version: string | null | undefined
+): UbuntuRelease | undefined {
+  return UBUNTU_RELEASES.find((entry) => entry.release === version);
+}
+
+/** The `uname -m` names one release runs on. */
+export function kernelArchitectures(release: UbuntuRelease): string[] {
+  return release.architectures.map(
+    (name) => KERNEL_ARCHITECTURES[name] ?? name
+  );
+}

package/src/remote-bootstrap.ts

@@ -4,10 +4,26 @@ import { join } from 'node:path';
 import { prepareSource } from './machine/source';
 import { isRevision } from './machine/state';
 import { sshArgs, shellQuote, sshFailure } from './ssh';
-import type { FleetNode, NodeReport, Runner, Check } from './types';
+import type {
+  FleetNode,
+  NodeReport,
+  Runner,
+  Check,
+  CommandResult,
+} from './types';
+import {
+  followOperation,
+  realClock,
+  retrying,
+  RETRY_PAUSES_MS,
+  transient,
+  type Clock,
+  type Follow,
+} from './remote-operation';
 import { record } from './config';
 import { installedDefinition, type FleetDefinition } from './definition';
 import { cleanCheck, validCheck } from './protocol';
+import { UBUNTU_RELEASES } from './platform/ubuntu-release';
 
 export interface BootstrapPayload {
   revision: string;
@@ -94,9 +110,11 @@ async function bundleRevision(
 export async function bootstrapRemote(
   node: FleetNode,
   payload: BootstrapPayload,
-  run: Runner
+  run: Runner,
+  follow: Follow = {}
 ): Promise<NodeReport> {
-  const cached = await cachedRevision(node, payload.revision, run);
+  const clock = follow.clock ?? realClock;
+  const cached = await cachedRevision(node, payload.revision, run, clock);
   if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
   const transfer = cached
     ? { mode: 'cached', bundle: '' }
@@ -110,30 +128,72 @@ export async function bootstrapRemote(
     payload.digest,
     node.id,
     transfer.mode,
+    UBUNTU_RELEASES.map((entry) => entry.release).join(' '),
   ]
     .map(shellQuote)
     .join(' ');
-  const result = await run([...sshArgs(node), command], {
-    input: bootstrapInput(payload.config, transfer.bundle),
-    timeoutMs: 5_400_000,
-  });
-  const evidence = parseEvidence(result);
-  if (evidence) return { node: node.id, checks: evidence };
-  if (!result.ok) return { node: node.id, checks: [sshFailure(result)] };
-  return {
-    node: node.id,
-    checks: [
-      {
-        id: 'bootstrap:protocol',
-        status: 'fail',
-        summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
-        remedy: {
-          action:
-            'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
-        },
+  const checks = await bootstrapSessions(
+    node,
+    payload,
+    { command, input: bootstrapInput(payload.config, transfer.bundle) },
+    { run, follow: { ...follow, clock } }
+  );
+  return { node: node.id, checks };
+}
+
+/** Opens the bootstrap session, and again after a lost one unless the
+ * worker's own operation can be followed to its end instead.
+ */
+async function bootstrapSessions(
+  node: FleetNode,
+  payload: BootstrapPayload,
+  session: { command: string; input: string },
+  context: { run: Runner; follow: Follow & { clock: Clock } }
+): Promise<Check[]> {
+  const { run, follow } = context;
+  let result: CommandResult | undefined;
+  for (const pause of [0, ...RETRY_PAUSES_MS]) {
+    if (result) {
+      // The session was lost, not the operation: the worker keeps running
+      // it, so the manager follows it rather than starting another.
+      await follow.clock.pause(pause);
+      const followed = await followOperation(
+        node,
+        payload.revision,
+        run,
+        follow
+      );
+      if (followed) return followed;
+    }
+    result = await run([...sshArgs(node), session.command], {
+      input: session.input,
+      timeoutMs: 5_400_000,
+    });
+    const evidence = parseEvidence(result);
+    if (evidence && locked(evidence))
+      return (
+        (await followOperation(node, payload.revision, run, follow)) ?? evidence
+      );
+    if (evidence) return evidence;
+    if (result.reason !== 'exit' || !transient(result)) break;
+  }
+  if (!result!.ok) return [sshFailure(result!)];
+  return [
+    {
+      id: 'bootstrap:protocol',
+      status: 'fail',
+      summary: 'REMOTE_BOOTSTRAP_EVIDENCE_INVALID',
+      remedy: {
+        action:
+          'Inspect the source-owned remote operation and lock. Do not assume a manager disconnect stopped installation.',
       },
-    ],
-  };
+    },
+  ];
+}
+
+/** Another operation holds the worker: its lock was the first thing tried. */
+function locked(checks: Check[]): boolean {
+  return checks.some((check) => check.summary === 'FLEET_REMOTE_LOCKED');
 }
 
 /** What the bootstrap reads on stdin: one line with the fleet configuration
@@ -151,10 +211,14 @@ export function bootstrapInput(
 async function cachedRevision(
   node: FleetNode,
   revision: string,
-  run: Runner
+  run: Runner,
+  clock: Clock
 ): Promise<boolean | Check> {
   const command = `d="$HOME/.local/share/personal/fleet/revisions/${revision}"; test ! -L "$d" && test -d "$d/.git" && test "$(git -C "$d" rev-parse HEAD 2>/dev/null)" = '${revision}' && test "$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)" = "$d" && clean=$(git -C "$d" status --porcelain 2>/dev/null) && test -z "$clean" && printf ready`;
-  const result = await run([...sshArgs(node), command], { timeoutMs: 20_000 });
+  const result = await retrying(
+    () => run([...sshArgs(node), command], { timeoutMs: 20_000 }),
+    clock
+  );
   if (!result.ok) {
     const failure = sshFailure(result);
     if (result.reason !== 'exit' || failure.id !== 'ssh:probe') return failure;

package/src/remote-operation.ts

@@ -0,0 +1,213 @@
+import { record } from './config';
+import { cleanCheck, validCheck } from './protocol';
+import { sshArgs, sshFailure } from './ssh';
+import type { Check, CommandResult, FleetNode, Runner } from './types';
+
+/** How the manager waits: a pause, and the time now. Tests answer both. */
+export interface Clock {
+  pause(ms: number): Promise<void>;
+  now(): number;
+}
+
+export const realClock: Clock = {
+  pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  now: () => Date.now(),
+};
+
+/** A session that could not be opened, or was lost, is tried again after
+ * these pauses: three attempts over thirty seconds.
+ */
+export const RETRY_PAUSES_MS = [10_000, 20_000];
+/** How often an operation the manager follows is looked at again. */
+export const FOLLOW_EVERY_MS = 15_000;
+/** As long as a bootstrap may run. */
+export const FOLLOW_AT_MOST_MS = 5_400_000;
+
+/** Failures of the connection, not of the worker: worth another attempt. A
+ * changed host key, a refused login or an unknown name never are.
+ */
+export function transient(result: CommandResult): boolean {
+  if (result.ok) return false;
+  const { id } = sshFailure(result);
+  return ['ssh:disconnected', 'ssh:network', 'ssh:timeout'].includes(id);
+}
+
+/** Runs a short SSH session, again after a pause while its connection fails. */
+export async function retrying(
+  attempt: () => Promise<CommandResult>,
+  clock: Clock
+): Promise<CommandResult> {
+  let result = await attempt();
+  for (const pause of RETRY_PAUSES_MS) {
+    if (!transient(result)) return result;
+    await clock.pause(pause);
+    result = await attempt();
+  }
+  return result;
+}
+
+/** What a worker says about the fleet operation on it. */
+export interface Operation {
+  /** The operation lock is held by a running process. */
+  held: boolean;
+  /** The process that recorded the operation is still running. */
+  alive: boolean;
+  revision: string | null;
+  phase: string | null;
+  /** Seconds since the epoch, on the worker's clock. */
+  started: number | null;
+  finished: boolean;
+  /** What node convergence reported, once the operation finished. */
+  result: Check[] | null;
+}
+
+/** Read-only. Prints whether the lock is held, whether the recorded process
+ * runs, the operation record and its result, one per line. Holding the lock
+ * for a moment to test it is not possible while another holds it, and takes
+ * nothing from anyone when nobody does.
+ */
+export const OPERATION_PROBE = [
+  'r="$HOME/.local/share/personal/fleet"',
+  'l="$r/remote-operation.lock"',
+  'held=no',
+  'if [ -f "$l" ] && [ ! -L "$l" ]; then if command -v flock >/dev/null 2>&1; then flock -n "$l" true || held=yes; else python3 -c \'import fcntl, sys; fcntl.flock(open(sys.argv[1], "a"), fcntl.LOCK_EX | fcntl.LOCK_NB)\' "$l" 2>/dev/null || held=yes; fi; fi',
+  'op=$(head -c 4096 "$r/remote-operation.json" 2>/dev/null | head -n 1)',
+  'pid=$(printf %s "$op" | sed -n \'s/.*"pid":\\([0-9][0-9]*\\).*/\\1/p\')',
+  'alive=no',
+  'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=yes; fi',
+  'printf \'held=%s\\nalive=%s\\noperation=%s\\nresult=\' "$held" "$alive" "$op"',
+  'head -c 1048576 "$r/remote-operation-result.json" 2>/dev/null | head -n 1',
+  "printf '\\n'",
+].join('\n');
+
+function field(stdout: string, name: string): string | null {
+  const line = stdout.split('\n').find((entry) => entry.startsWith(`${name}=`));
+  return line === undefined ? null : line.slice(name.length + 1);
+}
+
+function json(text: string | null): unknown {
+  try {
+    return text ? JSON.parse(text) : null;
+  } catch {
+    return null;
+  }
+}
+
+function evidence(value: unknown): Check[] | null {
+  return record(value) &&
+    value['version'] === 1 &&
+    Array.isArray(value['checks']) &&
+    value['checks'].length > 0 &&
+    value['checks'].length <= 200 &&
+    value['checks'].every(validCheck)
+    ? value['checks'].map(cleanCheck)
+    : null;
+}
+
+export function parseOperation(stdout: string): Operation | null {
+  const held = field(stdout, 'held');
+  const alive = field(stdout, 'alive');
+  if (held === null || alive === null) return null;
+  const operation = json(field(stdout, 'operation'));
+  const op = record(operation) ? operation : {};
+  const text = (value: unknown, pattern: RegExp): string | null =>
+    typeof value === 'string' && pattern.test(value) ? value : null;
+  return {
+    held: held === 'yes',
+    alive: alive === 'yes',
+    revision: text(op['revision'], /^[a-f0-9]{40}$/),
+    phase: text(op['phase'], /^[a-z0-9-]{1,40}$/),
+    started: typeof op['started'] === 'number' ? op['started'] : null,
+    finished: typeof op['finished'] === 'number',
+    result: evidence(json(field(stdout, 'result'))),
+  };
+}
+
+export async function inspectOperation(
+  node: FleetNode,
+  run: Runner,
+  clock: Clock
+): Promise<Operation | null> {
+  const result = await retrying(
+    () => run([...sshArgs(node), OPERATION_PROBE], { timeoutMs: 20_000 }),
+    clock
+  );
+  return result.ok ? parseOperation(result.stdout) : null;
+}
+
+/** An operation of this revision whose process still runs: this is the run
+ * the manager lost, or one another manager run started; either way it is
+ * the run to follow, not one to start beside it.
+ */
+function followable(operation: Operation, revision: string): boolean {
+  return (
+    operation.held &&
+    operation.alive &&
+    !operation.finished &&
+    operation.revision === revision
+  );
+}
+
+function finishedWith(operation: Operation, revision: string): Check[] | null {
+  return !operation.held &&
+    operation.finished &&
+    operation.revision === revision
+    ? operation.result
+    : null;
+}
+
+export interface Follow {
+  clock?: Clock;
+  /** Told the phase and how long the operation has run, at every look. */
+  progress?: (phase: string) => void;
+}
+
+/** After a lost session or a held lock: the result of the operation of this
+ * revision on the worker, followed to its end when it still runs. Null when
+ * there is none to follow: another revision, another holder, or nothing.
+ */
+export async function followOperation(
+  node: FleetNode,
+  revision: string,
+  run: Runner,
+  options: Follow = {}
+): Promise<Check[] | null> {
+  const clock = options.clock ?? realClock;
+  const deadline = clock.now() + FOLLOW_AT_MOST_MS;
+  let operation = await inspectOperation(node, run, clock);
+  if (!operation) return null;
+  const done = finishedWith(operation, revision);
+  if (done) return [attached(operation, 'finished'), ...done];
+  if (!followable(operation, revision)) return null;
+  let last: Operation = operation;
+  while (clock.now() < deadline) {
+    options.progress?.(describe(last, clock));
+    await clock.pause(FOLLOW_EVERY_MS);
+    operation = await inspectOperation(node, run, clock);
+    if (!operation) continue;
+    last = operation;
+    const result = finishedWith(operation, revision);
+    if (result) return [attached(operation, 'followed'), ...result];
+    if (!followable(operation, revision)) return null;
+  }
+  return null;
+}
+
+function describe(operation: Operation, clock: Clock): string {
+  const elapsed =
+    operation.started === null
+      ? ''
+      : `, ${Math.max(0, Math.round(clock.now() / 1000 - operation.started))} s`;
+  return `Following the running operation (${operation.phase ?? 'unknown phase'}${elapsed})`;
+}
+
+function attached(_operation: Operation, how: 'finished' | 'followed'): Check {
+  return {
+    id: 'bootstrap:attached',
+    status: 'pass',
+    summary:
+      how === 'followed'
+        ? 'The manager lost or could not take this machine’s operation of the same revision, followed it to its end and reports its result.'
+        : 'This machine finished the operation of the same revision the manager lost; its result is reported.',
+  };
+}

package/src/schedule-check.ts

@@ -0,0 +1,40 @@
+import type { History } from './state/history';
+import type { Check } from './types';
+
+/** How often the manager's schedule runs fleet-sync. */
+export const SCHEDULE_INTERVAL_SECONDS = 1800;
+
+/** `manager:schedule`: a schedule that is on must have succeeded within two
+ * of its intervals. A schedule that was never turned on is no check at all.
+ */
+export function scheduleCheck(history: History, now: Date): Check | null {
+  const { schedule } = history;
+  if (!schedule?.enabled) return null;
+  const window = 2 * schedule.intervalSeconds * 1000;
+  const lastSuccess = history.runs
+    .filter((run) => run.trigger === 'schedule' && run.ok)
+    .map((run) => Date.parse(run.finishedAt))
+    .reduce((latest, at) => Math.max(latest, at), Number.NEGATIVE_INFINITY);
+  // A schedule turned on a moment ago has had no chance to run yet.
+  const since = Math.max(lastSuccess, Date.parse(schedule.changedAt));
+  const minutes = Math.round(schedule.intervalSeconds / 60);
+  if (now.getTime() - since <= window)
+    return {
+      id: 'manager:schedule',
+      status: 'pass',
+      summary: Number.isFinite(lastSuccess)
+        ? `The schedule last converged the fleet at ${new Date(lastSuccess).toISOString()}.`
+        : 'The schedule was turned on and has not had to run yet.',
+    };
+  return {
+    id: 'manager:schedule',
+    status: 'fail',
+    summary: Number.isFinite(lastSuccess)
+      ? `The schedule has not converged the fleet since ${new Date(lastSuccess).toISOString()}, more than two intervals of ${minutes} minutes.`
+      : `The schedule has not converged the fleet once since it was turned on, more than two intervals of ${minutes} minutes ago.`,
+    remedy: {
+      action:
+        'Run devfleet fleet-sync --schedule status on the manager to see the last scheduled runs, then devfleet fleet-report for the failing checks. Re-enable it with devfleet fleet-sync --schedule enable --apply --yes when the manager was replaced.',
+    },
+  };
+}

package/src/secrets.ts

@@ -12,7 +12,20 @@ export const SECRET_NAMES = [
   'monitoring-tunnel-token',
   'proxy-tunnel-token',
 ] as const;
-export type SecretName = (typeof SECRET_NAMES)[number];
+/** Secrets only a feature the fleet declares needs: a Vault location may
+ * leave them out, and they are then unavailable.
+ */
+export const OPTIONAL_SECRET_NAMES = [
+  // The off-machine storage of the monitoring hub's backups
+  // (services.monitoring.backups).
+  'monitoring-backup-access-key',
+  'monitoring-backup-secret-key',
+  // Where fleet-watch notifies (notify.targets).
+  'notify-webhook-url',
+  'notify-ntfy-url',
+] as const;
+export type SecretName =
+  (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];
 
 /** A secret's location in a Vault KV v2 mount. */
 export type VaultLocation = { path: string; field: string };
@@ -24,8 +37,13 @@ export type SecretsConfig =
       provider: 'vault';
       address: string;
       mount: string;
-      locations: Record<SecretName, VaultLocation>;
+      locations: VaultLocations;
     };
+export type VaultLocations = Record<
+  (typeof SECRET_NAMES)[number],
+  VaultLocation
+> &
+  Partial<Record<(typeof OPTIONAL_SECRET_NAMES)[number], VaultLocation>>;
 
 /** Runs a command and returns its stdout, or null when it fails. */
 export type SecretCommand = (
@@ -79,6 +97,7 @@ export function secretsProvider(
       );
   return async (name) => {
     const location = config.locations[name];
+    if (!location) return null;
     return present(
       await run([
         'vault',
@@ -142,13 +161,12 @@ const PARSERS: Record<string, (value: Fields) => SecretsConfig | null> = {
   },
 };
 
-function vaultLocations(
-  value: unknown
-): Record<SecretName, VaultLocation> | null {
+function vaultLocations(value: unknown): VaultLocations | null {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
   const fields = value as Fields;
-  if (!only(fields, [...SECRET_NAMES])) return null;
-  const valid = SECRET_NAMES.every((name) => {
+  if (!only(fields, [...SECRET_NAMES, ...OPTIONAL_SECRET_NAMES])) return null;
+  const optional = OPTIONAL_SECRET_NAMES.filter((name) => name in fields);
+  const valid = [...SECRET_NAMES, ...optional].every((name) => {
     const location = fields[name] as Fields | undefined;
     return (
       location !== null &&
@@ -158,7 +176,7 @@ function vaultLocations(
       text(location['field'])
     );
   });
-  return valid ? (fields as Record<SecretName, VaultLocation>) : null;
+  return valid ? (fields as VaultLocations) : null;
 }
 
 /** The `secrets` block of a fleet configuration, or null when it is invalid. */

package/src/ssh.ts

@@ -20,9 +20,9 @@ export function sshArgs(node: FleetNode): string[] {
     '-o',
     'ConnectionAttempts=1',
     '-o',
-    'ServerAliveInterval=5',
+    'ServerAliveInterval=15',
     '-o',
-    'ServerAliveCountMax=2',
+    'ServerAliveCountMax=4',
     '-o',
     'ClearAllForwardings=yes',
     '-o',
@@ -66,6 +66,14 @@ const FAILURES = [
     action:
       'Use ssh-add -l and ssh -vvv ALIAS locally; check User, IdentityFile, IdentitiesOnly, agent/keychain access and authorized_keys. Do not share debug logs without review.',
   },
+  {
+    pattern:
+      /Connection (?:to \S+ )?closed|Connection reset|Broken pipe|Timeout, server \S+ not responding|client_loop: send disconnect/i,
+    id: 'ssh:disconnected',
+    summary: 'The SSH session was lost while the operation ran.',
+    action:
+      'The worker keeps running what it started. Rerun fleet-sync: it follows the operation of the same revision to its end. Check the network between the manager and the worker if it repeats.',
+  },
   {
     pattern: /Connection refused|No route to host|Network is unreachable/i,
     id: 'ssh:network',