@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/cli/fleet-sync.ts

@@ -20,13 +20,22 @@ import {
 } from '../machine/state';
 import { loadEnrollment } from '../state/enrollment';
 import { legacyHome } from '../state/paths';
+import {
+  appendRun,
+  summarizeRun,
+  type SyncRun,
+  type Trigger,
+} from '../state/history';
+import { capScheduleLogs } from './schedule-logs';
 import {
   DIVERGED_REMEDY,
+  mutateState,
   refresh,
   stateContext,
   type Standing,
 } from '../state/sync';
 import { runRecord } from './fleet-sync-state';
+import { planSources, planText, syncPlan } from './fleet-sync-plan';
 import { runCommand } from '../runner';
 import type { Runner } from '../types';
 import type { Settings } from './settings';
@@ -36,6 +45,7 @@ import { FleetLockError } from '../machine/lock-error';
 import { selected } from './fleet-sync-remote';
 import { managerIdentity } from './fleet-sync-manager';
 import { managerRunScript, rehome } from './fleet-sync-checkout';
+import { syncAccounts } from './fleet-sync-accounts';
 import { packageInfo } from '../package-info';
 import {
   forwardingDisabled,
@@ -53,6 +63,8 @@ export interface FleetSyncOptions {
   node?: string[];
   all?: boolean;
   schedule?: ScheduleAction;
+  /** `accounts`: only the account-derived configuration, on every node. */
+  only?: string;
   /** Internal: this revision already converged the whole fleet, so no canary
    * has to prove it first and every worker converges at once.
    */
@@ -80,11 +92,44 @@ export async function fleetSync(
   if (!(await onManager(run)))
     return offManager(options, settings, run, forward);
   if (options.schedule === 'status') return scheduleStatus(settings, run);
-  return managerSync(options, settings, {
-    run,
-    root: machineRoot(),
-    restart: () => fleetSync(options, settings, run, forward),
-  });
+  if (options.only === 'accounts') return onlyAccounts(options, settings, run);
+  const sync = () =>
+    managerSync(options, settings, {
+      run,
+      root: machineRoot(),
+      restart: sync,
+    });
+  if (options.dryRun || options.schedule) return sync();
+  const trigger = runTrigger();
+  if (trigger === 'schedule') capScheduleLogs(join(machineRoot(), 'logs'));
+  const startedAt = new Date();
+  const outcome = await sync();
+  await recordRun(
+    summarizeRun(
+      outcome.data,
+      { startedAt, finishedAt: new Date() },
+      trigger,
+      outcome.exitCode
+    ),
+    run
+  );
+  return outcome;
+}
+
+/** Runs the manager's schedule starts say so in their environment. */
+function runTrigger(env = process.env): Trigger {
+  return env['DEVFLEET_TRIGGER'] === 'schedule' ? 'schedule' : 'operator';
+}
+
+/** Appends the run to the history in the fleet state and uploads it. A
+ * history that cannot be written never changes the run's result.
+ */
+async function recordRun(entry: SyncRun, run: Runner): Promise<void> {
+  try {
+    await mutateState(stateContext(run), () => appendRun(entry));
+  } catch {
+    // Reported by the next fleet-state or fleet-sync run.
+  }
 }
 
 export interface ManagerSync {
@@ -128,9 +173,9 @@ export async function managerSync(
       revision
     );
   if (rehomed !== 'here') return rehomed;
-  if (options.dryRun) return selected({ ...options, revision }, settings, run);
-  // The bucket is the source of truth: a run starts from its version of the
-  // configuration, the roster and the approval, not from a stale local copy.
+  // The bucket is the source of truth: a run, and a preview, starts from its
+  // version of the configuration, the roster and the approval, not from a
+  // stale local copy. Refreshing never uploads.
   const fresh = await refresh(stateContext(run));
   if (fresh.standing === 'diverged')
     return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
@@ -140,6 +185,13 @@ export async function managerSync(
     reloadFleet();
     return restart();
   }
+  if (options.dryRun)
+    return preview({ ...options, revision }, settings, {
+      run,
+      root,
+      applied: state?.appliedRevision ?? null,
+      standing: fresh.standing,
+    });
   return authorizeAndApply(root, state, options, {
     settings,
     run,
@@ -148,6 +200,40 @@ export async function managerSync(
   });
 }
 
+/** The preview: the read-only report of every machine, and the plan of what
+ * a sync would change on them.
+ */
+async function preview(
+  options: FleetSyncOptions & { revision: string },
+  settings: Settings,
+  context: {
+    run: Runner;
+    root: string;
+    applied: string | null;
+    standing: Standing;
+  }
+): Promise<CommandOutcome> {
+  const outcome = await selected(options, settings, context.run);
+  const report = outcome.data as { nodes?: { node?: unknown }[] };
+  const plan = await syncPlan(
+    {
+      from: context.applied,
+      to: options.revision,
+      state: context.standing,
+      nodes: (report.nodes ?? [])
+        .map((node) => node.node)
+        .filter((node): node is string => typeof node === 'string'),
+      sources: planSources(settings.workspace, context.root),
+    },
+    context.run
+  );
+  return {
+    ...outcome,
+    data: { ...(outcome.data as object), plan },
+    text: [outcome.text, planText(plan)].filter(Boolean).join('\n'),
+  };
+}
+
 interface ApplyContext {
   settings: Settings;
   run: Runner;
@@ -289,11 +375,43 @@ function validate(options: FleetSyncOptions): void {
     throw new FleetUsageError(
       '--revision requires an exact lowercase 40-character commit SHA.'
     );
+  validateOnly(options);
   if (options.dryRun && (options.apply || options.yes))
     throw new FleetUsageError(
       '--dry-run cannot be combined with apply confirmation.'
     );
 }
+function validateOnly(options: FleetSyncOptions): void {
+  if (options.only !== undefined && options.only !== 'accounts')
+    throw new FleetUsageError('--only takes one scope: accounts.');
+  if (options.only && (options.dryRun || options.schedule))
+    throw new FleetUsageError(
+      '--only accounts applies at once; it takes neither --dry-run nor --schedule.'
+    );
+}
+
+/** `--only accounts`: workers keep the revision they run; only their
+ * accounts change.
+ */
+async function onlyAccounts(
+  options: FleetSyncOptions,
+  settings: Settings,
+  run: Runner
+): Promise<CommandOutcome> {
+  const root = machineRoot();
+  await ensurePrivateRoot(root);
+  const state = await loadEnrollment(legacyHome() ? root : null);
+  const revision =
+    state?.approvedRevision ??
+    (await selectedRevision(options.revision, undefined, settings, run));
+  if (!isRevision(revision))
+    return actionRequired(
+      'FLEET_REVISION_REQUIRED',
+      'No revision is applied yet. Run a full fleet-sync first.'
+    );
+  return syncAccounts(options, settings, { revision, run });
+}
+
 function interactive(options: FleetSyncOptions): boolean {
   return (
     ![

package/src/cli/fleet-watch.ts

@@ -0,0 +1,198 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fleet, secret } from '../fleet-config';
+import {
+  deliver,
+  observe,
+  parseMemory,
+  transitions,
+  type Notification,
+  type NotifyConfig,
+} from '../notify';
+import { connection } from '../state/connection';
+import type { Bucket } from '../state/bucket';
+import type { SecretsProvider } from '../secrets';
+import type { Check, NodeReport } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetWatchOptions {
+  test?: boolean;
+}
+
+/** What the watcher remembers between runs, kept in the bucket beside the
+ * fleet state. Its time is the heartbeat an off-machine check reads.
+ */
+export interface WatchStore {
+  read: () => Promise<string | null>;
+  write: (text: string) => Promise<void>;
+}
+
+export interface FleetWatchServices {
+  notify: NotifyConfig | undefined;
+  /** The machines as `fleet-status --workers` sees them now. */
+  status: () => Promise<NodeReport[]>;
+  store: WatchStore | null;
+  secrets: SecretsProvider;
+  send: typeof fetch;
+  now: () => Date;
+}
+
+/** Compares the fleet's health with the previous run and notifies on
+ * transitions only. `--test` sends one test notification and nothing else.
+ */
+export async function fleetWatch(
+  options: FleetWatchOptions,
+  services: FleetWatchServices
+): Promise<CommandOutcome> {
+  const { notify } = services;
+  if (options.test) {
+    const checks = notify
+      ? await deliver(
+          notify,
+          { event: 'test', node: 'fleet-watch', checks: [] },
+          services.secrets,
+          services.send
+        )
+      : [unconfigured('fail')];
+    return outcome(checks, [
+      { event: 'test', node: 'fleet-watch', checks: [] },
+    ]);
+  }
+  const previous = services.store
+    ? parseMemory(await services.store.read().catch(() => null))
+    : null;
+  const observed = observe(await services.status());
+  const { notifications, memory } = transitions(
+    previous,
+    observed,
+    services.now(),
+    notify?.repeatHours ?? 0
+  );
+  const checks: Check[] = [];
+  for (const notification of notifications)
+    if (notify)
+      checks.push(
+        ...(await deliver(
+          notify,
+          notification,
+          services.secrets,
+          services.send
+        ))
+      );
+  if (!notify) checks.push(unconfigured('warn'));
+  checks.push(await heartbeat(services.store, JSON.stringify(memory)));
+  return outcome(checks, notifications);
+}
+
+function unconfigured(status: 'warn' | 'fail'): Check {
+  return {
+    id: 'notify:configured',
+    status,
+    summary: 'NOTIFY_NOT_CONFIGURED',
+    remedy: {
+      action:
+        'Add a "notify" block to fleet.json with its targets, and store each target address as its secret (notify-webhook-url, notify-ntfy-url).',
+    },
+  };
+}
+
+async function heartbeat(
+  store: WatchStore | null,
+  text: string
+): Promise<Check> {
+  const id = 'watch:heartbeat';
+  if (!store)
+    return {
+      id,
+      status: 'warn',
+      summary: 'WATCH_HEARTBEAT_NO_BUCKET',
+      remedy: {
+        action:
+          'Name the bucket under "state" in fleet.json: the heartbeat and what the watcher remembers are kept there.',
+      },
+    };
+  return store.write(text).then(
+    () => ({
+      id,
+      status: 'pass' as const,
+      summary: 'The heartbeat and the health of each machine were stored.',
+    }),
+    () => ({
+      id,
+      status: 'fail' as const,
+      summary: 'WATCH_HEARTBEAT_UNWRITTEN',
+      remedy: {
+        action:
+          'Check the bucket credentials this run was given (devfleet fleet-state shows the connection).',
+      },
+    })
+  );
+}
+
+function outcome(
+  checks: Check[],
+  notifications: Notification[]
+): CommandOutcome {
+  const ok = !checks.some((check) => check.status === 'fail');
+  const lines = [
+    ...notifications.map(
+      (entry) =>
+        `${entry.event} ${entry.node}${entry.checks.length ? `: ${entry.checks.join(', ')}` : ''}`
+    ),
+    ...checks.map((check) => `${check.status} ${check.id} ${check.summary}`),
+  ];
+  return {
+    exitCode: ok ? 0 : 1,
+    data: { version: 1, command: 'fleet-watch', ok, notifications, checks },
+    text: lines.join('\n') || 'Nothing changed.',
+  };
+}
+
+/** The watcher's object in the bucket, beside the fleet state. */
+export const WATCH_OBJECT = 'watch.json';
+
+export function bucketStore(bucket: Bucket | null): WatchStore | null {
+  if (!bucket) return null;
+  if (bucket.kind === 'directory') {
+    const path = join(bucket.path, WATCH_OBJECT);
+    return {
+      read: () => readFile(path, 'utf8').catch(() => null),
+      write: async (text) => {
+        await mkdir(dirname(path), { recursive: true });
+        await writeFile(path, text, { mode: 0o600 });
+      },
+    };
+  }
+  const client = new Bun.S3Client({
+    bucket: bucket.bucket,
+    ...(bucket.endpoint ? { endpoint: bucket.endpoint } : {}),
+    ...(bucket.region ? { region: bucket.region } : {}),
+    ...(bucket.accessKeyId && bucket.secretAccessKey
+      ? {
+          accessKeyId: bucket.accessKeyId,
+          secretAccessKey: bucket.secretAccessKey,
+        }
+      : {}),
+  });
+  const file = client.file(`${bucket.prefix}/${WATCH_OBJECT}`);
+  return {
+    read: async () => ((await file.exists()) ? file.text() : null),
+    write: async (text) => {
+      await file.write(text, { type: 'application/json' });
+    },
+  };
+}
+
+/** The services of a real run, on the manager. */
+export function watchServices(
+  status: () => Promise<NodeReport[]>
+): FleetWatchServices {
+  return {
+    notify: fleet.notify,
+    status,
+    store: bucketStore(connection().bucket),
+    secrets: secret,
+    send: fetch,
+    now: () => new Date(),
+  };
+}

package/src/cli/fleet.ts

@@ -14,6 +14,8 @@ import { addNode, nodeFromOptions, readRoster } from './roster';
 import { executeTailnet } from './tailnet';
 import { probeSource } from './probe-source';
 import { executeRepairCommand } from './repair';
+import { scheduleCheck } from '../schedule-check';
+import { readHistory } from '../state/history';
 
 export interface FleetServices {
   run: Runner;
@@ -125,6 +127,8 @@ async function diagnose(
     services.run
   );
   if (!options.local && settings.missingRoster) reports.push(missingRoster());
+  const schedule = scheduleCheck(readHistory(), new Date());
+  if (schedule) reports[0]?.checks.push(schedule);
   const report = {
     ...makeReport('doctor', reports),
     coverage: {

package/src/cli/register-fleet.ts

@@ -7,6 +7,10 @@ import { executeFleet } from './fleet';
 import { fleetSync, type FleetSyncOptions } from './fleet-sync';
 import { syncNode, type NodeSyncOptions } from './node-sync';
 import { fleetState, type FleetStateOptions } from './fleet-state';
+import { fleetAccounts, type FleetAccountsOptions } from './fleet-accounts';
+import { fleetReport, type FleetReportOptions } from './fleet-report';
+import { fleetWatch, watchServices } from './fleet-watch';
+import type { NodeReport } from '../types';
 
 type Complete = (outcome: CommandOutcome) => void;
 
@@ -198,6 +202,10 @@ function registerSync({ program, defaults, complete }: Registration) {
       '--schedule <action>',
       'Manager-only schedule: enable, pause, resume, disable, status'
     )
+    .option(
+      '--only <scope>',
+      'accounts: only the account-derived configuration, on every node at once, without a full sync'
+    )
     .action(async (_options, command: Command) => {
       const options = command.optsWithGlobals<
         FleetSyncOptions & Partial<Settings>
@@ -277,6 +285,52 @@ function registerState({ program, complete }: Registration) {
     });
 }
 
+function registerReport({ program, complete }: Registration) {
+  program
+    .command('fleet-report')
+    .description(
+      'Report how reliably fleet-sync ran: success rate, duration, failing checks and time not ready'
+    )
+    .option('--since <duration>', 'How far back: 30d (default), 12h or 90m')
+    .action((_options, command: Command) => {
+      complete(fleetReport(command.optsWithGlobals<FleetReportOptions>()));
+    });
+}
+
+function registerWatch({ program, defaults, complete }: Registration) {
+  program
+    .command('fleet-watch')
+    .description(
+      "Check the workers and notify the fleet's targets when one becomes unhealthy or recovers"
+    )
+    .option('--test', 'Send one test notification to every target and stop')
+    .addHelpText(
+      'after',
+      '\nRuns fleet-status --workers, compares it with the previous run kept in the bucket and notifies only on a change (and again every repeatHours while a worker stays unhealthy). Targets come from "notify" in fleet.json; their addresses are secrets. Every run stores a heartbeat in the bucket for an off-machine check.'
+    )
+    .action(async (_options, command: Command) => {
+      const options = command.optsWithGlobals<
+        CliOptions & { test?: boolean }
+      >();
+      const settings = await withSettings(defaults, options);
+      const status = async (): Promise<NodeReport[]> => {
+        const report = await executeFleet(
+          'health',
+          { ...options, workers: true, managed: true },
+          settings
+        );
+        const data = report.data as { nodes?: NodeReport[] };
+        return data.nodes ?? [];
+      };
+      complete(
+        await fleetWatch(
+          { ...(options.test ? { test: true } : {}) },
+          watchServices(status)
+        )
+      );
+    });
+}
+
 export function registerFleet(
   program: Command,
   defaults: Settings,
@@ -291,6 +345,38 @@ export function registerFleet(
   );
   registerSync(registration);
   registerState(registration);
+  registerReport(registration);
+  registerWatch(registration);
   registerRepair(registration);
   registerNodes(registration);
+  registerAccounts(registration);
+}
+
+function registerAccounts({ program, defaults, complete }: Registration) {
+  program
+    .command('fleet-accounts [args...]')
+    .description(
+      'List the provider accounts signed in on the CLI proxy and their state; add, sign in again or remove one'
+    )
+    .option('--diff', 'Also compare what this machine offers with the roster')
+    .option('--add', 'Sign a new account in: --add PROVIDER')
+    .option('--relogin', 'Sign one account in again: --relogin PROVIDER INDEX')
+    .option('--remove', 'Remove one account: --remove PROVIDER INDEX')
+    .option(
+      '--remove-native-logins',
+      'Remove provider logins agent CLIs keep outside the proxy on this machine; needs --yes'
+    )
+    .option('--yes', 'Confirm --remove-native-logins')
+    .action(async (args: string[], _options, command: Command) => {
+      const options = command.optsWithGlobals<
+        FleetAccountsOptions & CliOptions
+      >();
+      complete(
+        await fleetAccounts(
+          options,
+          args,
+          await withSettings(defaults, options)
+        )
+      );
+    });
 }

package/src/cli/registry.ts

@@ -50,6 +50,13 @@ export const COMMANDS: CommandSpec[] = [
       'Read-only canonical worker readiness, dependencies, services and authentication; lists every remaining gap.',
       ['workers', 'readiness', 'health', 'canonical'],
     ],
+    [
+      'fleet-watch',
+      'Notify when a worker becomes unhealthy or recovers',
+      'Runs fleet-status --workers, compares it with the previous run kept in the bucket and notifies the targets fleet.json names under "notify" on a change only. Stores a heartbeat for an off-machine check. --test sends one test notification.',
+      ['notify', 'alert', 'webhook', 'ntfy', 'heartbeat', 'watch', 'monitor'],
+      true,
+    ],
     [
       'fleet-discover',
       'Discover Tailscale computers',
@@ -90,6 +97,12 @@ export const COMMANDS: CommandSpec[] = [
       ['state', 'bucket', 'restore', 'recover', 'disposable'],
       true,
     ],
+    [
+      'fleet-report',
+      'Report how reliably fleet-sync runs',
+      'Read-only: success rate, mean duration, the most frequent failing check IDs and how long each machine was not ready, from the history of fleet-sync runs kept in the fleet state. --since 30d (default), 12h or 90m.',
+      ['history', 'reliability', 'report', 'failures', 'sync'],
+    ],
     [
       'fleet-node-sync',
       'Sync this node',
@@ -135,6 +148,19 @@ export const COMMANDS: CommandSpec[] = [
       ['onboarding', 'register', 'new', 'setup', 'add'],
       true,
     ],
+    [
+      'fleet-accounts',
+      'List provider accounts and their state',
+      'Read-only: the provider accounts signed in on the CLI proxy (provider, index, label, state, when a cooldown clears), the roster version and the proxy:account:<provider> checks. --diff also compares what this machine offers with the roster.',
+      ['accounts', 'providers', 'login', 'quota', 'cooldown', 'roster'],
+    ],
+    [
+      'fleet-accounts --add',
+      'Add, sign in again or remove a provider account',
+      '--add PROVIDER, --relogin PROVIDER INDEX or --remove PROVIDER INDEX changes one account on the CLI proxy (signing in is the only manual step), then propagates the accounts to every node with fleet-sync --only accounts.',
+      ['accounts', 'login', 'relogin', 'remove', 'add', 'sign in'],
+      true,
+    ],
   ]),
   ...group('T3', [
     [

package/src/cli/schedule-logs.ts

@@ -0,0 +1,36 @@
+import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** The most a scheduled job's log may hold; a longer one keeps its newest
+ * quarter.
+ */
+export const SCHEDULE_LOG_BYTES = 1024 * 1024;
+
+/** Caps the schedule's own logs (`<label>.out.log`, `<label>.err.log`) at the
+ * start of every scheduled run. launchd and systemd append to them, so the
+ * newest lines are what is kept. Nothing else in the directory is touched.
+ */
+export function capScheduleLogs(
+  directory: string,
+  limit = SCHEDULE_LOG_BYTES
+): void {
+  let names: string[];
+  try {
+    names = readdirSync(directory);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    if (!/^[a-z0-9-]+\.(out|err)\.log$/.test(name)) continue;
+    const path = join(directory, name);
+    try {
+      const stat = statSync(path);
+      if (!stat.isFile() || stat.size <= limit) continue;
+      const kept = readFileSync(path).subarray(stat.size - limit / 4);
+      const start = kept.indexOf(10) + 1;
+      writeFileSync(path, kept.subarray(start), { mode: 0o600 });
+    } catch {
+      // A log that cannot be capped is capped by the next run.
+    }
+  }
+}

package/src/clock.ts

@@ -0,0 +1,63 @@
+import policy from '../ansible/ubuntu/policy.json';
+import type { HealthSample } from './machine-health';
+import type { Check } from './types';
+
+const { warnOffsetSeconds, failOffsetSeconds } = policy.clock;
+
+/** A skewed clock breaks TLS, token validation, apt signatures and Vault
+ * logins with errors that never mention time. The clock passes when it is
+ * synchronised to a time source and, where a reference time was measured,
+ * within the offset floors of `policy.json`.
+ *
+ * `offsetSeconds` is the difference to the reference (the manager's clock),
+ * when one was measured.
+ */
+export function clockCheck(
+  id: string,
+  sample: Pick<HealthSample, 'clockSynced' | 'timeService'>,
+  offsetSeconds?: number
+): Check {
+  const offset =
+    offsetSeconds === undefined ? undefined : Math.abs(offsetSeconds);
+  const measured =
+    offset === undefined ? '' : `, ${offset.toFixed(1)} s from the manager`;
+  const remedy = clockRemedy(sample.timeService);
+  if (sample.clockSynced === undefined && offset === undefined)
+    return {
+      id,
+      status: 'skip',
+      summary: 'CLOCK_UNOBSERVED: the clock synchronisation could not be read.',
+    };
+  if (offset !== undefined && offset > failOffsetSeconds)
+    return { id, status: 'fail', summary: `CLOCK_SKEWED${measured}`, remedy };
+  if (sample.clockSynced === 0 && sample.timeService === 0)
+    return {
+      id,
+      status: 'fail',
+      summary: `CLOCK_NOT_SYNCHRONISED: no time service is enabled${measured}`,
+      remedy,
+    };
+  if (sample.clockSynced === 0)
+    return {
+      id,
+      status: 'warn',
+      summary: `CLOCK_NOT_SYNCHRONISED${measured}`,
+      remedy,
+    };
+  if (offset !== undefined && offset > warnOffsetSeconds)
+    return { id, status: 'warn', summary: `CLOCK_DRIFTING${measured}`, remedy };
+  return {
+    id,
+    status: 'pass',
+    summary: `The clock is synchronised to a time source${measured}.`,
+  };
+}
+
+function clockRemedy(timeService: number | undefined): { action: string } {
+  return {
+    action:
+      timeService === 0
+        ? 'Enable the time service: apply the approved fleet-sync revision, which keeps chrony or systemd-timesyncd enabled on workers (timedatectl set-ntp true is the same setting).'
+        : 'Check that the time service reaches a time server (timedatectl timesync-status) and that UDP port 123 is not blocked.',
+  };
+}