@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/cli/register-fleet.ts

@@ -10,7 +10,8 @@ import { fleetState, type FleetStateOptions } from './fleet-state';
 import { fleetAccounts, type FleetAccountsOptions } from './fleet-accounts';
 import { fleetReport, type FleetReportOptions } from './fleet-report';
 import { fleetWatch, watchServices } from './fleet-watch';
-import type { NodeReport } from '../types';
+import type { Check, NodeReport } from '../types';
+import { fleet } from '../fleet-config';
 import { registerGithubVerify } from './github-verify';
 import { fleetRestore, type FleetRestoreOptions } from './fleet-restore';
 import { fleetValidate, type FleetValidateOptions } from './fleet-validate';
@@ -30,6 +31,12 @@ import {
   lifecycleServices,
   type NodeLifecycleOptions,
 } from './fleet-node-lifecycle';
+import {
+  fleetNodePark,
+  fleetNodeResume,
+  parkServices,
+  type NodeParkOptions,
+} from './fleet-node-park';
 
 type Complete = (outcome: CommandOutcome) => void;
 
@@ -198,6 +205,10 @@ function registerSync({ program, defaults, complete }: Registration) {
       '--revision <sha>',
       'Converge to an exact source commit; defaults to the commit the dev-fleet package you run was published from'
     )
+    .option(
+      '--rollback',
+      'Converge to the revision applied before the current one (fleet state); the revision, tools and T3 go back, system packages roll forward only'
+    )
     .option(
       '--node <id>',
       'Explicit enrolled remote target; repeatable',
@@ -208,6 +219,10 @@ function registerSync({ program, defaults, complete }: Registration) {
       '--vault-password-login',
       'Opt-in: log a worker without a token of its own in with the shared Vault user, on its own terminal'
     )
+    .option(
+      '--restart-busy',
+      'Restart a worker T3 for a new release or policy even while agent turns run, which ends them; by default the restart waits until T3 is idle'
+    )
     .option(
       '--canary <id>',
       'The worker that proves a new revision first when it is reachable; defaults to canary in fleet.json, then the first worker'
@@ -225,6 +240,14 @@ function registerSync({ program, defaults, complete }: Registration) {
       'Accepted for compatibility; fleet-sync applies by default (schedule writes still need --apply --yes)'
     )
     .option('--yes', 'Accepted for compatibility with --apply')
+    .option(
+      '--maintenance-reboot',
+      'Reboot each worker whose installed updates wait for a reboot: one at a time, canary first, only while it runs no agent turn, verified ready before the next'
+    )
+    .option(
+      '--force',
+      'With --maintenance-reboot: reboot a worker even while agent turns run (they end)'
+    )
     .option(
       '--take-over',
       'Make this machine the manager while another one still holds the manager lease'
@@ -309,11 +332,11 @@ function registerState({ program, complete }: Registration) {
     .option('--yes', 'With --restore: discard changes the bucket does not hold')
     .option(
       '--timestamp <time>',
-      'With --restore: the version the bucket held at this time, uploaded again as the current one (a damaged replica)'
+      'With --restore: the version the bucket held at this time, uploaded again as the current one (a damaged replica); refused while another manager holds the manager lease, unless with --take-over'
     )
     .option(
       '--take-over',
-      'With --init or --import: change the fleet state while another manager still holds the manager lease'
+      'With --init, --import or --restore --timestamp: change the fleet state while another manager still holds the manager lease'
     )
     .addHelpText(
       'after',
@@ -361,10 +384,52 @@ function registerWatch({ program, defaults, complete }: Registration) {
         const data = report.data as { nodes?: NodeReport[] };
         return data.nodes ?? [];
       };
+      const autoPark = async (reports: NodeReport[]): Promise<Check[]> => {
+        const checks: Check[] = [];
+        for (const report of reports) {
+          if (
+            !report.checks.some(
+              (check) => check.id === 'worker:idle' && check.status === 'warn'
+            )
+          )
+            continue;
+          const worker = fleet.workers.find(
+            (entry) =>
+              entry.id === report.node ||
+              entry.network?.tailscaleName === report.node
+          );
+          if (!worker) continue;
+          const parked = await fleetNodePark(
+            { id: worker.id },
+            parkServices(settings.config, () =>
+              Promise.reject(new Error('fleet-watch never resumes a worker'))
+            )
+          );
+          checks.push({
+            id: 'worker:auto-park',
+            status: parked.exitCode === 0 ? 'pass' : 'warn',
+            summary:
+              parked.exitCode === 0
+                ? `${worker.id} was idle longer than declared and is parked.`
+                : 'WORKER_AUTO_PARK_REFUSED',
+            ...(parked.exitCode === 0
+              ? {}
+              : {
+                  remedy: {
+                    action: `fleet-node-park refused ${worker.id}; run devfleet fleet-node-park --id ${worker.id} to see why.`,
+                  },
+                }),
+          });
+        }
+        return checks;
+      };
       complete(
         await fleetWatch(
           { ...(options.test ? { test: true } : {}) },
-          watchServices(status)
+          {
+            ...watchServices(status),
+            ...(fleet.parking?.autoPark ? { autoPark } : {}),
+          }
         )
       );
     });
@@ -448,6 +513,52 @@ function registerLifecycle({ program, defaults, complete }: Registration) {
     )
     .option('--yes', 'Confirm an explicitly requested --apply operation')
     .action(action('retire'));
+  const parking =
+    (name: 'park' | 'resume') =>
+    async (_options: unknown, command: Command) => {
+      const options = command.optsWithGlobals<NodeParkOptions & CliOptions>();
+      const settings = await withSettings(defaults, options);
+      const services = parkServices(settings.config, (id) =>
+        fleetSync(
+          {
+            node: [id],
+            apply: true,
+            yes: true,
+            nonInteractive: true,
+            ...(options.json ? { json: true } : {}),
+          },
+          {
+            ...defaults,
+            ...(options.config ? { config: options.config } : {}),
+          }
+        )
+      );
+      complete(
+        name === 'park'
+          ? await fleetNodePark(options, services)
+          : await fleetNodeResume(options, services)
+      );
+    };
+  program
+    .command('fleet-node-park')
+    .description(
+      'Park a worker that is stopped or destroyed on purpose: it is no longer contacted, never the canary and does not fail the fleet'
+    )
+    .requiredOption('--id <name>', 'The declared worker to park')
+    .option(
+      '--force',
+      'Park although agent turns are under way on it, or cannot be counted'
+    )
+    .option('--dry-run', 'Say what would change; change nothing')
+    .action(parking('park'));
+  program
+    .command('fleet-node-resume')
+    .description(
+      'Bring a parked worker back: run the declared start command, then fleet-sync that worker and verify it'
+    )
+    .requiredOption('--id <name>', 'The declared worker to resume')
+    .option('--dry-run', 'Say what would change; change nothing')
+    .action(parking('resume'));
   program
     .command('fleet-node-seed')
     .description(

package/src/cli/registry.ts

@@ -207,6 +207,20 @@ export const COMMANDS: CommandSpec[] = [
       ['seed', 'cloud-init', 'autoinstall', 'new', 'bootstrap', 'provision'],
       true,
     ],
+    [
+      'fleet-node-park',
+      'Park a worker that is off on purpose',
+      'Record in the fleet state that a declared worker is stopped or destroyed on purpose (a rented machine nobody uses): fleet-sync and fleet-status list it as parked with the time since, never contact it or make it the canary, and it does not change the exit code. Refuses while agent turns run on it, unless --force. Runs the stop command fleet.json declares under parking.stop with the worker ID. --dry-run previews.',
+      ['park', 'stop', 'pause', 'idle', 'cost', 'rented', 'shutdown'],
+      true,
+    ],
+    [
+      'fleet-node-resume',
+      'Resume a parked worker',
+      'Run the start command fleet.json declares under parking.start with the worker ID, mark the worker active in the fleet state, then run fleet-sync --node ID, which converges and verifies it. The exit code is the sync’s. --dry-run previews.',
+      ['resume', 'start', 'unpark', 'wake', 'rented'],
+      true,
+    ],
     [
       'fleet-node-retire',
       'Retire a worker',

package/src/cli/tailnet.ts

@@ -5,13 +5,19 @@ import {
   appendManagerLinks,
   appendDashboardEvidence,
 } from './managed-health';
-import { discoverTailnet } from '../tailscale';
+import { discoverTailnet, type Tailnet } from '../tailscale';
+import type { FleetNode, NodeReport } from '../types';
+import { appendPublicReachability } from '../public-reachability';
 import { tailnetHealth } from '../tailnet-health';
 import { makeReport, renderReport, reportExitCode } from '../report';
 import { readRoster } from './roster';
 import { publicHubUrl } from '../monitoring/beszel';
 import { stableUrls } from './fleet-sync-output';
 import { timeout } from './options';
+import { appendUsage } from './worker-usage-report';
+import { appendTailnetLeftovers } from '../tailnet-cleanup';
+import { fleet } from '../fleet-config';
+import { withUsage } from '../worker-activity';
 import type { FleetServices } from './fleet';
 import type { Settings } from './settings';
 import type { CliOptions } from './options';
@@ -57,12 +63,21 @@ export async function executeTailnet(
   );
   if (options.managed)
     await appendManagerLinks(nodes, registered, services.run);
+  // From the manager, outside the tailnet: optional, it never blocks readiness.
+  if (options.workers) await appendPublicReachability(nodes, services.run);
   // The hub runs on the manager, which worker scope deliberately filters out of
   // `tailnet`; its visibility comes from the unfiltered discovery.
   const dashboard = options.workers
-    ? await appendDashboardEvidence(nodes, discovered)
+    ? await workerFleetEvidence(nodes, discovered, registered, services)
     : null;
-  const report = makeReport('health', nodes);
+  // Parked workers, and what each active one is doing (uptime, agent turns).
+  const usage = options.managed
+    ? await appendUsage(nodes, registered, {
+        run: services.run,
+        computers: discovered.computers,
+      })
+    : null;
+  const report = { ...makeReport('health', nodes), ...(usage ?? {}) };
   const exitCode = reportExitCode(report, true);
   const verified = exitCode === 0;
   return {
@@ -77,16 +92,34 @@ export async function executeTailnet(
         inspected: nodes.filter((node) => node.snapshot).length,
       },
     },
-    text: options.workers
-      ? workerHealthText(
-          { ...report, ok: verified },
-          coverage,
-          publicDashboard(dashboard)
-        )
-      : `${stableUrls(renderReport({ ...report, ok: verified }))}\nCoverage: ${coverage.computers} visible computers; ${coverage.excludedMobile} mobile devices excluded.\n${coverage.limitation}`,
+    text: withUsage(
+      options.workers
+        ? workerHealthText(
+            { ...report, ok: verified },
+            coverage,
+            publicDashboard(dashboard)
+          )
+        : `${stableUrls(renderReport({ ...report, ok: verified }))}\nCoverage: ${coverage.computers} visible computers; ${coverage.excludedMobile} mobile devices excluded.\n${coverage.limitation}`,
+      usage
+    ),
   };
 }
 
+/** What the canonical worker fleet is checked for beyond each worker: the
+ * dashboard's evidence, and devices tagged as workers that are no declared
+ * worker (#123). Returns the private hub address.
+ */
+async function workerFleetEvidence(
+  nodes: NodeReport[],
+  discovered: Tailnet,
+  registered: FleetNode[],
+  services: FleetServices
+): Promise<string | null> {
+  const dashboard = await appendDashboardEvidence(nodes, discovered);
+  await appendTailnetLeftovers(nodes, fleet, services.tailnet, registered);
+  return dashboard;
+}
+
 /** Shown to people: the stable public name, never the tailnet address. */
 function publicDashboard(privateHub: string | null): string | null {
   return privateHub ? publicHubUrl() : null;

package/src/cli/ui-data.ts

@@ -86,7 +86,9 @@ function nodeChecks(value: unknown): CheckRow[] {
 }
 
 export function isStatus(value: unknown): value is string {
-  return ['pass', 'warn', 'fail', 'skip'].includes(String(value));
+  return ['pass', 'warn', 'fail', 'skip', 'not-applicable'].includes(
+    String(value)
+  );
 }
 
 export function isCancellation(error: unknown): boolean {

package/src/cli/worker-usage-report.ts

@@ -0,0 +1,63 @@
+import { fleet } from '../fleet-config';
+import { readParked } from '../state/workers';
+import type { TailnetComputer } from '../tailscale';
+import type { FleetNode, NodeReport, Runner } from '../types';
+import { fleetUsage, usageSummary, type WorkerUsage } from '../worker-activity';
+import { parkedCheck, parkedOnline } from '../worker-parking';
+
+/** The workers fleet-status contacts: every one that is not parked. */
+export function activeNodes(nodes: FleetNode[]): FleetNode[] {
+  const parked = readParked();
+  return nodes.filter((node) => !parked[node.id]);
+}
+
+function reportOf(reports: NodeReport[], id: string): NodeReport {
+  const worker = fleet.workers.find((entry) => entry.id === id);
+  const found = reports.find(
+    (entry) =>
+      entry.node === id ||
+      (worker?.network && entry.node === worker.network.tailscaleName)
+  );
+  if (found) return found;
+  const added: NodeReport = { node: id, checks: [] };
+  reports.push(added);
+  return added;
+}
+
+/** Adds to fleet-status what it says about each worker beyond its health:
+ * parked workers as parked (and a warning when one is online anyway), and
+ * for every active worker it reaches its uptime, agent turns, last agent
+ * activity and `worker:idle`. Returns the `workers` of the JSON output and
+ * its `usage` summary.
+ */
+export async function appendUsage(
+  reports: NodeReport[],
+  nodes: FleetNode[],
+  {
+    run,
+    computers = [],
+    now = new Date(),
+  }: { run: Runner; computers?: TailnetComputer[]; now?: Date }
+): Promise<{ workers: WorkerUsage[]; usage: ReturnType<typeof usageSummary> }> {
+  const parked = readParked();
+  for (const worker of fleet.workers) {
+    const at = parked[worker.id];
+    if (!at) continue;
+    const report = reportOf(reports, worker.id);
+    report.checks.push(parkedCheck(worker.id, at, now));
+    const online = parkedOnline(
+      worker.id,
+      worker.network?.tailscaleName,
+      computers
+    );
+    if (online) report.checks.push(online);
+  }
+  const { workers, checks } = await fleetUsage(fleet, {
+    nodes: nodes.filter((node) => !parked[node.id]),
+    parked,
+    run,
+    now,
+  });
+  for (const [id, check] of checks) reportOf(reports, id).checks.push(check);
+  return { workers, usage: usageSummary(workers) };
+}

package/src/clock.ts

@@ -53,6 +53,48 @@ export function clockCheck(
   };
 }
 
+/** The offset of a worker's clock from the manager's, measured by the
+ * manager around one SSH round trip: the worker read its clock between
+ * `sentAt` and `receivedAt` (milliseconds on the manager's clock), so the
+ * offset is only known to lie in that window. The check judges the smallest
+ * offset the window allows, so a slow link never reads as a skewed clock.
+ */
+export function clockOffsetCheck(
+  remoteSeconds: number,
+  sentAt: number,
+  receivedAt: number
+): Check {
+  const remote = remoteSeconds * 1000;
+  const least =
+    remote > receivedAt
+      ? (remote - receivedAt) / 1000
+      : remote < sentAt
+        ? (sentAt - remote) / 1000
+        : 0;
+  const direction = remote > receivedAt ? 'ahead of' : 'behind';
+  const measured = `${least.toFixed(1)} s ${direction} the manager`;
+  const remedy = clockRemedy(undefined);
+  if (least > failOffsetSeconds)
+    return {
+      id: 'worker:clock-offset',
+      status: 'fail',
+      summary: `CLOCK_SKEWED ${measured}`,
+      remedy,
+    };
+  if (least > warnOffsetSeconds)
+    return {
+      id: 'worker:clock-offset',
+      status: 'warn',
+      summary: `CLOCK_DRIFTING ${measured}`,
+      remedy,
+    };
+  return {
+    id: 'worker:clock-offset',
+    status: 'pass',
+    summary: `The clock is within ${warnOffsetSeconds} s of the manager's.`,
+  };
+}
+
 function clockRemedy(timeService: number | undefined): { action: string } {
   return {
     action: