@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/cli/fleet-node-park.ts

@@ -0,0 +1,238 @@
+import type { FleetDefinition, MachineDefinition } from '../definition';
+import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import { makeReport, renderReport } from '../report';
+import { runCommand } from '../runner';
+import { changeWorkers, readWorkerStates } from '../state/workers';
+import { stateContext, type StateContext } from '../state/sync';
+import type { Check, Runner } from '../types';
+import { duration, workerActivity } from '../worker-activity';
+import type { CommandOutcome } from './contracts';
+import { readRoster } from './roster';
+
+export interface NodeParkOptions {
+  id?: string;
+  /** Park although agent turns are under way, or cannot be counted. */
+  force?: boolean;
+  dryRun?: boolean;
+}
+
+export interface NodeParkServices {
+  run: Runner;
+  /** The roster: the fleet state's, or a file `--config` names. */
+  roster: string;
+  definition: () => FleetDefinition;
+  state: () => StateContext;
+  now: () => Date;
+  /** A normal fleet-sync of one worker, which verifies it. */
+  sync: (id: string) => Promise<CommandOutcome>;
+}
+
+export function parkServices(
+  roster: string,
+  sync: (id: string) => Promise<CommandOutcome>
+): NodeParkServices {
+  return {
+    run: runCommand,
+    roster,
+    definition: () => fleet,
+    state: () => stateContext(runCommand),
+    now: () => new Date(),
+    sync,
+  };
+}
+
+/** A hook may start or stop a machine through an infrastructure tool; it
+ * gets half an hour.
+ */
+const HOOK_TIMEOUT_MS = 1_800_000;
+
+function declared(definition: FleetDefinition, id: string | undefined) {
+  const worker = definition.workers.find((entry) => entry.id === id);
+  if (!worker)
+    throw new FleetUsageError('--id must name a worker fleet.json declares.');
+  return worker;
+}
+
+function outcome(
+  command: string,
+  id: string,
+  checks: Check[],
+  extra: Record<string, unknown> = {}
+): CommandOutcome {
+  const report = { ...makeReport(command, [{ node: id, checks }]), ...extra };
+  return {
+    exitCode: checks.some((check) => check.status === 'fail') ? 1 : 0,
+    data: report,
+    text: renderReport(report),
+  };
+}
+
+/** Runs the declared stop or start command with the worker ID last. */
+async function hook(
+  worker: MachineDefinition,
+  which: 'stop' | 'start',
+  definition: FleetDefinition,
+  run: Runner
+): Promise<Check | null> {
+  const command = definition.parking?.[which];
+  if (!command) return null;
+  const result = await run([...command, worker.id], {
+    timeoutMs: HOOK_TIMEOUT_MS,
+  });
+  return result.ok
+    ? {
+        id: 'worker:park-hook',
+        status: 'pass',
+        summary: `The declared ${which} command ran for ${worker.id}.`,
+      }
+    : {
+        id: 'worker:park-hook',
+        status: 'fail',
+        summary:
+          which === 'stop'
+            ? 'WORKER_STOP_HOOK_FAILED'
+            : 'WORKER_START_HOOK_FAILED',
+        remedy: {
+          action: `The ${which} command parking.${which} in fleet.json names failed for ${worker.id}, so its state was not changed. Run it by hand to see why, then run this command again.`,
+        },
+      };
+}
+
+/** Why a worker must not be parked now, unless forced: agent turns under
+ * way, or a worker that answers but whose turns cannot be counted. A worker
+ * that does not answer is off already.
+ */
+async function busy(
+  worker: MachineDefinition,
+  services: NodeParkServices
+): Promise<Check | null> {
+  const node = (await readRoster(services.roster))?.nodes.find(
+    (entry) => entry.id === worker.id
+  );
+  const activity = node ? await workerActivity(node, services.run) : null;
+  if (!activity || activity.runningSessions === 0) return null;
+  return {
+    id: 'worker:parked',
+    status: 'fail',
+    summary:
+      activity.runningSessions === null
+        ? 'WORKER_ACTIVITY_UNREADABLE'
+        : 'WORKER_BUSY',
+    remedy: {
+      action:
+        activity.runningSessions === null
+          ? `${worker.id}'s T3 could not say whether agent turns are under way, so it was not parked. Look at it, then repeat with --force to park it anyway.`
+          : `${worker.id} runs ${activity.runningSessions} agent turn(s); stopping it would end them. Wait for them to finish, or repeat with --force.`,
+    },
+  };
+}
+
+/** Parks a worker: it is stopped or destroyed on purpose, so it is no longer
+ * contacted, never the canary and does not fail the fleet. Refuses while
+ * agent turns run on it, unless --force. Runs the declared stop command.
+ */
+export async function fleetNodePark(
+  options: NodeParkOptions,
+  services: NodeParkServices
+): Promise<CommandOutcome> {
+  const definition = services.definition();
+  const worker = declared(definition, options.id);
+  const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
+  if (parkedAt)
+    return outcome('fleet-node-park', worker.id, [
+      {
+        id: 'worker:parked',
+        status: 'pass',
+        summary: `${worker.id} is parked since ${parkedAt}.`,
+      },
+    ]);
+  if (!options.force) {
+    const refused = await busy(worker, services);
+    if (refused) return outcome('fleet-node-park', worker.id, [refused]);
+  }
+  if (options.dryRun)
+    return outcome('fleet-node-park', worker.id, [
+      {
+        id: 'worker:parked',
+        status: 'skip',
+        summary: `Planned: ${definition.parking?.stop ? 'run the declared stop command and ' : ''}park ${worker.id}.`,
+      },
+    ]);
+  const stopped = await hook(worker, 'stop', definition, services.run);
+  if (stopped?.status === 'fail')
+    return outcome('fleet-node-park', worker.id, [stopped]);
+  const at = services.now().toISOString();
+  await changeWorkers(services.state(), (current) => ({
+    [worker.id]: { ...current[worker.id], parkedAt: at },
+  }));
+  return outcome('fleet-node-park', worker.id, [
+    ...(stopped ? [stopped] : []),
+    {
+      id: 'worker:parked',
+      status: 'pass',
+      summary: `${worker.id} is parked since ${at}: fleet-sync and fleet-status no longer contact it.`,
+    },
+  ]);
+}
+
+/** Brings a parked worker back: runs the declared start command, marks it
+ * active, then runs a normal fleet-sync of that worker, which verifies it.
+ * The exit code is the sync's.
+ */
+export async function fleetNodeResume(
+  options: NodeParkOptions,
+  services: NodeParkServices
+): Promise<CommandOutcome> {
+  const definition = services.definition();
+  const worker = declared(definition, options.id);
+  const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
+  if (options.dryRun)
+    return outcome('fleet-node-resume', worker.id, [
+      {
+        id: 'worker:parked',
+        status: 'skip',
+        summary: parkedAt
+          ? `Planned: ${definition.parking?.start ? 'run the declared start command, ' : ''}mark ${worker.id} active and run fleet-sync --node ${worker.id}.`
+          : `Planned: ${worker.id} is active already; run fleet-sync --node ${worker.id}.`,
+      },
+    ]);
+  const checks: Check[] = [];
+  if (parkedAt) {
+    const started = await hook(worker, 'start', definition, services.run);
+    if (started?.status === 'fail')
+      return outcome('fleet-node-resume', worker.id, [started]);
+    if (started) checks.push(started);
+    await changeWorkers(services.state(), (current) => {
+      const { parkedAt: _parked, ...rest } = current[worker.id] ?? {};
+      return { [worker.id]: rest };
+    });
+    const since = (services.now().getTime() - Date.parse(parkedAt)) / 1000;
+    checks.push({
+      id: 'worker:parked',
+      status: 'pass',
+      summary: `${worker.id} is active again after ${duration(Math.max(0, since))} parked.`,
+    });
+  }
+  const synced = await services.sync(worker.id);
+  const report = synced.data as { ok?: boolean };
+  checks.push(
+    synced.exitCode === 0 && report.ok !== false
+      ? {
+          id: 'rollout:node',
+          status: 'pass',
+          summary: `fleet-sync --node ${worker.id} converged and verified it.`,
+        }
+      : {
+          id: 'rollout:node',
+          status: 'fail',
+          summary: 'WORKER_RESUME_FAILED',
+          remedy: {
+            action: `fleet-sync --node ${worker.id} did not end ready; its report (sync in this output) says why. Fix that, then run devfleet fleet-sync --node ${worker.id}.`,
+          },
+        }
+  );
+  return outcome('fleet-node-resume', worker.id, checks, {
+    sync: synced.data,
+  });
+}

package/src/cli/fleet-node-seed.ts

@@ -0,0 +1,167 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { FleetDefinition } from '../definition';
+import { tailscalePins, ubuntuRelease } from '../platform/ubuntu-release';
+import { FleetUsageError } from '../errors';
+import { fleet, secret } from '../fleet-config';
+import type { SecretsProvider } from '../secrets';
+import {
+  controlRemedy,
+  fleetTailnetControl,
+  loginServer,
+  TailnetControlError,
+  type ControlHttp,
+} from '../tailnet-control';
+import type { Check } from '../types';
+import { generateHostKey, renderSeed, seedUserData } from '../worker-seed';
+import type { CommandOutcome } from './contracts';
+import {
+  keyName,
+  lifecycleServices,
+  pinHostKey,
+  type NodeLifecycleServices,
+} from './fleet-node-lifecycle';
+
+export interface NodeSeedOptions {
+  id?: string;
+  output?: string;
+  publicKey?: string;
+  autoinstall?: boolean;
+  apply?: boolean;
+  yes?: boolean;
+}
+
+export interface NodeSeedServices extends Pick<
+  NodeLifecycleServices,
+  'run' | 'playbook' | 'knownHosts' | 'state' | 'now'
+> {
+  definition: () => FleetDefinition;
+  secret: SecretsProvider;
+  home: string;
+  /** The tailnet control plane's HTTP; answered in scenarios. */
+  http?: ControlHttp;
+}
+
+export function seedServices(roster: string): NodeSeedServices {
+  const { run, playbook, knownHosts, state, now } = lifecycleServices(roster);
+  return {
+    run,
+    playbook,
+    knownHosts,
+    state,
+    now,
+    definition: () => fleet,
+    secret,
+    home: homedir(),
+  };
+}
+
+/** Writes the first-boot seed of a declared worker to a private file: the
+ * seed carries a single-use auth key, so it is never printed. With --apply
+ * --yes the manager pins the host key the seed installs, so the new machine
+ * is trusted without a first-use step.
+ */
+export async function fleetNodeSeed(
+  options: NodeSeedOptions,
+  services: NodeSeedServices
+): Promise<CommandOutcome> {
+  const worker = services
+    .definition()
+    .workers.find((entry) => entry.id === options.id);
+  if (!worker)
+    throw new FleetUsageError('--id must name a worker fleet.json declares.');
+  if (!options.output)
+    throw new FleetUsageError(
+      '--output PATH is required: the seed holds a single-use auth key and is never printed.'
+    );
+  const keyFile =
+    options.publicKey ?? join(services.home, '.ssh/id_ed25519.pub');
+  const managerKey = await readFile(keyFile, 'utf8').catch(() => {
+    throw new FleetUsageError(
+      `No manager public key at ${keyFile}; name one with --public-key PATH.`
+    );
+  });
+  const tailnet = services.definition().tailnet;
+  const authKey = await seedAuthKey(worker.id, services);
+  const release = ubuntuRelease(services.definition().ubuntu.release);
+  if (!release)
+    throw new FleetUsageError(
+      'fleet.json names an Ubuntu release the support matrix does not list.'
+    );
+  const hostKey = await generateHostKey(worker.id, services.run);
+  const seed = renderSeed(
+    await seedUserData({
+      worker,
+      managerKey,
+      authKey,
+      hostKey,
+      tailscale: tailscalePins(release),
+      codename: release.codename,
+      ...(loginServer(tailnet) ? { loginServer: loginServer(tailnet)! } : {}),
+    }),
+    Boolean(options.autoinstall)
+  );
+  await writeFile(options.output, seed, { mode: 0o600 });
+  const host = await keyName(worker, services.run);
+  const publicKey = hostKey.public.split(' ').slice(0, 2).join(' ');
+  // Pinned before the machine answers: in the fleet state for a worker
+  // reached from declared data, else in the operator's known_hosts.
+  const pin: Check =
+    options.apply && options.yes
+      ? await pinHostKey(worker, host, publicKey, services)
+      : {
+          id: 'ssh:host-key',
+          status: 'skip',
+          summary: `Planned: pin the seed's host key for ${host}; run again with --apply --yes.`,
+        };
+  return {
+    exitCode: pin.status === 'fail' ? 1 : 0,
+    data: {
+      version: 1,
+      command: 'fleet-node-seed',
+      ok: pin.status !== 'fail',
+      worker: worker.id,
+      output: options.output,
+      format: options.autoinstall ? 'autoinstall' : 'cloud-init',
+      hostKey: publicKey,
+      checks: [pin],
+    },
+    text: [
+      `Wrote the ${options.autoinstall ? 'autoinstall' : 'cloud-init'} seed of ${worker.id} to ${options.output} (private; it holds a single-use auth key).`,
+      `Host key: ${publicKey}`,
+      `${pin.status.toUpperCase()} ${pin.id}: ${pin.summary}`,
+      'Boot a stock Ubuntu image with it; once the machine is on the tailnet, add it with fleet-node-add and run devfleet fleet-sync --node ' +
+        worker.id +
+        '.',
+    ].join('\n'),
+  };
+}
+
+/** The key the seed carries: one minted for this worker by the control plane
+ * fleet.json declares, or else the secret tailnet-auth-key.
+ */
+async function seedAuthKey(
+  worker: string,
+  services: NodeSeedServices
+): Promise<string> {
+  const config = services.definition().tailnet;
+  if (config)
+    try {
+      const control = await fleetTailnetControl(
+        config,
+        services.secret,
+        services.http
+      );
+      return (await control!.createAuthKey({ worker })).key;
+    } catch (error) {
+      if (!(error instanceof TailnetControlError)) throw error;
+      throw new FleetUsageError(`${error.code}: ${controlRemedy(error.code)}`);
+    }
+  const key = await services.secret('tailnet-auth-key');
+  if (!key)
+    throw new FleetUsageError(
+      'The secrets provider holds no tailnet-auth-key. Declare tailnet in fleet.json with the secret tailnet-admin so a key is minted per worker, or store a short-lived, single-use, tagged Tailscale auth key there first.'
+    );
+  return key;
+}

package/src/cli/fleet-state.ts

@@ -486,13 +486,14 @@ async function restore(
   options: FleetStateOptions,
   services: FleetStateServices
 ): Promise<CommandOutcome> {
-  const target = context(services);
+  const target = context(services, options);
   if (!target.bucket)
     throw new FleetUsageError(
       'No bucket is configured, so there is nothing to restore from. Name it under "state" in fleet.json, or run devfleet fleet-state --init --bucket NAME --endpoint https://HOST.'
     );
   if (options.timestamp !== undefined) {
     // The bucket's current version is replaced too: --yes is the consent.
+    // Another manager's lease is not, and needs --take-over.
     if (!options.yes)
       throw new FleetUsageError(
         '--restore --timestamp replaces the fleet state in the bucket and on this machine with an earlier version. Repeat with --yes.'

package/src/cli/fleet-sync-checkout.ts

@@ -1,5 +1,6 @@
 import { spawn } from 'node:child_process';
 import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { prepareSource } from '../machine/source';
 import { PACKAGE_ROOT, packageInfo } from '../package-info';
@@ -16,6 +17,12 @@ export const RUN_CHECKOUT_ENV = 'PERSONAL_FLEET_RUN_CHECKOUT';
 
 const RUN_SCRIPT = 'bootstrap/fleet-manager-run.sh';
 
+/** Where the run script keeps the manager's fleet-owned checkouts, one per
+ * revision (`$fleet/manager-runs/<revision>`).
+ */
+export const managerRuns = (): string =>
+  join(homedir(), '.local/share/personal/fleet/manager-runs');
+
 export const inRunCheckout = (revision: string): boolean =>
   process.env[RUN_CHECKOUT_ENV] === revision;