@crvouga/dev-fleet 0.1.0 → 0.3.0

package/src/cli/roster.ts

@@ -1,22 +1,23 @@
-import { lstat, mkdir, open, readFile, rename, unlink } from 'node:fs/promises';
+import { mkdir, open, unlink } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { randomUUID } from 'node:crypto';
-import { parseConfig, record } from '../config';
+import { parseConfig } from '../config';
 import { FleetUsageError } from '../errors';
 import type { FleetConfig, FleetNode } from '../types';
 import { capabilities } from './options';
 import type { CliOptions } from './options';
+import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+import {
+  mutateState,
+  stateContext,
+  type Standing,
+  type StateContext,
+} from '../state/sync';
 
 export async function readRoster(path: string): Promise<FleetConfig | null> {
   try {
-    const info = await lstat(path);
-    if (!info.isFile() || info.size > 1024 * 1024)
-      throw new FleetUsageError(
-        'Roster must be a regular file smaller than 1 MiB; symlinks are not supported.'
-      );
-    return parseConfig(JSON.parse(await readFile(path, 'utf8')) as unknown);
+    const text = await readRosterText(path);
+    return text === null ? null : parseConfig(JSON.parse(text) as unknown);
   } catch (error) {
-    if (record(error) && error['code'] === 'ENOENT') return null;
     if (error instanceof FleetUsageError) throw error;
     throw new FleetUsageError(
       'Cannot read a valid roster. Use fleet-nodes --schema or choose --config PATH.'
@@ -24,6 +25,21 @@ export async function readRoster(path: string): Promise<FleetConfig | null> {
   }
 }
 
+/** The roster a fleet-sync needs: a missing one is an error, not an empty fleet. */
+export async function loadRoster(path: string): Promise<FleetConfig> {
+  let value: unknown;
+  try {
+    const text = await readRosterText(path);
+    if (text === null) throw new Error();
+    value = JSON.parse(text);
+  } catch {
+    throw new FleetUsageError(
+      'Cannot read fleet config. Run devfleet fleet-node-add or use --config PATH.'
+    );
+  }
+  return parseConfig(value);
+}
+
 export function nodeFromOptions(options: CliOptions): FleetNode {
   if (!options.id || !options.ssh || !options.path)
     throw new FleetUsageError(
@@ -46,18 +62,32 @@ function appendNode(
   return parseConfig({ version: 1, nodes: [...(existing?.nodes ?? []), node] });
 }
 
+export interface AddedNode {
+  roster: FleetConfig;
+  /** How the fleet state stands with the bucket; absent for a roster file. */
+  standing?: Standing;
+}
+
 export async function addNode(
   path: string,
   node: FleetNode,
-  apply: boolean
-): Promise<FleetConfig> {
-  if (!apply) return appendNode(await readRoster(path), node);
+  apply: boolean,
+  context: () => StateContext = stateContext
+): Promise<AddedNode> {
+  const save = async (): Promise<FleetConfig> => {
+    const roster = appendNode(await readRoster(path), node);
+    await writeRosterText(path, `${JSON.stringify(roster, null, 2)}\n`);
+    return roster;
+  };
+  if (!apply) return { roster: appendNode(await readRoster(path), node) };
+  if (path === STATE_ROSTER) {
+    const { value, standing } = await mutateState(context(), save);
+    return { roster: value, standing };
+  }
   await mkdir(dirname(path), { recursive: true, mode: 0o700 });
   const lock = await acquireLock(path);
   try {
-    const roster = appendNode(await readRoster(path), node);
-    await atomicWrite(path, roster);
-    return roster;
+    return { roster: await save() };
   } finally {
     await lock.close();
     await unlink(`${path}.lock`);
@@ -73,17 +103,3 @@ async function acquireLock(path: string) {
     );
   }
 }
-
-async function atomicWrite(path: string, roster: FleetConfig): Promise<void> {
-  const temporary = `${path}.${randomUUID()}.tmp`;
-  const handle = await open(temporary, 'wx', 0o600);
-  try {
-    await handle.writeFile(`${JSON.stringify(roster, null, 2)}\n`);
-    await handle.sync();
-    await handle.close();
-    await rename(temporary, path);
-  } finally {
-    await handle.close();
-    await unlink(temporary).catch(() => undefined);
-  }
-}

package/src/cli/settings.ts

@@ -1,10 +1,11 @@
 import { access } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
-import { homedir } from 'node:os';
-import { fileURLToPath } from 'node:url';
+import { PACKAGE_ROOT, packageInfo } from '../package-info';
+import { STATE_ROSTER } from '../state/roster';
 
 export interface Settings {
   workspace: string;
+  /** A roster file, or `state:roster` for the roster in the fleet state. */
   config: string;
 }
 
@@ -30,16 +31,17 @@ async function findUp(start: string, name: string): Promise<string | null> {
 /** A dev-fleet checkout, as opposed to any other project that has a mise.toml. */
 const CHECKOUT_MARKER = 'bootstrap/fleet-manager-run.sh';
 
-/** Where this CLI's own files live: the package root, also when run by npx. */
-const packageRoot = (): string =>
-  fileURLToPath(new URL('../..', import.meta.url));
-
+/** Where the CLI runs from. A published package is its own workspace, whatever
+ * project it is run in; code run from a checkout uses the dev-fleet checkout
+ * the command is started in (development).
+ */
 async function findCheckout(start: string): Promise<string> {
+  if (packageInfo().revision) return PACKAGE_ROOT;
   let current = resolve(start);
   while (true) {
     if (await exists(join(current, CHECKOUT_MARKER))) return current;
     const parent = dirname(current);
-    if (parent === current) return packageRoot();
+    if (parent === current) return PACKAGE_ROOT;
     current = parent;
   }
 }
@@ -54,11 +56,12 @@ export async function resolveSettings(
       env['PERSONAL_WORKSPACE'] ??
       (await findCheckout(cwd))
   );
-  const config = resolve(
+  // A roster file is named explicitly; otherwise it is kept in the fleet state.
+  const named =
     overrides.config ??
-      env['PERSONAL_FLEET_CONFIG'] ??
-      (await findUp(cwd, '.fleet.local.json')) ??
-      join(homedir(), '.config/personal/fleet.json')
-  );
+    env['PERSONAL_FLEET_CONFIG'] ??
+    (await findUp(cwd, '.fleet.local.json'));
+  const config =
+    !named || named === STATE_ROSTER ? STATE_ROSTER : resolve(named);
   return { workspace, config };
 }

package/src/cli/source-operations.ts

@@ -1,6 +1,8 @@
 import type { Command } from 'commander';
 import type { CommandOutcome } from './contracts';
 import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
 import { join } from 'node:path';
 import type { Settings } from './settings';
 
@@ -30,23 +32,45 @@ export function sourceOperation(args: string[]): string | undefined {
     : undefined;
 }
 
+/** These owners install from Git (a service keeps the commit it runs), so
+ * they run from a checkout: the one this CLI is in, or, for an installed
+ * package, the fleet-owned checkout of the revision fleet-sync applied on
+ * this machine. Null before the first fleet-sync.
+ */
+export function ownerCheckout(
+  workspace: string,
+  home = homedir()
+): string | null {
+  if (existsSync(join(workspace, '.git'))) return workspace;
+  const applied = join(home, '.local/share/personal/fleet/current');
+  return existsSync(join(applied, '.git')) ? applied : null;
+}
+
+const CHECKOUT_REQUIRED =
+  'This command runs from the revision fleet-sync applied on this machine, and none is applied yet. Run devfleet fleet-sync first.\n';
+
 export async function forwardSourceOperation(
   args: string[],
   settings: Settings
 ): Promise<number> {
   const operation = SOURCE_OPERATIONS[args[0]!]!;
   const [runtime, ...ownerArgs] = operation.command;
+  const checkout = ownerCheckout(settings.workspace);
+  if (!checkout) {
+    process.stderr.write(CHECKOUT_REQUIRED);
+    return 1;
+  }
   const resolved = ownerArgs.map((arg) =>
-    arg.includes('/') ? join(settings.workspace, arg) : arg
+    arg.includes('/') ? join(checkout, arg) : arg
   );
   return new Promise((resolve) => {
     const child = spawn(runtime!, [...resolved, ...args.slice(1)], {
-      cwd: settings.workspace,
+      cwd: checkout,
       stdio: 'inherit',
     });
     child.on('error', () => {
       process.stderr.write(
-        'This command requires the dev-fleet checkout and its toolchain.\n'
+        'This command requires python3 and the toolchain fleet-sync installs.\n'
       );
       resolve(1);
     });

package/src/cli/tailnet.ts

@@ -9,7 +9,7 @@ import { discoverTailnet } from '../tailscale';
 import { tailnetHealth } from '../tailnet-health';
 import { makeReport, renderReport, reportExitCode } from '../report';
 import { readRoster } from './roster';
-import { PUBLIC_HUB_URL } from '../monitoring/beszel';
+import { publicHubUrl } from '../monitoring/beszel';
 import { stableUrls } from './fleet-sync-output';
 import { timeout } from './options';
 import type { FleetServices } from './fleet';
@@ -89,7 +89,7 @@ export async function executeTailnet(
 
 /** Shown to people: the stable public name, never the tailnet address. */
 function publicDashboard(privateHub: string | null): string | null {
-  return privateHub ? PUBLIC_HUB_URL : null;
+  return privateHub ? publicHubUrl() : null;
 }
 
 async function healthContext(

package/src/definition.ts

@@ -4,6 +4,13 @@ import { FleetUsageError } from './errors';
 import { parseConfig, record } from './config';
 import type { FleetConfig } from './types';
 import { parseSecretsConfig, type SecretsConfig } from './secrets';
+import { parseLocation, type BucketLocation } from './state/bucket';
+import {
+  parseIdes,
+  parseT3,
+  type IdeId,
+  type T3Definition,
+} from './ide/definition';
 
 export interface MachineDefinition {
   id: string;
@@ -26,16 +33,20 @@ export interface ManagerDefinition {
 }
 export interface FleetDefinition {
   version: 3;
-  /** The Git repository that holds this fleet's configuration and pins. */
+  /** The Git repository machines take dev-fleet's code from. */
   source: { repository: string };
   /** Where this fleet keeps its secrets. */
   secrets: SecretsConfig;
+  /** The bucket that holds the fleet state. Its credentials are never part
+   * of the configuration: they come from the environment of the command.
+   */
+  state?: BucketLocation;
   manager: ManagerDefinition;
   /** Stateless, replaceable Linux boxes; clients of the manager's services. */
   workers: MachineDefinition[];
   services: {
     /** Public HTTPS origin of the CLI proxy every node uses. */
-    proxy: { endpoint: string };
+    proxy: { endpoint: string; accounts?: AccountPolicy };
     monitoring: {
       hubPort: 8443;
       /** Public dashboard origin. */
@@ -47,8 +58,28 @@ export interface FleetDefinition {
     };
   };
   worker?: WorkerPolicy;
+  /** The IDEs every machine runs; `["t3"]` when not named (idesOf). */
+  ides?: IdeId[];
+  /** What every machine's T3 holds beyond its provider instances. */
+  t3?: T3Definition;
   ubuntu: { release: '26.04'; architectures: ['amd64', 'arm64'] };
 }
+/** How provider accounts are watched and chosen (src/accounts/). */
+export interface AccountPolicy {
+  /** Fewer usable accounts of a provider than this is a warning; none fails. */
+  minimumUsable?: number;
+  /** Per provider (`claude`, `codex`, `antigravity`): which account an
+   * unpinned request goes to first. Accounts cooling down are skipped.
+   */
+  selection?: Record<string, AccountSelection>;
+}
+export interface AccountSelection {
+  strategy: 'fill-first' | 'round-robin' | 'least-recently-limited';
+  /** Account labels (sign-in emails) first, in this order; the others
+   * follow in the order of their routing prefix.
+   */
+  order?: string[];
+}
 const ID = /^[a-z][a-z0-9-]{0,62}$/;
 const HOST = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$/;
 function fail(): never {
@@ -137,11 +168,14 @@ export function parseDefinition(value: unknown): FleetDefinition {
       'version',
       'source',
       'secrets',
+      'state',
       'manager',
       'workers',
       'services',
       'ubuntu',
       'worker',
+      'ides',
+      't3',
     ])
   )
     return fail();
@@ -155,6 +189,7 @@ export function parseDefinition(value: unknown): FleetDefinition {
     version: 3,
     source: source(value['source']),
     secrets: secrets(value['secrets']),
+    ...(value['state'] === undefined ? {} : { state: state(value['state']) }),
     manager: lead,
     workers,
     services: services(value['services']),
@@ -162,6 +197,9 @@ export function parseDefinition(value: unknown): FleetDefinition {
     ...(value['worker'] === undefined
       ? {}
       : { worker: parseWorkerPolicy(value['worker']) }),
+    // Kept only when named: an earlier revision must still read the copy.
+    ...(value['ides'] === undefined ? {} : { ides: parseIdes(value['ides']) }),
+    ...(value['t3'] === undefined ? {} : { t3: parseT3(value['t3']) }),
   };
 }
 /** Singleton services name no host: they always run on the manager. */
@@ -170,15 +208,66 @@ function services(value: unknown): FleetDefinition['services'] {
   const { proxy, monitoring } = value;
   if (
     !record(proxy) ||
-    !keys(proxy, ['endpoint']) ||
+    !keys(proxy, ['endpoint', 'accounts']) ||
     !origin(proxy['endpoint'])
   )
     return fail();
   return {
-    proxy: { endpoint: proxy['endpoint'] },
+    proxy: {
+      endpoint: proxy['endpoint'],
+      ...(proxy['accounts'] === undefined
+        ? {}
+        : { accounts: accountPolicy(proxy['accounts']) }),
+    },
     monitoring: monitoringService(monitoring),
   };
 }
+const STRATEGIES = ['fill-first', 'round-robin', 'least-recently-limited'];
+function accountPolicy(value: unknown): AccountPolicy {
+  if (!record(value) || !keys(value, ['minimumUsable', 'selection']))
+    return fail();
+  const { minimumUsable, selection } = value;
+  if (
+    minimumUsable !== undefined &&
+    !(Number.isInteger(minimumUsable) && Number(minimumUsable) >= 1)
+  )
+    return fail();
+  return {
+    ...(minimumUsable === undefined
+      ? {}
+      : { minimumUsable: minimumUsable as number }),
+    ...(selection === undefined
+      ? {}
+      : { selection: selectionPolicies(selection) }),
+  };
+}
+function selectionPolicies(value: unknown): Record<string, AccountSelection> {
+  if (!record(value)) return fail();
+  const providers: Record<string, AccountSelection> = {};
+  for (const [provider, entry] of Object.entries(value)) {
+    if (!/^[a-z][a-z0-9-]{0,31}$/.test(provider)) return fail();
+    providers[provider] = accountSelection(entry);
+  }
+  return providers;
+}
+function accountSelection(entry: unknown): AccountSelection {
+  if (
+    !record(entry) ||
+    !keys(entry, ['strategy', 'order']) ||
+    !STRATEGIES.includes(String(entry['strategy']))
+  )
+    return fail();
+  const { order } = entry;
+  if (
+    order !== undefined &&
+    !(Array.isArray(order) && order.every((label) => account(label)))
+  )
+    return fail();
+  return {
+    strategy: entry['strategy'] as AccountSelection['strategy'],
+    ...(order ? { order: order as string[] } : {}),
+  };
+}
 function monitoringService(
   value: unknown
 ): FleetDefinition['services']['monitoring'] {
@@ -198,6 +287,18 @@ const account = (value: unknown): value is string =>
 function secrets(value: unknown): SecretsConfig {
   return parseSecretsConfig(value) ?? fail();
 }
+/** Says what is wrong with the block itself: a bucket has several settings. */
+function state(value: unknown): BucketLocation {
+  const bucket = parseLocation(value);
+  if (bucket.kind === 'directory') return { directory: bucket.path };
+  const { bucket: name, endpoint, region } = bucket;
+  return {
+    bucket: name,
+    ...(endpoint ? { endpoint } : {}),
+    ...(region ? { region } : {}),
+    prefix: bucket.prefix,
+  };
+}
 function source(value: unknown): FleetDefinition['source'] {
   if (
     !record(value) ||
@@ -221,6 +322,17 @@ function baseline(ubuntu: unknown): FleetDefinition['ubuntu'] {
   return ubuntu as FleetDefinition['ubuntu'];
 }
 
+/** The configuration as every machine keeps it at
+ * ~/.config/dev-fleet/fleet.json: without the bucket. Only the devfleet CLI
+ * on the manager reaches the bucket, and a launcher that still runs the
+ * revision from before the block existed must go on accepting the file while
+ * a sync is under way.
+ */
+export function installedDefinition(definition: FleetDefinition): string {
+  const { state: _bucket, ...machine } = definition;
+  return JSON.stringify(machine, null, 2) + '\n';
+}
+
 export function loadDefinition(path: string): FleetDefinition {
   try {
     return parseDefinition(JSON.parse(readFileSync(path, 'utf8')));

package/src/diagnostics.ts

@@ -71,7 +71,7 @@ export function diagnostic(node: string, check: Check): Diagnostic {
 }
 function scope(id: string): Diagnostic['scope'] {
   if (
-    /^(t3:manager-endpoint|ssh:|manager:|rollout:|source:|tailscale:(reachability|identity|enrolled-identity))/.test(
+    /^(t3:manager-(endpoint|origin)|ssh:|manager:|rollout:|source:|tailscale:(reachability|identity|enrolled-identity))/.test(
       id
     )
   )

package/src/fleet-config.ts

@@ -7,8 +7,12 @@ import {
   writeFileSync,
 } from 'node:fs';
 import { homedir } from 'node:os';
-import { join } from 'node:path';
-import { parseDefinition, type FleetDefinition } from './definition';
+import { dirname, join } from 'node:path';
+import {
+  installedDefinition,
+  parseDefinition,
+  type FleetDefinition,
+} from './definition';
 import { FleetUsageError } from './errors';
 import { pinnedPath } from './agents/toolchain';
 import type { Runner } from './types';
@@ -17,14 +21,62 @@ import {
   type SecretCommand,
   type SecretsProvider,
 } from './secrets';
+import { isOperator, stateDefinition } from './state/operator';
+import type { BucketLocation } from './state/bucket';
+import { definitionPath, stateRoot } from './state/paths';
+
+/** Where every machine keeps the fleet configuration fleet-sync approved.
+ * Launchers and scheduled jobs get no environment and read it from here.
+ */
+const installedPath = (home: string): string =>
+  join(home, '.config/dev-fleet/fleet.json');
+
+/** Where this process reads the fleet configuration from, in this order:
+ * - `file`: the file `DEVFLEET_CONFIG` names. The devfleet CLI sets it to the
+ *   `fleet.json` of the project it is run from (src/cli/project.ts);
+ * - `state`: the `definition` document in the manager's fleet state, the
+ *   configuration fleet-sync last applied, which only the devfleet CLI
+ *   reads. `path` is its copy as a file;
+ * - `installed`: the copy fleet-sync installed on this machine.
+ */
+export interface DefinitionSource {
+  kind: 'file' | 'state' | 'installed';
+  path: string;
+}
 
-/** Where every machine keeps the fleet configuration. fleet-sync installs the
- * approved copy here, so launchers and scheduled jobs find it without any
- * environment. `DEVFLEET_CONFIG` names another file (the manager's checkout of
- * its configuration repository, or a test fixture).
+export function definitionSource(
+  env = process.env,
+  home = homedir()
+): DefinitionSource {
+  const root = stateRoot(env, home);
+  const copy = definitionPath(root);
+  const named = env['DEVFLEET_CONFIG'];
+  // A devfleet command names the state's copy for the commands it starts.
+  const inherited = named === copy && isOperator();
+  if (named && !inherited) return { kind: 'file', path: named };
+  if (stateDefinition(root) !== null) return { kind: 'state', path: copy };
+  return { kind: 'installed', path: installedPath(home) };
+}
+
+function writePrivate(path: string, content: string): boolean {
+  if (existsSync(path) && readFileSync(path, 'utf8') === content) return false;
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  writeFileSync(`${path}.new`, content, { mode: 0o600 });
+  renameSync(`${path}.new`, path);
+  return true;
+}
+
+/** The file that holds the configuration this process runs with, for the
+ * subprocesses that take a path. Every command this process starts reads the
+ * same one.
  */
 export function configPath(env = process.env, home = homedir()): string {
-  return env['DEVFLEET_CONFIG'] || join(home, '.config/dev-fleet/fleet.json');
+  const source = definitionSource(env, home);
+  if (source.kind !== 'state') return source.path;
+  const body = stateDefinition(stateRoot(env, home));
+  if (body !== null) writePrivate(source.path, body);
+  env['DEVFLEET_CONFIG'] = source.path;
+  return source.path;
 }
 
 /** Installs the configuration this process runs with as the machine's copy,
@@ -32,30 +84,64 @@ export function configPath(env = process.env, home = homedir()): string {
  * Returns whether the file changed.
  */
 export function installConfig(home = homedir()): boolean {
-  const target = join(home, '.config/dev-fleet/fleet.json');
-  const content = JSON.stringify(fleet, null, 2) + '\n';
-  if (existsSync(target) && readFileSync(target, 'utf8') === content)
-    return false;
-  mkdirSync(join(home, '.config/dev-fleet'), { recursive: true, mode: 0o700 });
-  writeFileSync(`${target}.new`, content, { mode: 0o600 });
-  renameSync(`${target}.new`, target);
-  return true;
+  return writePrivate(installedPath(home), installedDefinition(fleet));
+}
+
+function read(path: string): FleetDefinition {
+  let value: unknown;
+  try {
+    value = JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    throw new FleetUsageError(
+      `The fleet configuration at ${path} is not readable JSON.`
+    );
+  }
+  return parseDefinition(value);
 }
 
 function load(): FleetDefinition {
   const path = configPath();
   if (!existsSync(path))
     throw new FleetUsageError(
-      `No fleet configuration at ${path}. Copy examples/fleet.json, edit it for your machines, and set DEVFLEET_CONFIG or place it there.`
+      `No fleet configuration. Run devfleet from the project that holds your fleet.json (devfleet fleet-state --example prints one to start from), or name the file with DEVFLEET_CONFIG. A manager whose bucket already holds one restores it with devfleet fleet-state --init. Looked at ${path}.`
     );
-  return parseDefinition(JSON.parse(readFileSync(path, 'utf8')));
+  return read(path);
+}
+
+/** Where the configuration says the fleet state is kept, read from the file
+ * each time: null when it names no bucket, or when this machine has no
+ * configuration yet.
+ */
+export function declaredState(
+  env = process.env,
+  home = homedir()
+): BucketLocation | null {
+  const path = configPath(env, home);
+  return existsSync(path) ? (read(path).state ?? null) : null;
+}
+
+let loaded: FleetDefinition | undefined;
+const current = (): FleetDefinition => (loaded ??= load());
+
+/** Forgets the configuration read so far; the next use reads it again. For
+ * a command that has just replaced the fleet state under itself.
+ */
+export function reloadFleet(): void {
+  loaded = undefined;
 }
 
 /** This fleet's own configuration: its machines, endpoints, accounts and source
  * repository. Fleet code never names any of them; it reads them here, and this
- * is the only module that loads the file.
+ * is the only module that loads the file. It is read on first use, so a
+ * command that needs no configuration runs on a machine that has none yet.
  */
-export const fleet = load();
+export const fleet: FleetDefinition = new Proxy({} as FleetDefinition, {
+  get: (_, key) => Reflect.get(current(), key),
+  has: (_, key) => Reflect.has(current(), key),
+  ownKeys: () => Reflect.ownKeys(current()),
+  getOwnPropertyDescriptor: (_, key) =>
+    Reflect.getOwnPropertyDescriptor(current(), key),
+});
 
 /** Reads one of this fleet's secrets for this node. Tools run from the pinned
  * toolchain, whatever PATH the caller (T3, launchd, systemd) started with.