@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/definition.ts

@@ -1,15 +1,29 @@
-import { parseWorkerPolicy, type WorkerPolicy } from './worker-policy';
+import {
+  parseWorkerPolicy,
+  toolProfiles,
+  type WorkerPolicy,
+} from './worker-policy';
+import type { ToolProfile } from '../bootstrap/worker-tools';
 import { readFileSync } from 'node:fs';
 import { FleetUsageError } from './errors';
 import { parseConfig, record } from './config';
 import type { FleetConfig } from './types';
 import { parseSecretsConfig, type SecretsConfig } from './secrets';
 import { parseLocation, type BucketLocation } from './state/bucket';
+import {
+  parseIdes,
+  parseT3,
+  type IdeId,
+  type T3Definition,
+} from './ide/definition';
+import { parseNotify, type NotifyConfig } from './notify';
 
 export interface MachineDefinition {
   id: string;
   ssh: string;
   os: 'linux' | 'macOS' | 'windows';
+  /** Tool profiles this worker adds to the fleet-wide ones. */
+  toolProfiles?: ToolProfile[];
   network?: {
     tailscaleName: string;
     user: string;
@@ -40,7 +54,7 @@ export interface FleetDefinition {
   workers: MachineDefinition[];
   services: {
     /** Public HTTPS origin of the CLI proxy every node uses. */
-    proxy: { endpoint: string };
+    proxy: { endpoint: string; accounts?: AccountPolicy };
     monitoring: {
       hubPort: 8443;
       /** Public dashboard origin. */
@@ -49,11 +63,62 @@ export interface FleetDefinition {
       managerAccount: string;
       /** Read-only Beszel account visitors are signed in as. */
       viewerAccount: string;
+      /** Scheduled off-machine backups fleet-sync sets on the hub. */
+      backups?: HubBackups;
     };
   };
   worker?: WorkerPolicy;
+  /** The IDEs every machine runs; `["t3"]` when not named (idesOf). */
+  ides?: IdeId[];
+  /** What every machine's T3 holds beyond its provider instances. */
+  t3?: T3Definition;
+  /** The Git identity and GitHub credential helper every node, manager
+   * included, commits and pushes with.
+   */
+  git?: GitIdentity;
+  /** Where fleet-watch sends notifications; absent, nobody is notified. */
+  notify?: NotifyConfig;
   ubuntu: { release: '26.04'; architectures: ['amd64', 'arm64'] };
 }
+/** How provider accounts are watched and chosen (src/accounts/). */
+export interface AccountPolicy {
+  /** Fewer usable accounts of a provider than this is a warning; none fails. */
+  minimumUsable?: number;
+  /** Per provider (`claude`, `codex`, `antigravity`): which account an
+   * unpinned request goes to first. Accounts cooling down are skipped.
+   */
+  selection?: Record<string, AccountSelection>;
+}
+export interface AccountSelection {
+  strategy: 'fill-first' | 'round-robin' | 'least-recently-limited';
+  /** Account labels (sign-in emails) first, in this order; the others
+   * follow in the order of their routing prefix.
+   */
+  order?: string[];
+}
+export interface HubBackups {
+  /** Cron expression of the hub's backup schedule. */
+  schedule: string;
+  /** How many scheduled backups the hub keeps. */
+  keep: number;
+  /** S3-compatible storage off the manager; its keys are secrets. */
+  storage: {
+    bucket: string;
+    endpoint: string;
+    region: string;
+    forcePathStyle?: boolean;
+  };
+}
+export interface GitIdentity {
+  /** `user.name` of every commit. */
+  name: string;
+  /** `user.email` of every commit. */
+  email: string;
+  /** A repository each node must reach through git with its own
+   * credentials, proving it can push (`worker:git-push-auth`).
+   */
+  verifyRepository?: string;
+}
 const ID = /^[a-z][a-z0-9-]{0,62}$/;
 const HOST = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$/;
 function fail(): never {
@@ -90,7 +155,10 @@ function validPort(value: unknown): boolean {
   );
 }
 function worker(value: unknown): MachineDefinition {
-  if (!record(value) || !keys(value, ['id', 'ssh', 'os', 'network']))
+  if (
+    !record(value) ||
+    !keys(value, ['id', 'ssh', 'os', 'network', 'toolProfiles'])
+  )
     return fail();
   if (typeof value['id'] !== 'string' || !ID.test(value['id'])) return fail();
   if (typeof value['ssh'] !== 'string' || !HOST.test(value['ssh']))
@@ -102,6 +170,9 @@ function worker(value: unknown): MachineDefinition {
     ssh: value['ssh'],
     os: 'linux',
     ...(value['network'] ? { network: network(value['network']) } : {}),
+    ...(value['toolProfiles'] === undefined
+      ? {}
+      : { toolProfiles: toolProfiles(value['toolProfiles']) ?? fail() }),
   };
 }
 function manager(value: unknown): ManagerDefinition {
@@ -148,6 +219,10 @@ export function parseDefinition(value: unknown): FleetDefinition {
       'services',
       'ubuntu',
       'worker',
+      'ides',
+      't3',
+      'git',
+      'notify',
     ])
   )
     return fail();
@@ -166,9 +241,25 @@ export function parseDefinition(value: unknown): FleetDefinition {
     workers,
     services: services(value['services']),
     ubuntu: baseline(value['ubuntu']),
+    ...optionalSections(value),
+  };
+}
+/** The sections a fleet may leave out, kept only when named: an earlier
+ * revision must still read the copy.
+ */
+function optionalSections(
+  value: Record<string, unknown>
+): Pick<FleetDefinition, 'worker' | 'ides' | 't3' | 'git' | 'notify'> {
+  return {
     ...(value['worker'] === undefined
       ? {}
       : { worker: parseWorkerPolicy(value['worker']) }),
+    ...(value['ides'] === undefined ? {} : { ides: parseIdes(value['ides']) }),
+    ...(value['t3'] === undefined ? {} : { t3: parseT3(value['t3']) }),
+    ...(value['git'] === undefined ? {} : { git: gitIdentity(value['git']) }),
+    ...(value['notify'] === undefined
+      ? {}
+      : { notify: parseNotify(value['notify']) ?? fail() }),
   };
 }
 /** Singleton services name no host: they always run on the manager. */
@@ -177,31 +268,159 @@ function services(value: unknown): FleetDefinition['services'] {
   const { proxy, monitoring } = value;
   if (
     !record(proxy) ||
-    !keys(proxy, ['endpoint']) ||
+    !keys(proxy, ['endpoint', 'accounts']) ||
     !origin(proxy['endpoint'])
   )
     return fail();
   return {
-    proxy: { endpoint: proxy['endpoint'] },
+    proxy: {
+      endpoint: proxy['endpoint'],
+      ...(proxy['accounts'] === undefined
+        ? {}
+        : { accounts: accountPolicy(proxy['accounts']) }),
+    },
     monitoring: monitoringService(monitoring),
   };
 }
+const STRATEGIES = ['fill-first', 'round-robin', 'least-recently-limited'];
+function accountPolicy(value: unknown): AccountPolicy {
+  if (!record(value) || !keys(value, ['minimumUsable', 'selection']))
+    return fail();
+  const { minimumUsable, selection } = value;
+  if (
+    minimumUsable !== undefined &&
+    !(Number.isInteger(minimumUsable) && Number(minimumUsable) >= 1)
+  )
+    return fail();
+  return {
+    ...(minimumUsable === undefined
+      ? {}
+      : { minimumUsable: minimumUsable as number }),
+    ...(selection === undefined
+      ? {}
+      : { selection: selectionPolicies(selection) }),
+  };
+}
+function selectionPolicies(value: unknown): Record<string, AccountSelection> {
+  if (!record(value)) return fail();
+  const providers: Record<string, AccountSelection> = {};
+  for (const [provider, entry] of Object.entries(value)) {
+    if (!/^[a-z][a-z0-9-]{0,31}$/.test(provider)) return fail();
+    providers[provider] = accountSelection(entry);
+  }
+  return providers;
+}
+function accountSelection(entry: unknown): AccountSelection {
+  if (
+    !record(entry) ||
+    !keys(entry, ['strategy', 'order']) ||
+    !STRATEGIES.includes(String(entry['strategy']))
+  )
+    return fail();
+  const { order } = entry;
+  if (
+    order !== undefined &&
+    !(Array.isArray(order) && order.every((label) => account(label)))
+  )
+    return fail();
+  return {
+    strategy: entry['strategy'] as AccountSelection['strategy'],
+    ...(order ? { order: order as string[] } : {}),
+  };
+}
 function monitoringService(
   value: unknown
 ): FleetDefinition['services']['monitoring'] {
-  const fields = ['hubPort', 'publicUrl', 'managerAccount', 'viewerAccount'];
+  const fields = [
+    'hubPort',
+    'publicUrl',
+    'managerAccount',
+    'viewerAccount',
+    'backups',
+  ];
   if (!record(value) || !keys(value, fields) || value['hubPort'] !== 8443)
     return fail();
   const { publicUrl, managerAccount, viewerAccount } = value;
   if (!origin(publicUrl) || !account(managerAccount) || !account(viewerAccount))
     return fail();
-  return { hubPort: 8443, publicUrl, managerAccount, viewerAccount };
+  return {
+    hubPort: 8443,
+    publicUrl,
+    managerAccount,
+    viewerAccount,
+    ...(value['backups'] === undefined
+      ? {}
+      : { backups: hubBackups(value['backups']) }),
+  };
+}
+const text = (value: unknown, pattern: RegExp): value is string =>
+  typeof value === 'string' && pattern.test(value);
+function hubBackups(value: unknown): HubBackups {
+  if (!record(value) || !keys(value, ['schedule', 'keep', 'storage']))
+    return fail();
+  const { schedule, keep } = value;
+  if (
+    !text(schedule, /^[0-9*/,-]+( [0-9*/,-]+){4}$/) ||
+    !Number.isInteger(keep) ||
+    Number(keep) < 1 ||
+    Number(keep) > 365
+  )
+    return fail();
+  return {
+    schedule,
+    keep: Number(keep),
+    storage: backupStorage(value['storage']),
+  };
+}
+function backupStorage(value: unknown): HubBackups['storage'] {
+  if (
+    !record(value) ||
+    !keys(value, ['bucket', 'endpoint', 'region', 'forcePathStyle'])
+  )
+    return fail();
+  const { bucket, endpoint, region, forcePathStyle } = value;
+  if (
+    !text(bucket, /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/) ||
+    !text(endpoint, /^https:\/\/[A-Za-z0-9.-]{1,253}(:[0-9]{1,5})?$/) ||
+    !text(region, /^[a-z0-9-]{1,32}$/) ||
+    (forcePathStyle !== undefined && typeof forcePathStyle !== 'boolean')
+  )
+    return fail();
+  return {
+    bucket,
+    endpoint,
+    region,
+    ...(forcePathStyle === undefined ? {} : { forcePathStyle }),
+  };
 }
 const origin = (value: unknown): value is string =>
   typeof value === 'string' && /^https:\/\/[A-Za-z0-9.-]{1,253}$/.test(value);
 const account = (value: unknown): value is string =>
   typeof value === 'string' &&
   /^[^\s@]{1,64}@[A-Za-z0-9.-]{1,253}$/.test(value);
+function gitIdentity(value: unknown): GitIdentity {
+  if (!record(value) || !keys(value, ['name', 'email', 'verifyRepository']))
+    return fail();
+  const { name, email, verifyRepository } = value;
+  // Written into a quoted Git configuration value: no quote, backslash or
+  // line break.
+  if (typeof name !== 'string' || !/^[^"\\\n\r]{1,100}$/.test(name))
+    return fail();
+  if (!account(email)) return fail();
+  if (
+    verifyRepository !== undefined &&
+    (typeof verifyRepository !== 'string' ||
+      !/^https:\/\/github\.com\/[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/.test(
+        verifyRepository
+      ))
+  )
+    return fail();
+  return {
+    name,
+    email,
+    ...(verifyRepository === undefined ? {} : { verifyRepository }),
+  };
+}
 function secrets(value: unknown): SecretsConfig {
   return parseSecretsConfig(value) ?? fail();
 }

package/src/git-identity.ts

@@ -0,0 +1,160 @@
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import { runAnsible } from './platform/ansible';
+import type { GitIdentity } from './definition';
+import type { Check, Runner } from './types';
+
+/** `ansible/github/git.yml` writes the declared identity and the GitHub
+ * credential helper in the node's own Git configuration; the two checks
+ * below measure them afterwards. The same on the manager and every worker.
+ */
+export async function convergeGitIdentity(
+  root: string,
+  git: GitIdentity | undefined,
+  run: Runner,
+  home: string = homedir()
+): Promise<Check[]> {
+  if (!git) return [];
+  const result = await runAnsible(
+    run,
+    resolve(root, 'ansible/github/git.yml'),
+    {
+      cwd: resolve(root, 'ansible/github'),
+      timeoutMs: 120_000,
+      input: JSON.stringify({
+        git_name: git.name,
+        git_email: git.email,
+        git_home: home,
+      }),
+    }
+  );
+  return [
+    {
+      id: 'git:configuration',
+      status: result.ok ? 'pass' : 'fail',
+      summary: result.ok
+        ? 'The declared Git identity and the GitHub credential helper are written in ~/.config/git/config.'
+        : 'GIT_CONFIGURATION_INCOMPLETE',
+      ...(result.ok
+        ? {}
+        : {
+            remedy: {
+              action:
+                'Inspect ~/.config/git/config and ~/.gitconfig on this node: a ~/.gitconfig that sets user.name, user.email or a GitHub credential helper wins over the fleet. Remove those lines and rerun fleet-sync.',
+            },
+          }),
+    },
+  ];
+}
+
+/** Git with no prompt: a missing credential fails instead of waiting. */
+const QUIET = {
+  GIT_TERMINAL_PROMPT: '0',
+  GCM_INTERACTIVE: 'never',
+  GIT_ASKPASS: '',
+  SSH_ASKPASS: '',
+};
+
+async function effective(run: Runner, key: string): Promise<string> {
+  const result = await run(['git', 'config', '--global', '--get-all', key], {
+    timeoutMs: 10_000,
+  });
+  return result.ok ? (result.stdout.trim().split('\n').pop() ?? '') : '';
+}
+
+/** Commits carry the declared author: the effective `user.name` and
+ * `user.email` outside any repository are the fleet's.
+ */
+export async function gitIdentityCheck(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check> {
+  const id = 'worker:git-identity';
+  if (!git)
+    return {
+      id,
+      status: 'warn',
+      summary: 'GIT_IDENTITY_UNDECLARED',
+      remedy: {
+        action:
+          'Declare git.name and git.email in fleet.json and rerun fleet-sync; until then commits on this node carry whatever author it happens to have.',
+      },
+    };
+  const name = await effective(run, 'user.name');
+  const email = await effective(run, 'user.email');
+  if (name === git.name && email === git.email)
+    return {
+      id,
+      status: 'pass',
+      summary: `Commits on this node are authored as ${git.name} <${git.email}>.`,
+    };
+  return {
+    id,
+    status: 'fail',
+    summary: !name || !email ? 'GIT_IDENTITY_MISSING' : 'GIT_IDENTITY_MISMATCH',
+    remedy: {
+      action:
+        'Rerun fleet-sync: it writes the identity in ~/.config/git/config. A ~/.gitconfig that sets another user.name or user.email wins over it; remove those lines.',
+    },
+  };
+}
+
+/** A push is authorized through git itself, never through gh: Git's
+ * credential helper must answer for github.com, and the declared repository
+ * must answer an authenticated listing. The credential is read and dropped;
+ * it is never kept, printed or reported.
+ */
+export async function gitPushAuthCheck(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check> {
+  const id = 'worker:git-push-auth';
+  const fill = await run(['git', 'credential', 'fill'], {
+    input: 'protocol=https\nhost=github.com\n\n',
+    env: QUIET,
+    timeoutMs: 20_000,
+  });
+  if (!fill.ok || !/^password=./m.test(fill.stdout))
+    return {
+      id,
+      status: 'fail',
+      summary: 'GIT_CREDENTIAL_HELPER_MISSING',
+      remedy: {
+        action:
+          'Git has no credential for github.com, so an agent cannot push over HTTPS. Rerun fleet-sync, which declares gh as the credential helper; if gh is signed out, the worker signs in again from its own Vault session (auth:github).',
+      },
+    };
+  if (git?.verifyRepository) {
+    const listed = await run(
+      ['git', 'ls-remote', '--heads', git.verifyRepository],
+      { env: QUIET, timeoutMs: 30_000 }
+    );
+    if (!listed.ok)
+      return {
+        id,
+        status: 'fail',
+        summary:
+          listed.reason === 'timeout'
+            ? 'GIT_REMOTE_UNREACHABLE'
+            : 'GIT_REMOTE_REFUSED',
+        remedy: {
+          action:
+            "The declared repository (git.verifyRepository) did not answer git with this node's credentials. Check that the fleet's GitHub account can push to it and that github.com is reachable.",
+        },
+      };
+  }
+  return {
+    id,
+    status: 'pass',
+    summary: git?.verifyRepository
+      ? "Git's credential helper answers for github.com and the declared repository answers through git."
+      : "Git's credential helper answers for github.com; declare git.verifyRepository to prove a repository too.",
+  };
+}
+
+export async function gitChecks(
+  git: GitIdentity | undefined,
+  run: Runner
+): Promise<Check[]> {
+  return [await gitIdentityCheck(git, run), await gitPushAuthCheck(git, run)];
+}

package/src/ide/definition.ts

@@ -0,0 +1,86 @@
+import { FleetUsageError } from '../errors';
+import { record } from '../config';
+
+/** The IDEs dev-fleet supports. The support matrix in the README lists the
+ * same; enabling one on the fleet is `ides` in fleet.json, not new code.
+ */
+export const SUPPORTED_IDES = ['t3'] as const;
+export type IdeId = (typeof SUPPORTED_IDES)[number];
+
+/** What `t3` in fleet.json declares for every machine's T3. */
+export interface T3Definition {
+  /** Top-level T3 settings every machine holds the same: tool servers,
+   * projects, selected preferences. A key not named here stays local.
+   */
+  profile?: Record<string, unknown>;
+  /** Models T3 must offer for every account of a tool, by tool. */
+  requiredModels?: Record<string, string[]>;
+}
+
+const KEY = /^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/;
+/** Settings another part of dev-fleet owns. */
+const RESERVED = ['providerInstances'];
+
+function invalid(detail: string): never {
+  throw new FleetUsageError(`Invalid fleet.json: ${detail}`);
+}
+
+/** The IDEs a fleet runs: those it names, or T3 alone. */
+export const idesOf = (definition: { ides?: readonly IdeId[] }): IdeId[] => [
+  ...(definition.ides ?? ['t3']),
+];
+
+export function parseIdes(value: unknown): IdeId[] {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    new Set(value).size !== value.length ||
+    value.some((id) => !SUPPORTED_IDES.includes(id as IdeId))
+  )
+    invalid(`ides lists each of ${SUPPORTED_IDES.join(', ')} at most once.`);
+  return value as IdeId[];
+}
+
+export function parseT3(value: unknown): T3Definition {
+  if (
+    !record(value) ||
+    Object.keys(value).some(
+      (key) => !['profile', 'requiredModels'].includes(key)
+    )
+  )
+    invalid('t3 holds only profile and requiredModels.');
+  const result: T3Definition = {};
+  if (value['profile'] !== undefined) {
+    const profile = value['profile'];
+    if (
+      !record(profile) ||
+      Object.keys(profile).some(
+        (key) => !KEY.test(key) || RESERVED.includes(key)
+      ) ||
+      JSON.stringify(profile).length > 65_536
+    )
+      invalid(
+        't3.profile names top-level T3 settings (not providerInstances) and stays under 64 KiB.'
+      );
+    result.profile = profile;
+  }
+  if (value['requiredModels'] !== undefined) {
+    const required = value['requiredModels'];
+    if (
+      !record(required) ||
+      Object.entries(required).some(
+        ([tool, models]) =>
+          !KEY.test(tool) ||
+          !Array.isArray(models) ||
+          models.some(
+            (model) =>
+              typeof model !== 'string' ||
+              !/^[A-Za-z0-9._:-]{1,128}$/.test(model)
+          )
+      )
+    )
+      invalid('t3.requiredModels lists model names by tool.');
+    result.requiredModels = required as Record<string, string[]>;
+  }
+  return result;
+}

package/src/ide/ide.ts

@@ -0,0 +1,43 @@
+import type { Check, FleetNode, Runner } from '../types';
+
+/** What every supported IDE is asked for. An implementation owns its own
+ * install, service and policy (an Ansible role per IDE, with the common
+ * variable contract: version, checksum, policy, instances, profile) and is
+ * reached only through these calls, so the orchestration never names one.
+ */
+export interface Ide {
+  /** The name the fleet configuration enables it by, under `ides`. */
+  readonly id: string;
+  /** The IDE's settings on this machine: provider instances from the account
+   * roster, the declared profile and the models it offers. Run on the manager
+   * and on every worker by the machine itself.
+   */
+  settings(context: MachineContext): Promise<Check[]>;
+  /** The manager's own IDE: its settings, its pinned version and its policy. */
+  manager(context: MachineContext): Promise<Check[]>;
+  /** What the manager measures of a worker's IDE: remote access, pairing and
+   * the version the worker runs.
+   */
+  worker(node: FleetNode, context: WorkerContext): Promise<Check[]>;
+}
+
+export interface MachineContext {
+  /** The node ID this machine is in the fleet. */
+  id: string;
+  /** The checkout this run applies. */
+  workspace: string;
+  apply: boolean;
+  run: Runner;
+}
+
+export interface WorkerContext {
+  run: Runner;
+  /** The manager's checkout of the revision this sync applies. */
+  workspace: string;
+  /** The checks this sync already reported for the worker. */
+  earlier: Check[];
+  /** Records what identifies the worker's IDE in the fleet state, the first
+   * time it is seen through a verified link.
+   */
+  remember?: (node: string, identity: string) => Promise<void>;
+}

package/src/ide/index.ts

@@ -0,0 +1,5 @@
+import type { Ide } from './ide';
+import { t3Ide } from './t3';
+
+/** Every IDE dev-fleet implements, by the ID fleet.json enables it with. */
+export const IDES: Readonly<Record<string, Ide>> = { t3: t3Ide() };

package/src/ide/orchestrate.ts

@@ -0,0 +1,50 @@
+import type { Check, FleetNode } from '../types';
+import type { Ide, MachineContext, WorkerContext } from './ide';
+
+/** The implementations the fleet enables, in the order it names them. An ID
+ * nobody implements is refused before anything runs.
+ */
+export function enabled(
+  ids: readonly string[],
+  registry: Readonly<Record<string, Ide>>
+): Ide[] {
+  return ids.map((id) => {
+    const ide = registry[id];
+    if (!ide) throw new Error(`Unsupported IDE: ${id}`);
+    return ide;
+  });
+}
+
+export async function settingsChecks(
+  ides: Ide[],
+  context: MachineContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides) checks.push(...(await ide.settings(context)));
+  return checks;
+}
+
+export async function managerChecks(
+  ides: Ide[],
+  context: MachineContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides) checks.push(...(await ide.manager(context)));
+  return checks;
+}
+
+export async function workerChecks(
+  ides: Ide[],
+  node: FleetNode,
+  context: WorkerContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides)
+    checks.push(
+      ...(await ide.worker(node, {
+        ...context,
+        earlier: [...context.earlier, ...checks],
+      }))
+    );
+  return checks;
+}