@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/cli/fleet-accounts.ts

@@ -0,0 +1,253 @@
+import { rm } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import { runCommand } from '../runner';
+import { blocking, type Check, type Runner } from '../types';
+import {
+  nativeLoginCheck,
+  nativeLogins,
+  providerAccountChecks,
+  rosterCurrentCheck,
+} from '../accounts/checks';
+import { exposedVersion, t3Settings } from '../accounts/context';
+import { parseRoster, type ProxyRoster } from '../accounts/roster';
+import { exposedInstances } from '../accounts/sessions';
+import { forwardSourceOperation, ownerCheckout } from './source-operations';
+import type { CommandOutcome } from './contracts';
+import type { Settings } from './settings';
+import { fleetSync } from './fleet-sync';
+
+export interface FleetAccountsOptions {
+  diff?: boolean;
+  add?: boolean;
+  relogin?: boolean;
+  remove?: boolean;
+  removeNativeLogins?: boolean;
+  yes?: boolean;
+}
+
+export interface AccountsDependencies {
+  run?: Runner;
+  /** Runs a proxy command attached to the terminal (a sign-in). */
+  operate?: (args: string[]) => Promise<number>;
+  /** `fleet-sync --only accounts`. */
+  propagate?: () => Promise<CommandOutcome>;
+  home?: string;
+  settingsFile?: string;
+}
+
+const PROVIDER = /^[a-z][a-z0-9-]{0,31}$/;
+
+/** `devfleet fleet-accounts`: the provider account roster the proxy holds,
+ * each account's state, and the account checks. `--add`, `--relogin` and
+ * `--remove` change one account on the proxy and propagate the result to
+ * every node at once.
+ */
+export async function fleetAccounts(
+  options: FleetAccountsOptions,
+  args: string[],
+  settings: Settings,
+  dependencies: AccountsDependencies = {}
+): Promise<CommandOutcome> {
+  const home = dependencies.home ?? homedir();
+  if (options.removeNativeLogins)
+    return removeNativeLogins(Boolean(options.yes), home);
+  const change = lifecycle(options, args);
+  const changed = change
+    ? await applyChange(change, settings, dependencies)
+    : undefined;
+  if (changed && !changed.data) return failedChange();
+  return report(options, settings, dependencies, { change, changed });
+}
+
+async function report(
+  options: FleetAccountsOptions,
+  settings: Settings,
+  dependencies: AccountsDependencies,
+  {
+    change,
+    changed,
+  }: { change: string[] | null; changed: CommandOutcome | undefined }
+): Promise<CommandOutcome> {
+  const home = dependencies.home ?? homedir();
+  const roster = await readRoster(
+    dependencies.run ?? runCommand,
+    settings.workspace
+  );
+  const checks: Check[] = providerAccountChecks(
+    roster,
+    fleet.services.proxy.accounts?.minimumUsable
+  );
+  if (options.diff)
+    checks.push(...(await thisMachine(roster, home, dependencies)));
+  const ok = !checks.some(blocking) && !changed?.exitCode;
+  return {
+    exitCode: ok ? 0 : 1,
+    data: {
+      version: 1,
+      command: 'fleet-accounts',
+      ok,
+      ...listing(roster),
+      checks,
+      ...(change && {
+        changed: change.slice(0, 2),
+        propagation: changed?.data,
+      }),
+    },
+  };
+}
+
+/** The roster as listed: no routing prefix, no file. */
+function listing(roster: ProxyRoster) {
+  return {
+    rosterVersion: roster.rosterVersion,
+    managementApi: roster.managementApi,
+    accounts: roster.accounts.map(
+      ({ provider, index, label, state, until }) => ({
+        provider,
+        index,
+        label,
+        state,
+        until,
+      })
+    ),
+  };
+}
+
+/** Runs the change on the proxy, then propagates it to every node. A change
+ * the proxy did not complete answers with no data: nothing was propagated.
+ */
+async function applyChange(
+  change: string[],
+  settings: Settings,
+  dependencies: AccountsDependencies
+): Promise<CommandOutcome> {
+  const operate =
+    dependencies.operate ??
+    ((forwarded: string[]) =>
+      forwardSourceOperation(['cliproxy', ...forwarded], settings));
+  if ((await operate(change)) !== 0) return { exitCode: 1, data: null };
+  return (
+    dependencies.propagate ??
+    (() => fleetSync({ only: 'accounts', json: true }, settings))
+  )();
+}
+
+function failedChange(): CommandOutcome {
+  return {
+    exitCode: 1,
+    data: {
+      version: 1,
+      command: 'fleet-accounts',
+      ok: false,
+      error: 'ACCOUNT_CHANGE_FAILED',
+      summary:
+        'The proxy did not complete the change; an interrupted sign-in leaves no credential behind. Nothing was propagated.',
+    },
+  };
+}
+
+/** `--diff`: what this machine offers against the roster. */
+async function thisMachine(
+  roster: ProxyRoster,
+  home: string,
+  dependencies: AccountsDependencies
+): Promise<Check[]> {
+  const exposed = await exposedInstances(
+    dependencies.settingsFile ?? t3Settings()
+  );
+  return [
+    rosterCurrentCheck(roster.rosterVersion, exposedVersion(exposed)),
+    nativeLoginCheck(nativeLogins(home)),
+  ];
+}
+
+/** The proxy command a lifecycle flag stands for, or null for a listing. */
+function lifecycle(
+  options: FleetAccountsOptions,
+  [provider, index]: string[]
+): string[] | null {
+  const chosen = (['add', 'relogin', 'remove'] as const).filter(
+    (flag) => options[flag]
+  );
+  if (chosen.length === 0) return null;
+  if (chosen.length > 1)
+    throw new FleetUsageError('Choose one of --add, --relogin and --remove.');
+  const flag = chosen[0]!;
+  if (!provider || !PROVIDER.test(provider))
+    throw new FleetUsageError(
+      `--${flag} needs a provider, such as claude or codex.`
+    );
+  if (flag === 'add') return ['add-account', provider];
+  if (!index || !/^[1-9][0-9]{0,3}$/.test(index))
+    throw new FleetUsageError(
+      `--${flag} needs the account's index, as devfleet fleet-accounts lists it.`
+    );
+  return [flag === 'relogin' ? 'relogin' : 'remove-account', provider, index];
+}
+
+/** The roster as the proxy on this manager reports it. */
+async function readRoster(
+  run: Runner,
+  workspace: string
+): Promise<ProxyRoster> {
+  const checkout = ownerCheckout(workspace);
+  if (!checkout)
+    throw new FleetUsageError(
+      'The account roster is kept by the proxy on the manager. Run devfleet fleet-accounts there, after a first fleet-sync.'
+    );
+  const result = await run(
+    [
+      'python3',
+      '-B',
+      join(checkout, 'cliproxy/scripts/cli.py'),
+      'accounts',
+      '--json',
+    ],
+    { cwd: checkout, timeoutMs: 60_000 }
+  );
+  try {
+    if (!result.ok) throw new Error();
+    return parseRoster(result.stdout);
+  } catch {
+    throw new FleetUsageError(
+      'The proxy did not report its accounts. Check it with devfleet cliproxy status.'
+    );
+  }
+}
+
+/** Opt-in: deletes the provider logins agent CLIs keep outside the proxy. */
+async function removeNativeLogins(
+  yes: boolean,
+  home: string
+): Promise<CommandOutcome> {
+  const found = nativeLogins(home);
+  if (!yes)
+    return {
+      exitCode: found.length ? 1 : 0,
+      data: {
+        version: 1,
+        command: 'fleet-accounts',
+        ok: found.length === 0,
+        applied: false,
+        checks: [nativeLoginCheck(found)],
+        next: found.length
+          ? 'Add --yes to remove these logins from this machine.'
+          : undefined,
+      },
+    };
+  for (const login of found) await rm(login.path, { force: true });
+  return {
+    exitCode: 0,
+    data: {
+      version: 1,
+      command: 'fleet-accounts',
+      ok: true,
+      applied: true,
+      removed: [...new Set(found.map((login) => login.cli))].sort(),
+      checks: [nativeLoginCheck(nativeLogins(home))],
+    },
+  };
+}

package/src/cli/fleet-report.ts

@@ -0,0 +1,126 @@
+import { FleetUsageError } from '../errors';
+import { readHistory, type SyncRun } from '../state/history';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetReportOptions {
+  since?: string;
+  json?: boolean;
+}
+
+const UNITS: Record<string, number> = {
+  m: 60_000,
+  h: 3_600_000,
+  d: 86_400_000,
+};
+
+/** `30d`, `12h` or `90m`: how far back the report looks. */
+export function parseSince(value = '30d'): number {
+  const match = /^([1-9]\d{0,4})([mhd])$/.exec(value);
+  if (!match)
+    throw new FleetUsageError(
+      '--since takes a duration such as 30d, 12h or 90m.'
+    );
+  return Number(match[1]) * UNITS[match[2]!]!;
+}
+
+export interface Reliability {
+  runs: number;
+  succeeded: number;
+  successRate: number | null;
+  meanDurationMs: number | null;
+  failingChecks: { id: string; runs: number }[];
+  notReady: { node: string; ms: number }[];
+}
+
+/** How long each machine was not ready: from the end of a run that left it
+ * not ready to the end of the next run that found it ready (or until now).
+ */
+function notReadyTime(runs: SyncRun[], now: Date): Map<string, number> {
+  const since = new Map<string, number>();
+  const total = new Map<string, number>();
+  const add = (node: string, until: number) => {
+    total.set(node, (total.get(node) ?? 0) + until - since.get(node)!);
+    since.delete(node);
+  };
+  for (const run of runs) {
+    const at = Date.parse(run.finishedAt);
+    for (const node of [...since.keys()])
+      if (!run.notReady.includes(node)) add(node, at);
+    for (const node of run.notReady) if (!since.has(node)) since.set(node, at);
+  }
+  for (const node of [...since.keys()]) add(node, now.getTime());
+  return total;
+}
+
+export function reliability(runs: SyncRun[], now: Date): Reliability {
+  const succeeded = runs.filter((run) => run.ok).length;
+  const counts = new Map<string, number>();
+  for (const run of runs)
+    for (const id of run.failing) counts.set(id, (counts.get(id) ?? 0) + 1);
+  return {
+    runs: runs.length,
+    succeeded,
+    successRate: runs.length ? succeeded / runs.length : null,
+    meanDurationMs: runs.length
+      ? Math.round(
+          runs.reduce((sum, run) => sum + run.durationMs, 0) / runs.length
+        )
+      : null,
+    failingChecks: [...counts]
+      .map(([id, count]) => ({ id, runs: count }))
+      .sort((a, b) => b.runs - a.runs || a.id.localeCompare(b.id))
+      .slice(0, 10),
+    notReady: [...notReadyTime(runs, now)]
+      .map(([node, ms]) => ({ node, ms }))
+      .sort((a, b) => b.ms - a.ms || a.node.localeCompare(b.node)),
+  };
+}
+
+/** Read-only: the reliability of fleet-sync over a period, from the run
+ * history kept in the fleet state.
+ */
+export function fleetReport(
+  options: FleetReportOptions,
+  env = process.env,
+  now = new Date()
+): CommandOutcome {
+  const since = options.since ?? '30d';
+  const from = now.getTime() - parseSince(since);
+  const runs = readHistory(env).runs.filter(
+    (run) => Date.parse(run.finishedAt) >= from
+  );
+  const summary = reliability(runs, now);
+  const data = {
+    version: 1,
+    command: 'fleet-report',
+    ok: true,
+    since,
+    ...summary,
+    entries: runs,
+  };
+  return { exitCode: 0, data, text: render(since, summary) };
+}
+
+function minutes(ms: number): string {
+  return `${Math.round(ms / 60_000)} min`;
+}
+
+function render(since: string, summary: Reliability): string {
+  if (!summary.runs) return `No fleet-sync run in the last ${since}.`;
+  const lines = [
+    `fleet-sync over the last ${since}: ${summary.runs} runs, ${summary.succeeded} succeeded (${Math.round((summary.successRate ?? 0) * 100)}%), ${Math.round((summary.meanDurationMs ?? 0) / 1000)} s on average.`,
+  ];
+  if (summary.failingChecks.length)
+    lines.push(
+      'Most frequent failing checks:',
+      ...summary.failingChecks.map(
+        (check) => `  ${check.id}  ${check.runs} runs`
+      )
+    );
+  if (summary.notReady.length)
+    lines.push(
+      'Time not ready:',
+      ...summary.notReady.map((node) => `  ${node.node}  ${minutes(node.ms)}`)
+    );
+  return lines.join('\n');
+}

package/src/cli/fleet-sync-accounts.ts

@@ -0,0 +1,58 @@
+import { enrolledRoster } from '../definition';
+import { selectNodes } from '../config';
+import { fleet as definition } from '../fleet-config';
+import { bundleProbe } from '../transport';
+import { machineProviders } from '../t3/providers';
+import { accountContext, nodeAccountChecks } from '../accounts/context';
+import { propagateAccounts } from '../accounts/propagate';
+import { loadRoster } from './roster';
+import { ownerCheckout } from './source-operations';
+import type { Check, FleetNode, Runner } from '../types';
+import type { FleetSyncOptions } from './fleet-sync';
+import type { Settings } from './settings';
+import type { CommandOutcome } from './contracts';
+
+/** `fleet-sync --only accounts` on the manager. */
+export async function syncAccounts(
+  options: FleetSyncOptions,
+  settings: Settings,
+  { revision, run }: { revision: string; run: Runner }
+): Promise<CommandOutcome> {
+  let nodes: FleetNode[] = [];
+  try {
+    nodes = selectNodes(
+      enrolledRoster(definition, await loadRoster(settings.config)),
+      options.node ?? []
+    );
+  } catch {
+    // No enrolled worker yet: the manager alone.
+  }
+  return propagateAccounts({
+    nodes,
+    revision,
+    source: await bundleProbe(),
+    manager: () => managerAccounts(settings.workspace),
+    run,
+  });
+}
+
+/** The manager's own account instances, from the checkout it runs. */
+export async function managerAccounts(workspace: string): Promise<Check[]> {
+  const checkout = ownerCheckout(workspace);
+  if (!checkout)
+    return [
+      {
+        id: 't3:providers',
+        status: 'fail',
+        summary: 'No revision is applied on the manager yet.',
+        remedy: {
+          action: 'Run a full fleet-sync first.',
+          command: 'devfleet fleet-sync',
+        },
+      },
+    ];
+  return [
+    await machineProviders(definition.manager.id, checkout, true),
+    ...(await nodeAccountChecks(await accountContext())),
+  ];
+}

package/src/cli/fleet-sync-manager.ts

@@ -6,14 +6,18 @@ import { convergeProxy } from '../proxy-converge';
 import { attempt } from '../node-converge-shared';
 import { quotaLimited, verifyFleetProviders } from '../agents/fleet-verify';
 import { verifyNativeAgents } from '../agents/native-verify';
+import { accountContext, nodeAccountChecks } from '../accounts/context';
 import { convergeManagerHub } from '../monitoring/manager-hub';
 import { enrollManagerMonitoring } from './fleet-sync-monitoring';
 import { installConfig } from '../fleet-config';
-import { machineProviders } from '../t3/providers';
-import { providerNames } from '../t3/names-check';
+import { IDES } from '../ide';
+import { enabled, managerChecks } from '../ide/orchestrate';
+import { idesOf } from '../ide/definition';
+import { convergeGitIdentity, gitChecks } from '../git-identity';
 import macos from '../../ansible/macos/policy.json';
 import { ansibleCommand } from '../platform/ansible';
 import { PACKAGE_ROOT } from '../package-info';
+import { managerHealth } from '../manager-health';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 
@@ -39,8 +43,16 @@ export async function convergeManager(
     if (input.apply) installConfig();
     checks.push(await managerPower(run, input.workspace));
     checks.push(await managerDocker(run));
-    checks.push(await machineProviders(id, input.workspace, input.apply));
-    checks.push(await providerNames(id, input.workspace, input.apply));
+    // Each enabled IDE: its settings, pinned version and policy.
+    checks.push(
+      ...(await managerChecks(enabled(idesOf(definition), IDES), {
+        id,
+        workspace: input.workspace,
+        apply: input.apply,
+        run,
+      }))
+    );
+    checks.push(...(await managerHealth(run)));
     if (input.apply) {
       // Same provider profiles and the same Vault gateway credential as every
       // worker: the instances above launch from these profiles, and the check
@@ -50,15 +62,30 @@ export async function convergeManager(
           managerAgentProfiles(id, input.workspace, run)
         ))
       );
+      // The same Git identity and GitHub credentials as every worker.
+      checks.push(
+        ...(await attempt('git:configuration', () =>
+          convergeGitIdentity(input.workspace, definition.git, run)
+        ))
+      );
+      if (definition.git)
+        checks.push(...(await gitChecks(definition.git, run)));
       const providers = await attempt('provider:verification', () =>
         verifyFleetProviders(id)
       );
       checks.push(...providers);
       // Same agents, versions and shared accounts as every worker.
+      const accounts = await accountContext();
+      checks.push(
+        ...(await attempt('accounts:verification', () =>
+          nodeAccountChecks(accounts)
+        ))
+      );
       checks.push(
         ...(await attempt('agent:verification', () =>
           verifyNativeAgents(id, input.workspace, run, {
             outOfQuota: quotaLimited(providers),
+            ...(accounts ? { accounts } : {}),
           })
         ))
       );

package/src/cli/fleet-sync-output.ts

@@ -19,9 +19,10 @@ const COLUMNS: { title: string; matches: (id: string) => boolean }[] = [
     matches: (id) =>
       [
         't3:service',
-        't3:policy',
+        't3:policy-staged',
+        't3:policy-loaded',
+        't3:running-version',
         't3:providers',
-        't3:antigravity-key',
         't3:private-route',
         'worker:t3-recovery',
         'worker:t3-health',