@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/node-converge-linux.ts

@@ -6,15 +6,24 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
 import { verifyResourceSafety } from './platform/resource-verify';
 import { convergeT3Route, T3_ROUTE } from './platform/t3-route';
 import { convergeProxy } from './proxy-converge';
-import { providerNames } from './t3/names-check';
-import { machineProviders } from './t3/providers';
+import { IDES } from './ide';
+import { enabled, settingsChecks } from './ide/orchestrate';
+import { idesOf } from './ide/definition';
+import { fleet } from './fleet-config';
 import { quotaLimited, verifyFleetProviders } from './agents/fleet-verify';
 import { verifyNativeAgents } from './agents/native-verify';
+import { accountContext, nodeAccountChecks } from './accounts/context';
 import { managedT3, managedT3Desktop } from '../bootstrap/native-t3';
-import { convergeT3Service, t3StartedAt } from './platform/t3-service';
+import {
+  convergeT3Service,
+  runningVersionCheck,
+  t3StartedAt,
+} from './platform/t3-service';
 import { userInfo } from 'node:os';
 import { resolve } from 'node:path';
 import { auditUbuntu } from './platform/ubuntu-audit';
+import { convergeGitIdentity } from './git-identity';
+import { workerSelection } from '../bootstrap/worker-tools';
 import {
   applyAgentProfiles,
   attempt,
@@ -52,7 +61,7 @@ export async function convergeLinux(
       await applyTailscale(root, run),
     ]))
   );
-  const { baseline, verified } = await baselineStep(root, run);
+  const { baseline, verified } = await baselineStep(root, run, node);
   checks.push(...baseline, ...verified);
   checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
   checks.push(
@@ -60,6 +69,11 @@ export async function convergeLinux(
       configureWorkstation(root, run)
     ))
   );
+  checks.push(
+    ...(await attempt('git:configuration', () =>
+      convergeGitIdentity(root, fleet.git, run)
+    ))
+  );
   checks.push(
     ...(await attempt('configuration:apply', () =>
       applyAgentProfiles(root, node, revision, run)
@@ -74,12 +88,19 @@ export async function convergeLinux(
     verifyFleetProviders(node)
   );
   checks.push(...providers);
+  const accounts = await accountContext();
+  checks.push(
+    ...(await attempt('accounts:verification', () =>
+      nodeAccountChecks(accounts)
+    ))
+  );
   checks.push(
     ...(await attempt('agent:verification', () =>
       verifyNativeAgents(node, root, run, {
         outOfQuota: quotaLimited(providers),
         // Where T3 starts them: the environment of its user service.
         t3Service: true,
+        ...(accounts ? { accounts } : {}),
       })
     ))
   );
@@ -96,7 +117,8 @@ export async function convergeLinux(
  */
 async function baselineStep(
   root: string,
-  run: Runner
+  run: Runner,
+  node: string
 ): Promise<{ baseline: Check[]; verified: Check[] }> {
   const measure = async () => [
     ...(await attempt('ubuntu:boot-policy', () =>
@@ -108,14 +130,14 @@ async function baselineStep(
   ];
   let cached = false;
   let baseline = await attempt('ubuntu:baseline', async () => {
-    const applied = await applyBaseline(root, run, false);
+    const applied = await applyBaseline(root, run, false, node);
     cached = applied.cached;
     return applied.checks;
   });
   let verified = await measure();
   if (cached && verified.some((check) => check.status === 'fail')) {
     baseline = await attempt('ubuntu:baseline', async () => {
-      return (await applyBaseline(root, run, true)).checks;
+      return (await applyBaseline(root, run, true, node)).checks;
     });
     verified = await measure();
   }
@@ -125,7 +147,8 @@ async function baselineStep(
 async function applyBaseline(
   root: string,
   run: Runner,
-  force: boolean
+  force: boolean,
+  node: string
 ): Promise<{ cached: boolean; checks: Check[] }> {
   const user = userInfo();
   const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -140,6 +163,8 @@ async function applyBaseline(
         development_user: user.username,
         development_uid: user.uid,
         development_home: user.homedir,
+        // The web profile brings headless Chromium (tasks/browser.yml).
+        worker_tool_profiles: workerSelection(fleet, node).profiles,
       }),
     },
     {
@@ -236,13 +261,20 @@ async function convergeT3(
         restarted: startedAt !== (await t3StartedAt(run)),
       })
     );
-  // Settings are watched by T3: providers arrive and stale ones leave at once.
-  checks.push(await machineProviders(node, root, true));
-  // Names live in the same settings, so they conform without a restart.
-  checks.push(await providerNames(node, root, true));
+  // Settings are watched by T3: providers, names and the profile apply
+  // without a restart.
+  checks.push(
+    ...(await settingsChecks(enabled(idesOf(fleet), IDES), {
+      id: node,
+      workspace: root,
+      apply: true,
+      run,
+    }))
+  );
   const serviceReady = checks.some(
     (check) => check.id === 't3:service' && check.status === 'pass'
   );
+  if (serviceReady) checks.push(await runningVersionCheck(run, userInfo().uid));
   if (serviceReady)
     checks.push(
       ...(await convergeT3Route(run, () => publishT3Route(root, run)))

package/src/notify.ts

@@ -0,0 +1,239 @@
+import { record } from './config';
+import type { SecretName, SecretsProvider } from './secrets';
+import type { Check, NodeReport } from './types';
+
+/** Where fleet-watch sends a notification. The address of each target is a
+ * secret, read by purpose: a webhook or ntfy URL often carries its own token.
+ */
+export const NOTIFY_TARGETS = {
+  webhook: 'notify-webhook-url',
+  ntfy: 'notify-ntfy-url',
+} as const satisfies Record<string, SecretName>;
+export type NotifyTarget = keyof typeof NOTIFY_TARGETS;
+
+export interface NotifyConfig {
+  targets: NotifyTarget[];
+  /** Notify again about a machine that stays unhealthy this often; 0 never. */
+  repeatHours: number;
+}
+
+const DEFAULT_REPEAT_HOURS = 24;
+
+/** The `notify` block of a fleet configuration, or null when it is invalid. */
+export function parseNotify(value: unknown): NotifyConfig | null {
+  if (!record(value)) return null;
+  if (
+    Object.keys(value).some((key) => !['targets', 'repeatHours'].includes(key))
+  )
+    return null;
+  const { targets, repeatHours = DEFAULT_REPEAT_HOURS } = value;
+  if (
+    !Array.isArray(targets) ||
+    targets.length === 0 ||
+    new Set(targets).size !== targets.length ||
+    !targets.every(
+      (target) =>
+        typeof target === 'string' && Object.hasOwn(NOTIFY_TARGETS, target)
+    ) ||
+    typeof repeatHours !== 'number' ||
+    !Number.isInteger(repeatHours) ||
+    repeatHours < 0 ||
+    repeatHours > 720
+  )
+    return null;
+  return { targets: targets as NotifyTarget[], repeatHours };
+}
+
+/** What the watcher remembers of each machine between two runs. */
+export interface WatchMemory {
+  version: 1;
+  /** When the watcher last ran: the heartbeat an off-machine check reads. */
+  heartbeatAt: string;
+  nodes: Record<
+    string,
+    { state: 'ready' | 'degraded'; since: string; notifiedAt: string | null }
+  >;
+}
+
+/** One machine as fleet-status saw it: the IDs of its failing checks. */
+export interface Observation {
+  node: string;
+  failing: string[];
+}
+
+export interface Notification {
+  event: 'degraded' | 'recovered' | 'still-degraded' | 'test';
+  node: string;
+  checks: string[];
+}
+
+/** Only what a report concluded: the required checks that failed. */
+export function observe(nodes: NodeReport[]): Observation[] {
+  return nodes.map((node) => ({
+    node: node.node,
+    failing: node.checks
+      .filter((check) => check.status === 'fail' && check.required !== false)
+      .map((check) => check.id),
+  }));
+}
+
+/** Notifications on transitions only (ready to degraded and back), and a
+ * reminder for a machine that stays degraded longer than the repeat interval.
+ * A machine never seen before counts as ready.
+ */
+export function transitions(
+  previous: WatchMemory | null,
+  observed: Observation[],
+  now: Date,
+  repeatHours: number
+): { notifications: Notification[]; memory: WatchMemory } {
+  const at = now.toISOString();
+  const notifications: Notification[] = [];
+  const nodes: WatchMemory['nodes'] = {};
+  for (const { node, failing } of observed) {
+    const before = previous?.nodes[node];
+    const state = failing.length ? 'degraded' : 'ready';
+    const checks = failing.filter(safe);
+    if (state === (before?.state ?? 'ready')) {
+      const due = state === 'degraded' && repeatDue(before, now, repeatHours);
+      if (due) notifications.push({ event: 'still-degraded', node, checks });
+      nodes[node] = before
+        ? { ...before, ...(due ? { notifiedAt: at } : {}) }
+        : { state, since: at, notifiedAt: null };
+      continue;
+    }
+    notifications.push({
+      event: state === 'degraded' ? 'degraded' : 'recovered',
+      node,
+      checks,
+    });
+    nodes[node] = { state, since: at, notifiedAt: at };
+  }
+  return { notifications, memory: { version: 1, heartbeatAt: at, nodes } };
+}
+
+function repeatDue(
+  before: WatchMemory['nodes'][string] | undefined,
+  now: Date,
+  repeatHours: number
+): boolean {
+  if (repeatHours <= 0 || !before?.notifiedAt) return false;
+  return (
+    now.getTime() - Date.parse(before.notifiedAt) >= repeatHours * 3_600_000
+  );
+}
+
+/** The memory stored last time, or null when there is none to trust. */
+export function parseMemory(text: string | null): WatchMemory | null {
+  if (!text) return null;
+  try {
+    const value: unknown = JSON.parse(text);
+    if (
+      !record(value) ||
+      value['version'] !== 1 ||
+      typeof value['heartbeatAt'] !== 'string' ||
+      !record(value['nodes'])
+    )
+      return null;
+    return value as unknown as WatchMemory;
+  } catch {
+    return null;
+  }
+}
+
+/** Node and check IDs are the only words a notification carries. */
+const safe = (id: string): boolean => /^[A-Za-z0-9:._@-]{1,128}$/.test(id);
+
+const WORDS: Record<Notification['event'], string> = {
+  degraded: 'is degraded',
+  recovered: 'is ready again',
+  'still-degraded': 'is still degraded',
+  test: 'sent a test notification',
+};
+
+export function notificationText(notification: Notification): string {
+  const node = safe(notification.node) ? notification.node : 'a machine';
+  const checks = notification.checks.filter(safe);
+  return `dev-fleet: ${node} ${WORDS[notification.event]}${
+    checks.length ? `. Failing checks: ${checks.join(', ')}` : ''
+  }`;
+}
+
+/** Sends one notification to every target. The result names the target and
+ * whether it took the notification: never its address or its answer.
+ */
+export async function deliver(
+  config: NotifyConfig,
+  notification: Notification,
+  secrets: SecretsProvider,
+  send: typeof fetch = fetch
+): Promise<Check[]> {
+  const text = notificationText(notification);
+  const checks: Check[] = [];
+  for (const target of config.targets) {
+    const id = `notify:${target}`;
+    const url = await secrets(NOTIFY_TARGETS[target]);
+    if (!url || !/^https?:\/\//.test(url)) {
+      checks.push({
+        id,
+        status: 'fail',
+        summary: 'NOTIFY_TARGET_UNAVAILABLE',
+        remedy: {
+          action: `Store the address of the ${target} target as the secret "${NOTIFY_TARGETS[target]}" with the secret provider fleet.json names.`,
+        },
+      });
+      continue;
+    }
+    const ok = await send(url, request(target, notification, text))
+      .then((response) => response.ok)
+      .catch(() => false);
+    checks.push(
+      ok
+        ? {
+            id,
+            status: 'pass',
+            summary: `The ${target} target took the notification.`,
+          }
+        : {
+            id,
+            status: 'fail',
+            summary: 'NOTIFY_DELIVERY_FAILED',
+            remedy: {
+              action: `Check that the ${target} target answers, then run devfleet fleet-watch --test.`,
+            },
+          }
+    );
+  }
+  return checks;
+}
+
+function request(
+  target: NotifyTarget,
+  notification: Notification,
+  text: string
+): RequestInit {
+  const signal = AbortSignal.timeout(15_000);
+  if (target === 'ntfy')
+    return {
+      method: 'POST',
+      body: text,
+      headers: {
+        Title: `dev-fleet: ${notification.event}`,
+        Tags:
+          notification.event === 'recovered' ? 'white_check_mark' : 'warning',
+      },
+      signal,
+    };
+  return {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      source: 'dev-fleet',
+      event: notification.event,
+      node: safe(notification.node) ? notification.node : null,
+      checks: notification.checks.filter(safe),
+      text,
+    }),
+    signal,
+  };
+}

package/src/platform/darwin.ts

@@ -176,19 +176,6 @@ export class DarwinPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    return join(
-      homedir(),
-      'Library/Application Support/Code/User/settings.json'
-    );
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return [
-      '/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code',
-    ].filter((p) => existsSync(p));
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return installLaunchdJob(job);
   }

package/src/platform/fallback.ts

@@ -78,14 +78,6 @@ export class FallbackPlatform implements Platform {
     return [];
   }
 
-  vscodeSettingsPath(): string | null {
-    return null;
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return [];
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return {
       ok: false,

package/src/platform/linux.ts

@@ -115,16 +115,6 @@ export class LinuxPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    const base =
-      process.env['XDG_CONFIG_HOME']?.trim() || join(homedir(), '.config');
-    return join(base, 'Code/User/settings.json');
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return ['/usr/share/code/bin/code'].filter((p) => existsSync(p));
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return installSystemdJob(job);
   }

package/src/platform/t3-policy.ts

@@ -1,6 +1,8 @@
 import { runAnsible } from './ansible';
 import { desired } from '../t3/desired';
 import policy from '../../ansible/ubuntu/t3/policy.json';
+import ubuntu from '../../ansible/ubuntu/policy.json';
+import { restartDeferral } from './t3-service';
 import { basename, dirname, join, resolve } from 'node:path';
 import { rename, rm, symlink } from 'node:fs/promises';
 import { userInfo } from 'node:os';
@@ -34,6 +36,7 @@ export async function stageT3Policy(
       t3_policy_document: document,
       development_user: user.username,
       development_uid: user.uid,
+      development_home: user.homedir,
     }),
     timeoutMs: 120_000,
   });
@@ -84,14 +87,21 @@ export const T3_RESTARTED_SUMMARY =
   'T3 was restarted to load the new managed policy; the running server confirms it.';
 
 /** `t3-policy.yml` restarts a running T3 that has not loaded the staged policy
- * (ending its running turns), so every node offers the same providers after
- * every sync. This measures the outcome; `restarted` says whether the app
- * restarted during this sync and so needs time to reconnect.
+ * so every node offers the same providers, but only while it runs no agent
+ * turn: a busy T3 keeps its turns and the play records the restart as
+ * pending, for personal-worker-recovery to apply at the first idle check.
+ * This measures the outcome; `restarted` says whether the app restarted
+ * during this sync and so needs time to reconnect.
  */
 export async function loadedT3Policy(
   executable: string,
   run: Runner,
-  options: { restarted: boolean; pause?: () => Promise<void> }
+  options: {
+    restarted: boolean;
+    pause?: () => Promise<void>;
+    uid?: number;
+    now?: number;
+  }
 ): Promise<Check> {
   // A restarted desktop app takes a few seconds to come back with its server.
   const check = await verifyT3Policy(
@@ -103,8 +113,25 @@ export async function loadedT3Policy(
           setTimeout(done, options.restarted ? 5000 : 1000)
         ))
   );
-  if (check.status !== 'pass')
-    return { ...check, summary: 'T3_POLICY_RESTART_FAILED' };
+  if (check.status !== 'pass') {
+    const deferral = await restartDeferral(
+      run,
+      options.uid ?? userInfo().uid,
+      options.now
+    );
+    if (!deferral.pending)
+      return { ...check, summary: 'T3_POLICY_RESTART_FAILED' };
+    return {
+      ...check,
+      status: deferral.overdue ? 'fail' : 'warn',
+      summary: 'T3_POLICY_RESTART_PENDING',
+      remedy: {
+        action: deferral.overdue
+          ? `Agent turns kept T3 from loading the staged policy for ${ubuntu.t3RestartMaxDeferralHours} hours. Finish or stop them, or restart t3code-desktop.service on the worker (this ends its running turns).`
+          : 'Agent turns are running, so T3 keeps them and loads the staged policy at the first check of personal-worker-recovery with no turn running.',
+      },
+    };
+  }
   return options.restarted
     ? { ...check, summary: T3_RESTARTED_SUMMARY }
     : check;

package/src/platform/t3-service.ts

@@ -1,3 +1,4 @@
+import policy from '../../ansible/ubuntu/policy.json';
 import type { Runner, Check } from '../types';
 
 /** The user unit that runs the pinned desktop app (UI and embedded server). */
@@ -44,6 +45,109 @@ export async function convergeT3Service(
   ];
 }
 
+/** The recovery script, installed outside every revision (`tasks/recovery.yml`). */
+export const T3_HEALTH_SCRIPT = '/usr/local/lib/personal-fleet/t3-health.sh';
+
+/** The record of the package the desktop app was unpacked from
+ * (`t3-desktop.yml`): `T3-Code-<version>-<arch>.deb`. */
+export const T3_DESKTOP_RELEASE =
+  '/opt/T3 Code (Nightly)/.personal-fleet-version';
+
+/** Where a play records a T3 restart it deferred because agent turns were
+ * running (`tasks/t3-restart.yml`): the second it was first deferred.
+ * personal-worker-recovery clears it when it restarts the idle app. */
+export function t3RestartMarker(uid: number): string {
+  return `/run/user/${uid}/personal-t3-restart-pending`;
+}
+
+/** A restart deferred while agent turns run, and whether it has waited longer
+ * than the policy allows. */
+export async function restartDeferral(
+  run: Runner,
+  uid: number,
+  now: number = Date.now()
+): Promise<{ pending: boolean; overdue: boolean }> {
+  const marker = await run(['cat', t3RestartMarker(uid)]);
+  if (!marker.ok) return { pending: false, overdue: false };
+  const since = Number(marker.stdout.trim());
+  const limit = policy.t3RestartMaxDeferralHours * 3600;
+  return {
+    pending: true,
+    overdue: Number.isFinite(since) && now / 1000 - since > limit,
+  };
+}
+
+/** The release the running server reports, compared with the one installed.
+ * Asks the server itself, so it needs no `t3` on the search path. A new
+ * release is installed beside a running app, which restarts into it only
+ * while it runs no agent turn: until then this is T3_RESTART_PENDING, and a
+ * failure once the restart has waited longer than the policy allows. */
+export async function runningVersionCheck(
+  run: Runner,
+  uid: number,
+  now: number = Date.now()
+): Promise<Check> {
+  const server = await run(
+    [
+      'curl',
+      '--fail',
+      '--silent',
+      '--max-time',
+      '10',
+      'http://127.0.0.1:3773/.well-known/t3/environment',
+    ],
+    { timeoutMs: 15_000 }
+  );
+  const installed = await run(['cat', T3_DESKTOP_RELEASE]);
+  const running = serverVersion(server.ok ? server.stdout : '');
+  const release = /^T3-Code-(.+)-[a-z0-9]+\.deb$/.exec(
+    installed.ok ? installed.stdout.trim() : ''
+  )?.[1];
+  const id = 't3:running-version';
+  if (!running || !release)
+    return {
+      id,
+      status: 'fail',
+      summary: 'T3_RUNNING_VERSION_UNKNOWN',
+      remedy: {
+        action:
+          'The running server or the installed release could not be read. Check worker:t3-health, then rerun fleet-sync to reinstall the pinned desktop app.',
+      },
+    };
+  if (running === release)
+    return {
+      id,
+      status: 'pass',
+      summary: `The running T3 server is the installed release ${release}.`,
+    };
+  const deferral = await restartDeferral(run, uid, now);
+  return {
+    id,
+    status: deferral.overdue ? 'fail' : 'warn',
+    summary: 'T3_RESTART_PENDING',
+    remedy: {
+      action: deferral.overdue
+        ? `T3 still runs ${running} after ${policy.t3RestartMaxDeferralHours} hours because agent turns kept running. Finish or stop them, or restart ${T3_DESKTOP_UNIT} on the worker (this ends its running turns).`
+        : `T3 runs ${running}; ${release} is installed. personal-worker-recovery restarts it at the first check with no agent turn running.`,
+    },
+  };
+}
+
+function serverVersion(body: string): string | undefined {
+  try {
+    const value: unknown = JSON.parse(body);
+    const version =
+      typeof value === 'object' && value !== null
+        ? (value as Record<string, unknown>)['serverVersion']
+        : undefined;
+    return typeof version === 'string' && /^[\w.+-]+$/.test(version)
+      ? version
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 /** When the desktop app last started; changes exactly when it restarts. */
 export async function t3StartedAt(run: Runner): Promise<string> {
   const shown = await run([

package/src/platform/types.ts

@@ -110,10 +110,6 @@ export type Platform = {
 
   /** App-bundled Tailscale CLI binaries that exist (used when `tailscale` is not on PATH). */
   tailscaleAppBinaries(): readonly string[];
-  /** VS Code user `settings.json`, or `null` when unknown on this OS. */
-  vscodeSettingsPath(): string | null;
-  /** VS Code `code` CLI binaries shipped inside the installed app. */
-  vscodeCliBinaries(): readonly string[];
 
   /** Install (or replace) a periodic job. Never throws — returns a result. */
   installScheduledJob(job: ScheduledJob): NotifierTestResult;

package/src/platform/windows.ts

@@ -110,20 +110,6 @@ export class WindowsPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    const base =
-      process.env['APPDATA']?.trim() || join(homedir(), 'AppData/Roaming');
-    return join(base, 'Code/User/settings.json');
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    const base =
-      process.env['LOCALAPPDATA']?.trim() || join(homedir(), 'AppData/Local');
-    return [join(base, 'Programs/Microsoft VS Code/bin/code.cmd')].filter((p) =>
-      existsSync(p)
-    );
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return {
       ok: false,