@crvouga/dev-fleet 0.2.0 → 0.3.0

package/src/t3/version.ts

@@ -0,0 +1,119 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { sshArgs } from '../ssh';
+import type { Check, FleetNode, Runner } from '../types';
+
+/** The T3 release every machine runs: `t3_version` in mise.toml. */
+export async function pinnedT3Version(workspace: string): Promise<string> {
+  const toml = await readFile(join(workspace, 'mise.toml'), 'utf8');
+  const version = /^t3_version = "([^"]+)"$/m.exec(toml)?.[1];
+  if (!version) throw new Error('mise.toml pins no T3 version.');
+  return version;
+}
+
+/** Where T3 notes the version it last started as, under its home. */
+export const STARTED_VERSION = 'userdata/desktop-last-started-version';
+
+const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$/;
+
+/** `t3:version`: the T3 a machine runs is the one the fleet pins, so the
+ * manager and the workers it drives never run different releases.
+ */
+export function versionCheck(
+  running: string | null,
+  pinned: string,
+  where: string,
+  install: string
+): Check {
+  const observed = running?.trim() ?? '';
+  if (!VERSION.test(observed))
+    return {
+      id: 't3:version',
+      status: 'warn',
+      summary: 'T3_VERSION_UNKNOWN',
+      remedy: {
+        action: `T3 on ${where} has not recorded the version it runs. Start T3 there once, then run fleet-sync.`,
+      },
+    };
+  if (observed !== pinned)
+    return {
+      id: 't3:version',
+      status: 'warn',
+      summary: 'T3_VERSION_SKEW',
+      remedy: {
+        action: `T3 on ${where} runs ${observed}; the fleet pins ${pinned}. ${install}`,
+      },
+    };
+  return {
+    id: 't3:version',
+    status: 'pass',
+    summary: `T3 on ${where} runs ${pinned}, the release every machine runs.`,
+  };
+}
+
+export async function managerVersion(
+  t3Home: string,
+  workspace: string
+): Promise<Check> {
+  const running = await readFile(join(t3Home, STARTED_VERSION), 'utf8').catch(
+    () => null
+  );
+  return versionCheck(
+    running,
+    await pinnedT3Version(workspace),
+    'the manager',
+    'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -K ansible/macos/manager.yml) it no longer updates itself.'
+  );
+}
+
+export async function workerVersion(
+  node: FleetNode,
+  run: Runner,
+  workspace: string
+): Promise<Check> {
+  const result = await run(
+    [...sshArgs(node), `cat "$HOME/.t3/${STARTED_VERSION}"`],
+    { timeoutMs: 15_000 }
+  );
+  return versionCheck(
+    result.ok ? result.stdout : null,
+    await pinnedT3Version(workspace),
+    `worker ${node.id}`,
+    'Run fleet-sync for that worker: it installs the pinned desktop app and restarts T3 when its policy changed.'
+  );
+}
+
+/** Where T3 on macOS reads its administrator policy. */
+export const MACOS_POLICY = '/Library/Application Support/T3Code/policy.json';
+
+/** `t3:updates`: T3's own updater is off by the fleet's policy, so it changes
+ * only when the pin does.
+ */
+export async function updatesCheck(
+  policyPath: string,
+  stage: string
+): Promise<Check> {
+  let mode: unknown;
+  try {
+    const policy = JSON.parse(await readFile(policyPath, 'utf8')) as {
+      limits?: { updates?: { mode?: unknown } };
+    };
+    mode = policy.limits?.updates?.mode;
+  } catch {
+    mode = undefined;
+  }
+  return mode === 'off'
+    ? {
+        id: 't3:updates',
+        status: 'pass',
+        summary: 'T3’s own updater is off by the fleet’s policy.',
+      }
+    : {
+        id: 't3:updates',
+        status: 'warn',
+        summary: 'T3_UPDATES_UNMANAGED',
+        remedy: {
+          action: `T3 on this machine may update itself away from the pinned release. Stage the fleet's policy once, as an administrator: ${stage}`,
+        },
+      };
+}

package/src/t3-linked-check.ts

@@ -103,7 +103,7 @@ function linkedEvidence(
   };
 }
 
-async function observedIdentity(
+export async function observedIdentity(
   node: FleetNode,
   run: Runner
 ): Promise<string | undefined> {

package/src/transport.ts

@@ -21,7 +21,7 @@ export async function bundleProbe(): Promise<string> {
 export interface RemoteOptions {
   source: string;
   timeoutMs: number;
-  action: 'probe' | 'sync' | 'repair' | 'configure';
+  action: 'probe' | 'sync' | 'repair' | 'configure' | 'accounts';
   configuration?: ConfigurationRequest;
   repairs?: AutomatedRepair[];
   expected?: Fingerprint;
@@ -41,7 +41,8 @@ export async function inspectRemote(
       return { node: node.id, checks: [sshFailure(preflight)] };
   }
   const target =
-    options.action === 'probe' && options.expected
+    (options.action === 'probe' || options.action === 'accounts') &&
+    options.expected
       ? await managedProbeTarget(node, options.expected.head, run)
       : { node, checks: [] };
   const command = remoteCommand(target.node, {

package/src/worker-host-checks.ts

@@ -4,6 +4,7 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
 import { verifyResourceSafety } from './platform/resource-verify';
 import policy from '../ansible/ubuntu/policy.json';
 import { userInfo } from 'node:os';
+import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
 import type { Check, Runner } from './types';
 
 function check(id: string, ok: boolean, summary: string): Check {
@@ -120,18 +121,13 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
     )
   );
   checks.push(await t3HealthCheck(run));
+  checks.push(await runningVersionCheck(run, userInfo().uid));
   const renewal = await run([
     'systemctl',
     '--user',
     'is-active',
     'personal-worker-auth.timer',
   ]);
-  const recovery = await run([
-    'systemctl',
-    '--user',
-    'is-active',
-    'personal-worker-recovery.timer',
-  ]);
   checks.push(
     check(
       'credential-renewal',
@@ -139,17 +135,44 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
       'Worker credential renewal timer is active.'
     )
   );
-  checks.push(
-    check(
-      'service-recovery',
-      recovery.ok,
-      'Worker service recovery timer is active.'
-    )
-  );
+  checks.push(await serviceRecoveryCheck(run));
   checks.push(await powerRecoveryCheck(run));
   return checks;
 }
 
+/** The recovery timer is active, the script its unit runs exists, and the
+ * last run of the unit succeeded: a watchdog that fails on every tick is not
+ * recovery. */
+export async function serviceRecoveryCheck(run: Runner): Promise<Check> {
+  const timer = await run([
+    'systemctl',
+    '--user',
+    'is-active',
+    'personal-worker-recovery.timer',
+  ]);
+  const script = await run(['test', '-r', T3_HEALTH_SCRIPT]);
+  const unit = await run([
+    'systemctl',
+    '--user',
+    'show',
+    'personal-worker-recovery.service',
+    '--property=Result',
+  ]);
+  const result = /^Result=(.*)$/m.exec(unit.stdout)?.[1] ?? 'unknown';
+  const summary = !timer.ok
+    ? 'WORKER_RECOVERY_TIMER_INACTIVE'
+    : !script.ok
+      ? 'WORKER_RECOVERY_SCRIPT_MISSING'
+      : !unit.ok || result !== 'success'
+        ? `WORKER_RECOVERY_LAST_RUN_FAILED (${/^[\w-]+$/.test(result) ? result : 'unknown'})`
+        : 'Worker service recovery timer is active and its last run succeeded.';
+  return check(
+    'service-recovery',
+    timer.ok && script.ok && unit.ok && result === 'success',
+    summary
+  );
+}
+
 /** The server answers its local API now, not merely that a process exists. */
 async function t3HealthCheck(run: Runner): Promise<Check> {
   const probe = await run(