@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/state/store.ts

@@ -17,9 +17,11 @@ import { FleetUsageError } from '../errors';
  */
 export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
 /** Kept when there is something to keep; a fleet without them is complete:
- * `history` is the record of recent fleet-sync runs and of the schedule.
+ * `history` is the record of recent fleet-sync runs and of the schedule;
+ * `workers` what the manager knows of each worker beyond fleet.json: its
+ * pinned SSH host key and whether it is parked (src/state/workers.ts).
  */
-export const OPTIONAL_DOCUMENTS = ['history'] as const;
+export const OPTIONAL_DOCUMENTS = ['history', 'workers'] as const;
 export type DocumentName = (typeof DOCUMENTS)[number];
 /** Every document the fleet state may hold. */
 export type StoredName = DocumentName | (typeof OPTIONAL_DOCUMENTS)[number];

package/src/state/sync.ts

@@ -349,9 +349,14 @@ export function pull(context: StateContext): Promise<Standing> {
 }
 
 /** Replaces this copy with the version the bucket held at `timestamp`, then
- * uploads it as the bucket's current version: the way back from a replica
- * whose latest version is damaged. `unreachable` when the bucket holds no
- * readable version at that time.
+ * uploads it as the bucket's current version, which takes the manager lease:
+ * the way back from a replica whose latest version is damaged. `unreachable`
+ * when the bucket holds no readable version at that time.
+ *
+ * Refused, before anything changes, while another manager holds the lease in
+ * the bucket's current version, unless told to take it over. When that
+ * version cannot be read the lease of this machine's copy, the last one it
+ * saw, decides instead.
  */
 export function pullAt(
   context: StateContext,
@@ -367,6 +372,17 @@ export function pullAt(
       'No bucket is configured, so there is no earlier version to restore.'
     );
   return locked(context.root, async () => {
+    const database = databasePath(context.root);
+    const current = await fetchRemote({ ...context, bucket });
+    try {
+      const held = leaseIn(
+        context,
+        current.status === 'present' ? current.file : database
+      );
+      if (held) throw new LeaseHeldError(leaseHeldMessage(held));
+    } finally {
+      if (current.status === 'present') discard(current.file);
+    }
     const remote = await fetchRemote(
       { ...context, bucket },
       true,
@@ -374,10 +390,17 @@ export function pullAt(
     );
     if (remote.status !== 'present')
       return remote.status === 'absent' ? 'empty' : remote.status;
-    const database = databasePath(context.root);
     discard(database);
     renameSync(remote.file, database);
     chmodSync(database, 0o600);
+    // The earlier version names whoever held the lease then.
+    if (context.holder) {
+      const now = (context.now ?? (() => new Date()))();
+      writeLease(database, {
+        holder: context.holder,
+        expiresAt: new Date(now.getTime() + LEASE_MS).toISOString(),
+      });
+    }
     return upload({ ...context, bucket }, true);
   });
 }
@@ -408,11 +431,17 @@ export class LeaseHeldError extends FleetUsageError {}
  * and this machine was not told to take it over. Reads local files only: call
  * it on the bucket's version, after a refresh.
  */
-export function leaseConflict(
+export function leaseConflict(context: StateContext, now?: Date): Lease | null {
+  return leaseIn(context, databasePath(context.root), now);
+}
+
+/** The lease another manager holds over the database at `file`. */
+function leaseIn(
   context: StateContext,
+  file: string,
   now: Date = (context.now ?? (() => new Date()))()
 ): Lease | null {
-  const lease = readLease(databasePath(context.root));
+  const lease = readLease(file);
   if (!context.holder || !lease || lease.holder === context.holder) return null;
   if (Date.parse(lease.expiresAt) <= now.getTime() || context.takeOver)
     return null;

package/src/state/workers.ts

@@ -0,0 +1,140 @@
+import { record } from '../config';
+import { HOST_KEY } from '../definition';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+import { mutateState, type StateContext, type Standing } from './sync';
+
+/** What the manager knows of one worker that fleet.json does not say. */
+export interface WorkerState {
+  /** The pinned SSH host key, "TYPE BASE64", and when it was pinned. */
+  hostKey?: string;
+  hostKeyPinnedAt?: string;
+  /** When the worker was parked: stopped or destroyed on purpose. Absent,
+   * the worker is active.
+   */
+  parkedAt?: string;
+  /** `records`: the worker was enrolled from a worker record, so it is
+   * retired by itself once its record disappears.
+   */
+  origin?: 'records';
+  /** When fleet-node-replace put a new machine in the worker's place: the
+   * manager pairs its T3 with the new machine's T3 environment as soon as
+   * it runs, then forgets this. A worker enrolled for the first time is
+   * paired by a person.
+   */
+  t3PairingRequestedAt?: string;
+}
+
+/** The `workers` document of the fleet state. */
+export interface WorkerStates {
+  version: 1;
+  workers: Record<string, WorkerState>;
+}
+
+const ID = /^[a-z][a-z0-9-]{0,62}$/;
+const isTime = (value: unknown): value is string =>
+  typeof value === 'string' && !Number.isNaN(Date.parse(value));
+
+const RULES: Record<keyof WorkerState, (value: unknown) => boolean> = {
+  hostKey: (value) => typeof value === 'string' && HOST_KEY.test(value),
+  hostKeyPinnedAt: isTime,
+  parkedAt: isTime,
+  origin: (value) => value === 'records',
+  t3PairingRequestedAt: isTime,
+};
+
+function workerState(value: unknown): WorkerState | null {
+  if (!record(value)) return null;
+  const broken = Object.entries(RULES).some(
+    ([field, valid]) => value[field] !== undefined && !valid(value[field])
+  );
+  if (broken) return null;
+  const { hostKey, hostKeyPinnedAt, parkedAt, origin, t3PairingRequestedAt } =
+    value;
+  return {
+    ...(typeof hostKey === 'string' ? { hostKey } : {}),
+    ...(isTime(hostKeyPinnedAt) ? { hostKeyPinnedAt } : {}),
+    ...(isTime(parkedAt) ? { parkedAt } : {}),
+    ...(origin === 'records' ? { origin } : {}),
+    ...(isTime(t3PairingRequestedAt) ? { t3PairingRequestedAt } : {}),
+  };
+}
+
+/** The document, or an empty one. An entry that cannot be read is dropped:
+ * an unreadable pin pins nothing, so the worker is refused, never trusted.
+ */
+export function parseWorkerStates(text: string | null): WorkerStates {
+  let value: unknown = null;
+  try {
+    value = text === null ? null : JSON.parse(text);
+  } catch {
+    value = null;
+  }
+  const workers: Record<string, WorkerState> = {};
+  if (record(value) && record(value['workers']))
+    for (const [id, entry] of Object.entries(value['workers'])) {
+      const parsed = workerState(entry);
+      if (ID.test(id) && parsed) workers[id] = parsed;
+    }
+  return { version: 1, workers };
+}
+
+export function readWorkerStates(env = process.env): WorkerStates {
+  return parseWorkerStates(
+    readDocument(databasePath(stateRoot(env)), 'workers')
+  );
+}
+
+/** The host keys pinned in the fleet state, by worker ID. */
+export function readPins(env = process.env): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(readWorkerStates(env).workers).flatMap(([id, entry]) =>
+      entry.hostKey ? [[id, entry.hostKey]] : []
+    )
+  );
+}
+
+/** When each parked worker was parked, by worker ID. */
+export function readParked(env = process.env): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(readWorkerStates(env).workers).flatMap(([id, entry]) =>
+      entry.parkedAt ? [[id, entry.parkedAt]] : []
+    )
+  );
+}
+
+function writeWorkerStates(states: WorkerStates, env = process.env): void {
+  const workers = Object.fromEntries(
+    Object.entries(states.workers)
+      .filter(([, entry]) => Object.keys(entry).length > 0)
+      .sort(([a], [b]) => a.localeCompare(b))
+  );
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'workers',
+    `${JSON.stringify({ version: 1, workers }, null, 2)}\n`
+  );
+}
+
+/** Changes the entries of some workers on top of the bucket's version and
+ * uploads the result. `change` returns the new entry of each worker it
+ * changes, or null to forget the worker.
+ */
+export async function changeWorkers(
+  context: StateContext,
+  change: (
+    current: Record<string, WorkerState>
+  ) => Record<string, WorkerState | null>
+): Promise<{ workers: Record<string, WorkerState>; standing: Standing }> {
+  const { value, standing } = await mutateState(context, () => {
+    const states = readWorkerStates();
+    const updates = change(states.workers);
+    const workers = { ...states.workers };
+    for (const [id, entry] of Object.entries(updates))
+      if (entry === null) delete workers[id];
+      else workers[id] = entry;
+    writeWorkerStates({ version: 1, workers });
+    return workers;
+  });
+  return { workers: value, standing };
+}

package/src/t3/pairing.ts

@@ -1,6 +1,9 @@
 import { parseConfig } from '../config';
 import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
 import { mutateState, stateContext, type StateContext } from '../state/sync';
+import { changeWorkers, readWorkerStates } from '../state/workers';
+import type { Check, FleetNode, Runner } from '../types';
+import { pairWorker, pairingCheck, type ManagerT3 } from './peer-link';
 
 /** Records the T3 environment a worker runs in the roster, the first time a
  * verified link from the manager reaches it. From then on the link check
@@ -29,3 +32,59 @@ export async function rememberEnvironment(
   if (path !== STATE_ROSTER) return save();
   return (await mutateState(context(), save)).value;
 }
+
+/** What pairing a worker needs: where the roster is, how commands run, the
+ * manager's T3 and the fleet state. */
+export interface PairingServices {
+  roster: string;
+  run: Runner;
+  manager: ManagerT3;
+  context: () => StateContext;
+  now: () => Date;
+}
+
+/** Pairs the manager's T3 with the environment a worker runs now, without a
+ * person (src/t3/peer-link.ts), and records it in the roster. Until it is
+ * paired, the fleet state keeps the request, so the next fleet-sync pairs
+ * it once the worker's T3 runs; once paired, the request is forgotten.
+ */
+export async function pairAndRecord(
+  node: FleetNode,
+  services: PairingServices
+): Promise<Check> {
+  const pairing = await pairWorker(node, services.run, services.manager);
+  if ('paired' in pairing)
+    await rememberEnvironment(
+      services.roster,
+      node.id,
+      pairing.paired,
+      services.context
+    );
+  const requested = services.now().toISOString();
+  await changeWorkers(services.context(), (current) => {
+    const { t3PairingRequestedAt: earlier, ...rest } = current[node.id] ?? {};
+    return {
+      [node.id]:
+        'paired' in pairing
+          ? rest
+          : { ...rest, t3PairingRequestedAt: earlier ?? requested },
+    };
+  });
+  return pairingCheck(node.id, pairing);
+}
+
+/** fleet-sync's part: pairs a worker only when fleet-node-replace asked for
+ * it. A worker enrolled for the first time is paired by a person.
+ */
+export async function pairIfRequested(
+  node: FleetNode,
+  services: Omit<PairingServices, 'context' | 'now'> &
+    Partial<Pick<PairingServices, 'context'>>
+): Promise<Check | undefined> {
+  if (!readWorkerStates().workers[node.id]?.t3PairingRequestedAt) return;
+  return pairAndRecord(node, {
+    context: () => stateContext(services.run),
+    ...services,
+    now: () => new Date(),
+  });
+}

package/src/t3/peer-link.ts

@@ -0,0 +1,333 @@
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { record } from '../config';
+import { sshArgs } from '../ssh';
+import { observedIdentity } from '../t3-linked-check';
+import { discoverTailnet } from '../tailscale';
+import type { Check, FleetNode, Runner } from '../types';
+
+/** Pairing the manager's T3 with a worker's T3 environment, with no person.
+ *
+ * A pairing code is a one-time secret, so it never enters a command line,
+ * a report or a log. The worker mints it (`t3 auth pairing create --json`
+ * over the SSH session the manager already has), and its stdout is read in
+ * memory. The manager's T3 receives it in the body of one request to its
+ * own server on loopback: the `peerLinks.link` call of its WebSocket API,
+ * which the T3 Settings (Connections) use, authorized by a session issued
+ * for this one call and revoked after it. `t3 environment link` takes the
+ * code only as `--pairing-code`, so it is not used.
+ */
+
+/** What the manager's T3 is asked to do: link the environment at `url`. */
+export interface LinkRequest {
+  url: string;
+  pairingCode: string;
+  /** The environment the worker answered as over SSH: T3 refuses an
+   * address that answers as another one. */
+  expectedEnvironmentId: string;
+}
+
+export type LinkOutcome =
+  | { environmentId: string }
+  /** Why T3 refused, as its own reason (`pairing_rejected`, `unreachable`,
+   * `disabled`, …), or `unavailable` when the manager's T3 did not answer. */
+  | { refused: string };
+
+/** The manager's own T3. */
+export interface ManagerT3 {
+  link(request: LinkRequest): Promise<LinkOutcome>;
+}
+
+/** The two requests to the manager's T3 server, injected by scenarios. */
+export interface T3Transport {
+  /** POSTs to the server with the session as a bearer token; the JSON body. */
+  post(url: string, bearer: string): Promise<unknown>;
+  /** Sends one message on a WebSocket and answers with the first message
+   * that ends request `id`. */
+  call(url: string, message: object, id: string): Promise<unknown>;
+}
+
+/** The access the manager's agents get on the worker: T3's default. */
+const ACCESS = 'approval-required';
+/** Scopes of the session that makes the call, and how long it lives at most. */
+const SESSION = [
+  '--ttl',
+  '2m',
+  '--scope',
+  'access:write',
+  '--label',
+  'dev-fleet pairing',
+];
+/** The WebSocket protocol of the pinned T3 server. */
+const ORCHESTRATION_PROTOCOL = '2';
+const CODE = /^[A-Za-z0-9_-]{6,128}$/;
+const ENVIRONMENT =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/** The manager's T3, reached through its local server. */
+export function managerT3(
+  run: Runner,
+  options: {
+    home?: string;
+    transport?: T3Transport;
+    read?: (path: string) => Promise<string>;
+  } = {}
+): ManagerT3 {
+  const home = resolve(
+    options.home ?? process.env['T3CODE_HOME'] ?? join(homedir(), '.t3')
+  );
+  const transport = options.transport ?? webTransport;
+  const read = options.read ?? ((path: string) => readFile(path, 'utf8'));
+  return {
+    async link(request) {
+      const origin = await loopbackOrigin(read, home);
+      if (!origin) return { refused: 'unavailable' };
+      const issued = await run(
+        ['t3', 'auth', 'session', 'issue', '--json', ...SESSION],
+        { timeoutMs: 30_000 }
+      );
+      const session = parseSession(issued.ok ? issued.stdout : '');
+      if (!session) return { refused: 'unavailable' };
+      try {
+        const ticket = await transport.post(
+          `${origin}/api/auth/websocket-ticket/rpc`,
+          session.token
+        );
+        if (!record(ticket) || typeof ticket['ticket'] !== 'string')
+          return { refused: 'unavailable' };
+        const socket = new URL('/ws', origin);
+        socket.protocol = socket.protocol === 'https:' ? 'wss:' : 'ws:';
+        socket.searchParams.set(
+          'orchestrationProtocol',
+          ORCHESTRATION_PROTOCOL
+        );
+        socket.searchParams.set('wsTicket', ticket['ticket']);
+        const reply = await transport.call(
+          socket.href,
+          {
+            _tag: 'Request',
+            id: '1',
+            tag: 'peerLinks.link',
+            payload: {
+              url: request.url,
+              pairingCode: request.pairingCode,
+              access: ACCESS,
+              expectedEnvironmentId: request.expectedEnvironmentId,
+            },
+            headers: [],
+          },
+          '1'
+        );
+        return outcomeOf(reply);
+      } catch {
+        return { refused: 'unavailable' };
+      } finally {
+        await run(['t3', 'auth', 'session', 'revoke', session.id], {
+          timeoutMs: 30_000,
+        });
+      }
+    },
+  };
+}
+
+/** The origin the manager's T3 server records for itself, when it is on
+ * this machine: a code is never sent anywhere else. */
+async function loopbackOrigin(
+  read: (path: string) => Promise<string>,
+  home: string
+): Promise<string | undefined> {
+  try {
+    const runtime: unknown = JSON.parse(
+      await read(join(home, 'userdata/server-runtime.json'))
+    );
+    if (!record(runtime) || typeof runtime['origin'] !== 'string') return;
+    const origin = new URL(runtime['origin']);
+    return origin.protocol === 'http:' &&
+      ['127.0.0.1', 'localhost', '[::1]'].includes(origin.hostname)
+      ? origin.origin
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function parseSession(text: string): { id: string; token: string } | null {
+  try {
+    const value: unknown = JSON.parse(text);
+    return record(value) &&
+      typeof value['sessionId'] === 'string' &&
+      /^[A-Za-z0-9_-]{1,128}$/.test(value['sessionId']) &&
+      typeof value['token'] === 'string' &&
+      value['token'].length > 0
+      ? { id: value['sessionId'], token: value['token'] }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** The end of the call: the link T3 made, or why it refused. */
+function outcomeOf(reply: unknown): LinkOutcome {
+  if (!record(reply) || reply['_tag'] !== 'Exit' || !record(reply['exit']))
+    return { refused: 'unavailable' };
+  const exit = reply['exit'];
+  if (exit['_tag'] === 'Success' && record(exit['value'])) {
+    const id = exit['value']['environmentId'];
+    return typeof id === 'string' && ENVIRONMENT.test(id)
+      ? { environmentId: id }
+      : { refused: 'unavailable' };
+  }
+  return { refused: refusal(exit['cause']) };
+}
+
+/** T3's own reason for a failed call: a word, never its message. */
+function refusal(cause: unknown): string {
+  const first: unknown = Array.isArray(cause) ? cause[0] : undefined;
+  const error = record(first) ? first['error'] : undefined;
+  const reason = record(error) ? error['reason'] : undefined;
+  return typeof reason === 'string' && /^[a-z_]{1,40}$/.test(reason)
+    ? reason
+    : 'unavailable';
+}
+
+const webTransport: T3Transport = {
+  async post(url, bearer) {
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: { authorization: `Bearer ${bearer}` },
+      signal: AbortSignal.timeout(10_000),
+    });
+    return response.ok ? ((await response.json()) as unknown) : null;
+  },
+  call(url, message, id) {
+    return new Promise((resolve, reject) => {
+      const socket = new WebSocket(url);
+      const timer = setTimeout(() => {
+        socket.close();
+        reject(new Error('timeout'));
+      }, 60_000);
+      const end = (value: unknown, error?: Error) => {
+        clearTimeout(timer);
+        socket.close();
+        if (error) reject(error);
+        else resolve(value);
+      };
+      socket.onopen = () => socket.send(JSON.stringify(message));
+      socket.onmessage = (event) => {
+        try {
+          const reply: unknown = JSON.parse(String(event.data));
+          if (
+            record(reply) &&
+            reply['_tag'] === 'Exit' &&
+            reply['requestId'] === id
+          )
+            end(reply);
+        } catch {
+          // Not the reply; keep waiting.
+        }
+      };
+      socket.onerror = () => end(null, new Error('socket'));
+      socket.onclose = () => end(null, new Error('closed'));
+    });
+  },
+};
+
+/** Run on the worker, as its development user, over SSH: mints a pairing
+ * code for the manager with the worker's own T3 (the CLI on PATH, or the
+ * pinned one the toolchain installed) and prints it as JSON on stdout. */
+export const MINT_PAIRING_CODE =
+  't3="$(command -v t3 || ls -d "$HOME"/.local/share/personal/tools/t3-*/t3 2>/dev/null | sort -V | tail -n 1)" && ' +
+  'exec "$t3" auth pairing create --json --ttl 5m --label "dev-fleet manager"';
+
+/** Mints a one-time pairing code on the worker. The code stays in memory. */
+async function mintPairingCode(
+  node: FleetNode,
+  run: Runner
+): Promise<string | undefined> {
+  const minted = await run([...sshArgs(node), MINT_PAIRING_CODE], {
+    timeoutMs: 30_000,
+  });
+  if (!minted.ok) return undefined;
+  try {
+    const value: unknown = JSON.parse(minted.stdout);
+    return record(value) &&
+      typeof value['credential'] === 'string' &&
+      CODE.test(value['credential'])
+      ? value['credential']
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export type Pairing =
+  | { paired: string }
+  /** The worker's T3 does not answer yet: nothing was minted. */
+  | { waiting: true }
+  | { failed: string };
+
+/** Pairs the manager's T3 with the T3 environment `node` runs now: the
+ * environment the worker answers as over SSH, at its private HTTPS address
+ * on the tailnet. */
+export async function pairWorker(
+  node: FleetNode,
+  run: Runner,
+  manager: ManagerT3
+): Promise<Pairing> {
+  const identity = await observedIdentity(node, run).catch(() => undefined);
+  if (!identity) return { waiting: true };
+  const tailnet = await discoverTailnet(run);
+  const peer = tailnet.computers.find(
+    (computer) => computer.id === node.tailscaleId && computer.online
+  );
+  if (!peer || !/^[a-z0-9.-]+\.ts\.net$/.test(peer.dns))
+    return { failed: 'unreachable' };
+  const pairingCode = await mintPairingCode(node, run);
+  if (!pairingCode) return { failed: 'mint' };
+  const linked = await manager.link({
+    url: `https://${peer.dns}`,
+    pairingCode,
+    expectedEnvironmentId: identity,
+  });
+  if ('refused' in linked) return { failed: linked.refused };
+  return linked.environmentId === identity
+    ? { paired: identity }
+    : { failed: 'identity' };
+}
+
+/** What each refusal asks a person to do. */
+function remedy(id: string, reason: string): string {
+  switch (reason) {
+    case 'mint':
+      return `The T3 of ${id} did not mint a pairing code. Run devfleet fleet-sync --node ${id} so it runs the pinned T3, then run the command again.`;
+    case 'disabled':
+      return "Linked environments is off in the manager's T3. Turn it on in its Settings (Labs), or stage the fleet's T3 policy, which turns it on, with ansible/macos/manager-privilege.yml as an administrator; then run the command again.";
+    case 'unavailable':
+      return "The manager's T3 did not answer on this machine. Open the T3 app (or start its service), then run the command again.";
+    default:
+      return `The T3 of ${id} refused the link (${reason}). Check that it runs the T3 policy fleet-sync stages (Linked environments on) and answers at its private address, then run devfleet fleet-sync --node ${id}.`;
+  }
+}
+
+/** The `t3:link` check of a pairing. */
+export function pairingCheck(id: string, pairing: Pairing): Check {
+  if ('paired' in pairing)
+    return {
+      id: 't3:link',
+      status: 'pass',
+      summary: `The manager's T3 is paired with the T3 environment ${pairing.paired} of ${id}.`,
+    };
+  if ('waiting' in pairing)
+    return {
+      id: 't3:link',
+      status: 'skip',
+      summary: `The T3 of ${id} does not answer yet; devfleet fleet-sync --node ${id} pairs the manager with it once it runs.`,
+    };
+  return {
+    id: 't3:link',
+    status: 'fail',
+    summary: 'T3_LINK_PAIRING_FAILED',
+    remedy: { action: remedy(id, pairing.failed) },
+  };
+}

package/src/t3/version.ts

@@ -62,7 +62,7 @@ export async function managerVersion(
     running,
     await pinnedT3Version(workspace),
     'the manager',
-    'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -K ansible/macos/manager.yml) it no longer updates itself.'
+    'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -e user=$(id -un) -K ansible/macos/manager-privilege.yml) it no longer updates itself.'
   );
 }