@crvouga/dev-fleet 0.7.0 → 0.8.0

package/src/machine/state.ts

@@ -20,6 +20,12 @@ export interface Enrollment {
   approvedRevision: string;
   appliedRevision?: string;
   previousRevision?: string;
+  /** The revision each worker last converged to, by fleet id. A worker
+   * keeps its progress when the fleet as a whole is not ready.
+   */
+  workers?: Record<string, string>;
+  /** Tombstones: when each retired worker left the fleet, by fleet id. */
+  retired?: Record<string, string>;
 }
 
 export const isRevision = (value: unknown): value is string =>
@@ -156,6 +162,30 @@ export async function ensurePrivateRoot(root: string): Promise<void> {
   }
 }
 
+function validWorkers(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, revision]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) && isRevision(revision)
+      ))
+  );
+}
+
+function validRetired(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, at]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) &&
+          typeof at === 'string' &&
+          !Number.isNaN(Date.parse(at))
+      ))
+  );
+}
+
 /** A valid enrollment record, or null. */
 export function parseEnrollment(data: unknown): Enrollment | null {
   if (!record(data) || data['version'] !== 1) return null;
@@ -167,7 +197,9 @@ export function parseEnrollment(data: unknown): Enrollment | null {
   return isRevision(data['approvedRevision']) &&
     ['appliedRevision', 'previousRevision'].every(
       (key) => data[key] === undefined || isRevision(data[key])
-    )
+    ) &&
+    validWorkers(data['workers']) &&
+    validRetired(data['retired'])
     ? (data as unknown as Enrollment)
     : null;
 }

package/src/manager-sync.ts

@@ -37,6 +37,8 @@ export interface ManagerReport {
   ok: boolean;
   applied: boolean;
   desiredRevision: string | null;
+  /** The worker that proved a revision new to the fleet first, when one did. */
+  canary?: string;
   checks: Check[];
   nodes: ManagerNode[];
   candidates: number;
@@ -196,12 +198,10 @@ async function applyNodes(
   deploy: DeployNode
 ): Promise<void> {
   const workers = report.nodes.filter((node) => node.role !== 'manager');
-  // A revision new to the fleet is proven on one worker first. When the run
-  // selects only later workers, the first selected one proves it instead:
-  // narrowing a run never skips the canary.
   const canary = input.canary
-    ? (workers.find((node) => node.node === input.canary) ?? workers[0])
+    ? chooseCanary(workers, input.canary, input.nodes)
     : undefined;
+  if (canary) report.canary = canary.node;
   let blocked = report.checks.some(blocking);
   const step = async (result: ManagerNode) => {
     const deployable = mayDeploy(result, input.nodes);
@@ -219,6 +219,26 @@ async function applyNodes(
   await Promise.all(workers.filter((node) => node !== canary).map(step));
 }
 
+/** A revision new to the fleet is proven on one worker first: the preferred
+ * one, or when it cannot be reached (powered off, being rebuilt, offline)
+ * the next worker in declared order that can. An unreachable worker is
+ * reported as such and never blocks the others; only a canary that was
+ * deployed and failed stops promotion. When the run selects only later
+ * workers, the first selected one proves it instead: narrowing a run never
+ * skips the canary.
+ */
+function chooseCanary(
+  workers: ManagerNode[],
+  preferred: string,
+  nodes: FleetNode[]
+): ManagerNode | undefined {
+  const first = workers.find((node) => node.node === preferred);
+  const order = first
+    ? [first, ...workers.filter((node) => node !== first)]
+    : workers;
+  return order.find((node) => mayDeploy(node, nodes));
+}
+
 /** One node's rollout step; true when its remote operation ran. */
 async function rollOut(
   result: ManagerNode,

package/src/secrets.ts

@@ -26,6 +26,9 @@ export const OPTIONAL_SECRET_NAMES = [
   // Where fleet-watch notifies (notify.targets).
   'notify-webhook-url',
   'notify-ntfy-url',
+  // A short-lived, single-use, tagged Tailscale auth key a new worker joins
+  // the tailnet with (fleet-node-seed).
+  'tailnet-auth-key',
 ] as const;
 export type SecretName =
   (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];

package/src/worker-credentials.ts

@@ -0,0 +1,201 @@
+import { fleet } from './fleet-config';
+import { resolveVaultConfig, type VaultConfig } from './vault-config';
+import type { Check, Runner } from './types';
+
+/** Each worker's own Vault token is kept in a document of its own, beside
+ * the fleet's shared one: `<project>/workers/<id>`, field `token`, with the
+ * token's `accessor` so the manager can revoke it without holding it. A
+ * policy that grants one worker its document grants it nothing of another's.
+ */
+export interface WorkerTokenLocation {
+  mount: string;
+  path: string;
+  field: 'token';
+}
+
+export function workerTokenLocation(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig()
+): WorkerTokenLocation {
+  return {
+    mount: vault.mount,
+    path: `${vault.project}/workers/${node}`,
+    field: 'token',
+  };
+}
+
+/** The Vault policy each worker token is issued with. */
+export const workerPolicyName = (node: string): string =>
+  `fleet-worker-${node}`;
+
+/** The least a worker needs: the fleet's shared document (the proxy key and
+ * what the providers read) and its own token document. Nothing grants a
+ * worker the documents of the others, so one worker's token cannot read
+ * another's. Token renewal and lookup come from Vault's default policy.
+ */
+export function workerVaultPolicy(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig(),
+  shared: string = sharedDocument()
+): string {
+  const own = workerTokenLocation(node, vault);
+  return [
+    `# dev-fleet: the policy of the worker ${node}. Written by dev-fleet; do not edit.`,
+    `path "${vault.mount}/data/${shared}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    `path "${own.mount}/data/${own.path}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    '',
+  ].join('\n');
+}
+
+function sharedDocument(): string {
+  const secrets = fleet.secrets;
+  return secrets.provider === 'vault'
+    ? secrets.locations['proxy-client-key'].path
+    : '';
+}
+
+const TIMEOUT = { timeoutMs: 20_000 };
+
+/** Issues a worker its own periodic, orphan token under its own policy and
+ * keeps it in its own document. The token reaches `vault kv put` on standard
+ * input, never as an argument. Run on the manager, logged in to Vault.
+ */
+export async function issueWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const policy = await run(
+    ['vault', 'policy', 'write', address, workerPolicyName(node), '-'],
+    { ...TIMEOUT, input: workerVaultPolicy(node, vault) }
+  );
+  if (!policy.ok) return credentialFailure('VAULT_WORKER_POLICY_FAILED');
+  const created = await run(
+    [
+      'vault',
+      'token',
+      'create',
+      address,
+      `-policy=${workerPolicyName(node)}`,
+      '-orphan',
+      '-period=768h',
+      `-display-name=${workerPolicyName(node)}`,
+      '-format=json',
+    ],
+    TIMEOUT
+  );
+  const issued = created.ok ? parseIssued(created.stdout) : null;
+  if (!issued) return credentialFailure('VAULT_WORKER_TOKEN_NOT_ISSUED');
+  const location = workerTokenLocation(node, vault);
+  const stored = await run(
+    [
+      'vault',
+      'kv',
+      'put',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+      'token=-',
+      `accessor=${issued.accessor}`,
+    ],
+    { ...TIMEOUT, input: issued.token }
+  );
+  if (!stored.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_STORED');
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node} has a token of its own, readable by nothing but itself and the manager.`,
+  };
+}
+
+function parseIssued(
+  stdout: string
+): { token: string; accessor: string } | null {
+  try {
+    const auth = (JSON.parse(stdout) as { auth?: Record<string, unknown> })
+      .auth;
+    const token = auth?.['client_token'];
+    const accessor = auth?.['accessor'];
+    return typeof token === 'string' &&
+      typeof accessor === 'string' &&
+      /^[\x21-\x7e]{16,4096}$/.test(token) &&
+      /^[A-Za-z0-9._-]{8,256}$/.test(accessor)
+      ? { token, accessor }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Revokes a worker's token by its accessor (the manager never holds the
+ * token to do so), then removes its document and its policy. A worker that
+ * has none already is revoked: running it twice changes nothing.
+ */
+export async function revokeWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const location = workerTokenLocation(node, vault);
+  const read = await run(
+    [
+      'vault',
+      'kv',
+      'get',
+      address,
+      `-mount=${location.mount}`,
+      '-field=accessor',
+      location.path,
+    ],
+    TIMEOUT
+  );
+  const accessor = read.ok ? read.stdout.trim() : '';
+  if (accessor) {
+    if (!/^[A-Za-z0-9._-]{8,256}$/.test(accessor))
+      return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+    const revoked = await run(
+      ['vault', 'token', 'revoke', address, '-accessor', accessor],
+      TIMEOUT
+    );
+    if (!revoked.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+  }
+  await run(
+    [
+      'vault',
+      'kv',
+      'metadata',
+      'delete',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+    ],
+    TIMEOUT
+  );
+  await run(
+    ['vault', 'policy', 'delete', address, workerPolicyName(node)],
+    TIMEOUT
+  );
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node}'s token is revoked and its document and policy removed.`,
+  };
+}
+
+function credentialFailure(summary: string): Check {
+  return {
+    id: 'auth:worker-token',
+    status: 'fail',
+    summary,
+    remedy: {
+      action:
+        'Log the manager in to Vault with a token that may write policies, issue tokens and write the fleet mount, then run the command again.',
+    },
+  };
+}

package/src/worker-manifest.ts

@@ -0,0 +1,201 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { PACKAGE_ROOT } from './package-info';
+import { shellQuote, sshArgs, sshFailure } from './ssh';
+import type { Check, FleetNode, Runner } from './types';
+
+/** What a worker is made of, measured: every installed package and its
+ * version, the tools mise has active, and whether each always-on service is
+ * enabled and running. Two workers built from the same revision have the same
+ * manifest, so its digest proves a rebuild reproduces a machine, and its
+ * difference from a fresh build shows what a long-lived worker gained.
+ */
+export interface WorkerManifest {
+  version: 1;
+  packages: Record<string, string>;
+  tools: Record<string, string>;
+  units: Record<string, string>;
+}
+
+/** The policy a worker's manifest is measured against. */
+interface UbuntuPolicy {
+  packages: Record<string, string>;
+  alwaysOnServices: string[];
+}
+
+export async function ubuntuPolicy(root = PACKAGE_ROOT): Promise<UbuntuPolicy> {
+  return JSON.parse(
+    await readFile(join(root, 'ansible/ubuntu/policy.json'), 'utf8')
+  ) as UbuntuPolicy;
+}
+
+/** A read-only POSIX probe: it lists, and changes nothing. */
+export function manifestProbe(services: string[]): string {
+  const units = services.map(shellQuote).join(' ');
+  return [
+    'export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH"',
+    'echo @packages',
+    "dpkg-query -W -f='${binary:Package}=${Version}\\n' 2>/dev/null",
+    'echo @tools',
+    "mise ls --current --json 2>/dev/null | tr -d '\\n'; echo",
+    'echo @units',
+    `for unit in ${units}; do printf '%s=%s/%s\\n' "$unit" "$(systemctl is-enabled "$unit" 2>/dev/null)" "$(systemctl is-active "$unit" 2>/dev/null)"; done`,
+  ].join('; ');
+}
+
+const sorted = (entries: [string, string][]): Record<string, string> =>
+  Object.fromEntries(entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+
+function pairs(lines: string[]): [string, string][] {
+  return lines
+    .map((line) => line.trim())
+    .filter((line) => line.includes('='))
+    .map((line) => {
+      const at = line.indexOf('=');
+      return [line.slice(0, at), line.slice(at + 1)] as [string, string];
+    });
+}
+
+function tools(line: string | undefined): Record<string, string> {
+  try {
+    const listed = JSON.parse(line ?? '{}') as Record<
+      string,
+      { version?: string }[]
+    >;
+    return sorted(
+      Object.entries(listed).map(([tool, versions]) => [
+        tool,
+        versions
+          .map((entry) => entry.version ?? '')
+          .sort()
+          .join(','),
+      ])
+    );
+  } catch {
+    return {};
+  }
+}
+
+/** The manifest the probe printed. Order never matters: everything is sorted. */
+export function parseManifest(stdout: string): WorkerManifest {
+  const sections: Record<string, string[]> = {};
+  let current = '';
+  for (const line of stdout.split('\n')) {
+    if (/^@[a-z]+$/.test(line.trim())) {
+      current = line.trim().slice(1);
+      sections[current] = [];
+    } else if (current) sections[current]!.push(line);
+  }
+  return {
+    version: 1,
+    packages: sorted(pairs(sections['packages'] ?? [])),
+    tools: tools(sections['tools']?.find((line) => line.trim())),
+    units: sorted(pairs(sections['units'] ?? [])),
+  };
+}
+
+/** The same manifest always has the same digest. */
+export function manifestDigest(manifest: WorkerManifest): string {
+  return createHash('sha256').update(JSON.stringify(manifest)).digest('hex');
+}
+
+export interface ManifestDifference {
+  section: 'packages' | 'tools' | 'units';
+  name: string;
+  expected: string | null;
+  actual: string | null;
+}
+
+/** What `live` has that `reference` does not, lacks, or has otherwise. */
+export function diffManifests(
+  reference: WorkerManifest,
+  live: WorkerManifest
+): ManifestDifference[] {
+  const differences: ManifestDifference[] = [];
+  for (const section of ['packages', 'tools', 'units'] as const) {
+    const names = new Set([
+      ...Object.keys(reference[section]),
+      ...Object.keys(live[section]),
+    ]);
+    for (const name of [...names].sort()) {
+      const expected = reference[section][name] ?? null;
+      const actual = live[section][name] ?? null;
+      if (expected !== actual)
+        differences.push({ section, name, expected, actual });
+    }
+  }
+  return differences;
+}
+
+/** Where a manifest departs from what the policy declares: a pinned package
+ * at another version or missing, an always-on service not enabled and active.
+ */
+export function declaredDifferences(
+  manifest: WorkerManifest,
+  policy: UbuntuPolicy
+): ManifestDifference[] {
+  const differences: ManifestDifference[] = [];
+  for (const [name, version] of Object.entries(policy.packages)) {
+    const actual = manifest.packages[name] ?? null;
+    if (actual !== version)
+      differences.push({
+        section: 'packages',
+        name,
+        expected: version,
+        actual,
+      });
+  }
+  for (const unit of policy.alwaysOnServices) {
+    const actual = manifest.units[unit] ?? null;
+    if (actual !== 'enabled/active')
+      differences.push({
+        section: 'units',
+        name: unit,
+        expected: 'enabled/active',
+        actual,
+      });
+  }
+  return differences;
+}
+
+/** `worker:manifest`: the worker is what a fresh build of the same revision
+ * is (when a reference is given) and what the policy declares.
+ */
+export function manifestCheck(differences: ManifestDifference[]): Check {
+  if (!differences.length)
+    return {
+      id: 'worker:manifest',
+      status: 'pass',
+      summary: 'The worker matches the declared pins and the fresh build.',
+    };
+  const shown = differences
+    .slice(0, 20)
+    .map(
+      (difference) =>
+        `${difference.section}:${difference.name} ${difference.expected ?? 'absent'} -> ${difference.actual ?? 'absent'}`
+    );
+  return {
+    id: 'worker:manifest',
+    status: 'fail',
+    summary: `WORKER_MANIFEST_DRIFT: ${differences.length} difference(s): ${shown.join('; ')}`,
+    remedy: {
+      action:
+        'Something was installed, removed or changed outside fleet-sync. Rebuild the worker (fleet-node-replace) or declare the change in dev-fleet.',
+    },
+  };
+}
+
+/** Measures one worker's manifest over the verified SSH connection. */
+export async function measureManifest(
+  node: FleetNode,
+  run: Runner,
+  policy: UbuntuPolicy
+): Promise<{ manifest: WorkerManifest } | { check: Check }> {
+  const result = await run(
+    [...sshArgs(node), manifestProbe(policy.alwaysOnServices)],
+    { timeoutMs: 60_000 }
+  );
+  if (!result.ok) return { check: sshFailure(result) };
+  return { manifest: parseManifest(result.stdout) };
+}

package/src/worker-policy.ts

@@ -34,33 +34,27 @@ function validCapabilities(value: unknown): boolean {
     REQUIRED.every((item) => value.includes(item))
   );
 }
-function validPolicy(value: Record<string, unknown>): boolean {
+const FIELDS: Record<string, (value: unknown) => boolean> = {
+  requiredCapabilities: validCapabilities,
+  githubLogin: (value) =>
+    typeof value === 'string' && /^[a-zA-Z0-9-]{1,39}$/.test(value),
+  vaultUser: (value) =>
+    typeof value === 'string' && /^[a-zA-Z0-9_.-]{1,64}$/.test(value),
+  toolchain: (value) => value === 'mise.lock',
+  agentProfile: (value) => value === 'proxy',
+  unattendedSudo: (value) => value === true,
+  persistentT3: (value) => value === true,
+  neverSleep: (value) => value === true,
+  toolProfiles: (value) => value === undefined || toolProfiles(value) !== null,
+  tools: (value) => value === undefined || toolNames(value),
+};
+/** The first field of a worker policy that breaks its rule, or null. */
+export function policyFault(value: Record<string, unknown>): string | null {
+  const unknown = Object.keys(value).find((key) => !(key in FIELDS));
+  if (unknown !== undefined) return unknown;
   return (
-    Object.keys(value).every((key) =>
-      [
-        'requiredCapabilities',
-        'githubLogin',
-        'vaultUser',
-        'toolchain',
-        'agentProfile',
-        'unattendedSudo',
-        'persistentT3',
-        'neverSleep',
-        'toolProfiles',
-        'tools',
-      ].includes(key)
-    ) &&
-    validTools(value) &&
-    validCapabilities(value['requiredCapabilities']) &&
-    typeof value['githubLogin'] === 'string' &&
-    /^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
-    typeof value['vaultUser'] === 'string' &&
-    /^[a-zA-Z0-9_.-]{1,64}$/.test(value['vaultUser']) &&
-    value['toolchain'] === 'mise.lock' &&
-    value['agentProfile'] === 'proxy' &&
-    ['unattendedSudo', 'persistentT3', 'neverSleep'].every(
-      (key) => value[key] === true
-    )
+    Object.entries(FIELDS).find(([key, valid]) => !valid(value[key]))?.[0] ??
+    null
   );
 }
 /** A list of known tool profiles, each once; null when it is not one. */
@@ -71,14 +65,6 @@ export function toolProfiles(value: unknown): ToolProfile[] | null {
     ? (value as ToolProfile[])
     : null;
 }
-/** The optional tool profiles and single tools every worker adds. */
-function validTools(value: Record<string, unknown>): boolean {
-  return (
-    (value['toolProfiles'] === undefined ||
-      toolProfiles(value['toolProfiles']) !== null) &&
-    (value['tools'] === undefined || toolNames(value['tools']))
-  );
-}
 function toolNames(value: unknown): boolean {
   return (
     Array.isArray(value) &&
@@ -89,7 +75,7 @@ function toolNames(value: unknown): boolean {
   );
 }
 export function parseWorkerPolicy(value: unknown): WorkerPolicy {
-  if (!record(value) || !validPolicy(value))
+  if (!record(value) || policyFault(value) !== null)
     throw new FleetUsageError(
       'Invalid mandatory worker policy. Workers require the locked toolchain, common services, unattended sudo, persistent T3 and no sleep.'
     );