@crvouga/dev-fleet 0.7.0 → 0.9.0

package/schema/worker-record.schema.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "dev-fleet worker records, version 1",
+  "description": "Workers as the tool that created them describes them, without naming the tool or the provider. fleet.json names a file or a command that returns this document under workerRecords; dev-fleet only reads it, never creates a machine and never calls a provider. Each record is a declared worker reached from its data: its host key is pinned as given. A record needs a tailnetName or an address. A worker whose record disappears (a file that is gone holds no record) is retired by the next fleet-sync.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["version", "workers"],
+  "properties": {
+    "$schema": { "type": "string" },
+    "version": { "const": 1 },
+    "workers": {
+      "type": "array",
+      "maxItems": 64,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["id", "user"],
+        "properties": {
+          "id": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9-]{0,62}$",
+            "description": "The worker's fleet ID. No other worker, inline or from a record, may have it."
+          },
+          "tailnetName": {
+            "type": "string",
+            "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$",
+            "description": "The tailnet name the machine joins under, without the trailing dot."
+          },
+          "address": {
+            "type": "string",
+            "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$",
+            "description": "A name or address SSH reaches the machine at, when it has no tailnet name."
+          },
+          "user": {
+            "type": "string",
+            "pattern": "^[a-z_][a-z0-9_-]{0,31}$",
+            "description": "The non-root account with passwordless sudo the manager signs in as."
+          },
+          "port": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535,
+            "description": "The port sshd listens on. Default 22."
+          },
+          "hostKey": {
+            "type": "string",
+            "pattern": "^(ssh-ed25519|ecdsa-sha2-nistp256|ssh-rsa) [A-Za-z0-9+/=]{16,}$",
+            "description": "The machine's SSH host public key, \"TYPE BASE64\": generated before the machine exists (devfleet fleet-node-seed prints it) or read from the machine by the tool that created it. It is pinned, never learned on first use. Without it the key the fleet state pins is used (devfleet fleet-node-seed --apply --yes), and until there is one every session is refused."
+          },
+          "t3Port": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535,
+            "description": "The port the worker's T3 server listens on. Default 3773."
+          },
+          "labels": {
+            "type": "object",
+            "additionalProperties": { "type": "string" },
+            "description": "Free-form labels; only displayed."
+          },
+          "billing": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["hourlyRate"],
+            "properties": {
+              "hourlyRate": { "type": "number", "minimum": 0 },
+              "currency": { "type": "string", "pattern": "^[A-Z]{3}$" },
+              "note": { "type": "string" }
+            },
+            "description": "What the machine costs while it runs, as you declare it; fleet-status sums it for idle workers."
+          }
+        }
+      }
+    }
+  }
+}

package/src/agents/converge.ts

@@ -1,15 +1,14 @@
 import { AgentConfigurationError } from './errors';
 import { mkdir, rm } from 'node:fs/promises';
 import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { digest, registry, type Selection } from './registry';
 import { render } from './render';
-import {
-  atomicWrite,
-  backup,
-  fileState,
-  readOwned,
-  safeParents,
-} from './files';
+import { readOwned, fileState, safeParents } from './files';
+import { runAnsible } from '../platform/ansible';
+import { PACKAGE_ROOT } from '../package-info';
+import { runCommand } from '../runner';
+import type { CommandResult, Runner } from '../types';
 
 export type Receipt = {
   version: 1;
@@ -19,6 +18,58 @@ export type Receipt = {
 };
 export type Context = { root: string; revision: string; selection: Selection };
 
+/** What `ansible/agents/agents.yml` is given to write one profile. TypeScript
+ * decides here, beforehand, what is backed up and what each file must hash
+ * to; the play renders the files from its templates and asserts them.
+ */
+export interface ProfileVariables {
+  agent_root: string;
+  agent_node: string;
+  agent_tool: string;
+  agent_profile: string;
+  agent_model: string;
+  /** Empty for a direct profile: it never names the proxy. */
+  agent_proxy_endpoint: string;
+  /** The model of every proxy route, in the registry's order. */
+  agent_proxy_models: string[];
+  /** The SHA-256 of each file as src/agents/render.ts specifies it. */
+  agent_files: Record<string, string>;
+  /** What the receipt records beside those digests. */
+  agent_revision: string;
+  agent_fingerprint: string;
+  /** Files to copy into ROOT/backups before they are replaced. */
+  agent_backups: { source: string; dest: string }[];
+}
+
+/** Runs the agents play with these variables. */
+export type ProfilePlay = (
+  variables: ProfileVariables
+) => Promise<CommandResult>;
+
+/** The package's agents play, with the Ansible this machine runs. `scratch`
+ * is where its variables file is kept while it runs (the fleet's private
+ * run directory by default).
+ */
+export function agentsPlay(
+  run: Runner = runCommand,
+  options: { check?: boolean; scratch?: string } = {}
+): ProfilePlay {
+  const directory = join(PACKAGE_ROOT, 'ansible', 'agents');
+  return (variables) =>
+    runAnsible(
+      run,
+      join(directory, 'agents.yml'),
+      {
+        cwd: directory,
+        input: JSON.stringify(variables),
+        // Inside the two minutes the sync gives `agents apply`.
+        timeoutMs: 90_000,
+        ...(options.check ? { check: true } : {}),
+      },
+      options.scratch
+    );
+}
+
 export async function plan(context: Context) {
   const { root, selection } = context;
   const directory = join(
@@ -46,7 +97,16 @@ export async function plan(context: Context) {
   return { directory, receiptPath, previous, files, fingerprint };
 }
 
-export async function convergeAgents(context: Context, adopt = false) {
+/** Applies one profile. A file someone edited is refused unless `adopt`;
+ * then the agents play writes the profile, and what it wrote is measured
+ * against the specification before this reports success. With nothing to
+ * change, the play is not run.
+ */
+export async function convergeAgents(
+  context: Context,
+  adopt = false,
+  play: ProfilePlay = agentsPlay()
+) {
   await safeParents(context.root);
   await mkdir(context.root, { recursive: true, mode: 0o700 });
   const lock = join(context.root, '.apply-lock');
@@ -63,23 +123,25 @@ export async function convergeAgents(context: Context, adopt = false) {
       throw new AgentConfigurationError(
         'Unmanaged or modified configuration. Review locally, then use --adopt to back up and replace only this profile.'
       );
-    for (const file of result.files) {
-      if (file.state === 'unchanged') continue;
-      if ((await readOwned(file.path)) !== null)
-        await backup(file.path, context.root);
-      await atomicWrite(file.path, file.content);
+    const { receipt, encoded } = receiptFor(result, context.revision);
+    const changing = result.files.filter((file) => file.state !== 'unchanged');
+    if (changing.length || (await readOwned(result.receiptPath)) !== encoded) {
+      const applied = await play(
+        profileVariables(context, receipt, {
+          backups: changing
+            .filter((file) => file.state !== 'create')
+            .map((file) => ({
+              source: file.path,
+              dest: join(context.root, 'backups', randomUUID()),
+            })),
+        })
+      );
+      if (!applied.ok)
+        throw new AgentConfigurationError(
+          'The agents play (ansible/agents/agents.yml) failed for this profile; no receipt was advanced and its output is not reported. Inspect the profile with agents preview, then apply again.'
+        );
+      await measure(context, encoded);
     }
-    const receipt: Receipt = {
-      version: 1,
-      revision: context.revision,
-      fingerprint: result.fingerprint,
-      files: Object.fromEntries(
-        result.files.map((file) => [file.name, digest(file.content)])
-      ),
-    };
-    const encoded = JSON.stringify(receipt, null, 2) + '\n';
-    if ((await readOwned(result.receiptPath)) !== encoded)
-      await atomicWrite(result.receiptPath, encoded);
     return {
       ...publicPlan(result, context.revision),
       appliedRevision: context.revision,
@@ -90,6 +152,60 @@ export async function convergeAgents(context: Context, adopt = false) {
   }
 }
 
+/** The receipt an apply of this plan records, and its exact text. */
+export function receiptFor(
+  result: Awaited<ReturnType<typeof plan>>,
+  revision: string
+): { receipt: Receipt; encoded: string } {
+  const receipt: Receipt = {
+    version: 1,
+    revision,
+    fingerprint: result.fingerprint,
+    files: Object.fromEntries(
+      result.files.map((file) => [file.name, digest(file.content)])
+    ),
+  };
+  return { receipt, encoded: JSON.stringify(receipt, null, 2) + '\n' };
+}
+
+export function profileVariables(
+  context: Context,
+  receipt: Receipt,
+  options: { backups: { source: string; dest: string }[] } = { backups: [] }
+): ProfileVariables {
+  const { selection } = context;
+  return {
+    agent_root: context.root,
+    agent_node: selection.node,
+    agent_tool: selection.tool,
+    agent_profile: selection.profile,
+    agent_model: selection.model,
+    agent_proxy_endpoint:
+      selection.profile === 'proxy' ? registry.proxy.endpoint : '',
+    agent_proxy_models: Object.values(registry.proxy.routes).map(
+      (route) => route.model
+    ),
+    agent_files: receipt.files,
+    agent_revision: receipt.revision,
+    agent_fingerprint: receipt.fingerprint,
+    agent_backups: options.backups,
+  };
+}
+
+/** After the play: every file is what was specified, byte for byte, and the
+ * receipt is the one this apply decided on.
+ */
+async function measure(context: Context, encoded: string): Promise<void> {
+  const after = await plan(context);
+  if (
+    after.files.some((file) => file.state !== 'unchanged') ||
+    (await readOwned(after.receiptPath)) !== encoded
+  )
+    throw new AgentConfigurationError(
+      'After the agents play, the profile on disk is not what dev-fleet specifies. Inspect it with agents preview before applying again.'
+    );
+}
+
 export function publicPlan(
   result: Awaited<ReturnType<typeof plan>>,
   revision: string

package/src/agents/files.ts

@@ -1,16 +1,6 @@
 import { AgentConfigurationError } from './errors';
-import {
-  lstat,
-  mkdir,
-  readFile,
-  rename,
-  rm,
-  writeFile,
-  copyFile,
-  chmod,
-} from 'node:fs/promises';
-import { dirname, join, parse } from 'node:path';
-import { randomUUID } from 'node:crypto';
+import { lstat, readFile } from 'node:fs/promises';
+import { dirname, parse } from 'node:path';
 import { digest } from './registry';
 
 export async function safeParents(path: string): Promise<void> {
@@ -42,26 +32,6 @@ export async function readOwned(path: string): Promise<string | null> {
   }
 }
 
-export async function atomicWrite(path: string, value: string): Promise<void> {
-  await safeParents(path);
-  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
-  const temp = `${path}.${randomUUID()}.tmp`;
-  try {
-    await writeFile(temp, value, { flag: 'wx', mode: 0o600 });
-    await rename(temp, path);
-  } finally {
-    await rm(temp, { force: true });
-  }
-}
-
-export async function backup(path: string, root: string): Promise<void> {
-  const dir = join(root, 'backups');
-  await mkdir(dir, { recursive: true, mode: 0o700 });
-  const target = join(dir, randomUUID());
-  await copyFile(path, target);
-  await chmod(target, 0o600);
-}
-
 export function fileState(
   current: string | null,
   desired: string,

package/src/agents/native-verify.ts

@@ -1,6 +1,9 @@
 import { mkdtemp, readFile, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
+import { homedir, tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { currentPlatform } from '../platform';
+import { agentRoot, render } from './render';
+import { select } from './registry';
 import type { Check, Runner } from '../types';
 import { pinnedPath } from './toolchain';
 import { registry } from './registry';
@@ -110,9 +113,13 @@ export async function verifyNativeAgents(
      * account change or key rotation re-verifies at once.
      */
     accounts?: AccountContext;
+    /** Where the agent profiles are rendered; this user's by default. */
+    profiles?: string;
   } = {}
 ): Promise<Check[]> {
   const { outOfQuota = new Set<string>(), cache = {} } = options;
+  const profiles =
+    options.profiles ?? agentRoot(homedir(), currentPlatform().name);
   const pins = parsePins(await readFile(join(workspace, 'mise.toml'), 'utf8'));
   const environment = options.t3Service
     ? await serviceLaunch(run)
@@ -136,7 +143,8 @@ export async function verifyNativeAgents(
                   registry,
                   await sourceKey(launcherSources(workspace, agent)),
                   options.accounts?.rosterVersion ?? '',
-                  options.accounts?.keyVersion ?? ''
+                  options.accounts?.keyVersion ?? '',
+                  await configurationKey(profiles, node, agent.binary)
                 ),
                 () =>
                   sessionCheck(agent, { node, workspace, cwd, launch }, run),
@@ -157,6 +165,7 @@ export async function verifyNativeAgents(
                 launch,
                 cache,
                 run,
+                profiles,
               })
           )
         : [];
@@ -207,6 +216,7 @@ async function accountSession(
     launch: Launch;
     cache: VerifyCache;
     run: Runner;
+    profiles: string;
   }
 ): Promise<'ok' | 'cooling-down' | 'failed'> {
   const spec = ACCOUNT_LAUNCH[instance.tool];
@@ -218,7 +228,8 @@ async function accountSession(
       instance,
       spec,
       accounts.rosterVersion,
-      accounts.keyVersion
+      accounts.keyVersion,
+      await configurationKey(context.profiles, context.node, instance.tool)
     ),
     async () => {
       const agents = join(context.workspace, 'agents');
@@ -484,6 +495,26 @@ function sessionRemedy(agent: Agent): string {
 }
 
 /** The launcher and profile code a session depends on. */
+/** Digest of the agent's proxy profile as it is on this machine: the files
+ * its session reads. A profile rendered again, edited or removed changes it,
+ * so the session is proven again at once. Empty for a tool with no profile.
+ */
+export async function configurationKey(
+  profiles: string,
+  node: string,
+  tool: string
+): Promise<string> {
+  try {
+    const selection = select(node, tool, 'proxy');
+    const directory = join(profiles, node, tool, 'proxy');
+    return await sourceKey(
+      Object.keys(render(selection)).map((name) => join(directory, name))
+    );
+  } catch {
+    return '';
+  }
+}
+
 function launcherSources(workspace: string, agent: Agent): string[] {
   const agents = join(workspace, 'agents');
   const lib = join(workspace, 'src/agents');

package/src/agents/secrets.ts

@@ -1,13 +1,48 @@
+import { chmod, mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
 import { AgentConfigurationError } from './errors';
 import { secret } from '../fleet-config';
 
+/** Where a node keeps the last proxy client key its secrets provider gave it,
+ * readable by its user alone. Disposable: the next successful read rewrites it.
+ */
+export const lastProxyKeyPath = (home = homedir()): string =>
+  join(home, '.cache/dev-fleet/proxy-client-key');
+
 /** The key every node presents to the CLI proxy, from this fleet's secrets
- * provider with this node's own access.
+ * provider with this node's own access. The provider stays the source: a key
+ * rotated there is used at the next launch. When the provider cannot answer
+ * (Vault unreachable, or this node's session expired) the last key it gave is
+ * used instead, so an outage of the secrets provider never takes every agent
+ * down with it; the node's own Vault checks still fail until it is repaired.
  */
-export async function proxyKey(): Promise<string> {
-  const value = await secret('proxy-client-key');
-  if (value) return value;
+export async function proxyKey(
+  options: {
+    read?: () => Promise<string | null>;
+    path?: string;
+  } = {}
+): Promise<string> {
+  const read = options.read ?? (() => secret('proxy-client-key'));
+  const path = options.path ?? lastProxyKeyPath();
+  const value = await read().catch(() => null);
+  if (value) {
+    await remember(path, value).catch(() => undefined);
+    return value;
+  }
+  const last = (await readFile(path, 'utf8').catch(() => '')).trim();
+  if (last) return last;
   throw new AgentConfigurationError(
     "Required secret unavailable: proxy-client-key. Restore this node's access to the fleet's secrets provider (see `secrets` in the fleet configuration)."
   );
 }
+
+async function remember(path: string, value: string): Promise<void> {
+  if ((await readFile(path, 'utf8').catch(() => '')).trim() === value) return;
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  await chmod(dirname(path), 0o700);
+  // Written aside and renamed, so a launch never reads half a key.
+  const staged = `${path}.${process.pid}`;
+  await writeFile(staged, `${value}\n`, { mode: 0o600 });
+  await rename(staged, path);
+}

package/src/check-owners.ts

@@ -19,6 +19,9 @@ const DECLARATION =
 const HARDWARE =
   'It is a property of the hardware, the firmware or the operating system release, which no command changes.';
 
+const PARKING =
+  'Whether a worker runs is the operator’s choice: park it with fleet-node-park or resume it with fleet-node-resume.';
+
 export const CHECK_OWNERS: CheckOwner[] = [
   // Toolchain and workstation: automatic.
   { pattern: 'tool:*', repair: 'workspace' },
@@ -68,12 +71,46 @@ export const CHECK_OWNERS: CheckOwner[] = [
       'Re-converge the worker with fleet-sync: it applies the whole worker policy at the approved revision.',
   },
   { pattern: 'worker:power-loss-recovery', manual: HARDWARE },
+  {
+    pattern: 'worker:account',
+    manual:
+      'fleet-sync creates the declared account over a root login; when it cannot, the account is created by the operator on the machine, or declared in fleet.json.',
+  },
+  {
+    pattern: 'worker:reboot',
+    manual:
+      'A reboot ends running agent turns, so it waits for them: rerun fleet-sync --maintenance-reboot once they end, or with --force.',
+  },
+  {
+    pattern: 'tailscale:enrolment-key',
+    manual:
+      'The tailnet-admin credential and the tag it may use are the operator’s, kept in the secrets provider and the control plane.',
+  },
+  {
+    pattern: 'tailscale:key-expiry-policy',
+    manual:
+      'The tailnet-admin credential and the tag it may use are the operator’s, kept in the secrets provider and the control plane.',
+  },
+  {
+    pattern: 'tailscale:device',
+    manual:
+      'Removing a device needs the tailnet-admin credential and a declared control plane, both the operator’s; without them a person removes it in the admin console.',
+  },
+  {
+    pattern: 'tailscale:leftover',
+    manual:
+      'Whether a device tagged as a worker belongs to the fleet is the operator’s choice: declare it, or remove it from the tailnet.',
+  },
+  // Whether a worker runs is the operator's choice (fleet-node-park).
+  { pattern: 'worker:idle', manual: PARKING },
+  { pattern: 'worker:park*', manual: PARKING },
+  { pattern: 'worker:auto-park', manual: PARKING },
   { pattern: 'worker:github-principal', manual: IDENTITY },
   { pattern: 'worker:vault-renewal', manual: IDENTITY },
   {
     pattern: 'manager:*',
     manual:
-      'The manager converges itself in fleet-sync; a setting that needs an administrator is applied with ansible-playbook -K ansible/macos/manager.yml.',
+      'The manager converges itself in fleet-sync; what needs an administrator is granted once with ansible-playbook -e user=$(id -un) -K ansible/macos/manager-privilege.yml.',
   },
   { pattern: 'manager:identity', manual: DECLARATION },
   {
@@ -104,7 +141,7 @@ export const CHECK_OWNERS: CheckOwner[] = [
   {
     pattern: 't3:*',
     manual:
-      'T3 pairing and its private route are approved by a person; fleet-sync applies the rest.',
+      'A worker’s first T3 pairing and its private route are approved by a person; a replaced worker is paired by fleet-node-replace or the next fleet-sync, and fleet-sync applies the rest.',
   },
   { pattern: 'vscode:*', manual: IDENTITY },
   {
@@ -130,6 +167,7 @@ export const CHECK_OWNERS: CheckOwner[] = [
   { pattern: 'github:rate-limit', manual: EVIDENCE },
   { pattern: 'github:workflow', manual: EVIDENCE },
   { pattern: 'fleet:*', manual: DECLARATION },
+  { pattern: 'definition:*', manual: DECLARATION },
   { pattern: 'enrollment:*', manual: DECLARATION },
   { pattern: 'notify:*', manual: DECLARATION },
   {
@@ -143,11 +181,33 @@ export const CHECK_OWNERS: CheckOwner[] = [
       'The roster reaches every node with devfleet fleet-sync --only accounts, run from the manager.',
   },
   { pattern: 'accounts:native-login', manual: IDENTITY },
+  { pattern: 'accounts:verification', manual: EVIDENCE },
   { pattern: 'configuration:*', manual: EVIDENCE },
   { pattern: 'source:*', manual: EVIDENCE },
   { pattern: 'bootstrap:*', manual: EVIDENCE },
   { pattern: 'rollout:*', manual: EVIDENCE },
   { pattern: 'repair:*', manual: EVIDENCE },
+  // A roster computer's checkout compared with the manager's.
+  {
+    pattern: 'revision',
+    manual:
+      'The checkout of that computer differs from the manager’s: reconcile its branch, or converge it with fleet-sync; dev-fleet never resets a working tree.',
+  },
+  {
+    pattern: 'sync:*',
+    manual:
+      'The checkout of that computer differs from the manager’s: reconcile its lockfiles and pins, or converge it with fleet-sync; dev-fleet never resets a working tree.',
+  },
+  {
+    pattern: 'restore:*',
+    manual:
+      'fleet-restore rebuilds a replacement manager from the bucket; what it needs first (the project’s fleet.json, the bucket’s credentials, Git, Docker Desktop, a tailnet sign-in) is the operator’s: follow the remedy, then run fleet-restore again.',
+  },
+  {
+    pattern: 'watch:*',
+    manual:
+      'The heartbeat is kept in the bucket fleet.json names: name it, or restore access to it, then run fleet-watch again.',
+  },
 ];
 
 const matches = (pattern: string, id: string): boolean =>