@crvouga/dev-fleet 0.8.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +65 -9
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +55 -9
  14. package/ansible/macos/manager.yml +154 -7
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/monitoring/agent-darwin.yml +3 -0
  19. package/ansible/monitoring/agent.yml +2 -0
  20. package/ansible/monitoring/hub-darwin.yml +3 -0
  21. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  22. package/ansible/net/remote-probe.sh +4 -1
  23. package/ansible/platforms.json +2 -2
  24. package/ansible/pyproject.toml +5 -0
  25. package/ansible/ubuntu/baseline.yml +39 -13
  26. package/ansible/ubuntu/disk-gc.py +6 -2
  27. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  28. package/ansible/ubuntu/policy.json +125 -6
  29. package/ansible/ubuntu/reboot.yml +89 -0
  30. package/ansible/ubuntu/recovery.yml +13 -3
  31. package/ansible/ubuntu/t3/policy.json +3 -0
  32. package/ansible/ubuntu/t3-desktop.yml +23 -2
  33. package/ansible/ubuntu/t3-headless.yml +128 -0
  34. package/ansible/ubuntu/t3-health.sh +24 -10
  35. package/ansible/ubuntu/t3-idle.py +37 -2
  36. package/ansible/ubuntu/t3-policy.yml +4 -1
  37. package/ansible/ubuntu/t3-route.yml +1 -0
  38. package/ansible/ubuntu/tailscale.yml +54 -8
  39. package/ansible/ubuntu/tasks/browser.yml +3 -0
  40. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  41. package/ansible/ubuntu/tasks/docker.yml +53 -6
  42. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  43. package/ansible/ubuntu/tasks/features.yml +22 -0
  44. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  45. package/ansible/ubuntu/tasks/install.yml +2 -1
  46. package/ansible/ubuntu/tasks/packages.yml +4 -2
  47. package/ansible/ubuntu/tasks/release.yml +6 -0
  48. package/ansible/ubuntu/tasks/resources.yml +6 -4
  49. package/ansible/ubuntu/tasks/services.yml +2 -2
  50. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  51. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  52. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  53. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  54. package/ansible/ubuntu/tasks/user.yml +1 -0
  55. package/ansible/ubuntu/worker-account.yml +117 -0
  56. package/ansible/uv.lock +265 -0
  57. package/bin/devfleet.mjs +42 -4
  58. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  59. package/bootstrap/global-config.ts +29 -3
  60. package/bootstrap/package-smoke.ts +127 -0
  61. package/bootstrap/pins.ts +189 -0
  62. package/bootstrap/release-notes.ts +88 -0
  63. package/bootstrap/release-version.ts +41 -7
  64. package/bootstrap/worker-features.ts +114 -0
  65. package/bootstrap/worker-seed.yaml +7 -3
  66. package/bootstrap/worker-tools.ts +39 -9
  67. package/cliproxy/README.md +1 -1
  68. package/cliproxy/docs/health-workflow.example.yml +20 -143
  69. package/docs/authentication.md +19 -11
  70. package/docs/consumer-setup.md +42 -27
  71. package/docs/convergence.md +67 -0
  72. package/docs/declared.md +61 -34
  73. package/docs/machine-setup.md +73 -7
  74. package/docs/manager-setup.md +10 -3
  75. package/docs/operations.md +64 -9
  76. package/docs/releasing.md +49 -1
  77. package/docs/runbooks.md +98 -7
  78. package/docs/state.md +23 -11
  79. package/docs/support-matrix.md +40 -4
  80. package/docs/topology.md +67 -6
  81. package/docs/ubuntu-sync.md +78 -28
  82. package/docs/worker-contract.md +122 -19
  83. package/docs/worker-requirements.md +107 -0
  84. package/examples/fleet.json +1 -0
  85. package/examples/workers-from-infrastructure/main.tf +132 -0
  86. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  87. package/llms.txt +3 -1
  88. package/mise.lock +29 -0
  89. package/mise.toml +5 -3
  90. package/package.json +2 -2
  91. package/schema/check-ids.json +1967 -170
  92. package/schema/check-ids.schema.json +34 -0
  93. package/schema/fleet.schema.json +327 -8
  94. package/schema/output/check.schema.json +1 -1
  95. package/schema/output/fleet-state-definition.schema.json +14 -0
  96. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  97. package/schema/output/fleet-sync.schema.json +18 -0
  98. package/schema/output/report.schema.json +3 -2
  99. package/schema/worker-record.schema.json +76 -0
  100. package/src/agents/converge.ts +140 -24
  101. package/src/agents/files.ts +2 -32
  102. package/src/agents/native-verify.ts +34 -3
  103. package/src/agents/secrets.ts +39 -4
  104. package/src/check-owners.ts +61 -2
  105. package/src/check-registry.ts +630 -0
  106. package/src/cli/agent-guide.ts +3 -3
  107. package/src/cli/cli.ts +4 -0
  108. package/src/cli/fleet-manifest.ts +9 -3
  109. package/src/cli/fleet-node-lifecycle.ts +313 -64
  110. package/src/cli/fleet-node-park.ts +238 -0
  111. package/src/cli/fleet-node-seed.ts +51 -35
  112. package/src/cli/fleet-state.ts +2 -1
  113. package/src/cli/fleet-sync-checkout.ts +7 -0
  114. package/src/cli/fleet-sync-enrollment.ts +193 -32
  115. package/src/cli/fleet-sync-forward.ts +4 -0
  116. package/src/cli/fleet-sync-manager.ts +103 -33
  117. package/src/cli/fleet-sync-output.ts +31 -11
  118. package/src/cli/fleet-sync-plan.ts +65 -9
  119. package/src/cli/fleet-sync-records.ts +113 -0
  120. package/src/cli/fleet-sync-remote.ts +151 -9
  121. package/src/cli/fleet-sync-schedule.ts +14 -4
  122. package/src/cli/fleet-sync-state.ts +28 -16
  123. package/src/cli/fleet-sync-vault.ts +77 -0
  124. package/src/cli/fleet-sync-worker.ts +214 -0
  125. package/src/cli/fleet-sync.ts +150 -19
  126. package/src/cli/fleet-validate.ts +59 -21
  127. package/src/cli/fleet-watch.ts +7 -1
  128. package/src/cli/fleet.ts +87 -5
  129. package/src/cli/managed-health.ts +25 -1
  130. package/src/cli/register-fleet.ts +115 -4
  131. package/src/cli/registry.ts +14 -0
  132. package/src/cli/tailnet.ts +43 -10
  133. package/src/cli/ui-data.ts +3 -1
  134. package/src/cli/worker-usage-report.ts +63 -0
  135. package/src/clock.ts +42 -0
  136. package/src/definition.ts +332 -9
  137. package/src/diagnostics.ts +5 -2
  138. package/src/exposure.ts +102 -0
  139. package/src/fleet-config.ts +3 -1
  140. package/src/git-identity.ts +76 -3
  141. package/src/ide/ide.ts +5 -0
  142. package/src/ide/t3.ts +12 -2
  143. package/src/input-hashes.ts +6 -0
  144. package/src/machine/machine-class.ts +59 -0
  145. package/src/machine/platform.ts +90 -11
  146. package/src/machine/state.ts +37 -15
  147. package/src/machine/support-matrix.ts +11 -9
  148. package/src/machine-health.ts +28 -5
  149. package/src/manager-health.ts +148 -2
  150. package/src/manager-sync.ts +65 -4
  151. package/src/monitoring/beszel.ts +8 -8
  152. package/src/monitoring/enroll.ts +8 -1
  153. package/src/monitoring/hub-access.ts +10 -4
  154. package/src/monitoring/systems.ts +138 -0
  155. package/src/network-definition.ts +13 -4
  156. package/src/node-converge-linux.ts +136 -27
  157. package/src/node-converge.ts +13 -3
  158. package/src/platform/ansible-receipt.ts +17 -2
  159. package/src/platform/ansible.ts +40 -4
  160. package/src/platform/manager-host.ts +25 -8
  161. package/src/platform/resource-verify.ts +18 -7
  162. package/src/platform/t3-policy.ts +24 -10
  163. package/src/platform/t3-service.ts +109 -34
  164. package/src/platform/ubuntu-release.ts +24 -0
  165. package/src/platform/ubuntu-verify.ts +31 -3
  166. package/src/platform/ubuntu-worker.ts +58 -0
  167. package/src/platform/worker-platform.ts +72 -0
  168. package/src/protocol.ts +1 -1
  169. package/src/public-reachability.ts +186 -0
  170. package/src/remote-bootstrap.ts +50 -12
  171. package/src/remote-operation.ts +18 -4
  172. package/src/repair-plan.ts +23 -12
  173. package/src/report.ts +2 -1
  174. package/src/schedule-check.ts +21 -2
  175. package/src/secrets.ts +4 -0
  176. package/src/service-auth.ts +55 -48
  177. package/src/ssh-connection.ts +141 -0
  178. package/src/ssh.ts +63 -24
  179. package/src/state/store.ts +4 -2
  180. package/src/state/sync.ts +35 -6
  181. package/src/state/workers.ts +140 -0
  182. package/src/t3/pairing.ts +59 -0
  183. package/src/t3/peer-link.ts +333 -0
  184. package/src/t3/version.ts +1 -1
  185. package/src/tailnet-cleanup.ts +185 -0
  186. package/src/tailnet-control.ts +346 -0
  187. package/src/tailscale.ts +1 -1
  188. package/src/toolchain-checks.ts +11 -1
  189. package/src/types.ts +4 -2
  190. package/src/worker-account.ts +63 -0
  191. package/src/worker-activity.ts +303 -0
  192. package/src/worker-capability-checks.ts +55 -16
  193. package/src/worker-capacity.ts +31 -0
  194. package/src/worker-exposure.ts +388 -0
  195. package/src/worker-features.ts +201 -0
  196. package/src/worker-host-checks.ts +71 -20
  197. package/src/worker-manifest.ts +27 -4
  198. package/src/worker-parking.ts +70 -0
  199. package/src/worker-policy.ts +17 -1
  200. package/src/worker-readiness.ts +15 -3
  201. package/src/worker-reboot.ts +129 -0
  202. package/src/worker-records.ts +305 -0
  203. package/src/worker-seed.ts +16 -11
  204. package/src/worker-sizing.ts +76 -0
  205. package/src/cli/schedule-logs.ts +0 -36
@@ -0,0 +1,238 @@
1
+ import type { FleetDefinition, MachineDefinition } from '../definition';
2
+ import { FleetUsageError } from '../errors';
3
+ import { fleet } from '../fleet-config';
4
+ import { makeReport, renderReport } from '../report';
5
+ import { runCommand } from '../runner';
6
+ import { changeWorkers, readWorkerStates } from '../state/workers';
7
+ import { stateContext, type StateContext } from '../state/sync';
8
+ import type { Check, Runner } from '../types';
9
+ import { duration, workerActivity } from '../worker-activity';
10
+ import type { CommandOutcome } from './contracts';
11
+ import { readRoster } from './roster';
12
+
13
+ export interface NodeParkOptions {
14
+ id?: string;
15
+ /** Park although agent turns are under way, or cannot be counted. */
16
+ force?: boolean;
17
+ dryRun?: boolean;
18
+ }
19
+
20
+ export interface NodeParkServices {
21
+ run: Runner;
22
+ /** The roster: the fleet state's, or a file `--config` names. */
23
+ roster: string;
24
+ definition: () => FleetDefinition;
25
+ state: () => StateContext;
26
+ now: () => Date;
27
+ /** A normal fleet-sync of one worker, which verifies it. */
28
+ sync: (id: string) => Promise<CommandOutcome>;
29
+ }
30
+
31
+ export function parkServices(
32
+ roster: string,
33
+ sync: (id: string) => Promise<CommandOutcome>
34
+ ): NodeParkServices {
35
+ return {
36
+ run: runCommand,
37
+ roster,
38
+ definition: () => fleet,
39
+ state: () => stateContext(runCommand),
40
+ now: () => new Date(),
41
+ sync,
42
+ };
43
+ }
44
+
45
+ /** A hook may start or stop a machine through an infrastructure tool; it
46
+ * gets half an hour.
47
+ */
48
+ const HOOK_TIMEOUT_MS = 1_800_000;
49
+
50
+ function declared(definition: FleetDefinition, id: string | undefined) {
51
+ const worker = definition.workers.find((entry) => entry.id === id);
52
+ if (!worker)
53
+ throw new FleetUsageError('--id must name a worker fleet.json declares.');
54
+ return worker;
55
+ }
56
+
57
+ function outcome(
58
+ command: string,
59
+ id: string,
60
+ checks: Check[],
61
+ extra: Record<string, unknown> = {}
62
+ ): CommandOutcome {
63
+ const report = { ...makeReport(command, [{ node: id, checks }]), ...extra };
64
+ return {
65
+ exitCode: checks.some((check) => check.status === 'fail') ? 1 : 0,
66
+ data: report,
67
+ text: renderReport(report),
68
+ };
69
+ }
70
+
71
+ /** Runs the declared stop or start command with the worker ID last. */
72
+ async function hook(
73
+ worker: MachineDefinition,
74
+ which: 'stop' | 'start',
75
+ definition: FleetDefinition,
76
+ run: Runner
77
+ ): Promise<Check | null> {
78
+ const command = definition.parking?.[which];
79
+ if (!command) return null;
80
+ const result = await run([...command, worker.id], {
81
+ timeoutMs: HOOK_TIMEOUT_MS,
82
+ });
83
+ return result.ok
84
+ ? {
85
+ id: 'worker:park-hook',
86
+ status: 'pass',
87
+ summary: `The declared ${which} command ran for ${worker.id}.`,
88
+ }
89
+ : {
90
+ id: 'worker:park-hook',
91
+ status: 'fail',
92
+ summary:
93
+ which === 'stop'
94
+ ? 'WORKER_STOP_HOOK_FAILED'
95
+ : 'WORKER_START_HOOK_FAILED',
96
+ remedy: {
97
+ action: `The ${which} command parking.${which} in fleet.json names failed for ${worker.id}, so its state was not changed. Run it by hand to see why, then run this command again.`,
98
+ },
99
+ };
100
+ }
101
+
102
+ /** Why a worker must not be parked now, unless forced: agent turns under
103
+ * way, or a worker that answers but whose turns cannot be counted. A worker
104
+ * that does not answer is off already.
105
+ */
106
+ async function busy(
107
+ worker: MachineDefinition,
108
+ services: NodeParkServices
109
+ ): Promise<Check | null> {
110
+ const node = (await readRoster(services.roster))?.nodes.find(
111
+ (entry) => entry.id === worker.id
112
+ );
113
+ const activity = node ? await workerActivity(node, services.run) : null;
114
+ if (!activity || activity.runningSessions === 0) return null;
115
+ return {
116
+ id: 'worker:parked',
117
+ status: 'fail',
118
+ summary:
119
+ activity.runningSessions === null
120
+ ? 'WORKER_ACTIVITY_UNREADABLE'
121
+ : 'WORKER_BUSY',
122
+ remedy: {
123
+ action:
124
+ activity.runningSessions === null
125
+ ? `${worker.id}'s T3 could not say whether agent turns are under way, so it was not parked. Look at it, then repeat with --force to park it anyway.`
126
+ : `${worker.id} runs ${activity.runningSessions} agent turn(s); stopping it would end them. Wait for them to finish, or repeat with --force.`,
127
+ },
128
+ };
129
+ }
130
+
131
+ /** Parks a worker: it is stopped or destroyed on purpose, so it is no longer
132
+ * contacted, never the canary and does not fail the fleet. Refuses while
133
+ * agent turns run on it, unless --force. Runs the declared stop command.
134
+ */
135
+ export async function fleetNodePark(
136
+ options: NodeParkOptions,
137
+ services: NodeParkServices
138
+ ): Promise<CommandOutcome> {
139
+ const definition = services.definition();
140
+ const worker = declared(definition, options.id);
141
+ const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
142
+ if (parkedAt)
143
+ return outcome('fleet-node-park', worker.id, [
144
+ {
145
+ id: 'worker:parked',
146
+ status: 'pass',
147
+ summary: `${worker.id} is parked since ${parkedAt}.`,
148
+ },
149
+ ]);
150
+ if (!options.force) {
151
+ const refused = await busy(worker, services);
152
+ if (refused) return outcome('fleet-node-park', worker.id, [refused]);
153
+ }
154
+ if (options.dryRun)
155
+ return outcome('fleet-node-park', worker.id, [
156
+ {
157
+ id: 'worker:parked',
158
+ status: 'skip',
159
+ summary: `Planned: ${definition.parking?.stop ? 'run the declared stop command and ' : ''}park ${worker.id}.`,
160
+ },
161
+ ]);
162
+ const stopped = await hook(worker, 'stop', definition, services.run);
163
+ if (stopped?.status === 'fail')
164
+ return outcome('fleet-node-park', worker.id, [stopped]);
165
+ const at = services.now().toISOString();
166
+ await changeWorkers(services.state(), (current) => ({
167
+ [worker.id]: { ...current[worker.id], parkedAt: at },
168
+ }));
169
+ return outcome('fleet-node-park', worker.id, [
170
+ ...(stopped ? [stopped] : []),
171
+ {
172
+ id: 'worker:parked',
173
+ status: 'pass',
174
+ summary: `${worker.id} is parked since ${at}: fleet-sync and fleet-status no longer contact it.`,
175
+ },
176
+ ]);
177
+ }
178
+
179
+ /** Brings a parked worker back: runs the declared start command, marks it
180
+ * active, then runs a normal fleet-sync of that worker, which verifies it.
181
+ * The exit code is the sync's.
182
+ */
183
+ export async function fleetNodeResume(
184
+ options: NodeParkOptions,
185
+ services: NodeParkServices
186
+ ): Promise<CommandOutcome> {
187
+ const definition = services.definition();
188
+ const worker = declared(definition, options.id);
189
+ const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
190
+ if (options.dryRun)
191
+ return outcome('fleet-node-resume', worker.id, [
192
+ {
193
+ id: 'worker:parked',
194
+ status: 'skip',
195
+ summary: parkedAt
196
+ ? `Planned: ${definition.parking?.start ? 'run the declared start command, ' : ''}mark ${worker.id} active and run fleet-sync --node ${worker.id}.`
197
+ : `Planned: ${worker.id} is active already; run fleet-sync --node ${worker.id}.`,
198
+ },
199
+ ]);
200
+ const checks: Check[] = [];
201
+ if (parkedAt) {
202
+ const started = await hook(worker, 'start', definition, services.run);
203
+ if (started?.status === 'fail')
204
+ return outcome('fleet-node-resume', worker.id, [started]);
205
+ if (started) checks.push(started);
206
+ await changeWorkers(services.state(), (current) => {
207
+ const { parkedAt: _parked, ...rest } = current[worker.id] ?? {};
208
+ return { [worker.id]: rest };
209
+ });
210
+ const since = (services.now().getTime() - Date.parse(parkedAt)) / 1000;
211
+ checks.push({
212
+ id: 'worker:parked',
213
+ status: 'pass',
214
+ summary: `${worker.id} is active again after ${duration(Math.max(0, since))} parked.`,
215
+ });
216
+ }
217
+ const synced = await services.sync(worker.id);
218
+ const report = synced.data as { ok?: boolean };
219
+ checks.push(
220
+ synced.exitCode === 0 && report.ok !== false
221
+ ? {
222
+ id: 'rollout:node',
223
+ status: 'pass',
224
+ summary: `fleet-sync --node ${worker.id} converged and verified it.`,
225
+ }
226
+ : {
227
+ id: 'rollout:node',
228
+ status: 'fail',
229
+ summary: 'WORKER_RESUME_FAILED',
230
+ remedy: {
231
+ action: `fleet-sync --node ${worker.id} did not end ready; its report (sync in this output) says why. Fix that, then run devfleet fleet-sync --node ${worker.id}.`,
232
+ },
233
+ }
234
+ );
235
+ return outcome('fleet-node-resume', worker.id, checks, {
236
+ sync: synced.data,
237
+ });
238
+ }
@@ -2,21 +2,24 @@ import { readFile, writeFile } from 'node:fs/promises';
2
2
  import { homedir } from 'node:os';
3
3
  import { join } from 'node:path';
4
4
  import type { FleetDefinition } from '../definition';
5
- import { ubuntuRelease } from '../platform/ubuntu-release';
5
+ import { tailscalePins, ubuntuRelease } from '../platform/ubuntu-release';
6
6
  import { FleetUsageError } from '../errors';
7
7
  import { fleet, secret } from '../fleet-config';
8
8
  import type { SecretsProvider } from '../secrets';
9
- import type { Check } from '../types';
10
9
  import {
11
- generateHostKey,
12
- renderSeed,
13
- seedUserData,
14
- tailscalePins,
15
- } from '../worker-seed';
10
+ controlRemedy,
11
+ fleetTailnetControl,
12
+ loginServer,
13
+ TailnetControlError,
14
+ type ControlHttp,
15
+ } from '../tailnet-control';
16
+ import type { Check } from '../types';
17
+ import { generateHostKey, renderSeed, seedUserData } from '../worker-seed';
16
18
  import type { CommandOutcome } from './contracts';
17
19
  import {
18
20
  keyName,
19
21
  lifecycleServices,
22
+ pinHostKey,
20
23
  type NodeLifecycleServices,
21
24
  } from './fleet-node-lifecycle';
22
25
 
@@ -31,19 +34,23 @@ export interface NodeSeedOptions {
31
34
 
32
35
  export interface NodeSeedServices extends Pick<
33
36
  NodeLifecycleServices,
34
- 'run' | 'playbook' | 'knownHosts'
37
+ 'run' | 'playbook' | 'knownHosts' | 'state' | 'now'
35
38
  > {
36
39
  definition: () => FleetDefinition;
37
40
  secret: SecretsProvider;
38
41
  home: string;
42
+ /** The tailnet control plane's HTTP; answered in scenarios. */
43
+ http?: ControlHttp;
39
44
  }
40
45
 
41
46
  export function seedServices(roster: string): NodeSeedServices {
42
- const { run, playbook, knownHosts } = lifecycleServices(roster);
47
+ const { run, playbook, knownHosts, state, now } = lifecycleServices(roster);
43
48
  return {
44
49
  run,
45
50
  playbook,
46
51
  knownHosts,
52
+ state,
53
+ now,
47
54
  definition: () => fleet,
48
55
  secret,
49
56
  home: homedir(),
@@ -75,11 +82,8 @@ export async function fleetNodeSeed(
75
82
  `No manager public key at ${keyFile}; name one with --public-key PATH.`
76
83
  );
77
84
  });
78
- const authKey = await services.secret('tailnet-auth-key');
79
- if (!authKey)
80
- throw new FleetUsageError(
81
- 'The secrets provider holds no tailnet-auth-key. Store a short-lived, single-use, tagged Tailscale auth key there first.'
82
- );
85
+ const tailnet = services.definition().tailnet;
86
+ const authKey = await seedAuthKey(worker.id, services);
83
87
  const release = ubuntuRelease(services.definition().ubuntu.release);
84
88
  if (!release)
85
89
  throw new FleetUsageError(
@@ -92,36 +96,20 @@ export async function fleetNodeSeed(
92
96
  managerKey,
93
97
  authKey,
94
98
  hostKey,
95
- tailscale: await tailscalePins(),
99
+ tailscale: tailscalePins(release),
96
100
  codename: release.codename,
101
+ ...(loginServer(tailnet) ? { loginServer: loginServer(tailnet)! } : {}),
97
102
  }),
98
103
  Boolean(options.autoinstall)
99
104
  );
100
105
  await writeFile(options.output, seed, { mode: 0o600 });
101
106
  const host = await keyName(worker, services.run);
102
107
  const publicKey = hostKey.public.split(' ').slice(0, 2).join(' ');
108
+ // Pinned before the machine answers: in the fleet state for a worker
109
+ // reached from declared data, else in the operator's known_hosts.
103
110
  const pin: Check =
104
111
  options.apply && options.yes
105
- ? (await services.playbook('macos/worker-host-key.yml', {
106
- host,
107
- pin: 'present',
108
- key: publicKey,
109
- known_hosts: services.knownHosts,
110
- }))
111
- ? {
112
- id: 'ssh:host-key',
113
- status: 'pass',
114
- summary: `The manager trusts the seed's host key for ${host}.`,
115
- }
116
- : {
117
- id: 'ssh:host-key',
118
- status: 'fail',
119
- summary: 'SSH_HOST_KEY_NOT_CHANGED',
120
- remedy: {
121
- action:
122
- 'The pinned Ansible could not change known_hosts; run the command again.',
123
- },
124
- }
112
+ ? await pinHostKey(worker, host, publicKey, services)
125
113
  : {
126
114
  id: 'ssh:host-key',
127
115
  status: 'skip',
@@ -149,3 +137,31 @@ export async function fleetNodeSeed(
149
137
  ].join('\n'),
150
138
  };
151
139
  }
140
+
141
+ /** The key the seed carries: one minted for this worker by the control plane
142
+ * fleet.json declares, or else the secret tailnet-auth-key.
143
+ */
144
+ async function seedAuthKey(
145
+ worker: string,
146
+ services: NodeSeedServices
147
+ ): Promise<string> {
148
+ const config = services.definition().tailnet;
149
+ if (config)
150
+ try {
151
+ const control = await fleetTailnetControl(
152
+ config,
153
+ services.secret,
154
+ services.http
155
+ );
156
+ return (await control!.createAuthKey({ worker })).key;
157
+ } catch (error) {
158
+ if (!(error instanceof TailnetControlError)) throw error;
159
+ throw new FleetUsageError(`${error.code}: ${controlRemedy(error.code)}`);
160
+ }
161
+ const key = await services.secret('tailnet-auth-key');
162
+ if (!key)
163
+ throw new FleetUsageError(
164
+ 'The secrets provider holds no tailnet-auth-key. Declare tailnet in fleet.json with the secret tailnet-admin so a key is minted per worker, or store a short-lived, single-use, tagged Tailscale auth key there first.'
165
+ );
166
+ return key;
167
+ }
@@ -486,13 +486,14 @@ async function restore(
486
486
  options: FleetStateOptions,
487
487
  services: FleetStateServices
488
488
  ): Promise<CommandOutcome> {
489
- const target = context(services);
489
+ const target = context(services, options);
490
490
  if (!target.bucket)
491
491
  throw new FleetUsageError(
492
492
  'No bucket is configured, so there is nothing to restore from. Name it under "state" in fleet.json, or run devfleet fleet-state --init --bucket NAME --endpoint https://HOST.'
493
493
  );
494
494
  if (options.timestamp !== undefined) {
495
495
  // The bucket's current version is replaced too: --yes is the consent.
496
+ // Another manager's lease is not, and needs --take-over.
496
497
  if (!options.yes)
497
498
  throw new FleetUsageError(
498
499
  '--restore --timestamp replaces the fleet state in the bucket and on this machine with an earlier version. Repeat with --yes.'
@@ -1,5 +1,6 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { readFileSync } from 'node:fs';
3
+ import { homedir } from 'node:os';
3
4
  import { join } from 'node:path';
4
5
  import { prepareSource } from '../machine/source';
5
6
  import { PACKAGE_ROOT, packageInfo } from '../package-info';
@@ -16,6 +17,12 @@ export const RUN_CHECKOUT_ENV = 'PERSONAL_FLEET_RUN_CHECKOUT';
16
17
 
17
18
  const RUN_SCRIPT = 'bootstrap/fleet-manager-run.sh';
18
19
 
20
+ /** Where the run script keeps the manager's fleet-owned checkouts, one per
21
+ * revision (`$fleet/manager-runs/<revision>`).
22
+ */
23
+ export const managerRuns = (): string =>
24
+ join(homedir(), '.local/share/personal/fleet/manager-runs');
25
+
19
26
  export const inRunCheckout = (revision: string): boolean =>
20
27
  process.env[RUN_CHECKOUT_ENV] === revision;
21
28
 
@@ -1,21 +1,58 @@
1
1
  import { mkdir, rm } from 'node:fs/promises';
2
2
  import { loadRoster } from './roster';
3
- import { enrolledRoster } from '../definition';
3
+ import { enrolledRoster, sshTarget, type FleetDefinition } from '../definition';
4
4
  import { FleetUsageError } from '../errors';
5
5
  import { dirname } from 'node:path';
6
- import { fleet as definition } from '../fleet-config';
6
+ import { fleet } from '../fleet-config';
7
7
  import { discoverTailnet } from '../tailscale';
8
8
  import { matchComputers } from '../tailnet-matching';
9
9
  import { sshArgs, sshFailure } from '../ssh';
10
10
  import { ensurePrivateRoot } from '../machine/state';
11
- import type { FleetNode, Runner } from '../types';
11
+ import { createWorkerAccount } from '../worker-account';
12
+ import { packagePlaybook, type PackagePlaybook } from '../platform/ansible';
13
+ import type { Check, FleetNode, Runner } from '../types';
12
14
  import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
15
+ import { readParked } from '../state/workers';
13
16
 
14
- /** Interactive enrollment/resume only. Existing paths and explicit identities
15
- * are preserved. Tailnet candidates never enter the declared membership.
17
+ /** What an enrollment did: the workers whose roster entry it added or
18
+ * completed, and the checks that say why another one was not enrolled (or
19
+ * what was done to reach it).
16
20
  */
17
- export async function enrollManager(path: string, run: Runner): Promise<void> {
18
- const canonical = definition;
21
+ export interface EnrollmentResult {
22
+ enrolled: string[];
23
+ checks: { node: string; check: Check }[];
24
+ }
25
+
26
+ export interface EnrollmentOptions {
27
+ /** A person is at the terminal and sees the list of what is enrolled. */
28
+ interactive: boolean;
29
+ /** A worker not on the tailnet yet joins it during this run: the fleet
30
+ * declares a tailnet control plane that issues single-use keys.
31
+ */
32
+ joinable?: boolean;
33
+ definition?: FleetDefinition;
34
+ /** Applies a playbook of the package from the manager. */
35
+ playbook?: PackagePlaybook;
36
+ /** When each parked worker was parked; default the fleet state's. */
37
+ parked?: Record<string, string>;
38
+ }
39
+
40
+ /** Enrolls declared workers, and resumes earlier enrollments. Existing paths
41
+ * and explicit identities are preserved. Tailnet candidates never enter the
42
+ * declared membership. An interactive run enrolls what a fresh
43
+ * host-key-verified SSH session reaches. A run without a person enrolls a
44
+ * worker only when every fact is verified: its pinned host key matched (the
45
+ * session is strict), `sudo -n` works, and exactly one tailnet device matches
46
+ * it, or none when this run joins it to the tailnet itself. Anything
47
+ * ambiguous stays out, with the code that says why; nothing prompts.
48
+ */
49
+ export async function enrollManager(
50
+ path: string,
51
+ run: Runner,
52
+ options: EnrollmentOptions = { interactive: true }
53
+ ): Promise<EnrollmentResult> {
54
+ const canonical = options.definition ?? fleet;
55
+ const result: EnrollmentResult = { enrolled: [], checks: [] };
19
56
  const original = await existingEnrollment(path);
20
57
  const saved =
21
58
  original === null
@@ -23,54 +60,178 @@ export async function enrollManager(path: string, run: Runner): Promise<void> {
23
60
  : enrolledRoster(canonical, await loadRoster(path)).nodes;
24
61
  const nodes: FleetNode[] = canonical.workers.map((node) => ({
25
62
  id: node.id,
26
- ssh: node.ssh,
63
+ ssh: sshTarget(node),
27
64
  workspace: '/',
28
65
  required: ['t3', 'toolchain', 'workstation'],
29
66
  }));
67
+ // A parked worker is off on purpose: it is enrolled once it is resumed.
68
+ const parked = options.parked ?? readParked();
30
69
  const pending = nodes.filter(
31
- (node) => !saved.find((entry) => entry.id === node.id)?.tailscaleId
70
+ (node) =>
71
+ !parked[node.id] &&
72
+ !saved.find((entry) => entry.id === node.id)?.tailscaleId
32
73
  );
33
- if (!pending.length) return;
74
+ if (!pending.length) return result;
34
75
  const tailnet = await discoverTailnet(run);
35
76
  const matches = await matchComputers(nodes, tailnet.computers, run);
36
77
  const enrolled: FleetNode[] = [];
37
78
  for (const node of pending) {
38
- const peers = tailnet.computers.filter((peer) =>
39
- matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
40
- );
41
- if (
42
- ambiguous(
43
- peers.map((peer) => peer.id),
44
- matches
45
- )
46
- )
47
- continue;
48
- const connection = await inspectConnection(node, peers[0]?.id, run);
79
+ // A worker already in the roster reports its own state in the run.
80
+ const known = saved.some((entry) => entry.id === node.id);
81
+ const connection = await enrollOne(node, {
82
+ run,
83
+ options,
84
+ peers: tailnet.computers
85
+ .filter((peer) =>
86
+ matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
87
+ )
88
+ .map((peer) => peer.id),
89
+ matches,
90
+ user: canonical.workers.find((worker) => worker.id === node.id)?.network
91
+ ?.user,
92
+ note: (check) => {
93
+ if (!known) result.checks.push({ node: node.id, check });
94
+ },
95
+ });
49
96
  if (connection) enrolled.push(connection);
50
97
  }
51
98
  const merged = mergeEnrollment(saved, enrolled);
52
- if (JSON.stringify(merged) === JSON.stringify(saved)) return;
99
+ if (JSON.stringify(merged) === JSON.stringify(saved)) return result;
53
100
  // Only canonical nodes reached over host-key-verified SSH get here, so
54
101
  // enrollment needs no extra confirmation; the list is still shown.
55
- process.stderr.write(
56
- `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => ` ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
57
- );
102
+ if (options.interactive)
103
+ process.stderr.write(
104
+ `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => ` ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
105
+ );
58
106
  await persistEnrollment(path, original, merged);
107
+ result.enrolled = merged
108
+ .filter(
109
+ (node) =>
110
+ JSON.stringify(node) !==
111
+ JSON.stringify(saved.find((entry) => entry.id === node.id))
112
+ )
113
+ .map((node) => node.id);
114
+ return result;
59
115
  }
60
116
 
117
+ /** One worker's connection, when every fact this run needs is verified. */
118
+ async function enrollOne(
119
+ node: FleetNode,
120
+ context: {
121
+ run: Runner;
122
+ options: EnrollmentOptions;
123
+ /** The tailnet devices that match this worker. */
124
+ peers: string[];
125
+ matches: Map<string, FleetNode[]>;
126
+ user: string | undefined;
127
+ note: (check: Check) => void;
128
+ }
129
+ ): Promise<FleetNode | null> {
130
+ const { run, options, peers, note } = context;
131
+ if (ambiguous(peers, context.matches)) {
132
+ if (!options.interactive) note(IDENTITY_CONFLICT);
133
+ return null;
134
+ }
135
+ if (!options.interactive && !peers.length && !options.joinable) {
136
+ note(NOT_ON_TAILNET);
137
+ return null;
138
+ }
139
+ const connection = await inspectConnection(node, peers[0], {
140
+ run,
141
+ options,
142
+ user: context.user,
143
+ note,
144
+ quiet: !options.interactive,
145
+ });
146
+ if (!connection || options.interactive) return connection;
147
+ const privilege = await passwordlessSudo(connection, run);
148
+ if (privilege) note(privilege);
149
+ return privilege ? null : connection;
150
+ }
151
+
152
+ const IDENTITY_CONFLICT: Check = {
153
+ id: 'tailscale:identity',
154
+ status: 'fail',
155
+ summary: 'TAILSCALE_IDENTITY_CONFLICT',
156
+ remedy: {
157
+ action:
158
+ 'More than one tailnet device matches this worker, so it was not enrolled. Remove the stale device from the tailnet, then rerun fleet-sync.',
159
+ },
160
+ };
161
+
162
+ const NOT_ON_TAILNET: Check = {
163
+ id: 'tailscale:enrolled-identity',
164
+ status: 'fail',
165
+ summary: 'TAILSCALE_ENROLLMENT_REQUIRED',
166
+ remedy: {
167
+ action:
168
+ 'No tailnet device matches this worker and fleet.json declares no tailnet control plane to join it with, so a run without a person did not enroll it. Declare tailnet with the secret tailnet-admin, join the machine yourself, or run fleet-sync once in a terminal.',
169
+ },
170
+ };
171
+
172
+ /** A run without a person never prompts for sudo: the account has it, or the
173
+ * worker is not enrolled.
174
+ */
175
+ async function passwordlessSudo(
176
+ node: FleetNode,
177
+ run: Runner
178
+ ): Promise<Check | null> {
179
+ const result = await run([...sshArgs(node), 'sudo -n true'], {
180
+ timeoutMs: 15_000,
181
+ });
182
+ if (result.ok) return null;
183
+ if (result.reason !== 'exit' || !/sudo:/i.test(result.stderr))
184
+ return sshFailure(result);
185
+ return {
186
+ id: 'bootstrap:privilege',
187
+ status: 'fail',
188
+ summary: 'SUDO_AUTHORIZATION_REQUIRED',
189
+ remedy: {
190
+ action:
191
+ 'The worker account has no passwordless sudo, so a run without a person did not enroll it. Grant it (a fleet-node-seed seed does), or run fleet-sync once in a terminal to authorize it.',
192
+ },
193
+ };
194
+ }
195
+
196
+ /** The account's home over a fresh strict session. A machine whose image
197
+ * lets only root in refuses the declared account: it is created once over
198
+ * the root login, and the session is opened again.
199
+ */
61
200
  async function inspectConnection(
62
201
  node: FleetNode,
63
202
  peerId: string | undefined,
64
- run: Runner
203
+ context: {
204
+ run: Runner;
205
+ options: EnrollmentOptions;
206
+ user: string | undefined;
207
+ note: (check: Check) => void;
208
+ quiet: boolean;
209
+ }
65
210
  ): Promise<FleetNode | null> {
66
- const home = await run([...sshArgs(node), 'printf "%s" "$HOME"'], {
67
- timeoutMs: 20_000,
68
- });
211
+ const { run } = context;
212
+ const probe = () =>
213
+ run([...sshArgs(node), 'printf "%s" "$HOME"'], { timeoutMs: 20_000 });
214
+ let home = await probe();
215
+ if (!home.ok && sshFailure(home).id === 'ssh:auth' && context.user) {
216
+ const account = await createWorkerAccount(
217
+ { id: node.id, ssh: node.ssh, login: 'root' },
218
+ context.user,
219
+ context.options.playbook ?? packagePlaybook(run)
220
+ );
221
+ // Only an account that was created is reported here: a root login that
222
+ // is refused as well stays the SSH failure below.
223
+ if (account.status === 'pass') {
224
+ context.note(account);
225
+ home = await probe();
226
+ }
227
+ }
69
228
  if (!home.ok) {
70
229
  const check = sshFailure(home);
71
- process.stderr.write(
72
- `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
73
- );
230
+ if (context.quiet) context.note(check);
231
+ else
232
+ process.stderr.write(
233
+ `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
234
+ );
74
235
  return null;
75
236
  }
76
237
  if (!/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(home.stdout)) return null;