@crvouga/dev-fleet 0.8.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +65 -9
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +55 -9
  14. package/ansible/macos/manager.yml +154 -7
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/monitoring/agent-darwin.yml +3 -0
  19. package/ansible/monitoring/agent.yml +2 -0
  20. package/ansible/monitoring/hub-darwin.yml +3 -0
  21. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  22. package/ansible/net/remote-probe.sh +4 -1
  23. package/ansible/platforms.json +2 -2
  24. package/ansible/pyproject.toml +5 -0
  25. package/ansible/ubuntu/baseline.yml +39 -13
  26. package/ansible/ubuntu/disk-gc.py +6 -2
  27. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  28. package/ansible/ubuntu/policy.json +125 -6
  29. package/ansible/ubuntu/reboot.yml +89 -0
  30. package/ansible/ubuntu/recovery.yml +13 -3
  31. package/ansible/ubuntu/t3/policy.json +3 -0
  32. package/ansible/ubuntu/t3-desktop.yml +23 -2
  33. package/ansible/ubuntu/t3-headless.yml +128 -0
  34. package/ansible/ubuntu/t3-health.sh +24 -10
  35. package/ansible/ubuntu/t3-idle.py +37 -2
  36. package/ansible/ubuntu/t3-policy.yml +4 -1
  37. package/ansible/ubuntu/t3-route.yml +1 -0
  38. package/ansible/ubuntu/tailscale.yml +54 -8
  39. package/ansible/ubuntu/tasks/browser.yml +3 -0
  40. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  41. package/ansible/ubuntu/tasks/docker.yml +53 -6
  42. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  43. package/ansible/ubuntu/tasks/features.yml +22 -0
  44. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  45. package/ansible/ubuntu/tasks/install.yml +2 -1
  46. package/ansible/ubuntu/tasks/packages.yml +4 -2
  47. package/ansible/ubuntu/tasks/release.yml +6 -0
  48. package/ansible/ubuntu/tasks/resources.yml +6 -4
  49. package/ansible/ubuntu/tasks/services.yml +2 -2
  50. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  51. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  52. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  53. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  54. package/ansible/ubuntu/tasks/user.yml +1 -0
  55. package/ansible/ubuntu/worker-account.yml +117 -0
  56. package/ansible/uv.lock +265 -0
  57. package/bin/devfleet.mjs +42 -4
  58. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  59. package/bootstrap/global-config.ts +29 -3
  60. package/bootstrap/package-smoke.ts +127 -0
  61. package/bootstrap/pins.ts +189 -0
  62. package/bootstrap/release-notes.ts +88 -0
  63. package/bootstrap/release-version.ts +41 -7
  64. package/bootstrap/worker-features.ts +114 -0
  65. package/bootstrap/worker-seed.yaml +7 -3
  66. package/bootstrap/worker-tools.ts +39 -9
  67. package/cliproxy/README.md +1 -1
  68. package/cliproxy/docs/health-workflow.example.yml +20 -143
  69. package/docs/authentication.md +19 -11
  70. package/docs/consumer-setup.md +42 -27
  71. package/docs/convergence.md +67 -0
  72. package/docs/declared.md +61 -34
  73. package/docs/machine-setup.md +73 -7
  74. package/docs/manager-setup.md +10 -3
  75. package/docs/operations.md +64 -9
  76. package/docs/releasing.md +49 -1
  77. package/docs/runbooks.md +98 -7
  78. package/docs/state.md +23 -11
  79. package/docs/support-matrix.md +40 -4
  80. package/docs/topology.md +67 -6
  81. package/docs/ubuntu-sync.md +78 -28
  82. package/docs/worker-contract.md +122 -19
  83. package/docs/worker-requirements.md +107 -0
  84. package/examples/fleet.json +1 -0
  85. package/examples/workers-from-infrastructure/main.tf +132 -0
  86. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  87. package/llms.txt +3 -1
  88. package/mise.lock +29 -0
  89. package/mise.toml +5 -3
  90. package/package.json +2 -2
  91. package/schema/check-ids.json +1967 -170
  92. package/schema/check-ids.schema.json +34 -0
  93. package/schema/fleet.schema.json +327 -8
  94. package/schema/output/check.schema.json +1 -1
  95. package/schema/output/fleet-state-definition.schema.json +14 -0
  96. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  97. package/schema/output/fleet-sync.schema.json +18 -0
  98. package/schema/output/report.schema.json +3 -2
  99. package/schema/worker-record.schema.json +76 -0
  100. package/src/agents/converge.ts +140 -24
  101. package/src/agents/files.ts +2 -32
  102. package/src/agents/native-verify.ts +34 -3
  103. package/src/agents/secrets.ts +39 -4
  104. package/src/check-owners.ts +61 -2
  105. package/src/check-registry.ts +630 -0
  106. package/src/cli/agent-guide.ts +3 -3
  107. package/src/cli/cli.ts +4 -0
  108. package/src/cli/fleet-manifest.ts +9 -3
  109. package/src/cli/fleet-node-lifecycle.ts +313 -64
  110. package/src/cli/fleet-node-park.ts +238 -0
  111. package/src/cli/fleet-node-seed.ts +51 -35
  112. package/src/cli/fleet-state.ts +2 -1
  113. package/src/cli/fleet-sync-checkout.ts +7 -0
  114. package/src/cli/fleet-sync-enrollment.ts +193 -32
  115. package/src/cli/fleet-sync-forward.ts +4 -0
  116. package/src/cli/fleet-sync-manager.ts +103 -33
  117. package/src/cli/fleet-sync-output.ts +31 -11
  118. package/src/cli/fleet-sync-plan.ts +65 -9
  119. package/src/cli/fleet-sync-records.ts +113 -0
  120. package/src/cli/fleet-sync-remote.ts +151 -9
  121. package/src/cli/fleet-sync-schedule.ts +14 -4
  122. package/src/cli/fleet-sync-state.ts +28 -16
  123. package/src/cli/fleet-sync-vault.ts +77 -0
  124. package/src/cli/fleet-sync-worker.ts +214 -0
  125. package/src/cli/fleet-sync.ts +150 -19
  126. package/src/cli/fleet-validate.ts +59 -21
  127. package/src/cli/fleet-watch.ts +7 -1
  128. package/src/cli/fleet.ts +87 -5
  129. package/src/cli/managed-health.ts +25 -1
  130. package/src/cli/register-fleet.ts +115 -4
  131. package/src/cli/registry.ts +14 -0
  132. package/src/cli/tailnet.ts +43 -10
  133. package/src/cli/ui-data.ts +3 -1
  134. package/src/cli/worker-usage-report.ts +63 -0
  135. package/src/clock.ts +42 -0
  136. package/src/definition.ts +332 -9
  137. package/src/diagnostics.ts +5 -2
  138. package/src/exposure.ts +102 -0
  139. package/src/fleet-config.ts +3 -1
  140. package/src/git-identity.ts +76 -3
  141. package/src/ide/ide.ts +5 -0
  142. package/src/ide/t3.ts +12 -2
  143. package/src/input-hashes.ts +6 -0
  144. package/src/machine/machine-class.ts +59 -0
  145. package/src/machine/platform.ts +90 -11
  146. package/src/machine/state.ts +37 -15
  147. package/src/machine/support-matrix.ts +11 -9
  148. package/src/machine-health.ts +28 -5
  149. package/src/manager-health.ts +148 -2
  150. package/src/manager-sync.ts +65 -4
  151. package/src/monitoring/beszel.ts +8 -8
  152. package/src/monitoring/enroll.ts +8 -1
  153. package/src/monitoring/hub-access.ts +10 -4
  154. package/src/monitoring/systems.ts +138 -0
  155. package/src/network-definition.ts +13 -4
  156. package/src/node-converge-linux.ts +136 -27
  157. package/src/node-converge.ts +13 -3
  158. package/src/platform/ansible-receipt.ts +17 -2
  159. package/src/platform/ansible.ts +40 -4
  160. package/src/platform/manager-host.ts +25 -8
  161. package/src/platform/resource-verify.ts +18 -7
  162. package/src/platform/t3-policy.ts +24 -10
  163. package/src/platform/t3-service.ts +109 -34
  164. package/src/platform/ubuntu-release.ts +24 -0
  165. package/src/platform/ubuntu-verify.ts +31 -3
  166. package/src/platform/ubuntu-worker.ts +58 -0
  167. package/src/platform/worker-platform.ts +72 -0
  168. package/src/protocol.ts +1 -1
  169. package/src/public-reachability.ts +186 -0
  170. package/src/remote-bootstrap.ts +50 -12
  171. package/src/remote-operation.ts +18 -4
  172. package/src/repair-plan.ts +23 -12
  173. package/src/report.ts +2 -1
  174. package/src/schedule-check.ts +21 -2
  175. package/src/secrets.ts +4 -0
  176. package/src/service-auth.ts +55 -48
  177. package/src/ssh-connection.ts +141 -0
  178. package/src/ssh.ts +63 -24
  179. package/src/state/store.ts +4 -2
  180. package/src/state/sync.ts +35 -6
  181. package/src/state/workers.ts +140 -0
  182. package/src/t3/pairing.ts +59 -0
  183. package/src/t3/peer-link.ts +333 -0
  184. package/src/t3/version.ts +1 -1
  185. package/src/tailnet-cleanup.ts +185 -0
  186. package/src/tailnet-control.ts +346 -0
  187. package/src/tailscale.ts +1 -1
  188. package/src/toolchain-checks.ts +11 -1
  189. package/src/types.ts +4 -2
  190. package/src/worker-account.ts +63 -0
  191. package/src/worker-activity.ts +303 -0
  192. package/src/worker-capability-checks.ts +55 -16
  193. package/src/worker-capacity.ts +31 -0
  194. package/src/worker-exposure.ts +388 -0
  195. package/src/worker-features.ts +201 -0
  196. package/src/worker-host-checks.ts +71 -20
  197. package/src/worker-manifest.ts +27 -4
  198. package/src/worker-parking.ts +70 -0
  199. package/src/worker-policy.ts +17 -1
  200. package/src/worker-readiness.ts +15 -3
  201. package/src/worker-reboot.ts +129 -0
  202. package/src/worker-records.ts +305 -0
  203. package/src/worker-seed.ts +16 -11
  204. package/src/worker-sizing.ts +76 -0
  205. package/src/cli/schedule-logs.ts +0 -36
@@ -18,17 +18,27 @@ import {
18
18
  convergeT3Service,
19
19
  runningVersionCheck,
20
20
  t3StartedAt,
21
+ t3Unit,
21
22
  } from './platform/t3-service';
23
+ import {
24
+ featuresCheck,
25
+ presentParts,
26
+ workerFeatures,
27
+ type WorkerFeatures,
28
+ } from './worker-features';
22
29
  import { userInfo } from 'node:os';
23
30
  import { resolve } from 'node:path';
24
31
  import { auditUbuntu } from './platform/ubuntu-audit';
32
+ import { detectMachineClass } from './machine/machine-class';
25
33
  import { convergeGitIdentity } from './git-identity';
26
34
  import { workerSelection } from '../bootstrap/worker-tools';
35
+ import { workerExposure } from './exposure';
27
36
  import {
28
37
  applyAgentProfiles,
29
38
  attempt,
30
39
  configureWorkstation,
31
40
  } from './node-converge-shared';
41
+ import { loginServer } from './tailnet-control';
32
42
  import type { Check, Runner } from './types';
33
43
 
34
44
  /** Ubuntu worker convergence: Ansible owns OS policy, services and recovery; the
@@ -36,10 +46,11 @@ import type { Check, Runner } from './types';
36
46
  */
37
47
  export async function convergeLinux(
38
48
  root: string,
39
- target: { node: string; revision: string },
49
+ target: { node: string; revision: string; restartBusy?: boolean },
40
50
  run: Runner
41
51
  ): Promise<Check[]> {
42
52
  const { node, revision } = target;
53
+ const restartBusy = target.restartBusy === true;
43
54
  const audit = await auditUbuntu(run);
44
55
  if (!audit.supported || !audit.passwordlessSudo)
45
56
  return [
@@ -58,12 +69,23 @@ export async function convergeLinux(
58
69
  const checks: Check[] = [];
59
70
  checks.push(
60
71
  ...(await attempt('tailscale:configuration', async () => [
61
- await applyTailscale(root, run),
72
+ await applyTailscale(root, node, run),
62
73
  ]))
63
74
  );
75
+ const features = workerFeatures(fleet, node);
76
+ const before = await presentParts(run, userInfo().username, features);
64
77
  const { baseline, verified } = await baselineStep(root, run, node);
65
78
  checks.push(...baseline, ...verified);
66
- checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
79
+ checks.push(
80
+ featuresCheck(
81
+ features,
82
+ await presentParts(run, userInfo().username, features),
83
+ before
84
+ )
85
+ );
86
+ checks.push(
87
+ ...(await t3AfterBaseline(root, { node, baseline, run, restartBusy }))
88
+ );
67
89
  checks.push(
68
90
  ...(await attempt('workstation:configuration', () =>
69
91
  configureWorkstation(root, run, node)
@@ -120,9 +142,10 @@ async function baselineStep(
120
142
  run: Runner,
121
143
  node: string
122
144
  ): Promise<{ baseline: Check[]; verified: Check[] }> {
145
+ const { machine } = await detectMachineClass(run);
123
146
  const measure = async () => [
124
147
  ...(await attempt('ubuntu:boot-policy', () =>
125
- verifyUbuntuBoot(userInfo().username, run)
148
+ verifyUbuntuBoot(userInfo().username, run, machine)
126
149
  )),
127
150
  ...(await attempt('ubuntu:resource-safety', () =>
128
151
  verifyResourceSafety(userInfo().homedir, run)
@@ -151,6 +174,7 @@ async function applyBaseline(
151
174
  node: string
152
175
  ): Promise<{ cached: boolean; checks: Check[] }> {
153
176
  const user = userInfo();
177
+ const features = workerFeatures(fleet, node);
154
178
  const ubuntu = resolve(root, 'ansible/ubuntu');
155
179
  const result = await runAnsibleWithReceipt(
156
180
  run,
@@ -165,13 +189,18 @@ async function applyBaseline(
165
189
  development_home: user.homedir,
166
190
  // The web profile brings headless Chromium (tasks/browser.yml).
167
191
  worker_tool_profiles: workerSelection(fleet, node).profiles,
192
+ // What the firewall opens beyond the tailnet (tasks/firewall.yml).
193
+ worker_exposure: workerExposure(fleet, node).policy,
194
+ // What the declared features leave out (tasks/features.yml).
195
+ worker_containers: features.containers,
196
+ worker_session: features.session,
168
197
  }),
169
198
  },
170
199
  {
171
200
  name: 'ubuntu-baseline',
172
201
  sources: [ubuntu, resolve(root, 'ansible/github')],
173
202
  force,
174
- verify: () => baselineServicesActive(run),
203
+ verify: () => baselineServicesActive(run, features),
175
204
  }
176
205
  );
177
206
  return {
@@ -191,14 +220,17 @@ async function applyBaseline(
191
220
  }
192
221
 
193
222
  /** Cheap probe for what the baseline owns: system services and user timers. */
194
- async function baselineServicesActive(run: Runner): Promise<boolean> {
223
+ async function baselineServicesActive(
224
+ run: Runner,
225
+ features: WorkerFeatures
226
+ ): Promise<boolean> {
195
227
  const system = await run([
196
228
  'systemctl',
197
229
  'is-active',
198
230
  '--quiet',
199
231
  'ssh.service',
200
232
  'tailscaled.service',
201
- 'docker.service',
233
+ ...(features.containers ? ['docker.service'] : []),
202
234
  ]);
203
235
  if (!system.ok) return false;
204
236
  const user = await run([
@@ -212,12 +244,27 @@ async function baselineServicesActive(run: Runner): Promise<boolean> {
212
244
  return user.ok;
213
245
  }
214
246
 
215
- async function applyTailscale(root: string, run: Runner): Promise<Check> {
247
+ /** A worker that was never logged in joins under its declared ID when the
248
+ * seed or the bootstrap left it a single-use key; the key carries the fleet's
249
+ * tag.
250
+ */
251
+ async function applyTailscale(
252
+ root: string,
253
+ node: string,
254
+ run: Runner
255
+ ): Promise<Check> {
216
256
  const ubuntu = resolve(root, 'ansible/ubuntu');
217
257
  const tailscale = await runAnsibleWithReceipt(
218
258
  run,
219
259
  'tailscale.yml',
220
- { cwd: ubuntu, timeoutMs: 300_000, input: '{}' },
260
+ {
261
+ cwd: ubuntu,
262
+ timeoutMs: 300_000,
263
+ input: JSON.stringify({
264
+ tailnet_hostname: node,
265
+ tailnet_login_server: loginServer(fleet.tailnet) ?? '',
266
+ }),
267
+ },
221
268
  {
222
269
  name: 'ubuntu-tailscale',
223
270
  sources: [ubuntu],
@@ -243,23 +290,30 @@ async function applyTailscale(root: string, run: Runner): Promise<Check> {
243
290
  async function convergeT3(
244
291
  root: string,
245
292
  node: string,
246
- run: Runner
293
+ run: Runner,
294
+ restartBusy: boolean
247
295
  ): Promise<Check[]> {
248
296
  const artifact = managedT3();
249
- const startedAt = await t3StartedAt(run);
250
- const checks = await convergeT3Service(
251
- artifact,
252
- run,
253
- async () => {
254
- const staged = await stageT3Policy(root, artifact.path, run, node);
297
+ const { session } = workerFeatures(fleet, node);
298
+ const startedAt = await t3StartedAt(run, session);
299
+ const checks = await convergeT3Service(artifact, run, {
300
+ prepare: async () => {
301
+ const staged = await stageT3Policy(root, artifact.path, run, {
302
+ node,
303
+ restartBusy,
304
+ });
255
305
  return staged.status === 'pass' ? [] : [staged];
256
306
  },
257
- () => installT3Desktop(root, run)
258
- );
307
+ install: () =>
308
+ session === 'headless'
309
+ ? installT3Headless(root, artifact.path, run, restartBusy)
310
+ : installT3Desktop(root, run, restartBusy),
311
+ session,
312
+ });
259
313
  if (checks.every((check) => check.status === 'pass'))
260
314
  checks.push(
261
315
  await loadedT3Policy(artifact.path, run, {
262
- restarted: startedAt !== (await t3StartedAt(run)),
316
+ restarted: startedAt !== (await t3StartedAt(run, session)),
263
317
  })
264
318
  );
265
319
  // Settings are watched by T3: providers, names and the profile apply
@@ -275,12 +329,16 @@ async function convergeT3(
275
329
  const serviceReady = checks.some(
276
330
  (check) => check.id === 't3:service' && check.status === 'pass'
277
331
  );
278
- if (serviceReady) checks.push(await runningVersionCheck(run, userInfo().uid));
332
+ if (serviceReady)
333
+ checks.push(
334
+ await runningVersionCheck(run, userInfo().uid, Date.now(), session)
335
+ );
279
336
  if (serviceReady)
280
337
  checks.push(
281
338
  ...(await convergeT3Route(run, () => publishT3Route(root, run)))
282
339
  );
283
- if (serviceReady) checks.push(...(await enableWorkerRecovery(root, run)));
340
+ if (serviceReady)
341
+ checks.push(...(await enableWorkerRecovery(root, run, session)));
284
342
  return checks;
285
343
  }
286
344
 
@@ -293,7 +351,11 @@ async function publishT3Route(root: string, run: Runner): Promise<boolean> {
293
351
  return result.ok;
294
352
  }
295
353
 
296
- async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
354
+ async function installT3Desktop(
355
+ root: string,
356
+ run: Runner,
357
+ restartBusy: boolean
358
+ ): Promise<boolean> {
297
359
  const user = userInfo();
298
360
  const desktop = managedT3Desktop();
299
361
  const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -311,6 +373,7 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
311
373
  t3_desktop_url: desktop.url,
312
374
  t3_desktop_name: desktop.name,
313
375
  t3_desktop_sha256: desktop.sha256,
376
+ t3_restart_busy: restartBusy,
314
377
  }),
315
378
  },
316
379
  {
@@ -341,11 +404,55 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
341
404
  return result.ok;
342
405
  }
343
406
 
407
+ /** A headless worker's T3: the pinned server the toolchain installed, as a
408
+ * lingering user service (t3-headless.yml). */
409
+ async function installT3Headless(
410
+ root: string,
411
+ server: string,
412
+ run: Runner,
413
+ restartBusy: boolean
414
+ ): Promise<boolean> {
415
+ const user = userInfo();
416
+ const ubuntu = resolve(root, 'ansible/ubuntu');
417
+ const unit = t3Unit('headless');
418
+ const result = await runAnsibleWithReceipt(
419
+ run,
420
+ resolve(ubuntu, 't3-headless.yml'),
421
+ {
422
+ cwd: ubuntu,
423
+ timeoutMs: 600_000,
424
+ input: JSON.stringify({
425
+ development_source: root,
426
+ development_user: user.username,
427
+ development_uid: user.uid,
428
+ development_home: user.homedir,
429
+ t3_server_path: server,
430
+ t3_restart_busy: restartBusy,
431
+ }),
432
+ },
433
+ {
434
+ name: 'ubuntu-t3-headless',
435
+ sources: [ubuntu],
436
+ // A stopped or disabled server reruns the play, which starts it.
437
+ verify: async () =>
438
+ (await run(['systemctl', '--user', 'is-enabled', '--quiet', unit]))
439
+ .ok &&
440
+ (await run(['systemctl', '--user', 'is-active', '--quiet', unit])).ok,
441
+ }
442
+ );
443
+ return result.ok;
444
+ }
445
+
344
446
  async function t3AfterBaseline(
345
447
  root: string,
346
- context: { node: string; baseline: Check[]; run: Runner }
448
+ context: {
449
+ node: string;
450
+ baseline: Check[];
451
+ run: Runner;
452
+ restartBusy: boolean;
453
+ }
347
454
  ): Promise<Check[]> {
348
- const { node, baseline, run } = context;
455
+ const { node, baseline, run, restartBusy } = context;
349
456
  if (baseline.some((check) => check.status !== 'pass'))
350
457
  return [
351
458
  {
@@ -358,12 +465,13 @@ async function t3AfterBaseline(
358
465
  },
359
466
  },
360
467
  ];
361
- return attempt('t3:service', () => convergeT3(root, node, run));
468
+ return attempt('t3:service', () => convergeT3(root, node, run, restartBusy));
362
469
  }
363
470
 
364
471
  async function enableWorkerRecovery(
365
472
  root: string,
366
- run: Runner
473
+ run: Runner,
474
+ session: WorkerFeatures['session']
367
475
  ): Promise<Check[]> {
368
476
  const user = userInfo();
369
477
  const ubuntu = resolve(root, 'ansible/ubuntu');
@@ -378,6 +486,7 @@ async function enableWorkerRecovery(
378
486
  development_user: user.username,
379
487
  development_uid: user.uid,
380
488
  development_home: user.homedir,
489
+ ...(session === 'headless' ? { t3_unit: t3Unit(session) } : {}),
381
490
  }),
382
491
  },
383
492
  {
@@ -400,7 +509,7 @@ async function enableWorkerRecovery(
400
509
  id: 'worker:service-recovery',
401
510
  status: result.ok ? 'pass' : 'fail',
402
511
  summary: result.ok
403
- ? 'T3 recovery enabled: a stopped desktop app is started and an unresponsive one restarted after three failed one-minute health checks.'
512
+ ? `T3 recovery enabled: a stopped ${session === 'headless' ? 'server' : 'desktop app'} is started and an unresponsive one restarted after three failed one-minute health checks.`
404
513
  : 'WORKER_RECOVERY_SETUP_FAILED',
405
514
  },
406
515
  ];
@@ -1,6 +1,7 @@
1
1
  import { runLockedCommand as runCommand } from './runner';
2
2
  import { isRevision } from './machine/state';
3
3
  import { convergeLinux } from './node-converge-linux';
4
+ import { T3_RESTART_MODES } from './platform/t3-service';
4
5
  import type { Check } from './types';
5
6
 
6
7
  /** Internal remote entry, run on a worker only. The manager owns the operation
@@ -8,14 +9,23 @@ import type { Check } from './types';
8
9
  * prints child output. Workers are Linux.
9
10
  */
10
11
  async function main(): Promise<Check[]> {
11
- const [node, revision] = process.argv.slice(2);
12
- if (!node || !/^[a-z][a-z0-9-]{0,62}$/.test(node) || !isRevision(revision))
12
+ const [node, revision, t3Restart = 'keep-turns'] = process.argv.slice(2);
13
+ if (
14
+ !node ||
15
+ !/^[a-z][a-z0-9-]{0,62}$/.test(node) ||
16
+ !isRevision(revision) ||
17
+ !(T3_RESTART_MODES as readonly string[]).includes(t3Restart)
18
+ )
13
19
  throw new Error();
14
20
  const root = process.cwd();
15
21
  const head = await runCommand(['git', 'rev-parse', 'HEAD']);
16
22
  if (!head.ok || head.stdout.trim() !== revision) throw new Error();
17
23
  if (process.platform === 'linux')
18
- return convergeLinux(root, { node, revision }, runCommand);
24
+ return convergeLinux(
25
+ root,
26
+ { node, revision, restartBusy: t3Restart === 'restart-busy' },
27
+ runCommand
28
+ );
19
29
  return [
20
30
  {
21
31
  id: 'platform:unsupported',
@@ -19,12 +19,18 @@ export interface Receipt {
19
19
  force?: boolean;
20
20
  /** Receipt directory; defaults to the user's fleet state. */
21
21
  directory?: string;
22
+ /** Run the playbook in check mode first, and apply only when that passes. */
23
+ checkFirst?: boolean;
24
+ /** Where the playbook's variables are written while it runs. */
25
+ scratch?: string;
22
26
  now?: number;
23
27
  }
24
28
 
25
29
  export interface ReceiptResult extends CommandResult {
26
30
  /** True when the playbook was skipped on an unchanged, verified receipt. */
27
31
  cached: boolean;
32
+ /** Which run the result is of: the check-mode run when it failed. */
33
+ stage?: 'check' | 'apply';
28
34
  }
29
35
 
30
36
  export function receiptDirectory(home = homedir()): string {
@@ -56,10 +62,19 @@ export async function runAnsibleWithReceipt(
56
62
  )
57
63
  return { ok: true, reason: 'exit', stdout: '', stderr: '', cached: true };
58
64
  await rm(path, { force: true }).catch(() => undefined);
59
- const result = await runAnsible(run, playbook, options);
65
+ if (receipt.checkFirst) {
66
+ const checked = await runAnsible(
67
+ run,
68
+ playbook,
69
+ { ...options, check: true },
70
+ receipt.scratch
71
+ );
72
+ if (!checked.ok) return { ...checked, cached: false, stage: 'check' };
73
+ }
74
+ const result = await runAnsible(run, playbook, options, receipt.scratch);
60
75
  if (result.ok && key !== null)
61
76
  await record(directory, path, { version: 1, key, appliedAt: now });
62
- return { ...result, cached: false };
77
+ return { ...result, cached: false, stage: 'apply' };
63
78
  }
64
79
 
65
80
  /** Best effort: a missing receipt only means the next run applies in full. */
@@ -30,6 +30,14 @@ export function ansibleCommand(
30
30
  ];
31
31
  }
32
32
 
33
+ /** The package's Ansible configuration (ansible/ansible.cfg): how Ansible
34
+ * reaches a worker over SSH (timeouts, retries, pipelining, connection
35
+ * reuse) beside the options src/ssh-connection.ts gives every session.
36
+ */
37
+ export function ansibleConfig(root = PACKAGE_ROOT): NodeJS.ProcessEnv {
38
+ return { ANSIBLE_CONFIG: join(root, 'ansible', 'ansible.cfg') };
39
+ }
40
+
33
41
  /** Private fleet state on disk, never /tmp: a worker's /tmp is RAM-backed with a
34
42
  * per-user quota that agent caches can exhaust, and the baseline that bounds it
35
43
  * must still be able to run then.
@@ -45,7 +53,11 @@ export function variablesDirectory(home = homedir()): string {
45
53
  export async function runAnsible(
46
54
  run: Runner,
47
55
  playbook: string,
48
- options: CommandOptions & { input: string },
56
+ options: CommandOptions & {
57
+ input: string;
58
+ /** Run in check mode: report what would change, change nothing. */
59
+ check?: boolean;
60
+ },
49
61
  scratch = variablesDirectory()
50
62
  ): Promise<CommandResult> {
51
63
  if (Buffer.byteLength(options.input) > 256 * 1024)
@@ -55,12 +67,36 @@ export async function runAnsible(
55
67
  const path = join(directory, 'variables.json');
56
68
  try {
57
69
  await writeFile(path, options.input, { mode: 0o600, flag: 'wx' });
58
- const { input: _input, ...commandOptions } = options;
70
+ const { input: _input, check, ...commandOptions } = options;
59
71
  return await run(
60
- [...ansibleCommand(), playbook, '--extra-vars', '@' + path],
61
- commandOptions
72
+ [
73
+ ...ansibleCommand(),
74
+ playbook,
75
+ ...(check ? ['--check'] : []),
76
+ '--extra-vars',
77
+ '@' + path,
78
+ ],
79
+ { ...commandOptions, env: { ...ansibleConfig(), ...commandOptions.env } }
62
80
  );
63
81
  } finally {
64
82
  await rm(directory, { recursive: true, force: true });
65
83
  }
66
84
  }
85
+
86
+ /** A playbook of the package, applied from the manager with these variables:
87
+ * one that addresses a worker over SSH itself (ansible/ubuntu/reboot.yml).
88
+ * Scenarios answer it in place of Ansible.
89
+ */
90
+ export type PackagePlaybook = (
91
+ name: string,
92
+ variables: object,
93
+ timeoutMs: number
94
+ ) => Promise<CommandResult>;
95
+
96
+ export function packagePlaybook(run: Runner): PackagePlaybook {
97
+ return (name, variables, timeoutMs) =>
98
+ runAnsible(run, join(PACKAGE_ROOT, 'ansible', name), {
99
+ input: JSON.stringify(variables),
100
+ timeoutMs,
101
+ });
102
+ }
@@ -15,6 +15,14 @@ export interface BootItem {
15
15
  detail: string;
16
16
  }
17
17
 
18
+ /** `notApplicable` names settings the machine lacks (a virtual Mac has no
19
+ * womp): the playbook skips them, so they are not drift.
20
+ */
21
+ export interface PowerDrift {
22
+ drifted: string[];
23
+ notApplicable: string[];
24
+ }
25
+
18
26
  /** Where the manager runs, behind one interface: how it is named, how it is
19
27
  * kept awake, how its services are supervised and where they keep data. The
20
28
  * macOS host is the default and the reference.
@@ -29,8 +37,9 @@ export interface ManagerHost {
29
37
  dataDirectory(home: string): string;
30
38
  /** The machine's local host name, as `manager.id` must name it. */
31
39
  localHostName(run: Runner): Promise<string | null>;
32
- /** The power settings that differ from the manager's policy. */
33
- powerDrift(run: Runner): Promise<string[]>;
40
+ /** The power settings that differ from the manager's policy, and those
41
+ * this machine does not have at all. */
42
+ powerDrift(run: Runner): Promise<PowerDrift>;
34
43
  /** What brings the manager's services back after a power cut, unattended. */
35
44
  bootRecovery(run: Runner, home: string): Promise<BootItem[]>;
36
45
  /** How a person applies what bootRecovery measures. */
@@ -78,10 +87,18 @@ export const MACOS_MANAGER: ManagerHost = {
78
87
  return result.ok ? hostId(result.stdout.trim()) : null;
79
88
  },
80
89
  async powerDrift(run) {
81
- const observed = await acPower(run);
82
- return Object.entries(macos.pmset)
83
- .filter(([key, value]) => observed(key) !== value)
84
- .map(([key]) => key);
90
+ const result = await run(['pmset', '-g', 'custom']);
91
+ const ac = result.ok ? result.stdout.split('AC Power:')[1] : undefined;
92
+ const observed = (key: string) =>
93
+ new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac ?? '')?.[1];
94
+ const absent = (key: string) =>
95
+ ac !== undefined && observed(key) === undefined;
96
+ return {
97
+ drifted: Object.entries(macos.pmset)
98
+ .filter(([key, value]) => !absent(key) && observed(key) !== value)
99
+ .map(([key]) => key),
100
+ notApplicable: Object.keys(macos.pmset).filter(absent),
101
+ };
85
102
  },
86
103
  async bootRecovery(run, home) {
87
104
  const observed = await acPower(run);
@@ -126,7 +143,7 @@ export const MACOS_MANAGER: ManagerHost = {
126
143
  ];
127
144
  },
128
145
  bootRemedy: (workspace) =>
129
- `Apply the manager policy, as an administrator: cd ${workspace} && ${ansibleCommand('darwin').join(' ')} -K ansible/macos/manager.yml (autorestart, Docker Desktop at login), then fleet-sync (LaunchAgents). FileVault and automatic login are a person’s choice: docs/manager-recovery.md.`,
146
+ `fleet-sync applies ansible/macos/manager.yml on every run (autorestart, Docker Desktop at login, LaunchAgents); autorestart needs the one grant an administrator gives: cd ${workspace} && ${ansibleCommand('darwin').join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml. FileVault and automatic login are a person’s choice: docs/manager-recovery.md.`,
130
147
  };
131
148
 
132
149
  /** The sleep states a Linux manager must never enter. */
@@ -153,7 +170,7 @@ export const LINUX_MANAGER: ManagerHost = {
153
170
  const result = await run(['systemctl', 'is-enabled', target]);
154
171
  if (result.stdout.trim() !== 'masked') drifted.push(target);
155
172
  }
156
- return drifted;
173
+ return { drifted, notApplicable: [] };
157
174
  },
158
175
  async bootRecovery(run) {
159
176
  const user = userInfo().username;
@@ -1,4 +1,4 @@
1
- import policy from '../../ansible/ubuntu/policy.json';
1
+ import { threshold } from '../worker-sizing';
2
2
  import type { Check, Runner } from '../types';
3
3
 
4
4
  const AGENT_DROP_IN =
@@ -9,7 +9,6 @@ const GUARDS = [
9
9
  'personal-fleet-disk-gc.timer',
10
10
  ];
11
11
  const MIB = 1024 * 1024;
12
- const FLOOR_MIB = policy.resourceSafety.diskGuardFloorMiB;
13
12
 
14
13
  /** A failure code and remedy, or undefined when the probe passes. */
15
14
  type Probe = (run: Runner) => Promise<[string, string] | undefined>;
@@ -61,17 +60,29 @@ const swapActive: Probe = async (run) => {
61
60
  ];
62
61
  };
63
62
 
63
+ /** The disk guard's floor scales with the filesystem
64
+ * (`sizing.thresholds.diskGuardFloor`); one of unknown size gets the upper
65
+ * bound. */
64
66
  const diskAboveFloor =
65
67
  (path: string): Probe =>
66
68
  async (run) => {
67
- const df = await run(['df', '--output=avail', '-B1', path]);
68
- const free = Number(df.stdout.trim().split('\n').at(-1));
69
- if (!df.ok || !Number.isSafeInteger(free))
69
+ const df = await run(['df', '--output=avail,size', '-B1', path]);
70
+ const [free, size] = (df.stdout.trim().split('\n').at(-1) ?? '')
71
+ .trim()
72
+ .split(/\s+/)
73
+ .map(Number);
74
+ if (!df.ok || free === undefined || !Number.isSafeInteger(free))
70
75
  return ['DISK_FREE_UNREADABLE', `Free space on ${path} is unreadable.`];
71
- if (free >= FLOOR_MIB * MIB) return undefined;
76
+ const floor = threshold(
77
+ 'diskGuardFloor',
78
+ size !== undefined && Number.isSafeInteger(size)
79
+ ? Math.floor(size / MIB)
80
+ : Number.MAX_SAFE_INTEGER
81
+ );
82
+ if (free >= floor * MIB) return undefined;
72
83
  return [
73
84
  'DISK_FREE_BELOW_FLOOR',
74
- `${path} has ${Math.floor(free / MIB)} MiB free, under the ${FLOOR_MIB} MiB floor. The disk GC already removed idle build output and caches; remove stale worktrees or data, or run "sudo systemctl start personal-fleet-disk-gc.service" and read its journal.`,
85
+ `${path} has ${Math.floor(free / MIB)} MiB free, under the ${floor} MiB floor. The disk GC already removed idle build output and caches; remove stale worktrees or data, or run "sudo systemctl start personal-fleet-disk-gc.service" and read its journal.`,
75
86
  ];
76
87
  };
77
88
 
@@ -8,27 +8,40 @@ import { rename, rm, symlink } from 'node:fs/promises';
8
8
  import { userInfo } from 'node:os';
9
9
  import type { Runner, Check } from '../types';
10
10
 
11
+ /** The administrator policy a worker's T3 runs: the fleet's
12
+ * (`ansible/ubuntu/t3/policy.json`, which keeps linked environments on so
13
+ * the manager pairs with a replaced worker without a person) with the
14
+ * worker's provider instances locked.
15
+ */
16
+ export function workerPolicy(instances: Record<string, unknown>) {
17
+ return {
18
+ ...policy,
19
+ settings: {
20
+ ...policy.settings,
21
+ ...Object.fromEntries(
22
+ Object.entries(instances).map(([id, value]) => [
23
+ `providerInstances.${id}`,
24
+ { value, locked: true },
25
+ ])
26
+ ),
27
+ },
28
+ };
29
+ }
30
+
11
31
  export async function stageT3Policy(
12
32
  root: string,
13
33
  executable: string,
14
34
  run: Runner,
15
- node: string
35
+ target: { node: string; restartBusy?: boolean }
16
36
  ): Promise<Check> {
37
+ const { node } = target;
17
38
  const instances = await desired(
18
39
  resolve(root, 'agents/t3.json'),
19
40
  await stableWorkspace(root),
20
41
  node
21
42
  );
22
43
  const user = userInfo();
23
- const document = {
24
- ...policy,
25
- settings: Object.fromEntries(
26
- Object.entries(instances).map(([id, value]) => [
27
- `providerInstances.${id}`,
28
- { value, locked: true },
29
- ])
30
- ),
31
- };
44
+ const document = workerPolicy(instances);
32
45
  const result = await runAnsible(run, 't3-policy.yml', {
33
46
  cwd: resolve(root, 'ansible/ubuntu'),
34
47
  input: JSON.stringify({
@@ -37,6 +50,7 @@ export async function stageT3Policy(
37
50
  development_user: user.username,
38
51
  development_uid: user.uid,
39
52
  development_home: user.homedir,
53
+ t3_restart_busy: target.restartBusy === true,
40
54
  }),
41
55
  timeoutMs: 120_000,
42
56
  });