@crvouga/dev-fleet 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/README.md +65 -9
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +55 -9
  14. package/ansible/macos/manager.yml +154 -7
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/monitoring/agent-darwin.yml +3 -0
  19. package/ansible/monitoring/agent.yml +2 -0
  20. package/ansible/monitoring/hub-darwin.yml +3 -0
  21. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  22. package/ansible/net/remote-probe.sh +4 -1
  23. package/ansible/platforms.json +2 -2
  24. package/ansible/pyproject.toml +5 -0
  25. package/ansible/ubuntu/baseline.yml +39 -13
  26. package/ansible/ubuntu/disk-gc.py +6 -2
  27. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  28. package/ansible/ubuntu/policy.json +125 -6
  29. package/ansible/ubuntu/reboot.yml +89 -0
  30. package/ansible/ubuntu/recovery.yml +13 -3
  31. package/ansible/ubuntu/t3/policy.json +3 -0
  32. package/ansible/ubuntu/t3-desktop.yml +23 -2
  33. package/ansible/ubuntu/t3-headless.yml +128 -0
  34. package/ansible/ubuntu/t3-health.sh +24 -10
  35. package/ansible/ubuntu/t3-idle.py +37 -2
  36. package/ansible/ubuntu/t3-policy.yml +4 -1
  37. package/ansible/ubuntu/t3-route.yml +1 -0
  38. package/ansible/ubuntu/tailscale.yml +54 -8
  39. package/ansible/ubuntu/tasks/browser.yml +3 -0
  40. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  41. package/ansible/ubuntu/tasks/docker.yml +53 -6
  42. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  43. package/ansible/ubuntu/tasks/features.yml +22 -0
  44. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  45. package/ansible/ubuntu/tasks/install.yml +2 -1
  46. package/ansible/ubuntu/tasks/packages.yml +4 -2
  47. package/ansible/ubuntu/tasks/release.yml +6 -0
  48. package/ansible/ubuntu/tasks/resources.yml +6 -4
  49. package/ansible/ubuntu/tasks/services.yml +2 -2
  50. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  51. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  52. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  53. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  54. package/ansible/ubuntu/tasks/user.yml +1 -0
  55. package/ansible/ubuntu/worker-account.yml +117 -0
  56. package/ansible/uv.lock +265 -0
  57. package/bin/devfleet.mjs +42 -4
  58. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  59. package/bootstrap/global-config.ts +29 -3
  60. package/bootstrap/package-smoke.ts +127 -0
  61. package/bootstrap/pins.ts +189 -0
  62. package/bootstrap/release-notes.ts +88 -0
  63. package/bootstrap/release-version.ts +41 -7
  64. package/bootstrap/worker-features.ts +114 -0
  65. package/bootstrap/worker-seed.yaml +7 -3
  66. package/bootstrap/worker-tools.ts +39 -9
  67. package/cliproxy/README.md +1 -1
  68. package/cliproxy/docs/health-workflow.example.yml +20 -143
  69. package/docs/authentication.md +19 -11
  70. package/docs/consumer-setup.md +42 -27
  71. package/docs/convergence.md +67 -0
  72. package/docs/declared.md +61 -34
  73. package/docs/machine-setup.md +73 -7
  74. package/docs/manager-setup.md +10 -3
  75. package/docs/operations.md +64 -9
  76. package/docs/releasing.md +49 -1
  77. package/docs/runbooks.md +98 -7
  78. package/docs/state.md +23 -11
  79. package/docs/support-matrix.md +40 -4
  80. package/docs/topology.md +67 -6
  81. package/docs/ubuntu-sync.md +78 -28
  82. package/docs/worker-contract.md +122 -19
  83. package/docs/worker-requirements.md +107 -0
  84. package/examples/fleet.json +1 -0
  85. package/examples/workers-from-infrastructure/main.tf +132 -0
  86. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  87. package/llms.txt +3 -1
  88. package/mise.lock +29 -0
  89. package/mise.toml +5 -3
  90. package/package.json +2 -2
  91. package/schema/check-ids.json +1967 -170
  92. package/schema/check-ids.schema.json +34 -0
  93. package/schema/fleet.schema.json +327 -8
  94. package/schema/output/check.schema.json +1 -1
  95. package/schema/output/fleet-state-definition.schema.json +14 -0
  96. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  97. package/schema/output/fleet-sync.schema.json +18 -0
  98. package/schema/output/report.schema.json +3 -2
  99. package/schema/worker-record.schema.json +76 -0
  100. package/src/agents/converge.ts +140 -24
  101. package/src/agents/files.ts +2 -32
  102. package/src/agents/native-verify.ts +34 -3
  103. package/src/agents/secrets.ts +39 -4
  104. package/src/check-owners.ts +61 -2
  105. package/src/check-registry.ts +630 -0
  106. package/src/cli/agent-guide.ts +3 -3
  107. package/src/cli/cli.ts +4 -0
  108. package/src/cli/fleet-manifest.ts +9 -3
  109. package/src/cli/fleet-node-lifecycle.ts +313 -64
  110. package/src/cli/fleet-node-park.ts +238 -0
  111. package/src/cli/fleet-node-seed.ts +51 -35
  112. package/src/cli/fleet-state.ts +2 -1
  113. package/src/cli/fleet-sync-checkout.ts +7 -0
  114. package/src/cli/fleet-sync-enrollment.ts +193 -32
  115. package/src/cli/fleet-sync-forward.ts +4 -0
  116. package/src/cli/fleet-sync-manager.ts +103 -33
  117. package/src/cli/fleet-sync-output.ts +31 -11
  118. package/src/cli/fleet-sync-plan.ts +65 -9
  119. package/src/cli/fleet-sync-records.ts +113 -0
  120. package/src/cli/fleet-sync-remote.ts +151 -9
  121. package/src/cli/fleet-sync-schedule.ts +14 -4
  122. package/src/cli/fleet-sync-state.ts +28 -16
  123. package/src/cli/fleet-sync-vault.ts +77 -0
  124. package/src/cli/fleet-sync-worker.ts +214 -0
  125. package/src/cli/fleet-sync.ts +150 -19
  126. package/src/cli/fleet-validate.ts +59 -21
  127. package/src/cli/fleet-watch.ts +7 -1
  128. package/src/cli/fleet.ts +87 -5
  129. package/src/cli/managed-health.ts +25 -1
  130. package/src/cli/register-fleet.ts +115 -4
  131. package/src/cli/registry.ts +14 -0
  132. package/src/cli/tailnet.ts +43 -10
  133. package/src/cli/ui-data.ts +3 -1
  134. package/src/cli/worker-usage-report.ts +63 -0
  135. package/src/clock.ts +42 -0
  136. package/src/definition.ts +332 -9
  137. package/src/diagnostics.ts +5 -2
  138. package/src/exposure.ts +102 -0
  139. package/src/fleet-config.ts +3 -1
  140. package/src/git-identity.ts +76 -3
  141. package/src/ide/ide.ts +5 -0
  142. package/src/ide/t3.ts +12 -2
  143. package/src/input-hashes.ts +6 -0
  144. package/src/machine/machine-class.ts +59 -0
  145. package/src/machine/platform.ts +90 -11
  146. package/src/machine/state.ts +37 -15
  147. package/src/machine/support-matrix.ts +11 -9
  148. package/src/machine-health.ts +28 -5
  149. package/src/manager-health.ts +148 -2
  150. package/src/manager-sync.ts +65 -4
  151. package/src/monitoring/beszel.ts +8 -8
  152. package/src/monitoring/enroll.ts +8 -1
  153. package/src/monitoring/hub-access.ts +10 -4
  154. package/src/monitoring/systems.ts +138 -0
  155. package/src/network-definition.ts +13 -4
  156. package/src/node-converge-linux.ts +136 -27
  157. package/src/node-converge.ts +13 -3
  158. package/src/platform/ansible-receipt.ts +17 -2
  159. package/src/platform/ansible.ts +40 -4
  160. package/src/platform/manager-host.ts +25 -8
  161. package/src/platform/resource-verify.ts +18 -7
  162. package/src/platform/t3-policy.ts +24 -10
  163. package/src/platform/t3-service.ts +109 -34
  164. package/src/platform/ubuntu-release.ts +24 -0
  165. package/src/platform/ubuntu-verify.ts +31 -3
  166. package/src/platform/ubuntu-worker.ts +58 -0
  167. package/src/platform/worker-platform.ts +72 -0
  168. package/src/protocol.ts +1 -1
  169. package/src/public-reachability.ts +186 -0
  170. package/src/remote-bootstrap.ts +50 -12
  171. package/src/remote-operation.ts +18 -4
  172. package/src/repair-plan.ts +23 -12
  173. package/src/report.ts +2 -1
  174. package/src/schedule-check.ts +21 -2
  175. package/src/secrets.ts +4 -0
  176. package/src/service-auth.ts +55 -48
  177. package/src/ssh-connection.ts +141 -0
  178. package/src/ssh.ts +63 -24
  179. package/src/state/store.ts +4 -2
  180. package/src/state/sync.ts +35 -6
  181. package/src/state/workers.ts +140 -0
  182. package/src/t3/pairing.ts +59 -0
  183. package/src/t3/peer-link.ts +333 -0
  184. package/src/t3/version.ts +1 -1
  185. package/src/tailnet-cleanup.ts +185 -0
  186. package/src/tailnet-control.ts +346 -0
  187. package/src/tailscale.ts +1 -1
  188. package/src/toolchain-checks.ts +11 -1
  189. package/src/types.ts +4 -2
  190. package/src/worker-account.ts +63 -0
  191. package/src/worker-activity.ts +303 -0
  192. package/src/worker-capacity.ts +31 -0
  193. package/src/worker-exposure.ts +388 -0
  194. package/src/worker-features.ts +201 -0
  195. package/src/worker-host-checks.ts +71 -20
  196. package/src/worker-manifest.ts +27 -4
  197. package/src/worker-parking.ts +70 -0
  198. package/src/worker-policy.ts +17 -1
  199. package/src/worker-readiness.ts +15 -3
  200. package/src/worker-reboot.ts +129 -0
  201. package/src/worker-records.ts +305 -0
  202. package/src/worker-seed.ts +16 -11
  203. package/src/worker-sizing.ts +76 -0
  204. package/src/cli/schedule-logs.ts +0 -36
@@ -2,7 +2,8 @@ import { createHash } from 'node:crypto';
2
2
  import { IDES } from '../ide';
3
3
  import { enabled, workerChecks } from '../ide/orchestrate';
4
4
  import { idesOf } from '../ide/definition';
5
- import { rememberEnvironment } from '../t3/pairing';
5
+ import { pairIfRequested, rememberEnvironment } from '../t3/pairing';
6
+ import { managerT3 } from '../t3/peer-link';
6
7
  import { authorizePrivilege } from './fleet-sync-privilege';
7
8
  import { authorizeVault } from './fleet-sync-vault';
8
9
  import { managerFailure } from '../manager-failure';
@@ -15,13 +16,32 @@ import { FleetUsageError } from '../errors';
15
16
  import { selectNodes } from '../config';
16
17
  import { loadRoster } from './roster';
17
18
  import { enrolledRoster } from '../definition';
19
+ import { readParked, readWorkerStates } from '../state/workers';
20
+ import { withoutParked } from '../worker-parking';
21
+ import { reconcileRecords } from './fleet-sync-records';
18
22
  import type { FleetDefinition } from '../definition';
19
23
  import { fleet as definition } from '../fleet-config';
20
24
  import { synchronizeManager } from '../manager-sync';
21
25
  import { inspectBootstrap } from '../manager-sync';
22
- import { prepareBootstrap, bootstrapRemote } from '../remote-bootstrap';
26
+ import {
27
+ prepareBootstrap,
28
+ bootstrapRemote,
29
+ readsTailnetKey,
30
+ } from '../remote-bootstrap';
31
+ import { secret } from '../fleet-config';
32
+ import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
33
+ import { discoverTailnet, type TailnetComputer } from '../tailscale';
34
+ import { deployWorker } from './fleet-sync-worker';
35
+ import { packagePlaybook } from '../platform/ansible';
23
36
  import { machineRoot } from './fleet-sync';
24
- import type { Check, Runner } from '../types';
37
+ import {
38
+ blocking,
39
+ type Check,
40
+ type FleetNode,
41
+ type NodeReport,
42
+ type Runner,
43
+ } from '../types';
44
+ import { diagnostic } from '../diagnostics';
25
45
  import type { DeployNode, ManagerReport } from '../manager-sync';
26
46
  import type { FleetSyncOptions } from './fleet-sync';
27
47
  import type { Settings } from './settings';
@@ -39,14 +59,35 @@ export async function selected(
39
59
  if (!options.revision)
40
60
  throw new FleetUsageError('An approved source revision is required.');
41
61
  const canonical = definition;
62
+ // Workers whose record disappeared are retired first (planned in a
63
+ // preview), so they are no longer in the roster.
64
+ const records = await reconcileRecords(canonical, settings.config, run, {
65
+ apply: Boolean(options.apply),
66
+ });
42
67
  let roster;
43
68
  try {
44
- roster = enrolledRoster(canonical, await loadRoster(settings.config));
69
+ const declared = new Set(canonical.workers.map((worker) => worker.id));
70
+ const loaded = await loadRoster(settings.config);
71
+ roster = enrolledRoster(canonical, {
72
+ ...loaded,
73
+ nodes: loaded.nodes.filter(
74
+ (node) =>
75
+ declared.has(node.id) ||
76
+ readWorkerStates().workers[node.id]?.origin !== 'records'
77
+ ),
78
+ });
45
79
  } catch {
46
80
  return enrollmentRequired(canonical, options.revision);
47
81
  }
48
- const nodes = selectNodes(roster, options.node ?? []);
49
- const scoped = scopeDefinition(canonical, options.node);
82
+ // A parked worker is not contacted and never the canary.
83
+ const active = withoutParked(
84
+ scopeDefinition(canonical, options.node),
85
+ selectNodes(roster, options.node ?? []),
86
+ readParked(),
87
+ new Date()
88
+ );
89
+ const { nodes, definition: scoped } = active;
90
+ const notes = [...records, ...active.checks];
50
91
  const ui = interfaceOptions(options);
51
92
  const planFingerprint = createHash('sha256')
52
93
  .update(
@@ -65,12 +106,16 @@ export async function selected(
65
106
  'Manager connection bindings or target policy changed after approval. Inspect a fresh plan and approve it; no source preparation or node mutation occurred.'
66
107
  )
67
108
  );
68
- const payload = options.apply
109
+ const prepared = options.apply
69
110
  ? await prepareBootstrap(machineRoot(), options.revision, run, {
70
111
  definition: canonical,
71
112
  checkout: settings.workspace,
72
113
  })
73
114
  : null;
115
+ const payload =
116
+ prepared && options.restartBusy
117
+ ? { ...prepared, restartBusy: true }
118
+ : prepared;
74
119
  const started = Date.now();
75
120
  const report = await synchronizeManager(
76
121
  {
@@ -79,6 +124,7 @@ export async function selected(
79
124
  revision: options.revision,
80
125
  apply: Boolean(options.apply),
81
126
  ...canary(canonical, options),
127
+ ...serial(options),
82
128
  progress: (node, phase) => ui.progress.step(node, phase),
83
129
  finished: (node) => ui.progress.finished(node),
84
130
  manager: () =>
@@ -97,12 +143,54 @@ export async function selected(
97
143
  ...ui,
98
144
  workspace: settings.workspace,
99
145
  roster: settings.config,
146
+ ...rebootOptions(options),
100
147
  })
101
148
  ).finally(() => ui.progress.stop());
149
+ if (options.enrollment) reportEnrollment(report, options.enrollment);
150
+ report.checks.push(...notes);
151
+ // A worker that could not be retired, like any other failed check of the
152
+ // fleet, means the run did not converge: it is not recorded as applied.
153
+ if (notes.some(blocking)) report.ok = false;
102
154
  if (settle) report.checks.push(await settle(report));
103
155
  return outcome(report, planFingerprint, started);
104
156
  }
105
157
 
158
+ /** Maintenance reboots go one worker at a time, and a worker that does not
159
+ * come back ready stops the others.
160
+ */
161
+ function serial(options: FleetSyncOptions): { serial?: true } {
162
+ return options.maintenanceReboot ? { serial: true } : {};
163
+ }
164
+
165
+ function rebootOptions(options: FleetSyncOptions): {
166
+ readyBefore: string[];
167
+ maintenanceReboot?: true;
168
+ force?: true;
169
+ } {
170
+ return {
171
+ readyBefore: options.readyBefore ?? [],
172
+ ...(options.maintenanceReboot ? { maintenanceReboot: true } : {}),
173
+ ...(options.force ? { force: true } : {}),
174
+ };
175
+ }
176
+
177
+ /** The workers this run enrolled, and why another was not: the checks land
178
+ * on the worker they are about.
179
+ */
180
+ export function reportEnrollment(
181
+ report: ManagerReport,
182
+ enrollment: NonNullable<FleetSyncOptions['enrollment']>
183
+ ): void {
184
+ report.enrolled = enrollment.enrolled;
185
+ for (const { node, check } of enrollment.checks) {
186
+ const entry = report.nodes.find((candidate) => candidate.node === node);
187
+ if (!entry) continue;
188
+ entry.checks.unshift(check);
189
+ entry.diagnostics = entry.checks.map((each) => diagnostic(node, each));
190
+ if (blocking(check)) report.ok = false;
191
+ }
192
+ }
193
+
106
194
  /** A revision new to the fleet is proven on one worker before the rest: the
107
195
  * one `--canary` or fleet.json names, else the first declared worker, and
108
196
  * the next reachable one when that cannot be reached. Repeating a revision
@@ -176,8 +264,15 @@ function deployWithMonitoring(
176
264
  workspace: string;
177
265
  roster: string;
178
266
  passwordLogin: boolean;
267
+ readyBefore: string[];
268
+ maintenanceReboot?: boolean;
269
+ force?: boolean;
179
270
  }
180
271
  ): DeployNode {
272
+ // Asked once per run, and only when a worker needs them.
273
+ let control: Promise<TailnetControl | null> | undefined;
274
+ let computers: Promise<TailnetComputer[]> | undefined;
275
+ const converge = convergeWorker(payload, canonical, run, ui);
181
276
  return async (node, revision) => {
182
277
  if (!payload || payload.revision !== revision)
183
278
  return {
@@ -194,10 +289,50 @@ function deployWithMonitoring(
194
289
  },
195
290
  ],
196
291
  };
292
+ return deployWorker(node, revision, {
293
+ definition: canonical,
294
+ playbook: packagePlaybook(run),
295
+ converge,
296
+ control: () =>
297
+ (control ??= fleetTailnetControl(canonical.tailnet, secret)),
298
+ computers: () =>
299
+ (computers ??= discoverTailnet(run).then((found) => found.computers)),
300
+ readyBefore: ui.readyBefore,
301
+ ...(ui.maintenanceReboot ? { maintenanceReboot: true } : {}),
302
+ ...(ui.force ? { force: true } : {}),
303
+ acceptsTailnetKey: readsTailnetKey(payload.script),
304
+ });
305
+ };
306
+ }
307
+
308
+ /** One converge of a worker: privilege, Vault, the bootstrap, then what is
309
+ * verified from the manager and the worker's monitoring.
310
+ */
311
+ function convergeWorker(
312
+ payload: Awaited<ReturnType<typeof prepareBootstrap>>,
313
+ canonical: FleetDefinition,
314
+ run: Runner,
315
+ ui: {
316
+ progress: SyncProgress;
317
+ interactiveAuthorization: boolean;
318
+ workspace: string;
319
+ roster: string;
320
+ passwordLogin: boolean;
321
+ }
322
+ ) {
323
+ return async (
324
+ node: FleetNode,
325
+ revision: string,
326
+ tailnetKey?: string
327
+ ): Promise<NodeReport> => {
197
328
  // Password prompts need a quiet terminal: no spinner while authorizing.
198
329
  if (ui.interactiveAuthorization) {
199
330
  ui.progress.stop();
200
- const checks = await authorizePrivilege(node, payload.authorizeSudo, run);
331
+ const checks = await authorizePrivilege(
332
+ node,
333
+ payload!.authorizeSudo,
334
+ run
335
+ );
201
336
  if (checks.length) return { node: node.id, checks };
202
337
  }
203
338
  const vault = {
@@ -207,8 +342,9 @@ function deployWithMonitoring(
207
342
  if (ui.interactiveAuthorization) ui.progress.stop();
208
343
  const before = await authorizeVault(node, vault, run);
209
344
  ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
210
- const result = await bootstrapRemote(node, payload, run, {
345
+ const result = await bootstrapRemote(node, payload!, run, {
211
346
  progress: (line) => ui.progress.step(node.id, line),
347
+ ...(tailnetKey ? { tailnetKey } : {}),
212
348
  });
213
349
  // A fresh node gets vault from the bootstrap; log it in now. Its provider
214
350
  // checks in this run predate the login, so the next sync verifies them.
@@ -226,6 +362,12 @@ function deployWithMonitoring(
226
362
  remember: async (id, identity) => {
227
363
  await rememberEnvironment(ui.roster, id, identity);
228
364
  },
365
+ pair: (target) =>
366
+ pairIfRequested(target, {
367
+ roster: ui.roster,
368
+ run,
369
+ manager: managerT3(run),
370
+ }),
229
371
  }))
230
372
  );
231
373
  if (
@@ -9,13 +9,19 @@ import {
9
9
  recordSchedule,
10
10
  type ScheduleRecord,
11
11
  } from '../state/history';
12
- import { scheduleCheck, SCHEDULE_INTERVAL_SECONDS } from '../schedule-check';
12
+ import macos from '../../ansible/macos/policy.json';
13
+ import { fleet } from '../fleet-config';
14
+ import {
15
+ scheduleCheck,
16
+ scheduleIntervals,
17
+ SCHEDULE_INTERVAL_SECONDS,
18
+ } from '../schedule-check';
13
19
  import type { Runner } from '../types';
14
20
  import type { CommandOutcome } from './contracts';
15
21
  import { machineRoot } from './fleet-sync';
16
22
 
17
23
  /** The launchd label and systemd unit already on running managers. */
18
- export const SCHEDULE_LABEL = 'personal-fleet-sync';
24
+ export const SCHEDULE_LABEL = macos.fleetSync.label;
19
25
  const LABEL = SCHEDULE_LABEL;
20
26
  export type ScheduleAction =
21
27
  'enable' | 'pause' | 'resume' | 'disable' | 'status';
@@ -31,6 +37,8 @@ interface ScheduleInput {
31
37
  /** The manager's caches; tests name their own. */
32
38
  root?: string;
33
39
  now?: Date;
40
+ /** How often the schedule runs; `manager.schedule.syncMinutes` otherwise. */
41
+ intervalSeconds?: number;
34
42
  }
35
43
 
36
44
  /** The published package a schedule installs and runs: its name, exact
@@ -205,14 +213,16 @@ async function enableSchedule(
205
213
  return required(
206
214
  'The exact CLI package could not be installed in the managed manager runtime. No schedule was enabled.'
207
215
  );
216
+ const intervalSeconds =
217
+ input.intervalSeconds ?? scheduleIntervals(fleet.manager).syncSeconds;
208
218
  const result = platform.installScheduledJob({
209
219
  label: LABEL,
210
- intervalSeconds: SCHEDULE_INTERVAL_SECONDS,
220
+ intervalSeconds,
211
221
  command: scheduledCommand(published, input.revision, prefix),
212
222
  logDir: join(root, 'logs'),
213
223
  env: scheduledEnvironment(),
214
224
  });
215
- if (result.ok) remember(input, true);
225
+ if (result.ok) remember(input, true, intervalSeconds);
216
226
  return {
217
227
  exitCode: result.ok ? 0 : 1,
218
228
  data: {
@@ -1,6 +1,6 @@
1
1
  import type { Enrollment } from '../machine/state';
2
2
  import { replicaCheck } from '../state/check';
3
- import { storeEnrollment } from '../state/enrollment';
3
+ import { loadEnrollment, storeEnrollment } from '../state/enrollment';
4
4
  import { adoptEarlier } from '../state/legacy';
5
5
  import { databasePath, stateRoot } from '../state/paths';
6
6
  import { writeDocument } from '../state/store';
@@ -23,9 +23,10 @@ export interface RunRecord {
23
23
  approve(): void;
24
24
  /** Once the result is known. The check says whether the bucket holds it.
25
25
  * `applied` is the revision each worker that converged now holds: it is
26
- * kept even when the fleet as a whole is not ready.
26
+ * kept even when the fleet as a whole is not ready, and it is all a run
27
+ * scoped with --node records.
27
28
  */
28
- settle(converged: boolean, applied?: Record<string, string>): Promise<Check>;
29
+ settle(ready: boolean, applied?: Record<string, string>): Promise<Check>;
29
30
  /** For a run that stopped before its result: its approval is still uploaded. */
30
31
  abandon(): Promise<void>;
31
32
  }
@@ -42,6 +43,10 @@ export function runRecord(
42
43
  * worker, a replacement manager) uses the one last applied.
43
44
  */
44
45
  definition?: string;
46
+ /** A run scoped with --node covers only the workers it selected: it
47
+ * records their revisions, never one applied for the whole fleet.
48
+ */
49
+ scoped?: boolean;
45
50
  }
46
51
  ): RunRecord {
47
52
  const env = state.env ?? process.env;
@@ -73,26 +78,33 @@ export function runRecord(
73
78
  }
74
79
  claimLease(state.context);
75
80
  },
76
- async settle(converged, applied = {}) {
81
+ async settle(ready, applied = {}) {
77
82
  settled = true;
78
- const workers = Object.keys(applied).length
79
- ? { workers: { ...previous?.workers, ...applied } }
80
- : {};
81
- if (!converged && workers.workers)
82
- storeEnrollment({ ...approval, ...workers }, env);
83
- if (converged)
83
+ const converged = ready && state.scoped !== true;
84
+ if (converged || Object.keys(applied).length) {
85
+ // What was recorded since the approval stands: the run itself
86
+ // retires a worker whose record was removed, and another command
87
+ // may have. Only what this run decided is written over it.
88
+ const stored = (await loadEnrollment(null, env)) ?? approval;
84
89
  storeEnrollment(
85
90
  {
86
- ...approval,
87
- ...workers,
88
- appliedRevision: revision,
89
- ...(previous?.appliedRevision &&
90
- previous.appliedRevision !== revision
91
- ? { previousRevision: previous.appliedRevision }
91
+ ...stored,
92
+ ...(Object.keys(applied).length
93
+ ? { workers: { ...stored.workers, ...applied } }
94
+ : {}),
95
+ ...(converged
96
+ ? {
97
+ appliedRevision: revision,
98
+ ...(previous?.appliedRevision &&
99
+ previous.appliedRevision !== revision
100
+ ? { previousRevision: previous.appliedRevision }
101
+ : {}),
102
+ }
92
103
  : {}),
93
104
  },
94
105
  env
95
106
  );
107
+ }
96
108
  // The fleet converged either way: a bucket problem is reported, not fatal.
97
109
  return replicaCheck(await publish().catch(() => 'unreachable' as const), {
98
110
  advisory: true,
@@ -1,8 +1,11 @@
1
1
  import { spawnSync } from 'node:child_process';
2
+ import { homedir } from 'node:os';
3
+ import { join } from 'node:path';
2
4
  import { fleet as definition } from '../fleet-config';
3
5
  import { sshArgs, shellQuote } from '../ssh';
4
6
  import { resolveVaultConfig, usesVault } from '../vault-config';
5
7
  import { workerTokenLocation } from '../worker-credentials';
8
+ import { runAnsible } from '../platform/ansible';
6
9
  import type { Check, FleetNode, Runner } from '../types';
7
10
 
8
11
  const TOOL_PATH =
@@ -213,3 +216,77 @@ function required(node: FleetNode): Check {
213
216
  },
214
217
  };
215
218
  }
219
+
220
+ /**
221
+ * The manager's own Vault session renews itself every ten minutes, like each
222
+ * worker's: `ansible/macos/vault-renew.yml`, applied on every applying sync.
223
+ * Measured as the session answering and the job being loaded; a lapsed
224
+ * session needs one login by a person, and until then agents on the manager
225
+ * use the proxy client key they last read (src/agents/secrets.ts).
226
+ */
227
+ export async function holdManagerVault(
228
+ run: Runner,
229
+ options: {
230
+ apply: boolean;
231
+ root: string;
232
+ home?: string;
233
+ uid?: number;
234
+ scratch?: string;
235
+ }
236
+ ): Promise<Check[]> {
237
+ if (!usesVault()) return [];
238
+ const { addr } = resolveVaultConfig();
239
+ const home = options.home ?? homedir();
240
+ if (options.apply)
241
+ await runAnsible(
242
+ run,
243
+ join(options.root, 'ansible/macos/vault-renew.yml'),
244
+ {
245
+ input: JSON.stringify({
246
+ vault_address: addr,
247
+ mise: join(home, '.local/bin/mise'),
248
+ }),
249
+ timeoutMs: 120_000,
250
+ },
251
+ options.scratch
252
+ );
253
+ const loaded = (
254
+ await run(
255
+ [
256
+ 'launchctl',
257
+ 'print',
258
+ `gui/${options.uid ?? process.getuid?.() ?? 0}/${MANAGER_VAULT_RENEWAL}`,
259
+ ],
260
+ { timeoutMs: 10_000 }
261
+ )
262
+ ).ok;
263
+ const session = (await run(lookup(), { timeoutMs: 20_000 })).ok;
264
+ return [vaultRenewalCheck(session, loaded, addr)];
265
+ }
266
+
267
+ function vaultRenewalCheck(
268
+ session: boolean,
269
+ loaded: boolean,
270
+ addr: string
271
+ ): Check {
272
+ if (session && loaded)
273
+ return {
274
+ id: 'manager:vault-renewal',
275
+ status: 'pass',
276
+ summary:
277
+ 'The manager is logged in to Vault and renews its session every ten minutes.',
278
+ };
279
+ return {
280
+ id: 'manager:vault-renewal',
281
+ status: 'fail',
282
+ summary: session ? 'VAULT_RENEWAL_MISSING' : 'VAULT_LOGIN_REQUIRED',
283
+ remedy: {
284
+ action: session
285
+ ? `The manager's Vault session is not renewed and lapses at its TTL. Run devfleet fleet-sync on the manager; it loads ${MANAGER_VAULT_RENEWAL}.`
286
+ : `Log the manager in once, in your own terminal: vault login -address=${addr} -method=userpass username=${definition.worker?.vaultUser ?? 'YOUR_USERNAME'}. Then run devfleet fleet-sync. Until then the manager's agents use the proxy client key they last read, and new worker tokens cannot be issued.`,
287
+ },
288
+ };
289
+ }
290
+
291
+ /** The launchd label of the manager's Vault renewal job (vault-renew.yml). */
292
+ export const MANAGER_VAULT_RENEWAL = 'dev.devfleet.vault-renew';
@@ -0,0 +1,214 @@
1
+ import type { FleetDefinition } from '../definition';
2
+ import {
3
+ controlRemedy,
4
+ TailnetControlError,
5
+ type TailnetControl,
6
+ } from '../tailnet-control';
7
+ import type { TailnetComputer } from '../tailscale';
8
+ import type { PackagePlaybook } from '../platform/ansible';
9
+ import type { Check, FleetNode, NodeReport } from '../types';
10
+ import { createWorkerAccount } from '../worker-account';
11
+ import {
12
+ rebootNeeded,
13
+ rebootWorker,
14
+ updatesPending,
15
+ type RebootReason,
16
+ } from '../worker-reboot';
17
+
18
+ /** What one worker's deployment needs around its converge: the converge
19
+ * itself (bootstrap, verification, monitoring), the tailnet control plane
20
+ * when the fleet declares one, the tailnet as the manager sees it, and what
21
+ * the run was asked to do.
22
+ */
23
+ export interface WorkerDeployment {
24
+ definition: FleetDefinition;
25
+ /** Applies a playbook of the package from the manager. */
26
+ playbook: PackagePlaybook;
27
+ /** One converge of the worker; `tailnetKey` is handed to its bootstrap. */
28
+ converge: (
29
+ node: FleetNode,
30
+ revision: string,
31
+ tailnetKey?: string
32
+ ) => Promise<NodeReport>;
33
+ /** Null when the fleet declares no tailnet control plane. */
34
+ control: () => Promise<TailnetControl | null>;
35
+ /** The devices of the tailnet, as the manager's Tailscale lists them. */
36
+ computers: () => Promise<TailnetComputer[]>;
37
+ /** The workers that converged in an earlier run. */
38
+ readyBefore: string[];
39
+ maintenanceReboot?: boolean;
40
+ force?: boolean;
41
+ /** Whether the bootstrap of this revision reads a tailnet key. */
42
+ acceptsTailnetKey: boolean;
43
+ }
44
+
45
+ /** Deploys one worker and, around its converge: hands a worker that is not
46
+ * on the tailnet a single-use key, keeps an enrolled worker's node key from
47
+ * expiring, creates the declared account on an image that lets only root
48
+ * in, and reboots the worker once when its first converge needs it (or for
49
+ * maintenance, when asked), converging it again to verify it.
50
+ */
51
+ export async function deployWorker(
52
+ node: FleetNode,
53
+ revision: string,
54
+ deployment: WorkerDeployment
55
+ ): Promise<NodeReport> {
56
+ const before: Check[] = [];
57
+ const key = await enrolmentKey(node, deployment, before);
58
+ before.push(...(await keyExpiry(node, deployment)));
59
+ let target = node;
60
+ let result = await deployment.converge(target, revision, key);
61
+ const account = await rootOnly(node, result, deployment);
62
+ if (account) {
63
+ before.push(account.check);
64
+ if (account.node) {
65
+ target = account.node;
66
+ result = await deployment.converge(target, revision, key);
67
+ }
68
+ }
69
+ const reason = rebootReason(node.id, result.checks, deployment);
70
+ if (reason) {
71
+ const reboot = await rebootWorker(
72
+ target,
73
+ {
74
+ revision,
75
+ reason,
76
+ ...(reason === 'maintenance' && deployment.force
77
+ ? { force: true }
78
+ : {}),
79
+ },
80
+ deployment.playbook
81
+ );
82
+ if (reboot.status === 'pass')
83
+ result = await deployment.converge(target, revision);
84
+ result.checks.unshift(reboot);
85
+ }
86
+ result.checks.unshift(...before);
87
+ return result;
88
+ }
89
+
90
+ /** Why this worker is rebooted now, if it is. A worker that converged
91
+ * before is never rebooted for its first converge; only an explicit
92
+ * maintenance run reboots it.
93
+ */
94
+ function rebootReason(
95
+ id: string,
96
+ checks: Check[],
97
+ deployment: WorkerDeployment
98
+ ): RebootReason | null {
99
+ if (deployment.maintenanceReboot && updatesPending(checks))
100
+ return 'maintenance';
101
+ if (!deployment.readyBefore.includes(id) && rebootNeeded(checks))
102
+ return 'first-converge';
103
+ return null;
104
+ }
105
+
106
+ /** A single-use, pre-authorised, tagged key for a worker the roster has no
107
+ * tailnet identity for yet. The key is never reported.
108
+ */
109
+ async function enrolmentKey(
110
+ node: FleetNode,
111
+ deployment: WorkerDeployment,
112
+ checks: Check[]
113
+ ): Promise<string | undefined> {
114
+ if (node.tailscaleId || !deployment.acceptsTailnetKey) return undefined;
115
+ try {
116
+ const control = await deployment.control();
117
+ if (!control) return undefined;
118
+ const minted = await control.createAuthKey({ worker: node.id });
119
+ checks.push({
120
+ id: 'tailscale:enrolment-key',
121
+ status: 'pass',
122
+ summary: `A single-use, pre-authorised key tagged ${deployment.definition.tailnet!.tag}, expiring within minutes, was handed to the worker on the bootstrap's stdin.`,
123
+ });
124
+ return minted.key;
125
+ } catch (error) {
126
+ if (!(error instanceof TailnetControlError)) throw error;
127
+ checks.push(controlFailure('tailscale:enrolment-key', error));
128
+ return undefined;
129
+ }
130
+ }
131
+
132
+ /** An enrolled worker's node key never expires: the control plane disables
133
+ * expiry for a worker whose key has one (tagged workers have none on the
134
+ * hosted Tailscale). The tailscale:key-expiry check still measures it.
135
+ */
136
+ async function keyExpiry(
137
+ node: FleetNode,
138
+ deployment: WorkerDeployment
139
+ ): Promise<Check[]> {
140
+ if (!node.tailscaleId) return [];
141
+ try {
142
+ const control = await deployment.control();
143
+ if (!control) return [];
144
+ const peer = (await deployment.computers()).find(
145
+ (computer) => computer.id === node.tailscaleId
146
+ );
147
+ if (peer?.keyExpiry === undefined) return [];
148
+ await control.disableKeyExpiry(node.tailscaleId);
149
+ return [
150
+ {
151
+ id: 'tailscale:key-expiry-policy',
152
+ status: 'pass',
153
+ summary:
154
+ 'Key expiry of this worker was disabled through the tailnet control plane; tailscale:key-expiry measures it again on the next run.',
155
+ },
156
+ ];
157
+ } catch (error) {
158
+ if (!(error instanceof TailnetControlError)) throw error;
159
+ return [
160
+ {
161
+ ...controlFailure('tailscale:key-expiry-policy', error),
162
+ status: 'warn',
163
+ },
164
+ ];
165
+ }
166
+ }
167
+
168
+ function controlFailure(id: string, error: TailnetControlError): Check {
169
+ return {
170
+ id,
171
+ status: 'fail',
172
+ summary: error.code,
173
+ remedy: { action: controlRemedy(error.code) },
174
+ };
175
+ }
176
+
177
+ /** The bootstrap refused to run as root: the alias logs in as root. With a
178
+ * declared account, it is created over that login and the run continues as
179
+ * it (`user@alias`); the next runs need the alias to name it, since the SSH
180
+ * policy turns root login off.
181
+ */
182
+ async function rootOnly(
183
+ node: FleetNode,
184
+ result: NodeReport,
185
+ deployment: WorkerDeployment
186
+ ): Promise<{ check: Check; node?: FleetNode } | null> {
187
+ if (
188
+ !result.checks.some(
189
+ (check) => check.summary === 'ROOT_DEVELOPMENT_USER_REJECTED'
190
+ )
191
+ )
192
+ return null;
193
+ const user = deployment.definition.workers.find(
194
+ (worker) => worker.id === node.id
195
+ )?.network?.user;
196
+ const check = await createWorkerAccount(
197
+ { id: node.id, ssh: node.ssh, login: '' },
198
+ user,
199
+ deployment.playbook
200
+ );
201
+ if (check.status !== 'pass') return { check };
202
+ return {
203
+ check: {
204
+ ...check,
205
+ status: 'warn',
206
+ summary: `${check.summary} This run continued as ${user}@${node.ssh}; set User ${user} for the SSH alias ${node.ssh}, since root login is turned off.`,
207
+ },
208
+ node: {
209
+ ...node,
210
+ ssh: `${user}@${node.ssh}`,
211
+ workspace: `/home/${user}/.local/share/personal/fleet`,
212
+ },
213
+ };
214
+ }