@crvouga/dev-fleet 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +88 -7
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +83 -0
  14. package/ansible/macos/manager.yml +164 -9
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/macos/worker-host-key.yml +33 -0
  19. package/ansible/monitoring/agent-darwin.yml +3 -0
  20. package/ansible/monitoring/agent.yml +2 -0
  21. package/ansible/monitoring/hub-darwin.yml +3 -0
  22. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  23. package/ansible/net/remote-probe.sh +4 -1
  24. package/ansible/platforms.json +2 -2
  25. package/ansible/pyproject.toml +5 -0
  26. package/ansible/ubuntu/baseline.yml +39 -13
  27. package/ansible/ubuntu/disk-gc.py +6 -2
  28. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  29. package/ansible/ubuntu/policy.json +125 -6
  30. package/ansible/ubuntu/reboot.yml +89 -0
  31. package/ansible/ubuntu/recovery.yml +13 -3
  32. package/ansible/ubuntu/t3/policy.json +3 -0
  33. package/ansible/ubuntu/t3-desktop.yml +23 -2
  34. package/ansible/ubuntu/t3-headless.yml +128 -0
  35. package/ansible/ubuntu/t3-health.sh +24 -10
  36. package/ansible/ubuntu/t3-idle.py +37 -2
  37. package/ansible/ubuntu/t3-policy.yml +4 -1
  38. package/ansible/ubuntu/t3-route.yml +1 -0
  39. package/ansible/ubuntu/tailscale.yml +54 -8
  40. package/ansible/ubuntu/tasks/browser.yml +3 -0
  41. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  42. package/ansible/ubuntu/tasks/docker.yml +53 -6
  43. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  44. package/ansible/ubuntu/tasks/features.yml +22 -0
  45. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  46. package/ansible/ubuntu/tasks/install.yml +2 -1
  47. package/ansible/ubuntu/tasks/packages.yml +4 -2
  48. package/ansible/ubuntu/tasks/release.yml +6 -0
  49. package/ansible/ubuntu/tasks/resources.yml +6 -4
  50. package/ansible/ubuntu/tasks/services.yml +2 -2
  51. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  52. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  53. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  54. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  55. package/ansible/ubuntu/tasks/user.yml +1 -0
  56. package/ansible/ubuntu/worker-account.yml +117 -0
  57. package/ansible/uv.lock +265 -0
  58. package/bin/devfleet.mjs +42 -4
  59. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  60. package/bootstrap/global-config.ts +29 -3
  61. package/bootstrap/package-smoke.ts +127 -0
  62. package/bootstrap/pins.ts +189 -0
  63. package/bootstrap/release-notes.ts +88 -0
  64. package/bootstrap/release-version.ts +41 -7
  65. package/bootstrap/worker-features.ts +114 -0
  66. package/bootstrap/worker-seed.yaml +65 -0
  67. package/bootstrap/worker-tools.ts +39 -9
  68. package/cliproxy/README.md +1 -1
  69. package/cliproxy/docs/health-workflow.example.yml +20 -143
  70. package/docs/authentication.md +21 -11
  71. package/docs/consumer-setup.md +74 -58
  72. package/docs/convergence.md +67 -0
  73. package/docs/declared.md +61 -34
  74. package/docs/machine-setup.md +73 -7
  75. package/docs/manager-setup.md +34 -0
  76. package/docs/operations.md +64 -9
  77. package/docs/releasing.md +49 -1
  78. package/docs/runbooks.md +112 -7
  79. package/docs/state.md +23 -11
  80. package/docs/support-matrix.md +40 -4
  81. package/docs/topology.md +82 -12
  82. package/docs/ubuntu-sync.md +98 -42
  83. package/docs/worker-contract.md +122 -19
  84. package/docs/worker-requirements.md +107 -0
  85. package/examples/fleet.json +1 -0
  86. package/examples/workers-from-infrastructure/main.tf +132 -0
  87. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  88. package/llms.txt +3 -1
  89. package/mise.lock +29 -0
  90. package/mise.toml +5 -3
  91. package/package.json +2 -2
  92. package/schema/check-ids.json +1991 -162
  93. package/schema/check-ids.schema.json +34 -0
  94. package/schema/fleet.schema.json +349 -10
  95. package/schema/output/check.schema.json +1 -1
  96. package/schema/output/fleet-state-definition.schema.json +14 -0
  97. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  98. package/schema/output/fleet-sync.schema.json +18 -0
  99. package/schema/output/report.schema.json +3 -2
  100. package/schema/worker-record.schema.json +76 -0
  101. package/src/agents/converge.ts +140 -24
  102. package/src/agents/files.ts +2 -32
  103. package/src/agents/native-verify.ts +34 -3
  104. package/src/agents/secrets.ts +39 -4
  105. package/src/check-owners.ts +62 -2
  106. package/src/check-registry.ts +630 -0
  107. package/src/cli/agent-guide.ts +3 -3
  108. package/src/cli/cli.ts +4 -0
  109. package/src/cli/fleet-manifest.ts +108 -0
  110. package/src/cli/fleet-node-lifecycle.ts +704 -0
  111. package/src/cli/fleet-node-park.ts +238 -0
  112. package/src/cli/fleet-node-seed.ts +167 -0
  113. package/src/cli/fleet-state.ts +2 -1
  114. package/src/cli/fleet-sync-checkout.ts +7 -0
  115. package/src/cli/fleet-sync-enrollment.ts +193 -32
  116. package/src/cli/fleet-sync-forward.ts +6 -0
  117. package/src/cli/fleet-sync-manager.ts +123 -8
  118. package/src/cli/fleet-sync-output.ts +31 -11
  119. package/src/cli/fleet-sync-plan.ts +65 -9
  120. package/src/cli/fleet-sync-records.ts +113 -0
  121. package/src/cli/fleet-sync-remote.ts +172 -15
  122. package/src/cli/fleet-sync-schedule.ts +14 -4
  123. package/src/cli/fleet-sync-state.ts +31 -10
  124. package/src/cli/fleet-sync-vault.ts +152 -26
  125. package/src/cli/fleet-sync-worker.ts +214 -0
  126. package/src/cli/fleet-sync.ts +172 -21
  127. package/src/cli/fleet-validate.ts +277 -0
  128. package/src/cli/fleet-watch.ts +7 -1
  129. package/src/cli/fleet.ts +87 -5
  130. package/src/cli/managed-health.ts +25 -1
  131. package/src/cli/register-fleet.ts +251 -4
  132. package/src/cli/registry.ts +47 -0
  133. package/src/cli/tailnet.ts +43 -10
  134. package/src/cli/ui-data.ts +3 -1
  135. package/src/cli/worker-usage-report.ts +63 -0
  136. package/src/clock.ts +42 -0
  137. package/src/definition.ts +762 -232
  138. package/src/diagnostics.ts +5 -2
  139. package/src/exposure.ts +102 -0
  140. package/src/fleet-config.ts +3 -1
  141. package/src/git-identity.ts +76 -3
  142. package/src/ide/ide.ts +5 -0
  143. package/src/ide/t3.ts +12 -2
  144. package/src/input-hashes.ts +6 -0
  145. package/src/machine/machine-class.ts +59 -0
  146. package/src/machine/platform.ts +90 -11
  147. package/src/machine/state.ts +70 -16
  148. package/src/machine/support-matrix.ts +11 -9
  149. package/src/machine-health.ts +28 -5
  150. package/src/manager-health.ts +148 -2
  151. package/src/manager-sync.ts +89 -8
  152. package/src/monitoring/beszel.ts +8 -8
  153. package/src/monitoring/enroll.ts +8 -1
  154. package/src/monitoring/hub-access.ts +10 -4
  155. package/src/monitoring/systems.ts +138 -0
  156. package/src/network-definition.ts +13 -4
  157. package/src/node-converge-linux.ts +136 -27
  158. package/src/node-converge.ts +13 -3
  159. package/src/platform/ansible-receipt.ts +17 -2
  160. package/src/platform/ansible.ts +40 -4
  161. package/src/platform/manager-host.ts +25 -8
  162. package/src/platform/resource-verify.ts +18 -7
  163. package/src/platform/t3-policy.ts +24 -10
  164. package/src/platform/t3-service.ts +109 -34
  165. package/src/platform/ubuntu-release.ts +24 -0
  166. package/src/platform/ubuntu-verify.ts +31 -3
  167. package/src/platform/ubuntu-worker.ts +58 -0
  168. package/src/platform/worker-platform.ts +72 -0
  169. package/src/protocol.ts +1 -1
  170. package/src/public-reachability.ts +186 -0
  171. package/src/remote-bootstrap.ts +50 -12
  172. package/src/remote-operation.ts +18 -4
  173. package/src/repair-plan.ts +23 -12
  174. package/src/report.ts +2 -1
  175. package/src/schedule-check.ts +21 -2
  176. package/src/secrets.ts +7 -0
  177. package/src/service-auth.ts +55 -48
  178. package/src/ssh-connection.ts +141 -0
  179. package/src/ssh.ts +63 -24
  180. package/src/state/store.ts +4 -2
  181. package/src/state/sync.ts +35 -6
  182. package/src/state/workers.ts +140 -0
  183. package/src/t3/pairing.ts +59 -0
  184. package/src/t3/peer-link.ts +333 -0
  185. package/src/t3/version.ts +1 -1
  186. package/src/tailnet-cleanup.ts +185 -0
  187. package/src/tailnet-control.ts +346 -0
  188. package/src/tailscale.ts +1 -1
  189. package/src/toolchain-checks.ts +11 -1
  190. package/src/types.ts +4 -2
  191. package/src/worker-account.ts +63 -0
  192. package/src/worker-activity.ts +303 -0
  193. package/src/worker-capacity.ts +31 -0
  194. package/src/worker-credentials.ts +201 -0
  195. package/src/worker-exposure.ts +388 -0
  196. package/src/worker-features.ts +201 -0
  197. package/src/worker-host-checks.ts +71 -20
  198. package/src/worker-manifest.ts +224 -0
  199. package/src/worker-parking.ts +70 -0
  200. package/src/worker-policy.ts +38 -36
  201. package/src/worker-readiness.ts +15 -3
  202. package/src/worker-reboot.ts +129 -0
  203. package/src/worker-records.ts +305 -0
  204. package/src/worker-seed.ts +143 -0
  205. package/src/worker-sizing.ts +76 -0
  206. package/src/cli/schedule-logs.ts +0 -36
@@ -1,6 +1,7 @@
1
1
  import { join } from 'node:path';
2
2
  import type { Standing } from '../state/sync';
3
3
  import type { Runner } from '../types';
4
+ import { featureRemovals, workerFeatures } from '../worker-features';
4
5
 
5
6
  /** A tool whose pin changes between the applied and the selected revision. */
6
7
  export interface ToolChange {
@@ -31,6 +32,9 @@ export interface SyncPlan {
31
32
  tools: ToolChange[];
32
33
  restarts: Restart[];
33
34
  nodes: { node: string; from: string | null; to: string; changes: boolean }[];
35
+ /** What a sync removes from a worker because the fleet configuration
36
+ * turned one of its features off since it was last applied. */
37
+ removals: { node: string; removes: string[] }[];
34
38
  }
35
39
 
36
40
  /** The coding agents' CLIs: a new version is used by the next session, and a
@@ -84,16 +88,24 @@ export function toolChanges(
84
88
  }
85
89
 
86
90
  /** What restarts follow from the changes, as the playbooks and the toolchain
87
- * setup act on them.
91
+ * setup act on them. A worker's T3 restarts only while it runs no agent turn
92
+ * (`ansible/ubuntu/tasks/t3-restart.yml`) unless `--restart-busy` asks for it
93
+ * anyway: only then do running turns end.
88
94
  */
89
- export function restarts(tools: ToolChange[], files: string[]): Restart[] {
95
+ export function restarts(
96
+ tools: ToolChange[],
97
+ files: string[],
98
+ restartBusy = false
99
+ ): Restart[] {
90
100
  const found: Restart[] = [];
101
+ const when = restartBusy
102
+ ? 'at once (--restart-busy): its running agent turns end.'
103
+ : 'once it runs no agent turn: running turns keep going and the restart is pending until then.';
91
104
  if (tools.some((change) => change.tool === 't3'))
92
105
  found.push({
93
106
  service: 't3',
94
- reason:
95
- 'T3 is upgraded and restarted on every machine; its running agent turns end.',
96
- endsRunningTurns: true,
107
+ reason: `T3 is upgraded on every worker and restarted ${when}`,
108
+ endsRunningTurns: restartBusy,
97
109
  });
98
110
  else if (
99
111
  files.some((file) =>
@@ -102,9 +114,8 @@ export function restarts(tools: ToolChange[], files: string[]): Restart[] {
102
114
  )
103
115
  found.push({
104
116
  service: 't3',
105
- reason:
106
- 'The managed T3 policy may change: a T3 that has not loaded it is restarted, and its running agent turns end.',
107
- endsRunningTurns: true,
117
+ reason: `The managed T3 policy may change: a T3 that has not loaded it is restarted ${when}`,
118
+ endsRunningTurns: restartBusy,
108
119
  });
109
120
  const agents = tools.filter((change) => AGENT_TOOLS.includes(change.tool));
110
121
  if (agents.length)
@@ -136,6 +147,11 @@ export async function syncPlan(
136
147
  state: Standing;
137
148
  nodes: string[];
138
149
  sources: string[];
150
+ /** `--restart-busy`: a T3 restart does not wait for idle. */
151
+ restartBusy?: boolean;
152
+ /** The fleet configuration last applied (from the fleet state), when
153
+ * there is one, and the one this sync applies. */
154
+ definitions?: { applied: unknown; selected: unknown };
139
155
  },
140
156
  run: Runner
141
157
  ): Promise<SyncPlan> {
@@ -163,16 +179,54 @@ export async function syncPlan(
163
179
  revision: { from: input.from, to: input.to },
164
180
  known: changes !== null,
165
181
  tools: changes?.tools ?? [],
166
- restarts: changes ? restarts(changes.tools, changes.files) : [],
182
+ restarts: changes
183
+ ? restarts(changes.tools, changes.files, input.restartBusy === true)
184
+ : [],
167
185
  nodes: input.nodes.map((node) => ({
168
186
  node,
169
187
  from: input.from,
170
188
  to: input.to,
171
189
  changes: differs,
172
190
  })),
191
+ removals: input.definitions ? removals(input.nodes, input.definitions) : [],
173
192
  };
174
193
  }
175
194
 
195
+ /** Per worker, what turning its features off removes. Nothing is known to
196
+ * be removed from a worker the applied configuration does not name. */
197
+ function removals(
198
+ nodes: string[],
199
+ definitions: { applied: unknown; selected: unknown }
200
+ ): SyncPlan['removals'] {
201
+ const workers = (definition: unknown): Record<string, unknown>[] => {
202
+ const list =
203
+ typeof definition === 'object' && definition !== null
204
+ ? (definition as Record<string, unknown>)['workers']
205
+ : undefined;
206
+ return Array.isArray(list)
207
+ ? list.filter(
208
+ (entry): entry is Record<string, unknown> =>
209
+ typeof entry === 'object' && entry !== null
210
+ )
211
+ : [];
212
+ };
213
+ return nodes.flatMap((node) => {
214
+ const before = workers(definitions.applied).find(
215
+ (entry) => entry['id'] === node
216
+ );
217
+ if (!before) return [];
218
+ const network = workers(definitions.selected).find(
219
+ (entry) => entry['id'] === node
220
+ )?.['network'] as { user?: unknown } | undefined;
221
+ const removes = featureRemovals(
222
+ workerFeatures(definitions.applied, node),
223
+ workerFeatures(definitions.selected, node),
224
+ typeof network?.user === 'string' ? network.user : 'the worker user'
225
+ );
226
+ return removes.length ? [{ node, removes }] : [];
227
+ });
228
+ }
229
+
176
230
  export const planSources = (workspace: string, root: string): string[] => [
177
231
  workspace,
178
232
  join(root, 'source'),
@@ -196,5 +250,7 @@ export function planText(plan: SyncPlan): string {
196
250
  );
197
251
  for (const restart of plan.restarts)
198
252
  lines.push(` Restarts ${restart.service}: ${restart.reason}`);
253
+ for (const removal of plan.removals ?? [])
254
+ lines.push(` Removes from ${removal.node}: ${removal.removes.join(', ')}`);
199
255
  return lines.join('\n');
200
256
  }
@@ -0,0 +1,113 @@
1
+ import type { FleetDefinition, MachineDefinition } from '../definition';
2
+ import { readRosterText } from '../state/roster';
3
+ import { changeWorkers, readWorkerStates } from '../state/workers';
4
+ import { parseConfig } from '../config';
5
+ import type { Check, FleetNode, Runner } from '../types';
6
+ import {
7
+ lifecycleServices,
8
+ retireWorker,
9
+ type NodeLifecycleServices,
10
+ } from './fleet-node-lifecycle';
11
+
12
+ /** The workers of the fleet as worker records describe them come and go
13
+ * with the records: a record that appears is a declared worker, enrolled by
14
+ * the next sync like any other; one that disappears is retired by the same
15
+ * path as fleet-node-retire. fleet-sync calls this before it reads the
16
+ * roster. It records which workers came from records, so that only those
17
+ * are ever retired by themselves; a worker removed from fleet.json by hand
18
+ * must still be retired first.
19
+ */
20
+ export async function reconcileRecords(
21
+ definition: FleetDefinition,
22
+ roster: string,
23
+ run: Runner,
24
+ {
25
+ apply,
26
+ services = lifecycleServices,
27
+ }: {
28
+ apply: boolean;
29
+ services?: (roster: string) => NodeLifecycleServices;
30
+ }
31
+ ): Promise<Check[]> {
32
+ const states = readWorkerStates().workers;
33
+ const declared = new Set(definition.workers.map((worker) => worker.id));
34
+ const vanished = Object.entries(states)
35
+ .filter(([id, entry]) => entry.origin === 'records' && !declared.has(id))
36
+ .map(([id]) => id);
37
+ const entries = await rosterEntries(roster);
38
+ const lifecycle = { ...services(roster), run };
39
+ const checks: Check[] = [];
40
+ for (const id of vanished) {
41
+ const retired = await retireWorker(
42
+ gone(id, entries.get(id)),
43
+ entries.get(id),
44
+ lifecycle,
45
+ apply
46
+ );
47
+ const steps = (retired.data as { nodes: { checks: Check[] }[] }).nodes[0]!
48
+ .checks;
49
+ const failed = steps.filter((step) => step.status === 'fail');
50
+ checks.push(
51
+ failed.length
52
+ ? {
53
+ id: 'enrollment:record',
54
+ status: 'fail',
55
+ summary: 'WORKER_RECORD_RETIREMENT_FAILED',
56
+ remedy: {
57
+ action: `${id}'s worker record disappeared, and retiring it failed at ${failed.map((step) => step.id).join(', ')}. Follow that remedy, then run fleet-sync again.`,
58
+ },
59
+ }
60
+ : {
61
+ id: 'enrollment:record',
62
+ status: apply ? 'pass' : 'skip',
63
+ summary: `${apply ? '' : 'Planned: '}${id}'s worker record disappeared, so it is retired: ${steps.map((step) => step.id).join(', ')}.`,
64
+ }
65
+ );
66
+ }
67
+ const fresh = definition.workers.filter(
68
+ (worker) => worker.origin === 'records' && !states[worker.id]?.origin
69
+ );
70
+ if (apply && fresh.length)
71
+ await changeWorkers(lifecycle.state(), (current) =>
72
+ Object.fromEntries(
73
+ fresh.map((worker) => [
74
+ worker.id,
75
+ { ...current[worker.id], origin: 'records' as const },
76
+ ])
77
+ )
78
+ );
79
+ return checks;
80
+ }
81
+
82
+ async function rosterEntries(path: string): Promise<Map<string, FleetNode>> {
83
+ const text = await readRosterText(path).catch(() => null);
84
+ if (text === null) return new Map();
85
+ try {
86
+ return new Map(
87
+ parseConfig(JSON.parse(text) as unknown).nodes.map((node) => [
88
+ node.id,
89
+ node,
90
+ ])
91
+ );
92
+ } catch {
93
+ return new Map();
94
+ }
95
+ }
96
+
97
+ /** A worker whose record is gone, as far as retiring it needs: reached from
98
+ * declared data (so its pin is the fleet state's), under the name the roster
99
+ * last knew. Its user and ports are never used to retire it.
100
+ */
101
+ function gone(id: string, entry: FleetNode | undefined): MachineDefinition {
102
+ return {
103
+ id,
104
+ os: 'linux',
105
+ origin: 'records',
106
+ network: {
107
+ tailscaleName: entry?.ssh ?? id,
108
+ user: 'root',
109
+ sshPort: 22,
110
+ t3Port: 3773,
111
+ },
112
+ };
113
+ }
@@ -2,7 +2,8 @@ import { createHash } from 'node:crypto';
2
2
  import { IDES } from '../ide';
3
3
  import { enabled, workerChecks } from '../ide/orchestrate';
4
4
  import { idesOf } from '../ide/definition';
5
- import { rememberEnvironment } from '../t3/pairing';
5
+ import { pairIfRequested, rememberEnvironment } from '../t3/pairing';
6
+ import { managerT3 } from '../t3/peer-link';
6
7
  import { authorizePrivilege } from './fleet-sync-privilege';
7
8
  import { authorizeVault } from './fleet-sync-vault';
8
9
  import { managerFailure } from '../manager-failure';
@@ -15,13 +16,32 @@ import { FleetUsageError } from '../errors';
15
16
  import { selectNodes } from '../config';
16
17
  import { loadRoster } from './roster';
17
18
  import { enrolledRoster } from '../definition';
19
+ import { readParked, readWorkerStates } from '../state/workers';
20
+ import { withoutParked } from '../worker-parking';
21
+ import { reconcileRecords } from './fleet-sync-records';
18
22
  import type { FleetDefinition } from '../definition';
19
23
  import { fleet as definition } from '../fleet-config';
20
24
  import { synchronizeManager } from '../manager-sync';
21
25
  import { inspectBootstrap } from '../manager-sync';
22
- import { prepareBootstrap, bootstrapRemote } from '../remote-bootstrap';
26
+ import {
27
+ prepareBootstrap,
28
+ bootstrapRemote,
29
+ readsTailnetKey,
30
+ } from '../remote-bootstrap';
31
+ import { secret } from '../fleet-config';
32
+ import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
33
+ import { discoverTailnet, type TailnetComputer } from '../tailscale';
34
+ import { deployWorker } from './fleet-sync-worker';
35
+ import { packagePlaybook } from '../platform/ansible';
23
36
  import { machineRoot } from './fleet-sync';
24
- import type { Check, Runner } from '../types';
37
+ import {
38
+ blocking,
39
+ type Check,
40
+ type FleetNode,
41
+ type NodeReport,
42
+ type Runner,
43
+ } from '../types';
44
+ import { diagnostic } from '../diagnostics';
25
45
  import type { DeployNode, ManagerReport } from '../manager-sync';
26
46
  import type { FleetSyncOptions } from './fleet-sync';
27
47
  import type { Settings } from './settings';
@@ -39,14 +59,35 @@ export async function selected(
39
59
  if (!options.revision)
40
60
  throw new FleetUsageError('An approved source revision is required.');
41
61
  const canonical = definition;
62
+ // Workers whose record disappeared are retired first (planned in a
63
+ // preview), so they are no longer in the roster.
64
+ const records = await reconcileRecords(canonical, settings.config, run, {
65
+ apply: Boolean(options.apply),
66
+ });
42
67
  let roster;
43
68
  try {
44
- roster = enrolledRoster(canonical, await loadRoster(settings.config));
69
+ const declared = new Set(canonical.workers.map((worker) => worker.id));
70
+ const loaded = await loadRoster(settings.config);
71
+ roster = enrolledRoster(canonical, {
72
+ ...loaded,
73
+ nodes: loaded.nodes.filter(
74
+ (node) =>
75
+ declared.has(node.id) ||
76
+ readWorkerStates().workers[node.id]?.origin !== 'records'
77
+ ),
78
+ });
45
79
  } catch {
46
80
  return enrollmentRequired(canonical, options.revision);
47
81
  }
48
- const nodes = selectNodes(roster, options.node ?? []);
49
- const scoped = scopeDefinition(canonical, options.node);
82
+ // A parked worker is not contacted and never the canary.
83
+ const active = withoutParked(
84
+ scopeDefinition(canonical, options.node),
85
+ selectNodes(roster, options.node ?? []),
86
+ readParked(),
87
+ new Date()
88
+ );
89
+ const { nodes, definition: scoped } = active;
90
+ const notes = [...records, ...active.checks];
50
91
  const ui = interfaceOptions(options);
51
92
  const planFingerprint = createHash('sha256')
52
93
  .update(
@@ -65,12 +106,16 @@ export async function selected(
65
106
  'Manager connection bindings or target policy changed after approval. Inspect a fresh plan and approve it; no source preparation or node mutation occurred.'
66
107
  )
67
108
  );
68
- const payload = options.apply
109
+ const prepared = options.apply
69
110
  ? await prepareBootstrap(machineRoot(), options.revision, run, {
70
111
  definition: canonical,
71
112
  checkout: settings.workspace,
72
113
  })
73
114
  : null;
115
+ const payload =
116
+ prepared && options.restartBusy
117
+ ? { ...prepared, restartBusy: true }
118
+ : prepared;
74
119
  const started = Date.now();
75
120
  const report = await synchronizeManager(
76
121
  {
@@ -79,6 +124,7 @@ export async function selected(
79
124
  revision: options.revision,
80
125
  apply: Boolean(options.apply),
81
126
  ...canary(canonical, options),
127
+ ...serial(options),
82
128
  progress: (node, phase) => ui.progress.step(node, phase),
83
129
  finished: (node) => ui.progress.finished(node),
84
130
  manager: () =>
@@ -97,22 +143,74 @@ export async function selected(
97
143
  ...ui,
98
144
  workspace: settings.workspace,
99
145
  roster: settings.config,
146
+ ...rebootOptions(options),
100
147
  })
101
148
  ).finally(() => ui.progress.stop());
149
+ if (options.enrollment) reportEnrollment(report, options.enrollment);
150
+ report.checks.push(...notes);
151
+ // A worker that could not be retired, like any other failed check of the
152
+ // fleet, means the run did not converge: it is not recorded as applied.
153
+ if (notes.some(blocking)) report.ok = false;
102
154
  if (settle) report.checks.push(await settle(report));
103
155
  return outcome(report, planFingerprint, started);
104
156
  }
105
157
 
106
- /** A revision new to the fleet is proven on the first declared worker before
107
- * the rest; repeating one that already converged everywhere needs no canary,
108
- * so every worker converges at once.
158
+ /** Maintenance reboots go one worker at a time, and a worker that does not
159
+ * come back ready stops the others.
160
+ */
161
+ function serial(options: FleetSyncOptions): { serial?: true } {
162
+ return options.maintenanceReboot ? { serial: true } : {};
163
+ }
164
+
165
+ function rebootOptions(options: FleetSyncOptions): {
166
+ readyBefore: string[];
167
+ maintenanceReboot?: true;
168
+ force?: true;
169
+ } {
170
+ return {
171
+ readyBefore: options.readyBefore ?? [],
172
+ ...(options.maintenanceReboot ? { maintenanceReboot: true } : {}),
173
+ ...(options.force ? { force: true } : {}),
174
+ };
175
+ }
176
+
177
+ /** The workers this run enrolled, and why another was not: the checks land
178
+ * on the worker they are about.
179
+ */
180
+ export function reportEnrollment(
181
+ report: ManagerReport,
182
+ enrollment: NonNullable<FleetSyncOptions['enrollment']>
183
+ ): void {
184
+ report.enrolled = enrollment.enrolled;
185
+ for (const { node, check } of enrollment.checks) {
186
+ const entry = report.nodes.find((candidate) => candidate.node === node);
187
+ if (!entry) continue;
188
+ entry.checks.unshift(check);
189
+ entry.diagnostics = entry.checks.map((each) => diagnostic(node, each));
190
+ if (blocking(check)) report.ok = false;
191
+ }
192
+ }
193
+
194
+ /** A revision new to the fleet is proven on one worker before the rest: the
195
+ * one `--canary` or fleet.json names, else the first declared worker, and
196
+ * the next reachable one when that cannot be reached. Repeating a revision
197
+ * that already converged everywhere needs no canary, so every worker
198
+ * converges at once.
109
199
  */
110
200
  function canary(
111
201
  canonical: FleetDefinition,
112
202
  options: FleetSyncOptions
113
203
  ): { canary?: string } {
114
- const first = canonical.workers[0];
115
- return first && !options.rolledOut ? { canary: first.id } : {};
204
+ if (
205
+ options.canary &&
206
+ !canonical.workers.some((worker) => worker.id === options.canary)
207
+ )
208
+ throw new FleetUsageError(
209
+ '--canary names a worker fleet.json does not declare.'
210
+ );
211
+ const preferred =
212
+ options.canary ?? canonical.canary ?? canonical.workers[0]?.id;
213
+ return preferred && !options.rolledOut ? { canary: preferred } : {};
116
214
  }
117
215
 
118
216
  /** Saves the full JSON report, then renders the short human summary. */
@@ -165,8 +263,16 @@ function deployWithMonitoring(
165
263
  /** The manager's checkout, and the roster an identity is recorded in. */
166
264
  workspace: string;
167
265
  roster: string;
266
+ passwordLogin: boolean;
267
+ readyBefore: string[];
268
+ maintenanceReboot?: boolean;
269
+ force?: boolean;
168
270
  }
169
271
  ): DeployNode {
272
+ // Asked once per run, and only when a worker needs them.
273
+ let control: Promise<TailnetControl | null> | undefined;
274
+ let computers: Promise<TailnetComputer[]> | undefined;
275
+ const converge = convergeWorker(payload, canonical, run, ui);
170
276
  return async (node, revision) => {
171
277
  if (!payload || payload.revision !== revision)
172
278
  return {
@@ -183,18 +289,62 @@ function deployWithMonitoring(
183
289
  },
184
290
  ],
185
291
  };
292
+ return deployWorker(node, revision, {
293
+ definition: canonical,
294
+ playbook: packagePlaybook(run),
295
+ converge,
296
+ control: () =>
297
+ (control ??= fleetTailnetControl(canonical.tailnet, secret)),
298
+ computers: () =>
299
+ (computers ??= discoverTailnet(run).then((found) => found.computers)),
300
+ readyBefore: ui.readyBefore,
301
+ ...(ui.maintenanceReboot ? { maintenanceReboot: true } : {}),
302
+ ...(ui.force ? { force: true } : {}),
303
+ acceptsTailnetKey: readsTailnetKey(payload.script),
304
+ });
305
+ };
306
+ }
307
+
308
+ /** One converge of a worker: privilege, Vault, the bootstrap, then what is
309
+ * verified from the manager and the worker's monitoring.
310
+ */
311
+ function convergeWorker(
312
+ payload: Awaited<ReturnType<typeof prepareBootstrap>>,
313
+ canonical: FleetDefinition,
314
+ run: Runner,
315
+ ui: {
316
+ progress: SyncProgress;
317
+ interactiveAuthorization: boolean;
318
+ workspace: string;
319
+ roster: string;
320
+ passwordLogin: boolean;
321
+ }
322
+ ) {
323
+ return async (
324
+ node: FleetNode,
325
+ revision: string,
326
+ tailnetKey?: string
327
+ ): Promise<NodeReport> => {
186
328
  // Password prompts need a quiet terminal: no spinner while authorizing.
187
329
  if (ui.interactiveAuthorization) {
188
330
  ui.progress.stop();
189
- const checks = await authorizePrivilege(node, payload.authorizeSudo, run);
331
+ const checks = await authorizePrivilege(
332
+ node,
333
+ payload!.authorizeSudo,
334
+ run
335
+ );
190
336
  if (checks.length) return { node: node.id, checks };
191
337
  }
192
- const vault = { interactive: ui.interactiveAuthorization };
338
+ const vault = {
339
+ interactive: ui.interactiveAuthorization,
340
+ passwordLogin: ui.passwordLogin,
341
+ };
193
342
  if (ui.interactiveAuthorization) ui.progress.stop();
194
343
  const before = await authorizeVault(node, vault, run);
195
344
  ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
196
- const result = await bootstrapRemote(node, payload, run, {
345
+ const result = await bootstrapRemote(node, payload!, run, {
197
346
  progress: (line) => ui.progress.step(node.id, line),
347
+ ...(tailnetKey ? { tailnetKey } : {}),
198
348
  });
199
349
  // A fresh node gets vault from the bootstrap; log it in now. Its provider
200
350
  // checks in this run predate the login, so the next sync verifies them.
@@ -212,6 +362,12 @@ function deployWithMonitoring(
212
362
  remember: async (id, identity) => {
213
363
  await rememberEnvironment(ui.roster, id, identity);
214
364
  },
365
+ pair: (target) =>
366
+ pairIfRequested(target, {
367
+ roster: ui.roster,
368
+ run,
369
+ manager: managerT3(run),
370
+ }),
215
371
  }))
216
372
  );
217
373
  if (
@@ -242,6 +398,7 @@ function scopeDefinition(canonical: FleetDefinition, ids?: string[]) {
242
398
  function interfaceOptions(options: FleetSyncOptions) {
243
399
  return {
244
400
  progress: syncProgress(!options.json),
401
+ passwordLogin: Boolean(options.vaultPasswordLogin),
245
402
  interactiveAuthorization: Boolean(
246
403
  options.interactiveAuthorization &&
247
404
  !options.json &&
@@ -9,13 +9,19 @@ import {
9
9
  recordSchedule,
10
10
  type ScheduleRecord,
11
11
  } from '../state/history';
12
- import { scheduleCheck, SCHEDULE_INTERVAL_SECONDS } from '../schedule-check';
12
+ import macos from '../../ansible/macos/policy.json';
13
+ import { fleet } from '../fleet-config';
14
+ import {
15
+ scheduleCheck,
16
+ scheduleIntervals,
17
+ SCHEDULE_INTERVAL_SECONDS,
18
+ } from '../schedule-check';
13
19
  import type { Runner } from '../types';
14
20
  import type { CommandOutcome } from './contracts';
15
21
  import { machineRoot } from './fleet-sync';
16
22
 
17
23
  /** The launchd label and systemd unit already on running managers. */
18
- export const SCHEDULE_LABEL = 'personal-fleet-sync';
24
+ export const SCHEDULE_LABEL = macos.fleetSync.label;
19
25
  const LABEL = SCHEDULE_LABEL;
20
26
  export type ScheduleAction =
21
27
  'enable' | 'pause' | 'resume' | 'disable' | 'status';
@@ -31,6 +37,8 @@ interface ScheduleInput {
31
37
  /** The manager's caches; tests name their own. */
32
38
  root?: string;
33
39
  now?: Date;
40
+ /** How often the schedule runs; `manager.schedule.syncMinutes` otherwise. */
41
+ intervalSeconds?: number;
34
42
  }
35
43
 
36
44
  /** The published package a schedule installs and runs: its name, exact
@@ -205,14 +213,16 @@ async function enableSchedule(
205
213
  return required(
206
214
  'The exact CLI package could not be installed in the managed manager runtime. No schedule was enabled.'
207
215
  );
216
+ const intervalSeconds =
217
+ input.intervalSeconds ?? scheduleIntervals(fleet.manager).syncSeconds;
208
218
  const result = platform.installScheduledJob({
209
219
  label: LABEL,
210
- intervalSeconds: SCHEDULE_INTERVAL_SECONDS,
220
+ intervalSeconds,
211
221
  command: scheduledCommand(published, input.revision, prefix),
212
222
  logDir: join(root, 'logs'),
213
223
  env: scheduledEnvironment(),
214
224
  });
215
- if (result.ok) remember(input, true);
225
+ if (result.ok) remember(input, true, intervalSeconds);
216
226
  return {
217
227
  exitCode: result.ok ? 0 : 1,
218
228
  data: {
@@ -1,6 +1,6 @@
1
1
  import type { Enrollment } from '../machine/state';
2
2
  import { replicaCheck } from '../state/check';
3
- import { storeEnrollment } from '../state/enrollment';
3
+ import { loadEnrollment, storeEnrollment } from '../state/enrollment';
4
4
  import { adoptEarlier } from '../state/legacy';
5
5
  import { databasePath, stateRoot } from '../state/paths';
6
6
  import { writeDocument } from '../state/store';
@@ -21,8 +21,12 @@ import type { Check } from '../types';
21
21
  export interface RunRecord {
22
22
  /** Before any machine changes: this revision is the approved one. */
23
23
  approve(): void;
24
- /** Once the result is known. The check says whether the bucket holds it. */
25
- settle(converged: boolean): Promise<Check>;
24
+ /** Once the result is known. The check says whether the bucket holds it.
25
+ * `applied` is the revision each worker that converged now holds: it is
26
+ * kept even when the fleet as a whole is not ready, and it is all a run
27
+ * scoped with --node records.
28
+ */
29
+ settle(ready: boolean, applied?: Record<string, string>): Promise<Check>;
26
30
  /** For a run that stopped before its result: its approval is still uploaded. */
27
31
  abandon(): Promise<void>;
28
32
  }
@@ -39,6 +43,10 @@ export function runRecord(
39
43
  * worker, a replacement manager) uses the one last applied.
40
44
  */
41
45
  definition?: string;
46
+ /** A run scoped with --node covers only the workers it selected: it
47
+ * records their revisions, never one applied for the whole fleet.
48
+ */
49
+ scoped?: boolean;
42
50
  }
43
51
  ): RunRecord {
44
52
  const env = state.env ?? process.env;
@@ -70,20 +78,33 @@ export function runRecord(
70
78
  }
71
79
  claimLease(state.context);
72
80
  },
73
- async settle(converged) {
81
+ async settle(ready, applied = {}) {
74
82
  settled = true;
75
- if (converged)
83
+ const converged = ready && state.scoped !== true;
84
+ if (converged || Object.keys(applied).length) {
85
+ // What was recorded since the approval stands: the run itself
86
+ // retires a worker whose record was removed, and another command
87
+ // may have. Only what this run decided is written over it.
88
+ const stored = (await loadEnrollment(null, env)) ?? approval;
76
89
  storeEnrollment(
77
90
  {
78
- ...approval,
79
- appliedRevision: revision,
80
- ...(previous?.appliedRevision &&
81
- previous.appliedRevision !== revision
82
- ? { previousRevision: previous.appliedRevision }
91
+ ...stored,
92
+ ...(Object.keys(applied).length
93
+ ? { workers: { ...stored.workers, ...applied } }
94
+ : {}),
95
+ ...(converged
96
+ ? {
97
+ appliedRevision: revision,
98
+ ...(previous?.appliedRevision &&
99
+ previous.appliedRevision !== revision
100
+ ? { previousRevision: previous.appliedRevision }
101
+ : {}),
102
+ }
83
103
  : {}),
84
104
  },
85
105
  env
86
106
  );
107
+ }
87
108
  // The fleet converged either way: a bucket problem is reported, not fatal.
88
109
  return replicaCheck(await publish().catch(() => 'unreachable' as const), {
89
110
  advisory: true,