@crvouga/dev-fleet 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -9
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +55 -9
- package/ansible/macos/manager.yml +154 -7
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +7 -3
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +19 -11
- package/docs/consumer-setup.md +42 -27
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +10 -3
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +98 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +67 -6
- package/docs/ubuntu-sync.md +78 -28
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1967 -170
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +327 -8
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +61 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +9 -3
- package/src/cli/fleet-node-lifecycle.ts +313 -64
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +51 -35
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +4 -0
- package/src/cli/fleet-sync-manager.ts +103 -33
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +151 -9
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +28 -16
- package/src/cli/fleet-sync-vault.ts +77 -0
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +150 -19
- package/src/cli/fleet-validate.ts +59 -21
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +115 -4
- package/src/cli/registry.ts +14 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +332 -9
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +37 -15
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +65 -4
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +4 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capacity.ts +31 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +27 -4
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +17 -1
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +16 -11
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
|
@@ -2,7 +2,8 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { IDES } from '../ide';
|
|
3
3
|
import { enabled, workerChecks } from '../ide/orchestrate';
|
|
4
4
|
import { idesOf } from '../ide/definition';
|
|
5
|
-
import { rememberEnvironment } from '../t3/pairing';
|
|
5
|
+
import { pairIfRequested, rememberEnvironment } from '../t3/pairing';
|
|
6
|
+
import { managerT3 } from '../t3/peer-link';
|
|
6
7
|
import { authorizePrivilege } from './fleet-sync-privilege';
|
|
7
8
|
import { authorizeVault } from './fleet-sync-vault';
|
|
8
9
|
import { managerFailure } from '../manager-failure';
|
|
@@ -15,13 +16,32 @@ import { FleetUsageError } from '../errors';
|
|
|
15
16
|
import { selectNodes } from '../config';
|
|
16
17
|
import { loadRoster } from './roster';
|
|
17
18
|
import { enrolledRoster } from '../definition';
|
|
19
|
+
import { readParked, readWorkerStates } from '../state/workers';
|
|
20
|
+
import { withoutParked } from '../worker-parking';
|
|
21
|
+
import { reconcileRecords } from './fleet-sync-records';
|
|
18
22
|
import type { FleetDefinition } from '../definition';
|
|
19
23
|
import { fleet as definition } from '../fleet-config';
|
|
20
24
|
import { synchronizeManager } from '../manager-sync';
|
|
21
25
|
import { inspectBootstrap } from '../manager-sync';
|
|
22
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
prepareBootstrap,
|
|
28
|
+
bootstrapRemote,
|
|
29
|
+
readsTailnetKey,
|
|
30
|
+
} from '../remote-bootstrap';
|
|
31
|
+
import { secret } from '../fleet-config';
|
|
32
|
+
import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
|
|
33
|
+
import { discoverTailnet, type TailnetComputer } from '../tailscale';
|
|
34
|
+
import { deployWorker } from './fleet-sync-worker';
|
|
35
|
+
import { packagePlaybook } from '../platform/ansible';
|
|
23
36
|
import { machineRoot } from './fleet-sync';
|
|
24
|
-
import
|
|
37
|
+
import {
|
|
38
|
+
blocking,
|
|
39
|
+
type Check,
|
|
40
|
+
type FleetNode,
|
|
41
|
+
type NodeReport,
|
|
42
|
+
type Runner,
|
|
43
|
+
} from '../types';
|
|
44
|
+
import { diagnostic } from '../diagnostics';
|
|
25
45
|
import type { DeployNode, ManagerReport } from '../manager-sync';
|
|
26
46
|
import type { FleetSyncOptions } from './fleet-sync';
|
|
27
47
|
import type { Settings } from './settings';
|
|
@@ -39,14 +59,35 @@ export async function selected(
|
|
|
39
59
|
if (!options.revision)
|
|
40
60
|
throw new FleetUsageError('An approved source revision is required.');
|
|
41
61
|
const canonical = definition;
|
|
62
|
+
// Workers whose record disappeared are retired first (planned in a
|
|
63
|
+
// preview), so they are no longer in the roster.
|
|
64
|
+
const records = await reconcileRecords(canonical, settings.config, run, {
|
|
65
|
+
apply: Boolean(options.apply),
|
|
66
|
+
});
|
|
42
67
|
let roster;
|
|
43
68
|
try {
|
|
44
|
-
|
|
69
|
+
const declared = new Set(canonical.workers.map((worker) => worker.id));
|
|
70
|
+
const loaded = await loadRoster(settings.config);
|
|
71
|
+
roster = enrolledRoster(canonical, {
|
|
72
|
+
...loaded,
|
|
73
|
+
nodes: loaded.nodes.filter(
|
|
74
|
+
(node) =>
|
|
75
|
+
declared.has(node.id) ||
|
|
76
|
+
readWorkerStates().workers[node.id]?.origin !== 'records'
|
|
77
|
+
),
|
|
78
|
+
});
|
|
45
79
|
} catch {
|
|
46
80
|
return enrollmentRequired(canonical, options.revision);
|
|
47
81
|
}
|
|
48
|
-
|
|
49
|
-
const
|
|
82
|
+
// A parked worker is not contacted and never the canary.
|
|
83
|
+
const active = withoutParked(
|
|
84
|
+
scopeDefinition(canonical, options.node),
|
|
85
|
+
selectNodes(roster, options.node ?? []),
|
|
86
|
+
readParked(),
|
|
87
|
+
new Date()
|
|
88
|
+
);
|
|
89
|
+
const { nodes, definition: scoped } = active;
|
|
90
|
+
const notes = [...records, ...active.checks];
|
|
50
91
|
const ui = interfaceOptions(options);
|
|
51
92
|
const planFingerprint = createHash('sha256')
|
|
52
93
|
.update(
|
|
@@ -65,12 +106,16 @@ export async function selected(
|
|
|
65
106
|
'Manager connection bindings or target policy changed after approval. Inspect a fresh plan and approve it; no source preparation or node mutation occurred.'
|
|
66
107
|
)
|
|
67
108
|
);
|
|
68
|
-
const
|
|
109
|
+
const prepared = options.apply
|
|
69
110
|
? await prepareBootstrap(machineRoot(), options.revision, run, {
|
|
70
111
|
definition: canonical,
|
|
71
112
|
checkout: settings.workspace,
|
|
72
113
|
})
|
|
73
114
|
: null;
|
|
115
|
+
const payload =
|
|
116
|
+
prepared && options.restartBusy
|
|
117
|
+
? { ...prepared, restartBusy: true }
|
|
118
|
+
: prepared;
|
|
74
119
|
const started = Date.now();
|
|
75
120
|
const report = await synchronizeManager(
|
|
76
121
|
{
|
|
@@ -79,6 +124,7 @@ export async function selected(
|
|
|
79
124
|
revision: options.revision,
|
|
80
125
|
apply: Boolean(options.apply),
|
|
81
126
|
...canary(canonical, options),
|
|
127
|
+
...serial(options),
|
|
82
128
|
progress: (node, phase) => ui.progress.step(node, phase),
|
|
83
129
|
finished: (node) => ui.progress.finished(node),
|
|
84
130
|
manager: () =>
|
|
@@ -97,12 +143,54 @@ export async function selected(
|
|
|
97
143
|
...ui,
|
|
98
144
|
workspace: settings.workspace,
|
|
99
145
|
roster: settings.config,
|
|
146
|
+
...rebootOptions(options),
|
|
100
147
|
})
|
|
101
148
|
).finally(() => ui.progress.stop());
|
|
149
|
+
if (options.enrollment) reportEnrollment(report, options.enrollment);
|
|
150
|
+
report.checks.push(...notes);
|
|
151
|
+
// A worker that could not be retired, like any other failed check of the
|
|
152
|
+
// fleet, means the run did not converge: it is not recorded as applied.
|
|
153
|
+
if (notes.some(blocking)) report.ok = false;
|
|
102
154
|
if (settle) report.checks.push(await settle(report));
|
|
103
155
|
return outcome(report, planFingerprint, started);
|
|
104
156
|
}
|
|
105
157
|
|
|
158
|
+
/** Maintenance reboots go one worker at a time, and a worker that does not
|
|
159
|
+
* come back ready stops the others.
|
|
160
|
+
*/
|
|
161
|
+
function serial(options: FleetSyncOptions): { serial?: true } {
|
|
162
|
+
return options.maintenanceReboot ? { serial: true } : {};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function rebootOptions(options: FleetSyncOptions): {
|
|
166
|
+
readyBefore: string[];
|
|
167
|
+
maintenanceReboot?: true;
|
|
168
|
+
force?: true;
|
|
169
|
+
} {
|
|
170
|
+
return {
|
|
171
|
+
readyBefore: options.readyBefore ?? [],
|
|
172
|
+
...(options.maintenanceReboot ? { maintenanceReboot: true } : {}),
|
|
173
|
+
...(options.force ? { force: true } : {}),
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The workers this run enrolled, and why another was not: the checks land
|
|
178
|
+
* on the worker they are about.
|
|
179
|
+
*/
|
|
180
|
+
export function reportEnrollment(
|
|
181
|
+
report: ManagerReport,
|
|
182
|
+
enrollment: NonNullable<FleetSyncOptions['enrollment']>
|
|
183
|
+
): void {
|
|
184
|
+
report.enrolled = enrollment.enrolled;
|
|
185
|
+
for (const { node, check } of enrollment.checks) {
|
|
186
|
+
const entry = report.nodes.find((candidate) => candidate.node === node);
|
|
187
|
+
if (!entry) continue;
|
|
188
|
+
entry.checks.unshift(check);
|
|
189
|
+
entry.diagnostics = entry.checks.map((each) => diagnostic(node, each));
|
|
190
|
+
if (blocking(check)) report.ok = false;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
106
194
|
/** A revision new to the fleet is proven on one worker before the rest: the
|
|
107
195
|
* one `--canary` or fleet.json names, else the first declared worker, and
|
|
108
196
|
* the next reachable one when that cannot be reached. Repeating a revision
|
|
@@ -176,8 +264,15 @@ function deployWithMonitoring(
|
|
|
176
264
|
workspace: string;
|
|
177
265
|
roster: string;
|
|
178
266
|
passwordLogin: boolean;
|
|
267
|
+
readyBefore: string[];
|
|
268
|
+
maintenanceReboot?: boolean;
|
|
269
|
+
force?: boolean;
|
|
179
270
|
}
|
|
180
271
|
): DeployNode {
|
|
272
|
+
// Asked once per run, and only when a worker needs them.
|
|
273
|
+
let control: Promise<TailnetControl | null> | undefined;
|
|
274
|
+
let computers: Promise<TailnetComputer[]> | undefined;
|
|
275
|
+
const converge = convergeWorker(payload, canonical, run, ui);
|
|
181
276
|
return async (node, revision) => {
|
|
182
277
|
if (!payload || payload.revision !== revision)
|
|
183
278
|
return {
|
|
@@ -194,10 +289,50 @@ function deployWithMonitoring(
|
|
|
194
289
|
},
|
|
195
290
|
],
|
|
196
291
|
};
|
|
292
|
+
return deployWorker(node, revision, {
|
|
293
|
+
definition: canonical,
|
|
294
|
+
playbook: packagePlaybook(run),
|
|
295
|
+
converge,
|
|
296
|
+
control: () =>
|
|
297
|
+
(control ??= fleetTailnetControl(canonical.tailnet, secret)),
|
|
298
|
+
computers: () =>
|
|
299
|
+
(computers ??= discoverTailnet(run).then((found) => found.computers)),
|
|
300
|
+
readyBefore: ui.readyBefore,
|
|
301
|
+
...(ui.maintenanceReboot ? { maintenanceReboot: true } : {}),
|
|
302
|
+
...(ui.force ? { force: true } : {}),
|
|
303
|
+
acceptsTailnetKey: readsTailnetKey(payload.script),
|
|
304
|
+
});
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** One converge of a worker: privilege, Vault, the bootstrap, then what is
|
|
309
|
+
* verified from the manager and the worker's monitoring.
|
|
310
|
+
*/
|
|
311
|
+
function convergeWorker(
|
|
312
|
+
payload: Awaited<ReturnType<typeof prepareBootstrap>>,
|
|
313
|
+
canonical: FleetDefinition,
|
|
314
|
+
run: Runner,
|
|
315
|
+
ui: {
|
|
316
|
+
progress: SyncProgress;
|
|
317
|
+
interactiveAuthorization: boolean;
|
|
318
|
+
workspace: string;
|
|
319
|
+
roster: string;
|
|
320
|
+
passwordLogin: boolean;
|
|
321
|
+
}
|
|
322
|
+
) {
|
|
323
|
+
return async (
|
|
324
|
+
node: FleetNode,
|
|
325
|
+
revision: string,
|
|
326
|
+
tailnetKey?: string
|
|
327
|
+
): Promise<NodeReport> => {
|
|
197
328
|
// Password prompts need a quiet terminal: no spinner while authorizing.
|
|
198
329
|
if (ui.interactiveAuthorization) {
|
|
199
330
|
ui.progress.stop();
|
|
200
|
-
const checks = await authorizePrivilege(
|
|
331
|
+
const checks = await authorizePrivilege(
|
|
332
|
+
node,
|
|
333
|
+
payload!.authorizeSudo,
|
|
334
|
+
run
|
|
335
|
+
);
|
|
201
336
|
if (checks.length) return { node: node.id, checks };
|
|
202
337
|
}
|
|
203
338
|
const vault = {
|
|
@@ -207,8 +342,9 @@ function deployWithMonitoring(
|
|
|
207
342
|
if (ui.interactiveAuthorization) ui.progress.stop();
|
|
208
343
|
const before = await authorizeVault(node, vault, run);
|
|
209
344
|
ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
|
|
210
|
-
const result = await bootstrapRemote(node, payload
|
|
345
|
+
const result = await bootstrapRemote(node, payload!, run, {
|
|
211
346
|
progress: (line) => ui.progress.step(node.id, line),
|
|
347
|
+
...(tailnetKey ? { tailnetKey } : {}),
|
|
212
348
|
});
|
|
213
349
|
// A fresh node gets vault from the bootstrap; log it in now. Its provider
|
|
214
350
|
// checks in this run predate the login, so the next sync verifies them.
|
|
@@ -226,6 +362,12 @@ function deployWithMonitoring(
|
|
|
226
362
|
remember: async (id, identity) => {
|
|
227
363
|
await rememberEnvironment(ui.roster, id, identity);
|
|
228
364
|
},
|
|
365
|
+
pair: (target) =>
|
|
366
|
+
pairIfRequested(target, {
|
|
367
|
+
roster: ui.roster,
|
|
368
|
+
run,
|
|
369
|
+
manager: managerT3(run),
|
|
370
|
+
}),
|
|
229
371
|
}))
|
|
230
372
|
);
|
|
231
373
|
if (
|
|
@@ -9,13 +9,19 @@ import {
|
|
|
9
9
|
recordSchedule,
|
|
10
10
|
type ScheduleRecord,
|
|
11
11
|
} from '../state/history';
|
|
12
|
-
import
|
|
12
|
+
import macos from '../../ansible/macos/policy.json';
|
|
13
|
+
import { fleet } from '../fleet-config';
|
|
14
|
+
import {
|
|
15
|
+
scheduleCheck,
|
|
16
|
+
scheduleIntervals,
|
|
17
|
+
SCHEDULE_INTERVAL_SECONDS,
|
|
18
|
+
} from '../schedule-check';
|
|
13
19
|
import type { Runner } from '../types';
|
|
14
20
|
import type { CommandOutcome } from './contracts';
|
|
15
21
|
import { machineRoot } from './fleet-sync';
|
|
16
22
|
|
|
17
23
|
/** The launchd label and systemd unit already on running managers. */
|
|
18
|
-
export const SCHEDULE_LABEL =
|
|
24
|
+
export const SCHEDULE_LABEL = macos.fleetSync.label;
|
|
19
25
|
const LABEL = SCHEDULE_LABEL;
|
|
20
26
|
export type ScheduleAction =
|
|
21
27
|
'enable' | 'pause' | 'resume' | 'disable' | 'status';
|
|
@@ -31,6 +37,8 @@ interface ScheduleInput {
|
|
|
31
37
|
/** The manager's caches; tests name their own. */
|
|
32
38
|
root?: string;
|
|
33
39
|
now?: Date;
|
|
40
|
+
/** How often the schedule runs; `manager.schedule.syncMinutes` otherwise. */
|
|
41
|
+
intervalSeconds?: number;
|
|
34
42
|
}
|
|
35
43
|
|
|
36
44
|
/** The published package a schedule installs and runs: its name, exact
|
|
@@ -205,14 +213,16 @@ async function enableSchedule(
|
|
|
205
213
|
return required(
|
|
206
214
|
'The exact CLI package could not be installed in the managed manager runtime. No schedule was enabled.'
|
|
207
215
|
);
|
|
216
|
+
const intervalSeconds =
|
|
217
|
+
input.intervalSeconds ?? scheduleIntervals(fleet.manager).syncSeconds;
|
|
208
218
|
const result = platform.installScheduledJob({
|
|
209
219
|
label: LABEL,
|
|
210
|
-
intervalSeconds
|
|
220
|
+
intervalSeconds,
|
|
211
221
|
command: scheduledCommand(published, input.revision, prefix),
|
|
212
222
|
logDir: join(root, 'logs'),
|
|
213
223
|
env: scheduledEnvironment(),
|
|
214
224
|
});
|
|
215
|
-
if (result.ok) remember(input, true);
|
|
225
|
+
if (result.ok) remember(input, true, intervalSeconds);
|
|
216
226
|
return {
|
|
217
227
|
exitCode: result.ok ? 0 : 1,
|
|
218
228
|
data: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Enrollment } from '../machine/state';
|
|
2
2
|
import { replicaCheck } from '../state/check';
|
|
3
|
-
import { storeEnrollment } from '../state/enrollment';
|
|
3
|
+
import { loadEnrollment, storeEnrollment } from '../state/enrollment';
|
|
4
4
|
import { adoptEarlier } from '../state/legacy';
|
|
5
5
|
import { databasePath, stateRoot } from '../state/paths';
|
|
6
6
|
import { writeDocument } from '../state/store';
|
|
@@ -23,9 +23,10 @@ export interface RunRecord {
|
|
|
23
23
|
approve(): void;
|
|
24
24
|
/** Once the result is known. The check says whether the bucket holds it.
|
|
25
25
|
* `applied` is the revision each worker that converged now holds: it is
|
|
26
|
-
* kept even when the fleet as a whole is not ready
|
|
26
|
+
* kept even when the fleet as a whole is not ready, and it is all a run
|
|
27
|
+
* scoped with --node records.
|
|
27
28
|
*/
|
|
28
|
-
settle(
|
|
29
|
+
settle(ready: boolean, applied?: Record<string, string>): Promise<Check>;
|
|
29
30
|
/** For a run that stopped before its result: its approval is still uploaded. */
|
|
30
31
|
abandon(): Promise<void>;
|
|
31
32
|
}
|
|
@@ -42,6 +43,10 @@ export function runRecord(
|
|
|
42
43
|
* worker, a replacement manager) uses the one last applied.
|
|
43
44
|
*/
|
|
44
45
|
definition?: string;
|
|
46
|
+
/** A run scoped with --node covers only the workers it selected: it
|
|
47
|
+
* records their revisions, never one applied for the whole fleet.
|
|
48
|
+
*/
|
|
49
|
+
scoped?: boolean;
|
|
45
50
|
}
|
|
46
51
|
): RunRecord {
|
|
47
52
|
const env = state.env ?? process.env;
|
|
@@ -73,26 +78,33 @@ export function runRecord(
|
|
|
73
78
|
}
|
|
74
79
|
claimLease(state.context);
|
|
75
80
|
},
|
|
76
|
-
async settle(
|
|
81
|
+
async settle(ready, applied = {}) {
|
|
77
82
|
settled = true;
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
const converged = ready && state.scoped !== true;
|
|
84
|
+
if (converged || Object.keys(applied).length) {
|
|
85
|
+
// What was recorded since the approval stands: the run itself
|
|
86
|
+
// retires a worker whose record was removed, and another command
|
|
87
|
+
// may have. Only what this run decided is written over it.
|
|
88
|
+
const stored = (await loadEnrollment(null, env)) ?? approval;
|
|
84
89
|
storeEnrollment(
|
|
85
90
|
{
|
|
86
|
-
...
|
|
87
|
-
...
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
? {
|
|
91
|
+
...stored,
|
|
92
|
+
...(Object.keys(applied).length
|
|
93
|
+
? { workers: { ...stored.workers, ...applied } }
|
|
94
|
+
: {}),
|
|
95
|
+
...(converged
|
|
96
|
+
? {
|
|
97
|
+
appliedRevision: revision,
|
|
98
|
+
...(previous?.appliedRevision &&
|
|
99
|
+
previous.appliedRevision !== revision
|
|
100
|
+
? { previousRevision: previous.appliedRevision }
|
|
101
|
+
: {}),
|
|
102
|
+
}
|
|
92
103
|
: {}),
|
|
93
104
|
},
|
|
94
105
|
env
|
|
95
106
|
);
|
|
107
|
+
}
|
|
96
108
|
// The fleet converged either way: a bucket problem is reported, not fatal.
|
|
97
109
|
return replicaCheck(await publish().catch(() => 'unreachable' as const), {
|
|
98
110
|
advisory: true,
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
2
4
|
import { fleet as definition } from '../fleet-config';
|
|
3
5
|
import { sshArgs, shellQuote } from '../ssh';
|
|
4
6
|
import { resolveVaultConfig, usesVault } from '../vault-config';
|
|
5
7
|
import { workerTokenLocation } from '../worker-credentials';
|
|
8
|
+
import { runAnsible } from '../platform/ansible';
|
|
6
9
|
import type { Check, FleetNode, Runner } from '../types';
|
|
7
10
|
|
|
8
11
|
const TOOL_PATH =
|
|
@@ -213,3 +216,77 @@ function required(node: FleetNode): Check {
|
|
|
213
216
|
},
|
|
214
217
|
};
|
|
215
218
|
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* The manager's own Vault session renews itself every ten minutes, like each
|
|
222
|
+
* worker's: `ansible/macos/vault-renew.yml`, applied on every applying sync.
|
|
223
|
+
* Measured as the session answering and the job being loaded; a lapsed
|
|
224
|
+
* session needs one login by a person, and until then agents on the manager
|
|
225
|
+
* use the proxy client key they last read (src/agents/secrets.ts).
|
|
226
|
+
*/
|
|
227
|
+
export async function holdManagerVault(
|
|
228
|
+
run: Runner,
|
|
229
|
+
options: {
|
|
230
|
+
apply: boolean;
|
|
231
|
+
root: string;
|
|
232
|
+
home?: string;
|
|
233
|
+
uid?: number;
|
|
234
|
+
scratch?: string;
|
|
235
|
+
}
|
|
236
|
+
): Promise<Check[]> {
|
|
237
|
+
if (!usesVault()) return [];
|
|
238
|
+
const { addr } = resolveVaultConfig();
|
|
239
|
+
const home = options.home ?? homedir();
|
|
240
|
+
if (options.apply)
|
|
241
|
+
await runAnsible(
|
|
242
|
+
run,
|
|
243
|
+
join(options.root, 'ansible/macos/vault-renew.yml'),
|
|
244
|
+
{
|
|
245
|
+
input: JSON.stringify({
|
|
246
|
+
vault_address: addr,
|
|
247
|
+
mise: join(home, '.local/bin/mise'),
|
|
248
|
+
}),
|
|
249
|
+
timeoutMs: 120_000,
|
|
250
|
+
},
|
|
251
|
+
options.scratch
|
|
252
|
+
);
|
|
253
|
+
const loaded = (
|
|
254
|
+
await run(
|
|
255
|
+
[
|
|
256
|
+
'launchctl',
|
|
257
|
+
'print',
|
|
258
|
+
`gui/${options.uid ?? process.getuid?.() ?? 0}/${MANAGER_VAULT_RENEWAL}`,
|
|
259
|
+
],
|
|
260
|
+
{ timeoutMs: 10_000 }
|
|
261
|
+
)
|
|
262
|
+
).ok;
|
|
263
|
+
const session = (await run(lookup(), { timeoutMs: 20_000 })).ok;
|
|
264
|
+
return [vaultRenewalCheck(session, loaded, addr)];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function vaultRenewalCheck(
|
|
268
|
+
session: boolean,
|
|
269
|
+
loaded: boolean,
|
|
270
|
+
addr: string
|
|
271
|
+
): Check {
|
|
272
|
+
if (session && loaded)
|
|
273
|
+
return {
|
|
274
|
+
id: 'manager:vault-renewal',
|
|
275
|
+
status: 'pass',
|
|
276
|
+
summary:
|
|
277
|
+
'The manager is logged in to Vault and renews its session every ten minutes.',
|
|
278
|
+
};
|
|
279
|
+
return {
|
|
280
|
+
id: 'manager:vault-renewal',
|
|
281
|
+
status: 'fail',
|
|
282
|
+
summary: session ? 'VAULT_RENEWAL_MISSING' : 'VAULT_LOGIN_REQUIRED',
|
|
283
|
+
remedy: {
|
|
284
|
+
action: session
|
|
285
|
+
? `The manager's Vault session is not renewed and lapses at its TTL. Run devfleet fleet-sync on the manager; it loads ${MANAGER_VAULT_RENEWAL}.`
|
|
286
|
+
: `Log the manager in once, in your own terminal: vault login -address=${addr} -method=userpass username=${definition.worker?.vaultUser ?? 'YOUR_USERNAME'}. Then run devfleet fleet-sync. Until then the manager's agents use the proxy client key they last read, and new worker tokens cannot be issued.`,
|
|
287
|
+
},
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** The launchd label of the manager's Vault renewal job (vault-renew.yml). */
|
|
292
|
+
export const MANAGER_VAULT_RENEWAL = 'dev.devfleet.vault-renew';
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import type { FleetDefinition } from '../definition';
|
|
2
|
+
import {
|
|
3
|
+
controlRemedy,
|
|
4
|
+
TailnetControlError,
|
|
5
|
+
type TailnetControl,
|
|
6
|
+
} from '../tailnet-control';
|
|
7
|
+
import type { TailnetComputer } from '../tailscale';
|
|
8
|
+
import type { PackagePlaybook } from '../platform/ansible';
|
|
9
|
+
import type { Check, FleetNode, NodeReport } from '../types';
|
|
10
|
+
import { createWorkerAccount } from '../worker-account';
|
|
11
|
+
import {
|
|
12
|
+
rebootNeeded,
|
|
13
|
+
rebootWorker,
|
|
14
|
+
updatesPending,
|
|
15
|
+
type RebootReason,
|
|
16
|
+
} from '../worker-reboot';
|
|
17
|
+
|
|
18
|
+
/** What one worker's deployment needs around its converge: the converge
|
|
19
|
+
* itself (bootstrap, verification, monitoring), the tailnet control plane
|
|
20
|
+
* when the fleet declares one, the tailnet as the manager sees it, and what
|
|
21
|
+
* the run was asked to do.
|
|
22
|
+
*/
|
|
23
|
+
export interface WorkerDeployment {
|
|
24
|
+
definition: FleetDefinition;
|
|
25
|
+
/** Applies a playbook of the package from the manager. */
|
|
26
|
+
playbook: PackagePlaybook;
|
|
27
|
+
/** One converge of the worker; `tailnetKey` is handed to its bootstrap. */
|
|
28
|
+
converge: (
|
|
29
|
+
node: FleetNode,
|
|
30
|
+
revision: string,
|
|
31
|
+
tailnetKey?: string
|
|
32
|
+
) => Promise<NodeReport>;
|
|
33
|
+
/** Null when the fleet declares no tailnet control plane. */
|
|
34
|
+
control: () => Promise<TailnetControl | null>;
|
|
35
|
+
/** The devices of the tailnet, as the manager's Tailscale lists them. */
|
|
36
|
+
computers: () => Promise<TailnetComputer[]>;
|
|
37
|
+
/** The workers that converged in an earlier run. */
|
|
38
|
+
readyBefore: string[];
|
|
39
|
+
maintenanceReboot?: boolean;
|
|
40
|
+
force?: boolean;
|
|
41
|
+
/** Whether the bootstrap of this revision reads a tailnet key. */
|
|
42
|
+
acceptsTailnetKey: boolean;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Deploys one worker and, around its converge: hands a worker that is not
|
|
46
|
+
* on the tailnet a single-use key, keeps an enrolled worker's node key from
|
|
47
|
+
* expiring, creates the declared account on an image that lets only root
|
|
48
|
+
* in, and reboots the worker once when its first converge needs it (or for
|
|
49
|
+
* maintenance, when asked), converging it again to verify it.
|
|
50
|
+
*/
|
|
51
|
+
export async function deployWorker(
|
|
52
|
+
node: FleetNode,
|
|
53
|
+
revision: string,
|
|
54
|
+
deployment: WorkerDeployment
|
|
55
|
+
): Promise<NodeReport> {
|
|
56
|
+
const before: Check[] = [];
|
|
57
|
+
const key = await enrolmentKey(node, deployment, before);
|
|
58
|
+
before.push(...(await keyExpiry(node, deployment)));
|
|
59
|
+
let target = node;
|
|
60
|
+
let result = await deployment.converge(target, revision, key);
|
|
61
|
+
const account = await rootOnly(node, result, deployment);
|
|
62
|
+
if (account) {
|
|
63
|
+
before.push(account.check);
|
|
64
|
+
if (account.node) {
|
|
65
|
+
target = account.node;
|
|
66
|
+
result = await deployment.converge(target, revision, key);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const reason = rebootReason(node.id, result.checks, deployment);
|
|
70
|
+
if (reason) {
|
|
71
|
+
const reboot = await rebootWorker(
|
|
72
|
+
target,
|
|
73
|
+
{
|
|
74
|
+
revision,
|
|
75
|
+
reason,
|
|
76
|
+
...(reason === 'maintenance' && deployment.force
|
|
77
|
+
? { force: true }
|
|
78
|
+
: {}),
|
|
79
|
+
},
|
|
80
|
+
deployment.playbook
|
|
81
|
+
);
|
|
82
|
+
if (reboot.status === 'pass')
|
|
83
|
+
result = await deployment.converge(target, revision);
|
|
84
|
+
result.checks.unshift(reboot);
|
|
85
|
+
}
|
|
86
|
+
result.checks.unshift(...before);
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Why this worker is rebooted now, if it is. A worker that converged
|
|
91
|
+
* before is never rebooted for its first converge; only an explicit
|
|
92
|
+
* maintenance run reboots it.
|
|
93
|
+
*/
|
|
94
|
+
function rebootReason(
|
|
95
|
+
id: string,
|
|
96
|
+
checks: Check[],
|
|
97
|
+
deployment: WorkerDeployment
|
|
98
|
+
): RebootReason | null {
|
|
99
|
+
if (deployment.maintenanceReboot && updatesPending(checks))
|
|
100
|
+
return 'maintenance';
|
|
101
|
+
if (!deployment.readyBefore.includes(id) && rebootNeeded(checks))
|
|
102
|
+
return 'first-converge';
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** A single-use, pre-authorised, tagged key for a worker the roster has no
|
|
107
|
+
* tailnet identity for yet. The key is never reported.
|
|
108
|
+
*/
|
|
109
|
+
async function enrolmentKey(
|
|
110
|
+
node: FleetNode,
|
|
111
|
+
deployment: WorkerDeployment,
|
|
112
|
+
checks: Check[]
|
|
113
|
+
): Promise<string | undefined> {
|
|
114
|
+
if (node.tailscaleId || !deployment.acceptsTailnetKey) return undefined;
|
|
115
|
+
try {
|
|
116
|
+
const control = await deployment.control();
|
|
117
|
+
if (!control) return undefined;
|
|
118
|
+
const minted = await control.createAuthKey({ worker: node.id });
|
|
119
|
+
checks.push({
|
|
120
|
+
id: 'tailscale:enrolment-key',
|
|
121
|
+
status: 'pass',
|
|
122
|
+
summary: `A single-use, pre-authorised key tagged ${deployment.definition.tailnet!.tag}, expiring within minutes, was handed to the worker on the bootstrap's stdin.`,
|
|
123
|
+
});
|
|
124
|
+
return minted.key;
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (!(error instanceof TailnetControlError)) throw error;
|
|
127
|
+
checks.push(controlFailure('tailscale:enrolment-key', error));
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** An enrolled worker's node key never expires: the control plane disables
|
|
133
|
+
* expiry for a worker whose key has one (tagged workers have none on the
|
|
134
|
+
* hosted Tailscale). The tailscale:key-expiry check still measures it.
|
|
135
|
+
*/
|
|
136
|
+
async function keyExpiry(
|
|
137
|
+
node: FleetNode,
|
|
138
|
+
deployment: WorkerDeployment
|
|
139
|
+
): Promise<Check[]> {
|
|
140
|
+
if (!node.tailscaleId) return [];
|
|
141
|
+
try {
|
|
142
|
+
const control = await deployment.control();
|
|
143
|
+
if (!control) return [];
|
|
144
|
+
const peer = (await deployment.computers()).find(
|
|
145
|
+
(computer) => computer.id === node.tailscaleId
|
|
146
|
+
);
|
|
147
|
+
if (peer?.keyExpiry === undefined) return [];
|
|
148
|
+
await control.disableKeyExpiry(node.tailscaleId);
|
|
149
|
+
return [
|
|
150
|
+
{
|
|
151
|
+
id: 'tailscale:key-expiry-policy',
|
|
152
|
+
status: 'pass',
|
|
153
|
+
summary:
|
|
154
|
+
'Key expiry of this worker was disabled through the tailnet control plane; tailscale:key-expiry measures it again on the next run.',
|
|
155
|
+
},
|
|
156
|
+
];
|
|
157
|
+
} catch (error) {
|
|
158
|
+
if (!(error instanceof TailnetControlError)) throw error;
|
|
159
|
+
return [
|
|
160
|
+
{
|
|
161
|
+
...controlFailure('tailscale:key-expiry-policy', error),
|
|
162
|
+
status: 'warn',
|
|
163
|
+
},
|
|
164
|
+
];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function controlFailure(id: string, error: TailnetControlError): Check {
|
|
169
|
+
return {
|
|
170
|
+
id,
|
|
171
|
+
status: 'fail',
|
|
172
|
+
summary: error.code,
|
|
173
|
+
remedy: { action: controlRemedy(error.code) },
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The bootstrap refused to run as root: the alias logs in as root. With a
|
|
178
|
+
* declared account, it is created over that login and the run continues as
|
|
179
|
+
* it (`user@alias`); the next runs need the alias to name it, since the SSH
|
|
180
|
+
* policy turns root login off.
|
|
181
|
+
*/
|
|
182
|
+
async function rootOnly(
|
|
183
|
+
node: FleetNode,
|
|
184
|
+
result: NodeReport,
|
|
185
|
+
deployment: WorkerDeployment
|
|
186
|
+
): Promise<{ check: Check; node?: FleetNode } | null> {
|
|
187
|
+
if (
|
|
188
|
+
!result.checks.some(
|
|
189
|
+
(check) => check.summary === 'ROOT_DEVELOPMENT_USER_REJECTED'
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
return null;
|
|
193
|
+
const user = deployment.definition.workers.find(
|
|
194
|
+
(worker) => worker.id === node.id
|
|
195
|
+
)?.network?.user;
|
|
196
|
+
const check = await createWorkerAccount(
|
|
197
|
+
{ id: node.id, ssh: node.ssh, login: '' },
|
|
198
|
+
user,
|
|
199
|
+
deployment.playbook
|
|
200
|
+
);
|
|
201
|
+
if (check.status !== 'pass') return { check };
|
|
202
|
+
return {
|
|
203
|
+
check: {
|
|
204
|
+
...check,
|
|
205
|
+
status: 'warn',
|
|
206
|
+
summary: `${check.summary} This run continued as ${user}@${node.ssh}; set User ${user} for the SSH alias ${node.ssh}, since root login is turned off.`,
|
|
207
|
+
},
|
|
208
|
+
node: {
|
|
209
|
+
...node,
|
|
210
|
+
ssh: `${user}@${node.ssh}`,
|
|
211
|
+
workspace: `/home/${user}/.local/share/personal/fleet`,
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
}
|