@crvouga/dev-fleet 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -9
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +55 -9
- package/ansible/macos/manager.yml +154 -7
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +7 -3
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +19 -11
- package/docs/consumer-setup.md +42 -27
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +10 -3
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +98 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +67 -6
- package/docs/ubuntu-sync.md +78 -28
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1967 -170
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +327 -8
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +61 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +9 -3
- package/src/cli/fleet-node-lifecycle.ts +313 -64
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +51 -35
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +4 -0
- package/src/cli/fleet-sync-manager.ts +103 -33
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +151 -9
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +28 -16
- package/src/cli/fleet-sync-vault.ts +77 -0
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +150 -19
- package/src/cli/fleet-validate.ts +59 -21
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +115 -4
- package/src/cli/registry.ts +14 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +332 -9
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +37 -15
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +65 -4
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +4 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capability-checks.ts +55 -16
- package/src/worker-capacity.ts +31 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +27 -4
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +17 -1
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +16 -11
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
|
@@ -18,17 +18,27 @@ import {
|
|
|
18
18
|
convergeT3Service,
|
|
19
19
|
runningVersionCheck,
|
|
20
20
|
t3StartedAt,
|
|
21
|
+
t3Unit,
|
|
21
22
|
} from './platform/t3-service';
|
|
23
|
+
import {
|
|
24
|
+
featuresCheck,
|
|
25
|
+
presentParts,
|
|
26
|
+
workerFeatures,
|
|
27
|
+
type WorkerFeatures,
|
|
28
|
+
} from './worker-features';
|
|
22
29
|
import { userInfo } from 'node:os';
|
|
23
30
|
import { resolve } from 'node:path';
|
|
24
31
|
import { auditUbuntu } from './platform/ubuntu-audit';
|
|
32
|
+
import { detectMachineClass } from './machine/machine-class';
|
|
25
33
|
import { convergeGitIdentity } from './git-identity';
|
|
26
34
|
import { workerSelection } from '../bootstrap/worker-tools';
|
|
35
|
+
import { workerExposure } from './exposure';
|
|
27
36
|
import {
|
|
28
37
|
applyAgentProfiles,
|
|
29
38
|
attempt,
|
|
30
39
|
configureWorkstation,
|
|
31
40
|
} from './node-converge-shared';
|
|
41
|
+
import { loginServer } from './tailnet-control';
|
|
32
42
|
import type { Check, Runner } from './types';
|
|
33
43
|
|
|
34
44
|
/** Ubuntu worker convergence: Ansible owns OS policy, services and recovery; the
|
|
@@ -36,10 +46,11 @@ import type { Check, Runner } from './types';
|
|
|
36
46
|
*/
|
|
37
47
|
export async function convergeLinux(
|
|
38
48
|
root: string,
|
|
39
|
-
target: { node: string; revision: string },
|
|
49
|
+
target: { node: string; revision: string; restartBusy?: boolean },
|
|
40
50
|
run: Runner
|
|
41
51
|
): Promise<Check[]> {
|
|
42
52
|
const { node, revision } = target;
|
|
53
|
+
const restartBusy = target.restartBusy === true;
|
|
43
54
|
const audit = await auditUbuntu(run);
|
|
44
55
|
if (!audit.supported || !audit.passwordlessSudo)
|
|
45
56
|
return [
|
|
@@ -58,12 +69,23 @@ export async function convergeLinux(
|
|
|
58
69
|
const checks: Check[] = [];
|
|
59
70
|
checks.push(
|
|
60
71
|
...(await attempt('tailscale:configuration', async () => [
|
|
61
|
-
await applyTailscale(root, run),
|
|
72
|
+
await applyTailscale(root, node, run),
|
|
62
73
|
]))
|
|
63
74
|
);
|
|
75
|
+
const features = workerFeatures(fleet, node);
|
|
76
|
+
const before = await presentParts(run, userInfo().username, features);
|
|
64
77
|
const { baseline, verified } = await baselineStep(root, run, node);
|
|
65
78
|
checks.push(...baseline, ...verified);
|
|
66
|
-
checks.push(
|
|
79
|
+
checks.push(
|
|
80
|
+
featuresCheck(
|
|
81
|
+
features,
|
|
82
|
+
await presentParts(run, userInfo().username, features),
|
|
83
|
+
before
|
|
84
|
+
)
|
|
85
|
+
);
|
|
86
|
+
checks.push(
|
|
87
|
+
...(await t3AfterBaseline(root, { node, baseline, run, restartBusy }))
|
|
88
|
+
);
|
|
67
89
|
checks.push(
|
|
68
90
|
...(await attempt('workstation:configuration', () =>
|
|
69
91
|
configureWorkstation(root, run, node)
|
|
@@ -120,9 +142,10 @@ async function baselineStep(
|
|
|
120
142
|
run: Runner,
|
|
121
143
|
node: string
|
|
122
144
|
): Promise<{ baseline: Check[]; verified: Check[] }> {
|
|
145
|
+
const { machine } = await detectMachineClass(run);
|
|
123
146
|
const measure = async () => [
|
|
124
147
|
...(await attempt('ubuntu:boot-policy', () =>
|
|
125
|
-
verifyUbuntuBoot(userInfo().username, run)
|
|
148
|
+
verifyUbuntuBoot(userInfo().username, run, machine)
|
|
126
149
|
)),
|
|
127
150
|
...(await attempt('ubuntu:resource-safety', () =>
|
|
128
151
|
verifyResourceSafety(userInfo().homedir, run)
|
|
@@ -151,6 +174,7 @@ async function applyBaseline(
|
|
|
151
174
|
node: string
|
|
152
175
|
): Promise<{ cached: boolean; checks: Check[] }> {
|
|
153
176
|
const user = userInfo();
|
|
177
|
+
const features = workerFeatures(fleet, node);
|
|
154
178
|
const ubuntu = resolve(root, 'ansible/ubuntu');
|
|
155
179
|
const result = await runAnsibleWithReceipt(
|
|
156
180
|
run,
|
|
@@ -165,13 +189,18 @@ async function applyBaseline(
|
|
|
165
189
|
development_home: user.homedir,
|
|
166
190
|
// The web profile brings headless Chromium (tasks/browser.yml).
|
|
167
191
|
worker_tool_profiles: workerSelection(fleet, node).profiles,
|
|
192
|
+
// What the firewall opens beyond the tailnet (tasks/firewall.yml).
|
|
193
|
+
worker_exposure: workerExposure(fleet, node).policy,
|
|
194
|
+
// What the declared features leave out (tasks/features.yml).
|
|
195
|
+
worker_containers: features.containers,
|
|
196
|
+
worker_session: features.session,
|
|
168
197
|
}),
|
|
169
198
|
},
|
|
170
199
|
{
|
|
171
200
|
name: 'ubuntu-baseline',
|
|
172
201
|
sources: [ubuntu, resolve(root, 'ansible/github')],
|
|
173
202
|
force,
|
|
174
|
-
verify: () => baselineServicesActive(run),
|
|
203
|
+
verify: () => baselineServicesActive(run, features),
|
|
175
204
|
}
|
|
176
205
|
);
|
|
177
206
|
return {
|
|
@@ -191,14 +220,17 @@ async function applyBaseline(
|
|
|
191
220
|
}
|
|
192
221
|
|
|
193
222
|
/** Cheap probe for what the baseline owns: system services and user timers. */
|
|
194
|
-
async function baselineServicesActive(
|
|
223
|
+
async function baselineServicesActive(
|
|
224
|
+
run: Runner,
|
|
225
|
+
features: WorkerFeatures
|
|
226
|
+
): Promise<boolean> {
|
|
195
227
|
const system = await run([
|
|
196
228
|
'systemctl',
|
|
197
229
|
'is-active',
|
|
198
230
|
'--quiet',
|
|
199
231
|
'ssh.service',
|
|
200
232
|
'tailscaled.service',
|
|
201
|
-
'docker.service',
|
|
233
|
+
...(features.containers ? ['docker.service'] : []),
|
|
202
234
|
]);
|
|
203
235
|
if (!system.ok) return false;
|
|
204
236
|
const user = await run([
|
|
@@ -212,12 +244,27 @@ async function baselineServicesActive(run: Runner): Promise<boolean> {
|
|
|
212
244
|
return user.ok;
|
|
213
245
|
}
|
|
214
246
|
|
|
215
|
-
|
|
247
|
+
/** A worker that was never logged in joins under its declared ID when the
|
|
248
|
+
* seed or the bootstrap left it a single-use key; the key carries the fleet's
|
|
249
|
+
* tag.
|
|
250
|
+
*/
|
|
251
|
+
async function applyTailscale(
|
|
252
|
+
root: string,
|
|
253
|
+
node: string,
|
|
254
|
+
run: Runner
|
|
255
|
+
): Promise<Check> {
|
|
216
256
|
const ubuntu = resolve(root, 'ansible/ubuntu');
|
|
217
257
|
const tailscale = await runAnsibleWithReceipt(
|
|
218
258
|
run,
|
|
219
259
|
'tailscale.yml',
|
|
220
|
-
{
|
|
260
|
+
{
|
|
261
|
+
cwd: ubuntu,
|
|
262
|
+
timeoutMs: 300_000,
|
|
263
|
+
input: JSON.stringify({
|
|
264
|
+
tailnet_hostname: node,
|
|
265
|
+
tailnet_login_server: loginServer(fleet.tailnet) ?? '',
|
|
266
|
+
}),
|
|
267
|
+
},
|
|
221
268
|
{
|
|
222
269
|
name: 'ubuntu-tailscale',
|
|
223
270
|
sources: [ubuntu],
|
|
@@ -243,23 +290,30 @@ async function applyTailscale(root: string, run: Runner): Promise<Check> {
|
|
|
243
290
|
async function convergeT3(
|
|
244
291
|
root: string,
|
|
245
292
|
node: string,
|
|
246
|
-
run: Runner
|
|
293
|
+
run: Runner,
|
|
294
|
+
restartBusy: boolean
|
|
247
295
|
): Promise<Check[]> {
|
|
248
296
|
const artifact = managedT3();
|
|
249
|
-
const
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
297
|
+
const { session } = workerFeatures(fleet, node);
|
|
298
|
+
const startedAt = await t3StartedAt(run, session);
|
|
299
|
+
const checks = await convergeT3Service(artifact, run, {
|
|
300
|
+
prepare: async () => {
|
|
301
|
+
const staged = await stageT3Policy(root, artifact.path, run, {
|
|
302
|
+
node,
|
|
303
|
+
restartBusy,
|
|
304
|
+
});
|
|
255
305
|
return staged.status === 'pass' ? [] : [staged];
|
|
256
306
|
},
|
|
257
|
-
() =>
|
|
258
|
-
|
|
307
|
+
install: () =>
|
|
308
|
+
session === 'headless'
|
|
309
|
+
? installT3Headless(root, artifact.path, run, restartBusy)
|
|
310
|
+
: installT3Desktop(root, run, restartBusy),
|
|
311
|
+
session,
|
|
312
|
+
});
|
|
259
313
|
if (checks.every((check) => check.status === 'pass'))
|
|
260
314
|
checks.push(
|
|
261
315
|
await loadedT3Policy(artifact.path, run, {
|
|
262
|
-
restarted: startedAt !== (await t3StartedAt(run)),
|
|
316
|
+
restarted: startedAt !== (await t3StartedAt(run, session)),
|
|
263
317
|
})
|
|
264
318
|
);
|
|
265
319
|
// Settings are watched by T3: providers, names and the profile apply
|
|
@@ -275,12 +329,16 @@ async function convergeT3(
|
|
|
275
329
|
const serviceReady = checks.some(
|
|
276
330
|
(check) => check.id === 't3:service' && check.status === 'pass'
|
|
277
331
|
);
|
|
278
|
-
if (serviceReady)
|
|
332
|
+
if (serviceReady)
|
|
333
|
+
checks.push(
|
|
334
|
+
await runningVersionCheck(run, userInfo().uid, Date.now(), session)
|
|
335
|
+
);
|
|
279
336
|
if (serviceReady)
|
|
280
337
|
checks.push(
|
|
281
338
|
...(await convergeT3Route(run, () => publishT3Route(root, run)))
|
|
282
339
|
);
|
|
283
|
-
if (serviceReady)
|
|
340
|
+
if (serviceReady)
|
|
341
|
+
checks.push(...(await enableWorkerRecovery(root, run, session)));
|
|
284
342
|
return checks;
|
|
285
343
|
}
|
|
286
344
|
|
|
@@ -293,7 +351,11 @@ async function publishT3Route(root: string, run: Runner): Promise<boolean> {
|
|
|
293
351
|
return result.ok;
|
|
294
352
|
}
|
|
295
353
|
|
|
296
|
-
async function installT3Desktop(
|
|
354
|
+
async function installT3Desktop(
|
|
355
|
+
root: string,
|
|
356
|
+
run: Runner,
|
|
357
|
+
restartBusy: boolean
|
|
358
|
+
): Promise<boolean> {
|
|
297
359
|
const user = userInfo();
|
|
298
360
|
const desktop = managedT3Desktop();
|
|
299
361
|
const ubuntu = resolve(root, 'ansible/ubuntu');
|
|
@@ -311,6 +373,7 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
|
|
|
311
373
|
t3_desktop_url: desktop.url,
|
|
312
374
|
t3_desktop_name: desktop.name,
|
|
313
375
|
t3_desktop_sha256: desktop.sha256,
|
|
376
|
+
t3_restart_busy: restartBusy,
|
|
314
377
|
}),
|
|
315
378
|
},
|
|
316
379
|
{
|
|
@@ -341,11 +404,55 @@ async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
|
|
|
341
404
|
return result.ok;
|
|
342
405
|
}
|
|
343
406
|
|
|
407
|
+
/** A headless worker's T3: the pinned server the toolchain installed, as a
|
|
408
|
+
* lingering user service (t3-headless.yml). */
|
|
409
|
+
async function installT3Headless(
|
|
410
|
+
root: string,
|
|
411
|
+
server: string,
|
|
412
|
+
run: Runner,
|
|
413
|
+
restartBusy: boolean
|
|
414
|
+
): Promise<boolean> {
|
|
415
|
+
const user = userInfo();
|
|
416
|
+
const ubuntu = resolve(root, 'ansible/ubuntu');
|
|
417
|
+
const unit = t3Unit('headless');
|
|
418
|
+
const result = await runAnsibleWithReceipt(
|
|
419
|
+
run,
|
|
420
|
+
resolve(ubuntu, 't3-headless.yml'),
|
|
421
|
+
{
|
|
422
|
+
cwd: ubuntu,
|
|
423
|
+
timeoutMs: 600_000,
|
|
424
|
+
input: JSON.stringify({
|
|
425
|
+
development_source: root,
|
|
426
|
+
development_user: user.username,
|
|
427
|
+
development_uid: user.uid,
|
|
428
|
+
development_home: user.homedir,
|
|
429
|
+
t3_server_path: server,
|
|
430
|
+
t3_restart_busy: restartBusy,
|
|
431
|
+
}),
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
name: 'ubuntu-t3-headless',
|
|
435
|
+
sources: [ubuntu],
|
|
436
|
+
// A stopped or disabled server reruns the play, which starts it.
|
|
437
|
+
verify: async () =>
|
|
438
|
+
(await run(['systemctl', '--user', 'is-enabled', '--quiet', unit]))
|
|
439
|
+
.ok &&
|
|
440
|
+
(await run(['systemctl', '--user', 'is-active', '--quiet', unit])).ok,
|
|
441
|
+
}
|
|
442
|
+
);
|
|
443
|
+
return result.ok;
|
|
444
|
+
}
|
|
445
|
+
|
|
344
446
|
async function t3AfterBaseline(
|
|
345
447
|
root: string,
|
|
346
|
-
context: {
|
|
448
|
+
context: {
|
|
449
|
+
node: string;
|
|
450
|
+
baseline: Check[];
|
|
451
|
+
run: Runner;
|
|
452
|
+
restartBusy: boolean;
|
|
453
|
+
}
|
|
347
454
|
): Promise<Check[]> {
|
|
348
|
-
const { node, baseline, run } = context;
|
|
455
|
+
const { node, baseline, run, restartBusy } = context;
|
|
349
456
|
if (baseline.some((check) => check.status !== 'pass'))
|
|
350
457
|
return [
|
|
351
458
|
{
|
|
@@ -358,12 +465,13 @@ async function t3AfterBaseline(
|
|
|
358
465
|
},
|
|
359
466
|
},
|
|
360
467
|
];
|
|
361
|
-
return attempt('t3:service', () => convergeT3(root, node, run));
|
|
468
|
+
return attempt('t3:service', () => convergeT3(root, node, run, restartBusy));
|
|
362
469
|
}
|
|
363
470
|
|
|
364
471
|
async function enableWorkerRecovery(
|
|
365
472
|
root: string,
|
|
366
|
-
run: Runner
|
|
473
|
+
run: Runner,
|
|
474
|
+
session: WorkerFeatures['session']
|
|
367
475
|
): Promise<Check[]> {
|
|
368
476
|
const user = userInfo();
|
|
369
477
|
const ubuntu = resolve(root, 'ansible/ubuntu');
|
|
@@ -378,6 +486,7 @@ async function enableWorkerRecovery(
|
|
|
378
486
|
development_user: user.username,
|
|
379
487
|
development_uid: user.uid,
|
|
380
488
|
development_home: user.homedir,
|
|
489
|
+
...(session === 'headless' ? { t3_unit: t3Unit(session) } : {}),
|
|
381
490
|
}),
|
|
382
491
|
},
|
|
383
492
|
{
|
|
@@ -400,7 +509,7 @@ async function enableWorkerRecovery(
|
|
|
400
509
|
id: 'worker:service-recovery',
|
|
401
510
|
status: result.ok ? 'pass' : 'fail',
|
|
402
511
|
summary: result.ok
|
|
403
|
-
?
|
|
512
|
+
? `T3 recovery enabled: a stopped ${session === 'headless' ? 'server' : 'desktop app'} is started and an unresponsive one restarted after three failed one-minute health checks.`
|
|
404
513
|
: 'WORKER_RECOVERY_SETUP_FAILED',
|
|
405
514
|
},
|
|
406
515
|
];
|
package/src/node-converge.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { runLockedCommand as runCommand } from './runner';
|
|
2
2
|
import { isRevision } from './machine/state';
|
|
3
3
|
import { convergeLinux } from './node-converge-linux';
|
|
4
|
+
import { T3_RESTART_MODES } from './platform/t3-service';
|
|
4
5
|
import type { Check } from './types';
|
|
5
6
|
|
|
6
7
|
/** Internal remote entry, run on a worker only. The manager owns the operation
|
|
@@ -8,14 +9,23 @@ import type { Check } from './types';
|
|
|
8
9
|
* prints child output. Workers are Linux.
|
|
9
10
|
*/
|
|
10
11
|
async function main(): Promise<Check[]> {
|
|
11
|
-
const [node, revision] = process.argv.slice(2);
|
|
12
|
-
if (
|
|
12
|
+
const [node, revision, t3Restart = 'keep-turns'] = process.argv.slice(2);
|
|
13
|
+
if (
|
|
14
|
+
!node ||
|
|
15
|
+
!/^[a-z][a-z0-9-]{0,62}$/.test(node) ||
|
|
16
|
+
!isRevision(revision) ||
|
|
17
|
+
!(T3_RESTART_MODES as readonly string[]).includes(t3Restart)
|
|
18
|
+
)
|
|
13
19
|
throw new Error();
|
|
14
20
|
const root = process.cwd();
|
|
15
21
|
const head = await runCommand(['git', 'rev-parse', 'HEAD']);
|
|
16
22
|
if (!head.ok || head.stdout.trim() !== revision) throw new Error();
|
|
17
23
|
if (process.platform === 'linux')
|
|
18
|
-
return convergeLinux(
|
|
24
|
+
return convergeLinux(
|
|
25
|
+
root,
|
|
26
|
+
{ node, revision, restartBusy: t3Restart === 'restart-busy' },
|
|
27
|
+
runCommand
|
|
28
|
+
);
|
|
19
29
|
return [
|
|
20
30
|
{
|
|
21
31
|
id: 'platform:unsupported',
|
|
@@ -19,12 +19,18 @@ export interface Receipt {
|
|
|
19
19
|
force?: boolean;
|
|
20
20
|
/** Receipt directory; defaults to the user's fleet state. */
|
|
21
21
|
directory?: string;
|
|
22
|
+
/** Run the playbook in check mode first, and apply only when that passes. */
|
|
23
|
+
checkFirst?: boolean;
|
|
24
|
+
/** Where the playbook's variables are written while it runs. */
|
|
25
|
+
scratch?: string;
|
|
22
26
|
now?: number;
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
export interface ReceiptResult extends CommandResult {
|
|
26
30
|
/** True when the playbook was skipped on an unchanged, verified receipt. */
|
|
27
31
|
cached: boolean;
|
|
32
|
+
/** Which run the result is of: the check-mode run when it failed. */
|
|
33
|
+
stage?: 'check' | 'apply';
|
|
28
34
|
}
|
|
29
35
|
|
|
30
36
|
export function receiptDirectory(home = homedir()): string {
|
|
@@ -56,10 +62,19 @@ export async function runAnsibleWithReceipt(
|
|
|
56
62
|
)
|
|
57
63
|
return { ok: true, reason: 'exit', stdout: '', stderr: '', cached: true };
|
|
58
64
|
await rm(path, { force: true }).catch(() => undefined);
|
|
59
|
-
|
|
65
|
+
if (receipt.checkFirst) {
|
|
66
|
+
const checked = await runAnsible(
|
|
67
|
+
run,
|
|
68
|
+
playbook,
|
|
69
|
+
{ ...options, check: true },
|
|
70
|
+
receipt.scratch
|
|
71
|
+
);
|
|
72
|
+
if (!checked.ok) return { ...checked, cached: false, stage: 'check' };
|
|
73
|
+
}
|
|
74
|
+
const result = await runAnsible(run, playbook, options, receipt.scratch);
|
|
60
75
|
if (result.ok && key !== null)
|
|
61
76
|
await record(directory, path, { version: 1, key, appliedAt: now });
|
|
62
|
-
return { ...result, cached: false };
|
|
77
|
+
return { ...result, cached: false, stage: 'apply' };
|
|
63
78
|
}
|
|
64
79
|
|
|
65
80
|
/** Best effort: a missing receipt only means the next run applies in full. */
|
package/src/platform/ansible.ts
CHANGED
|
@@ -30,6 +30,14 @@ export function ansibleCommand(
|
|
|
30
30
|
];
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/** The package's Ansible configuration (ansible/ansible.cfg): how Ansible
|
|
34
|
+
* reaches a worker over SSH (timeouts, retries, pipelining, connection
|
|
35
|
+
* reuse) beside the options src/ssh-connection.ts gives every session.
|
|
36
|
+
*/
|
|
37
|
+
export function ansibleConfig(root = PACKAGE_ROOT): NodeJS.ProcessEnv {
|
|
38
|
+
return { ANSIBLE_CONFIG: join(root, 'ansible', 'ansible.cfg') };
|
|
39
|
+
}
|
|
40
|
+
|
|
33
41
|
/** Private fleet state on disk, never /tmp: a worker's /tmp is RAM-backed with a
|
|
34
42
|
* per-user quota that agent caches can exhaust, and the baseline that bounds it
|
|
35
43
|
* must still be able to run then.
|
|
@@ -45,7 +53,11 @@ export function variablesDirectory(home = homedir()): string {
|
|
|
45
53
|
export async function runAnsible(
|
|
46
54
|
run: Runner,
|
|
47
55
|
playbook: string,
|
|
48
|
-
options: CommandOptions & {
|
|
56
|
+
options: CommandOptions & {
|
|
57
|
+
input: string;
|
|
58
|
+
/** Run in check mode: report what would change, change nothing. */
|
|
59
|
+
check?: boolean;
|
|
60
|
+
},
|
|
49
61
|
scratch = variablesDirectory()
|
|
50
62
|
): Promise<CommandResult> {
|
|
51
63
|
if (Buffer.byteLength(options.input) > 256 * 1024)
|
|
@@ -55,12 +67,36 @@ export async function runAnsible(
|
|
|
55
67
|
const path = join(directory, 'variables.json');
|
|
56
68
|
try {
|
|
57
69
|
await writeFile(path, options.input, { mode: 0o600, flag: 'wx' });
|
|
58
|
-
const { input: _input, ...commandOptions } = options;
|
|
70
|
+
const { input: _input, check, ...commandOptions } = options;
|
|
59
71
|
return await run(
|
|
60
|
-
[
|
|
61
|
-
|
|
72
|
+
[
|
|
73
|
+
...ansibleCommand(),
|
|
74
|
+
playbook,
|
|
75
|
+
...(check ? ['--check'] : []),
|
|
76
|
+
'--extra-vars',
|
|
77
|
+
'@' + path,
|
|
78
|
+
],
|
|
79
|
+
{ ...commandOptions, env: { ...ansibleConfig(), ...commandOptions.env } }
|
|
62
80
|
);
|
|
63
81
|
} finally {
|
|
64
82
|
await rm(directory, { recursive: true, force: true });
|
|
65
83
|
}
|
|
66
84
|
}
|
|
85
|
+
|
|
86
|
+
/** A playbook of the package, applied from the manager with these variables:
|
|
87
|
+
* one that addresses a worker over SSH itself (ansible/ubuntu/reboot.yml).
|
|
88
|
+
* Scenarios answer it in place of Ansible.
|
|
89
|
+
*/
|
|
90
|
+
export type PackagePlaybook = (
|
|
91
|
+
name: string,
|
|
92
|
+
variables: object,
|
|
93
|
+
timeoutMs: number
|
|
94
|
+
) => Promise<CommandResult>;
|
|
95
|
+
|
|
96
|
+
export function packagePlaybook(run: Runner): PackagePlaybook {
|
|
97
|
+
return (name, variables, timeoutMs) =>
|
|
98
|
+
runAnsible(run, join(PACKAGE_ROOT, 'ansible', name), {
|
|
99
|
+
input: JSON.stringify(variables),
|
|
100
|
+
timeoutMs,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
@@ -15,6 +15,14 @@ export interface BootItem {
|
|
|
15
15
|
detail: string;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
/** `notApplicable` names settings the machine lacks (a virtual Mac has no
|
|
19
|
+
* womp): the playbook skips them, so they are not drift.
|
|
20
|
+
*/
|
|
21
|
+
export interface PowerDrift {
|
|
22
|
+
drifted: string[];
|
|
23
|
+
notApplicable: string[];
|
|
24
|
+
}
|
|
25
|
+
|
|
18
26
|
/** Where the manager runs, behind one interface: how it is named, how it is
|
|
19
27
|
* kept awake, how its services are supervised and where they keep data. The
|
|
20
28
|
* macOS host is the default and the reference.
|
|
@@ -29,8 +37,9 @@ export interface ManagerHost {
|
|
|
29
37
|
dataDirectory(home: string): string;
|
|
30
38
|
/** The machine's local host name, as `manager.id` must name it. */
|
|
31
39
|
localHostName(run: Runner): Promise<string | null>;
|
|
32
|
-
/** The power settings that differ from the manager's policy
|
|
33
|
-
|
|
40
|
+
/** The power settings that differ from the manager's policy, and those
|
|
41
|
+
* this machine does not have at all. */
|
|
42
|
+
powerDrift(run: Runner): Promise<PowerDrift>;
|
|
34
43
|
/** What brings the manager's services back after a power cut, unattended. */
|
|
35
44
|
bootRecovery(run: Runner, home: string): Promise<BootItem[]>;
|
|
36
45
|
/** How a person applies what bootRecovery measures. */
|
|
@@ -78,10 +87,18 @@ export const MACOS_MANAGER: ManagerHost = {
|
|
|
78
87
|
return result.ok ? hostId(result.stdout.trim()) : null;
|
|
79
88
|
},
|
|
80
89
|
async powerDrift(run) {
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
90
|
+
const result = await run(['pmset', '-g', 'custom']);
|
|
91
|
+
const ac = result.ok ? result.stdout.split('AC Power:')[1] : undefined;
|
|
92
|
+
const observed = (key: string) =>
|
|
93
|
+
new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac ?? '')?.[1];
|
|
94
|
+
const absent = (key: string) =>
|
|
95
|
+
ac !== undefined && observed(key) === undefined;
|
|
96
|
+
return {
|
|
97
|
+
drifted: Object.entries(macos.pmset)
|
|
98
|
+
.filter(([key, value]) => !absent(key) && observed(key) !== value)
|
|
99
|
+
.map(([key]) => key),
|
|
100
|
+
notApplicable: Object.keys(macos.pmset).filter(absent),
|
|
101
|
+
};
|
|
85
102
|
},
|
|
86
103
|
async bootRecovery(run, home) {
|
|
87
104
|
const observed = await acPower(run);
|
|
@@ -126,7 +143,7 @@ export const MACOS_MANAGER: ManagerHost = {
|
|
|
126
143
|
];
|
|
127
144
|
},
|
|
128
145
|
bootRemedy: (workspace) =>
|
|
129
|
-
`
|
|
146
|
+
`fleet-sync applies ansible/macos/manager.yml on every run (autorestart, Docker Desktop at login, LaunchAgents); autorestart needs the one grant an administrator gives: cd ${workspace} && ${ansibleCommand('darwin').join(' ')} -e user=$(id -un) -K ansible/macos/manager-privilege.yml. FileVault and automatic login are a person’s choice: docs/manager-recovery.md.`,
|
|
130
147
|
};
|
|
131
148
|
|
|
132
149
|
/** The sleep states a Linux manager must never enter. */
|
|
@@ -153,7 +170,7 @@ export const LINUX_MANAGER: ManagerHost = {
|
|
|
153
170
|
const result = await run(['systemctl', 'is-enabled', target]);
|
|
154
171
|
if (result.stdout.trim() !== 'masked') drifted.push(target);
|
|
155
172
|
}
|
|
156
|
-
return drifted;
|
|
173
|
+
return { drifted, notApplicable: [] };
|
|
157
174
|
},
|
|
158
175
|
async bootRecovery(run) {
|
|
159
176
|
const user = userInfo().username;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { threshold } from '../worker-sizing';
|
|
2
2
|
import type { Check, Runner } from '../types';
|
|
3
3
|
|
|
4
4
|
const AGENT_DROP_IN =
|
|
@@ -9,7 +9,6 @@ const GUARDS = [
|
|
|
9
9
|
'personal-fleet-disk-gc.timer',
|
|
10
10
|
];
|
|
11
11
|
const MIB = 1024 * 1024;
|
|
12
|
-
const FLOOR_MIB = policy.resourceSafety.diskGuardFloorMiB;
|
|
13
12
|
|
|
14
13
|
/** A failure code and remedy, or undefined when the probe passes. */
|
|
15
14
|
type Probe = (run: Runner) => Promise<[string, string] | undefined>;
|
|
@@ -61,17 +60,29 @@ const swapActive: Probe = async (run) => {
|
|
|
61
60
|
];
|
|
62
61
|
};
|
|
63
62
|
|
|
63
|
+
/** The disk guard's floor scales with the filesystem
|
|
64
|
+
* (`sizing.thresholds.diskGuardFloor`); one of unknown size gets the upper
|
|
65
|
+
* bound. */
|
|
64
66
|
const diskAboveFloor =
|
|
65
67
|
(path: string): Probe =>
|
|
66
68
|
async (run) => {
|
|
67
|
-
const df = await run(['df', '--output=avail', '-B1', path]);
|
|
68
|
-
const free =
|
|
69
|
-
|
|
69
|
+
const df = await run(['df', '--output=avail,size', '-B1', path]);
|
|
70
|
+
const [free, size] = (df.stdout.trim().split('\n').at(-1) ?? '')
|
|
71
|
+
.trim()
|
|
72
|
+
.split(/\s+/)
|
|
73
|
+
.map(Number);
|
|
74
|
+
if (!df.ok || free === undefined || !Number.isSafeInteger(free))
|
|
70
75
|
return ['DISK_FREE_UNREADABLE', `Free space on ${path} is unreadable.`];
|
|
71
|
-
|
|
76
|
+
const floor = threshold(
|
|
77
|
+
'diskGuardFloor',
|
|
78
|
+
size !== undefined && Number.isSafeInteger(size)
|
|
79
|
+
? Math.floor(size / MIB)
|
|
80
|
+
: Number.MAX_SAFE_INTEGER
|
|
81
|
+
);
|
|
82
|
+
if (free >= floor * MIB) return undefined;
|
|
72
83
|
return [
|
|
73
84
|
'DISK_FREE_BELOW_FLOOR',
|
|
74
|
-
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${
|
|
85
|
+
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${floor} MiB floor. The disk GC already removed idle build output and caches; remove stale worktrees or data, or run "sudo systemctl start personal-fleet-disk-gc.service" and read its journal.`,
|
|
75
86
|
];
|
|
76
87
|
};
|
|
77
88
|
|
|
@@ -8,27 +8,40 @@ import { rename, rm, symlink } from 'node:fs/promises';
|
|
|
8
8
|
import { userInfo } from 'node:os';
|
|
9
9
|
import type { Runner, Check } from '../types';
|
|
10
10
|
|
|
11
|
+
/** The administrator policy a worker's T3 runs: the fleet's
|
|
12
|
+
* (`ansible/ubuntu/t3/policy.json`, which keeps linked environments on so
|
|
13
|
+
* the manager pairs with a replaced worker without a person) with the
|
|
14
|
+
* worker's provider instances locked.
|
|
15
|
+
*/
|
|
16
|
+
export function workerPolicy(instances: Record<string, unknown>) {
|
|
17
|
+
return {
|
|
18
|
+
...policy,
|
|
19
|
+
settings: {
|
|
20
|
+
...policy.settings,
|
|
21
|
+
...Object.fromEntries(
|
|
22
|
+
Object.entries(instances).map(([id, value]) => [
|
|
23
|
+
`providerInstances.${id}`,
|
|
24
|
+
{ value, locked: true },
|
|
25
|
+
])
|
|
26
|
+
),
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
11
31
|
export async function stageT3Policy(
|
|
12
32
|
root: string,
|
|
13
33
|
executable: string,
|
|
14
34
|
run: Runner,
|
|
15
|
-
node: string
|
|
35
|
+
target: { node: string; restartBusy?: boolean }
|
|
16
36
|
): Promise<Check> {
|
|
37
|
+
const { node } = target;
|
|
17
38
|
const instances = await desired(
|
|
18
39
|
resolve(root, 'agents/t3.json'),
|
|
19
40
|
await stableWorkspace(root),
|
|
20
41
|
node
|
|
21
42
|
);
|
|
22
43
|
const user = userInfo();
|
|
23
|
-
const document =
|
|
24
|
-
...policy,
|
|
25
|
-
settings: Object.fromEntries(
|
|
26
|
-
Object.entries(instances).map(([id, value]) => [
|
|
27
|
-
`providerInstances.${id}`,
|
|
28
|
-
{ value, locked: true },
|
|
29
|
-
])
|
|
30
|
-
),
|
|
31
|
-
};
|
|
44
|
+
const document = workerPolicy(instances);
|
|
32
45
|
const result = await runAnsible(run, 't3-policy.yml', {
|
|
33
46
|
cwd: resolve(root, 'ansible/ubuntu'),
|
|
34
47
|
input: JSON.stringify({
|
|
@@ -37,6 +50,7 @@ export async function stageT3Policy(
|
|
|
37
50
|
development_user: user.username,
|
|
38
51
|
development_uid: user.uid,
|
|
39
52
|
development_home: user.homedir,
|
|
53
|
+
t3_restart_busy: target.restartBusy === true,
|
|
40
54
|
}),
|
|
41
55
|
timeoutMs: 120_000,
|
|
42
56
|
});
|