@crvouga/dev-fleet 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +50 -16
  2. package/ansible/github/git.yml +66 -0
  3. package/ansible/macos/manager.yml +3 -0
  4. package/ansible/macos/policy.json +11 -1
  5. package/ansible/macos/t3-policy.yml +49 -0
  6. package/ansible/monitoring/compose.yml +10 -0
  7. package/ansible/monitoring/container-watchdog.py +118 -0
  8. package/ansible/monitoring/hub-darwin.yml +146 -0
  9. package/ansible/net/remote-probe.sh +5 -0
  10. package/ansible/ubuntu/baseline.yml +2 -0
  11. package/ansible/ubuntu/disk-gc.py +25 -3
  12. package/ansible/ubuntu/disk-guard.py +24 -5
  13. package/ansible/ubuntu/policy.json +51 -1
  14. package/ansible/ubuntu/t3-desktop.yml +19 -0
  15. package/ansible/ubuntu/t3-health.sh +93 -22
  16. package/ansible/ubuntu/t3-idle.py +42 -0
  17. package/ansible/ubuntu/t3-policy.yml +4 -11
  18. package/ansible/ubuntu/tasks/browser.yml +56 -0
  19. package/ansible/ubuntu/tasks/docker.yml +52 -0
  20. package/ansible/ubuntu/tasks/recovery.yml +60 -1
  21. package/ansible/ubuntu/tasks/resources.yml +2 -2
  22. package/ansible/ubuntu/tasks/services.yml +13 -0
  23. package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
  24. package/ansible/ubuntu/wip-guard.py +121 -0
  25. package/bootstrap/doctor.ts +32 -6
  26. package/bootstrap/fleet-remote-bootstrap.sh +31 -4
  27. package/bootstrap/native-t3.ts +22 -11
  28. package/bootstrap/setup.ts +23 -8
  29. package/bootstrap/worker-tools.ts +118 -0
  30. package/cliproxy/README.md +4 -4
  31. package/cliproxy/compose.yaml +4 -0
  32. package/cliproxy/scripts/accounts.py +216 -0
  33. package/cliproxy/scripts/checkpoint.py +30 -2
  34. package/cliproxy/scripts/cli.py +1 -1
  35. package/cliproxy/scripts/clientkeys.py +99 -0
  36. package/cliproxy/scripts/lease.py +80 -0
  37. package/cliproxy/scripts/ops +137 -25
  38. package/cliproxy/scripts/restore-verify +15 -0
  39. package/docs/accounts.md +154 -0
  40. package/docs/authentication.md +90 -0
  41. package/docs/declared.md +5 -1
  42. package/docs/durability.md +26 -0
  43. package/docs/machine-setup.md +26 -15
  44. package/docs/operations.md +87 -31
  45. package/docs/runbooks.md +380 -0
  46. package/docs/state.md +13 -7
  47. package/docs/topology.md +2 -0
  48. package/docs/ubuntu-sync.md +35 -28
  49. package/examples/fleet.json +19 -1
  50. package/llms.txt +3 -0
  51. package/mise.toml +10 -1
  52. package/package.json +2 -2
  53. package/schema/check-ids.json +522 -0
  54. package/schema/fleet.schema.json +244 -1
  55. package/schema/output/check.schema.json +23 -0
  56. package/schema/output/error.schema.json +14 -0
  57. package/schema/output/fleet-report.schema.json +79 -0
  58. package/schema/output/fleet-sync.schema.json +112 -0
  59. package/schema/output/report.schema.json +38 -0
  60. package/src/accounts/checks.ts +220 -0
  61. package/src/accounts/context.ts +76 -0
  62. package/src/accounts/propagate.ts +62 -0
  63. package/src/accounts/roster.ts +85 -0
  64. package/src/accounts/selection.ts +67 -0
  65. package/src/accounts/sessions.ts +120 -0
  66. package/src/agents/account-relay.ts +176 -26
  67. package/src/agents/launch.ts +84 -5
  68. package/src/agents/native-verify.ts +128 -2
  69. package/src/auth-checks.ts +3 -0
  70. package/src/cli/agent-guide.ts +5 -3
  71. package/src/cli/fleet-accounts.ts +253 -0
  72. package/src/cli/fleet-report.ts +126 -0
  73. package/src/cli/fleet-sync-accounts.ts +58 -0
  74. package/src/cli/fleet-sync-manager.ts +31 -4
  75. package/src/cli/fleet-sync-output.ts +3 -2
  76. package/src/cli/fleet-sync-plan.ts +200 -0
  77. package/src/cli/fleet-sync-remote.ts +23 -14
  78. package/src/cli/fleet-sync-schedule.ts +142 -35
  79. package/src/cli/fleet-sync.ts +126 -8
  80. package/src/cli/fleet-watch.ts +198 -0
  81. package/src/cli/fleet.ts +4 -0
  82. package/src/cli/register-fleet.ts +86 -0
  83. package/src/cli/registry.ts +26 -0
  84. package/src/cli/schedule-logs.ts +36 -0
  85. package/src/clock.ts +63 -0
  86. package/src/definition.ts +226 -7
  87. package/src/git-identity.ts +160 -0
  88. package/src/ide/definition.ts +86 -0
  89. package/src/ide/ide.ts +43 -0
  90. package/src/ide/index.ts +5 -0
  91. package/src/ide/orchestrate.ts +50 -0
  92. package/src/ide/t3.ts +111 -0
  93. package/src/machine-health.ts +51 -4
  94. package/src/manager-health.ts +205 -0
  95. package/src/manager-sync.ts +13 -17
  96. package/src/monitoring/hub-backup.ts +63 -17
  97. package/src/monitoring/manager-hub.ts +44 -2
  98. package/src/node-converge-linux.ts +44 -12
  99. package/src/notify.ts +239 -0
  100. package/src/platform/darwin.ts +0 -13
  101. package/src/platform/fallback.ts +0 -8
  102. package/src/platform/linux.ts +0 -10
  103. package/src/platform/t3-policy.ts +33 -6
  104. package/src/platform/t3-service.ts +104 -0
  105. package/src/platform/types.ts +0 -4
  106. package/src/platform/windows.ts +0 -14
  107. package/src/remote-bootstrap.ts +87 -25
  108. package/src/remote-entry.ts +15 -1
  109. package/src/remote-operation.ts +213 -0
  110. package/src/schedule-check.ts +40 -0
  111. package/src/secrets.ts +26 -8
  112. package/src/service-auth.ts +15 -5
  113. package/src/ssh.ts +10 -2
  114. package/src/state/history.ts +172 -0
  115. package/src/state/store.ts +11 -4
  116. package/src/t3/accounts.ts +7 -1
  117. package/src/t3/models.ts +133 -0
  118. package/src/t3/pairing.ts +31 -0
  119. package/src/t3/profile.ts +203 -0
  120. package/src/t3/version.ts +119 -0
  121. package/src/t3-linked-check.ts +1 -1
  122. package/src/tailscale.ts +51 -0
  123. package/src/toolchain-checks.ts +8 -3
  124. package/src/transport.ts +3 -2
  125. package/src/worker-capability-checks.ts +167 -0
  126. package/src/worker-capacity.ts +19 -7
  127. package/src/worker-host-checks.ts +68 -13
  128. package/src/worker-policy.ts +33 -0
  129. package/src/worker-readiness.ts +10 -1
@@ -0,0 +1,119 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { sshArgs } from '../ssh';
4
+ import type { Check, FleetNode, Runner } from '../types';
5
+
6
+ /** The T3 release every machine runs: `t3_version` in mise.toml. */
7
+ export async function pinnedT3Version(workspace: string): Promise<string> {
8
+ const toml = await readFile(join(workspace, 'mise.toml'), 'utf8');
9
+ const version = /^t3_version = "([^"]+)"$/m.exec(toml)?.[1];
10
+ if (!version) throw new Error('mise.toml pins no T3 version.');
11
+ return version;
12
+ }
13
+
14
+ /** Where T3 notes the version it last started as, under its home. */
15
+ export const STARTED_VERSION = 'userdata/desktop-last-started-version';
16
+
17
+ const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$/;
18
+
19
+ /** `t3:version`: the T3 a machine runs is the one the fleet pins, so the
20
+ * manager and the workers it drives never run different releases.
21
+ */
22
+ export function versionCheck(
23
+ running: string | null,
24
+ pinned: string,
25
+ where: string,
26
+ install: string
27
+ ): Check {
28
+ const observed = running?.trim() ?? '';
29
+ if (!VERSION.test(observed))
30
+ return {
31
+ id: 't3:version',
32
+ status: 'warn',
33
+ summary: 'T3_VERSION_UNKNOWN',
34
+ remedy: {
35
+ action: `T3 on ${where} has not recorded the version it runs. Start T3 there once, then run fleet-sync.`,
36
+ },
37
+ };
38
+ if (observed !== pinned)
39
+ return {
40
+ id: 't3:version',
41
+ status: 'warn',
42
+ summary: 'T3_VERSION_SKEW',
43
+ remedy: {
44
+ action: `T3 on ${where} runs ${observed}; the fleet pins ${pinned}. ${install}`,
45
+ },
46
+ };
47
+ return {
48
+ id: 't3:version',
49
+ status: 'pass',
50
+ summary: `T3 on ${where} runs ${pinned}, the release every machine runs.`,
51
+ };
52
+ }
53
+
54
+ export async function managerVersion(
55
+ t3Home: string,
56
+ workspace: string
57
+ ): Promise<Check> {
58
+ const running = await readFile(join(t3Home, STARTED_VERSION), 'utf8').catch(
59
+ () => null
60
+ );
61
+ return versionCheck(
62
+ running,
63
+ await pinnedT3Version(workspace),
64
+ 'the manager',
65
+ 'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -K ansible/macos/manager.yml) it no longer updates itself.'
66
+ );
67
+ }
68
+
69
+ export async function workerVersion(
70
+ node: FleetNode,
71
+ run: Runner,
72
+ workspace: string
73
+ ): Promise<Check> {
74
+ const result = await run(
75
+ [...sshArgs(node), `cat "$HOME/.t3/${STARTED_VERSION}"`],
76
+ { timeoutMs: 15_000 }
77
+ );
78
+ return versionCheck(
79
+ result.ok ? result.stdout : null,
80
+ await pinnedT3Version(workspace),
81
+ `worker ${node.id}`,
82
+ 'Run fleet-sync for that worker: it installs the pinned desktop app and restarts T3 when its policy changed.'
83
+ );
84
+ }
85
+
86
+ /** Where T3 on macOS reads its administrator policy. */
87
+ export const MACOS_POLICY = '/Library/Application Support/T3Code/policy.json';
88
+
89
+ /** `t3:updates`: T3's own updater is off by the fleet's policy, so it changes
90
+ * only when the pin does.
91
+ */
92
+ export async function updatesCheck(
93
+ policyPath: string,
94
+ stage: string
95
+ ): Promise<Check> {
96
+ let mode: unknown;
97
+ try {
98
+ const policy = JSON.parse(await readFile(policyPath, 'utf8')) as {
99
+ limits?: { updates?: { mode?: unknown } };
100
+ };
101
+ mode = policy.limits?.updates?.mode;
102
+ } catch {
103
+ mode = undefined;
104
+ }
105
+ return mode === 'off'
106
+ ? {
107
+ id: 't3:updates',
108
+ status: 'pass',
109
+ summary: 'T3’s own updater is off by the fleet’s policy.',
110
+ }
111
+ : {
112
+ id: 't3:updates',
113
+ status: 'warn',
114
+ summary: 'T3_UPDATES_UNMANAGED',
115
+ remedy: {
116
+ action: `T3 on this machine may update itself away from the pinned release. Stage the fleet's policy once, as an administrator: ${stage}`,
117
+ },
118
+ };
119
+ }
@@ -103,7 +103,7 @@ function linkedEvidence(
103
103
  };
104
104
  }
105
105
 
106
- async function observedIdentity(
106
+ export async function observedIdentity(
107
107
  node: FleetNode,
108
108
  run: Runner
109
109
  ): Promise<string | undefined> {
package/src/tailscale.ts CHANGED
@@ -10,6 +10,9 @@ export interface TailnetComputer {
10
10
  ips: string[];
11
11
  online: boolean;
12
12
  self: boolean;
13
+ /** When the node key expires, in milliseconds since the epoch; absent when
14
+ * it never does (tagged nodes, or expiry disabled in the admin console). */
15
+ keyExpiry?: number;
13
16
  }
14
17
  export interface Tailnet {
15
18
  computers: TailnetComputer[];
@@ -103,6 +106,7 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
103
106
  OS: os,
104
107
  TailscaleIPs: ips,
105
108
  Online: online,
109
+ KeyExpiry: keyExpiry,
106
110
  } = value;
107
111
  if (
108
112
  ![id, name, os].every(
@@ -129,6 +133,53 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
129
133
  ips: ips as string[],
130
134
  online,
131
135
  self,
136
+ ...expiry(keyExpiry),
137
+ };
138
+ }
139
+
140
+ function expiry(value: unknown): { keyExpiry?: number } {
141
+ if (typeof value !== 'string') return {};
142
+ const time = Date.parse(value);
143
+ // Tailscale writes the zero time for a key that never expires.
144
+ return Number.isFinite(time) && time > 0 ? { keyExpiry: time } : {};
145
+ }
146
+
147
+ const DAY_MS = 86_400_000;
148
+ const KEY_EXPIRY = { warnDays: 14, failDays: 3 };
149
+
150
+ /** A node drops off the tailnet on the day its key expires. The check warns
151
+ * two weeks ahead and fails three days ahead, like a T3 link that expires.
152
+ */
153
+ export function keyExpiryCheck(
154
+ computer: TailnetComputer | undefined,
155
+ now = Date.now()
156
+ ): Check {
157
+ const id = 'tailscale:key-expiry';
158
+ if (!computer)
159
+ return { id, status: 'skip', summary: 'TAILSCALE_KEY_EXPIRY_UNOBSERVED' };
160
+ if (computer.keyExpiry === undefined)
161
+ return {
162
+ id,
163
+ status: 'pass',
164
+ summary: 'The Tailscale node key of this machine does not expire.',
165
+ };
166
+ const days = Math.floor((computer.keyExpiry - now) / DAY_MS);
167
+ const remedy = {
168
+ action: computer.self
169
+ ? 'Renew the node key of the manager before it expires: tailscale up --force-reauth on the manager, or disable key expiry for it in the Tailscale admin console.'
170
+ : 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
171
+ };
172
+ if (days < 0)
173
+ return { id, status: 'fail', summary: 'TAILSCALE_KEY_EXPIRED', remedy };
174
+ const summary = `TAILSCALE_KEY_EXPIRES_SOON in ${days} days`;
175
+ if (days <= KEY_EXPIRY.failDays)
176
+ return { id, status: 'fail', summary, remedy };
177
+ if (days <= KEY_EXPIRY.warnDays)
178
+ return { id, status: 'warn', summary, remedy };
179
+ return {
180
+ id,
181
+ status: 'pass',
182
+ summary: `The Tailscale node key expires in ${days} days.`,
132
183
  };
133
184
  }
134
185
 
@@ -1,11 +1,16 @@
1
1
  import { join } from 'node:path';
2
+ import {
3
+ selectionArguments,
4
+ type ToolSelection,
5
+ } from '../bootstrap/worker-tools';
2
6
  import { record } from './config';
3
7
  import type { Check, Runner } from './types';
4
8
 
5
9
  export async function toolchainChecks(
6
10
  root: string,
7
11
  run: Runner,
8
- worker?: boolean
12
+ /** A worker's own tool selection; absent on the manager. */
13
+ worker?: ToolSelection
9
14
  ): Promise<Check[]> {
10
15
  const result = await run(doctorArguments(root, worker), {
11
16
  cwd: root,
@@ -97,12 +102,12 @@ function toolCheck(value: unknown): Check | null {
97
102
  };
98
103
  }
99
104
 
100
- function doctorArguments(root: string, worker?: boolean): string[] {
105
+ function doctorArguments(root: string, worker?: ToolSelection): string[] {
101
106
  return [
102
107
  'bun',
103
108
  join(root, 'bootstrap/doctor.ts'),
104
109
  '--json',
105
110
  '--docker',
106
- ...(worker ? ['--worker'] : []),
111
+ ...(worker ? ['--worker', ...selectionArguments(worker)] : []),
107
112
  ];
108
113
  }
package/src/transport.ts CHANGED
@@ -21,7 +21,7 @@ export async function bundleProbe(): Promise<string> {
21
21
  export interface RemoteOptions {
22
22
  source: string;
23
23
  timeoutMs: number;
24
- action: 'probe' | 'sync' | 'repair' | 'configure';
24
+ action: 'probe' | 'sync' | 'repair' | 'configure' | 'accounts';
25
25
  configuration?: ConfigurationRequest;
26
26
  repairs?: AutomatedRepair[];
27
27
  expected?: Fingerprint;
@@ -41,7 +41,8 @@ export async function inspectRemote(
41
41
  return { node: node.id, checks: [sshFailure(preflight)] };
42
42
  }
43
43
  const target =
44
- options.action === 'probe' && options.expected
44
+ (options.action === 'probe' || options.action === 'accounts') &&
45
+ options.expected
45
46
  ? await managedProbeTarget(node, options.expected.head, run)
46
47
  : { node, checks: [] };
47
48
  const command = remoteCommand(target.node, {
@@ -0,0 +1,167 @@
1
+ import policy from '../ansible/ubuntu/policy.json';
2
+ import type { Check, CommandResult, Runner } from './types';
3
+
4
+ /** What an agent does on a worker, tried the way it does it: start a
5
+ * container, build an image, launch a headless browser. Each check runs as
6
+ * the worker's user, in its session, and reports a code and a remedy, never
7
+ * what a command printed.
8
+ */
9
+
10
+ function failure(id: string, summary: string, action: string): Check {
11
+ return { id, status: 'fail', summary, remedy: { action } };
12
+ }
13
+
14
+ /** Docker's own words for the causes an agent meets most. */
15
+ function dockerFailure(result: CommandResult): {
16
+ code: string;
17
+ action: string;
18
+ } {
19
+ const said = `${result.stdout}\n${result.stderr}`;
20
+ if (result.reason === 'timeout')
21
+ return {
22
+ code: 'DOCKER_RUN_TIMEOUT',
23
+ action:
24
+ 'A container did not start and exit in time. Inspect the engine (systemctl status docker) and its disk; rerun fleet-sync.',
25
+ };
26
+ if (/permission denied[^\n]*docker\.sock/i.test(said))
27
+ return {
28
+ code: 'DOCKER_SOCKET_ACCESS_DENIED',
29
+ action:
30
+ "This session cannot use the Docker socket. fleet-sync adds the worker's user to the docker group; a session started before that keeps its old groups. Rerun fleet-sync, then restart the session (log out and in, or restart the T3 service).",
31
+ };
32
+ if (/cannot connect to the docker daemon/i.test(said))
33
+ return {
34
+ code: 'DOCKER_DAEMON_UNREACHABLE',
35
+ action:
36
+ 'The Docker engine is not running. Rerun fleet-sync, which starts and enables docker.service.',
37
+ };
38
+ if (/no such image|pull access denied|not found/i.test(said))
39
+ return {
40
+ code: 'DOCKER_SMOKE_IMAGE_MISSING',
41
+ action:
42
+ 'The pinned smoke-test image is not cached on this worker. Rerun fleet-sync, which pulls it once so this check works offline.',
43
+ };
44
+ if (/no space left on device/i.test(said))
45
+ return {
46
+ code: 'DOCKER_DISK_FULL',
47
+ action:
48
+ 'The image store is full. Free space under /var/lib/docker (the disk GC prunes old images) and rerun fleet-sync.',
49
+ };
50
+ return {
51
+ code: 'DOCKER_RUN_FAILED',
52
+ action:
53
+ 'Docker answers but cannot run a container here. Inspect cgroups and the engine log (journalctl -u docker), then rerun fleet-sync.',
54
+ };
55
+ }
56
+
57
+ /** Runs the pinned image offline, then builds a one-line image from it. */
58
+ export async function dockerRunCheck(run: Runner): Promise<Check> {
59
+ const id = 'worker:docker-run';
60
+ const { image, timeoutSeconds } = policy.dockerSmokeTest;
61
+ const timeoutMs = timeoutSeconds * 1000;
62
+ const started = await run(
63
+ [
64
+ 'docker',
65
+ 'run',
66
+ '--rm',
67
+ '--network',
68
+ 'none',
69
+ '--pull',
70
+ 'never',
71
+ image,
72
+ 'true',
73
+ ],
74
+ { timeoutMs }
75
+ );
76
+ if (!started.ok) {
77
+ const { code, action } = dockerFailure(started);
78
+ return failure(id, code, action);
79
+ }
80
+ const built = await run(
81
+ ['docker', 'build', '--quiet', '--pull=false', '--network', 'none', '-'],
82
+ { input: `FROM ${image}\nRUN true\n`, timeoutMs }
83
+ );
84
+ if (!built.ok) {
85
+ const { code, action } = dockerFailure(built);
86
+ return failure(
87
+ id,
88
+ code === 'DOCKER_RUN_FAILED' ? 'DOCKER_BUILD_FAILED' : code,
89
+ code === 'DOCKER_RUN_FAILED'
90
+ ? 'A container runs but an image does not build. Inspect Buildx (docker buildx ls) and the image store, then rerun fleet-sync.'
91
+ : action
92
+ );
93
+ }
94
+ return {
95
+ id,
96
+ status: 'pass',
97
+ summary:
98
+ 'A container of the pinned image ran and a one-line image built, offline, in this session.',
99
+ };
100
+ }
101
+
102
+ /** Where `ansible/ubuntu/tasks/browser.yml` unpacks the pinned build. */
103
+ export function headlessShell(arch: string = process.arch): string {
104
+ const { version, amd64, arm64 } = policy.browser.headlessShell;
105
+ const build = arch === 'arm64' ? arm64 : amd64;
106
+ const directory = build.archive.split('/').pop()!;
107
+ return `/opt/personal-fleet/browser/${version}/${directory}/chrome-headless-shell`;
108
+ }
109
+
110
+ /** A worker with the web profile launches the pinned headless Chromium,
111
+ * loads a blank page and exits, as a browser test would.
112
+ */
113
+ export async function browserCheck(
114
+ run: Runner,
115
+ web: boolean,
116
+ arch: string = process.arch
117
+ ): Promise<Check> {
118
+ const id = 'worker:browser';
119
+ if (!web)
120
+ return {
121
+ id,
122
+ status: 'skip',
123
+ summary:
124
+ 'This worker does not select the web tool profile, so no browser is installed for tests.',
125
+ };
126
+ const launched = await run(
127
+ [
128
+ headlessShell(arch),
129
+ '--headless',
130
+ '--no-sandbox',
131
+ '--disable-gpu',
132
+ '--dump-dom',
133
+ 'about:blank',
134
+ ],
135
+ { timeoutMs: policy.browser.launchTimeoutSeconds * 1000 }
136
+ );
137
+ if (launched.ok && /<html/i.test(launched.stdout))
138
+ return {
139
+ id,
140
+ status: 'pass',
141
+ summary: `Headless Chromium ${policy.browser.headlessShell.version} launched, loaded a blank page and exited.`,
142
+ };
143
+ const said = `${launched.stdout}\n${launched.stderr}`;
144
+ if (launched.reason === 'missing')
145
+ return failure(
146
+ id,
147
+ 'BROWSER_NOT_INSTALLED',
148
+ 'The pinned headless Chromium is not installed. Rerun fleet-sync, which installs it on workers with the web profile.'
149
+ );
150
+ if (/error while loading shared libraries/i.test(said))
151
+ return failure(
152
+ id,
153
+ 'BROWSER_LIBRARY_MISSING',
154
+ 'A system library headless Chromium loads is missing. Rerun fleet-sync, which installs the pinned libraries of the web profile.'
155
+ );
156
+ if (launched.reason === 'timeout')
157
+ return failure(
158
+ id,
159
+ 'BROWSER_LAUNCH_TIMEOUT',
160
+ 'Headless Chromium did not exit in time. Check the worker for memory pressure, then rerun fleet-sync.'
161
+ );
162
+ return failure(
163
+ id,
164
+ 'BROWSER_LAUNCH_FAILED',
165
+ 'Headless Chromium did not load a blank page. Run it by hand with --dump-dom about:blank to see why, then rerun fleet-sync.'
166
+ );
167
+ }
@@ -1,11 +1,14 @@
1
1
  import { join } from 'node:path';
2
+ import { clockCheck } from './clock';
2
3
  import { healthChecks, parseProbe } from './machine-health';
3
4
  import type { Check, Runner } from './types';
4
5
 
5
6
  export async function workerCapacity(
6
7
  root: string,
7
8
  node: string,
8
- run: Runner
9
+ run: Runner,
10
+ /** The manager's clock in seconds since the epoch, when it sent one. */
11
+ reference?: number
9
12
  ): Promise<Check[]> {
10
13
  const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
11
14
  timeoutMs: 10_000,
@@ -25,10 +28,19 @@ export async function workerCapacity(
25
28
  summary: 'Required CPU, memory and disk evidence is unavailable.',
26
29
  },
27
30
  ];
28
- return healthChecks(sample, node).map((check) => ({
29
- id: `worker:capacity:${check.id}`,
30
- status: check.severity,
31
- summary: check.detail,
32
- ...(check.fix ? { remedy: { action: check.fix } } : {}),
33
- }));
31
+ return [
32
+ ...healthChecks(sample, node).map((check) => ({
33
+ id: `worker:capacity:${check.id}`,
34
+ status: check.severity,
35
+ summary: check.detail,
36
+ ...(check.fix ? { remedy: { action: check.fix } } : {}),
37
+ })),
38
+ clockCheck(
39
+ 'worker:clock',
40
+ sample,
41
+ reference === undefined || sample.epoch === undefined
42
+ ? undefined
43
+ : sample.epoch - reference
44
+ ),
45
+ ];
34
46
  }
@@ -4,6 +4,7 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
4
4
  import { verifyResourceSafety } from './platform/resource-verify';
5
5
  import policy from '../ansible/ubuntu/policy.json';
6
6
  import { userInfo } from 'node:os';
7
+ import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
7
8
  import type { Check, Runner } from './types';
8
9
 
9
10
  function check(id: string, ok: boolean, summary: string): Check {
@@ -94,6 +95,7 @@ export async function workerHostChecks(
94
95
  )
95
96
  );
96
97
  }
98
+ checks.push(await rebootPendingCheck(audit.rebootPending, run));
97
99
  checks.push(...(await recoveryChecks(run)));
98
100
  checks.push(...(await verifyUbuntuBoot(user, run)));
99
101
  checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
@@ -120,18 +122,13 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
120
122
  )
121
123
  );
122
124
  checks.push(await t3HealthCheck(run));
125
+ checks.push(await runningVersionCheck(run, userInfo().uid));
123
126
  const renewal = await run([
124
127
  'systemctl',
125
128
  '--user',
126
129
  'is-active',
127
130
  'personal-worker-auth.timer',
128
131
  ]);
129
- const recovery = await run([
130
- 'systemctl',
131
- '--user',
132
- 'is-active',
133
- 'personal-worker-recovery.timer',
134
- ]);
135
132
  checks.push(
136
133
  check(
137
134
  'credential-renewal',
@@ -139,17 +136,44 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
139
136
  'Worker credential renewal timer is active.'
140
137
  )
141
138
  );
142
- checks.push(
143
- check(
144
- 'service-recovery',
145
- recovery.ok,
146
- 'Worker service recovery timer is active.'
147
- )
148
- );
139
+ checks.push(await serviceRecoveryCheck(run));
149
140
  checks.push(await powerRecoveryCheck(run));
150
141
  return checks;
151
142
  }
152
143
 
144
+ /** The recovery timer is active, the script its unit runs exists, and the
145
+ * last run of the unit succeeded: a watchdog that fails on every tick is not
146
+ * recovery. */
147
+ export async function serviceRecoveryCheck(run: Runner): Promise<Check> {
148
+ const timer = await run([
149
+ 'systemctl',
150
+ '--user',
151
+ 'is-active',
152
+ 'personal-worker-recovery.timer',
153
+ ]);
154
+ const script = await run(['test', '-r', T3_HEALTH_SCRIPT]);
155
+ const unit = await run([
156
+ 'systemctl',
157
+ '--user',
158
+ 'show',
159
+ 'personal-worker-recovery.service',
160
+ '--property=Result',
161
+ ]);
162
+ const result = /^Result=(.*)$/m.exec(unit.stdout)?.[1] ?? 'unknown';
163
+ const summary = !timer.ok
164
+ ? 'WORKER_RECOVERY_TIMER_INACTIVE'
165
+ : !script.ok
166
+ ? 'WORKER_RECOVERY_SCRIPT_MISSING'
167
+ : !unit.ok || result !== 'success'
168
+ ? `WORKER_RECOVERY_LAST_RUN_FAILED (${/^[\w-]+$/.test(result) ? result : 'unknown'})`
169
+ : 'Worker service recovery timer is active and its last run succeeded.';
170
+ return check(
171
+ 'service-recovery',
172
+ timer.ok && script.ok && unit.ok && result === 'success',
173
+ summary
174
+ );
175
+ }
176
+
153
177
  /** The server answers its local API now, not merely that a process exists. */
154
178
  async function t3HealthCheck(run: Runner): Promise<Check> {
155
179
  const probe = await run(
@@ -179,3 +203,34 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
179
203
  'T3 local API answers with its environment identity within 10 seconds.'
180
204
  );
181
205
  }
206
+
207
+ const DAY_SECONDS = 86_400;
208
+
209
+ /** Kernel and library updates apply only after a reboot. A pending reboot
210
+ * warns with its age, and fails once it is older than `policy.json` allows.
211
+ */
212
+ export async function rebootPendingCheck(
213
+ pending: boolean,
214
+ run: Runner,
215
+ now = Date.now()
216
+ ): Promise<Check> {
217
+ const id = 'worker:reboot-pending';
218
+ if (!pending) return { id, status: 'pass', summary: 'No reboot is pending.' };
219
+ const since = await run(['stat', '-c', '%Y', '/var/run/reboot-required']);
220
+ const seconds = Number(since.stdout.trim());
221
+ const days =
222
+ since.ok && /^\d+$/.test(since.stdout.trim())
223
+ ? Math.max(0, Math.floor((now / 1000 - seconds) / DAY_SECONDS))
224
+ : null;
225
+ const age = days === null ? '' : ` for ${days} days`;
226
+ return {
227
+ id,
228
+ status:
229
+ days !== null && days > policy.rebootPendingFailDays ? 'fail' : 'warn',
230
+ summary: `REBOOT_PENDING${age}: installed updates apply only after a reboot.`,
231
+ remedy: {
232
+ action:
233
+ 'Reboot the worker when no agent turn is running (sudo systemctl reboot on the worker), then confirm it returns to ready with devfleet fleet-status --workers.',
234
+ },
235
+ };
236
+ }
@@ -1,6 +1,7 @@
1
1
  import { CAPABILITIES, record } from './config';
2
2
  import { FleetUsageError } from './errors';
3
3
  import type { Capability } from './types';
4
+ import { TOOL_PROFILES, type ToolProfile } from '../bootstrap/worker-tools';
4
5
 
5
6
  export interface WorkerPolicy {
6
7
  requiredCapabilities: Capability[];
@@ -12,6 +13,10 @@ export interface WorkerPolicy {
12
13
  unattendedSudo: true;
13
14
  persistentT3: true;
14
15
  neverSleep: true;
16
+ /** Tool profiles every worker installs on top of the base tools. */
17
+ toolProfiles?: ToolProfile[];
18
+ /** Single tools pinned in dev-fleet's mise.toml every worker installs. */
19
+ tools?: string[];
15
20
  }
16
21
  const REQUIRED = [
17
22
  'github',
@@ -41,8 +46,11 @@ function validPolicy(value: Record<string, unknown>): boolean {
41
46
  'unattendedSudo',
42
47
  'persistentT3',
43
48
  'neverSleep',
49
+ 'toolProfiles',
50
+ 'tools',
44
51
  ].includes(key)
45
52
  ) &&
53
+ validTools(value) &&
46
54
  validCapabilities(value['requiredCapabilities']) &&
47
55
  typeof value['githubLogin'] === 'string' &&
48
56
  /^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
@@ -55,6 +63,31 @@ function validPolicy(value: Record<string, unknown>): boolean {
55
63
  )
56
64
  );
57
65
  }
66
+ /** A list of known tool profiles, each once; null when it is not one. */
67
+ export function toolProfiles(value: unknown): ToolProfile[] | null {
68
+ return Array.isArray(value) &&
69
+ new Set(value).size === value.length &&
70
+ value.every((item) => TOOL_PROFILES.includes(item as ToolProfile))
71
+ ? (value as ToolProfile[])
72
+ : null;
73
+ }
74
+ /** The optional tool profiles and single tools every worker adds. */
75
+ function validTools(value: Record<string, unknown>): boolean {
76
+ return (
77
+ (value['toolProfiles'] === undefined ||
78
+ toolProfiles(value['toolProfiles']) !== null) &&
79
+ (value['tools'] === undefined || toolNames(value['tools']))
80
+ );
81
+ }
82
+ function toolNames(value: unknown): boolean {
83
+ return (
84
+ Array.isArray(value) &&
85
+ new Set(value).size === value.length &&
86
+ value.every(
87
+ (item) => typeof item === 'string' && /^[a-z0-9@:/._-]{1,75}$/i.test(item)
88
+ )
89
+ );
90
+ }
58
91
  export function parseWorkerPolicy(value: unknown): WorkerPolicy {
59
92
  if (!record(value) || !validPolicy(value))
60
93
  throw new FleetUsageError(
@@ -1,7 +1,10 @@
1
1
  import { workerCapacity } from './worker-capacity';
2
2
  import { userInfo, hostname } from 'node:os';
3
3
  import { hostId } from './topology';
4
- import { configPath } from './fleet-config';
4
+ import { configPath, fleet } from './fleet-config';
5
+ import { workerSelection } from '../bootstrap/worker-tools';
6
+ import { browserCheck, dockerRunCheck } from './worker-capability-checks';
7
+ import { gitChecks } from './git-identity';
5
8
  import type { Check, Runner } from './types';
6
9
  import { workerHostChecks } from './worker-host-checks';
7
10
  import { workerAccessChecks } from './worker-access-checks';
@@ -59,6 +62,12 @@ export async function workerReadiness(
59
62
  ...host,
60
63
  ...access,
61
64
  proxy,
65
+ await dockerRunCheck(run),
66
+ await browserCheck(
67
+ run,
68
+ workerSelection(fleet, nodeId).profiles.includes('web')
69
+ ),
70
+ ...(await gitChecks(fleet.git, run)),
62
71
  ...(await workerCapacity(root, nodeId, run)),
63
72
  ];
64
73
  }