@crvouga/dev-fleet 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -16
- package/ansible/github/git.yml +66 -0
- package/ansible/macos/manager.yml +3 -0
- package/ansible/macos/policy.json +11 -1
- package/ansible/macos/t3-policy.yml +49 -0
- package/ansible/monitoring/compose.yml +10 -0
- package/ansible/monitoring/container-watchdog.py +118 -0
- package/ansible/monitoring/hub-darwin.yml +146 -0
- package/ansible/net/remote-probe.sh +5 -0
- package/ansible/ubuntu/baseline.yml +2 -0
- package/ansible/ubuntu/disk-gc.py +25 -3
- package/ansible/ubuntu/disk-guard.py +24 -5
- package/ansible/ubuntu/policy.json +51 -1
- package/ansible/ubuntu/t3-desktop.yml +19 -0
- package/ansible/ubuntu/t3-health.sh +93 -22
- package/ansible/ubuntu/t3-idle.py +42 -0
- package/ansible/ubuntu/t3-policy.yml +4 -11
- package/ansible/ubuntu/tasks/browser.yml +56 -0
- package/ansible/ubuntu/tasks/docker.yml +52 -0
- package/ansible/ubuntu/tasks/recovery.yml +60 -1
- package/ansible/ubuntu/tasks/resources.yml +2 -2
- package/ansible/ubuntu/tasks/services.yml +13 -0
- package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
- package/ansible/ubuntu/wip-guard.py +121 -0
- package/bootstrap/doctor.ts +32 -6
- package/bootstrap/fleet-remote-bootstrap.sh +31 -4
- package/bootstrap/native-t3.ts +22 -11
- package/bootstrap/setup.ts +23 -8
- package/bootstrap/worker-tools.ts +118 -0
- package/cliproxy/README.md +4 -4
- package/cliproxy/compose.yaml +4 -0
- package/cliproxy/scripts/accounts.py +216 -0
- package/cliproxy/scripts/checkpoint.py +30 -2
- package/cliproxy/scripts/cli.py +1 -1
- package/cliproxy/scripts/clientkeys.py +99 -0
- package/cliproxy/scripts/lease.py +80 -0
- package/cliproxy/scripts/ops +137 -25
- package/cliproxy/scripts/restore-verify +15 -0
- package/docs/accounts.md +154 -0
- package/docs/authentication.md +90 -0
- package/docs/declared.md +5 -1
- package/docs/durability.md +26 -0
- package/docs/machine-setup.md +26 -15
- package/docs/operations.md +87 -31
- package/docs/runbooks.md +380 -0
- package/docs/state.md +13 -7
- package/docs/topology.md +2 -0
- package/docs/ubuntu-sync.md +35 -28
- package/examples/fleet.json +19 -1
- package/llms.txt +3 -0
- package/mise.toml +10 -1
- package/package.json +2 -2
- package/schema/check-ids.json +522 -0
- package/schema/fleet.schema.json +244 -1
- package/schema/output/check.schema.json +23 -0
- package/schema/output/error.schema.json +14 -0
- package/schema/output/fleet-report.schema.json +79 -0
- package/schema/output/fleet-sync.schema.json +112 -0
- package/schema/output/report.schema.json +38 -0
- package/src/accounts/checks.ts +220 -0
- package/src/accounts/context.ts +76 -0
- package/src/accounts/propagate.ts +62 -0
- package/src/accounts/roster.ts +85 -0
- package/src/accounts/selection.ts +67 -0
- package/src/accounts/sessions.ts +120 -0
- package/src/agents/account-relay.ts +176 -26
- package/src/agents/launch.ts +84 -5
- package/src/agents/native-verify.ts +128 -2
- package/src/auth-checks.ts +3 -0
- package/src/cli/agent-guide.ts +5 -3
- package/src/cli/fleet-accounts.ts +253 -0
- package/src/cli/fleet-report.ts +126 -0
- package/src/cli/fleet-sync-accounts.ts +58 -0
- package/src/cli/fleet-sync-manager.ts +31 -4
- package/src/cli/fleet-sync-output.ts +3 -2
- package/src/cli/fleet-sync-plan.ts +200 -0
- package/src/cli/fleet-sync-remote.ts +23 -14
- package/src/cli/fleet-sync-schedule.ts +142 -35
- package/src/cli/fleet-sync.ts +126 -8
- package/src/cli/fleet-watch.ts +198 -0
- package/src/cli/fleet.ts +4 -0
- package/src/cli/register-fleet.ts +86 -0
- package/src/cli/registry.ts +26 -0
- package/src/cli/schedule-logs.ts +36 -0
- package/src/clock.ts +63 -0
- package/src/definition.ts +226 -7
- package/src/git-identity.ts +160 -0
- package/src/ide/definition.ts +86 -0
- package/src/ide/ide.ts +43 -0
- package/src/ide/index.ts +5 -0
- package/src/ide/orchestrate.ts +50 -0
- package/src/ide/t3.ts +111 -0
- package/src/machine-health.ts +51 -4
- package/src/manager-health.ts +205 -0
- package/src/manager-sync.ts +13 -17
- package/src/monitoring/hub-backup.ts +63 -17
- package/src/monitoring/manager-hub.ts +44 -2
- package/src/node-converge-linux.ts +44 -12
- package/src/notify.ts +239 -0
- package/src/platform/darwin.ts +0 -13
- package/src/platform/fallback.ts +0 -8
- package/src/platform/linux.ts +0 -10
- package/src/platform/t3-policy.ts +33 -6
- package/src/platform/t3-service.ts +104 -0
- package/src/platform/types.ts +0 -4
- package/src/platform/windows.ts +0 -14
- package/src/remote-bootstrap.ts +87 -25
- package/src/remote-entry.ts +15 -1
- package/src/remote-operation.ts +213 -0
- package/src/schedule-check.ts +40 -0
- package/src/secrets.ts +26 -8
- package/src/service-auth.ts +15 -5
- package/src/ssh.ts +10 -2
- package/src/state/history.ts +172 -0
- package/src/state/store.ts +11 -4
- package/src/t3/accounts.ts +7 -1
- package/src/t3/models.ts +133 -0
- package/src/t3/pairing.ts +31 -0
- package/src/t3/profile.ts +203 -0
- package/src/t3/version.ts +119 -0
- package/src/t3-linked-check.ts +1 -1
- package/src/tailscale.ts +51 -0
- package/src/toolchain-checks.ts +8 -3
- package/src/transport.ts +3 -2
- package/src/worker-capability-checks.ts +167 -0
- package/src/worker-capacity.ts +19 -7
- package/src/worker-host-checks.ts +68 -13
- package/src/worker-policy.ts +33 -0
- package/src/worker-readiness.ts +10 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { sshArgs } from '../ssh';
|
|
4
|
+
import type { Check, FleetNode, Runner } from '../types';
|
|
5
|
+
|
|
6
|
+
/** The T3 release every machine runs: `t3_version` in mise.toml. */
|
|
7
|
+
export async function pinnedT3Version(workspace: string): Promise<string> {
|
|
8
|
+
const toml = await readFile(join(workspace, 'mise.toml'), 'utf8');
|
|
9
|
+
const version = /^t3_version = "([^"]+)"$/m.exec(toml)?.[1];
|
|
10
|
+
if (!version) throw new Error('mise.toml pins no T3 version.');
|
|
11
|
+
return version;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Where T3 notes the version it last started as, under its home. */
|
|
15
|
+
export const STARTED_VERSION = 'userdata/desktop-last-started-version';
|
|
16
|
+
|
|
17
|
+
const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$/;
|
|
18
|
+
|
|
19
|
+
/** `t3:version`: the T3 a machine runs is the one the fleet pins, so the
|
|
20
|
+
* manager and the workers it drives never run different releases.
|
|
21
|
+
*/
|
|
22
|
+
export function versionCheck(
|
|
23
|
+
running: string | null,
|
|
24
|
+
pinned: string,
|
|
25
|
+
where: string,
|
|
26
|
+
install: string
|
|
27
|
+
): Check {
|
|
28
|
+
const observed = running?.trim() ?? '';
|
|
29
|
+
if (!VERSION.test(observed))
|
|
30
|
+
return {
|
|
31
|
+
id: 't3:version',
|
|
32
|
+
status: 'warn',
|
|
33
|
+
summary: 'T3_VERSION_UNKNOWN',
|
|
34
|
+
remedy: {
|
|
35
|
+
action: `T3 on ${where} has not recorded the version it runs. Start T3 there once, then run fleet-sync.`,
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
if (observed !== pinned)
|
|
39
|
+
return {
|
|
40
|
+
id: 't3:version',
|
|
41
|
+
status: 'warn',
|
|
42
|
+
summary: 'T3_VERSION_SKEW',
|
|
43
|
+
remedy: {
|
|
44
|
+
action: `T3 on ${where} runs ${observed}; the fleet pins ${pinned}. ${install}`,
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
return {
|
|
48
|
+
id: 't3:version',
|
|
49
|
+
status: 'pass',
|
|
50
|
+
summary: `T3 on ${where} runs ${pinned}, the release every machine runs.`,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function managerVersion(
|
|
55
|
+
t3Home: string,
|
|
56
|
+
workspace: string
|
|
57
|
+
): Promise<Check> {
|
|
58
|
+
const running = await readFile(join(t3Home, STARTED_VERSION), 'utf8').catch(
|
|
59
|
+
() => null
|
|
60
|
+
);
|
|
61
|
+
return versionCheck(
|
|
62
|
+
running,
|
|
63
|
+
await pinnedT3Version(workspace),
|
|
64
|
+
'the manager',
|
|
65
|
+
'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -K ansible/macos/manager.yml) it no longer updates itself.'
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function workerVersion(
|
|
70
|
+
node: FleetNode,
|
|
71
|
+
run: Runner,
|
|
72
|
+
workspace: string
|
|
73
|
+
): Promise<Check> {
|
|
74
|
+
const result = await run(
|
|
75
|
+
[...sshArgs(node), `cat "$HOME/.t3/${STARTED_VERSION}"`],
|
|
76
|
+
{ timeoutMs: 15_000 }
|
|
77
|
+
);
|
|
78
|
+
return versionCheck(
|
|
79
|
+
result.ok ? result.stdout : null,
|
|
80
|
+
await pinnedT3Version(workspace),
|
|
81
|
+
`worker ${node.id}`,
|
|
82
|
+
'Run fleet-sync for that worker: it installs the pinned desktop app and restarts T3 when its policy changed.'
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Where T3 on macOS reads its administrator policy. */
|
|
87
|
+
export const MACOS_POLICY = '/Library/Application Support/T3Code/policy.json';
|
|
88
|
+
|
|
89
|
+
/** `t3:updates`: T3's own updater is off by the fleet's policy, so it changes
|
|
90
|
+
* only when the pin does.
|
|
91
|
+
*/
|
|
92
|
+
export async function updatesCheck(
|
|
93
|
+
policyPath: string,
|
|
94
|
+
stage: string
|
|
95
|
+
): Promise<Check> {
|
|
96
|
+
let mode: unknown;
|
|
97
|
+
try {
|
|
98
|
+
const policy = JSON.parse(await readFile(policyPath, 'utf8')) as {
|
|
99
|
+
limits?: { updates?: { mode?: unknown } };
|
|
100
|
+
};
|
|
101
|
+
mode = policy.limits?.updates?.mode;
|
|
102
|
+
} catch {
|
|
103
|
+
mode = undefined;
|
|
104
|
+
}
|
|
105
|
+
return mode === 'off'
|
|
106
|
+
? {
|
|
107
|
+
id: 't3:updates',
|
|
108
|
+
status: 'pass',
|
|
109
|
+
summary: 'T3’s own updater is off by the fleet’s policy.',
|
|
110
|
+
}
|
|
111
|
+
: {
|
|
112
|
+
id: 't3:updates',
|
|
113
|
+
status: 'warn',
|
|
114
|
+
summary: 'T3_UPDATES_UNMANAGED',
|
|
115
|
+
remedy: {
|
|
116
|
+
action: `T3 on this machine may update itself away from the pinned release. Stage the fleet's policy once, as an administrator: ${stage}`,
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
package/src/t3-linked-check.ts
CHANGED
package/src/tailscale.ts
CHANGED
|
@@ -10,6 +10,9 @@ export interface TailnetComputer {
|
|
|
10
10
|
ips: string[];
|
|
11
11
|
online: boolean;
|
|
12
12
|
self: boolean;
|
|
13
|
+
/** When the node key expires, in milliseconds since the epoch; absent when
|
|
14
|
+
* it never does (tagged nodes, or expiry disabled in the admin console). */
|
|
15
|
+
keyExpiry?: number;
|
|
13
16
|
}
|
|
14
17
|
export interface Tailnet {
|
|
15
18
|
computers: TailnetComputer[];
|
|
@@ -103,6 +106,7 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
|
|
|
103
106
|
OS: os,
|
|
104
107
|
TailscaleIPs: ips,
|
|
105
108
|
Online: online,
|
|
109
|
+
KeyExpiry: keyExpiry,
|
|
106
110
|
} = value;
|
|
107
111
|
if (
|
|
108
112
|
![id, name, os].every(
|
|
@@ -129,6 +133,53 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
|
|
|
129
133
|
ips: ips as string[],
|
|
130
134
|
online,
|
|
131
135
|
self,
|
|
136
|
+
...expiry(keyExpiry),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function expiry(value: unknown): { keyExpiry?: number } {
|
|
141
|
+
if (typeof value !== 'string') return {};
|
|
142
|
+
const time = Date.parse(value);
|
|
143
|
+
// Tailscale writes the zero time for a key that never expires.
|
|
144
|
+
return Number.isFinite(time) && time > 0 ? { keyExpiry: time } : {};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const DAY_MS = 86_400_000;
|
|
148
|
+
const KEY_EXPIRY = { warnDays: 14, failDays: 3 };
|
|
149
|
+
|
|
150
|
+
/** A node drops off the tailnet on the day its key expires. The check warns
|
|
151
|
+
* two weeks ahead and fails three days ahead, like a T3 link that expires.
|
|
152
|
+
*/
|
|
153
|
+
export function keyExpiryCheck(
|
|
154
|
+
computer: TailnetComputer | undefined,
|
|
155
|
+
now = Date.now()
|
|
156
|
+
): Check {
|
|
157
|
+
const id = 'tailscale:key-expiry';
|
|
158
|
+
if (!computer)
|
|
159
|
+
return { id, status: 'skip', summary: 'TAILSCALE_KEY_EXPIRY_UNOBSERVED' };
|
|
160
|
+
if (computer.keyExpiry === undefined)
|
|
161
|
+
return {
|
|
162
|
+
id,
|
|
163
|
+
status: 'pass',
|
|
164
|
+
summary: 'The Tailscale node key of this machine does not expire.',
|
|
165
|
+
};
|
|
166
|
+
const days = Math.floor((computer.keyExpiry - now) / DAY_MS);
|
|
167
|
+
const remedy = {
|
|
168
|
+
action: computer.self
|
|
169
|
+
? 'Renew the node key of the manager before it expires: tailscale up --force-reauth on the manager, or disable key expiry for it in the Tailscale admin console.'
|
|
170
|
+
: 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
|
|
171
|
+
};
|
|
172
|
+
if (days < 0)
|
|
173
|
+
return { id, status: 'fail', summary: 'TAILSCALE_KEY_EXPIRED', remedy };
|
|
174
|
+
const summary = `TAILSCALE_KEY_EXPIRES_SOON in ${days} days`;
|
|
175
|
+
if (days <= KEY_EXPIRY.failDays)
|
|
176
|
+
return { id, status: 'fail', summary, remedy };
|
|
177
|
+
if (days <= KEY_EXPIRY.warnDays)
|
|
178
|
+
return { id, status: 'warn', summary, remedy };
|
|
179
|
+
return {
|
|
180
|
+
id,
|
|
181
|
+
status: 'pass',
|
|
182
|
+
summary: `The Tailscale node key expires in ${days} days.`,
|
|
132
183
|
};
|
|
133
184
|
}
|
|
134
185
|
|
package/src/toolchain-checks.ts
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import {
|
|
3
|
+
selectionArguments,
|
|
4
|
+
type ToolSelection,
|
|
5
|
+
} from '../bootstrap/worker-tools';
|
|
2
6
|
import { record } from './config';
|
|
3
7
|
import type { Check, Runner } from './types';
|
|
4
8
|
|
|
5
9
|
export async function toolchainChecks(
|
|
6
10
|
root: string,
|
|
7
11
|
run: Runner,
|
|
8
|
-
worker
|
|
12
|
+
/** A worker's own tool selection; absent on the manager. */
|
|
13
|
+
worker?: ToolSelection
|
|
9
14
|
): Promise<Check[]> {
|
|
10
15
|
const result = await run(doctorArguments(root, worker), {
|
|
11
16
|
cwd: root,
|
|
@@ -97,12 +102,12 @@ function toolCheck(value: unknown): Check | null {
|
|
|
97
102
|
};
|
|
98
103
|
}
|
|
99
104
|
|
|
100
|
-
function doctorArguments(root: string, worker?:
|
|
105
|
+
function doctorArguments(root: string, worker?: ToolSelection): string[] {
|
|
101
106
|
return [
|
|
102
107
|
'bun',
|
|
103
108
|
join(root, 'bootstrap/doctor.ts'),
|
|
104
109
|
'--json',
|
|
105
110
|
'--docker',
|
|
106
|
-
...(worker ? ['--worker'] : []),
|
|
111
|
+
...(worker ? ['--worker', ...selectionArguments(worker)] : []),
|
|
107
112
|
];
|
|
108
113
|
}
|
package/src/transport.ts
CHANGED
|
@@ -21,7 +21,7 @@ export async function bundleProbe(): Promise<string> {
|
|
|
21
21
|
export interface RemoteOptions {
|
|
22
22
|
source: string;
|
|
23
23
|
timeoutMs: number;
|
|
24
|
-
action: 'probe' | 'sync' | 'repair' | 'configure';
|
|
24
|
+
action: 'probe' | 'sync' | 'repair' | 'configure' | 'accounts';
|
|
25
25
|
configuration?: ConfigurationRequest;
|
|
26
26
|
repairs?: AutomatedRepair[];
|
|
27
27
|
expected?: Fingerprint;
|
|
@@ -41,7 +41,8 @@ export async function inspectRemote(
|
|
|
41
41
|
return { node: node.id, checks: [sshFailure(preflight)] };
|
|
42
42
|
}
|
|
43
43
|
const target =
|
|
44
|
-
options.action === 'probe'
|
|
44
|
+
(options.action === 'probe' || options.action === 'accounts') &&
|
|
45
|
+
options.expected
|
|
45
46
|
? await managedProbeTarget(node, options.expected.head, run)
|
|
46
47
|
: { node, checks: [] };
|
|
47
48
|
const command = remoteCommand(target.node, {
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import policy from '../ansible/ubuntu/policy.json';
|
|
2
|
+
import type { Check, CommandResult, Runner } from './types';
|
|
3
|
+
|
|
4
|
+
/** What an agent does on a worker, tried the way it does it: start a
|
|
5
|
+
* container, build an image, launch a headless browser. Each check runs as
|
|
6
|
+
* the worker's user, in its session, and reports a code and a remedy, never
|
|
7
|
+
* what a command printed.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
function failure(id: string, summary: string, action: string): Check {
|
|
11
|
+
return { id, status: 'fail', summary, remedy: { action } };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Docker's own words for the causes an agent meets most. */
|
|
15
|
+
function dockerFailure(result: CommandResult): {
|
|
16
|
+
code: string;
|
|
17
|
+
action: string;
|
|
18
|
+
} {
|
|
19
|
+
const said = `${result.stdout}\n${result.stderr}`;
|
|
20
|
+
if (result.reason === 'timeout')
|
|
21
|
+
return {
|
|
22
|
+
code: 'DOCKER_RUN_TIMEOUT',
|
|
23
|
+
action:
|
|
24
|
+
'A container did not start and exit in time. Inspect the engine (systemctl status docker) and its disk; rerun fleet-sync.',
|
|
25
|
+
};
|
|
26
|
+
if (/permission denied[^\n]*docker\.sock/i.test(said))
|
|
27
|
+
return {
|
|
28
|
+
code: 'DOCKER_SOCKET_ACCESS_DENIED',
|
|
29
|
+
action:
|
|
30
|
+
"This session cannot use the Docker socket. fleet-sync adds the worker's user to the docker group; a session started before that keeps its old groups. Rerun fleet-sync, then restart the session (log out and in, or restart the T3 service).",
|
|
31
|
+
};
|
|
32
|
+
if (/cannot connect to the docker daemon/i.test(said))
|
|
33
|
+
return {
|
|
34
|
+
code: 'DOCKER_DAEMON_UNREACHABLE',
|
|
35
|
+
action:
|
|
36
|
+
'The Docker engine is not running. Rerun fleet-sync, which starts and enables docker.service.',
|
|
37
|
+
};
|
|
38
|
+
if (/no such image|pull access denied|not found/i.test(said))
|
|
39
|
+
return {
|
|
40
|
+
code: 'DOCKER_SMOKE_IMAGE_MISSING',
|
|
41
|
+
action:
|
|
42
|
+
'The pinned smoke-test image is not cached on this worker. Rerun fleet-sync, which pulls it once so this check works offline.',
|
|
43
|
+
};
|
|
44
|
+
if (/no space left on device/i.test(said))
|
|
45
|
+
return {
|
|
46
|
+
code: 'DOCKER_DISK_FULL',
|
|
47
|
+
action:
|
|
48
|
+
'The image store is full. Free space under /var/lib/docker (the disk GC prunes old images) and rerun fleet-sync.',
|
|
49
|
+
};
|
|
50
|
+
return {
|
|
51
|
+
code: 'DOCKER_RUN_FAILED',
|
|
52
|
+
action:
|
|
53
|
+
'Docker answers but cannot run a container here. Inspect cgroups and the engine log (journalctl -u docker), then rerun fleet-sync.',
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Runs the pinned image offline, then builds a one-line image from it. */
|
|
58
|
+
export async function dockerRunCheck(run: Runner): Promise<Check> {
|
|
59
|
+
const id = 'worker:docker-run';
|
|
60
|
+
const { image, timeoutSeconds } = policy.dockerSmokeTest;
|
|
61
|
+
const timeoutMs = timeoutSeconds * 1000;
|
|
62
|
+
const started = await run(
|
|
63
|
+
[
|
|
64
|
+
'docker',
|
|
65
|
+
'run',
|
|
66
|
+
'--rm',
|
|
67
|
+
'--network',
|
|
68
|
+
'none',
|
|
69
|
+
'--pull',
|
|
70
|
+
'never',
|
|
71
|
+
image,
|
|
72
|
+
'true',
|
|
73
|
+
],
|
|
74
|
+
{ timeoutMs }
|
|
75
|
+
);
|
|
76
|
+
if (!started.ok) {
|
|
77
|
+
const { code, action } = dockerFailure(started);
|
|
78
|
+
return failure(id, code, action);
|
|
79
|
+
}
|
|
80
|
+
const built = await run(
|
|
81
|
+
['docker', 'build', '--quiet', '--pull=false', '--network', 'none', '-'],
|
|
82
|
+
{ input: `FROM ${image}\nRUN true\n`, timeoutMs }
|
|
83
|
+
);
|
|
84
|
+
if (!built.ok) {
|
|
85
|
+
const { code, action } = dockerFailure(built);
|
|
86
|
+
return failure(
|
|
87
|
+
id,
|
|
88
|
+
code === 'DOCKER_RUN_FAILED' ? 'DOCKER_BUILD_FAILED' : code,
|
|
89
|
+
code === 'DOCKER_RUN_FAILED'
|
|
90
|
+
? 'A container runs but an image does not build. Inspect Buildx (docker buildx ls) and the image store, then rerun fleet-sync.'
|
|
91
|
+
: action
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
id,
|
|
96
|
+
status: 'pass',
|
|
97
|
+
summary:
|
|
98
|
+
'A container of the pinned image ran and a one-line image built, offline, in this session.',
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Where `ansible/ubuntu/tasks/browser.yml` unpacks the pinned build. */
|
|
103
|
+
export function headlessShell(arch: string = process.arch): string {
|
|
104
|
+
const { version, amd64, arm64 } = policy.browser.headlessShell;
|
|
105
|
+
const build = arch === 'arm64' ? arm64 : amd64;
|
|
106
|
+
const directory = build.archive.split('/').pop()!;
|
|
107
|
+
return `/opt/personal-fleet/browser/${version}/${directory}/chrome-headless-shell`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** A worker with the web profile launches the pinned headless Chromium,
|
|
111
|
+
* loads a blank page and exits, as a browser test would.
|
|
112
|
+
*/
|
|
113
|
+
export async function browserCheck(
|
|
114
|
+
run: Runner,
|
|
115
|
+
web: boolean,
|
|
116
|
+
arch: string = process.arch
|
|
117
|
+
): Promise<Check> {
|
|
118
|
+
const id = 'worker:browser';
|
|
119
|
+
if (!web)
|
|
120
|
+
return {
|
|
121
|
+
id,
|
|
122
|
+
status: 'skip',
|
|
123
|
+
summary:
|
|
124
|
+
'This worker does not select the web tool profile, so no browser is installed for tests.',
|
|
125
|
+
};
|
|
126
|
+
const launched = await run(
|
|
127
|
+
[
|
|
128
|
+
headlessShell(arch),
|
|
129
|
+
'--headless',
|
|
130
|
+
'--no-sandbox',
|
|
131
|
+
'--disable-gpu',
|
|
132
|
+
'--dump-dom',
|
|
133
|
+
'about:blank',
|
|
134
|
+
],
|
|
135
|
+
{ timeoutMs: policy.browser.launchTimeoutSeconds * 1000 }
|
|
136
|
+
);
|
|
137
|
+
if (launched.ok && /<html/i.test(launched.stdout))
|
|
138
|
+
return {
|
|
139
|
+
id,
|
|
140
|
+
status: 'pass',
|
|
141
|
+
summary: `Headless Chromium ${policy.browser.headlessShell.version} launched, loaded a blank page and exited.`,
|
|
142
|
+
};
|
|
143
|
+
const said = `${launched.stdout}\n${launched.stderr}`;
|
|
144
|
+
if (launched.reason === 'missing')
|
|
145
|
+
return failure(
|
|
146
|
+
id,
|
|
147
|
+
'BROWSER_NOT_INSTALLED',
|
|
148
|
+
'The pinned headless Chromium is not installed. Rerun fleet-sync, which installs it on workers with the web profile.'
|
|
149
|
+
);
|
|
150
|
+
if (/error while loading shared libraries/i.test(said))
|
|
151
|
+
return failure(
|
|
152
|
+
id,
|
|
153
|
+
'BROWSER_LIBRARY_MISSING',
|
|
154
|
+
'A system library headless Chromium loads is missing. Rerun fleet-sync, which installs the pinned libraries of the web profile.'
|
|
155
|
+
);
|
|
156
|
+
if (launched.reason === 'timeout')
|
|
157
|
+
return failure(
|
|
158
|
+
id,
|
|
159
|
+
'BROWSER_LAUNCH_TIMEOUT',
|
|
160
|
+
'Headless Chromium did not exit in time. Check the worker for memory pressure, then rerun fleet-sync.'
|
|
161
|
+
);
|
|
162
|
+
return failure(
|
|
163
|
+
id,
|
|
164
|
+
'BROWSER_LAUNCH_FAILED',
|
|
165
|
+
'Headless Chromium did not load a blank page. Run it by hand with --dump-dom about:blank to see why, then rerun fleet-sync.'
|
|
166
|
+
);
|
|
167
|
+
}
|
package/src/worker-capacity.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
+
import { clockCheck } from './clock';
|
|
2
3
|
import { healthChecks, parseProbe } from './machine-health';
|
|
3
4
|
import type { Check, Runner } from './types';
|
|
4
5
|
|
|
5
6
|
export async function workerCapacity(
|
|
6
7
|
root: string,
|
|
7
8
|
node: string,
|
|
8
|
-
run: Runner
|
|
9
|
+
run: Runner,
|
|
10
|
+
/** The manager's clock in seconds since the epoch, when it sent one. */
|
|
11
|
+
reference?: number
|
|
9
12
|
): Promise<Check[]> {
|
|
10
13
|
const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
|
|
11
14
|
timeoutMs: 10_000,
|
|
@@ -25,10 +28,19 @@ export async function workerCapacity(
|
|
|
25
28
|
summary: 'Required CPU, memory and disk evidence is unavailable.',
|
|
26
29
|
},
|
|
27
30
|
];
|
|
28
|
-
return
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
return [
|
|
32
|
+
...healthChecks(sample, node).map((check) => ({
|
|
33
|
+
id: `worker:capacity:${check.id}`,
|
|
34
|
+
status: check.severity,
|
|
35
|
+
summary: check.detail,
|
|
36
|
+
...(check.fix ? { remedy: { action: check.fix } } : {}),
|
|
37
|
+
})),
|
|
38
|
+
clockCheck(
|
|
39
|
+
'worker:clock',
|
|
40
|
+
sample,
|
|
41
|
+
reference === undefined || sample.epoch === undefined
|
|
42
|
+
? undefined
|
|
43
|
+
: sample.epoch - reference
|
|
44
|
+
),
|
|
45
|
+
];
|
|
34
46
|
}
|
|
@@ -4,6 +4,7 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
|
|
|
4
4
|
import { verifyResourceSafety } from './platform/resource-verify';
|
|
5
5
|
import policy from '../ansible/ubuntu/policy.json';
|
|
6
6
|
import { userInfo } from 'node:os';
|
|
7
|
+
import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
|
|
7
8
|
import type { Check, Runner } from './types';
|
|
8
9
|
|
|
9
10
|
function check(id: string, ok: boolean, summary: string): Check {
|
|
@@ -94,6 +95,7 @@ export async function workerHostChecks(
|
|
|
94
95
|
)
|
|
95
96
|
);
|
|
96
97
|
}
|
|
98
|
+
checks.push(await rebootPendingCheck(audit.rebootPending, run));
|
|
97
99
|
checks.push(...(await recoveryChecks(run)));
|
|
98
100
|
checks.push(...(await verifyUbuntuBoot(user, run)));
|
|
99
101
|
checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
|
|
@@ -120,18 +122,13 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
|
|
|
120
122
|
)
|
|
121
123
|
);
|
|
122
124
|
checks.push(await t3HealthCheck(run));
|
|
125
|
+
checks.push(await runningVersionCheck(run, userInfo().uid));
|
|
123
126
|
const renewal = await run([
|
|
124
127
|
'systemctl',
|
|
125
128
|
'--user',
|
|
126
129
|
'is-active',
|
|
127
130
|
'personal-worker-auth.timer',
|
|
128
131
|
]);
|
|
129
|
-
const recovery = await run([
|
|
130
|
-
'systemctl',
|
|
131
|
-
'--user',
|
|
132
|
-
'is-active',
|
|
133
|
-
'personal-worker-recovery.timer',
|
|
134
|
-
]);
|
|
135
132
|
checks.push(
|
|
136
133
|
check(
|
|
137
134
|
'credential-renewal',
|
|
@@ -139,17 +136,44 @@ async function recoveryChecks(run: Runner): Promise<Check[]> {
|
|
|
139
136
|
'Worker credential renewal timer is active.'
|
|
140
137
|
)
|
|
141
138
|
);
|
|
142
|
-
checks.push(
|
|
143
|
-
check(
|
|
144
|
-
'service-recovery',
|
|
145
|
-
recovery.ok,
|
|
146
|
-
'Worker service recovery timer is active.'
|
|
147
|
-
)
|
|
148
|
-
);
|
|
139
|
+
checks.push(await serviceRecoveryCheck(run));
|
|
149
140
|
checks.push(await powerRecoveryCheck(run));
|
|
150
141
|
return checks;
|
|
151
142
|
}
|
|
152
143
|
|
|
144
|
+
/** The recovery timer is active, the script its unit runs exists, and the
|
|
145
|
+
* last run of the unit succeeded: a watchdog that fails on every tick is not
|
|
146
|
+
* recovery. */
|
|
147
|
+
export async function serviceRecoveryCheck(run: Runner): Promise<Check> {
|
|
148
|
+
const timer = await run([
|
|
149
|
+
'systemctl',
|
|
150
|
+
'--user',
|
|
151
|
+
'is-active',
|
|
152
|
+
'personal-worker-recovery.timer',
|
|
153
|
+
]);
|
|
154
|
+
const script = await run(['test', '-r', T3_HEALTH_SCRIPT]);
|
|
155
|
+
const unit = await run([
|
|
156
|
+
'systemctl',
|
|
157
|
+
'--user',
|
|
158
|
+
'show',
|
|
159
|
+
'personal-worker-recovery.service',
|
|
160
|
+
'--property=Result',
|
|
161
|
+
]);
|
|
162
|
+
const result = /^Result=(.*)$/m.exec(unit.stdout)?.[1] ?? 'unknown';
|
|
163
|
+
const summary = !timer.ok
|
|
164
|
+
? 'WORKER_RECOVERY_TIMER_INACTIVE'
|
|
165
|
+
: !script.ok
|
|
166
|
+
? 'WORKER_RECOVERY_SCRIPT_MISSING'
|
|
167
|
+
: !unit.ok || result !== 'success'
|
|
168
|
+
? `WORKER_RECOVERY_LAST_RUN_FAILED (${/^[\w-]+$/.test(result) ? result : 'unknown'})`
|
|
169
|
+
: 'Worker service recovery timer is active and its last run succeeded.';
|
|
170
|
+
return check(
|
|
171
|
+
'service-recovery',
|
|
172
|
+
timer.ok && script.ok && unit.ok && result === 'success',
|
|
173
|
+
summary
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
153
177
|
/** The server answers its local API now, not merely that a process exists. */
|
|
154
178
|
async function t3HealthCheck(run: Runner): Promise<Check> {
|
|
155
179
|
const probe = await run(
|
|
@@ -179,3 +203,34 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
|
|
|
179
203
|
'T3 local API answers with its environment identity within 10 seconds.'
|
|
180
204
|
);
|
|
181
205
|
}
|
|
206
|
+
|
|
207
|
+
const DAY_SECONDS = 86_400;
|
|
208
|
+
|
|
209
|
+
/** Kernel and library updates apply only after a reboot. A pending reboot
|
|
210
|
+
* warns with its age, and fails once it is older than `policy.json` allows.
|
|
211
|
+
*/
|
|
212
|
+
export async function rebootPendingCheck(
|
|
213
|
+
pending: boolean,
|
|
214
|
+
run: Runner,
|
|
215
|
+
now = Date.now()
|
|
216
|
+
): Promise<Check> {
|
|
217
|
+
const id = 'worker:reboot-pending';
|
|
218
|
+
if (!pending) return { id, status: 'pass', summary: 'No reboot is pending.' };
|
|
219
|
+
const since = await run(['stat', '-c', '%Y', '/var/run/reboot-required']);
|
|
220
|
+
const seconds = Number(since.stdout.trim());
|
|
221
|
+
const days =
|
|
222
|
+
since.ok && /^\d+$/.test(since.stdout.trim())
|
|
223
|
+
? Math.max(0, Math.floor((now / 1000 - seconds) / DAY_SECONDS))
|
|
224
|
+
: null;
|
|
225
|
+
const age = days === null ? '' : ` for ${days} days`;
|
|
226
|
+
return {
|
|
227
|
+
id,
|
|
228
|
+
status:
|
|
229
|
+
days !== null && days > policy.rebootPendingFailDays ? 'fail' : 'warn',
|
|
230
|
+
summary: `REBOOT_PENDING${age}: installed updates apply only after a reboot.`,
|
|
231
|
+
remedy: {
|
|
232
|
+
action:
|
|
233
|
+
'Reboot the worker when no agent turn is running (sudo systemctl reboot on the worker), then confirm it returns to ready with devfleet fleet-status --workers.',
|
|
234
|
+
},
|
|
235
|
+
};
|
|
236
|
+
}
|
package/src/worker-policy.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { CAPABILITIES, record } from './config';
|
|
2
2
|
import { FleetUsageError } from './errors';
|
|
3
3
|
import type { Capability } from './types';
|
|
4
|
+
import { TOOL_PROFILES, type ToolProfile } from '../bootstrap/worker-tools';
|
|
4
5
|
|
|
5
6
|
export interface WorkerPolicy {
|
|
6
7
|
requiredCapabilities: Capability[];
|
|
@@ -12,6 +13,10 @@ export interface WorkerPolicy {
|
|
|
12
13
|
unattendedSudo: true;
|
|
13
14
|
persistentT3: true;
|
|
14
15
|
neverSleep: true;
|
|
16
|
+
/** Tool profiles every worker installs on top of the base tools. */
|
|
17
|
+
toolProfiles?: ToolProfile[];
|
|
18
|
+
/** Single tools pinned in dev-fleet's mise.toml every worker installs. */
|
|
19
|
+
tools?: string[];
|
|
15
20
|
}
|
|
16
21
|
const REQUIRED = [
|
|
17
22
|
'github',
|
|
@@ -41,8 +46,11 @@ function validPolicy(value: Record<string, unknown>): boolean {
|
|
|
41
46
|
'unattendedSudo',
|
|
42
47
|
'persistentT3',
|
|
43
48
|
'neverSleep',
|
|
49
|
+
'toolProfiles',
|
|
50
|
+
'tools',
|
|
44
51
|
].includes(key)
|
|
45
52
|
) &&
|
|
53
|
+
validTools(value) &&
|
|
46
54
|
validCapabilities(value['requiredCapabilities']) &&
|
|
47
55
|
typeof value['githubLogin'] === 'string' &&
|
|
48
56
|
/^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
|
|
@@ -55,6 +63,31 @@ function validPolicy(value: Record<string, unknown>): boolean {
|
|
|
55
63
|
)
|
|
56
64
|
);
|
|
57
65
|
}
|
|
66
|
+
/** A list of known tool profiles, each once; null when it is not one. */
|
|
67
|
+
export function toolProfiles(value: unknown): ToolProfile[] | null {
|
|
68
|
+
return Array.isArray(value) &&
|
|
69
|
+
new Set(value).size === value.length &&
|
|
70
|
+
value.every((item) => TOOL_PROFILES.includes(item as ToolProfile))
|
|
71
|
+
? (value as ToolProfile[])
|
|
72
|
+
: null;
|
|
73
|
+
}
|
|
74
|
+
/** The optional tool profiles and single tools every worker adds. */
|
|
75
|
+
function validTools(value: Record<string, unknown>): boolean {
|
|
76
|
+
return (
|
|
77
|
+
(value['toolProfiles'] === undefined ||
|
|
78
|
+
toolProfiles(value['toolProfiles']) !== null) &&
|
|
79
|
+
(value['tools'] === undefined || toolNames(value['tools']))
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
function toolNames(value: unknown): boolean {
|
|
83
|
+
return (
|
|
84
|
+
Array.isArray(value) &&
|
|
85
|
+
new Set(value).size === value.length &&
|
|
86
|
+
value.every(
|
|
87
|
+
(item) => typeof item === 'string' && /^[a-z0-9@:/._-]{1,75}$/i.test(item)
|
|
88
|
+
)
|
|
89
|
+
);
|
|
90
|
+
}
|
|
58
91
|
export function parseWorkerPolicy(value: unknown): WorkerPolicy {
|
|
59
92
|
if (!record(value) || !validPolicy(value))
|
|
60
93
|
throw new FleetUsageError(
|
package/src/worker-readiness.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { workerCapacity } from './worker-capacity';
|
|
2
2
|
import { userInfo, hostname } from 'node:os';
|
|
3
3
|
import { hostId } from './topology';
|
|
4
|
-
import { configPath } from './fleet-config';
|
|
4
|
+
import { configPath, fleet } from './fleet-config';
|
|
5
|
+
import { workerSelection } from '../bootstrap/worker-tools';
|
|
6
|
+
import { browserCheck, dockerRunCheck } from './worker-capability-checks';
|
|
7
|
+
import { gitChecks } from './git-identity';
|
|
5
8
|
import type { Check, Runner } from './types';
|
|
6
9
|
import { workerHostChecks } from './worker-host-checks';
|
|
7
10
|
import { workerAccessChecks } from './worker-access-checks';
|
|
@@ -59,6 +62,12 @@ export async function workerReadiness(
|
|
|
59
62
|
...host,
|
|
60
63
|
...access,
|
|
61
64
|
proxy,
|
|
65
|
+
await dockerRunCheck(run),
|
|
66
|
+
await browserCheck(
|
|
67
|
+
run,
|
|
68
|
+
workerSelection(fleet, nodeId).profiles.includes('web')
|
|
69
|
+
),
|
|
70
|
+
...(await gitChecks(fleet.git, run)),
|
|
62
71
|
...(await workerCapacity(root, nodeId, run)),
|
|
63
72
|
];
|
|
64
73
|
}
|