@crvouga/dev-fleet 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -9
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +55 -9
- package/ansible/macos/manager.yml +154 -7
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +7 -3
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +19 -11
- package/docs/consumer-setup.md +42 -27
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +10 -3
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +98 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +67 -6
- package/docs/ubuntu-sync.md +78 -28
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1967 -170
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +327 -8
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +61 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +9 -3
- package/src/cli/fleet-node-lifecycle.ts +313 -64
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +51 -35
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +4 -0
- package/src/cli/fleet-sync-manager.ts +103 -33
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +151 -9
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +28 -16
- package/src/cli/fleet-sync-vault.ts +77 -0
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +150 -19
- package/src/cli/fleet-validate.ts +59 -21
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +115 -4
- package/src/cli/registry.ts +14 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +332 -9
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +37 -15
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +65 -4
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +4 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capacity.ts +31 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +27 -4
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +17 -1
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +16 -11
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
|
@@ -1,20 +1,41 @@
|
|
|
1
1
|
import policy from '../../ansible/ubuntu/policy.json';
|
|
2
2
|
import type { Runner, Check } from '../types';
|
|
3
|
+
import type { Session } from '../../bootstrap/worker-features';
|
|
3
4
|
|
|
4
5
|
/** The user unit that runs the pinned desktop app (UI and embedded server). */
|
|
5
6
|
export const T3_DESKTOP_UNIT = 't3code-desktop.service';
|
|
7
|
+
/** The lingering user unit that runs the pinned native server on a headless
|
|
8
|
+
* worker (`t3-headless.yml`). */
|
|
9
|
+
export const T3_HEADLESS_UNIT = 't3code-headless.service';
|
|
6
10
|
|
|
7
|
-
/**
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
/** The unit that runs T3 for a session type. */
|
|
12
|
+
export function t3Unit(session: Session = 'desktop'): string {
|
|
13
|
+
return session === 'headless' ? T3_HEADLESS_UNIT : T3_DESKTOP_UNIT;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Workers run one T3 process: on a desktop worker the desktop app, started
|
|
17
|
+
* with the auto-login graphical session (`t3-desktop.yml`); on a headless
|
|
18
|
+
* worker the pinned native server as a lingering user service
|
|
19
|
+
* (`t3-headless.yml`). The play installs it, retires the other session
|
|
20
|
+
* type's unit and starts it; this only refuses a foreign launcher beforehand
|
|
10
21
|
* and measures the result afterwards.
|
|
11
22
|
*/
|
|
12
23
|
export async function convergeT3Service(
|
|
13
24
|
artifact: { path: string; version: string },
|
|
14
25
|
run: Runner,
|
|
15
|
-
|
|
16
|
-
|
|
26
|
+
steps: {
|
|
27
|
+
/** Stages the policy; any check it returns stops here. */
|
|
28
|
+
prepare?: () => Promise<Check[]>;
|
|
29
|
+
/** Applies the session type's playbook. */
|
|
30
|
+
install?: () => Promise<boolean>;
|
|
31
|
+
session?: Session;
|
|
32
|
+
} = {}
|
|
17
33
|
): Promise<Check[]> {
|
|
34
|
+
const {
|
|
35
|
+
prepare = async () => [],
|
|
36
|
+
install = async () => true,
|
|
37
|
+
session = 'desktop',
|
|
38
|
+
} = steps;
|
|
18
39
|
const existing = await run(['t3', '--version']);
|
|
19
40
|
if (conflictingVersion(existing, artifact.version))
|
|
20
41
|
return pending(
|
|
@@ -23,27 +44,55 @@ export async function convergeT3Service(
|
|
|
23
44
|
);
|
|
24
45
|
const preparation = await prepare();
|
|
25
46
|
if (preparation.length) return preparation;
|
|
26
|
-
|
|
27
|
-
|
|
47
|
+
const words = SESSION_WORDS[session];
|
|
48
|
+
if (!(await install())) return pending(...words.installFailed);
|
|
49
|
+
const unit = t3Unit(session);
|
|
50
|
+
const active = await userctl(run, 'is-active', unit);
|
|
51
|
+
const enabled = await userctl(run, 'is-enabled', unit);
|
|
52
|
+
if (!active.ok || !enabled.ok) return pending(...words.stopped(unit));
|
|
53
|
+
return [{ id: 't3:service', status: 'pass', summary: words.running }];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** How a worker's T3 restarts for a new release or policy, as the manager's
|
|
57
|
+
* bootstrap passes it on to node convergence (`fleet-sync --restart-busy`).
|
|
58
|
+
* The words cross from one revision's script to another: an interface.
|
|
59
|
+
*/
|
|
60
|
+
export const T3_RESTART_MODES = ['keep-turns', 'restart-busy'] as const;
|
|
61
|
+
|
|
62
|
+
/** What the check says of each session type: its codes and remedies. */
|
|
63
|
+
const SESSION_WORDS: Record<
|
|
64
|
+
Session,
|
|
65
|
+
{
|
|
66
|
+
installFailed: [string, string];
|
|
67
|
+
stopped: (unit: string) => [string, string];
|
|
68
|
+
running: string;
|
|
69
|
+
}
|
|
70
|
+
> = {
|
|
71
|
+
desktop: {
|
|
72
|
+
installFailed: [
|
|
28
73
|
'T3_DESKTOP_INSTALL_FAILED',
|
|
29
|
-
'Inspect the t3-desktop.yml Ansible output (package download, apt, GDM auto-login, user unit). T3 data in ~/.t3 is untouched.'
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const enabled = await userctl(run, 'is-enabled', T3_DESKTOP_UNIT);
|
|
33
|
-
if (!active.ok || !enabled.ok)
|
|
34
|
-
return pending(
|
|
74
|
+
'Inspect the t3-desktop.yml Ansible output (package download, apt, GDM auto-login, user unit). T3 data in ~/.t3 is untouched.',
|
|
75
|
+
],
|
|
76
|
+
stopped: (unit) => [
|
|
35
77
|
'T3_DESKTOP_STOPPED',
|
|
36
|
-
`The desktop app needs the auto-login graphical session. Reboot or log in on the machine, then inspect journalctl --user -u ${
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
78
|
+
`The desktop app needs the auto-login graphical session. Reboot or log in on the machine, then inspect journalctl --user -u ${unit}; ~/.t3 is preserved.`,
|
|
79
|
+
],
|
|
80
|
+
running:
|
|
81
|
+
'Pinned T3 desktop app is enabled for the auto-login session and running. Private routing, policy and pairing remain separate checks.',
|
|
82
|
+
},
|
|
83
|
+
headless: {
|
|
84
|
+
installFailed: [
|
|
85
|
+
'T3_SERVER_INSTALL_FAILED',
|
|
86
|
+
'Inspect the t3-headless.yml Ansible output (the pinned server in the toolchain, the user unit). T3 data in ~/.t3 is untouched.',
|
|
87
|
+
],
|
|
88
|
+
stopped: (unit) => [
|
|
89
|
+
'T3_SERVER_STOPPED',
|
|
90
|
+
`The T3 server is a lingering user service and needs no login. Inspect journalctl --user -u ${unit} and loginctl show-user for Linger=yes; ~/.t3 is preserved.`,
|
|
91
|
+
],
|
|
92
|
+
running:
|
|
93
|
+
'Pinned T3 server is enabled as a lingering user service and running, with no display. Private routing, policy and pairing remain separate checks.',
|
|
94
|
+
},
|
|
95
|
+
};
|
|
47
96
|
|
|
48
97
|
/** The recovery script, installed outside every revision (`tasks/recovery.yml`). */
|
|
49
98
|
export const T3_HEALTH_SCRIPT = '/usr/local/lib/personal-fleet/t3-health.sh';
|
|
@@ -85,7 +134,8 @@ export async function restartDeferral(
|
|
|
85
134
|
export async function runningVersionCheck(
|
|
86
135
|
run: Runner,
|
|
87
136
|
uid: number,
|
|
88
|
-
now: number = Date.now()
|
|
137
|
+
now: number = Date.now(),
|
|
138
|
+
session: Session = 'desktop'
|
|
89
139
|
): Promise<Check> {
|
|
90
140
|
const server = await run(
|
|
91
141
|
[
|
|
@@ -98,11 +148,8 @@ export async function runningVersionCheck(
|
|
|
98
148
|
],
|
|
99
149
|
{ timeoutMs: 15_000 }
|
|
100
150
|
);
|
|
101
|
-
const installed = await run(['cat', T3_DESKTOP_RELEASE]);
|
|
102
151
|
const running = serverVersion(server.ok ? server.stdout : '');
|
|
103
|
-
const release =
|
|
104
|
-
installed.ok ? installed.stdout.trim() : ''
|
|
105
|
-
)?.[1];
|
|
152
|
+
const release = await installedRelease(run, session);
|
|
106
153
|
const id = 't3:running-version';
|
|
107
154
|
if (!running || !release)
|
|
108
155
|
return {
|
|
@@ -111,7 +158,7 @@ export async function runningVersionCheck(
|
|
|
111
158
|
summary: 'T3_RUNNING_VERSION_UNKNOWN',
|
|
112
159
|
remedy: {
|
|
113
160
|
action:
|
|
114
|
-
'The running server or the installed release could not be read. Check worker:t3-health, then rerun fleet-sync to reinstall the pinned
|
|
161
|
+
'The running server or the installed release could not be read. Check worker:t3-health, then rerun fleet-sync to reinstall the pinned T3.',
|
|
115
162
|
},
|
|
116
163
|
};
|
|
117
164
|
if (running === release)
|
|
@@ -127,12 +174,37 @@ export async function runningVersionCheck(
|
|
|
127
174
|
summary: 'T3_RESTART_PENDING',
|
|
128
175
|
remedy: {
|
|
129
176
|
action: deferral.overdue
|
|
130
|
-
? `T3 still runs ${running} after ${policy.t3RestartMaxDeferralHours} hours because agent turns kept running. Finish or stop them, or restart ${
|
|
177
|
+
? `T3 still runs ${running} after ${policy.t3RestartMaxDeferralHours} hours because agent turns kept running. Finish or stop them, or restart ${t3Unit(session)} on the worker (this ends its running turns).`
|
|
131
178
|
: `T3 runs ${running}; ${release} is installed. personal-worker-recovery restarts it at the first check with no agent turn running.`,
|
|
132
179
|
},
|
|
133
180
|
};
|
|
134
181
|
}
|
|
135
182
|
|
|
183
|
+
/** The release installed for the session: the desktop package the app was
|
|
184
|
+
* unpacked from, or the versioned directory the server unit runs. */
|
|
185
|
+
async function installedRelease(
|
|
186
|
+
run: Runner,
|
|
187
|
+
session: Session
|
|
188
|
+
): Promise<string | undefined> {
|
|
189
|
+
if (session === 'headless') {
|
|
190
|
+
const shown = await run([
|
|
191
|
+
'systemctl',
|
|
192
|
+
'--user',
|
|
193
|
+
'show',
|
|
194
|
+
'--property=ExecStart',
|
|
195
|
+
'--value',
|
|
196
|
+
T3_HEADLESS_UNIT,
|
|
197
|
+
]);
|
|
198
|
+
return shown.ok
|
|
199
|
+
? /\/t3-([\w.+-]+?)-linux-[a-z0-9]+\/t3\b/.exec(shown.stdout)?.[1]
|
|
200
|
+
: undefined;
|
|
201
|
+
}
|
|
202
|
+
const installed = await run(['cat', T3_DESKTOP_RELEASE]);
|
|
203
|
+
return /^T3-Code-(.+)-[a-z0-9]+\.deb$/.exec(
|
|
204
|
+
installed.ok ? installed.stdout.trim() : ''
|
|
205
|
+
)?.[1];
|
|
206
|
+
}
|
|
207
|
+
|
|
136
208
|
function serverVersion(body: string): string | undefined {
|
|
137
209
|
try {
|
|
138
210
|
const value: unknown = JSON.parse(body);
|
|
@@ -148,15 +220,18 @@ function serverVersion(body: string): string | undefined {
|
|
|
148
220
|
}
|
|
149
221
|
}
|
|
150
222
|
|
|
151
|
-
/** When
|
|
152
|
-
export async function t3StartedAt(
|
|
223
|
+
/** When T3 last started; changes exactly when it restarts. */
|
|
224
|
+
export async function t3StartedAt(
|
|
225
|
+
run: Runner,
|
|
226
|
+
session: Session = 'desktop'
|
|
227
|
+
): Promise<string> {
|
|
153
228
|
const shown = await run([
|
|
154
229
|
'systemctl',
|
|
155
230
|
'--user',
|
|
156
231
|
'show',
|
|
157
232
|
'--property=ActiveEnterTimestampMonotonic',
|
|
158
233
|
'--value',
|
|
159
|
-
|
|
234
|
+
t3Unit(session),
|
|
160
235
|
]);
|
|
161
236
|
return shown.ok ? shown.stdout.trim() : '';
|
|
162
237
|
}
|
|
@@ -9,6 +9,11 @@ export interface UbuntuRelease {
|
|
|
9
9
|
architectures: string[];
|
|
10
10
|
/** Pins of this release that differ from the policy's `packages`. */
|
|
11
11
|
packages?: Record<string, string>;
|
|
12
|
+
/** Pins of this release that differ from the policy's `browser.packages`. */
|
|
13
|
+
browserPackages?: Record<string, string>;
|
|
14
|
+
/** The digest of the key Tailscale signs this release's suite with, when
|
|
15
|
+
* it is not the policy's `tailscale.keySha256`. */
|
|
16
|
+
tailscaleKeySha256?: string;
|
|
12
17
|
}
|
|
13
18
|
|
|
14
19
|
export const UBUNTU_RELEASES: readonly UbuntuRelease[] = policy.releases;
|
|
@@ -21,6 +26,25 @@ export function packagePins(
|
|
|
21
26
|
return { ...policy.packages, ...release?.packages };
|
|
22
27
|
}
|
|
23
28
|
|
|
29
|
+
/** The browser library pins of one release, combined the same way. */
|
|
30
|
+
export function browserPackagePins(
|
|
31
|
+
release: UbuntuRelease | undefined
|
|
32
|
+
): Record<string, string> {
|
|
33
|
+
return { ...policy.browser.packages, ...release?.browserPackages };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** The Tailscale version and the digest of the key that signs one release's
|
|
37
|
+
* suite (as tasks/release.yml selects it for Ansible). */
|
|
38
|
+
export function tailscalePins(release: UbuntuRelease | undefined): {
|
|
39
|
+
version: string;
|
|
40
|
+
keySha256: string;
|
|
41
|
+
} {
|
|
42
|
+
return {
|
|
43
|
+
version: policy.tailscale.version,
|
|
44
|
+
keySha256: release?.tailscaleKeySha256 ?? policy.tailscale.keySha256,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
24
48
|
/** `uname -m` for each Debian architecture name. */
|
|
25
49
|
const KERNEL_ARCHITECTURES: Record<string, string> = {
|
|
26
50
|
amd64: 'x86_64',
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import type { Check, Runner } from '../types';
|
|
2
|
+
import type { MachineClass } from '../machine/machine-class';
|
|
2
3
|
|
|
3
|
-
|
|
4
|
+
/** What a lid does: a virtual machine has none, so these do not apply. */
|
|
5
|
+
const LID = {
|
|
4
6
|
HandleLidSwitch: 's "ignore"',
|
|
5
7
|
HandleLidSwitchDocked: 's "ignore"',
|
|
6
8
|
HandleLidSwitchExternalPower: 's "ignore"',
|
|
9
|
+
};
|
|
10
|
+
const LOGIND = {
|
|
7
11
|
IdleAction: 's "ignore"',
|
|
8
12
|
KillUserProcesses: 'b false',
|
|
9
13
|
StopIdleSessionUSec: 't 18446744073709551615',
|
|
@@ -17,9 +21,33 @@ const POWER_FILES = [
|
|
|
17
21
|
*/
|
|
18
22
|
export async function verifyUbuntuBoot(
|
|
19
23
|
user: string,
|
|
20
|
-
run: Runner
|
|
24
|
+
run: Runner,
|
|
25
|
+
machine: MachineClass = 'physical'
|
|
26
|
+
): Promise<Check[]> {
|
|
27
|
+
const virtual = machine === 'virtual';
|
|
28
|
+
// On a virtual machine the lid policy is not applicable; the sleep masks
|
|
29
|
+
// stay, since a guest can still be suspended by its own idle policy.
|
|
30
|
+
const lid: Check[] = virtual
|
|
31
|
+
? [
|
|
32
|
+
{
|
|
33
|
+
id: 'ubuntu:lid-policy',
|
|
34
|
+
status: 'not-applicable',
|
|
35
|
+
summary: 'Not applicable: a virtual machine has no lid.',
|
|
36
|
+
},
|
|
37
|
+
]
|
|
38
|
+
: [];
|
|
39
|
+
return [
|
|
40
|
+
...lid,
|
|
41
|
+
...(await bootPolicy(user, run, virtual ? LOGIND : { ...LID, ...LOGIND })),
|
|
42
|
+
];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function bootPolicy(
|
|
46
|
+
user: string,
|
|
47
|
+
run: Runner,
|
|
48
|
+
properties: Record<string, string>
|
|
21
49
|
): Promise<Check[]> {
|
|
22
|
-
for (const [property, expected] of Object.entries(
|
|
50
|
+
for (const [property, expected] of Object.entries(properties)) {
|
|
23
51
|
const observed = await run([
|
|
24
52
|
'busctl',
|
|
25
53
|
'get-property',
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Runner } from '../types';
|
|
2
|
+
import type { WorkerFeatures } from '../worker-features';
|
|
3
|
+
import {
|
|
4
|
+
browserPackagePins,
|
|
5
|
+
kernelArchitectures,
|
|
6
|
+
packagePins,
|
|
7
|
+
tailscalePins,
|
|
8
|
+
UBUNTU_RELEASES,
|
|
9
|
+
ubuntuRelease,
|
|
10
|
+
} from './ubuntu-release';
|
|
11
|
+
import type { WorkerPlatform, WorkerRelease } from './worker-platform';
|
|
12
|
+
|
|
13
|
+
/** Ubuntu: every release ansible/ubuntu/policy.json accepts, converged by
|
|
14
|
+
* the playbooks under ansible/ubuntu. A release's differences are data of
|
|
15
|
+
* its policy entry (tasks/release.yml selects it from the facts). */
|
|
16
|
+
export const UBUNTU_WORKER: WorkerPlatform = {
|
|
17
|
+
family: 'ubuntu',
|
|
18
|
+
label: 'Ubuntu',
|
|
19
|
+
policy: 'ansible/ubuntu/policy.json',
|
|
20
|
+
playbooks: 'ansible/ubuntu',
|
|
21
|
+
releases: UBUNTU_RELEASES,
|
|
22
|
+
mismatch: 'UBUNTU_BASELINE_MISMATCH',
|
|
23
|
+
packageManager: 'apt',
|
|
24
|
+
packagePins: (release) => packagePins(ubuntuRelease(release?.release)),
|
|
25
|
+
browserPackagePins: (release) =>
|
|
26
|
+
browserPackagePins(ubuntuRelease(release?.release)),
|
|
27
|
+
tailscalePins: (release) => tailscalePins(ubuntuRelease(release?.release)),
|
|
28
|
+
serviceManager: 'systemd',
|
|
29
|
+
sessionModel: 'systemd-user-linger',
|
|
30
|
+
release: (version) => ubuntuRelease(version),
|
|
31
|
+
kernelArchitectures: (release: WorkerRelease) =>
|
|
32
|
+
kernelArchitectures(ubuntuRelease(release.release) ?? release),
|
|
33
|
+
// Loaded when used: the platform probe reads this module on the manager,
|
|
34
|
+
// and the host checks pull in what only a worker runs.
|
|
35
|
+
hostChecks: async (
|
|
36
|
+
user: string,
|
|
37
|
+
run: Runner,
|
|
38
|
+
features?: WorkerFeatures,
|
|
39
|
+
powerRecoveryNote?: string
|
|
40
|
+
) =>
|
|
41
|
+
(await import('../worker-host-checks')).workerHostChecks(
|
|
42
|
+
user,
|
|
43
|
+
run,
|
|
44
|
+
features,
|
|
45
|
+
powerRecoveryNote
|
|
46
|
+
),
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** Every platform a worker may run. A machine of any other family is
|
|
50
|
+
* refused by the first check that can tell (docs/support-matrix.md). */
|
|
51
|
+
export const WORKER_PLATFORMS: readonly WorkerPlatform[] = [UBUNTU_WORKER];
|
|
52
|
+
|
|
53
|
+
/** The platform of an `ID` from /etc/os-release, when workers may run it. */
|
|
54
|
+
export function workerPlatform(
|
|
55
|
+
family: string | null | undefined
|
|
56
|
+
): WorkerPlatform | undefined {
|
|
57
|
+
return WORKER_PLATFORMS.find((platform) => platform.family === family);
|
|
58
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { Check, Runner } from '../types';
|
|
2
|
+
import type { WorkerFeatures } from '../worker-features';
|
|
3
|
+
|
|
4
|
+
/** One release of a worker platform, as its policy names it. */
|
|
5
|
+
export interface WorkerRelease {
|
|
6
|
+
/** `VERSION_ID` of /etc/os-release. */
|
|
7
|
+
release: string;
|
|
8
|
+
codename: string;
|
|
9
|
+
/** Debian architecture names, as `dpkg --print-architecture` reports them. */
|
|
10
|
+
architectures: string[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** What differs between the operating systems a worker may run. Everything
|
|
14
|
+
* the manager and the worker's own checks need to know about one platform is
|
|
15
|
+
* here; what a playbook needs is data of the platform's policy, selected on
|
|
16
|
+
* the machine from its facts. Ubuntu is the one implementation
|
|
17
|
+
* (`src/platform/ubuntu-worker.ts`); docs/support-matrix.md lists the
|
|
18
|
+
* platforms that are not, and what each would need.
|
|
19
|
+
*/
|
|
20
|
+
export interface WorkerPlatform {
|
|
21
|
+
/** `ID` of /etc/os-release, as the platform probe prints it. */
|
|
22
|
+
readonly family: string;
|
|
23
|
+
/** The name the support matrix gives it. */
|
|
24
|
+
readonly label: string;
|
|
25
|
+
/** The policy that names its releases and pins, from the package root. */
|
|
26
|
+
readonly policy: string;
|
|
27
|
+
/** The playbooks that converge it, from the package root. */
|
|
28
|
+
readonly playbooks: string;
|
|
29
|
+
/** The releases a worker may run, newest first. */
|
|
30
|
+
readonly releases: readonly WorkerRelease[];
|
|
31
|
+
/** The reason a machine of this family is refused before anything is
|
|
32
|
+
* installed: another release or another CPU. */
|
|
33
|
+
readonly mismatch: string;
|
|
34
|
+
|
|
35
|
+
/** Packages: the system package manager the pins are written for, and the
|
|
36
|
+
* exact pins of one release (common pins with the release's own over them). */
|
|
37
|
+
readonly packageManager: 'apt';
|
|
38
|
+
packagePins(release: WorkerRelease | undefined): Record<string, string>;
|
|
39
|
+
/** The libraries headless Chromium loads, pinned the same way. */
|
|
40
|
+
browserPackagePins(
|
|
41
|
+
release: WorkerRelease | undefined
|
|
42
|
+
): Record<string, string>;
|
|
43
|
+
/** The Tailscale version and the digest of the key that signs this
|
|
44
|
+
* release's package suite. */
|
|
45
|
+
tailscalePins(release: WorkerRelease | undefined): {
|
|
46
|
+
version: string;
|
|
47
|
+
keySha256: string;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/** Service manager: what keeps the always-on system services and the
|
|
51
|
+
* developer's own units running and restarts them. */
|
|
52
|
+
readonly serviceManager: 'systemd';
|
|
53
|
+
/** Session model: how the developer's processes (T3, agents) outlive a
|
|
54
|
+
* login. `systemd-user-linger`: a lingering systemd user manager, started
|
|
55
|
+
* at boot without anyone logged in, never stopped by logind. */
|
|
56
|
+
readonly sessionModel: 'systemd-user-linger';
|
|
57
|
+
|
|
58
|
+
/** The release a probe reported, when the platform accepts it. */
|
|
59
|
+
release(version: string | null | undefined): WorkerRelease | undefined;
|
|
60
|
+
/** The `uname -m` names one release runs on. */
|
|
61
|
+
kernelArchitectures(release: WorkerRelease): string[];
|
|
62
|
+
/** Checks: the measured host evidence of a worker of this platform,
|
|
63
|
+
* read-only, run on the worker itself. What its declared features leave
|
|
64
|
+
* out is not applicable; so is a physical machine's power recovery on a
|
|
65
|
+
* virtual one, which `powerRecoveryNote` may say who provides. */
|
|
66
|
+
hostChecks(
|
|
67
|
+
user: string,
|
|
68
|
+
run: Runner,
|
|
69
|
+
features?: WorkerFeatures,
|
|
70
|
+
powerRecoveryNote?: string
|
|
71
|
+
): Promise<Check[]>;
|
|
72
|
+
}
|
package/src/protocol.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { record } from './config';
|
|
|
2
2
|
import { FINGERPRINT_FILES } from './input-hashes';
|
|
3
3
|
import type { Check, Snapshot } from './types';
|
|
4
4
|
|
|
5
|
-
const STATUSES = ['pass', 'warn', 'fail', 'skip'];
|
|
5
|
+
const STATUSES = ['pass', 'warn', 'fail', 'skip', 'not-applicable'];
|
|
6
6
|
|
|
7
7
|
export function validCheck(value: unknown): value is Check {
|
|
8
8
|
if (
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { createConnection, isIP } from 'node:net';
|
|
2
|
+
import { record } from './config';
|
|
3
|
+
import { fleet } from './fleet-config';
|
|
4
|
+
import { workerExposure, type WorkerExposure } from './exposure';
|
|
5
|
+
import { MONITORING_AGENT_PORT } from './worker-exposure';
|
|
6
|
+
import type { Check, NodeReport, Runner } from './types';
|
|
7
|
+
|
|
8
|
+
/** Opens one TCP connection and closes it at once: whether it was accepted. */
|
|
9
|
+
export type Connect = (
|
|
10
|
+
host: string,
|
|
11
|
+
port: number,
|
|
12
|
+
timeoutMs: number
|
|
13
|
+
) => Promise<boolean>;
|
|
14
|
+
|
|
15
|
+
export const tcpConnect: Connect = (host, port, timeoutMs) =>
|
|
16
|
+
new Promise((resolve) => {
|
|
17
|
+
const socket = createConnection({ host, port });
|
|
18
|
+
const done = (accepted: boolean) => {
|
|
19
|
+
socket.destroy();
|
|
20
|
+
resolve(accepted);
|
|
21
|
+
};
|
|
22
|
+
socket.setTimeout(timeoutMs, () => done(false));
|
|
23
|
+
socket.once('connect', () => done(true));
|
|
24
|
+
socket.once('error', () => done(false));
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
const UNRESTRICTED = 'WORKER_PUBLIC_PORT_UNRESTRICTED';
|
|
28
|
+
|
|
29
|
+
const PRIVATE_V4: [number, number][] = [
|
|
30
|
+
[0x00000000, 8], // this network
|
|
31
|
+
[0x0a000000, 8],
|
|
32
|
+
[0x64400000, 10], // carrier-grade NAT and the tailnet
|
|
33
|
+
[0x7f000000, 8],
|
|
34
|
+
[0xa9fe0000, 16], // link-local
|
|
35
|
+
[0xac100000, 12],
|
|
36
|
+
[0xc0a80000, 16],
|
|
37
|
+
[0xe0000000, 3], // multicast and reserved
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
/** An address the internet routes to: not private, loopback, link-local,
|
|
41
|
+
* shared (which includes the tailnet), unique-local or multicast. */
|
|
42
|
+
export function isPublicAddress(address: string): boolean {
|
|
43
|
+
if (isIP(address) === 4) {
|
|
44
|
+
const value =
|
|
45
|
+
address
|
|
46
|
+
.split('.')
|
|
47
|
+
.reduce((total, part) => total * 256 + Number(part), 0) >>> 0;
|
|
48
|
+
return !PRIVATE_V4.some(
|
|
49
|
+
([network, bits]) =>
|
|
50
|
+
value >>> (32 - bits) === (network >>> (32 - bits)) >>> 0
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
if (isIP(address) === 6) {
|
|
54
|
+
const lower = address.toLowerCase();
|
|
55
|
+
const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
|
|
56
|
+
if (mapped) return isPublicAddress(mapped[1]!);
|
|
57
|
+
return !(
|
|
58
|
+
lower === '::' ||
|
|
59
|
+
lower === '::1' ||
|
|
60
|
+
/^fe[89ab]/.test(lower) ||
|
|
61
|
+
/^f[cd]/.test(lower) ||
|
|
62
|
+
/^ff/.test(lower)
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The public addresses Tailscale reaches a worker at (`Addrs` of its peer in
|
|
69
|
+
* `tailscale status --json`), each once. */
|
|
70
|
+
export function publicEndpoints(
|
|
71
|
+
status: unknown,
|
|
72
|
+
tailscaleName: string
|
|
73
|
+
): string[] {
|
|
74
|
+
if (!record(status) || !record(status['Peer'])) return [];
|
|
75
|
+
const name = tailscaleName.toLowerCase();
|
|
76
|
+
const peer = Object.values(status['Peer']).find(
|
|
77
|
+
(entry) =>
|
|
78
|
+
record(entry) &&
|
|
79
|
+
typeof entry['DNSName'] === 'string' &&
|
|
80
|
+
entry['DNSName'].replace(/\.$/, '').toLowerCase() === name
|
|
81
|
+
);
|
|
82
|
+
if (!record(peer) || !Array.isArray(peer['Addrs'])) return [];
|
|
83
|
+
const found = new Set<string>();
|
|
84
|
+
for (const entry of peer['Addrs'].slice(0, 32)) {
|
|
85
|
+
if (typeof entry !== 'string') continue;
|
|
86
|
+
const host = entry.replace(/:\d+$/, '').replace(/^\[|\]$/g, '');
|
|
87
|
+
if (isPublicAddress(host)) found.add(host);
|
|
88
|
+
}
|
|
89
|
+
return [...found];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** worker:public-reachability, from the manager: none of the fleet's ports
|
|
93
|
+
* accepts a connection on the worker's public address. Optional: the manager
|
|
94
|
+
* may share the worker's network, so a failure here never blocks readiness;
|
|
95
|
+
* worker:exposure is measured on the worker itself. Names the service and
|
|
96
|
+
* port, never the address.
|
|
97
|
+
*/
|
|
98
|
+
export async function publicReachabilityCheck(
|
|
99
|
+
rules: WorkerExposure,
|
|
100
|
+
addresses: string[],
|
|
101
|
+
connect: Connect = tcpConnect
|
|
102
|
+
): Promise<Check> {
|
|
103
|
+
if (addresses.length === 0)
|
|
104
|
+
return {
|
|
105
|
+
id: 'worker:public-reachability',
|
|
106
|
+
status: 'not-applicable',
|
|
107
|
+
required: false,
|
|
108
|
+
summary:
|
|
109
|
+
'Tailscale knows no public address for this worker: there is nothing outside the tailnet to try.',
|
|
110
|
+
};
|
|
111
|
+
const ports = [
|
|
112
|
+
...(rules.policy.publicSsh ? [] : [{ name: 'SSH', port: rules.sshPort }]),
|
|
113
|
+
{ name: 'T3', port: rules.t3Port },
|
|
114
|
+
{ name: 'the monitoring agent', port: MONITORING_AGENT_PORT },
|
|
115
|
+
].filter(
|
|
116
|
+
(entry) =>
|
|
117
|
+
!rules.policy.publicPorts.some(
|
|
118
|
+
(open) => open.protocol === 'tcp' && open.port === entry.port
|
|
119
|
+
)
|
|
120
|
+
);
|
|
121
|
+
const accepted: string[] = [];
|
|
122
|
+
for (const entry of ports)
|
|
123
|
+
for (const address of addresses)
|
|
124
|
+
if (await connect(address, entry.port, 3_000)) {
|
|
125
|
+
accepted.push(`${entry.name} (tcp ${entry.port})`);
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
const tried = ports.map((entry) => `${entry.name} (tcp ${entry.port})`);
|
|
129
|
+
if (accepted.length)
|
|
130
|
+
return {
|
|
131
|
+
id: 'worker:public-reachability',
|
|
132
|
+
status: 'fail',
|
|
133
|
+
required: false,
|
|
134
|
+
summary: `${UNRESTRICTED} ${accepted.join(', ')} accepted a connection on the worker's public address.`,
|
|
135
|
+
remedy: {
|
|
136
|
+
action:
|
|
137
|
+
'Run devfleet fleet-sync, which loads the fleet firewall on the worker, then read worker:exposure for the unit that listens. When the manager shares the worker’s network, confirm from a machine outside it before changing anything.',
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
return {
|
|
141
|
+
id: 'worker:public-reachability',
|
|
142
|
+
status: 'pass',
|
|
143
|
+
required: false,
|
|
144
|
+
summary: `No connection was accepted on the worker's public address: ${tried.join(', ')}.`,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Adds worker:public-reachability to the report of every declared worker
|
|
149
|
+
* the tailnet knows. */
|
|
150
|
+
export async function appendPublicReachability(
|
|
151
|
+
reports: NodeReport[],
|
|
152
|
+
run: Runner,
|
|
153
|
+
connect: Connect = tcpConnect
|
|
154
|
+
): Promise<void> {
|
|
155
|
+
const status = await run(['tailscale', 'status', '--json'], {
|
|
156
|
+
timeoutMs: 15_000,
|
|
157
|
+
});
|
|
158
|
+
let parsed: unknown = null;
|
|
159
|
+
try {
|
|
160
|
+
parsed = status.ok ? JSON.parse(status.stdout) : null;
|
|
161
|
+
} catch {
|
|
162
|
+
/* Raw status is never reported. */
|
|
163
|
+
}
|
|
164
|
+
for (const machine of fleet.workers) {
|
|
165
|
+
const name = machine.network?.tailscaleName;
|
|
166
|
+
const report = reports.find(
|
|
167
|
+
(entry) => entry.node === name || entry.node === machine.id
|
|
168
|
+
);
|
|
169
|
+
if (!report || !name) continue;
|
|
170
|
+
report.checks.push(
|
|
171
|
+
parsed === null
|
|
172
|
+
? {
|
|
173
|
+
id: 'worker:public-reachability',
|
|
174
|
+
status: 'skip',
|
|
175
|
+
required: false,
|
|
176
|
+
summary:
|
|
177
|
+
'Tailscale status is unavailable on the manager, so the public address of this worker is unknown.',
|
|
178
|
+
}
|
|
179
|
+
: await publicReachabilityCheck(
|
|
180
|
+
workerExposure(fleet, machine.id),
|
|
181
|
+
publicEndpoints(parsed, name),
|
|
182
|
+
connect
|
|
183
|
+
)
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
}
|