@crvouga/dev-fleet 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -37
- package/agents/setup.md +42 -12
- package/ansible/macos/manager.yml +7 -7
- package/ansible/macos/policy.json +2 -10
- package/ansible/monitoring/README.md +86 -25
- package/ansible/monitoring/agent.yml +40 -2
- package/ansible/monitoring/hub-route-darwin.yml +51 -0
- package/ansible/monitoring/lock.json +5 -0
- package/ansible/pyproject.toml +12 -0
- package/ansible/ubuntu/baseline.yml +1 -1
- package/ansible/ubuntu/disk-gc.py +336 -0
- package/ansible/ubuntu/policy.json +12 -3
- package/ansible/ubuntu/tailscale.yml +8 -5
- package/ansible/ubuntu/tasks/resources.yml +61 -1
- package/ansible/uv.lock +190 -0
- package/bootstrap/bootstrap.sh +25 -18
- package/bootstrap/doctor.ts +46 -1
- package/bootstrap/fleet-manager-run.sh +42 -15
- package/bootstrap/fleet-remote-bootstrap.sh +85 -12
- package/bootstrap/global-config.ts +64 -6
- package/bootstrap/install-launcher.sh +3 -0
- package/bootstrap/npm.ts +10 -2
- package/bootstrap/release-version.ts +157 -0
- package/docs/consumer-setup.md +408 -0
- package/docs/declared.md +89 -0
- package/docs/machine-setup.md +399 -0
- package/docs/operations.md +276 -0
- package/docs/releasing.md +103 -0
- package/docs/state.md +254 -0
- package/docs/topology.md +67 -0
- package/docs/ubuntu-sync.md +351 -0
- package/docs/worker-contract.md +67 -0
- package/examples/fleet.json +5 -0
- package/llms.txt +35 -0
- package/mise.lock +29 -0
- package/mise.toml +12 -1
- package/package.json +11 -2
- package/schema/fleet.schema.json +71 -15
- package/src/agents/fleet-verify.ts +9 -0
- package/src/agents/native-verify.ts +167 -22
- package/src/agents/provider-errors.ts +9 -2
- package/src/agents/registry.ts +7 -1
- package/src/cli/agent-guide.ts +39 -5
- package/src/cli/app.ts +11 -4
- package/src/cli/cli.ts +7 -0
- package/src/cli/command-index.ts +7 -0
- package/src/cli/fleet-state.ts +499 -0
- package/src/cli/fleet-sync-checkout.ts +56 -17
- package/src/cli/fleet-sync-enrollment.ts +19 -23
- package/src/cli/fleet-sync-forward.ts +1 -1
- package/src/cli/fleet-sync-manager.ts +64 -8
- package/src/cli/fleet-sync-monitoring.ts +5 -5
- package/src/cli/fleet-sync-output.ts +4 -1
- package/src/cli/fleet-sync-remote.ts +11 -5
- package/src/cli/fleet-sync-state.ts +86 -0
- package/src/cli/fleet-sync.ts +123 -57
- package/src/cli/fleet.ts +4 -2
- package/src/cli/managed-health.ts +4 -4
- package/src/cli/node-sync.ts +7 -0
- package/src/cli/program.ts +20 -4
- package/src/cli/project.ts +51 -0
- package/src/cli/register-fleet.ts +51 -1
- package/src/cli/registry.ts +28 -1
- package/src/cli/roster.ts +46 -30
- package/src/cli/settings.ts +15 -12
- package/src/cli/source-operations.ts +27 -3
- package/src/cli/tailnet.ts +2 -2
- package/src/definition.ts +31 -1
- package/src/diagnostics.ts +1 -1
- package/src/fleet-config.ts +105 -19
- package/src/machine/source.ts +3 -3
- package/src/machine/state.ts +15 -27
- package/src/manager-sync.ts +8 -3
- package/src/monitoring/agent-evidence.ts +146 -0
- package/src/monitoring/apply.ts +15 -3
- package/src/monitoring/beszel.ts +11 -4
- package/src/monitoring/enroll.ts +2 -1
- package/src/monitoring/hub-access.ts +4 -3
- package/src/monitoring/hub-backup.ts +121 -0
- package/src/monitoring/hub-viewer.ts +8 -8
- package/src/monitoring/hub.ts +3 -3
- package/src/monitoring/manager-hub.ts +30 -14
- package/src/node-converge-linux.ts +2 -0
- package/src/package-info.ts +45 -0
- package/src/platform/ansible.ts +19 -15
- package/src/platform/resource-verify.ts +8 -4
- package/src/platform/ubuntu-audit.ts +5 -2
- package/src/remote-bootstrap.ts +17 -19
- package/src/repair-catalog.ts +1 -1
- package/src/runner.ts +1 -1
- package/src/state/bucket.ts +208 -0
- package/src/state/check.ts +74 -0
- package/src/state/connection.ts +92 -0
- package/src/state/enrollment.ts +45 -0
- package/src/state/legacy.ts +89 -0
- package/src/state/litestream.ts +143 -0
- package/src/state/operator.ts +20 -0
- package/src/state/paths.ts +40 -0
- package/src/state/roster.ts +62 -0
- package/src/state/store.ts +169 -0
- package/src/state/sync.ts +352 -0
- package/src/t3-endpoint.ts +64 -8
- package/src/t3-linked-check.ts +23 -1
- package/src/toolchain-checks.ts +25 -0
- package/src/types.ts +4 -0
- package/src/worker-access-checks.ts +56 -3
- package/src/worker-host-checks.ts +5 -0
- package/src/workstation-checks.ts +1 -1
- package/src/index.ts +0 -6
- package/src/test-fixtures.ts +0 -36
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { record } from '../config';
|
|
2
|
+
import type { Check } from '../types';
|
|
3
|
+
import {
|
|
4
|
+
LOCAL_HUB,
|
|
5
|
+
api,
|
|
6
|
+
superuserToken,
|
|
7
|
+
type HubLogin,
|
|
8
|
+
type Session,
|
|
9
|
+
} from './hub-viewer';
|
|
10
|
+
|
|
11
|
+
/** A scheduled backup older than this means the schedule is not running. */
|
|
12
|
+
export const BACKUP_MAX_AGE_MS = 48 * 60 * 60 * 1000;
|
|
13
|
+
|
|
14
|
+
function backupCheck(
|
|
15
|
+
status: Check['status'],
|
|
16
|
+
summary: string,
|
|
17
|
+
action?: string
|
|
18
|
+
): Check {
|
|
19
|
+
return {
|
|
20
|
+
id: 'beszel:backup',
|
|
21
|
+
status,
|
|
22
|
+
summary,
|
|
23
|
+
...(action ? { remedy: { action } } : {}),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const SETTINGS =
|
|
28
|
+
'Open the hub as its superuser (/_/ on the manager), Settings > Backups.';
|
|
29
|
+
|
|
30
|
+
/** Reads, never changes, the hub's own backup settings and its list of
|
|
31
|
+
* backups through the stock PocketBase API of the pinned release (superuser
|
|
32
|
+
* only: GET /api/settings and GET /api/backups). Backups are configured in
|
|
33
|
+
* the hub's UI; this says whether they are scheduled, kept off the manager
|
|
34
|
+
* and recent. Storage names, keys and endpoints are never reported.
|
|
35
|
+
*/
|
|
36
|
+
export async function verifyHubBackups(
|
|
37
|
+
login: HubLogin | null,
|
|
38
|
+
fetcher: typeof fetch = fetch,
|
|
39
|
+
base: string = LOCAL_HUB,
|
|
40
|
+
now: number = Date.now()
|
|
41
|
+
): Promise<Check> {
|
|
42
|
+
if (!login)
|
|
43
|
+
return backupCheck(
|
|
44
|
+
'skip',
|
|
45
|
+
'BESZEL_BACKUP_UNVERIFIED',
|
|
46
|
+
"The fleet admin login is unavailable, so the hub's backups were not read. Log the manager's Vault session in and rerun fleet-sync."
|
|
47
|
+
);
|
|
48
|
+
let settings: unknown;
|
|
49
|
+
let backups: unknown;
|
|
50
|
+
try {
|
|
51
|
+
const anonymous: Session = { fetcher, base, token: '' };
|
|
52
|
+
const session = {
|
|
53
|
+
...anonymous,
|
|
54
|
+
token: await superuserToken(anonymous, login),
|
|
55
|
+
};
|
|
56
|
+
settings = await api(session, 'GET', '/api/settings');
|
|
57
|
+
backups = await api(session, 'GET', '/api/backups');
|
|
58
|
+
} catch {
|
|
59
|
+
return backupCheck(
|
|
60
|
+
'warn',
|
|
61
|
+
'BESZEL_BACKUP_UNVERIFIED',
|
|
62
|
+
`The hub's backup settings could not be read with the fleet admin login. ${SETTINGS} Details are withheld.`
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
const policy = backupPolicy(settings);
|
|
66
|
+
if (!policy || !Array.isArray(backups))
|
|
67
|
+
return backupCheck(
|
|
68
|
+
'warn',
|
|
69
|
+
'BESZEL_BACKUP_UNVERIFIED',
|
|
70
|
+
`The hub answered in a form this release does not expect. ${SETTINGS}`
|
|
71
|
+
);
|
|
72
|
+
if (!policy.cron)
|
|
73
|
+
return backupCheck(
|
|
74
|
+
'warn',
|
|
75
|
+
'BESZEL_BACKUP_UNSCHEDULED',
|
|
76
|
+
`The hub takes no scheduled backups, so its history, settings and machine identities are lost with the manager. ${SETTINGS} Set a schedule and how many to keep.`
|
|
77
|
+
);
|
|
78
|
+
if (!policy.offHost)
|
|
79
|
+
return backupCheck(
|
|
80
|
+
'warn',
|
|
81
|
+
'BESZEL_BACKUP_OFFHOST_REQUIRED',
|
|
82
|
+
`Scheduled backups stay on the manager's disk, which is not disaster recovery. ${SETTINGS} Store them in existing encrypted S3-compatible storage you already approved; do not create new storage or reuse unrelated credentials.`
|
|
83
|
+
);
|
|
84
|
+
const newest = newestBackup(backups);
|
|
85
|
+
if (newest === null || now - newest > BACKUP_MAX_AGE_MS)
|
|
86
|
+
return backupCheck(
|
|
87
|
+
'warn',
|
|
88
|
+
'BESZEL_BACKUP_STALE',
|
|
89
|
+
`No backup newer than ${BACKUP_MAX_AGE_MS / 3_600_000} hours is listed although a schedule is set. ${SETTINGS} Check the schedule and the storage's credentials, then create a backup by hand.`
|
|
90
|
+
);
|
|
91
|
+
return backupCheck(
|
|
92
|
+
'pass',
|
|
93
|
+
`Scheduled backups go to off-host storage, keeping ${policy.keep}; the newest is from ${new Date(newest).toISOString()}.`
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function backupPolicy(
|
|
98
|
+
settings: unknown
|
|
99
|
+
): { cron: string; keep: number; offHost: boolean } | null {
|
|
100
|
+
if (!record(settings) || !record(settings['backups'])) return null;
|
|
101
|
+
const backups = settings['backups'];
|
|
102
|
+
const s3 = record(backups['s3']) ? backups['s3'] : {};
|
|
103
|
+
return {
|
|
104
|
+
cron: typeof backups['cron'] === 'string' ? backups['cron'].trim() : '',
|
|
105
|
+
keep:
|
|
106
|
+
typeof backups['cronMaxKeep'] === 'number' ? backups['cronMaxKeep'] : 0,
|
|
107
|
+
offHost: s3['enabled'] === true,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** PocketBase lists `modified` as "2006-01-02 15:04:05.000Z". */
|
|
112
|
+
function newestBackup(backups: unknown[]): number | null {
|
|
113
|
+
const times = backups
|
|
114
|
+
.map((entry) =>
|
|
115
|
+
record(entry) && typeof entry['modified'] === 'string'
|
|
116
|
+
? Date.parse(entry['modified'].replace(' ', 'T'))
|
|
117
|
+
: NaN
|
|
118
|
+
)
|
|
119
|
+
.filter(Number.isFinite);
|
|
120
|
+
return times.length ? Math.max(...times) : null;
|
|
121
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto';
|
|
2
2
|
import { record } from '../config';
|
|
3
3
|
import type { Check } from '../types';
|
|
4
|
-
import {
|
|
4
|
+
import { publicViewerIdentity } from './hub-access';
|
|
5
5
|
|
|
6
6
|
/** Loopback address of the hub container on the manager. */
|
|
7
7
|
export const LOCAL_HUB = 'http://127.0.0.1:8090';
|
|
@@ -25,13 +25,13 @@ function viewerCheck(
|
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
interface Session {
|
|
28
|
+
export interface Session {
|
|
29
29
|
fetcher: Fetcher;
|
|
30
30
|
base: string;
|
|
31
31
|
token: string;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
async function api(
|
|
34
|
+
export async function api(
|
|
35
35
|
session: Session,
|
|
36
36
|
method: string,
|
|
37
37
|
path: string,
|
|
@@ -52,7 +52,7 @@ async function api(
|
|
|
52
52
|
return response.json() as Promise<unknown>;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
async function superuserToken(
|
|
55
|
+
export async function superuserToken(
|
|
56
56
|
session: Session,
|
|
57
57
|
login: HubLogin
|
|
58
58
|
): Promise<string> {
|
|
@@ -70,7 +70,7 @@ async function superuserToken(
|
|
|
70
70
|
async function findViewer(
|
|
71
71
|
session: Session
|
|
72
72
|
): Promise<{ id: string; role: unknown } | null> {
|
|
73
|
-
const filter = encodeURIComponent(`email="${
|
|
73
|
+
const filter = encodeURIComponent(`email="${publicViewerIdentity()}"`);
|
|
74
74
|
const list = await api(
|
|
75
75
|
session,
|
|
76
76
|
'GET',
|
|
@@ -86,7 +86,7 @@ async function findViewer(
|
|
|
86
86
|
async function createViewer(session: Session): Promise<void> {
|
|
87
87
|
const password = randomBytes(24).toString('base64url');
|
|
88
88
|
await api(session, 'POST', '/api/collections/users/records', {
|
|
89
|
-
email:
|
|
89
|
+
email: publicViewerIdentity(),
|
|
90
90
|
username: 'public-viewer',
|
|
91
91
|
password,
|
|
92
92
|
passwordConfirm: password,
|
|
@@ -130,13 +130,13 @@ export async function ensurePublicViewer(
|
|
|
130
130
|
if (verified?.role !== 'readonly') throw new Error('BESZEL_VIEWER_ROLE');
|
|
131
131
|
return viewerCheck(
|
|
132
132
|
'pass',
|
|
133
|
-
`Visitors are signed in automatically as ${
|
|
133
|
+
`Visitors are signed in automatically as ${publicViewerIdentity()} (read-only).`
|
|
134
134
|
);
|
|
135
135
|
} catch {
|
|
136
136
|
return viewerCheck(
|
|
137
137
|
'warn',
|
|
138
138
|
'BESZEL_VIEWER_UNVERIFIED',
|
|
139
|
-
`Could not create or verify the read-only ${
|
|
139
|
+
`Could not create or verify the read-only ${publicViewerIdentity()} user with the fleet admin login. Create it in the hub (Settings > Users, role Read-only); never make it an admin. Details are withheld.`
|
|
140
140
|
);
|
|
141
141
|
}
|
|
142
142
|
}
|
package/src/monitoring/hub.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { record } from '../config';
|
|
2
2
|
import type { Check, Runner } from '../types';
|
|
3
|
-
import {
|
|
3
|
+
import { publicHubUrl } from './beszel';
|
|
4
4
|
import { secretVia } from '../fleet-config';
|
|
5
5
|
|
|
6
6
|
/** Shared Beszel hub helpers. The hub runs only on the manager; see
|
|
@@ -11,14 +11,14 @@ export function publicEndpointCheck(connected: boolean): Check {
|
|
|
11
11
|
? {
|
|
12
12
|
id: 'beszel:public-endpoint',
|
|
13
13
|
status: 'pass',
|
|
14
|
-
summary: `Cloudflare connector for ${
|
|
14
|
+
summary: `Cloudflare connector for ${publicHubUrl()} was started from the Vault credential.`,
|
|
15
15
|
}
|
|
16
16
|
: {
|
|
17
17
|
id: 'beszel:public-endpoint',
|
|
18
18
|
status: 'warn',
|
|
19
19
|
summary: 'BESZEL_TUNNEL_CREDENTIAL_UNAVAILABLE',
|
|
20
20
|
remedy: {
|
|
21
|
-
action: `Create the Cloudflare tunnel and store its token in the fleet's secrets, then log the manager's Vault session in with read access to secret/beszel/tunnel and rerun fleet-sync. The private dashboard is unaffected; ${
|
|
21
|
+
action: `Create the Cloudflare tunnel and store its token in the fleet's secrets, then log the manager's Vault session in with read access to secret/beszel/tunnel and rerun fleet-sync. The private dashboard is unaffected; ${publicHubUrl()} stays offline until then.`,
|
|
22
22
|
},
|
|
23
23
|
};
|
|
24
24
|
}
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { delimiter, join, resolve } from 'node:path';
|
|
4
|
+
import { runAnsible } from '../platform/ansible';
|
|
4
5
|
import { runAnsibleWithReceipt } from '../platform/ansible-receipt';
|
|
5
6
|
import { failure, inspectRoute, publicEndpointCheck, tunnelToken } from './hub';
|
|
6
|
-
import {
|
|
7
|
+
import { publicViewerIdentity, fleetAccount } from './hub-access';
|
|
7
8
|
import { ensurePublicViewer, type HubLogin } from './hub-viewer';
|
|
9
|
+
import { verifyHubBackups } from './hub-backup';
|
|
8
10
|
import type { FleetDefinition } from '../definition';
|
|
9
11
|
import { hostId, hubUrl } from '../topology';
|
|
10
|
-
import {
|
|
12
|
+
import { publicHubUrl, privateHub } from './beszel';
|
|
11
13
|
import type { Check, Runner } from '../types';
|
|
12
14
|
|
|
13
15
|
/** Bundled CLI of the Tailscale macOS app when no `tailscale` is on PATH. */
|
|
14
16
|
export const TAILSCALE_APP_CLI =
|
|
15
17
|
'/Applications/Tailscale.app/Contents/MacOS/Tailscale';
|
|
16
18
|
const LOCAL_HUB = 'http://127.0.0.1:8090';
|
|
19
|
+
/** The one place the route is declared; `hub-route-darwin.yml` receives it. */
|
|
20
|
+
export const HUB_ROUTE = { port: '8443', proxy: LOCAL_HUB };
|
|
17
21
|
|
|
18
22
|
/** Per-user hub paths on the manager. Data stays in the human's home (and its
|
|
19
23
|
* normal backups); it is never copied to a worker.
|
|
@@ -52,12 +56,15 @@ export interface ManagerHubOptions {
|
|
|
52
56
|
receipts?: string;
|
|
53
57
|
/** Ensures the read-only AUTO_LOGIN user; defaults to the loopback hub API. */
|
|
54
58
|
viewer?: (login: HubLogin | null) => Promise<Check>;
|
|
59
|
+
/** Reads the hub's backup settings; defaults to the loopback hub API. */
|
|
60
|
+
backups?: (login: HubLogin | null) => Promise<Check>;
|
|
55
61
|
}
|
|
56
62
|
|
|
57
63
|
/** Converges the singleton Beszel hub locally on the manager: pinned Compose
|
|
58
64
|
* under Docker Desktop, the connector token from the manager's own Vault
|
|
59
65
|
* session (private variables file, never argv or output), a private Tailscale
|
|
60
|
-
* Serve route, then independent health checks locally and
|
|
66
|
+
* Serve route (its own playbook), then independent health checks locally and
|
|
67
|
+
* over the tailnet.
|
|
61
68
|
*/
|
|
62
69
|
export async function convergeManagerHub(
|
|
63
70
|
root: string,
|
|
@@ -105,7 +112,7 @@ export async function convergeManagerHub(
|
|
|
105
112
|
'Inspect Docker Desktop, the Compose image, file sharing for ~/Library/Application Support and disk space. Existing data is never deleted.'
|
|
106
113
|
);
|
|
107
114
|
const route = await servePrivateRoute(
|
|
108
|
-
hub,
|
|
115
|
+
{ hub, monitoring },
|
|
109
116
|
run,
|
|
110
117
|
options.tailscale === undefined ? tailscaleCommand() : options.tailscale
|
|
111
118
|
);
|
|
@@ -120,6 +127,7 @@ export async function convergeManagerHub(
|
|
|
120
127
|
},
|
|
121
128
|
publicEndpointCheck(tunnel !== null),
|
|
122
129
|
await ensureViewer(options, account),
|
|
130
|
+
await (options.backups ?? verifyHubBackups)(account),
|
|
123
131
|
];
|
|
124
132
|
}
|
|
125
133
|
|
|
@@ -131,8 +139,8 @@ function hubVariables(
|
|
|
131
139
|
): Record<string, string> {
|
|
132
140
|
return {
|
|
133
141
|
beszel_hub_url: hub,
|
|
134
|
-
beszel_public_url:
|
|
135
|
-
beszel_auto_login:
|
|
142
|
+
beszel_public_url: publicHubUrl(),
|
|
143
|
+
beszel_auto_login: publicViewerIdentity(),
|
|
136
144
|
beszel_data_dir: paths.data,
|
|
137
145
|
beszel_tunnel_dir: paths.tunnel,
|
|
138
146
|
...(tunnel ? { beszel_tunnel_token: tunnel } : {}),
|
|
@@ -175,8 +183,11 @@ async function preconditions(
|
|
|
175
183
|
return null;
|
|
176
184
|
}
|
|
177
185
|
|
|
186
|
+
/** `hub-route-darwin.yml` publishes the route when its port has no owner; this
|
|
187
|
+
* refuses a conflicting owner beforehand and measures the result.
|
|
188
|
+
*/
|
|
178
189
|
async function servePrivateRoute(
|
|
179
|
-
hub: string,
|
|
190
|
+
{ hub, monitoring }: { hub: string; monitoring: string },
|
|
180
191
|
run: Runner,
|
|
181
192
|
cli: string | null
|
|
182
193
|
): Promise<Check[] | null> {
|
|
@@ -191,21 +202,26 @@ async function servePrivateRoute(
|
|
|
191
202
|
'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
|
|
192
203
|
'Inspect Tailscale Serve permission and existing private routes before retry.'
|
|
193
204
|
);
|
|
194
|
-
const route = inspectRoute(status.stdout, hub);
|
|
205
|
+
const route = inspectRoute(status.stdout, hub, HUB_ROUTE);
|
|
195
206
|
if (Array.isArray(route)) return route;
|
|
196
207
|
if (!route) {
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
208
|
+
const published = await runAnsible(run, 'hub-route-darwin.yml', {
|
|
209
|
+
cwd: monitoring,
|
|
210
|
+
input: JSON.stringify({
|
|
211
|
+
beszel_tailscale_cli: cli,
|
|
212
|
+
beszel_route_port: HUB_ROUTE.port,
|
|
213
|
+
beszel_route_target: HUB_ROUTE.proxy,
|
|
214
|
+
}),
|
|
215
|
+
timeoutMs: 120_000,
|
|
216
|
+
});
|
|
217
|
+
if (!published.ok)
|
|
202
218
|
return failure(
|
|
203
219
|
'BESZEL_PRIVATE_ROUTE_REQUIRED',
|
|
204
220
|
'Allow Tailscale Serve (HTTPS certificates) for this tailnet and approve it in the Tailscale app if asked; no tailnet policy or public ingress is changed.'
|
|
205
221
|
);
|
|
206
222
|
}
|
|
207
223
|
const verified = await run([cli, 'serve', 'status', '--json']);
|
|
208
|
-
if (!verified.ok || inspectRoute(verified.stdout, hub) !== true)
|
|
224
|
+
if (!verified.ok || inspectRoute(verified.stdout, hub, HUB_ROUTE) !== true)
|
|
209
225
|
return failure(
|
|
210
226
|
'BESZEL_PRIVATE_ROUTE_UNVERIFIED',
|
|
211
227
|
'The requested route was not independently observed after application. Inspect Serve permissions and configuration; installer success is insufficient.'
|
|
@@ -78,6 +78,8 @@ export async function convergeLinux(
|
|
|
78
78
|
...(await attempt('agent:verification', () =>
|
|
79
79
|
verifyNativeAgents(node, root, run, {
|
|
80
80
|
outOfQuota: quotaLimited(providers),
|
|
81
|
+
// Where T3 starts them: the environment of its user service.
|
|
82
|
+
t3Service: true,
|
|
81
83
|
})
|
|
82
84
|
))
|
|
83
85
|
);
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
|
|
5
|
+
/** Where this code's own files live: the published package, or a checkout. */
|
|
6
|
+
export const PACKAGE_ROOT = fileURLToPath(new URL('..', import.meta.url));
|
|
7
|
+
|
|
8
|
+
export interface PackageInfo {
|
|
9
|
+
name: string;
|
|
10
|
+
version: string;
|
|
11
|
+
/** The commit this package was published from; null in a checkout. */
|
|
12
|
+
revision: string | null;
|
|
13
|
+
/** Where a defect or a feature request is raised (`bugs` in the manifest). */
|
|
14
|
+
issues: string | null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function issueTracker(bugs: unknown): string | null {
|
|
18
|
+
const url =
|
|
19
|
+
typeof bugs === 'object' && bugs !== null
|
|
20
|
+
? (bugs as { url?: unknown }).url
|
|
21
|
+
: bugs;
|
|
22
|
+
return typeof url === 'string' && /^https:\/\/\S+$/.test(url) ? url : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** What the running package says about itself. The release workflow stamps
|
|
26
|
+
* the version and the commit (`gitHead`) into the package it publishes, so an
|
|
27
|
+
* installed dev-fleet knows which revision every machine has to run. A
|
|
28
|
+
* checkout carries neither: its revision is its own HEAD. The issue tracker
|
|
29
|
+
* comes from the manifest too, so a fork sends its users to its own.
|
|
30
|
+
*/
|
|
31
|
+
export function packageInfo(root = PACKAGE_ROOT): PackageInfo {
|
|
32
|
+
const manifest = JSON.parse(
|
|
33
|
+
readFileSync(join(root, 'package.json'), 'utf8')
|
|
34
|
+
) as { name?: unknown; version?: unknown; gitHead?: unknown; bugs?: unknown };
|
|
35
|
+
const { gitHead } = manifest;
|
|
36
|
+
return {
|
|
37
|
+
name: String(manifest.name),
|
|
38
|
+
version: String(manifest.version),
|
|
39
|
+
revision:
|
|
40
|
+
typeof gitHead === 'string' && /^[a-f0-9]{40}$/.test(gitHead)
|
|
41
|
+
? gitHead
|
|
42
|
+
: null,
|
|
43
|
+
issues: issueTracker(manifest.bugs),
|
|
44
|
+
};
|
|
45
|
+
}
|
package/src/platform/ansible.ts
CHANGED
|
@@ -1,27 +1,31 @@
|
|
|
1
1
|
import { mkdir, mkdtemp, writeFile, rm } from 'node:fs/promises';
|
|
2
|
-
import {
|
|
3
|
-
import { delimiter, join } from 'node:path';
|
|
2
|
+
import { join } from 'node:path';
|
|
4
3
|
import { homedir } from 'node:os';
|
|
5
|
-
import
|
|
4
|
+
import { PACKAGE_ROOT } from '../package-info';
|
|
6
5
|
import type { Runner, CommandOptions, CommandResult } from '../types';
|
|
7
6
|
|
|
8
|
-
/**
|
|
9
|
-
*
|
|
7
|
+
/** The Ansible each machine runs playbooks with, pinned either way. A worker
|
|
8
|
+
* has the exact `ansible-core` package the Ubuntu policy names. The manager
|
|
9
|
+
* runs the project in `ansible/` (`pyproject.toml`, `uv.lock`): one release on
|
|
10
|
+
* one Python, every dependency checked against its locked hash, in a
|
|
11
|
+
* throwaway environment. No global Python environment is modified, nothing is
|
|
12
|
+
* written into the checkout, and an `ansible-playbook` that happens to be on
|
|
13
|
+
* PATH is never used.
|
|
10
14
|
*/
|
|
11
15
|
export function ansibleCommand(
|
|
12
|
-
|
|
13
|
-
|
|
16
|
+
platform: NodeJS.Platform = process.platform,
|
|
17
|
+
root = PACKAGE_ROOT
|
|
14
18
|
): string[] {
|
|
15
|
-
|
|
16
|
-
.split(delimiter)
|
|
17
|
-
.filter(Boolean)
|
|
18
|
-
.some((directory) => exists(join(directory, 'ansible-playbook')));
|
|
19
|
-
return found
|
|
19
|
+
return platform === 'linux'
|
|
20
20
|
? ['ansible-playbook']
|
|
21
21
|
: [
|
|
22
|
-
'
|
|
23
|
-
'
|
|
24
|
-
|
|
22
|
+
'uv',
|
|
23
|
+
'run',
|
|
24
|
+
'--project',
|
|
25
|
+
join(root, 'ansible'),
|
|
26
|
+
'--locked',
|
|
27
|
+
'--isolated',
|
|
28
|
+
'--quiet',
|
|
25
29
|
'ansible-playbook',
|
|
26
30
|
];
|
|
27
31
|
}
|
|
@@ -3,7 +3,11 @@ import type { Check, Runner } from '../types';
|
|
|
3
3
|
|
|
4
4
|
const AGENT_DROP_IN =
|
|
5
5
|
'/etc/systemd/user/t3code-.scope.d/60-personal-fleet.conf';
|
|
6
|
-
const GUARDS = [
|
|
6
|
+
const GUARDS = [
|
|
7
|
+
'systemd-oomd.service',
|
|
8
|
+
'personal-fleet-disk-guard.service',
|
|
9
|
+
'personal-fleet-disk-gc.timer',
|
|
10
|
+
];
|
|
7
11
|
const MIB = 1024 * 1024;
|
|
8
12
|
const FLOOR_MIB = policy.resourceSafety.diskGuardFloorMiB;
|
|
9
13
|
|
|
@@ -67,7 +71,7 @@ const diskAboveFloor =
|
|
|
67
71
|
if (free >= FLOOR_MIB * MIB) return undefined;
|
|
68
72
|
return [
|
|
69
73
|
'DISK_FREE_BELOW_FLOOR',
|
|
70
|
-
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${FLOOR_MIB} MiB floor. The
|
|
74
|
+
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${FLOOR_MIB} MiB floor. The disk GC already removed idle build output and caches; remove stale worktrees or data, or run "sudo systemctl start personal-fleet-disk-gc.service" and read its journal.`,
|
|
71
75
|
];
|
|
72
76
|
};
|
|
73
77
|
|
|
@@ -90,7 +94,7 @@ const tmpWritable: Probe = async (run) =>
|
|
|
90
94
|
|
|
91
95
|
/** Measured memory and disk safety on an Ubuntu worker: the running managers,
|
|
92
96
|
* not the files Ansible wrote. Free space below the guard floor fails because
|
|
93
|
-
* the
|
|
97
|
+
* what is left after the disk GC is work a person must decide about.
|
|
94
98
|
*/
|
|
95
99
|
export async function verifyResourceSafety(
|
|
96
100
|
home: string,
|
|
@@ -122,7 +126,7 @@ export async function verifyResourceSafety(
|
|
|
122
126
|
id: 'ubuntu:resource-safety',
|
|
123
127
|
status: 'pass',
|
|
124
128
|
summary:
|
|
125
|
-
'systemd-oomd and the disk
|
|
129
|
+
'systemd-oomd, the disk guard and the disk GC timer are running, user workloads are capped below physical memory, T3 agents have per-agent limits, swap is active, free disk is above the guard floor and /tmp is writable.',
|
|
126
130
|
},
|
|
127
131
|
];
|
|
128
132
|
}
|
|
@@ -31,12 +31,15 @@ export async function auditUbuntu(run: Runner): Promise<UbuntuAudit> {
|
|
|
31
31
|
}
|
|
32
32
|
const version =
|
|
33
33
|
/^VERSION_ID="?([0-9]+\.[0-9]+)"?$/m.exec(release.stdout)?.[1] ?? null;
|
|
34
|
-
const
|
|
34
|
+
const reported = arch.stdout.trim();
|
|
35
|
+
const architecture = policy.architectures.includes(reported)
|
|
36
|
+
? reported
|
|
37
|
+
: null;
|
|
35
38
|
return {
|
|
36
39
|
supported:
|
|
37
40
|
release.ok &&
|
|
38
41
|
/^ID=ubuntu$/m.test(release.stdout) &&
|
|
39
|
-
version ===
|
|
42
|
+
version === policy.release &&
|
|
40
43
|
architecture !== null,
|
|
41
44
|
release: version,
|
|
42
45
|
architecture,
|
package/src/remote-bootstrap.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { isRevision } from './machine/state';
|
|
|
6
6
|
import { sshArgs, shellQuote, sshFailure } from './ssh';
|
|
7
7
|
import type { FleetNode, NodeReport, Runner, Check } from './types';
|
|
8
8
|
import { record } from './config';
|
|
9
|
-
import type
|
|
9
|
+
import { installedDefinition, type FleetDefinition } from './definition';
|
|
10
10
|
import { cleanCheck, validCheck } from './protocol';
|
|
11
11
|
|
|
12
12
|
export interface BootstrapPayload {
|
|
@@ -48,16 +48,10 @@ export async function prepareBootstrap(
|
|
|
48
48
|
);
|
|
49
49
|
// The fleet configuration is data, not source: it travels beside the code.
|
|
50
50
|
return payload && options.definition
|
|
51
|
-
? { ...payload, config:
|
|
51
|
+
? { ...payload, config: installedDefinition(options.definition) }
|
|
52
52
|
: payload;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
/** Installs the approved fleet configuration where every launcher and job on
|
|
56
|
-
* the node reads it. It holds no secret.
|
|
57
|
-
*/
|
|
58
|
-
const INSTALL_CONFIG =
|
|
59
|
-
'umask 077; d="$HOME/.config/dev-fleet"; mkdir -p "$d" && cat >"$d/fleet.json.new" && mv "$d/fleet.json.new" "$d/fleet.json"';
|
|
60
|
-
|
|
61
55
|
/** Workers run one revision and never need history: the bundle holds only the
|
|
62
56
|
* approved commit and its files (about a quarter of the full history), and the
|
|
63
57
|
* receiving checkout is marked shallow at that commit. `git bundle create`
|
|
@@ -104,17 +98,9 @@ export async function bootstrapRemote(
|
|
|
104
98
|
): Promise<NodeReport> {
|
|
105
99
|
const cached = await cachedRevision(node, payload.revision, run);
|
|
106
100
|
if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
|
|
107
|
-
if (payload.config) {
|
|
108
|
-
const installed = await run([...sshArgs(node), INSTALL_CONFIG], {
|
|
109
|
-
input: payload.config,
|
|
110
|
-
timeoutMs: 20_000,
|
|
111
|
-
});
|
|
112
|
-
if (!installed.ok)
|
|
113
|
-
return { node: node.id, checks: [sshFailure(installed)] };
|
|
114
|
-
}
|
|
115
101
|
const transfer = cached
|
|
116
|
-
? { mode: 'cached',
|
|
117
|
-
: { mode: 'transfer',
|
|
102
|
+
? { mode: 'cached', bundle: '' }
|
|
103
|
+
: { mode: 'transfer', bundle: payload.bundle };
|
|
118
104
|
const command = [
|
|
119
105
|
'sh',
|
|
120
106
|
'-c',
|
|
@@ -128,7 +114,7 @@ export async function bootstrapRemote(
|
|
|
128
114
|
.map(shellQuote)
|
|
129
115
|
.join(' ');
|
|
130
116
|
const result = await run([...sshArgs(node), command], {
|
|
131
|
-
input: transfer.
|
|
117
|
+
input: bootstrapInput(payload.config, transfer.bundle),
|
|
132
118
|
timeoutMs: 5_400_000,
|
|
133
119
|
});
|
|
134
120
|
const evidence = parseEvidence(result);
|
|
@@ -150,6 +136,18 @@ export async function bootstrapRemote(
|
|
|
150
136
|
};
|
|
151
137
|
}
|
|
152
138
|
|
|
139
|
+
/** What the bootstrap reads on stdin: one line with the fleet configuration
|
|
140
|
+
* (installed under the remote lock, so overlapping runs never both write it),
|
|
141
|
+
* then the bundle when the worker lacks the revision.
|
|
142
|
+
*/
|
|
143
|
+
export function bootstrapInput(
|
|
144
|
+
config: string | undefined,
|
|
145
|
+
bundle: string
|
|
146
|
+
): string {
|
|
147
|
+
const line = config ? Buffer.from(config).toString('base64') : '';
|
|
148
|
+
return `${line}\n${bundle}`;
|
|
149
|
+
}
|
|
150
|
+
|
|
153
151
|
async function cachedRevision(
|
|
154
152
|
node: FleetNode,
|
|
155
153
|
revision: string,
|
package/src/repair-catalog.ts
CHANGED
|
@@ -47,7 +47,7 @@ export const REPAIR_RECIPES: RepairRecipe[] = [
|
|
|
47
47
|
owner: 'dev-fleet',
|
|
48
48
|
automatic: true,
|
|
49
49
|
action:
|
|
50
|
-
'Links ~/.local/bin/devfleet to this checkout. Nothing is copied and no configuration is changed.',
|
|
50
|
+
'Links ~/.local/bin/devfleet to this checkout of dev-fleet (refused for an installed package, which is not one). Nothing is copied and no configuration is changed.',
|
|
51
51
|
commands: [['sh', 'bootstrap/install-launcher.sh']],
|
|
52
52
|
},
|
|
53
53
|
{
|
package/src/runner.ts
CHANGED