@crvouga/dev-fleet 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -37
- package/agents/setup.md +42 -12
- package/ansible/macos/manager.yml +7 -7
- package/ansible/macos/policy.json +2 -10
- package/ansible/monitoring/README.md +86 -25
- package/ansible/monitoring/agent.yml +40 -2
- package/ansible/monitoring/hub-route-darwin.yml +51 -0
- package/ansible/monitoring/lock.json +5 -0
- package/ansible/pyproject.toml +12 -0
- package/ansible/ubuntu/baseline.yml +1 -1
- package/ansible/ubuntu/disk-gc.py +336 -0
- package/ansible/ubuntu/policy.json +12 -3
- package/ansible/ubuntu/tailscale.yml +8 -5
- package/ansible/ubuntu/tasks/resources.yml +61 -1
- package/ansible/uv.lock +190 -0
- package/bootstrap/bootstrap.sh +25 -18
- package/bootstrap/doctor.ts +46 -1
- package/bootstrap/fleet-manager-run.sh +42 -15
- package/bootstrap/fleet-remote-bootstrap.sh +85 -12
- package/bootstrap/global-config.ts +64 -6
- package/bootstrap/install-launcher.sh +3 -0
- package/bootstrap/npm.ts +10 -2
- package/bootstrap/release-version.ts +157 -0
- package/docs/consumer-setup.md +408 -0
- package/docs/declared.md +89 -0
- package/docs/machine-setup.md +399 -0
- package/docs/operations.md +276 -0
- package/docs/releasing.md +103 -0
- package/docs/state.md +254 -0
- package/docs/topology.md +67 -0
- package/docs/ubuntu-sync.md +351 -0
- package/docs/worker-contract.md +67 -0
- package/examples/fleet.json +5 -0
- package/llms.txt +35 -0
- package/mise.lock +29 -0
- package/mise.toml +12 -1
- package/package.json +11 -2
- package/schema/fleet.schema.json +71 -15
- package/src/agents/fleet-verify.ts +9 -0
- package/src/agents/native-verify.ts +167 -22
- package/src/agents/provider-errors.ts +9 -2
- package/src/agents/registry.ts +7 -1
- package/src/cli/agent-guide.ts +39 -5
- package/src/cli/app.ts +11 -4
- package/src/cli/cli.ts +7 -0
- package/src/cli/command-index.ts +7 -0
- package/src/cli/fleet-state.ts +499 -0
- package/src/cli/fleet-sync-checkout.ts +56 -17
- package/src/cli/fleet-sync-enrollment.ts +19 -23
- package/src/cli/fleet-sync-forward.ts +1 -1
- package/src/cli/fleet-sync-manager.ts +64 -8
- package/src/cli/fleet-sync-monitoring.ts +5 -5
- package/src/cli/fleet-sync-output.ts +4 -1
- package/src/cli/fleet-sync-remote.ts +11 -5
- package/src/cli/fleet-sync-state.ts +86 -0
- package/src/cli/fleet-sync.ts +123 -57
- package/src/cli/fleet.ts +4 -2
- package/src/cli/managed-health.ts +4 -4
- package/src/cli/node-sync.ts +7 -0
- package/src/cli/program.ts +20 -4
- package/src/cli/project.ts +51 -0
- package/src/cli/register-fleet.ts +51 -1
- package/src/cli/registry.ts +28 -1
- package/src/cli/roster.ts +46 -30
- package/src/cli/settings.ts +15 -12
- package/src/cli/source-operations.ts +27 -3
- package/src/cli/tailnet.ts +2 -2
- package/src/definition.ts +31 -1
- package/src/diagnostics.ts +1 -1
- package/src/fleet-config.ts +105 -19
- package/src/machine/source.ts +3 -3
- package/src/machine/state.ts +15 -27
- package/src/manager-sync.ts +8 -3
- package/src/monitoring/agent-evidence.ts +146 -0
- package/src/monitoring/apply.ts +15 -3
- package/src/monitoring/beszel.ts +11 -4
- package/src/monitoring/enroll.ts +2 -1
- package/src/monitoring/hub-access.ts +4 -3
- package/src/monitoring/hub-backup.ts +121 -0
- package/src/monitoring/hub-viewer.ts +8 -8
- package/src/monitoring/hub.ts +3 -3
- package/src/monitoring/manager-hub.ts +30 -14
- package/src/node-converge-linux.ts +2 -0
- package/src/package-info.ts +45 -0
- package/src/platform/ansible.ts +19 -15
- package/src/platform/resource-verify.ts +8 -4
- package/src/platform/ubuntu-audit.ts +5 -2
- package/src/remote-bootstrap.ts +17 -19
- package/src/repair-catalog.ts +1 -1
- package/src/runner.ts +1 -1
- package/src/state/bucket.ts +208 -0
- package/src/state/check.ts +74 -0
- package/src/state/connection.ts +92 -0
- package/src/state/enrollment.ts +45 -0
- package/src/state/legacy.ts +89 -0
- package/src/state/litestream.ts +143 -0
- package/src/state/operator.ts +20 -0
- package/src/state/paths.ts +40 -0
- package/src/state/roster.ts +62 -0
- package/src/state/store.ts +169 -0
- package/src/state/sync.ts +352 -0
- package/src/t3-endpoint.ts +64 -8
- package/src/t3-linked-check.ts +23 -1
- package/src/toolchain-checks.ts +25 -0
- package/src/types.ts +4 -0
- package/src/worker-access-checks.ts +56 -3
- package/src/worker-host-checks.ts +5 -0
- package/src/workstation-checks.ts +1 -1
- package/src/index.ts +0 -6
- package/src/test-fixtures.ts +0 -36
package/src/t3-endpoint.ts
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import { discoverTailnet } from './tailscale';
|
|
2
2
|
import type { Check, FleetNode, Runner } from './types';
|
|
3
3
|
|
|
4
|
+
/** An origin no browser of the fleet uses: the private route must not let
|
|
5
|
+
* a page served from it read the worker's T3.
|
|
6
|
+
*/
|
|
7
|
+
export const FOREIGN_ORIGIN = 'https://origin-check.invalid';
|
|
8
|
+
|
|
4
9
|
/** A fresh manager HTTPS request is transport evidence only. Never read
|
|
5
10
|
* pairing material or treat HTTP health as an authenticated tool session.
|
|
11
|
+
* A second request, from a foreign origin, must not be granted cross-origin
|
|
12
|
+
* access.
|
|
6
13
|
*/
|
|
7
14
|
export async function verifyT3Endpoint(
|
|
8
15
|
node: FleetNode,
|
|
@@ -21,17 +28,66 @@ export async function verifyT3Endpoint(
|
|
|
21
28
|
});
|
|
22
29
|
await response.body?.cancel();
|
|
23
30
|
if (response.status !== 200) return pending('T3_MANAGER_ENDPOINT_REJECTED');
|
|
24
|
-
return [
|
|
25
|
-
{
|
|
26
|
-
id: 't3:manager-endpoint',
|
|
27
|
-
status: 'pass',
|
|
28
|
-
summary:
|
|
29
|
-
'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
|
|
30
|
-
},
|
|
31
|
-
];
|
|
32
31
|
} catch {
|
|
33
32
|
return pending('T3_MANAGER_ENDPOINT_UNREACHABLE');
|
|
34
33
|
}
|
|
34
|
+
return [
|
|
35
|
+
{
|
|
36
|
+
id: 't3:manager-endpoint',
|
|
37
|
+
status: 'pass',
|
|
38
|
+
summary:
|
|
39
|
+
'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
|
|
40
|
+
},
|
|
41
|
+
await originCheck('https://' + peer.dns, request),
|
|
42
|
+
];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Grants cross-origin reads: it echoes the foreign origin, or allows any
|
|
46
|
+
* origin together with credentials.
|
|
47
|
+
*/
|
|
48
|
+
function permissive(headers: Headers): boolean {
|
|
49
|
+
const allowed = headers.get('access-control-allow-origin')?.trim();
|
|
50
|
+
return (
|
|
51
|
+
allowed === FOREIGN_ORIGIN ||
|
|
52
|
+
(allowed === '*' &&
|
|
53
|
+
headers.get('access-control-allow-credentials')?.trim() === 'true')
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function originCheck(
|
|
58
|
+
url: string,
|
|
59
|
+
request: (url: string, options: RequestInit) => Promise<Response>
|
|
60
|
+
): Promise<Check> {
|
|
61
|
+
const check = (status: Check['status'], summary: string): Check => ({
|
|
62
|
+
id: 't3:manager-origin',
|
|
63
|
+
status,
|
|
64
|
+
summary,
|
|
65
|
+
...(status === 'pass'
|
|
66
|
+
? {}
|
|
67
|
+
: {
|
|
68
|
+
remedy: {
|
|
69
|
+
action:
|
|
70
|
+
'The private T3 route must not grant cross-origin access to pages from other sites. Restore the stock T3 server and its private Serve route from source; never add a proxy that rewrites CORS headers.',
|
|
71
|
+
},
|
|
72
|
+
}),
|
|
73
|
+
});
|
|
74
|
+
try {
|
|
75
|
+
const response = await request(url, {
|
|
76
|
+
method: 'GET',
|
|
77
|
+
redirect: 'manual',
|
|
78
|
+
headers: { Origin: FOREIGN_ORIGIN },
|
|
79
|
+
signal: AbortSignal.timeout(10_000),
|
|
80
|
+
});
|
|
81
|
+
await response.body?.cancel();
|
|
82
|
+
return permissive(response.headers)
|
|
83
|
+
? check('fail', 'T3_MANAGER_ORIGIN_UNRESTRICTED')
|
|
84
|
+
: check(
|
|
85
|
+
'pass',
|
|
86
|
+
'A request from a foreign origin was not granted cross-origin access to the private T3 route.'
|
|
87
|
+
);
|
|
88
|
+
} catch {
|
|
89
|
+
return check('warn', 'T3_MANAGER_ORIGIN_UNVERIFIED');
|
|
90
|
+
}
|
|
35
91
|
}
|
|
36
92
|
function pending(code: string): Check[] {
|
|
37
93
|
return [
|
package/src/t3-linked-check.ts
CHANGED
|
@@ -22,7 +22,29 @@ export async function verifyT3Linked(
|
|
|
22
22
|
)();
|
|
23
23
|
check = await linkedOnce(node, run, now());
|
|
24
24
|
}
|
|
25
|
-
return check;
|
|
25
|
+
return check.status === 'fail' ? explained(node, run, check) : check;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A worker enrolled with one T3 environment that now runs another lost
|
|
29
|
+
* its identity (a reset data directory, a reinstall): the old link cannot
|
|
30
|
+
* reach it, and a new pairing is the remedy, not a reconnect.
|
|
31
|
+
*/
|
|
32
|
+
async function explained(
|
|
33
|
+
node: FleetNode,
|
|
34
|
+
run: Runner,
|
|
35
|
+
check: Check
|
|
36
|
+
): Promise<Check> {
|
|
37
|
+
if (!node.t3EnvironmentId) return check;
|
|
38
|
+
const observed = await observedIdentity(node, run).catch(() => undefined);
|
|
39
|
+
if (!observed || observed === node.t3EnvironmentId) return check;
|
|
40
|
+
return {
|
|
41
|
+
...check,
|
|
42
|
+
summary: 'T3_ENVIRONMENT_IDENTITY_CHANGED',
|
|
43
|
+
remedy: {
|
|
44
|
+
action:
|
|
45
|
+
'This worker now runs a different T3 environment than the one enrolled in the roster, so the manager’s link cannot reach it. Find out why its T3 data was replaced before anything else; then pair the new environment through T3 Connections and enroll it again. Fleet never copies or rewrites a T3 identity.',
|
|
46
|
+
},
|
|
47
|
+
};
|
|
26
48
|
}
|
|
27
49
|
|
|
28
50
|
async function linkedOnce(
|
package/src/toolchain-checks.ts
CHANGED
|
@@ -41,6 +41,29 @@ export async function toolchainChecks(
|
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/** Checks of the doctor that are not a tool version, with their own words. */
|
|
45
|
+
const OWN: Record<string, { pass: string; fail: string; action: string }> = {
|
|
46
|
+
'global-mise-config': {
|
|
47
|
+
pass: 'The global mise configuration is the one fleet-sync publishes.',
|
|
48
|
+
fail: 'The global mise configuration is missing, was replaced by another project or tool, or is not the one this revision publishes.',
|
|
49
|
+
action:
|
|
50
|
+
'Run devfleet fleet-sync from the project that holds fleet.json. On a fleet machine the global mise configuration belongs to dev-fleet: a project pins its own tools in a project mise.toml and never publishes a global one.',
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/** A doctor check with its own words, or null for a tool version. */
|
|
55
|
+
function ownCheck(id: string, status: Check['status']): Check | null {
|
|
56
|
+
if (!Object.hasOwn(OWN, id)) return null;
|
|
57
|
+
const own = OWN[id]!;
|
|
58
|
+
const ok = status === 'pass';
|
|
59
|
+
return {
|
|
60
|
+
id: `tool:${id}`,
|
|
61
|
+
status,
|
|
62
|
+
summary: ok ? own.pass : own.fail,
|
|
63
|
+
...(!ok ? { remedy: { action: own.action } } : {}),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
44
67
|
function toolCheck(value: unknown): Check | null {
|
|
45
68
|
if (
|
|
46
69
|
!record(value) ||
|
|
@@ -49,6 +72,8 @@ function toolCheck(value: unknown): Check | null {
|
|
|
49
72
|
!['pass', 'fail', 'warn', 'skip'].includes(String(value['status']))
|
|
50
73
|
)
|
|
51
74
|
return null;
|
|
75
|
+
const own = ownCheck(value['id'], value['status'] as Check['status']);
|
|
76
|
+
if (own) return own;
|
|
52
77
|
const ok = value['status'] === 'pass';
|
|
53
78
|
const version =
|
|
54
79
|
typeof value['detail'] === 'string' &&
|
package/src/types.ts
CHANGED
|
@@ -80,6 +80,10 @@ export interface CommandOptions {
|
|
|
80
80
|
input?: string;
|
|
81
81
|
timeoutMs?: number;
|
|
82
82
|
env?: NodeJS.ProcessEnv;
|
|
83
|
+
/** Run with `env` alone instead of on top of this process's environment:
|
|
84
|
+
* for a launch that must see only what a service would.
|
|
85
|
+
*/
|
|
86
|
+
isolatedEnv?: boolean;
|
|
83
87
|
}
|
|
84
88
|
|
|
85
89
|
export interface CommandResult {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Check, Runner } from './types';
|
|
2
|
-
import { fleet
|
|
2
|
+
import { fleet } from './fleet-config';
|
|
3
|
+
import { secretsProvider } from './secrets';
|
|
3
4
|
import { registry } from './agents/registry';
|
|
4
5
|
import { resolveVaultConfig, usesVault } from './vault-config';
|
|
5
6
|
import { record } from './config';
|
|
@@ -26,7 +27,29 @@ export async function workerAccessChecks(run: Runner): Promise<Check[]> {
|
|
|
26
27
|
...(usesVault() ? [await vaultRenewalCheck(run)] : []),
|
|
27
28
|
];
|
|
28
29
|
}
|
|
30
|
+
/** Vault refuses a read the token's policy does not allow with a 403. */
|
|
31
|
+
const PERMISSION_DENIED = /permission denied|Code: 403/i;
|
|
32
|
+
|
|
29
33
|
async function proxyAuthentication(run: Runner): Promise<Check> {
|
|
34
|
+
// Read through the declared provider, remembering only whether Vault said
|
|
35
|
+
// the policy forbids it: never what it printed.
|
|
36
|
+
let denied = false;
|
|
37
|
+
const secret = secretsProvider(fleet.secrets, {
|
|
38
|
+
run: async (argv) => {
|
|
39
|
+
const result = await run(argv, { timeoutMs: 20_000 });
|
|
40
|
+
if (!result.ok && PERMISSION_DENIED.test(result.stderr)) denied = true;
|
|
41
|
+
return result.ok ? result.stdout : null;
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
const forbidden: Check = {
|
|
45
|
+
id: 'worker:proxy-authentication',
|
|
46
|
+
status: 'fail',
|
|
47
|
+
summary: 'VAULT_POLICY_PERMISSION_REQUIRED',
|
|
48
|
+
remedy: {
|
|
49
|
+
action:
|
|
50
|
+
'This worker’s Vault identity was refused the proxy client key by its policy. Grant that identity read access to the declared location in the Vault policy you manage. fleet-sync seeds no secret, writes no policy and adds no secret store.',
|
|
51
|
+
},
|
|
52
|
+
};
|
|
30
53
|
const unavailable: Check = {
|
|
31
54
|
id: 'worker:proxy-authentication',
|
|
32
55
|
status: 'fail',
|
|
@@ -37,7 +60,8 @@ async function proxyAuthentication(run: Runner): Promise<Check> {
|
|
|
37
60
|
},
|
|
38
61
|
};
|
|
39
62
|
try {
|
|
40
|
-
const key = (await
|
|
63
|
+
const key = (await secret('proxy-client-key')) ?? '';
|
|
64
|
+
if (denied) return forbidden;
|
|
41
65
|
if (!/^[\x21-\x7e]{16,4096}$/.test(key)) return unavailable;
|
|
42
66
|
const result = await run(
|
|
43
67
|
[
|
|
@@ -103,13 +127,42 @@ async function vaultRenewalCheck(run: Runner): Promise<Check> {
|
|
|
103
127
|
token['ttl'] <= 1200
|
|
104
128
|
)
|
|
105
129
|
return unavailable;
|
|
130
|
+
if (!(await lastRenewalSucceeded(run)))
|
|
131
|
+
return {
|
|
132
|
+
...unavailable,
|
|
133
|
+
summary: 'VAULT_RENEWAL_FAILING',
|
|
134
|
+
remedy: {
|
|
135
|
+
action:
|
|
136
|
+
'The identity is renewable, but the last run of the renewal job failed. Inspect it on this worker with: journalctl --user -u personal-worker-auth.service. Its token expires unless a renewal succeeds; no credentials are copied between machines.',
|
|
137
|
+
},
|
|
138
|
+
};
|
|
106
139
|
return {
|
|
107
140
|
...unavailable,
|
|
108
141
|
status: 'pass',
|
|
109
142
|
summary:
|
|
110
|
-
'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer.',
|
|
143
|
+
'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer, and its last renewal succeeded.',
|
|
111
144
|
};
|
|
112
145
|
} catch {
|
|
113
146
|
return unavailable;
|
|
114
147
|
}
|
|
115
148
|
}
|
|
149
|
+
|
|
150
|
+
/** An active timer is not evidence that renewal works: the result of the
|
|
151
|
+
* job's last run is. A job that has not run yet has nothing to say.
|
|
152
|
+
*/
|
|
153
|
+
async function lastRenewalSucceeded(run: Runner): Promise<boolean> {
|
|
154
|
+
const shown = await run([
|
|
155
|
+
'systemctl',
|
|
156
|
+
'--user',
|
|
157
|
+
'show',
|
|
158
|
+
'personal-worker-auth.service',
|
|
159
|
+
'--property=Result,ExecMainStatus',
|
|
160
|
+
]);
|
|
161
|
+
if (!shown.ok) return true;
|
|
162
|
+
const result = /^Result=(.*)$/m.exec(shown.stdout)?.[1];
|
|
163
|
+
const status = /^ExecMainStatus=(.*)$/m.exec(shown.stdout)?.[1];
|
|
164
|
+
return (
|
|
165
|
+
(result === undefined || result === 'success') &&
|
|
166
|
+
(status === undefined || status === '0')
|
|
167
|
+
);
|
|
168
|
+
}
|
|
@@ -62,6 +62,11 @@ export async function workerHostChecks(
|
|
|
62
62
|
'Exact declarative Ubuntu package version.'
|
|
63
63
|
)
|
|
64
64
|
),
|
|
65
|
+
check(
|
|
66
|
+
'package:tailscale',
|
|
67
|
+
audit.packages['tailscale'] === policy.tailscale.version,
|
|
68
|
+
'Exact declarative Tailscale version.'
|
|
69
|
+
),
|
|
65
70
|
];
|
|
66
71
|
for (const name of policy.alwaysOnServices) {
|
|
67
72
|
const unit = await run([
|
|
@@ -22,7 +22,7 @@ export async function workstationChecks(
|
|
|
22
22
|
summary: 'The devfleet launcher is missing.',
|
|
23
23
|
remedy: {
|
|
24
24
|
action:
|
|
25
|
-
'Run fleet-sync from the manager
|
|
25
|
+
'Run fleet-sync from the manager: it links the launcher to the checkout of the revision it applies.',
|
|
26
26
|
},
|
|
27
27
|
},
|
|
28
28
|
];
|
package/src/index.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export { parseConfig, loadConfig, selectNodes } from './config';
|
|
2
|
-
export { doctor } from './manager';
|
|
3
|
-
export { probe } from './probe';
|
|
4
|
-
export { makeReport, reportExitCode, renderReport } from './report';
|
|
5
|
-
export { syncPlan, syncWorkspace } from './sync';
|
|
6
|
-
export type * from './types';
|
package/src/test-fixtures.ts
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|
2
|
-
import { tmpdir } from 'node:os';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
|
-
import type { CommandResult, Runner, FleetNode } from './types';
|
|
5
|
-
|
|
6
|
-
export const HEAD = 'a'.repeat(40);
|
|
7
|
-
export const NODE: FleetNode = {
|
|
8
|
-
id: 'dev-one',
|
|
9
|
-
ssh: 'dev-one',
|
|
10
|
-
workspace: '/home/dev/workspace',
|
|
11
|
-
required: [],
|
|
12
|
-
};
|
|
13
|
-
export const success = (stdout = ''): CommandResult => ({
|
|
14
|
-
ok: true,
|
|
15
|
-
stdout,
|
|
16
|
-
stderr: '',
|
|
17
|
-
reason: 'exit',
|
|
18
|
-
});
|
|
19
|
-
export const failure = (stderr = ''): CommandResult => ({
|
|
20
|
-
ok: false,
|
|
21
|
-
stdout: '',
|
|
22
|
-
stderr,
|
|
23
|
-
reason: 'exit',
|
|
24
|
-
});
|
|
25
|
-
export const gitRunner: Runner = async (command) =>
|
|
26
|
-
success(command.includes('rev-parse') ? HEAD : '');
|
|
27
|
-
|
|
28
|
-
export async function workspace(): Promise<string> {
|
|
29
|
-
const root = await mkdtemp(join(tmpdir(), "fleet test's "));
|
|
30
|
-
for (const file of ['package.json', 'bun.lock', 'mise.toml', 'mise.lock'])
|
|
31
|
-
await writeFile(join(root, file), '{}');
|
|
32
|
-
await mkdir(join(root, 'packages/example'), { recursive: true });
|
|
33
|
-
await writeFile(join(root, 'packages/example/package.json'), '{}');
|
|
34
|
-
await mkdir(join(root, '.mise/locks'), { recursive: true });
|
|
35
|
-
return root;
|
|
36
|
-
}
|