@crvouga/dev-fleet 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -37
- package/agents/setup.md +42 -12
- package/ansible/macos/manager.yml +7 -7
- package/ansible/macos/policy.json +2 -10
- package/ansible/monitoring/README.md +86 -25
- package/ansible/monitoring/agent.yml +40 -2
- package/ansible/monitoring/hub-route-darwin.yml +51 -0
- package/ansible/monitoring/lock.json +5 -0
- package/ansible/pyproject.toml +12 -0
- package/ansible/ubuntu/baseline.yml +1 -1
- package/ansible/ubuntu/disk-gc.py +336 -0
- package/ansible/ubuntu/policy.json +12 -3
- package/ansible/ubuntu/tailscale.yml +8 -5
- package/ansible/ubuntu/tasks/resources.yml +61 -1
- package/ansible/uv.lock +190 -0
- package/bootstrap/bootstrap.sh +25 -18
- package/bootstrap/doctor.ts +46 -1
- package/bootstrap/fleet-manager-run.sh +42 -15
- package/bootstrap/fleet-remote-bootstrap.sh +85 -12
- package/bootstrap/global-config.ts +64 -6
- package/bootstrap/install-launcher.sh +3 -0
- package/bootstrap/npm.ts +10 -2
- package/bootstrap/release-version.ts +157 -0
- package/docs/consumer-setup.md +408 -0
- package/docs/declared.md +89 -0
- package/docs/machine-setup.md +399 -0
- package/docs/operations.md +276 -0
- package/docs/releasing.md +103 -0
- package/docs/state.md +254 -0
- package/docs/topology.md +67 -0
- package/docs/ubuntu-sync.md +351 -0
- package/docs/worker-contract.md +67 -0
- package/examples/fleet.json +5 -0
- package/llms.txt +35 -0
- package/mise.lock +29 -0
- package/mise.toml +12 -1
- package/package.json +11 -2
- package/schema/fleet.schema.json +71 -15
- package/src/agents/fleet-verify.ts +9 -0
- package/src/agents/native-verify.ts +167 -22
- package/src/agents/provider-errors.ts +9 -2
- package/src/agents/registry.ts +7 -1
- package/src/cli/agent-guide.ts +39 -5
- package/src/cli/app.ts +11 -4
- package/src/cli/cli.ts +7 -0
- package/src/cli/command-index.ts +7 -0
- package/src/cli/fleet-state.ts +499 -0
- package/src/cli/fleet-sync-checkout.ts +56 -17
- package/src/cli/fleet-sync-enrollment.ts +19 -23
- package/src/cli/fleet-sync-forward.ts +1 -1
- package/src/cli/fleet-sync-manager.ts +64 -8
- package/src/cli/fleet-sync-monitoring.ts +5 -5
- package/src/cli/fleet-sync-output.ts +4 -1
- package/src/cli/fleet-sync-remote.ts +11 -5
- package/src/cli/fleet-sync-state.ts +86 -0
- package/src/cli/fleet-sync.ts +123 -57
- package/src/cli/fleet.ts +4 -2
- package/src/cli/managed-health.ts +4 -4
- package/src/cli/node-sync.ts +7 -0
- package/src/cli/program.ts +20 -4
- package/src/cli/project.ts +51 -0
- package/src/cli/register-fleet.ts +51 -1
- package/src/cli/registry.ts +28 -1
- package/src/cli/roster.ts +46 -30
- package/src/cli/settings.ts +15 -12
- package/src/cli/source-operations.ts +27 -3
- package/src/cli/tailnet.ts +2 -2
- package/src/definition.ts +31 -1
- package/src/diagnostics.ts +1 -1
- package/src/fleet-config.ts +105 -19
- package/src/machine/source.ts +3 -3
- package/src/machine/state.ts +15 -27
- package/src/manager-sync.ts +8 -3
- package/src/monitoring/agent-evidence.ts +146 -0
- package/src/monitoring/apply.ts +15 -3
- package/src/monitoring/beszel.ts +11 -4
- package/src/monitoring/enroll.ts +2 -1
- package/src/monitoring/hub-access.ts +4 -3
- package/src/monitoring/hub-backup.ts +121 -0
- package/src/monitoring/hub-viewer.ts +8 -8
- package/src/monitoring/hub.ts +3 -3
- package/src/monitoring/manager-hub.ts +30 -14
- package/src/node-converge-linux.ts +2 -0
- package/src/package-info.ts +45 -0
- package/src/platform/ansible.ts +19 -15
- package/src/platform/resource-verify.ts +8 -4
- package/src/platform/ubuntu-audit.ts +5 -2
- package/src/remote-bootstrap.ts +17 -19
- package/src/repair-catalog.ts +1 -1
- package/src/runner.ts +1 -1
- package/src/state/bucket.ts +208 -0
- package/src/state/check.ts +74 -0
- package/src/state/connection.ts +92 -0
- package/src/state/enrollment.ts +45 -0
- package/src/state/legacy.ts +89 -0
- package/src/state/litestream.ts +143 -0
- package/src/state/operator.ts +20 -0
- package/src/state/paths.ts +40 -0
- package/src/state/roster.ts +62 -0
- package/src/state/store.ts +169 -0
- package/src/state/sync.ts +352 -0
- package/src/t3-endpoint.ts +64 -8
- package/src/t3-linked-check.ts +23 -1
- package/src/toolchain-checks.ts +25 -0
- package/src/types.ts +4 -0
- package/src/worker-access-checks.ts +56 -3
- package/src/worker-host-checks.ts +5 -0
- package/src/workstation-checks.ts +1 -1
- package/src/index.ts +0 -6
- package/src/test-fixtures.ts +0 -36
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { loadConfig } from '../config';
|
|
1
|
+
import { mkdir, rm } from 'node:fs/promises';
|
|
2
|
+
import { loadRoster } from './roster';
|
|
4
3
|
import { enrolledRoster } from '../definition';
|
|
5
4
|
import { FleetUsageError } from '../errors';
|
|
6
5
|
import { dirname } from 'node:path';
|
|
@@ -10,6 +9,7 @@ import { matchComputers } from '../tailnet-matching';
|
|
|
10
9
|
import { sshArgs, sshFailure } from '../ssh';
|
|
11
10
|
import { ensurePrivateRoot } from '../machine/state';
|
|
12
11
|
import type { FleetNode, Runner } from '../types';
|
|
12
|
+
import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
|
|
13
13
|
|
|
14
14
|
/** Interactive enrollment/resume only. Existing paths and explicit identities
|
|
15
15
|
* are preserved. Tailnet candidates never enter the declared membership.
|
|
@@ -20,7 +20,7 @@ export async function enrollManager(path: string, run: Runner): Promise<void> {
|
|
|
20
20
|
const saved =
|
|
21
21
|
original === null
|
|
22
22
|
? []
|
|
23
|
-
: enrolledRoster(canonical, await
|
|
23
|
+
: enrolledRoster(canonical, await loadRoster(path)).nodes;
|
|
24
24
|
const nodes: FleetNode[] = canonical.workers.map((node) => ({
|
|
25
25
|
id: node.id,
|
|
26
26
|
ssh: node.ssh,
|
|
@@ -55,8 +55,6 @@ export async function enrollManager(path: string, run: Runner): Promise<void> {
|
|
|
55
55
|
process.stderr.write(
|
|
56
56
|
`Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => ` ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
|
|
57
57
|
);
|
|
58
|
-
await ensurePrivateRoot(dirname(path));
|
|
59
|
-
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
60
58
|
await persistEnrollment(path, original, merged);
|
|
61
59
|
}
|
|
62
60
|
|
|
@@ -95,13 +93,8 @@ function ambiguous(ids: string[], matches: Map<string, FleetNode[]>): boolean {
|
|
|
95
93
|
ids.length > 1 || (ids.length === 1 && matches.get(ids[0]!)?.length !== 1)
|
|
96
94
|
);
|
|
97
95
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
return await readFile(path, 'utf8');
|
|
101
|
-
} catch (error) {
|
|
102
|
-
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
|
103
|
-
throw error;
|
|
104
|
-
}
|
|
96
|
+
function existingEnrollment(path: string): Promise<string | null> {
|
|
97
|
+
return readRosterText(path);
|
|
105
98
|
}
|
|
106
99
|
export function mergeEnrollment(
|
|
107
100
|
saved: FleetNode[],
|
|
@@ -123,22 +116,25 @@ async function persistEnrollment(
|
|
|
123
116
|
original: string | null,
|
|
124
117
|
nodes: FleetNode[]
|
|
125
118
|
): Promise<void> {
|
|
126
|
-
const
|
|
127
|
-
await mkdir(lock, { mode: 0o700 });
|
|
128
|
-
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
129
|
-
try {
|
|
119
|
+
const write = async (): Promise<void> => {
|
|
130
120
|
if ((await existingEnrollment(path)) !== original)
|
|
131
121
|
throw new FleetUsageError(
|
|
132
122
|
'Enrollment changed during approval. Inspect it and rerun; no connection was overwritten.'
|
|
133
123
|
);
|
|
134
|
-
await
|
|
135
|
-
|
|
136
|
-
JSON.stringify({ version: 1, nodes }, null, 2) + '\n'
|
|
137
|
-
{ flag: 'wx', mode: 0o600 }
|
|
124
|
+
await writeRosterText(
|
|
125
|
+
path,
|
|
126
|
+
JSON.stringify({ version: 1, nodes }, null, 2) + '\n'
|
|
138
127
|
);
|
|
139
|
-
|
|
128
|
+
};
|
|
129
|
+
// The fleet state serializes its own writers; a roster file needs a lock.
|
|
130
|
+
if (path === STATE_ROSTER) return write();
|
|
131
|
+
await ensurePrivateRoot(dirname(path));
|
|
132
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
133
|
+
const lock = `${path}.enroll.lock`;
|
|
134
|
+
await mkdir(lock, { mode: 0o700 });
|
|
135
|
+
try {
|
|
136
|
+
await write();
|
|
140
137
|
} finally {
|
|
141
|
-
await rm(temporary, { force: true });
|
|
142
138
|
await rm(lock, { recursive: true });
|
|
143
139
|
}
|
|
144
140
|
}
|
|
@@ -91,7 +91,7 @@ export function managerCommand(
|
|
|
91
91
|
revision,
|
|
92
92
|
...forwardedFlags(options, undefined),
|
|
93
93
|
]
|
|
94
|
-
: ['
|
|
94
|
+
: ['devfleet', 'fleet-sync', ...forwardedFlags(options, revision)];
|
|
95
95
|
const color = process.stdout.isTTY && !process.env['NO_COLOR'];
|
|
96
96
|
// A login shell gives the manager's normal PATH (mise, Docker, Tailscale).
|
|
97
97
|
const script = `export ${NO_FORWARD_ENV}=1${color ? ' FORCE_COLOR=1' : ''}; exec ${args.map(shellQuote).join(' ')}`;
|
|
@@ -11,6 +11,11 @@ import { enrollManagerMonitoring } from './fleet-sync-monitoring';
|
|
|
11
11
|
import { installConfig } from '../fleet-config';
|
|
12
12
|
import { machineProviders } from '../t3/providers';
|
|
13
13
|
import { providerNames } from '../t3/names-check';
|
|
14
|
+
import macos from '../../ansible/macos/policy.json';
|
|
15
|
+
import { ansibleCommand } from '../platform/ansible';
|
|
16
|
+
import { PACKAGE_ROOT } from '../package-info';
|
|
17
|
+
import { readFileSync } from 'node:fs';
|
|
18
|
+
import { join } from 'node:path';
|
|
14
19
|
|
|
15
20
|
/** The manager converges itself locally (never over SSH) before any worker:
|
|
16
21
|
* it must be the declared manager, stay awake on AC power, and host every
|
|
@@ -32,7 +37,8 @@ export async function convergeManager(
|
|
|
32
37
|
if (identity.status === 'pass') {
|
|
33
38
|
// Launchers and scheduled jobs on the manager read the installed copy.
|
|
34
39
|
if (input.apply) installConfig();
|
|
35
|
-
checks.push(await managerPower(run));
|
|
40
|
+
checks.push(await managerPower(run, input.workspace));
|
|
41
|
+
checks.push(await managerDocker(run));
|
|
36
42
|
checks.push(await machineProviders(id, input.workspace, input.apply));
|
|
37
43
|
checks.push(await providerNames(id, input.workspace, input.apply));
|
|
38
44
|
if (input.apply) {
|
|
@@ -170,12 +176,22 @@ export async function managerIdentity(id: string, run: Runner): Promise<Check> {
|
|
|
170
176
|
};
|
|
171
177
|
}
|
|
172
178
|
|
|
173
|
-
/** Workers depend on the manager's services, so it never sleeps on AC power.
|
|
174
|
-
|
|
179
|
+
/** Workers depend on the manager's services, so it never sleeps on AC power.
|
|
180
|
+
* Measures every setting `ansible/macos/manager.yml` declares, from the same
|
|
181
|
+
* policy file the playbook applies.
|
|
182
|
+
*/
|
|
183
|
+
export async function managerPower(
|
|
184
|
+
run: Runner,
|
|
185
|
+
workspace = '.'
|
|
186
|
+
): Promise<Check> {
|
|
175
187
|
const result = await run(['pmset', '-g', 'custom']);
|
|
176
188
|
const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
|
|
177
|
-
const
|
|
178
|
-
|
|
189
|
+
const observed = (key: string) =>
|
|
190
|
+
new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
|
|
191
|
+
const drifted = Object.entries(macos.pmset)
|
|
192
|
+
.filter(([key, value]) => observed(key) !== value)
|
|
193
|
+
.map(([key]) => key);
|
|
194
|
+
return drifted.length === 0
|
|
179
195
|
? {
|
|
180
196
|
id: 'manager:power',
|
|
181
197
|
status: 'pass',
|
|
@@ -184,10 +200,50 @@ export async function managerPower(run: Runner): Promise<Check> {
|
|
|
184
200
|
: {
|
|
185
201
|
id: 'manager:power',
|
|
186
202
|
status: 'fail',
|
|
187
|
-
summary: '
|
|
203
|
+
summary: drifted.includes('sleep')
|
|
204
|
+
? 'MANAGER_SLEEPS_ON_POWER'
|
|
205
|
+
: `MANAGER_POWER_POLICY_DRIFT ${drifted.join(' ')}`,
|
|
188
206
|
remedy: {
|
|
189
|
-
action:
|
|
190
|
-
'Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once: ansible-playbook -K ansible/macos/manager.yml',
|
|
207
|
+
action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once, as an administrator: cd ${workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`,
|
|
191
208
|
},
|
|
192
209
|
};
|
|
193
210
|
}
|
|
211
|
+
|
|
212
|
+
/** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
|
|
213
|
+
export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
|
|
214
|
+
const { vars } = Bun.TOML.parse(
|
|
215
|
+
readFileSync(join(root, 'mise.toml'), 'utf8')
|
|
216
|
+
) as { vars: Record<string, string> };
|
|
217
|
+
return `Docker Desktop ${vars['docker_desktop_version']} (${vars['docker_desktop_build']})`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** The manager's services run in Docker Desktop, which no playbook installs
|
|
221
|
+
* and which updates itself: the pinned release is measured, and a different
|
|
222
|
+
* one warns instead of failing the run.
|
|
223
|
+
*/
|
|
224
|
+
export async function managerDocker(
|
|
225
|
+
run: Runner,
|
|
226
|
+
pinned = pinnedDockerDesktop()
|
|
227
|
+
): Promise<Check> {
|
|
228
|
+
const result = await run(
|
|
229
|
+
['docker', 'version', '--format', '{{.Server.Platform.Name}}'],
|
|
230
|
+
{ timeoutMs: 20_000 }
|
|
231
|
+
);
|
|
232
|
+
const observed = result.ok ? result.stdout.trim() : '';
|
|
233
|
+
if (observed === pinned)
|
|
234
|
+
return {
|
|
235
|
+
id: 'manager:docker',
|
|
236
|
+
status: 'pass',
|
|
237
|
+
summary: `The manager runs the pinned ${pinned}.`,
|
|
238
|
+
};
|
|
239
|
+
return {
|
|
240
|
+
id: 'manager:docker',
|
|
241
|
+
status: 'warn',
|
|
242
|
+
summary: observed
|
|
243
|
+
? 'MANAGER_DOCKER_DESKTOP_DRIFT'
|
|
244
|
+
: 'MANAGER_DOCKER_DESKTOP_UNOBSERVED',
|
|
245
|
+
remedy: {
|
|
246
|
+
action: `The manager's services are tested on ${pinned}. Install that release and turn off its automatic updates, or expect differences; fleet-sync does not install or replace Docker Desktop.`,
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
@@ -2,7 +2,7 @@ import type { FleetNode, Runner, Check } from '../types';
|
|
|
2
2
|
import type { FleetDefinition } from '../definition';
|
|
3
3
|
import { hubUrl, roleOf } from '../topology';
|
|
4
4
|
import { discoverTailnet } from '../tailscale';
|
|
5
|
-
import {
|
|
5
|
+
import { fleetIdentity, hubAccess } from '../monitoring/hub-access';
|
|
6
6
|
import { enrollBeszel } from '../monitoring/enroll';
|
|
7
7
|
import { verifyBeszel } from '../monitoring/beszel';
|
|
8
8
|
import { installRemoteAgent } from '../monitoring/remote';
|
|
@@ -142,15 +142,15 @@ async function hubReachable(hub: string, run: Runner): Promise<boolean> {
|
|
|
142
142
|
return probe.ok;
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
-
const
|
|
146
|
-
BESZEL_ACCOUNT_REQUIRED: `The hub rejected the fleet login ${
|
|
145
|
+
const remedies = (): Record<string, string> => ({
|
|
146
|
+
BESZEL_ACCOUNT_REQUIRED: `The hub rejected the fleet login ${fleetIdentity()}. A new hub on the manager creates it on first start from BESZEL_FLEET_AUTH; an older hub needs it once, by its superuser, as a verified users record with that Vault password. Never paste the password on a command line.`,
|
|
147
147
|
BESZEL_HUB_UNREACHABLE:
|
|
148
148
|
'The hub runs on the manager; keep it awake and Docker Desktop running, then rerun fleet-sync from the manager so it converges the hub first.',
|
|
149
149
|
BESZEL_NODE_TAILNET_REQUIRED:
|
|
150
150
|
'The worker is not visible on the tailnet. Sign it in to Tailscale and record its tailscaleId in the local roster, then rerun fleet-sync.',
|
|
151
151
|
BESZEL_NOT_ENROLLED:
|
|
152
152
|
'Only workers declared in fleet.json run the agent over SSH; fleet-sync installs the manager’s own agent locally after its hub.',
|
|
153
|
-
};
|
|
153
|
+
});
|
|
154
154
|
function unavailable(code: string): Check[] {
|
|
155
155
|
return [
|
|
156
156
|
{
|
|
@@ -159,7 +159,7 @@ function unavailable(code: string): Check[] {
|
|
|
159
159
|
summary: code,
|
|
160
160
|
remedy: {
|
|
161
161
|
action:
|
|
162
|
-
|
|
162
|
+
remedies()[code] ??
|
|
163
163
|
"Verify the manager's private hub is running and the manager has BESZEL_FLEET_AUTH through existing Vault authorization. The normal hub account must access these systems. Inspect duplicate registration/host conflicts before retry; no token or raw response is printed.",
|
|
164
164
|
},
|
|
165
165
|
},
|
|
@@ -30,7 +30,10 @@ const COLUMNS: { title: string; matches: (id: string) => boolean }[] = [
|
|
|
30
30
|
},
|
|
31
31
|
{
|
|
32
32
|
title: 'LINK',
|
|
33
|
-
matches: (id) =>
|
|
33
|
+
matches: (id) =>
|
|
34
|
+
['t3:manager-endpoint', 't3:manager-origin', 't3:paired-access'].includes(
|
|
35
|
+
id
|
|
36
|
+
),
|
|
34
37
|
},
|
|
35
38
|
{
|
|
36
39
|
title: 'PROXY',
|
|
@@ -11,7 +11,8 @@ import { saveReport } from './fleet-sync-report';
|
|
|
11
11
|
import { synchronizeMonitoring } from './fleet-sync-monitoring';
|
|
12
12
|
import { convergeManager } from './fleet-sync-manager';
|
|
13
13
|
import { FleetUsageError } from '../errors';
|
|
14
|
-
import {
|
|
14
|
+
import { selectNodes } from '../config';
|
|
15
|
+
import { loadRoster } from './roster';
|
|
15
16
|
import { enrolledRoster } from '../definition';
|
|
16
17
|
import type { FleetDefinition } from '../definition';
|
|
17
18
|
import { fleet as definition } from '../fleet-config';
|
|
@@ -19,23 +20,27 @@ import { synchronizeManager } from '../manager-sync';
|
|
|
19
20
|
import { inspectBootstrap } from '../manager-sync';
|
|
20
21
|
import { prepareBootstrap, bootstrapRemote } from '../remote-bootstrap';
|
|
21
22
|
import { machineRoot } from './fleet-sync';
|
|
22
|
-
import type { Runner } from '../types';
|
|
23
|
-
import type { DeployNode } from '../manager-sync';
|
|
23
|
+
import type { Check, Runner } from '../types';
|
|
24
|
+
import type { DeployNode, ManagerReport } from '../manager-sync';
|
|
24
25
|
import type { FleetSyncOptions } from './fleet-sync';
|
|
25
26
|
import type { Settings } from './settings';
|
|
26
27
|
import type { CommandOutcome } from './contracts';
|
|
27
28
|
|
|
29
|
+
/** `settle` runs once the fleet's result is known and before it is reported:
|
|
30
|
+
* it records the result and returns the check that says so.
|
|
31
|
+
*/
|
|
28
32
|
export async function selected(
|
|
29
33
|
options: FleetSyncOptions,
|
|
30
34
|
settings: Settings,
|
|
31
|
-
run: Runner
|
|
35
|
+
run: Runner,
|
|
36
|
+
settle?: (report: ManagerReport) => Promise<Check>
|
|
32
37
|
): Promise<CommandOutcome> {
|
|
33
38
|
if (!options.revision)
|
|
34
39
|
throw new FleetUsageError('An approved source revision is required.');
|
|
35
40
|
const canonical = definition;
|
|
36
41
|
let roster;
|
|
37
42
|
try {
|
|
38
|
-
roster = enrolledRoster(canonical, await
|
|
43
|
+
roster = enrolledRoster(canonical, await loadRoster(settings.config));
|
|
39
44
|
} catch {
|
|
40
45
|
return enrollmentRequired(canonical, options.revision);
|
|
41
46
|
}
|
|
@@ -89,6 +94,7 @@ export async function selected(
|
|
|
89
94
|
run,
|
|
90
95
|
deployWithMonitoring(payload, canonical, run, ui)
|
|
91
96
|
).finally(() => ui.progress.stop());
|
|
97
|
+
if (settle) report.checks.push(await settle(report));
|
|
92
98
|
return outcome(report, planFingerprint, started);
|
|
93
99
|
}
|
|
94
100
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { Enrollment } from '../machine/state';
|
|
2
|
+
import { replicaCheck } from '../state/check';
|
|
3
|
+
import { storeEnrollment } from '../state/enrollment';
|
|
4
|
+
import { adoptEarlier } from '../state/legacy';
|
|
5
|
+
import { databasePath, stateRoot } from '../state/paths';
|
|
6
|
+
import { writeDocument } from '../state/store';
|
|
7
|
+
import { pending, push, type Standing, type StateContext } from '../state/sync';
|
|
8
|
+
import type { Check } from '../types';
|
|
9
|
+
|
|
10
|
+
/** What one fleet-sync run records in the fleet state: the revision it was
|
|
11
|
+
* approved for and the configuration it runs with, then what it applied, then
|
|
12
|
+
* an upload so the bucket holds all of it.
|
|
13
|
+
*/
|
|
14
|
+
export interface RunRecord {
|
|
15
|
+
/** Before any machine changes: this revision is the approved one. */
|
|
16
|
+
approve(): void;
|
|
17
|
+
/** Once the result is known. The check says whether the bucket holds it. */
|
|
18
|
+
settle(converged: boolean): Promise<Check>;
|
|
19
|
+
/** For a run that stopped before its result: its approval is still uploaded. */
|
|
20
|
+
abandon(): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function runRecord(
|
|
24
|
+
previous: Enrollment | null,
|
|
25
|
+
revision: string,
|
|
26
|
+
state: {
|
|
27
|
+
context: StateContext;
|
|
28
|
+
standing: Standing;
|
|
29
|
+
env?: NodeJS.ProcessEnv;
|
|
30
|
+
/** The configuration this run reads from a project's fleet.json. Stored
|
|
31
|
+
* so that a run started anywhere else (the schedule, a hand-off from a
|
|
32
|
+
* worker, a replacement manager) uses the one last applied.
|
|
33
|
+
*/
|
|
34
|
+
definition?: string;
|
|
35
|
+
}
|
|
36
|
+
): RunRecord {
|
|
37
|
+
const env = state.env ?? process.env;
|
|
38
|
+
const approval: Enrollment = {
|
|
39
|
+
...previous,
|
|
40
|
+
version: 1,
|
|
41
|
+
id: 'manager',
|
|
42
|
+
approvedRevision: revision,
|
|
43
|
+
};
|
|
44
|
+
let settled = false;
|
|
45
|
+
/** Uploads only what this run changed; otherwise the bucket stands as it
|
|
46
|
+
* did when the run started.
|
|
47
|
+
*/
|
|
48
|
+
const publish = (): Promise<Standing> =>
|
|
49
|
+
pending(state.context)
|
|
50
|
+
? push(state.context)
|
|
51
|
+
: Promise.resolve(state.standing);
|
|
52
|
+
return {
|
|
53
|
+
approve: () => {
|
|
54
|
+
storeEnrollment(approval, env);
|
|
55
|
+
if (state.definition === undefined) return;
|
|
56
|
+
adoptEarlier(env);
|
|
57
|
+
writeDocument(
|
|
58
|
+
databasePath(stateRoot(env)),
|
|
59
|
+
'definition',
|
|
60
|
+
state.definition
|
|
61
|
+
);
|
|
62
|
+
},
|
|
63
|
+
async settle(converged) {
|
|
64
|
+
settled = true;
|
|
65
|
+
if (converged)
|
|
66
|
+
storeEnrollment(
|
|
67
|
+
{
|
|
68
|
+
...approval,
|
|
69
|
+
appliedRevision: revision,
|
|
70
|
+
...(previous?.appliedRevision &&
|
|
71
|
+
previous.appliedRevision !== revision
|
|
72
|
+
? { previousRevision: previous.appliedRevision }
|
|
73
|
+
: {}),
|
|
74
|
+
},
|
|
75
|
+
env
|
|
76
|
+
);
|
|
77
|
+
// The fleet converged either way: a bucket problem is reported, not fatal.
|
|
78
|
+
return replicaCheck(await publish().catch(() => 'unreachable' as const), {
|
|
79
|
+
advisory: true,
|
|
80
|
+
});
|
|
81
|
+
},
|
|
82
|
+
async abandon() {
|
|
83
|
+
if (!settled) await publish().catch(() => undefined);
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
package/src/cli/fleet-sync.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { localCheckoutRevision } from '../machine/local-source';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
definitionSource,
|
|
4
|
+
fleet as canonicalDefinition,
|
|
5
|
+
reloadFleet,
|
|
6
|
+
} from '../fleet-config';
|
|
3
7
|
import { managerFailure } from '../manager-failure';
|
|
4
8
|
import { managerText, stdoutColor } from './fleet-sync-output';
|
|
5
9
|
import { managerSchedule, type ScheduleAction } from './fleet-sync-schedule';
|
|
@@ -9,12 +13,20 @@ import { homedir } from 'node:os';
|
|
|
9
13
|
import { join } from 'node:path';
|
|
10
14
|
import { existsSync, renameSync } from 'node:fs';
|
|
11
15
|
import {
|
|
12
|
-
readEnrollment,
|
|
13
16
|
isRevision,
|
|
14
|
-
saveEnrollment,
|
|
15
17
|
withMachineLock,
|
|
16
18
|
ensurePrivateRoot,
|
|
19
|
+
type Enrollment,
|
|
17
20
|
} from '../machine/state';
|
|
21
|
+
import { loadEnrollment } from '../state/enrollment';
|
|
22
|
+
import { legacyHome } from '../state/paths';
|
|
23
|
+
import {
|
|
24
|
+
DIVERGED_REMEDY,
|
|
25
|
+
refresh,
|
|
26
|
+
stateContext,
|
|
27
|
+
type Standing,
|
|
28
|
+
} from '../state/sync';
|
|
29
|
+
import { runRecord } from './fleet-sync-state';
|
|
18
30
|
import { runCommand } from '../runner';
|
|
19
31
|
import type { Runner } from '../types';
|
|
20
32
|
import type { Settings } from './settings';
|
|
@@ -23,15 +35,14 @@ import { FleetUsageError } from '../errors';
|
|
|
23
35
|
import { FleetLockError } from '../machine/lock-error';
|
|
24
36
|
import { selected } from './fleet-sync-remote';
|
|
25
37
|
import { managerIdentity } from './fleet-sync-manager';
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
38
|
+
import { managerRunScript, rehome } from './fleet-sync-checkout';
|
|
39
|
+
import { packageInfo } from '../package-info';
|
|
28
40
|
import {
|
|
29
41
|
forwardingDisabled,
|
|
30
42
|
forwardToManager,
|
|
31
43
|
type ForwardSpawn,
|
|
32
44
|
} from './fleet-sync-forward';
|
|
33
45
|
|
|
34
|
-
declare const PERSONAL_SOURCE_REVISION: string | null;
|
|
35
46
|
export interface FleetSyncOptions {
|
|
36
47
|
revision?: string;
|
|
37
48
|
dryRun?: boolean;
|
|
@@ -69,9 +80,29 @@ export async function fleetSync(
|
|
|
69
80
|
if (!(await onManager(run)))
|
|
70
81
|
return offManager(options, settings, run, forward);
|
|
71
82
|
if (options.schedule === 'status') return scheduleStatus(settings, run);
|
|
72
|
-
|
|
83
|
+
return managerSync(options, settings, {
|
|
84
|
+
run,
|
|
85
|
+
root: machineRoot(),
|
|
86
|
+
restart: () => fleetSync(options, settings, run, forward),
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface ManagerSync {
|
|
91
|
+
run: Runner;
|
|
92
|
+
/** This machine's run lock and caches. */
|
|
93
|
+
root: string;
|
|
94
|
+
/** Starts the request over once the fleet state has been replaced. */
|
|
95
|
+
restart: () => Promise<CommandOutcome>;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** fleet-sync on the machine already known to be the manager. */
|
|
99
|
+
export async function managerSync(
|
|
100
|
+
options: FleetSyncOptions,
|
|
101
|
+
settings: Settings,
|
|
102
|
+
{ run, root, restart }: ManagerSync
|
|
103
|
+
): Promise<CommandOutcome> {
|
|
73
104
|
await ensurePrivateRoot(root);
|
|
74
|
-
const state = await
|
|
105
|
+
const state = await loadEnrollment(legacyHome() ? root : null);
|
|
75
106
|
const revision = await selectedRevision(
|
|
76
107
|
options.revision,
|
|
77
108
|
state?.approvedRevision,
|
|
@@ -81,61 +112,100 @@ export async function fleetSync(
|
|
|
81
112
|
if (!isRevision(revision))
|
|
82
113
|
return actionRequired(
|
|
83
114
|
'FLEET_REVISION_REQUIRED',
|
|
84
|
-
'
|
|
115
|
+
'No revision to converge to. Run the published dev-fleet package (it names the commit it was built from), or select a full commit SHA with --revision.'
|
|
85
116
|
);
|
|
86
117
|
if (options.schedule)
|
|
87
118
|
return scheduleRequest(options, revision, settings, run);
|
|
88
119
|
const rehomed = await rehome(
|
|
89
120
|
options,
|
|
90
|
-
{ revision, workspace: settings.workspace },
|
|
121
|
+
{ revision, workspace: settings.workspace, root },
|
|
91
122
|
run
|
|
92
123
|
);
|
|
93
|
-
if (rehomed
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if (
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
124
|
+
if (rehomed === 'unavailable')
|
|
125
|
+
return actionRequired(
|
|
126
|
+
'FLEET_SOURCE_UNAVAILABLE',
|
|
127
|
+
`Revision ${revision} could not be taken from ${canonicalDefinition.source.repository}, so nothing was changed. Check the network and Git access to it, and that the revision is pushed there; fleet-sync never converges from the package or working tree it was started in.`,
|
|
128
|
+
revision
|
|
129
|
+
);
|
|
130
|
+
if (rehomed !== 'here') return rehomed;
|
|
131
|
+
if (options.dryRun) return selected({ ...options, revision }, settings, run);
|
|
132
|
+
// The bucket is the source of truth: a run starts from its version of the
|
|
133
|
+
// configuration, the roster and the approval, not from a stale local copy.
|
|
134
|
+
const fresh = await refresh(stateContext(run));
|
|
135
|
+
if (fresh.standing === 'diverged')
|
|
136
|
+
return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
|
|
137
|
+
if (fresh.replaced) {
|
|
138
|
+
// Everything read so far came from the stale copy, the manager's own
|
|
139
|
+
// identity included.
|
|
140
|
+
reloadFleet();
|
|
141
|
+
return restart();
|
|
104
142
|
}
|
|
105
|
-
return
|
|
143
|
+
return authorizeAndApply(root, state, options, {
|
|
144
|
+
settings,
|
|
145
|
+
run,
|
|
146
|
+
revision,
|
|
147
|
+
standing: fresh.standing,
|
|
148
|
+
});
|
|
106
149
|
}
|
|
107
150
|
|
|
108
|
-
|
|
151
|
+
interface ApplyContext {
|
|
152
|
+
settings: Settings;
|
|
153
|
+
run: Runner;
|
|
154
|
+
revision: string;
|
|
155
|
+
/** How the fleet state stood with the bucket when this run started. */
|
|
156
|
+
standing: Standing;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Applies without a confirmation prompt; --dry-run is the preview. */
|
|
160
|
+
async function authorizeAndApply(
|
|
161
|
+
root: string,
|
|
162
|
+
state: Enrollment | null,
|
|
163
|
+
options: FleetSyncOptions,
|
|
164
|
+
context: ApplyContext
|
|
165
|
+
): Promise<CommandOutcome> {
|
|
166
|
+
const { settings, run, revision } = context;
|
|
167
|
+
const request = { ...options, revision, apply: true, yes: true };
|
|
168
|
+
if (!interactive(options)) return applyLocked(root, state, request, context);
|
|
169
|
+
await enrollManager(settings.config, run);
|
|
170
|
+
// Needed to hand each node its own Vault token; nodes report any gap.
|
|
171
|
+
await authorizeManager({ interactive: true }, run);
|
|
172
|
+
return applyLocked(
|
|
173
|
+
root,
|
|
174
|
+
state,
|
|
175
|
+
{ ...request, interactiveAuthorization: true },
|
|
176
|
+
context
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Records the approved revision, converges, records what was applied, then
|
|
181
|
+
* uploads the fleet state: the report says whether the bucket holds it.
|
|
182
|
+
*/
|
|
109
183
|
function applyLocked(
|
|
110
184
|
root: string,
|
|
111
|
-
state:
|
|
185
|
+
state: Enrollment | null,
|
|
112
186
|
request: FleetSyncOptions & { revision: string },
|
|
113
|
-
context:
|
|
187
|
+
context: ApplyContext
|
|
114
188
|
): Promise<CommandOutcome> {
|
|
115
|
-
const { revision } =
|
|
116
|
-
const
|
|
189
|
+
const { settings, run, revision } = context;
|
|
190
|
+
const record = runRecord(state, revision, {
|
|
191
|
+
context: stateContext(run),
|
|
192
|
+
standing: context.standing,
|
|
193
|
+
...(definitionSource().kind === 'file'
|
|
194
|
+
? { definition: JSON.stringify(canonicalDefinition, null, 2) + '\n' }
|
|
195
|
+
: {}),
|
|
196
|
+
});
|
|
117
197
|
return withMachineLock(root, async () => {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
);
|
|
130
|
-
if (result.exitCode === 0)
|
|
131
|
-
await saveEnrollment(root, {
|
|
132
|
-
...approval,
|
|
133
|
-
appliedRevision: revision,
|
|
134
|
-
...(state?.appliedRevision && state.appliedRevision !== revision
|
|
135
|
-
? { previousRevision: state.appliedRevision }
|
|
136
|
-
: {}),
|
|
137
|
-
});
|
|
138
|
-
return result;
|
|
198
|
+
record.approve();
|
|
199
|
+
try {
|
|
200
|
+
return await selected(
|
|
201
|
+
{ ...request, rolledOut: state?.appliedRevision === revision },
|
|
202
|
+
settings,
|
|
203
|
+
run,
|
|
204
|
+
(report) => record.settle(report.ok)
|
|
205
|
+
);
|
|
206
|
+
} finally {
|
|
207
|
+
await record.abandon();
|
|
208
|
+
}
|
|
139
209
|
}).catch((error: unknown) => {
|
|
140
210
|
if (error instanceof FleetLockError)
|
|
141
211
|
return actionRequired(
|
|
@@ -175,12 +245,7 @@ async function offManager(
|
|
|
175
245
|
revision ?? null
|
|
176
246
|
);
|
|
177
247
|
const runScript = revision
|
|
178
|
-
? await
|
|
179
|
-
settings.workspace,
|
|
180
|
-
revision,
|
|
181
|
-
'bootstrap/fleet-manager-run.sh',
|
|
182
|
-
run
|
|
183
|
-
)
|
|
248
|
+
? await managerRunScript(settings.workspace, revision, run)
|
|
184
249
|
: null;
|
|
185
250
|
return forwardToManager(definition, options, revision, {
|
|
186
251
|
spawn: forward,
|
|
@@ -242,10 +307,11 @@ function interactive(options: FleetSyncOptions): boolean {
|
|
|
242
307
|
);
|
|
243
308
|
}
|
|
244
309
|
|
|
310
|
+
/** The commit an installed package was published from: the version of
|
|
311
|
+
* dev-fleet you run is the revision the fleet moves to.
|
|
312
|
+
*/
|
|
245
313
|
function packageRevision(): string | undefined {
|
|
246
|
-
return
|
|
247
|
-
? PERSONAL_SOURCE_REVISION
|
|
248
|
-
: undefined;
|
|
314
|
+
return packageInfo().revision ?? undefined;
|
|
249
315
|
}
|
|
250
316
|
|
|
251
317
|
async function scheduleRequest(
|
package/src/cli/fleet.ts
CHANGED
|
@@ -89,15 +89,17 @@ async function register(
|
|
|
89
89
|
options: CliOptions
|
|
90
90
|
): Promise<CommandOutcome> {
|
|
91
91
|
const node = nodeFromOptions(options);
|
|
92
|
-
const
|
|
92
|
+
const added = await addNode(settings.config, node, options.apply === true);
|
|
93
93
|
return result({
|
|
94
94
|
version: 1,
|
|
95
95
|
command: 'add',
|
|
96
96
|
ok: true,
|
|
97
97
|
applied: options.apply === true,
|
|
98
98
|
path: settings.config,
|
|
99
|
+
// How the fleet state stands with the bucket after this change.
|
|
100
|
+
...(added.standing ? { state: added.standing } : {}),
|
|
99
101
|
node,
|
|
100
|
-
nodes:
|
|
102
|
+
nodes: added.roster.nodes,
|
|
101
103
|
next: options.apply
|
|
102
104
|
? 'Run fleet-nodes --onboard, then fleet-status --json.'
|
|
103
105
|
: 'Review this preview, then repeat with --apply --yes.',
|