@crvouga/dev-fleet 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -7
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +83 -0
- package/ansible/macos/manager.yml +164 -9
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/macos/worker-host-key.yml +33 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +65 -0
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +21 -11
- package/docs/consumer-setup.md +74 -58
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +34 -0
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +112 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +82 -12
- package/docs/ubuntu-sync.md +98 -42
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1991 -162
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +349 -10
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +62 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +108 -0
- package/src/cli/fleet-node-lifecycle.ts +704 -0
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +167 -0
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +6 -0
- package/src/cli/fleet-sync-manager.ts +123 -8
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +172 -15
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +31 -10
- package/src/cli/fleet-sync-vault.ts +152 -26
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +172 -21
- package/src/cli/fleet-validate.ts +277 -0
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +251 -4
- package/src/cli/registry.ts +47 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +762 -232
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +70 -16
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +89 -8
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +7 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capacity.ts +31 -0
- package/src/worker-credentials.ts +201 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +224 -0
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +38 -36
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +143 -0
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
2
|
import type { Standing } from '../state/sync';
|
|
3
3
|
import type { Runner } from '../types';
|
|
4
|
+
import { featureRemovals, workerFeatures } from '../worker-features';
|
|
4
5
|
|
|
5
6
|
/** A tool whose pin changes between the applied and the selected revision. */
|
|
6
7
|
export interface ToolChange {
|
|
@@ -31,6 +32,9 @@ export interface SyncPlan {
|
|
|
31
32
|
tools: ToolChange[];
|
|
32
33
|
restarts: Restart[];
|
|
33
34
|
nodes: { node: string; from: string | null; to: string; changes: boolean }[];
|
|
35
|
+
/** What a sync removes from a worker because the fleet configuration
|
|
36
|
+
* turned one of its features off since it was last applied. */
|
|
37
|
+
removals: { node: string; removes: string[] }[];
|
|
34
38
|
}
|
|
35
39
|
|
|
36
40
|
/** The coding agents' CLIs: a new version is used by the next session, and a
|
|
@@ -84,16 +88,24 @@ export function toolChanges(
|
|
|
84
88
|
}
|
|
85
89
|
|
|
86
90
|
/** What restarts follow from the changes, as the playbooks and the toolchain
|
|
87
|
-
* setup act on them.
|
|
91
|
+
* setup act on them. A worker's T3 restarts only while it runs no agent turn
|
|
92
|
+
* (`ansible/ubuntu/tasks/t3-restart.yml`) unless `--restart-busy` asks for it
|
|
93
|
+
* anyway: only then do running turns end.
|
|
88
94
|
*/
|
|
89
|
-
export function restarts(
|
|
95
|
+
export function restarts(
|
|
96
|
+
tools: ToolChange[],
|
|
97
|
+
files: string[],
|
|
98
|
+
restartBusy = false
|
|
99
|
+
): Restart[] {
|
|
90
100
|
const found: Restart[] = [];
|
|
101
|
+
const when = restartBusy
|
|
102
|
+
? 'at once (--restart-busy): its running agent turns end.'
|
|
103
|
+
: 'once it runs no agent turn: running turns keep going and the restart is pending until then.';
|
|
91
104
|
if (tools.some((change) => change.tool === 't3'))
|
|
92
105
|
found.push({
|
|
93
106
|
service: 't3',
|
|
94
|
-
reason:
|
|
95
|
-
|
|
96
|
-
endsRunningTurns: true,
|
|
107
|
+
reason: `T3 is upgraded on every worker and restarted ${when}`,
|
|
108
|
+
endsRunningTurns: restartBusy,
|
|
97
109
|
});
|
|
98
110
|
else if (
|
|
99
111
|
files.some((file) =>
|
|
@@ -102,9 +114,8 @@ export function restarts(tools: ToolChange[], files: string[]): Restart[] {
|
|
|
102
114
|
)
|
|
103
115
|
found.push({
|
|
104
116
|
service: 't3',
|
|
105
|
-
reason:
|
|
106
|
-
|
|
107
|
-
endsRunningTurns: true,
|
|
117
|
+
reason: `The managed T3 policy may change: a T3 that has not loaded it is restarted ${when}`,
|
|
118
|
+
endsRunningTurns: restartBusy,
|
|
108
119
|
});
|
|
109
120
|
const agents = tools.filter((change) => AGENT_TOOLS.includes(change.tool));
|
|
110
121
|
if (agents.length)
|
|
@@ -136,6 +147,11 @@ export async function syncPlan(
|
|
|
136
147
|
state: Standing;
|
|
137
148
|
nodes: string[];
|
|
138
149
|
sources: string[];
|
|
150
|
+
/** `--restart-busy`: a T3 restart does not wait for idle. */
|
|
151
|
+
restartBusy?: boolean;
|
|
152
|
+
/** The fleet configuration last applied (from the fleet state), when
|
|
153
|
+
* there is one, and the one this sync applies. */
|
|
154
|
+
definitions?: { applied: unknown; selected: unknown };
|
|
139
155
|
},
|
|
140
156
|
run: Runner
|
|
141
157
|
): Promise<SyncPlan> {
|
|
@@ -163,16 +179,54 @@ export async function syncPlan(
|
|
|
163
179
|
revision: { from: input.from, to: input.to },
|
|
164
180
|
known: changes !== null,
|
|
165
181
|
tools: changes?.tools ?? [],
|
|
166
|
-
restarts: changes
|
|
182
|
+
restarts: changes
|
|
183
|
+
? restarts(changes.tools, changes.files, input.restartBusy === true)
|
|
184
|
+
: [],
|
|
167
185
|
nodes: input.nodes.map((node) => ({
|
|
168
186
|
node,
|
|
169
187
|
from: input.from,
|
|
170
188
|
to: input.to,
|
|
171
189
|
changes: differs,
|
|
172
190
|
})),
|
|
191
|
+
removals: input.definitions ? removals(input.nodes, input.definitions) : [],
|
|
173
192
|
};
|
|
174
193
|
}
|
|
175
194
|
|
|
195
|
+
/** Per worker, what turning its features off removes. Nothing is known to
|
|
196
|
+
* be removed from a worker the applied configuration does not name. */
|
|
197
|
+
function removals(
|
|
198
|
+
nodes: string[],
|
|
199
|
+
definitions: { applied: unknown; selected: unknown }
|
|
200
|
+
): SyncPlan['removals'] {
|
|
201
|
+
const workers = (definition: unknown): Record<string, unknown>[] => {
|
|
202
|
+
const list =
|
|
203
|
+
typeof definition === 'object' && definition !== null
|
|
204
|
+
? (definition as Record<string, unknown>)['workers']
|
|
205
|
+
: undefined;
|
|
206
|
+
return Array.isArray(list)
|
|
207
|
+
? list.filter(
|
|
208
|
+
(entry): entry is Record<string, unknown> =>
|
|
209
|
+
typeof entry === 'object' && entry !== null
|
|
210
|
+
)
|
|
211
|
+
: [];
|
|
212
|
+
};
|
|
213
|
+
return nodes.flatMap((node) => {
|
|
214
|
+
const before = workers(definitions.applied).find(
|
|
215
|
+
(entry) => entry['id'] === node
|
|
216
|
+
);
|
|
217
|
+
if (!before) return [];
|
|
218
|
+
const network = workers(definitions.selected).find(
|
|
219
|
+
(entry) => entry['id'] === node
|
|
220
|
+
)?.['network'] as { user?: unknown } | undefined;
|
|
221
|
+
const removes = featureRemovals(
|
|
222
|
+
workerFeatures(definitions.applied, node),
|
|
223
|
+
workerFeatures(definitions.selected, node),
|
|
224
|
+
typeof network?.user === 'string' ? network.user : 'the worker user'
|
|
225
|
+
);
|
|
226
|
+
return removes.length ? [{ node, removes }] : [];
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
176
230
|
export const planSources = (workspace: string, root: string): string[] => [
|
|
177
231
|
workspace,
|
|
178
232
|
join(root, 'source'),
|
|
@@ -196,5 +250,7 @@ export function planText(plan: SyncPlan): string {
|
|
|
196
250
|
);
|
|
197
251
|
for (const restart of plan.restarts)
|
|
198
252
|
lines.push(` Restarts ${restart.service}: ${restart.reason}`);
|
|
253
|
+
for (const removal of plan.removals ?? [])
|
|
254
|
+
lines.push(` Removes from ${removal.node}: ${removal.removes.join(', ')}`);
|
|
199
255
|
return lines.join('\n');
|
|
200
256
|
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import type { FleetDefinition, MachineDefinition } from '../definition';
|
|
2
|
+
import { readRosterText } from '../state/roster';
|
|
3
|
+
import { changeWorkers, readWorkerStates } from '../state/workers';
|
|
4
|
+
import { parseConfig } from '../config';
|
|
5
|
+
import type { Check, FleetNode, Runner } from '../types';
|
|
6
|
+
import {
|
|
7
|
+
lifecycleServices,
|
|
8
|
+
retireWorker,
|
|
9
|
+
type NodeLifecycleServices,
|
|
10
|
+
} from './fleet-node-lifecycle';
|
|
11
|
+
|
|
12
|
+
/** The workers of the fleet as worker records describe them come and go
|
|
13
|
+
* with the records: a record that appears is a declared worker, enrolled by
|
|
14
|
+
* the next sync like any other; one that disappears is retired by the same
|
|
15
|
+
* path as fleet-node-retire. fleet-sync calls this before it reads the
|
|
16
|
+
* roster. It records which workers came from records, so that only those
|
|
17
|
+
* are ever retired by themselves; a worker removed from fleet.json by hand
|
|
18
|
+
* must still be retired first.
|
|
19
|
+
*/
|
|
20
|
+
export async function reconcileRecords(
|
|
21
|
+
definition: FleetDefinition,
|
|
22
|
+
roster: string,
|
|
23
|
+
run: Runner,
|
|
24
|
+
{
|
|
25
|
+
apply,
|
|
26
|
+
services = lifecycleServices,
|
|
27
|
+
}: {
|
|
28
|
+
apply: boolean;
|
|
29
|
+
services?: (roster: string) => NodeLifecycleServices;
|
|
30
|
+
}
|
|
31
|
+
): Promise<Check[]> {
|
|
32
|
+
const states = readWorkerStates().workers;
|
|
33
|
+
const declared = new Set(definition.workers.map((worker) => worker.id));
|
|
34
|
+
const vanished = Object.entries(states)
|
|
35
|
+
.filter(([id, entry]) => entry.origin === 'records' && !declared.has(id))
|
|
36
|
+
.map(([id]) => id);
|
|
37
|
+
const entries = await rosterEntries(roster);
|
|
38
|
+
const lifecycle = { ...services(roster), run };
|
|
39
|
+
const checks: Check[] = [];
|
|
40
|
+
for (const id of vanished) {
|
|
41
|
+
const retired = await retireWorker(
|
|
42
|
+
gone(id, entries.get(id)),
|
|
43
|
+
entries.get(id),
|
|
44
|
+
lifecycle,
|
|
45
|
+
apply
|
|
46
|
+
);
|
|
47
|
+
const steps = (retired.data as { nodes: { checks: Check[] }[] }).nodes[0]!
|
|
48
|
+
.checks;
|
|
49
|
+
const failed = steps.filter((step) => step.status === 'fail');
|
|
50
|
+
checks.push(
|
|
51
|
+
failed.length
|
|
52
|
+
? {
|
|
53
|
+
id: 'enrollment:record',
|
|
54
|
+
status: 'fail',
|
|
55
|
+
summary: 'WORKER_RECORD_RETIREMENT_FAILED',
|
|
56
|
+
remedy: {
|
|
57
|
+
action: `${id}'s worker record disappeared, and retiring it failed at ${failed.map((step) => step.id).join(', ')}. Follow that remedy, then run fleet-sync again.`,
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
: {
|
|
61
|
+
id: 'enrollment:record',
|
|
62
|
+
status: apply ? 'pass' : 'skip',
|
|
63
|
+
summary: `${apply ? '' : 'Planned: '}${id}'s worker record disappeared, so it is retired: ${steps.map((step) => step.id).join(', ')}.`,
|
|
64
|
+
}
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
const fresh = definition.workers.filter(
|
|
68
|
+
(worker) => worker.origin === 'records' && !states[worker.id]?.origin
|
|
69
|
+
);
|
|
70
|
+
if (apply && fresh.length)
|
|
71
|
+
await changeWorkers(lifecycle.state(), (current) =>
|
|
72
|
+
Object.fromEntries(
|
|
73
|
+
fresh.map((worker) => [
|
|
74
|
+
worker.id,
|
|
75
|
+
{ ...current[worker.id], origin: 'records' as const },
|
|
76
|
+
])
|
|
77
|
+
)
|
|
78
|
+
);
|
|
79
|
+
return checks;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function rosterEntries(path: string): Promise<Map<string, FleetNode>> {
|
|
83
|
+
const text = await readRosterText(path).catch(() => null);
|
|
84
|
+
if (text === null) return new Map();
|
|
85
|
+
try {
|
|
86
|
+
return new Map(
|
|
87
|
+
parseConfig(JSON.parse(text) as unknown).nodes.map((node) => [
|
|
88
|
+
node.id,
|
|
89
|
+
node,
|
|
90
|
+
])
|
|
91
|
+
);
|
|
92
|
+
} catch {
|
|
93
|
+
return new Map();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** A worker whose record is gone, as far as retiring it needs: reached from
|
|
98
|
+
* declared data (so its pin is the fleet state's), under the name the roster
|
|
99
|
+
* last knew. Its user and ports are never used to retire it.
|
|
100
|
+
*/
|
|
101
|
+
function gone(id: string, entry: FleetNode | undefined): MachineDefinition {
|
|
102
|
+
return {
|
|
103
|
+
id,
|
|
104
|
+
os: 'linux',
|
|
105
|
+
origin: 'records',
|
|
106
|
+
network: {
|
|
107
|
+
tailscaleName: entry?.ssh ?? id,
|
|
108
|
+
user: 'root',
|
|
109
|
+
sshPort: 22,
|
|
110
|
+
t3Port: 3773,
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
@@ -2,7 +2,8 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { IDES } from '../ide';
|
|
3
3
|
import { enabled, workerChecks } from '../ide/orchestrate';
|
|
4
4
|
import { idesOf } from '../ide/definition';
|
|
5
|
-
import { rememberEnvironment } from '../t3/pairing';
|
|
5
|
+
import { pairIfRequested, rememberEnvironment } from '../t3/pairing';
|
|
6
|
+
import { managerT3 } from '../t3/peer-link';
|
|
6
7
|
import { authorizePrivilege } from './fleet-sync-privilege';
|
|
7
8
|
import { authorizeVault } from './fleet-sync-vault';
|
|
8
9
|
import { managerFailure } from '../manager-failure';
|
|
@@ -15,13 +16,32 @@ import { FleetUsageError } from '../errors';
|
|
|
15
16
|
import { selectNodes } from '../config';
|
|
16
17
|
import { loadRoster } from './roster';
|
|
17
18
|
import { enrolledRoster } from '../definition';
|
|
19
|
+
import { readParked, readWorkerStates } from '../state/workers';
|
|
20
|
+
import { withoutParked } from '../worker-parking';
|
|
21
|
+
import { reconcileRecords } from './fleet-sync-records';
|
|
18
22
|
import type { FleetDefinition } from '../definition';
|
|
19
23
|
import { fleet as definition } from '../fleet-config';
|
|
20
24
|
import { synchronizeManager } from '../manager-sync';
|
|
21
25
|
import { inspectBootstrap } from '../manager-sync';
|
|
22
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
prepareBootstrap,
|
|
28
|
+
bootstrapRemote,
|
|
29
|
+
readsTailnetKey,
|
|
30
|
+
} from '../remote-bootstrap';
|
|
31
|
+
import { secret } from '../fleet-config';
|
|
32
|
+
import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
|
|
33
|
+
import { discoverTailnet, type TailnetComputer } from '../tailscale';
|
|
34
|
+
import { deployWorker } from './fleet-sync-worker';
|
|
35
|
+
import { packagePlaybook } from '../platform/ansible';
|
|
23
36
|
import { machineRoot } from './fleet-sync';
|
|
24
|
-
import
|
|
37
|
+
import {
|
|
38
|
+
blocking,
|
|
39
|
+
type Check,
|
|
40
|
+
type FleetNode,
|
|
41
|
+
type NodeReport,
|
|
42
|
+
type Runner,
|
|
43
|
+
} from '../types';
|
|
44
|
+
import { diagnostic } from '../diagnostics';
|
|
25
45
|
import type { DeployNode, ManagerReport } from '../manager-sync';
|
|
26
46
|
import type { FleetSyncOptions } from './fleet-sync';
|
|
27
47
|
import type { Settings } from './settings';
|
|
@@ -39,14 +59,35 @@ export async function selected(
|
|
|
39
59
|
if (!options.revision)
|
|
40
60
|
throw new FleetUsageError('An approved source revision is required.');
|
|
41
61
|
const canonical = definition;
|
|
62
|
+
// Workers whose record disappeared are retired first (planned in a
|
|
63
|
+
// preview), so they are no longer in the roster.
|
|
64
|
+
const records = await reconcileRecords(canonical, settings.config, run, {
|
|
65
|
+
apply: Boolean(options.apply),
|
|
66
|
+
});
|
|
42
67
|
let roster;
|
|
43
68
|
try {
|
|
44
|
-
|
|
69
|
+
const declared = new Set(canonical.workers.map((worker) => worker.id));
|
|
70
|
+
const loaded = await loadRoster(settings.config);
|
|
71
|
+
roster = enrolledRoster(canonical, {
|
|
72
|
+
...loaded,
|
|
73
|
+
nodes: loaded.nodes.filter(
|
|
74
|
+
(node) =>
|
|
75
|
+
declared.has(node.id) ||
|
|
76
|
+
readWorkerStates().workers[node.id]?.origin !== 'records'
|
|
77
|
+
),
|
|
78
|
+
});
|
|
45
79
|
} catch {
|
|
46
80
|
return enrollmentRequired(canonical, options.revision);
|
|
47
81
|
}
|
|
48
|
-
|
|
49
|
-
const
|
|
82
|
+
// A parked worker is not contacted and never the canary.
|
|
83
|
+
const active = withoutParked(
|
|
84
|
+
scopeDefinition(canonical, options.node),
|
|
85
|
+
selectNodes(roster, options.node ?? []),
|
|
86
|
+
readParked(),
|
|
87
|
+
new Date()
|
|
88
|
+
);
|
|
89
|
+
const { nodes, definition: scoped } = active;
|
|
90
|
+
const notes = [...records, ...active.checks];
|
|
50
91
|
const ui = interfaceOptions(options);
|
|
51
92
|
const planFingerprint = createHash('sha256')
|
|
52
93
|
.update(
|
|
@@ -65,12 +106,16 @@ export async function selected(
|
|
|
65
106
|
'Manager connection bindings or target policy changed after approval. Inspect a fresh plan and approve it; no source preparation or node mutation occurred.'
|
|
66
107
|
)
|
|
67
108
|
);
|
|
68
|
-
const
|
|
109
|
+
const prepared = options.apply
|
|
69
110
|
? await prepareBootstrap(machineRoot(), options.revision, run, {
|
|
70
111
|
definition: canonical,
|
|
71
112
|
checkout: settings.workspace,
|
|
72
113
|
})
|
|
73
114
|
: null;
|
|
115
|
+
const payload =
|
|
116
|
+
prepared && options.restartBusy
|
|
117
|
+
? { ...prepared, restartBusy: true }
|
|
118
|
+
: prepared;
|
|
74
119
|
const started = Date.now();
|
|
75
120
|
const report = await synchronizeManager(
|
|
76
121
|
{
|
|
@@ -79,6 +124,7 @@ export async function selected(
|
|
|
79
124
|
revision: options.revision,
|
|
80
125
|
apply: Boolean(options.apply),
|
|
81
126
|
...canary(canonical, options),
|
|
127
|
+
...serial(options),
|
|
82
128
|
progress: (node, phase) => ui.progress.step(node, phase),
|
|
83
129
|
finished: (node) => ui.progress.finished(node),
|
|
84
130
|
manager: () =>
|
|
@@ -97,22 +143,74 @@ export async function selected(
|
|
|
97
143
|
...ui,
|
|
98
144
|
workspace: settings.workspace,
|
|
99
145
|
roster: settings.config,
|
|
146
|
+
...rebootOptions(options),
|
|
100
147
|
})
|
|
101
148
|
).finally(() => ui.progress.stop());
|
|
149
|
+
if (options.enrollment) reportEnrollment(report, options.enrollment);
|
|
150
|
+
report.checks.push(...notes);
|
|
151
|
+
// A worker that could not be retired, like any other failed check of the
|
|
152
|
+
// fleet, means the run did not converge: it is not recorded as applied.
|
|
153
|
+
if (notes.some(blocking)) report.ok = false;
|
|
102
154
|
if (settle) report.checks.push(await settle(report));
|
|
103
155
|
return outcome(report, planFingerprint, started);
|
|
104
156
|
}
|
|
105
157
|
|
|
106
|
-
/**
|
|
107
|
-
*
|
|
108
|
-
|
|
158
|
+
/** Maintenance reboots go one worker at a time, and a worker that does not
|
|
159
|
+
* come back ready stops the others.
|
|
160
|
+
*/
|
|
161
|
+
function serial(options: FleetSyncOptions): { serial?: true } {
|
|
162
|
+
return options.maintenanceReboot ? { serial: true } : {};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function rebootOptions(options: FleetSyncOptions): {
|
|
166
|
+
readyBefore: string[];
|
|
167
|
+
maintenanceReboot?: true;
|
|
168
|
+
force?: true;
|
|
169
|
+
} {
|
|
170
|
+
return {
|
|
171
|
+
readyBefore: options.readyBefore ?? [],
|
|
172
|
+
...(options.maintenanceReboot ? { maintenanceReboot: true } : {}),
|
|
173
|
+
...(options.force ? { force: true } : {}),
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The workers this run enrolled, and why another was not: the checks land
|
|
178
|
+
* on the worker they are about.
|
|
179
|
+
*/
|
|
180
|
+
export function reportEnrollment(
|
|
181
|
+
report: ManagerReport,
|
|
182
|
+
enrollment: NonNullable<FleetSyncOptions['enrollment']>
|
|
183
|
+
): void {
|
|
184
|
+
report.enrolled = enrollment.enrolled;
|
|
185
|
+
for (const { node, check } of enrollment.checks) {
|
|
186
|
+
const entry = report.nodes.find((candidate) => candidate.node === node);
|
|
187
|
+
if (!entry) continue;
|
|
188
|
+
entry.checks.unshift(check);
|
|
189
|
+
entry.diagnostics = entry.checks.map((each) => diagnostic(node, each));
|
|
190
|
+
if (blocking(check)) report.ok = false;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** A revision new to the fleet is proven on one worker before the rest: the
|
|
195
|
+
* one `--canary` or fleet.json names, else the first declared worker, and
|
|
196
|
+
* the next reachable one when that cannot be reached. Repeating a revision
|
|
197
|
+
* that already converged everywhere needs no canary, so every worker
|
|
198
|
+
* converges at once.
|
|
109
199
|
*/
|
|
110
200
|
function canary(
|
|
111
201
|
canonical: FleetDefinition,
|
|
112
202
|
options: FleetSyncOptions
|
|
113
203
|
): { canary?: string } {
|
|
114
|
-
|
|
115
|
-
|
|
204
|
+
if (
|
|
205
|
+
options.canary &&
|
|
206
|
+
!canonical.workers.some((worker) => worker.id === options.canary)
|
|
207
|
+
)
|
|
208
|
+
throw new FleetUsageError(
|
|
209
|
+
'--canary names a worker fleet.json does not declare.'
|
|
210
|
+
);
|
|
211
|
+
const preferred =
|
|
212
|
+
options.canary ?? canonical.canary ?? canonical.workers[0]?.id;
|
|
213
|
+
return preferred && !options.rolledOut ? { canary: preferred } : {};
|
|
116
214
|
}
|
|
117
215
|
|
|
118
216
|
/** Saves the full JSON report, then renders the short human summary. */
|
|
@@ -165,8 +263,16 @@ function deployWithMonitoring(
|
|
|
165
263
|
/** The manager's checkout, and the roster an identity is recorded in. */
|
|
166
264
|
workspace: string;
|
|
167
265
|
roster: string;
|
|
266
|
+
passwordLogin: boolean;
|
|
267
|
+
readyBefore: string[];
|
|
268
|
+
maintenanceReboot?: boolean;
|
|
269
|
+
force?: boolean;
|
|
168
270
|
}
|
|
169
271
|
): DeployNode {
|
|
272
|
+
// Asked once per run, and only when a worker needs them.
|
|
273
|
+
let control: Promise<TailnetControl | null> | undefined;
|
|
274
|
+
let computers: Promise<TailnetComputer[]> | undefined;
|
|
275
|
+
const converge = convergeWorker(payload, canonical, run, ui);
|
|
170
276
|
return async (node, revision) => {
|
|
171
277
|
if (!payload || payload.revision !== revision)
|
|
172
278
|
return {
|
|
@@ -183,18 +289,62 @@ function deployWithMonitoring(
|
|
|
183
289
|
},
|
|
184
290
|
],
|
|
185
291
|
};
|
|
292
|
+
return deployWorker(node, revision, {
|
|
293
|
+
definition: canonical,
|
|
294
|
+
playbook: packagePlaybook(run),
|
|
295
|
+
converge,
|
|
296
|
+
control: () =>
|
|
297
|
+
(control ??= fleetTailnetControl(canonical.tailnet, secret)),
|
|
298
|
+
computers: () =>
|
|
299
|
+
(computers ??= discoverTailnet(run).then((found) => found.computers)),
|
|
300
|
+
readyBefore: ui.readyBefore,
|
|
301
|
+
...(ui.maintenanceReboot ? { maintenanceReboot: true } : {}),
|
|
302
|
+
...(ui.force ? { force: true } : {}),
|
|
303
|
+
acceptsTailnetKey: readsTailnetKey(payload.script),
|
|
304
|
+
});
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** One converge of a worker: privilege, Vault, the bootstrap, then what is
|
|
309
|
+
* verified from the manager and the worker's monitoring.
|
|
310
|
+
*/
|
|
311
|
+
function convergeWorker(
|
|
312
|
+
payload: Awaited<ReturnType<typeof prepareBootstrap>>,
|
|
313
|
+
canonical: FleetDefinition,
|
|
314
|
+
run: Runner,
|
|
315
|
+
ui: {
|
|
316
|
+
progress: SyncProgress;
|
|
317
|
+
interactiveAuthorization: boolean;
|
|
318
|
+
workspace: string;
|
|
319
|
+
roster: string;
|
|
320
|
+
passwordLogin: boolean;
|
|
321
|
+
}
|
|
322
|
+
) {
|
|
323
|
+
return async (
|
|
324
|
+
node: FleetNode,
|
|
325
|
+
revision: string,
|
|
326
|
+
tailnetKey?: string
|
|
327
|
+
): Promise<NodeReport> => {
|
|
186
328
|
// Password prompts need a quiet terminal: no spinner while authorizing.
|
|
187
329
|
if (ui.interactiveAuthorization) {
|
|
188
330
|
ui.progress.stop();
|
|
189
|
-
const checks = await authorizePrivilege(
|
|
331
|
+
const checks = await authorizePrivilege(
|
|
332
|
+
node,
|
|
333
|
+
payload!.authorizeSudo,
|
|
334
|
+
run
|
|
335
|
+
);
|
|
190
336
|
if (checks.length) return { node: node.id, checks };
|
|
191
337
|
}
|
|
192
|
-
const vault = {
|
|
338
|
+
const vault = {
|
|
339
|
+
interactive: ui.interactiveAuthorization,
|
|
340
|
+
passwordLogin: ui.passwordLogin,
|
|
341
|
+
};
|
|
193
342
|
if (ui.interactiveAuthorization) ui.progress.stop();
|
|
194
343
|
const before = await authorizeVault(node, vault, run);
|
|
195
344
|
ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
|
|
196
|
-
const result = await bootstrapRemote(node, payload
|
|
345
|
+
const result = await bootstrapRemote(node, payload!, run, {
|
|
197
346
|
progress: (line) => ui.progress.step(node.id, line),
|
|
347
|
+
...(tailnetKey ? { tailnetKey } : {}),
|
|
198
348
|
});
|
|
199
349
|
// A fresh node gets vault from the bootstrap; log it in now. Its provider
|
|
200
350
|
// checks in this run predate the login, so the next sync verifies them.
|
|
@@ -212,6 +362,12 @@ function deployWithMonitoring(
|
|
|
212
362
|
remember: async (id, identity) => {
|
|
213
363
|
await rememberEnvironment(ui.roster, id, identity);
|
|
214
364
|
},
|
|
365
|
+
pair: (target) =>
|
|
366
|
+
pairIfRequested(target, {
|
|
367
|
+
roster: ui.roster,
|
|
368
|
+
run,
|
|
369
|
+
manager: managerT3(run),
|
|
370
|
+
}),
|
|
215
371
|
}))
|
|
216
372
|
);
|
|
217
373
|
if (
|
|
@@ -242,6 +398,7 @@ function scopeDefinition(canonical: FleetDefinition, ids?: string[]) {
|
|
|
242
398
|
function interfaceOptions(options: FleetSyncOptions) {
|
|
243
399
|
return {
|
|
244
400
|
progress: syncProgress(!options.json),
|
|
401
|
+
passwordLogin: Boolean(options.vaultPasswordLogin),
|
|
245
402
|
interactiveAuthorization: Boolean(
|
|
246
403
|
options.interactiveAuthorization &&
|
|
247
404
|
!options.json &&
|
|
@@ -9,13 +9,19 @@ import {
|
|
|
9
9
|
recordSchedule,
|
|
10
10
|
type ScheduleRecord,
|
|
11
11
|
} from '../state/history';
|
|
12
|
-
import
|
|
12
|
+
import macos from '../../ansible/macos/policy.json';
|
|
13
|
+
import { fleet } from '../fleet-config';
|
|
14
|
+
import {
|
|
15
|
+
scheduleCheck,
|
|
16
|
+
scheduleIntervals,
|
|
17
|
+
SCHEDULE_INTERVAL_SECONDS,
|
|
18
|
+
} from '../schedule-check';
|
|
13
19
|
import type { Runner } from '../types';
|
|
14
20
|
import type { CommandOutcome } from './contracts';
|
|
15
21
|
import { machineRoot } from './fleet-sync';
|
|
16
22
|
|
|
17
23
|
/** The launchd label and systemd unit already on running managers. */
|
|
18
|
-
export const SCHEDULE_LABEL =
|
|
24
|
+
export const SCHEDULE_LABEL = macos.fleetSync.label;
|
|
19
25
|
const LABEL = SCHEDULE_LABEL;
|
|
20
26
|
export type ScheduleAction =
|
|
21
27
|
'enable' | 'pause' | 'resume' | 'disable' | 'status';
|
|
@@ -31,6 +37,8 @@ interface ScheduleInput {
|
|
|
31
37
|
/** The manager's caches; tests name their own. */
|
|
32
38
|
root?: string;
|
|
33
39
|
now?: Date;
|
|
40
|
+
/** How often the schedule runs; `manager.schedule.syncMinutes` otherwise. */
|
|
41
|
+
intervalSeconds?: number;
|
|
34
42
|
}
|
|
35
43
|
|
|
36
44
|
/** The published package a schedule installs and runs: its name, exact
|
|
@@ -205,14 +213,16 @@ async function enableSchedule(
|
|
|
205
213
|
return required(
|
|
206
214
|
'The exact CLI package could not be installed in the managed manager runtime. No schedule was enabled.'
|
|
207
215
|
);
|
|
216
|
+
const intervalSeconds =
|
|
217
|
+
input.intervalSeconds ?? scheduleIntervals(fleet.manager).syncSeconds;
|
|
208
218
|
const result = platform.installScheduledJob({
|
|
209
219
|
label: LABEL,
|
|
210
|
-
intervalSeconds
|
|
220
|
+
intervalSeconds,
|
|
211
221
|
command: scheduledCommand(published, input.revision, prefix),
|
|
212
222
|
logDir: join(root, 'logs'),
|
|
213
223
|
env: scheduledEnvironment(),
|
|
214
224
|
});
|
|
215
|
-
if (result.ok) remember(input, true);
|
|
225
|
+
if (result.ok) remember(input, true, intervalSeconds);
|
|
216
226
|
return {
|
|
217
227
|
exitCode: result.ok ? 0 : 1,
|
|
218
228
|
data: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Enrollment } from '../machine/state';
|
|
2
2
|
import { replicaCheck } from '../state/check';
|
|
3
|
-
import { storeEnrollment } from '../state/enrollment';
|
|
3
|
+
import { loadEnrollment, storeEnrollment } from '../state/enrollment';
|
|
4
4
|
import { adoptEarlier } from '../state/legacy';
|
|
5
5
|
import { databasePath, stateRoot } from '../state/paths';
|
|
6
6
|
import { writeDocument } from '../state/store';
|
|
@@ -21,8 +21,12 @@ import type { Check } from '../types';
|
|
|
21
21
|
export interface RunRecord {
|
|
22
22
|
/** Before any machine changes: this revision is the approved one. */
|
|
23
23
|
approve(): void;
|
|
24
|
-
/** Once the result is known. The check says whether the bucket holds it.
|
|
25
|
-
|
|
24
|
+
/** Once the result is known. The check says whether the bucket holds it.
|
|
25
|
+
* `applied` is the revision each worker that converged now holds: it is
|
|
26
|
+
* kept even when the fleet as a whole is not ready, and it is all a run
|
|
27
|
+
* scoped with --node records.
|
|
28
|
+
*/
|
|
29
|
+
settle(ready: boolean, applied?: Record<string, string>): Promise<Check>;
|
|
26
30
|
/** For a run that stopped before its result: its approval is still uploaded. */
|
|
27
31
|
abandon(): Promise<void>;
|
|
28
32
|
}
|
|
@@ -39,6 +43,10 @@ export function runRecord(
|
|
|
39
43
|
* worker, a replacement manager) uses the one last applied.
|
|
40
44
|
*/
|
|
41
45
|
definition?: string;
|
|
46
|
+
/** A run scoped with --node covers only the workers it selected: it
|
|
47
|
+
* records their revisions, never one applied for the whole fleet.
|
|
48
|
+
*/
|
|
49
|
+
scoped?: boolean;
|
|
42
50
|
}
|
|
43
51
|
): RunRecord {
|
|
44
52
|
const env = state.env ?? process.env;
|
|
@@ -70,20 +78,33 @@ export function runRecord(
|
|
|
70
78
|
}
|
|
71
79
|
claimLease(state.context);
|
|
72
80
|
},
|
|
73
|
-
async settle(
|
|
81
|
+
async settle(ready, applied = {}) {
|
|
74
82
|
settled = true;
|
|
75
|
-
|
|
83
|
+
const converged = ready && state.scoped !== true;
|
|
84
|
+
if (converged || Object.keys(applied).length) {
|
|
85
|
+
// What was recorded since the approval stands: the run itself
|
|
86
|
+
// retires a worker whose record was removed, and another command
|
|
87
|
+
// may have. Only what this run decided is written over it.
|
|
88
|
+
const stored = (await loadEnrollment(null, env)) ?? approval;
|
|
76
89
|
storeEnrollment(
|
|
77
90
|
{
|
|
78
|
-
...
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
91
|
+
...stored,
|
|
92
|
+
...(Object.keys(applied).length
|
|
93
|
+
? { workers: { ...stored.workers, ...applied } }
|
|
94
|
+
: {}),
|
|
95
|
+
...(converged
|
|
96
|
+
? {
|
|
97
|
+
appliedRevision: revision,
|
|
98
|
+
...(previous?.appliedRevision &&
|
|
99
|
+
previous.appliedRevision !== revision
|
|
100
|
+
? { previousRevision: previous.appliedRevision }
|
|
101
|
+
: {}),
|
|
102
|
+
}
|
|
83
103
|
: {}),
|
|
84
104
|
},
|
|
85
105
|
env
|
|
86
106
|
);
|
|
107
|
+
}
|
|
87
108
|
// The fleet converged either way: a bucket problem is reported, not fatal.
|
|
88
109
|
return replicaCheck(await publish().catch(() => 'unreachable' as const), {
|
|
89
110
|
advisory: true,
|