@crvouga/dev-fleet 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -9
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +55 -9
- package/ansible/macos/manager.yml +154 -7
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +7 -3
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +19 -11
- package/docs/consumer-setup.md +42 -27
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +10 -3
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +98 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +67 -6
- package/docs/ubuntu-sync.md +78 -28
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1967 -170
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +327 -8
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +61 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +9 -3
- package/src/cli/fleet-node-lifecycle.ts +313 -64
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +51 -35
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +4 -0
- package/src/cli/fleet-sync-manager.ts +103 -33
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +151 -9
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +28 -16
- package/src/cli/fleet-sync-vault.ts +77 -0
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +150 -19
- package/src/cli/fleet-validate.ts +59 -21
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +115 -4
- package/src/cli/registry.ts +14 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +332 -9
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +37 -15
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +65 -4
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +4 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capability-checks.ts +55 -16
- package/src/worker-capacity.ts +31 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +27 -4
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +17 -1
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +16 -11
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import type { FleetDefinition, MachineDefinition } from '../definition';
|
|
2
|
+
import { FleetUsageError } from '../errors';
|
|
3
|
+
import { fleet } from '../fleet-config';
|
|
4
|
+
import { makeReport, renderReport } from '../report';
|
|
5
|
+
import { runCommand } from '../runner';
|
|
6
|
+
import { changeWorkers, readWorkerStates } from '../state/workers';
|
|
7
|
+
import { stateContext, type StateContext } from '../state/sync';
|
|
8
|
+
import type { Check, Runner } from '../types';
|
|
9
|
+
import { duration, workerActivity } from '../worker-activity';
|
|
10
|
+
import type { CommandOutcome } from './contracts';
|
|
11
|
+
import { readRoster } from './roster';
|
|
12
|
+
|
|
13
|
+
export interface NodeParkOptions {
|
|
14
|
+
id?: string;
|
|
15
|
+
/** Park although agent turns are under way, or cannot be counted. */
|
|
16
|
+
force?: boolean;
|
|
17
|
+
dryRun?: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface NodeParkServices {
|
|
21
|
+
run: Runner;
|
|
22
|
+
/** The roster: the fleet state's, or a file `--config` names. */
|
|
23
|
+
roster: string;
|
|
24
|
+
definition: () => FleetDefinition;
|
|
25
|
+
state: () => StateContext;
|
|
26
|
+
now: () => Date;
|
|
27
|
+
/** A normal fleet-sync of one worker, which verifies it. */
|
|
28
|
+
sync: (id: string) => Promise<CommandOutcome>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function parkServices(
|
|
32
|
+
roster: string,
|
|
33
|
+
sync: (id: string) => Promise<CommandOutcome>
|
|
34
|
+
): NodeParkServices {
|
|
35
|
+
return {
|
|
36
|
+
run: runCommand,
|
|
37
|
+
roster,
|
|
38
|
+
definition: () => fleet,
|
|
39
|
+
state: () => stateContext(runCommand),
|
|
40
|
+
now: () => new Date(),
|
|
41
|
+
sync,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A hook may start or stop a machine through an infrastructure tool; it
|
|
46
|
+
* gets half an hour.
|
|
47
|
+
*/
|
|
48
|
+
const HOOK_TIMEOUT_MS = 1_800_000;
|
|
49
|
+
|
|
50
|
+
function declared(definition: FleetDefinition, id: string | undefined) {
|
|
51
|
+
const worker = definition.workers.find((entry) => entry.id === id);
|
|
52
|
+
if (!worker)
|
|
53
|
+
throw new FleetUsageError('--id must name a worker fleet.json declares.');
|
|
54
|
+
return worker;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function outcome(
|
|
58
|
+
command: string,
|
|
59
|
+
id: string,
|
|
60
|
+
checks: Check[],
|
|
61
|
+
extra: Record<string, unknown> = {}
|
|
62
|
+
): CommandOutcome {
|
|
63
|
+
const report = { ...makeReport(command, [{ node: id, checks }]), ...extra };
|
|
64
|
+
return {
|
|
65
|
+
exitCode: checks.some((check) => check.status === 'fail') ? 1 : 0,
|
|
66
|
+
data: report,
|
|
67
|
+
text: renderReport(report),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Runs the declared stop or start command with the worker ID last. */
|
|
72
|
+
async function hook(
|
|
73
|
+
worker: MachineDefinition,
|
|
74
|
+
which: 'stop' | 'start',
|
|
75
|
+
definition: FleetDefinition,
|
|
76
|
+
run: Runner
|
|
77
|
+
): Promise<Check | null> {
|
|
78
|
+
const command = definition.parking?.[which];
|
|
79
|
+
if (!command) return null;
|
|
80
|
+
const result = await run([...command, worker.id], {
|
|
81
|
+
timeoutMs: HOOK_TIMEOUT_MS,
|
|
82
|
+
});
|
|
83
|
+
return result.ok
|
|
84
|
+
? {
|
|
85
|
+
id: 'worker:park-hook',
|
|
86
|
+
status: 'pass',
|
|
87
|
+
summary: `The declared ${which} command ran for ${worker.id}.`,
|
|
88
|
+
}
|
|
89
|
+
: {
|
|
90
|
+
id: 'worker:park-hook',
|
|
91
|
+
status: 'fail',
|
|
92
|
+
summary:
|
|
93
|
+
which === 'stop'
|
|
94
|
+
? 'WORKER_STOP_HOOK_FAILED'
|
|
95
|
+
: 'WORKER_START_HOOK_FAILED',
|
|
96
|
+
remedy: {
|
|
97
|
+
action: `The ${which} command parking.${which} in fleet.json names failed for ${worker.id}, so its state was not changed. Run it by hand to see why, then run this command again.`,
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Why a worker must not be parked now, unless forced: agent turns under
|
|
103
|
+
* way, or a worker that answers but whose turns cannot be counted. A worker
|
|
104
|
+
* that does not answer is off already.
|
|
105
|
+
*/
|
|
106
|
+
async function busy(
|
|
107
|
+
worker: MachineDefinition,
|
|
108
|
+
services: NodeParkServices
|
|
109
|
+
): Promise<Check | null> {
|
|
110
|
+
const node = (await readRoster(services.roster))?.nodes.find(
|
|
111
|
+
(entry) => entry.id === worker.id
|
|
112
|
+
);
|
|
113
|
+
const activity = node ? await workerActivity(node, services.run) : null;
|
|
114
|
+
if (!activity || activity.runningSessions === 0) return null;
|
|
115
|
+
return {
|
|
116
|
+
id: 'worker:parked',
|
|
117
|
+
status: 'fail',
|
|
118
|
+
summary:
|
|
119
|
+
activity.runningSessions === null
|
|
120
|
+
? 'WORKER_ACTIVITY_UNREADABLE'
|
|
121
|
+
: 'WORKER_BUSY',
|
|
122
|
+
remedy: {
|
|
123
|
+
action:
|
|
124
|
+
activity.runningSessions === null
|
|
125
|
+
? `${worker.id}'s T3 could not say whether agent turns are under way, so it was not parked. Look at it, then repeat with --force to park it anyway.`
|
|
126
|
+
: `${worker.id} runs ${activity.runningSessions} agent turn(s); stopping it would end them. Wait for them to finish, or repeat with --force.`,
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Parks a worker: it is stopped or destroyed on purpose, so it is no longer
|
|
132
|
+
* contacted, never the canary and does not fail the fleet. Refuses while
|
|
133
|
+
* agent turns run on it, unless --force. Runs the declared stop command.
|
|
134
|
+
*/
|
|
135
|
+
export async function fleetNodePark(
|
|
136
|
+
options: NodeParkOptions,
|
|
137
|
+
services: NodeParkServices
|
|
138
|
+
): Promise<CommandOutcome> {
|
|
139
|
+
const definition = services.definition();
|
|
140
|
+
const worker = declared(definition, options.id);
|
|
141
|
+
const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
|
|
142
|
+
if (parkedAt)
|
|
143
|
+
return outcome('fleet-node-park', worker.id, [
|
|
144
|
+
{
|
|
145
|
+
id: 'worker:parked',
|
|
146
|
+
status: 'pass',
|
|
147
|
+
summary: `${worker.id} is parked since ${parkedAt}.`,
|
|
148
|
+
},
|
|
149
|
+
]);
|
|
150
|
+
if (!options.force) {
|
|
151
|
+
const refused = await busy(worker, services);
|
|
152
|
+
if (refused) return outcome('fleet-node-park', worker.id, [refused]);
|
|
153
|
+
}
|
|
154
|
+
if (options.dryRun)
|
|
155
|
+
return outcome('fleet-node-park', worker.id, [
|
|
156
|
+
{
|
|
157
|
+
id: 'worker:parked',
|
|
158
|
+
status: 'skip',
|
|
159
|
+
summary: `Planned: ${definition.parking?.stop ? 'run the declared stop command and ' : ''}park ${worker.id}.`,
|
|
160
|
+
},
|
|
161
|
+
]);
|
|
162
|
+
const stopped = await hook(worker, 'stop', definition, services.run);
|
|
163
|
+
if (stopped?.status === 'fail')
|
|
164
|
+
return outcome('fleet-node-park', worker.id, [stopped]);
|
|
165
|
+
const at = services.now().toISOString();
|
|
166
|
+
await changeWorkers(services.state(), (current) => ({
|
|
167
|
+
[worker.id]: { ...current[worker.id], parkedAt: at },
|
|
168
|
+
}));
|
|
169
|
+
return outcome('fleet-node-park', worker.id, [
|
|
170
|
+
...(stopped ? [stopped] : []),
|
|
171
|
+
{
|
|
172
|
+
id: 'worker:parked',
|
|
173
|
+
status: 'pass',
|
|
174
|
+
summary: `${worker.id} is parked since ${at}: fleet-sync and fleet-status no longer contact it.`,
|
|
175
|
+
},
|
|
176
|
+
]);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Brings a parked worker back: runs the declared start command, marks it
|
|
180
|
+
* active, then runs a normal fleet-sync of that worker, which verifies it.
|
|
181
|
+
* The exit code is the sync's.
|
|
182
|
+
*/
|
|
183
|
+
export async function fleetNodeResume(
|
|
184
|
+
options: NodeParkOptions,
|
|
185
|
+
services: NodeParkServices
|
|
186
|
+
): Promise<CommandOutcome> {
|
|
187
|
+
const definition = services.definition();
|
|
188
|
+
const worker = declared(definition, options.id);
|
|
189
|
+
const parkedAt = readWorkerStates().workers[worker.id]?.parkedAt;
|
|
190
|
+
if (options.dryRun)
|
|
191
|
+
return outcome('fleet-node-resume', worker.id, [
|
|
192
|
+
{
|
|
193
|
+
id: 'worker:parked',
|
|
194
|
+
status: 'skip',
|
|
195
|
+
summary: parkedAt
|
|
196
|
+
? `Planned: ${definition.parking?.start ? 'run the declared start command, ' : ''}mark ${worker.id} active and run fleet-sync --node ${worker.id}.`
|
|
197
|
+
: `Planned: ${worker.id} is active already; run fleet-sync --node ${worker.id}.`,
|
|
198
|
+
},
|
|
199
|
+
]);
|
|
200
|
+
const checks: Check[] = [];
|
|
201
|
+
if (parkedAt) {
|
|
202
|
+
const started = await hook(worker, 'start', definition, services.run);
|
|
203
|
+
if (started?.status === 'fail')
|
|
204
|
+
return outcome('fleet-node-resume', worker.id, [started]);
|
|
205
|
+
if (started) checks.push(started);
|
|
206
|
+
await changeWorkers(services.state(), (current) => {
|
|
207
|
+
const { parkedAt: _parked, ...rest } = current[worker.id] ?? {};
|
|
208
|
+
return { [worker.id]: rest };
|
|
209
|
+
});
|
|
210
|
+
const since = (services.now().getTime() - Date.parse(parkedAt)) / 1000;
|
|
211
|
+
checks.push({
|
|
212
|
+
id: 'worker:parked',
|
|
213
|
+
status: 'pass',
|
|
214
|
+
summary: `${worker.id} is active again after ${duration(Math.max(0, since))} parked.`,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const synced = await services.sync(worker.id);
|
|
218
|
+
const report = synced.data as { ok?: boolean };
|
|
219
|
+
checks.push(
|
|
220
|
+
synced.exitCode === 0 && report.ok !== false
|
|
221
|
+
? {
|
|
222
|
+
id: 'rollout:node',
|
|
223
|
+
status: 'pass',
|
|
224
|
+
summary: `fleet-sync --node ${worker.id} converged and verified it.`,
|
|
225
|
+
}
|
|
226
|
+
: {
|
|
227
|
+
id: 'rollout:node',
|
|
228
|
+
status: 'fail',
|
|
229
|
+
summary: 'WORKER_RESUME_FAILED',
|
|
230
|
+
remedy: {
|
|
231
|
+
action: `fleet-sync --node ${worker.id} did not end ready; its report (sync in this output) says why. Fix that, then run devfleet fleet-sync --node ${worker.id}.`,
|
|
232
|
+
},
|
|
233
|
+
}
|
|
234
|
+
);
|
|
235
|
+
return outcome('fleet-node-resume', worker.id, checks, {
|
|
236
|
+
sync: synced.data,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
@@ -2,21 +2,24 @@ import { readFile, writeFile } from 'node:fs/promises';
|
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import type { FleetDefinition } from '../definition';
|
|
5
|
-
import { ubuntuRelease } from '../platform/ubuntu-release';
|
|
5
|
+
import { tailscalePins, ubuntuRelease } from '../platform/ubuntu-release';
|
|
6
6
|
import { FleetUsageError } from '../errors';
|
|
7
7
|
import { fleet, secret } from '../fleet-config';
|
|
8
8
|
import type { SecretsProvider } from '../secrets';
|
|
9
|
-
import type { Check } from '../types';
|
|
10
9
|
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
10
|
+
controlRemedy,
|
|
11
|
+
fleetTailnetControl,
|
|
12
|
+
loginServer,
|
|
13
|
+
TailnetControlError,
|
|
14
|
+
type ControlHttp,
|
|
15
|
+
} from '../tailnet-control';
|
|
16
|
+
import type { Check } from '../types';
|
|
17
|
+
import { generateHostKey, renderSeed, seedUserData } from '../worker-seed';
|
|
16
18
|
import type { CommandOutcome } from './contracts';
|
|
17
19
|
import {
|
|
18
20
|
keyName,
|
|
19
21
|
lifecycleServices,
|
|
22
|
+
pinHostKey,
|
|
20
23
|
type NodeLifecycleServices,
|
|
21
24
|
} from './fleet-node-lifecycle';
|
|
22
25
|
|
|
@@ -31,19 +34,23 @@ export interface NodeSeedOptions {
|
|
|
31
34
|
|
|
32
35
|
export interface NodeSeedServices extends Pick<
|
|
33
36
|
NodeLifecycleServices,
|
|
34
|
-
'run' | 'playbook' | 'knownHosts'
|
|
37
|
+
'run' | 'playbook' | 'knownHosts' | 'state' | 'now'
|
|
35
38
|
> {
|
|
36
39
|
definition: () => FleetDefinition;
|
|
37
40
|
secret: SecretsProvider;
|
|
38
41
|
home: string;
|
|
42
|
+
/** The tailnet control plane's HTTP; answered in scenarios. */
|
|
43
|
+
http?: ControlHttp;
|
|
39
44
|
}
|
|
40
45
|
|
|
41
46
|
export function seedServices(roster: string): NodeSeedServices {
|
|
42
|
-
const { run, playbook, knownHosts } = lifecycleServices(roster);
|
|
47
|
+
const { run, playbook, knownHosts, state, now } = lifecycleServices(roster);
|
|
43
48
|
return {
|
|
44
49
|
run,
|
|
45
50
|
playbook,
|
|
46
51
|
knownHosts,
|
|
52
|
+
state,
|
|
53
|
+
now,
|
|
47
54
|
definition: () => fleet,
|
|
48
55
|
secret,
|
|
49
56
|
home: homedir(),
|
|
@@ -75,11 +82,8 @@ export async function fleetNodeSeed(
|
|
|
75
82
|
`No manager public key at ${keyFile}; name one with --public-key PATH.`
|
|
76
83
|
);
|
|
77
84
|
});
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
throw new FleetUsageError(
|
|
81
|
-
'The secrets provider holds no tailnet-auth-key. Store a short-lived, single-use, tagged Tailscale auth key there first.'
|
|
82
|
-
);
|
|
85
|
+
const tailnet = services.definition().tailnet;
|
|
86
|
+
const authKey = await seedAuthKey(worker.id, services);
|
|
83
87
|
const release = ubuntuRelease(services.definition().ubuntu.release);
|
|
84
88
|
if (!release)
|
|
85
89
|
throw new FleetUsageError(
|
|
@@ -92,36 +96,20 @@ export async function fleetNodeSeed(
|
|
|
92
96
|
managerKey,
|
|
93
97
|
authKey,
|
|
94
98
|
hostKey,
|
|
95
|
-
tailscale:
|
|
99
|
+
tailscale: tailscalePins(release),
|
|
96
100
|
codename: release.codename,
|
|
101
|
+
...(loginServer(tailnet) ? { loginServer: loginServer(tailnet)! } : {}),
|
|
97
102
|
}),
|
|
98
103
|
Boolean(options.autoinstall)
|
|
99
104
|
);
|
|
100
105
|
await writeFile(options.output, seed, { mode: 0o600 });
|
|
101
106
|
const host = await keyName(worker, services.run);
|
|
102
107
|
const publicKey = hostKey.public.split(' ').slice(0, 2).join(' ');
|
|
108
|
+
// Pinned before the machine answers: in the fleet state for a worker
|
|
109
|
+
// reached from declared data, else in the operator's known_hosts.
|
|
103
110
|
const pin: Check =
|
|
104
111
|
options.apply && options.yes
|
|
105
|
-
?
|
|
106
|
-
host,
|
|
107
|
-
pin: 'present',
|
|
108
|
-
key: publicKey,
|
|
109
|
-
known_hosts: services.knownHosts,
|
|
110
|
-
}))
|
|
111
|
-
? {
|
|
112
|
-
id: 'ssh:host-key',
|
|
113
|
-
status: 'pass',
|
|
114
|
-
summary: `The manager trusts the seed's host key for ${host}.`,
|
|
115
|
-
}
|
|
116
|
-
: {
|
|
117
|
-
id: 'ssh:host-key',
|
|
118
|
-
status: 'fail',
|
|
119
|
-
summary: 'SSH_HOST_KEY_NOT_CHANGED',
|
|
120
|
-
remedy: {
|
|
121
|
-
action:
|
|
122
|
-
'The pinned Ansible could not change known_hosts; run the command again.',
|
|
123
|
-
},
|
|
124
|
-
}
|
|
112
|
+
? await pinHostKey(worker, host, publicKey, services)
|
|
125
113
|
: {
|
|
126
114
|
id: 'ssh:host-key',
|
|
127
115
|
status: 'skip',
|
|
@@ -149,3 +137,31 @@ export async function fleetNodeSeed(
|
|
|
149
137
|
].join('\n'),
|
|
150
138
|
};
|
|
151
139
|
}
|
|
140
|
+
|
|
141
|
+
/** The key the seed carries: one minted for this worker by the control plane
|
|
142
|
+
* fleet.json declares, or else the secret tailnet-auth-key.
|
|
143
|
+
*/
|
|
144
|
+
async function seedAuthKey(
|
|
145
|
+
worker: string,
|
|
146
|
+
services: NodeSeedServices
|
|
147
|
+
): Promise<string> {
|
|
148
|
+
const config = services.definition().tailnet;
|
|
149
|
+
if (config)
|
|
150
|
+
try {
|
|
151
|
+
const control = await fleetTailnetControl(
|
|
152
|
+
config,
|
|
153
|
+
services.secret,
|
|
154
|
+
services.http
|
|
155
|
+
);
|
|
156
|
+
return (await control!.createAuthKey({ worker })).key;
|
|
157
|
+
} catch (error) {
|
|
158
|
+
if (!(error instanceof TailnetControlError)) throw error;
|
|
159
|
+
throw new FleetUsageError(`${error.code}: ${controlRemedy(error.code)}`);
|
|
160
|
+
}
|
|
161
|
+
const key = await services.secret('tailnet-auth-key');
|
|
162
|
+
if (!key)
|
|
163
|
+
throw new FleetUsageError(
|
|
164
|
+
'The secrets provider holds no tailnet-auth-key. Declare tailnet in fleet.json with the secret tailnet-admin so a key is minted per worker, or store a short-lived, single-use, tagged Tailscale auth key there first.'
|
|
165
|
+
);
|
|
166
|
+
return key;
|
|
167
|
+
}
|
package/src/cli/fleet-state.ts
CHANGED
|
@@ -486,13 +486,14 @@ async function restore(
|
|
|
486
486
|
options: FleetStateOptions,
|
|
487
487
|
services: FleetStateServices
|
|
488
488
|
): Promise<CommandOutcome> {
|
|
489
|
-
const target = context(services);
|
|
489
|
+
const target = context(services, options);
|
|
490
490
|
if (!target.bucket)
|
|
491
491
|
throw new FleetUsageError(
|
|
492
492
|
'No bucket is configured, so there is nothing to restore from. Name it under "state" in fleet.json, or run devfleet fleet-state --init --bucket NAME --endpoint https://HOST.'
|
|
493
493
|
);
|
|
494
494
|
if (options.timestamp !== undefined) {
|
|
495
495
|
// The bucket's current version is replaced too: --yes is the consent.
|
|
496
|
+
// Another manager's lease is not, and needs --take-over.
|
|
496
497
|
if (!options.yes)
|
|
497
498
|
throw new FleetUsageError(
|
|
498
499
|
'--restore --timestamp replaces the fleet state in the bucket and on this machine with an earlier version. Repeat with --yes.'
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
3
4
|
import { join } from 'node:path';
|
|
4
5
|
import { prepareSource } from '../machine/source';
|
|
5
6
|
import { PACKAGE_ROOT, packageInfo } from '../package-info';
|
|
@@ -16,6 +17,12 @@ export const RUN_CHECKOUT_ENV = 'PERSONAL_FLEET_RUN_CHECKOUT';
|
|
|
16
17
|
|
|
17
18
|
const RUN_SCRIPT = 'bootstrap/fleet-manager-run.sh';
|
|
18
19
|
|
|
20
|
+
/** Where the run script keeps the manager's fleet-owned checkouts, one per
|
|
21
|
+
* revision (`$fleet/manager-runs/<revision>`).
|
|
22
|
+
*/
|
|
23
|
+
export const managerRuns = (): string =>
|
|
24
|
+
join(homedir(), '.local/share/personal/fleet/manager-runs');
|
|
25
|
+
|
|
19
26
|
export const inRunCheckout = (revision: string): boolean =>
|
|
20
27
|
process.env[RUN_CHECKOUT_ENV] === revision;
|
|
21
28
|
|
|
@@ -1,21 +1,58 @@
|
|
|
1
1
|
import { mkdir, rm } from 'node:fs/promises';
|
|
2
2
|
import { loadRoster } from './roster';
|
|
3
|
-
import { enrolledRoster } from '../definition';
|
|
3
|
+
import { enrolledRoster, sshTarget, type FleetDefinition } from '../definition';
|
|
4
4
|
import { FleetUsageError } from '../errors';
|
|
5
5
|
import { dirname } from 'node:path';
|
|
6
|
-
import { fleet
|
|
6
|
+
import { fleet } from '../fleet-config';
|
|
7
7
|
import { discoverTailnet } from '../tailscale';
|
|
8
8
|
import { matchComputers } from '../tailnet-matching';
|
|
9
9
|
import { sshArgs, sshFailure } from '../ssh';
|
|
10
10
|
import { ensurePrivateRoot } from '../machine/state';
|
|
11
|
-
import
|
|
11
|
+
import { createWorkerAccount } from '../worker-account';
|
|
12
|
+
import { packagePlaybook, type PackagePlaybook } from '../platform/ansible';
|
|
13
|
+
import type { Check, FleetNode, Runner } from '../types';
|
|
12
14
|
import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
|
|
15
|
+
import { readParked } from '../state/workers';
|
|
13
16
|
|
|
14
|
-
/**
|
|
15
|
-
*
|
|
17
|
+
/** What an enrollment did: the workers whose roster entry it added or
|
|
18
|
+
* completed, and the checks that say why another one was not enrolled (or
|
|
19
|
+
* what was done to reach it).
|
|
16
20
|
*/
|
|
17
|
-
export
|
|
18
|
-
|
|
21
|
+
export interface EnrollmentResult {
|
|
22
|
+
enrolled: string[];
|
|
23
|
+
checks: { node: string; check: Check }[];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface EnrollmentOptions {
|
|
27
|
+
/** A person is at the terminal and sees the list of what is enrolled. */
|
|
28
|
+
interactive: boolean;
|
|
29
|
+
/** A worker not on the tailnet yet joins it during this run: the fleet
|
|
30
|
+
* declares a tailnet control plane that issues single-use keys.
|
|
31
|
+
*/
|
|
32
|
+
joinable?: boolean;
|
|
33
|
+
definition?: FleetDefinition;
|
|
34
|
+
/** Applies a playbook of the package from the manager. */
|
|
35
|
+
playbook?: PackagePlaybook;
|
|
36
|
+
/** When each parked worker was parked; default the fleet state's. */
|
|
37
|
+
parked?: Record<string, string>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Enrolls declared workers, and resumes earlier enrollments. Existing paths
|
|
41
|
+
* and explicit identities are preserved. Tailnet candidates never enter the
|
|
42
|
+
* declared membership. An interactive run enrolls what a fresh
|
|
43
|
+
* host-key-verified SSH session reaches. A run without a person enrolls a
|
|
44
|
+
* worker only when every fact is verified: its pinned host key matched (the
|
|
45
|
+
* session is strict), `sudo -n` works, and exactly one tailnet device matches
|
|
46
|
+
* it, or none when this run joins it to the tailnet itself. Anything
|
|
47
|
+
* ambiguous stays out, with the code that says why; nothing prompts.
|
|
48
|
+
*/
|
|
49
|
+
export async function enrollManager(
|
|
50
|
+
path: string,
|
|
51
|
+
run: Runner,
|
|
52
|
+
options: EnrollmentOptions = { interactive: true }
|
|
53
|
+
): Promise<EnrollmentResult> {
|
|
54
|
+
const canonical = options.definition ?? fleet;
|
|
55
|
+
const result: EnrollmentResult = { enrolled: [], checks: [] };
|
|
19
56
|
const original = await existingEnrollment(path);
|
|
20
57
|
const saved =
|
|
21
58
|
original === null
|
|
@@ -23,54 +60,178 @@ export async function enrollManager(path: string, run: Runner): Promise<void> {
|
|
|
23
60
|
: enrolledRoster(canonical, await loadRoster(path)).nodes;
|
|
24
61
|
const nodes: FleetNode[] = canonical.workers.map((node) => ({
|
|
25
62
|
id: node.id,
|
|
26
|
-
ssh: node
|
|
63
|
+
ssh: sshTarget(node),
|
|
27
64
|
workspace: '/',
|
|
28
65
|
required: ['t3', 'toolchain', 'workstation'],
|
|
29
66
|
}));
|
|
67
|
+
// A parked worker is off on purpose: it is enrolled once it is resumed.
|
|
68
|
+
const parked = options.parked ?? readParked();
|
|
30
69
|
const pending = nodes.filter(
|
|
31
|
-
(node) =>
|
|
70
|
+
(node) =>
|
|
71
|
+
!parked[node.id] &&
|
|
72
|
+
!saved.find((entry) => entry.id === node.id)?.tailscaleId
|
|
32
73
|
);
|
|
33
|
-
if (!pending.length) return;
|
|
74
|
+
if (!pending.length) return result;
|
|
34
75
|
const tailnet = await discoverTailnet(run);
|
|
35
76
|
const matches = await matchComputers(nodes, tailnet.computers, run);
|
|
36
77
|
const enrolled: FleetNode[] = [];
|
|
37
78
|
for (const node of pending) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
79
|
+
// A worker already in the roster reports its own state in the run.
|
|
80
|
+
const known = saved.some((entry) => entry.id === node.id);
|
|
81
|
+
const connection = await enrollOne(node, {
|
|
82
|
+
run,
|
|
83
|
+
options,
|
|
84
|
+
peers: tailnet.computers
|
|
85
|
+
.filter((peer) =>
|
|
86
|
+
matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
|
|
87
|
+
)
|
|
88
|
+
.map((peer) => peer.id),
|
|
89
|
+
matches,
|
|
90
|
+
user: canonical.workers.find((worker) => worker.id === node.id)?.network
|
|
91
|
+
?.user,
|
|
92
|
+
note: (check) => {
|
|
93
|
+
if (!known) result.checks.push({ node: node.id, check });
|
|
94
|
+
},
|
|
95
|
+
});
|
|
49
96
|
if (connection) enrolled.push(connection);
|
|
50
97
|
}
|
|
51
98
|
const merged = mergeEnrollment(saved, enrolled);
|
|
52
|
-
if (JSON.stringify(merged) === JSON.stringify(saved)) return;
|
|
99
|
+
if (JSON.stringify(merged) === JSON.stringify(saved)) return result;
|
|
53
100
|
// Only canonical nodes reached over host-key-verified SSH get here, so
|
|
54
101
|
// enrollment needs no extra confirmation; the list is still shown.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
)
|
|
102
|
+
if (options.interactive)
|
|
103
|
+
process.stderr.write(
|
|
104
|
+
`Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => ` ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
|
|
105
|
+
);
|
|
58
106
|
await persistEnrollment(path, original, merged);
|
|
107
|
+
result.enrolled = merged
|
|
108
|
+
.filter(
|
|
109
|
+
(node) =>
|
|
110
|
+
JSON.stringify(node) !==
|
|
111
|
+
JSON.stringify(saved.find((entry) => entry.id === node.id))
|
|
112
|
+
)
|
|
113
|
+
.map((node) => node.id);
|
|
114
|
+
return result;
|
|
59
115
|
}
|
|
60
116
|
|
|
117
|
+
/** One worker's connection, when every fact this run needs is verified. */
|
|
118
|
+
async function enrollOne(
|
|
119
|
+
node: FleetNode,
|
|
120
|
+
context: {
|
|
121
|
+
run: Runner;
|
|
122
|
+
options: EnrollmentOptions;
|
|
123
|
+
/** The tailnet devices that match this worker. */
|
|
124
|
+
peers: string[];
|
|
125
|
+
matches: Map<string, FleetNode[]>;
|
|
126
|
+
user: string | undefined;
|
|
127
|
+
note: (check: Check) => void;
|
|
128
|
+
}
|
|
129
|
+
): Promise<FleetNode | null> {
|
|
130
|
+
const { run, options, peers, note } = context;
|
|
131
|
+
if (ambiguous(peers, context.matches)) {
|
|
132
|
+
if (!options.interactive) note(IDENTITY_CONFLICT);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
if (!options.interactive && !peers.length && !options.joinable) {
|
|
136
|
+
note(NOT_ON_TAILNET);
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
const connection = await inspectConnection(node, peers[0], {
|
|
140
|
+
run,
|
|
141
|
+
options,
|
|
142
|
+
user: context.user,
|
|
143
|
+
note,
|
|
144
|
+
quiet: !options.interactive,
|
|
145
|
+
});
|
|
146
|
+
if (!connection || options.interactive) return connection;
|
|
147
|
+
const privilege = await passwordlessSudo(connection, run);
|
|
148
|
+
if (privilege) note(privilege);
|
|
149
|
+
return privilege ? null : connection;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const IDENTITY_CONFLICT: Check = {
|
|
153
|
+
id: 'tailscale:identity',
|
|
154
|
+
status: 'fail',
|
|
155
|
+
summary: 'TAILSCALE_IDENTITY_CONFLICT',
|
|
156
|
+
remedy: {
|
|
157
|
+
action:
|
|
158
|
+
'More than one tailnet device matches this worker, so it was not enrolled. Remove the stale device from the tailnet, then rerun fleet-sync.',
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
const NOT_ON_TAILNET: Check = {
|
|
163
|
+
id: 'tailscale:enrolled-identity',
|
|
164
|
+
status: 'fail',
|
|
165
|
+
summary: 'TAILSCALE_ENROLLMENT_REQUIRED',
|
|
166
|
+
remedy: {
|
|
167
|
+
action:
|
|
168
|
+
'No tailnet device matches this worker and fleet.json declares no tailnet control plane to join it with, so a run without a person did not enroll it. Declare tailnet with the secret tailnet-admin, join the machine yourself, or run fleet-sync once in a terminal.',
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
/** A run without a person never prompts for sudo: the account has it, or the
|
|
173
|
+
* worker is not enrolled.
|
|
174
|
+
*/
|
|
175
|
+
async function passwordlessSudo(
|
|
176
|
+
node: FleetNode,
|
|
177
|
+
run: Runner
|
|
178
|
+
): Promise<Check | null> {
|
|
179
|
+
const result = await run([...sshArgs(node), 'sudo -n true'], {
|
|
180
|
+
timeoutMs: 15_000,
|
|
181
|
+
});
|
|
182
|
+
if (result.ok) return null;
|
|
183
|
+
if (result.reason !== 'exit' || !/sudo:/i.test(result.stderr))
|
|
184
|
+
return sshFailure(result);
|
|
185
|
+
return {
|
|
186
|
+
id: 'bootstrap:privilege',
|
|
187
|
+
status: 'fail',
|
|
188
|
+
summary: 'SUDO_AUTHORIZATION_REQUIRED',
|
|
189
|
+
remedy: {
|
|
190
|
+
action:
|
|
191
|
+
'The worker account has no passwordless sudo, so a run without a person did not enroll it. Grant it (a fleet-node-seed seed does), or run fleet-sync once in a terminal to authorize it.',
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The account's home over a fresh strict session. A machine whose image
|
|
197
|
+
* lets only root in refuses the declared account: it is created once over
|
|
198
|
+
* the root login, and the session is opened again.
|
|
199
|
+
*/
|
|
61
200
|
async function inspectConnection(
|
|
62
201
|
node: FleetNode,
|
|
63
202
|
peerId: string | undefined,
|
|
64
|
-
|
|
203
|
+
context: {
|
|
204
|
+
run: Runner;
|
|
205
|
+
options: EnrollmentOptions;
|
|
206
|
+
user: string | undefined;
|
|
207
|
+
note: (check: Check) => void;
|
|
208
|
+
quiet: boolean;
|
|
209
|
+
}
|
|
65
210
|
): Promise<FleetNode | null> {
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
211
|
+
const { run } = context;
|
|
212
|
+
const probe = () =>
|
|
213
|
+
run([...sshArgs(node), 'printf "%s" "$HOME"'], { timeoutMs: 20_000 });
|
|
214
|
+
let home = await probe();
|
|
215
|
+
if (!home.ok && sshFailure(home).id === 'ssh:auth' && context.user) {
|
|
216
|
+
const account = await createWorkerAccount(
|
|
217
|
+
{ id: node.id, ssh: node.ssh, login: 'root' },
|
|
218
|
+
context.user,
|
|
219
|
+
context.options.playbook ?? packagePlaybook(run)
|
|
220
|
+
);
|
|
221
|
+
// Only an account that was created is reported here: a root login that
|
|
222
|
+
// is refused as well stays the SSH failure below.
|
|
223
|
+
if (account.status === 'pass') {
|
|
224
|
+
context.note(account);
|
|
225
|
+
home = await probe();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
69
228
|
if (!home.ok) {
|
|
70
229
|
const check = sshFailure(home);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
230
|
+
if (context.quiet) context.note(check);
|
|
231
|
+
else
|
|
232
|
+
process.stderr.write(
|
|
233
|
+
`[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
|
|
234
|
+
);
|
|
74
235
|
return null;
|
|
75
236
|
}
|
|
76
237
|
if (!/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(home.stdout)) return null;
|