@crvouga/dev-fleet 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -7
- package/agents/audit.md +18 -18
- package/ansible/agents/agents.yml +134 -0
- package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
- package/ansible/agents/templates/claude.settings.json.j2 +3 -0
- package/ansible/agents/templates/codex.config.toml.j2 +11 -0
- package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
- package/ansible/agents/templates/grok.config.toml.j2 +19 -0
- package/ansible/agents/templates/opencode.json.j2 +24 -0
- package/ansible/agents/templates/receipt.json.j2 +10 -0
- package/ansible/ansible.cfg +14 -0
- package/ansible/github/git.yml +82 -0
- package/ansible/macos/manager-privilege.yml +83 -0
- package/ansible/macos/manager.yml +164 -9
- package/ansible/macos/policy.json +13 -0
- package/ansible/macos/t3-policy.yml +5 -3
- package/ansible/macos/vault-renew.yml +85 -0
- package/ansible/macos/worker-host-key.yml +33 -0
- package/ansible/monitoring/agent-darwin.yml +3 -0
- package/ansible/monitoring/agent.yml +2 -0
- package/ansible/monitoring/hub-darwin.yml +3 -0
- package/ansible/monitoring/hub-route-darwin.yml +1 -0
- package/ansible/net/remote-probe.sh +4 -1
- package/ansible/platforms.json +2 -2
- package/ansible/pyproject.toml +5 -0
- package/ansible/ubuntu/baseline.yml +39 -13
- package/ansible/ubuntu/disk-gc.py +6 -2
- package/ansible/ubuntu/firewall.nft.j2 +50 -0
- package/ansible/ubuntu/policy.json +125 -6
- package/ansible/ubuntu/reboot.yml +89 -0
- package/ansible/ubuntu/recovery.yml +13 -3
- package/ansible/ubuntu/t3/policy.json +3 -0
- package/ansible/ubuntu/t3-desktop.yml +23 -2
- package/ansible/ubuntu/t3-headless.yml +128 -0
- package/ansible/ubuntu/t3-health.sh +24 -10
- package/ansible/ubuntu/t3-idle.py +37 -2
- package/ansible/ubuntu/t3-policy.yml +4 -1
- package/ansible/ubuntu/t3-route.yml +1 -0
- package/ansible/ubuntu/tailscale.yml +54 -8
- package/ansible/ubuntu/tasks/browser.yml +3 -0
- package/ansible/ubuntu/tasks/containers-off.yml +43 -0
- package/ansible/ubuntu/tasks/docker.yml +53 -6
- package/ansible/ubuntu/tasks/dpkg.yml +4 -0
- package/ansible/ubuntu/tasks/features.yml +22 -0
- package/ansible/ubuntu/tasks/firewall.yml +172 -0
- package/ansible/ubuntu/tasks/install.yml +2 -1
- package/ansible/ubuntu/tasks/packages.yml +4 -2
- package/ansible/ubuntu/tasks/release.yml +6 -0
- package/ansible/ubuntu/tasks/resources.yml +6 -4
- package/ansible/ubuntu/tasks/services.yml +2 -2
- package/ansible/ubuntu/tasks/sizing.yml +49 -0
- package/ansible/ubuntu/tasks/snapshot.yml +14 -3
- package/ansible/ubuntu/tasks/ssh.yml +61 -4
- package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
- package/ansible/ubuntu/tasks/user.yml +1 -0
- package/ansible/ubuntu/worker-account.yml +117 -0
- package/ansible/uv.lock +265 -0
- package/bin/devfleet.mjs +42 -4
- package/bootstrap/fleet-remote-bootstrap.sh +91 -25
- package/bootstrap/global-config.ts +29 -3
- package/bootstrap/package-smoke.ts +127 -0
- package/bootstrap/pins.ts +189 -0
- package/bootstrap/release-notes.ts +88 -0
- package/bootstrap/release-version.ts +41 -7
- package/bootstrap/worker-features.ts +114 -0
- package/bootstrap/worker-seed.yaml +65 -0
- package/bootstrap/worker-tools.ts +39 -9
- package/cliproxy/README.md +1 -1
- package/cliproxy/docs/health-workflow.example.yml +20 -143
- package/docs/authentication.md +21 -11
- package/docs/consumer-setup.md +74 -58
- package/docs/convergence.md +67 -0
- package/docs/declared.md +61 -34
- package/docs/machine-setup.md +73 -7
- package/docs/manager-setup.md +34 -0
- package/docs/operations.md +64 -9
- package/docs/releasing.md +49 -1
- package/docs/runbooks.md +112 -7
- package/docs/state.md +23 -11
- package/docs/support-matrix.md +40 -4
- package/docs/topology.md +82 -12
- package/docs/ubuntu-sync.md +98 -42
- package/docs/worker-contract.md +122 -19
- package/docs/worker-requirements.md +107 -0
- package/examples/fleet.json +1 -0
- package/examples/workers-from-infrastructure/main.tf +132 -0
- package/examples/workers-from-infrastructure/worker-records.json +14 -0
- package/llms.txt +3 -1
- package/mise.lock +29 -0
- package/mise.toml +5 -3
- package/package.json +2 -2
- package/schema/check-ids.json +1991 -162
- package/schema/check-ids.schema.json +34 -0
- package/schema/fleet.schema.json +349 -10
- package/schema/output/check.schema.json +1 -1
- package/schema/output/fleet-state-definition.schema.json +14 -0
- package/schema/output/fleet-sync-schedule.schema.json +26 -0
- package/schema/output/fleet-sync.schema.json +18 -0
- package/schema/output/report.schema.json +3 -2
- package/schema/worker-record.schema.json +76 -0
- package/src/agents/converge.ts +140 -24
- package/src/agents/files.ts +2 -32
- package/src/agents/native-verify.ts +34 -3
- package/src/agents/secrets.ts +39 -4
- package/src/check-owners.ts +62 -2
- package/src/check-registry.ts +630 -0
- package/src/cli/agent-guide.ts +3 -3
- package/src/cli/cli.ts +4 -0
- package/src/cli/fleet-manifest.ts +108 -0
- package/src/cli/fleet-node-lifecycle.ts +704 -0
- package/src/cli/fleet-node-park.ts +238 -0
- package/src/cli/fleet-node-seed.ts +167 -0
- package/src/cli/fleet-state.ts +2 -1
- package/src/cli/fleet-sync-checkout.ts +7 -0
- package/src/cli/fleet-sync-enrollment.ts +193 -32
- package/src/cli/fleet-sync-forward.ts +6 -0
- package/src/cli/fleet-sync-manager.ts +123 -8
- package/src/cli/fleet-sync-output.ts +31 -11
- package/src/cli/fleet-sync-plan.ts +65 -9
- package/src/cli/fleet-sync-records.ts +113 -0
- package/src/cli/fleet-sync-remote.ts +172 -15
- package/src/cli/fleet-sync-schedule.ts +14 -4
- package/src/cli/fleet-sync-state.ts +31 -10
- package/src/cli/fleet-sync-vault.ts +152 -26
- package/src/cli/fleet-sync-worker.ts +214 -0
- package/src/cli/fleet-sync.ts +172 -21
- package/src/cli/fleet-validate.ts +277 -0
- package/src/cli/fleet-watch.ts +7 -1
- package/src/cli/fleet.ts +87 -5
- package/src/cli/managed-health.ts +25 -1
- package/src/cli/register-fleet.ts +251 -4
- package/src/cli/registry.ts +47 -0
- package/src/cli/tailnet.ts +43 -10
- package/src/cli/ui-data.ts +3 -1
- package/src/cli/worker-usage-report.ts +63 -0
- package/src/clock.ts +42 -0
- package/src/definition.ts +762 -232
- package/src/diagnostics.ts +5 -2
- package/src/exposure.ts +102 -0
- package/src/fleet-config.ts +3 -1
- package/src/git-identity.ts +76 -3
- package/src/ide/ide.ts +5 -0
- package/src/ide/t3.ts +12 -2
- package/src/input-hashes.ts +6 -0
- package/src/machine/machine-class.ts +59 -0
- package/src/machine/platform.ts +90 -11
- package/src/machine/state.ts +70 -16
- package/src/machine/support-matrix.ts +11 -9
- package/src/machine-health.ts +28 -5
- package/src/manager-health.ts +148 -2
- package/src/manager-sync.ts +89 -8
- package/src/monitoring/beszel.ts +8 -8
- package/src/monitoring/enroll.ts +8 -1
- package/src/monitoring/hub-access.ts +10 -4
- package/src/monitoring/systems.ts +138 -0
- package/src/network-definition.ts +13 -4
- package/src/node-converge-linux.ts +136 -27
- package/src/node-converge.ts +13 -3
- package/src/platform/ansible-receipt.ts +17 -2
- package/src/platform/ansible.ts +40 -4
- package/src/platform/manager-host.ts +25 -8
- package/src/platform/resource-verify.ts +18 -7
- package/src/platform/t3-policy.ts +24 -10
- package/src/platform/t3-service.ts +109 -34
- package/src/platform/ubuntu-release.ts +24 -0
- package/src/platform/ubuntu-verify.ts +31 -3
- package/src/platform/ubuntu-worker.ts +58 -0
- package/src/platform/worker-platform.ts +72 -0
- package/src/protocol.ts +1 -1
- package/src/public-reachability.ts +186 -0
- package/src/remote-bootstrap.ts +50 -12
- package/src/remote-operation.ts +18 -4
- package/src/repair-plan.ts +23 -12
- package/src/report.ts +2 -1
- package/src/schedule-check.ts +21 -2
- package/src/secrets.ts +7 -0
- package/src/service-auth.ts +55 -48
- package/src/ssh-connection.ts +141 -0
- package/src/ssh.ts +63 -24
- package/src/state/store.ts +4 -2
- package/src/state/sync.ts +35 -6
- package/src/state/workers.ts +140 -0
- package/src/t3/pairing.ts +59 -0
- package/src/t3/peer-link.ts +333 -0
- package/src/t3/version.ts +1 -1
- package/src/tailnet-cleanup.ts +185 -0
- package/src/tailnet-control.ts +346 -0
- package/src/tailscale.ts +1 -1
- package/src/toolchain-checks.ts +11 -1
- package/src/types.ts +4 -2
- package/src/worker-account.ts +63 -0
- package/src/worker-activity.ts +303 -0
- package/src/worker-capacity.ts +31 -0
- package/src/worker-credentials.ts +201 -0
- package/src/worker-exposure.ts +388 -0
- package/src/worker-features.ts +201 -0
- package/src/worker-host-checks.ts +71 -20
- package/src/worker-manifest.ts +224 -0
- package/src/worker-parking.ts +70 -0
- package/src/worker-policy.ts +38 -36
- package/src/worker-readiness.ts +15 -3
- package/src/worker-reboot.ts +129 -0
- package/src/worker-records.ts +305 -0
- package/src/worker-seed.ts +143 -0
- package/src/worker-sizing.ts +76 -0
- package/src/cli/schedule-logs.ts +0 -36
package/src/cli/fleet.ts
CHANGED
|
@@ -16,13 +16,37 @@ import { probeSource } from './probe-source';
|
|
|
16
16
|
import { executeRepairCommand } from './repair';
|
|
17
17
|
import { scheduleCheck } from '../schedule-check';
|
|
18
18
|
import { readHistory } from '../state/history';
|
|
19
|
+
import { withUsage } from '../worker-activity';
|
|
20
|
+
import { activeNodes, appendUsage } from './worker-usage-report';
|
|
21
|
+
import { fleet, secret } from '../fleet-config';
|
|
22
|
+
import { appendTailnetLeftovers } from '../tailnet-cleanup';
|
|
23
|
+
import { fleetTailnetControl, type TailnetControl } from '../tailnet-control';
|
|
24
|
+
import { packageInfo } from '../package-info';
|
|
25
|
+
import { managerRuns } from './fleet-sync-checkout';
|
|
26
|
+
import { existsSync } from 'node:fs';
|
|
27
|
+
import { join } from 'node:path';
|
|
19
28
|
|
|
20
29
|
export interface FleetServices {
|
|
21
30
|
run: Runner;
|
|
22
31
|
source: () => Promise<string>;
|
|
32
|
+
/** The tailnet control plane fleet.json declares, asked for devices
|
|
33
|
+
* tagged as workers that no declared worker accounts for. Absent, or null,
|
|
34
|
+
* fleet-status says nothing about them.
|
|
35
|
+
*/
|
|
36
|
+
tailnet?: () => Promise<TailnetControl | null>;
|
|
37
|
+
/** The commit the running package names; absent or null in a checkout. */
|
|
38
|
+
revision?: string | null;
|
|
39
|
+
/** Where the manager keeps its fleet-owned checkout of each revision. */
|
|
40
|
+
runs?: string;
|
|
23
41
|
}
|
|
24
42
|
|
|
25
|
-
const SERVICES: FleetServices = {
|
|
43
|
+
const SERVICES: FleetServices = {
|
|
44
|
+
run: runCommand,
|
|
45
|
+
source: probeSource,
|
|
46
|
+
tailnet: () => fleetTailnetControl(fleet.tailnet, secret),
|
|
47
|
+
revision: packageInfo().revision,
|
|
48
|
+
runs: managerRuns(),
|
|
49
|
+
};
|
|
26
50
|
|
|
27
51
|
export async function executeFleet(
|
|
28
52
|
command: string,
|
|
@@ -114,11 +138,14 @@ async function diagnose(
|
|
|
114
138
|
settings: Settings & { missingRoster: boolean },
|
|
115
139
|
services: FleetServices
|
|
116
140
|
): Promise<CommandOutcome> {
|
|
117
|
-
|
|
141
|
+
// A parked worker is not contacted; it is listed as parked.
|
|
142
|
+
const active = activeNodes(nodes);
|
|
143
|
+
const source = active.length ? await services.source() : undefined;
|
|
144
|
+
const root = statusRoot(settings, services);
|
|
118
145
|
const reports = await doctor(
|
|
119
|
-
|
|
146
|
+
active,
|
|
120
147
|
{
|
|
121
|
-
root
|
|
148
|
+
root,
|
|
122
149
|
local: options.local === true,
|
|
123
150
|
required: capabilities(options.require),
|
|
124
151
|
timeoutMs: timeout(options),
|
|
@@ -126,11 +153,23 @@ async function diagnose(
|
|
|
126
153
|
},
|
|
127
154
|
services.run
|
|
128
155
|
);
|
|
156
|
+
explainMissingCheckout(reports, root);
|
|
129
157
|
if (!options.local && settings.missingRoster) reports.push(missingRoster());
|
|
130
158
|
const schedule = scheduleCheck(readHistory(), new Date());
|
|
131
159
|
if (schedule) reports[0]?.checks.push(schedule);
|
|
160
|
+
// What each worker is doing, unless only this manager is checked.
|
|
161
|
+
const usage = await usageOf(
|
|
162
|
+
reports,
|
|
163
|
+
active,
|
|
164
|
+
services.run,
|
|
165
|
+
skipsFleet(options, settings)
|
|
166
|
+
);
|
|
167
|
+
// Devices tagged as workers that are no declared worker (#123).
|
|
168
|
+
if (!skipsFleet(options, settings))
|
|
169
|
+
await appendTailnetLeftovers(reports, fleet, services.tailnet, nodes);
|
|
132
170
|
const report = {
|
|
133
171
|
...makeReport('doctor', reports),
|
|
172
|
+
...(usage ?? {}),
|
|
134
173
|
coverage: {
|
|
135
174
|
scope: options.local
|
|
136
175
|
? 'local'
|
|
@@ -146,10 +185,53 @@ async function diagnose(
|
|
|
146
185
|
return {
|
|
147
186
|
exitCode: reportExitCode(report, options.strict === true),
|
|
148
187
|
data: report,
|
|
149
|
-
text: renderReport(report),
|
|
188
|
+
text: withUsage(renderReport(report), usage),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** What the manager and the workers are measured against. The published
|
|
193
|
+
* package holds no Git history and no bun.lock, and whatever repository
|
|
194
|
+
* surrounds it is the consumer's project: from a package it is the
|
|
195
|
+
* fleet-owned checkout fleet-sync made of the commit the package names. A
|
|
196
|
+
* checkout (development) is its own reference.
|
|
197
|
+
*/
|
|
198
|
+
function statusRoot(settings: Settings, services: FleetServices): string {
|
|
199
|
+
const { revision, runs } = services;
|
|
200
|
+
return revision && runs ? join(runs, revision) : settings.workspace;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** The manager has not run fleet-sync at this package's commit yet (a
|
|
204
|
+
* checkout is its own reference, so it always exists).
|
|
205
|
+
*/
|
|
206
|
+
function explainMissingCheckout(reports: NodeReport[], root: string): void {
|
|
207
|
+
if (existsSync(root)) return;
|
|
208
|
+
const check = reports[0]?.checks.find((item) => item.id === 'workspace');
|
|
209
|
+
if (!check) return;
|
|
210
|
+
check.summary =
|
|
211
|
+
'This manager has no fleet-owned checkout of the commit this package names; it is made by fleet-sync.';
|
|
212
|
+
check.remedy = {
|
|
213
|
+
action:
|
|
214
|
+
'Run devfleet fleet-sync from the project that holds your fleet.json, then fleet-status again.',
|
|
150
215
|
};
|
|
151
216
|
}
|
|
152
217
|
|
|
218
|
+
/** Only this manager is checked: what is said of the fleet is skipped. */
|
|
219
|
+
function skipsFleet(
|
|
220
|
+
options: CliOptions,
|
|
221
|
+
settings: { missingRoster: boolean }
|
|
222
|
+
): boolean {
|
|
223
|
+
return Boolean(options.local || settings.missingRoster);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function usageOf(
|
|
227
|
+
reports: NodeReport[],
|
|
228
|
+
nodes: FleetNode[],
|
|
229
|
+
run: Runner,
|
|
230
|
+
skip: boolean
|
|
231
|
+
) {
|
|
232
|
+
return skip ? Promise.resolve(null) : appendUsage(reports, nodes, { run });
|
|
233
|
+
}
|
|
234
|
+
|
|
153
235
|
function missingRoster(): NodeReport {
|
|
154
236
|
return {
|
|
155
237
|
node: 'fleet',
|
|
@@ -3,6 +3,7 @@ import { verifyT3Endpoint } from '../t3-endpoint';
|
|
|
3
3
|
import type { Check, NodeReport, Runner } from '../types';
|
|
4
4
|
import { fleet as definition } from '../fleet-config';
|
|
5
5
|
import { enrolledRoster } from '../definition';
|
|
6
|
+
import { readParked } from '../state/workers';
|
|
6
7
|
import type { Tailnet } from '../tailscale';
|
|
7
8
|
import type { FleetNode } from '../types';
|
|
8
9
|
import {
|
|
@@ -12,6 +13,7 @@ import {
|
|
|
12
13
|
} from '../monitoring/beszel';
|
|
13
14
|
import { hubAccess } from '../monitoring/hub-access';
|
|
14
15
|
import { hubUrl } from '../topology';
|
|
16
|
+
import { leftoverSystems, listSystems } from '../monitoring/systems';
|
|
15
17
|
|
|
16
18
|
export interface DashboardAccess {
|
|
17
19
|
(hub: string): Promise<{ request: BeszelRequest }>;
|
|
@@ -76,6 +78,7 @@ async function nodeEvidence(
|
|
|
76
78
|
) ?? dashboard;
|
|
77
79
|
report.checks.push(...evidence.checks);
|
|
78
80
|
}
|
|
81
|
+
dashboard.checks.push(...(await leftovers(request)));
|
|
79
82
|
} catch (error) {
|
|
80
83
|
dashboard.checks.push({
|
|
81
84
|
id: 'beszel:access',
|
|
@@ -92,6 +95,21 @@ async function nodeEvidence(
|
|
|
92
95
|
}
|
|
93
96
|
}
|
|
94
97
|
|
|
98
|
+
/** Systems the hub watches that no declared worker (parked ones included)
|
|
99
|
+
* and not the manager is named after. A hub whose list cannot be read says
|
|
100
|
+
* nothing here; its access is reported above.
|
|
101
|
+
*/
|
|
102
|
+
async function leftovers(request: BeszelRequest): Promise<Check[]> {
|
|
103
|
+
try {
|
|
104
|
+
return leftoverSystems(await listSystems(request), [
|
|
105
|
+
definition.manager.id,
|
|
106
|
+
...definition.workers.map((worker) => worker.id),
|
|
107
|
+
]);
|
|
108
|
+
} catch {
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
95
113
|
async function hubReachable(
|
|
96
114
|
hub: string,
|
|
97
115
|
fetcher: typeof fetch
|
|
@@ -155,7 +173,13 @@ export function managedHealthScope(
|
|
|
155
173
|
roster: FleetNode[],
|
|
156
174
|
includeManager = true
|
|
157
175
|
) {
|
|
158
|
-
|
|
176
|
+
// A parked worker is off on purpose: it is not audited, not missing and
|
|
177
|
+
// its device is not expected online (fleet-status lists it as parked).
|
|
178
|
+
const parked = readParked();
|
|
179
|
+
const canonical = {
|
|
180
|
+
...definition,
|
|
181
|
+
workers: definition.workers.filter((entry) => !parked[entry.id]),
|
|
182
|
+
};
|
|
159
183
|
const declared = roster.filter((node) =>
|
|
160
184
|
canonical.workers.some((entry) => entry.id === node.id)
|
|
161
185
|
);
|
|
@@ -10,9 +10,33 @@ import { fleetState, type FleetStateOptions } from './fleet-state';
|
|
|
10
10
|
import { fleetAccounts, type FleetAccountsOptions } from './fleet-accounts';
|
|
11
11
|
import { fleetReport, type FleetReportOptions } from './fleet-report';
|
|
12
12
|
import { fleetWatch, watchServices } from './fleet-watch';
|
|
13
|
-
import type { NodeReport } from '../types';
|
|
13
|
+
import type { Check, NodeReport } from '../types';
|
|
14
|
+
import { fleet } from '../fleet-config';
|
|
14
15
|
import { registerGithubVerify } from './github-verify';
|
|
15
16
|
import { fleetRestore, type FleetRestoreOptions } from './fleet-restore';
|
|
17
|
+
import { fleetValidate, type FleetValidateOptions } from './fleet-validate';
|
|
18
|
+
import {
|
|
19
|
+
fleetManifest,
|
|
20
|
+
manifestServices,
|
|
21
|
+
type FleetManifestOptions,
|
|
22
|
+
} from './fleet-manifest';
|
|
23
|
+
import {
|
|
24
|
+
fleetNodeSeed,
|
|
25
|
+
seedServices,
|
|
26
|
+
type NodeSeedOptions,
|
|
27
|
+
} from './fleet-node-seed';
|
|
28
|
+
import {
|
|
29
|
+
fleetNodeReplace,
|
|
30
|
+
fleetNodeRetire,
|
|
31
|
+
lifecycleServices,
|
|
32
|
+
type NodeLifecycleOptions,
|
|
33
|
+
} from './fleet-node-lifecycle';
|
|
34
|
+
import {
|
|
35
|
+
fleetNodePark,
|
|
36
|
+
fleetNodeResume,
|
|
37
|
+
parkServices,
|
|
38
|
+
type NodeParkOptions,
|
|
39
|
+
} from './fleet-node-park';
|
|
16
40
|
|
|
17
41
|
type Complete = (outcome: CommandOutcome) => void;
|
|
18
42
|
|
|
@@ -181,12 +205,28 @@ function registerSync({ program, defaults, complete }: Registration) {
|
|
|
181
205
|
'--revision <sha>',
|
|
182
206
|
'Converge to an exact source commit; defaults to the commit the dev-fleet package you run was published from'
|
|
183
207
|
)
|
|
208
|
+
.option(
|
|
209
|
+
'--rollback',
|
|
210
|
+
'Converge to the revision applied before the current one (fleet state); the revision, tools and T3 go back, system packages roll forward only'
|
|
211
|
+
)
|
|
184
212
|
.option(
|
|
185
213
|
'--node <id>',
|
|
186
214
|
'Explicit enrolled remote target; repeatable',
|
|
187
215
|
collect,
|
|
188
216
|
[]
|
|
189
217
|
)
|
|
218
|
+
.option(
|
|
219
|
+
'--vault-password-login',
|
|
220
|
+
'Opt-in: log a worker without a token of its own in with the shared Vault user, on its own terminal'
|
|
221
|
+
)
|
|
222
|
+
.option(
|
|
223
|
+
'--restart-busy',
|
|
224
|
+
'Restart a worker T3 for a new release or policy even while agent turns run, which ends them; by default the restart waits until T3 is idle'
|
|
225
|
+
)
|
|
226
|
+
.option(
|
|
227
|
+
'--canary <id>',
|
|
228
|
+
'The worker that proves a new revision first when it is reachable; defaults to canary in fleet.json, then the first worker'
|
|
229
|
+
)
|
|
190
230
|
.option(
|
|
191
231
|
'--all',
|
|
192
232
|
'Explicit spelling of the default enrolled fleet scope; never discovered peers'
|
|
@@ -200,6 +240,14 @@ function registerSync({ program, defaults, complete }: Registration) {
|
|
|
200
240
|
'Accepted for compatibility; fleet-sync applies by default (schedule writes still need --apply --yes)'
|
|
201
241
|
)
|
|
202
242
|
.option('--yes', 'Accepted for compatibility with --apply')
|
|
243
|
+
.option(
|
|
244
|
+
'--maintenance-reboot',
|
|
245
|
+
'Reboot each worker whose installed updates wait for a reboot: one at a time, canary first, only while it runs no agent turn, verified ready before the next'
|
|
246
|
+
)
|
|
247
|
+
.option(
|
|
248
|
+
'--force',
|
|
249
|
+
'With --maintenance-reboot: reboot a worker even while agent turns run (they end)'
|
|
250
|
+
)
|
|
203
251
|
.option(
|
|
204
252
|
'--take-over',
|
|
205
253
|
'Make this machine the manager while another one still holds the manager lease'
|
|
@@ -284,11 +332,11 @@ function registerState({ program, complete }: Registration) {
|
|
|
284
332
|
.option('--yes', 'With --restore: discard changes the bucket does not hold')
|
|
285
333
|
.option(
|
|
286
334
|
'--timestamp <time>',
|
|
287
|
-
'With --restore: the version the bucket held at this time, uploaded again as the current one (a damaged replica)'
|
|
335
|
+
'With --restore: the version the bucket held at this time, uploaded again as the current one (a damaged replica); refused while another manager holds the manager lease, unless with --take-over'
|
|
288
336
|
)
|
|
289
337
|
.option(
|
|
290
338
|
'--take-over',
|
|
291
|
-
'With --init or --
|
|
339
|
+
'With --init, --import or --restore --timestamp: change the fleet state while another manager still holds the manager lease'
|
|
292
340
|
)
|
|
293
341
|
.addHelpText(
|
|
294
342
|
'after',
|
|
@@ -336,10 +384,52 @@ function registerWatch({ program, defaults, complete }: Registration) {
|
|
|
336
384
|
const data = report.data as { nodes?: NodeReport[] };
|
|
337
385
|
return data.nodes ?? [];
|
|
338
386
|
};
|
|
387
|
+
const autoPark = async (reports: NodeReport[]): Promise<Check[]> => {
|
|
388
|
+
const checks: Check[] = [];
|
|
389
|
+
for (const report of reports) {
|
|
390
|
+
if (
|
|
391
|
+
!report.checks.some(
|
|
392
|
+
(check) => check.id === 'worker:idle' && check.status === 'warn'
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
continue;
|
|
396
|
+
const worker = fleet.workers.find(
|
|
397
|
+
(entry) =>
|
|
398
|
+
entry.id === report.node ||
|
|
399
|
+
entry.network?.tailscaleName === report.node
|
|
400
|
+
);
|
|
401
|
+
if (!worker) continue;
|
|
402
|
+
const parked = await fleetNodePark(
|
|
403
|
+
{ id: worker.id },
|
|
404
|
+
parkServices(settings.config, () =>
|
|
405
|
+
Promise.reject(new Error('fleet-watch never resumes a worker'))
|
|
406
|
+
)
|
|
407
|
+
);
|
|
408
|
+
checks.push({
|
|
409
|
+
id: 'worker:auto-park',
|
|
410
|
+
status: parked.exitCode === 0 ? 'pass' : 'warn',
|
|
411
|
+
summary:
|
|
412
|
+
parked.exitCode === 0
|
|
413
|
+
? `${worker.id} was idle longer than declared and is parked.`
|
|
414
|
+
: 'WORKER_AUTO_PARK_REFUSED',
|
|
415
|
+
...(parked.exitCode === 0
|
|
416
|
+
? {}
|
|
417
|
+
: {
|
|
418
|
+
remedy: {
|
|
419
|
+
action: `fleet-node-park refused ${worker.id}; run devfleet fleet-node-park --id ${worker.id} to see why.`,
|
|
420
|
+
},
|
|
421
|
+
}),
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
return checks;
|
|
425
|
+
};
|
|
339
426
|
complete(
|
|
340
427
|
await fleetWatch(
|
|
341
428
|
{ ...(options.test ? { test: true } : {}) },
|
|
342
|
-
|
|
429
|
+
{
|
|
430
|
+
...watchServices(status),
|
|
431
|
+
...(fleet.parking?.autoPark ? { autoPark } : {}),
|
|
432
|
+
}
|
|
343
433
|
)
|
|
344
434
|
);
|
|
345
435
|
});
|
|
@@ -362,6 +452,160 @@ function registerRestore({ program, complete }: Registration) {
|
|
|
362
452
|
});
|
|
363
453
|
}
|
|
364
454
|
|
|
455
|
+
function registerValidate({ program, complete }: Registration) {
|
|
456
|
+
program
|
|
457
|
+
.command('fleet-validate')
|
|
458
|
+
.description(
|
|
459
|
+
'Validate fleet.json and name the field that breaks a rule; contacts nothing'
|
|
460
|
+
)
|
|
461
|
+
.option(
|
|
462
|
+
'--file <path>',
|
|
463
|
+
'The fleet.json to validate; defaults to the project’s'
|
|
464
|
+
)
|
|
465
|
+
.option(
|
|
466
|
+
'--ansible',
|
|
467
|
+
'Also render the Ansible inventory and syntax-check every playbook with the pinned Ansible'
|
|
468
|
+
)
|
|
469
|
+
.action(async (_options, command: Command) => {
|
|
470
|
+
complete(
|
|
471
|
+
await fleetValidate(command.optsWithGlobals<FleetValidateOptions>())
|
|
472
|
+
);
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function registerLifecycle({ program, defaults, complete }: Registration) {
|
|
477
|
+
const action =
|
|
478
|
+
(name: 'replace' | 'retire') =>
|
|
479
|
+
async (_options: unknown, command: Command) => {
|
|
480
|
+
const options = command.optsWithGlobals<
|
|
481
|
+
NodeLifecycleOptions & CliOptions
|
|
482
|
+
>();
|
|
483
|
+
const settings = await withSettings(defaults, options);
|
|
484
|
+
const services = lifecycleServices(settings.config);
|
|
485
|
+
complete(
|
|
486
|
+
name === 'replace'
|
|
487
|
+
? await fleetNodeReplace(options, services)
|
|
488
|
+
: await fleetNodeRetire(options, services)
|
|
489
|
+
);
|
|
490
|
+
};
|
|
491
|
+
program
|
|
492
|
+
.command('fleet-node-replace')
|
|
493
|
+
.description(
|
|
494
|
+
'Put a fresh machine in place of a worker under the same id; writes need --apply --yes'
|
|
495
|
+
)
|
|
496
|
+
.requiredOption('--id <name>', 'The declared worker being replaced')
|
|
497
|
+
.option(
|
|
498
|
+
'--host-key <key>',
|
|
499
|
+
'The new machine’s SSH host key, "TYPE BASE64", verified out of band or printed by fleet-node-seed'
|
|
500
|
+
)
|
|
501
|
+
.option('--apply', 'Change the manager and Vault; requires --yes')
|
|
502
|
+
.option('--yes', 'Confirm an explicitly requested --apply operation')
|
|
503
|
+
.action(action('replace'));
|
|
504
|
+
program
|
|
505
|
+
.command('fleet-node-retire')
|
|
506
|
+
.description(
|
|
507
|
+
'Take a worker out of the fleet: revoke its token, forget its host key and roster entry; writes need --apply --yes'
|
|
508
|
+
)
|
|
509
|
+
.requiredOption('--id <name>', 'The declared worker to retire')
|
|
510
|
+
.option(
|
|
511
|
+
'--apply',
|
|
512
|
+
'Change the manager, Vault and the fleet state; requires --yes'
|
|
513
|
+
)
|
|
514
|
+
.option('--yes', 'Confirm an explicitly requested --apply operation')
|
|
515
|
+
.action(action('retire'));
|
|
516
|
+
const parking =
|
|
517
|
+
(name: 'park' | 'resume') =>
|
|
518
|
+
async (_options: unknown, command: Command) => {
|
|
519
|
+
const options = command.optsWithGlobals<NodeParkOptions & CliOptions>();
|
|
520
|
+
const settings = await withSettings(defaults, options);
|
|
521
|
+
const services = parkServices(settings.config, (id) =>
|
|
522
|
+
fleetSync(
|
|
523
|
+
{
|
|
524
|
+
node: [id],
|
|
525
|
+
apply: true,
|
|
526
|
+
yes: true,
|
|
527
|
+
nonInteractive: true,
|
|
528
|
+
...(options.json ? { json: true } : {}),
|
|
529
|
+
},
|
|
530
|
+
{
|
|
531
|
+
...defaults,
|
|
532
|
+
...(options.config ? { config: options.config } : {}),
|
|
533
|
+
}
|
|
534
|
+
)
|
|
535
|
+
);
|
|
536
|
+
complete(
|
|
537
|
+
name === 'park'
|
|
538
|
+
? await fleetNodePark(options, services)
|
|
539
|
+
: await fleetNodeResume(options, services)
|
|
540
|
+
);
|
|
541
|
+
};
|
|
542
|
+
program
|
|
543
|
+
.command('fleet-node-park')
|
|
544
|
+
.description(
|
|
545
|
+
'Park a worker that is stopped or destroyed on purpose: it is no longer contacted, never the canary and does not fail the fleet'
|
|
546
|
+
)
|
|
547
|
+
.requiredOption('--id <name>', 'The declared worker to park')
|
|
548
|
+
.option(
|
|
549
|
+
'--force',
|
|
550
|
+
'Park although agent turns are under way on it, or cannot be counted'
|
|
551
|
+
)
|
|
552
|
+
.option('--dry-run', 'Say what would change; change nothing')
|
|
553
|
+
.action(parking('park'));
|
|
554
|
+
program
|
|
555
|
+
.command('fleet-node-resume')
|
|
556
|
+
.description(
|
|
557
|
+
'Bring a parked worker back: run the declared start command, then fleet-sync that worker and verify it'
|
|
558
|
+
)
|
|
559
|
+
.requiredOption('--id <name>', 'The declared worker to resume')
|
|
560
|
+
.option('--dry-run', 'Say what would change; change nothing')
|
|
561
|
+
.action(parking('resume'));
|
|
562
|
+
program
|
|
563
|
+
.command('fleet-node-seed')
|
|
564
|
+
.description(
|
|
565
|
+
'Write cloud-init (or autoinstall) user data a new worker joins the fleet with by itself'
|
|
566
|
+
)
|
|
567
|
+
.requiredOption('--id <name>', 'The declared worker to seed')
|
|
568
|
+
.requiredOption(
|
|
569
|
+
'--output <path>',
|
|
570
|
+
'Where to write the seed (mode 600); it holds a single-use auth key and is never printed'
|
|
571
|
+
)
|
|
572
|
+
.option(
|
|
573
|
+
'--public-key <path>',
|
|
574
|
+
'The manager public key to authorize; default ~/.ssh/id_ed25519.pub'
|
|
575
|
+
)
|
|
576
|
+
.option('--autoinstall', 'An Ubuntu autoinstall file, for bare metal')
|
|
577
|
+
.option(
|
|
578
|
+
'--apply',
|
|
579
|
+
'Also pin the seed host key on this manager; requires --yes'
|
|
580
|
+
)
|
|
581
|
+
.option('--yes', 'Confirm an explicitly requested --apply operation')
|
|
582
|
+
.action(async (_options: unknown, command: Command) => {
|
|
583
|
+
const options = command.optsWithGlobals<NodeSeedOptions & CliOptions>();
|
|
584
|
+
const settings = await withSettings(defaults, options);
|
|
585
|
+
complete(await fleetNodeSeed(options, seedServices(settings.config)));
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function registerManifest({ program, defaults, complete }: Registration) {
|
|
590
|
+
program
|
|
591
|
+
.command('fleet-manifest')
|
|
592
|
+
.description(
|
|
593
|
+
'Measure what a worker is made of and check it against the pins and a fresh build'
|
|
594
|
+
)
|
|
595
|
+
.requiredOption('--id <name>', 'The enrolled worker to measure')
|
|
596
|
+
.option(
|
|
597
|
+
'--reference <path>',
|
|
598
|
+
'A fleet-manifest --json of a fresh build of the same revision'
|
|
599
|
+
)
|
|
600
|
+
.action(async (_options: unknown, command: Command) => {
|
|
601
|
+
const options = command.optsWithGlobals<
|
|
602
|
+
FleetManifestOptions & CliOptions
|
|
603
|
+
>();
|
|
604
|
+
const settings = await withSettings(defaults, options);
|
|
605
|
+
complete(await fleetManifest(options, manifestServices(settings.config)));
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
|
|
365
609
|
export function registerFleet(
|
|
366
610
|
program: Command,
|
|
367
611
|
defaults: Settings,
|
|
@@ -379,6 +623,9 @@ export function registerFleet(
|
|
|
379
623
|
registerReport(registration);
|
|
380
624
|
registerWatch(registration);
|
|
381
625
|
registerRestore(registration);
|
|
626
|
+
registerValidate(registration);
|
|
627
|
+
registerLifecycle(registration);
|
|
628
|
+
registerManifest(registration);
|
|
382
629
|
registerRepair(registration);
|
|
383
630
|
registerNodes(registration);
|
|
384
631
|
registerAccounts(registration);
|
package/src/cli/registry.ts
CHANGED
|
@@ -116,6 +116,18 @@ export const COMMANDS: CommandSpec[] = [
|
|
|
116
116
|
['manager', 'restore', 'replace', 'lost', 'recovery'],
|
|
117
117
|
true,
|
|
118
118
|
],
|
|
119
|
+
[
|
|
120
|
+
'fleet-validate',
|
|
121
|
+
'Validate the fleet configuration',
|
|
122
|
+
'Read-only: judge fleet.json without contacting anything. An invalid file names the first rule it breaks as {code, path, expected, actual}. --ansible also renders the Ansible inventory and syntax-checks every playbook with the pinned Ansible.',
|
|
123
|
+
['validate', 'lint', 'schema', 'fleet.json', 'configuration', 'ansible'],
|
|
124
|
+
],
|
|
125
|
+
[
|
|
126
|
+
'fleet-manifest',
|
|
127
|
+
'Measure a worker and compare it with a fresh build',
|
|
128
|
+
'Read-only over verified SSH: every installed package, the active mise tools and the state of each always-on service, with a digest that is the same for two workers built from the same revision. The check worker:manifest reports every difference from the declared pins and, with --reference, from a fresh build.',
|
|
129
|
+
['manifest', 'drift', 'reproducible', 'rebuild', 'packages', 'audit'],
|
|
130
|
+
],
|
|
119
131
|
[
|
|
120
132
|
'fleet-node-sync',
|
|
121
133
|
'Sync this node',
|
|
@@ -181,6 +193,41 @@ export const COMMANDS: CommandSpec[] = [
|
|
|
181
193
|
['github', 'verify', 'pull request', 'push', 'permissions', 'workflow'],
|
|
182
194
|
true,
|
|
183
195
|
],
|
|
196
|
+
[
|
|
197
|
+
'fleet-node-replace',
|
|
198
|
+
'Replace a worker with a fresh machine',
|
|
199
|
+
'Preview, or with --apply --yes carry out, putting a fresh machine in place of a declared worker under the same id: verify the new device on the tailnet, pin its host key (--host-key), bind the roster entry to it and issue it a Vault token of its own, revoking the old one. Then fleet-sync --node ID converges it. Running it again changes nothing.',
|
|
200
|
+
['replace', 'rebuild', 'reinstall', 'host key', 'token', 'device'],
|
|
201
|
+
true,
|
|
202
|
+
],
|
|
203
|
+
[
|
|
204
|
+
'fleet-node-seed',
|
|
205
|
+
'Seed a new worker that joins by itself',
|
|
206
|
+
'Write cloud-init user data (or an Ubuntu autoinstall file with --autoinstall) for a declared worker to a private file: its account with the manager key, the privilege rule, a host key generated for it, the pinned Tailscale and a short-lived single-use auth key from the secrets provider (tailnet-auth-key). --apply --yes also pins that host key on the manager, so there is no trust on first use.',
|
|
207
|
+
['seed', 'cloud-init', 'autoinstall', 'new', 'bootstrap', 'provision'],
|
|
208
|
+
true,
|
|
209
|
+
],
|
|
210
|
+
[
|
|
211
|
+
'fleet-node-park',
|
|
212
|
+
'Park a worker that is off on purpose',
|
|
213
|
+
'Record in the fleet state that a declared worker is stopped or destroyed on purpose (a rented machine nobody uses): fleet-sync and fleet-status list it as parked with the time since, never contact it or make it the canary, and it does not change the exit code. Refuses while agent turns run on it, unless --force. Runs the stop command fleet.json declares under parking.stop with the worker ID. --dry-run previews.',
|
|
214
|
+
['park', 'stop', 'pause', 'idle', 'cost', 'rented', 'shutdown'],
|
|
215
|
+
true,
|
|
216
|
+
],
|
|
217
|
+
[
|
|
218
|
+
'fleet-node-resume',
|
|
219
|
+
'Resume a parked worker',
|
|
220
|
+
'Run the start command fleet.json declares under parking.start with the worker ID, mark the worker active in the fleet state, then run fleet-sync --node ID, which converges and verifies it. The exit code is the sync’s. --dry-run previews.',
|
|
221
|
+
['resume', 'start', 'unpark', 'wake', 'rented'],
|
|
222
|
+
true,
|
|
223
|
+
],
|
|
224
|
+
[
|
|
225
|
+
'fleet-node-retire',
|
|
226
|
+
'Retire a worker',
|
|
227
|
+
'Preview, or with --apply --yes carry out, taking a worker out of the fleet: revoke its Vault token, forget its pinned host key, remove its roster entry and record a tombstone in the fleet state. Then remove it from fleet.json.',
|
|
228
|
+
['retire', 'remove', 'decommission', 'revoke', 'delete'],
|
|
229
|
+
true,
|
|
230
|
+
],
|
|
184
231
|
]),
|
|
185
232
|
...group('T3', [
|
|
186
233
|
[
|
package/src/cli/tailnet.ts
CHANGED
|
@@ -5,13 +5,19 @@ import {
|
|
|
5
5
|
appendManagerLinks,
|
|
6
6
|
appendDashboardEvidence,
|
|
7
7
|
} from './managed-health';
|
|
8
|
-
import { discoverTailnet } from '../tailscale';
|
|
8
|
+
import { discoverTailnet, type Tailnet } from '../tailscale';
|
|
9
|
+
import type { FleetNode, NodeReport } from '../types';
|
|
10
|
+
import { appendPublicReachability } from '../public-reachability';
|
|
9
11
|
import { tailnetHealth } from '../tailnet-health';
|
|
10
12
|
import { makeReport, renderReport, reportExitCode } from '../report';
|
|
11
13
|
import { readRoster } from './roster';
|
|
12
14
|
import { publicHubUrl } from '../monitoring/beszel';
|
|
13
15
|
import { stableUrls } from './fleet-sync-output';
|
|
14
16
|
import { timeout } from './options';
|
|
17
|
+
import { appendUsage } from './worker-usage-report';
|
|
18
|
+
import { appendTailnetLeftovers } from '../tailnet-cleanup';
|
|
19
|
+
import { fleet } from '../fleet-config';
|
|
20
|
+
import { withUsage } from '../worker-activity';
|
|
15
21
|
import type { FleetServices } from './fleet';
|
|
16
22
|
import type { Settings } from './settings';
|
|
17
23
|
import type { CliOptions } from './options';
|
|
@@ -57,12 +63,21 @@ export async function executeTailnet(
|
|
|
57
63
|
);
|
|
58
64
|
if (options.managed)
|
|
59
65
|
await appendManagerLinks(nodes, registered, services.run);
|
|
66
|
+
// From the manager, outside the tailnet: optional, it never blocks readiness.
|
|
67
|
+
if (options.workers) await appendPublicReachability(nodes, services.run);
|
|
60
68
|
// The hub runs on the manager, which worker scope deliberately filters out of
|
|
61
69
|
// `tailnet`; its visibility comes from the unfiltered discovery.
|
|
62
70
|
const dashboard = options.workers
|
|
63
|
-
? await
|
|
71
|
+
? await workerFleetEvidence(nodes, discovered, registered, services)
|
|
64
72
|
: null;
|
|
65
|
-
|
|
73
|
+
// Parked workers, and what each active one is doing (uptime, agent turns).
|
|
74
|
+
const usage = options.managed
|
|
75
|
+
? await appendUsage(nodes, registered, {
|
|
76
|
+
run: services.run,
|
|
77
|
+
computers: discovered.computers,
|
|
78
|
+
})
|
|
79
|
+
: null;
|
|
80
|
+
const report = { ...makeReport('health', nodes), ...(usage ?? {}) };
|
|
66
81
|
const exitCode = reportExitCode(report, true);
|
|
67
82
|
const verified = exitCode === 0;
|
|
68
83
|
return {
|
|
@@ -77,16 +92,34 @@ export async function executeTailnet(
|
|
|
77
92
|
inspected: nodes.filter((node) => node.snapshot).length,
|
|
78
93
|
},
|
|
79
94
|
},
|
|
80
|
-
text:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
95
|
+
text: withUsage(
|
|
96
|
+
options.workers
|
|
97
|
+
? workerHealthText(
|
|
98
|
+
{ ...report, ok: verified },
|
|
99
|
+
coverage,
|
|
100
|
+
publicDashboard(dashboard)
|
|
101
|
+
)
|
|
102
|
+
: `${stableUrls(renderReport({ ...report, ok: verified }))}\nCoverage: ${coverage.computers} visible computers; ${coverage.excludedMobile} mobile devices excluded.\n${coverage.limitation}`,
|
|
103
|
+
usage
|
|
104
|
+
),
|
|
87
105
|
};
|
|
88
106
|
}
|
|
89
107
|
|
|
108
|
+
/** What the canonical worker fleet is checked for beyond each worker: the
|
|
109
|
+
* dashboard's evidence, and devices tagged as workers that are no declared
|
|
110
|
+
* worker (#123). Returns the private hub address.
|
|
111
|
+
*/
|
|
112
|
+
async function workerFleetEvidence(
|
|
113
|
+
nodes: NodeReport[],
|
|
114
|
+
discovered: Tailnet,
|
|
115
|
+
registered: FleetNode[],
|
|
116
|
+
services: FleetServices
|
|
117
|
+
): Promise<string | null> {
|
|
118
|
+
const dashboard = await appendDashboardEvidence(nodes, discovered);
|
|
119
|
+
await appendTailnetLeftovers(nodes, fleet, services.tailnet, registered);
|
|
120
|
+
return dashboard;
|
|
121
|
+
}
|
|
122
|
+
|
|
90
123
|
/** Shown to people: the stable public name, never the tailnet address. */
|
|
91
124
|
function publicDashboard(privateHub: string | null): string | null {
|
|
92
125
|
return privateHub ? publicHubUrl() : null;
|
package/src/cli/ui-data.ts
CHANGED
|
@@ -86,7 +86,9 @@ function nodeChecks(value: unknown): CheckRow[] {
|
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
export function isStatus(value: unknown): value is string {
|
|
89
|
-
return ['pass', 'warn', 'fail', 'skip'].includes(
|
|
89
|
+
return ['pass', 'warn', 'fail', 'skip', 'not-applicable'].includes(
|
|
90
|
+
String(value)
|
|
91
|
+
);
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
export function isCancellation(error: unknown): boolean {
|