@coreplane/switchboard 0.0.0 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +17 -1
- package/dist/assets/.dockerignore +27 -0
- package/dist/assets/.env.example +33 -0
- package/dist/assets/Dockerfile +111 -0
- package/dist/assets/config/config.example.yaml +359 -0
- package/dist/assets/deploy/bin/build-stamp.d.mts +15 -0
- package/dist/assets/deploy/bin/build-stamp.mjs +98 -0
- package/dist/assets/deploy/bin/cf-logs +32 -0
- package/dist/assets/deploy/cloudflare/package.json +29 -0
- package/dist/assets/deploy/cloudflare/preflight.mjs +243 -0
- package/dist/assets/deploy/cloudflare/tsconfig.json +18 -0
- package/dist/assets/deploy/cloudflare/worker.ts +382 -0
- package/dist/assets/deploy/cloudflare/wrangler.template.jsonc +67 -0
- package/dist/assets/deploy/cloudflare/write-build.d.mts +7 -0
- package/dist/assets/deploy/cloudflare/write-build.mjs +53 -0
- package/dist/assets/deploy/cloudflare-docs/package.json +18 -0
- package/dist/assets/deploy/cloudflare-docs/wrangler.template.jsonc +30 -0
- package/dist/assets/deploy/cloudflare-memory/package.json +25 -0
- package/dist/assets/deploy/cloudflare-memory/tsconfig.json +17 -0
- package/dist/assets/deploy/cloudflare-memory/worker.ts +2635 -0
- package/dist/assets/deploy/cloudflare-memory/wrangler.template.jsonc +50 -0
- package/dist/assets/deploy/cloudflare-resident/Dockerfile +91 -0
- package/dist/assets/deploy/cloudflare-resident/gc.ts +287 -0
- package/dist/assets/deploy/cloudflare-resident/node-async-hooks.d.ts +11 -0
- package/dist/assets/deploy/cloudflare-resident/package.json +29 -0
- package/dist/assets/deploy/cloudflare-resident/preflight.mjs +224 -0
- package/dist/assets/deploy/cloudflare-resident/tsconfig.json +19 -0
- package/dist/assets/deploy/cloudflare-resident/worker.ts +6637 -0
- package/dist/assets/deploy/cloudflare-resident/wrangler.template.jsonc +120 -0
- package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +67 -0
- package/dist/assets/deploy/cloudflare-sandbox/docker-wrapper.sh +37 -0
- package/dist/assets/deploy/cloudflare-sandbox/package.json +26 -0
- package/dist/assets/deploy/cloudflare-sandbox/tsconfig.json +20 -0
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +410 -0
- package/dist/assets/deploy/cloudflare-sandbox/wrangler.template.jsonc +67 -0
- package/dist/assets/deploy/profile.example.json +13 -0
- package/dist/assets/deploy/secrets.manifest.json +108 -0
- package/dist/assets/docker-entrypoint.sh +15 -0
- package/dist/assets/package-lock.json +18407 -0
- package/dist/assets/package.json +104 -0
- package/dist/assets/project.json +219 -0
- package/dist/assets/source.json +5 -0
- package/dist/assets/src/core/authz/actor.ts +100 -0
- package/dist/assets/src/core/authz/authorize.ts +169 -0
- package/dist/assets/src/core/authz/grants.ts +347 -0
- package/dist/assets/src/core/authz/policy.ts +281 -0
- package/dist/assets/src/core/authz/resource.ts +147 -0
- package/dist/assets/src/core/authz/types.ts +164 -0
- package/dist/assets/src/core/drain.ts +54 -0
- package/dist/assets/src/core/ingressTokens.ts +64 -0
- package/dist/assets/src/core/memory/engine.ts +115 -0
- package/dist/assets/src/core/memory/scorer.ts +147 -0
- package/dist/assets/src/core/memory/types.ts +120 -0
- package/dist/assets/src/core/normalizeSpans.ts +299 -0
- package/dist/assets/src/core/prDescriptionTypes.ts +54 -0
- package/dist/assets/src/core/redact.ts +113 -0
- package/dist/assets/src/core/runEvents.ts +537 -0
- package/dist/assets/src/core/runFriction.ts +665 -0
- package/dist/assets/src/core/runLedger/decisions.ts +126 -0
- package/dist/assets/src/core/runLedger/types.ts +177 -0
- package/dist/assets/src/core/runRecord.ts +627 -0
- package/dist/assets/src/core/runShape.ts +61 -0
- package/dist/assets/src/core/schedules.ts +452 -0
- package/dist/assets/src/core/time/formatDuration.ts +61 -0
- package/dist/assets/src/core/trace/attrs.ts +203 -0
- package/dist/assets/src/core/trace/classify.ts +49 -0
- package/dist/assets/src/core/trace/clock.ts +6 -0
- package/dist/assets/src/core/trace/context.ts +9 -0
- package/dist/assets/src/core/trace/ids.ts +23 -0
- package/dist/assets/src/core/trace/partition.ts +235 -0
- package/dist/assets/src/core/trace/sinks.ts +68 -0
- package/dist/assets/src/core/trace/streamSpans.ts +163 -0
- package/dist/assets/src/core/trace/traceparent.ts +29 -0
- package/dist/assets/src/core/trace/tracer.ts +247 -0
- package/dist/assets/src/core/trace/types.ts +125 -0
- package/dist/assets/src/core/trace/workerTrace.ts +97 -0
- package/dist/assets/src/deploy/buildStamp.ts +93 -0
- package/dist/assets/src/deploy/liveGate.ts +203 -0
- package/dist/assets/src/deploy/profile.ts +162 -0
- package/dist/assets/src/deploy/restart.ts +393 -0
- package/dist/assets/src/effort.ts +17 -0
- package/dist/assets/src/execution/bashTimeout.ts +78 -0
- package/dist/assets/src/execution/bindingPurge.ts +43 -0
- package/dist/assets/src/execution/residentBackupTransfer.ts +50 -0
- package/dist/assets/src/execution/residentCleanliness.ts +95 -0
- package/dist/assets/src/execution/residentCredentials.ts +81 -0
- package/dist/assets/src/execution/residentDepCache.ts +321 -0
- package/dist/assets/src/execution/residentDepsStore.ts +326 -0
- package/dist/assets/src/execution/residentDetach.ts +48 -0
- package/dist/assets/src/execution/residentDisk.ts +107 -0
- package/dist/assets/src/execution/residentDiskBudget.ts +448 -0
- package/dist/assets/src/execution/residentExecWrap.ts +100 -0
- package/dist/assets/src/execution/residentHead.ts +85 -0
- package/dist/assets/src/execution/residentReadonly.ts +72 -0
- package/dist/assets/src/execution/residentRefresh.ts +429 -0
- package/dist/assets/src/execution/residentRestoreExtract.ts +130 -0
- package/dist/assets/src/execution/residentState.ts +47 -0
- package/dist/assets/src/execution/residentStepReport.ts +98 -0
- package/dist/assets/src/execution/residentStepTrace.ts +97 -0
- package/dist/assets/src/execution/residentSteps.ts +99 -0
- package/dist/assets/src/execution/residentText.ts +83 -0
- package/dist/assets/src/execution/residentTrace.ts +119 -0
- package/dist/assets/src/execution/sandboxEnv.ts +42 -0
- package/dist/assets/src/execution/sandboxErrors.ts +159 -0
- package/dist/assets/src/execution/sandboxKeepalive.ts +118 -0
- package/dist/assets/src/execution/shellQuote.ts +8 -0
- package/dist/assets/src/mcp/registry.ts +242 -0
- package/dist/assets/src/providers/types.ts +152 -0
- package/dist/assets/web/dist/.vite/manifest.json +176 -0
- package/dist/assets/web/dist/assets/AppShell-Bk2gbvet.js +1 -0
- package/dist/assets/web/dist/assets/CostsPage-CTZcMYYx.js +1 -0
- package/dist/assets/web/dist/assets/NotFoundPage-C-BuaSm8.js +1 -0
- package/dist/assets/web/dist/assets/ResidentDetailPage-DvQ05AGa.js +1 -0
- package/dist/assets/web/dist/assets/ResidentsIndexPage-B3uxKUne.js +1 -0
- package/dist/assets/web/dist/assets/RunRoutePage-XVFj0XDc.css +1 -0
- package/dist/assets/web/dist/assets/RunRoutePage-ty94olNM.js +126 -0
- package/dist/assets/web/dist/assets/RunsIndexPage-CM-qxyQm.js +1 -0
- package/dist/assets/web/dist/assets/RunsTabs-C4krAL9o.js +1 -0
- package/dist/assets/web/dist/assets/ScheduledPage-C1psvLD4.js +1 -0
- package/dist/assets/web/dist/assets/StatusDot-DuoQnQeU.js +1 -0
- package/dist/assets/web/dist/assets/Tooltip-BfLPyxQy.js +1 -0
- package/dist/assets/web/dist/assets/favicon-DL1rdWJt.js +1 -0
- package/dist/assets/web/dist/assets/localIso-L06jV29p.js +1 -0
- package/dist/assets/web/dist/assets/main-Bnbk_Rsg.js +28 -0
- package/dist/assets/web/dist/assets/main-BsBGUyMH.css +2 -0
- package/dist/assets/web/dist/assets/residentDiskBudget-BMBKlYRH.js +1 -0
- package/dist/assets/web/dist/assets/seed-BglCRKLA.js +6 -0
- package/dist/assets/web/dist/assets/wallClock-Ckv3sKoR.js +1 -0
- package/dist/cli.js +34494 -0
- package/package.json +43 -10
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
import { parseIngressTokenMap, type IngressIdentity } from "../core/ingressTokens.js";
|
|
2
|
+
import { hasAction } from "../core/authz/authorize.js";
|
|
3
|
+
import type { GrantsLookup } from "../core/authz/actor.js";
|
|
4
|
+
import { LIVE_GATE_DEADLINE_MS, parseHealthz, type HealthzBody } from "./liveGate.js";
|
|
5
|
+
import { profileUrls, type DeploymentProfile } from "./profile.js";
|
|
6
|
+
|
|
7
|
+
// `deploy restart` — restart the bot container WITHOUT an image build, so a
|
|
8
|
+
// rotated bot secret goes live in seconds instead of a full `deploy all --only
|
|
9
|
+
// bot` (docs/reference/specs/slack-channel.md item 8). Cloudflare's model: `wrangler secret
|
|
10
|
+
// put` updates the Worker's env, but a running container keeps the env it
|
|
11
|
+
// started with, and a rollout only happens on an image/config change. The
|
|
12
|
+
// documented restart is the Container DO calling `stop()` (SIGTERM → the bot's
|
|
13
|
+
// graceful drain finishes in-flight runs and exits) and the NEXT request
|
|
14
|
+
// starting it again — with envVars computed at start time from the DO's
|
|
15
|
+
// current env (deploy/cloudflare/worker.ts). The Worker exposes that as
|
|
16
|
+
// `POST /admin/restart`; this module is the pure half shared by the Worker
|
|
17
|
+
// (authorization, the refusal decision, the wire shapes) and the CLI runner
|
|
18
|
+
// (src/deploy/run.ts `runBotRestart`), so both sides are unit-tested here and
|
|
19
|
+
// nothing in this file imports node:*.
|
|
20
|
+
//
|
|
21
|
+
// Authorization: the bearer must be an entry of the bot's own
|
|
22
|
+
// `SWITCHBOARD_INGRESS_TOKENS` map whose `http:<subject>` actor holds
|
|
23
|
+
// `deploy:write` in the bot's config (`grants`, authorization.md item 9) — the
|
|
24
|
+
// very action the `deploy.restart` command declares, so the Worker route is
|
|
25
|
+
// authorized exactly as `/api/deploy.restart` would be if the bot served it.
|
|
26
|
+
// Two halves, because the two sides hold different things: the Worker holds the
|
|
27
|
+
// token map (it fires scheduled runs with the `cron` entry) and can tell WHO a
|
|
28
|
+
// bearer is — `authenticateRestart`, 401/503 without touching the container —
|
|
29
|
+
// but the grants live in the container's config, so it asks the bot
|
|
30
|
+
// (`POST /admin/restart/authorize`, src/channels/adminRestartAuthorize.ts) whether
|
|
31
|
+
// that subject holds the action and relays the answer (`parseRestartAuthorization`).
|
|
32
|
+
// The Worker names the subject it authenticated in `RESTART_SUBJECT_HEADER` and
|
|
33
|
+
// the bot decides the grant for THAT subject without re-authenticating the
|
|
34
|
+
// bearer: the two sides can hold different generations of the token map (a
|
|
35
|
+
// `wrangler secret put` reaches the Worker's env at once and the container's
|
|
36
|
+
// only after a restart), so re-authenticating in the container would refuse the
|
|
37
|
+
// very rotation the restart exists to finish. The header is trustworthy because
|
|
38
|
+
// the container is reachable only through the Worker, which strips it from every
|
|
39
|
+
// proxied request (`stripRestartSubject`) and sets it only on its own internal
|
|
40
|
+
// call. The bot's own `/admin/crash` runs the whole check in one place
|
|
41
|
+
// (`authorizeRestart`).
|
|
42
|
+
//
|
|
43
|
+
// The route's URL is the installation's: the deployment profile names the
|
|
44
|
+
// bot's hostname, `planRestart` derives `https://<bot>/admin/restart` from it.
|
|
45
|
+
|
|
46
|
+
/** The operator's env var holding a `SWITCHBOARD_INGRESS_TOKENS` bearer with `deploy:write`. */
|
|
47
|
+
export const RESTART_TOKEN_ENV = "SWITCHBOARD_DEPLOY_TOKEN";
|
|
48
|
+
/** The scope the bearer's identity must carry — the `deploy.restart` command's own. */
|
|
49
|
+
export const RESTART_SCOPE = "deploy:write";
|
|
50
|
+
|
|
51
|
+
// ---- refusal decision (Worker side; the CLI relies on the 409) -----------------------------
|
|
52
|
+
|
|
53
|
+
export interface RestartVerdict {
|
|
54
|
+
allow: boolean;
|
|
55
|
+
forced: boolean;
|
|
56
|
+
/** What refuses (fail-closed: no JSON body, an impossible count). */
|
|
57
|
+
problems: string[];
|
|
58
|
+
/** What is said but does not refuse: runs in flight (they hand off), a drain under way. */
|
|
59
|
+
warnings: string[];
|
|
60
|
+
message: string;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Whether the container may be stopped now — the deploy preflight's rules
|
|
64
|
+
* (deploy/cloudflare/preflight.mjs `decide`) minus the rollout-state check (a
|
|
65
|
+
* restart is not a rollout). Since the handoff (docs/reference/specs/run-history.md item
|
|
66
|
+
* 39) runs in flight and a drain under way are WARNINGS, not refusals: SIGTERM
|
|
67
|
+
* hands every resumable run to the next generation. Fail closed on a body
|
|
68
|
+
* that is not JSON or an impossible count; `force` allows those anyway, with
|
|
69
|
+
* the warning.
|
|
70
|
+
*/
|
|
71
|
+
export function decideRestart(body: HealthzBody | undefined, opts: { force: boolean }): RestartVerdict {
|
|
72
|
+
const problems: string[] = [];
|
|
73
|
+
const warnings: string[] = [];
|
|
74
|
+
if (!body) {
|
|
75
|
+
problems.push(
|
|
76
|
+
"bot not answering with JSON on /healthz (container restarting, unreachable, or a Worker that predates the preflight)",
|
|
77
|
+
);
|
|
78
|
+
} else {
|
|
79
|
+
if (!Number.isInteger(body.inFlight) || (body.inFlight as number) < 0) {
|
|
80
|
+
problems.push(`bot reports an impossible inFlight=${JSON.stringify(body.inFlight)} (counter bug or old Worker)`);
|
|
81
|
+
} else if ((body.inFlight as number) > 0) {
|
|
82
|
+
warnings.push(
|
|
83
|
+
`${body.inFlight} run(s) in flight — handed to the next generation on SIGTERM (run-history item 39); they continue there`,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
if (body.draining === true)
|
|
87
|
+
warnings.push(
|
|
88
|
+
"bot is already draining (a deploy or an earlier restart is in progress) — it restarts on its own when the drain ends; a second stop is harmless",
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
const said =
|
|
92
|
+
warnings.length > 0 ? ` —\n${warnings.map((w) => ` - ${w}`).join("\n")}` : ": no runs in flight, not draining";
|
|
93
|
+
if (problems.length === 0) return { allow: true, forced: false, problems, warnings, message: `restart ok${said}` };
|
|
94
|
+
const detail = problems.map((p) => ` - ${p}`).join("\n");
|
|
95
|
+
if (opts.force)
|
|
96
|
+
return {
|
|
97
|
+
allow: true,
|
|
98
|
+
forced: true,
|
|
99
|
+
problems,
|
|
100
|
+
warnings,
|
|
101
|
+
message: `restart WARNING: stopping by force despite —\n${detail}`,
|
|
102
|
+
};
|
|
103
|
+
return {
|
|
104
|
+
allow: false,
|
|
105
|
+
forced: false,
|
|
106
|
+
problems,
|
|
107
|
+
warnings,
|
|
108
|
+
message: `restart REFUSED —\n${detail}\n wait and retry, or pass --force to stop blind`,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ---- authorization (Worker side) ---------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
/** Byte-wise equality whose running time depends only on the lengths, never on
|
|
115
|
+
* where the first difference is. */
|
|
116
|
+
export function constantTimeEqual(a: string, b: string): boolean {
|
|
117
|
+
const enc = new TextEncoder();
|
|
118
|
+
const ab = enc.encode(a);
|
|
119
|
+
const bb = enc.encode(b);
|
|
120
|
+
let diff = ab.length ^ bb.length;
|
|
121
|
+
for (let i = 0; i < Math.max(ab.length, bb.length); i++) diff |= (ab[i] ?? 0) ^ (bb[i] ?? 0);
|
|
122
|
+
return diff === 0;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Find the presented bearer among the configured tokens by comparing against
|
|
126
|
+
* EVERY entry (fixed work; a plain object-key lookup would let a probe learn
|
|
127
|
+
* which prefixes exist from timing). Returns the matched identity, if any. */
|
|
128
|
+
export function lookupConstantTime<T>(tokens: Record<string, T>, presented: string): T | undefined {
|
|
129
|
+
let found: T | undefined;
|
|
130
|
+
for (const [token, identity] of Object.entries(tokens)) if (constantTimeEqual(token, presented)) found = identity;
|
|
131
|
+
return found;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export type RestartAuth = { ok: true; subject: string } | { ok: false; status: 401 | 403 | 503; reason: string };
|
|
135
|
+
export type RestartAuthn = { ok: true; identity: IngressIdentity } | { ok: false; status: 401 | 503; reason: string };
|
|
136
|
+
|
|
137
|
+
/** The bot route the Worker asks before stopping the container: 200 `{ ok, subject }`
|
|
138
|
+
* when the bearer's actor holds `deploy:write`, else `authorizeRestart`'s 401/403/503. */
|
|
139
|
+
export const RESTART_AUTHORIZE_PATH = "/admin/restart/authorize";
|
|
140
|
+
/** The header the Worker sets on its internal authorize call, naming the subject it
|
|
141
|
+
* authenticated from its own token map; stripped from every proxied request. */
|
|
142
|
+
export const RESTART_SUBJECT_HEADER = "x-switchboard-restart-subject";
|
|
143
|
+
|
|
144
|
+
/** WHETHER, for a subject the Worker already authenticated: the bot's grants alone. */
|
|
145
|
+
export function authorizeRestartSubject(subject: string, grantsFor: GrantsLookup): RestartAuth {
|
|
146
|
+
if (subject.trim() === "") return { ok: false, status: 401, reason: "unauthorized: an empty subject" };
|
|
147
|
+
if (!hasAction(grantsFor(`http:${subject}`).actions, RESTART_SCOPE))
|
|
148
|
+
return {
|
|
149
|
+
ok: false,
|
|
150
|
+
status: 403,
|
|
151
|
+
reason: `forbidden: identity "${subject}" holds no ${RESTART_SCOPE} grant (grants["http:${subject}"] in config.yaml)`,
|
|
152
|
+
};
|
|
153
|
+
return { ok: true, subject };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** The request without the subject header — what the Worker forwards to the container for
|
|
157
|
+
* every route it does not answer itself, so a caller can never assert a subject. */
|
|
158
|
+
export function stripRestartSubject(request: Request): Request {
|
|
159
|
+
if (!request.headers.has(RESTART_SUBJECT_HEADER)) return request;
|
|
160
|
+
const headers = new Headers(request.headers);
|
|
161
|
+
headers.delete(RESTART_SUBJECT_HEADER);
|
|
162
|
+
return new Request(request, { headers });
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** WHO the bearer is — the Worker's half. Check `Authorization: Bearer <token>`
|
|
166
|
+
* against the raw `SWITCHBOARD_INGRESS_TOKENS` value: no usable map → 503 (the
|
|
167
|
+
* route is disabled, never open); no/unknown bearer → 401. Says nothing about
|
|
168
|
+
* what the identity may do. Never echoes token material. */
|
|
169
|
+
export function authenticateRestart(authorization: string | undefined, tokensRaw: string | undefined): RestartAuthn {
|
|
170
|
+
return authenticateIngressBearer(authorization, tokensRaw, "restart");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Who the bearer is, from the ingress token map — the same map every
|
|
174
|
+
* `/admin/*` route on the bot checks (`feature` names the route in the 503,
|
|
175
|
+
* which is what an operator sees when the map is missing). */
|
|
176
|
+
export function authenticateIngressBearer(
|
|
177
|
+
authorization: string | undefined,
|
|
178
|
+
tokensRaw: string | undefined,
|
|
179
|
+
feature: string,
|
|
180
|
+
): RestartAuthn {
|
|
181
|
+
const parsed = parseIngressTokenMap(tokensRaw);
|
|
182
|
+
if (!parsed.ok || Object.keys(parsed.tokens).length === 0) {
|
|
183
|
+
return {
|
|
184
|
+
ok: false,
|
|
185
|
+
status: 503,
|
|
186
|
+
reason: `${feature} disabled: SWITCHBOARD_INGRESS_TOKENS is ${parsed.ok ? "not set" : parsed.reason}`,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
const m = /^Bearer\s+(\S+)$/i.exec(authorization ?? "");
|
|
190
|
+
const identity = m ? lookupConstantTime(parsed.tokens, m[1]) : undefined;
|
|
191
|
+
if (!identity)
|
|
192
|
+
return {
|
|
193
|
+
ok: false,
|
|
194
|
+
status: 401,
|
|
195
|
+
reason: "unauthorized: a Bearer token from SWITCHBOARD_INGRESS_TOKENS is required",
|
|
196
|
+
};
|
|
197
|
+
return { ok: true, identity };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** WHO and WHETHER — the whole check, where the grants are (the bot). The
|
|
201
|
+
* bearer's `http:<subject>` actor must hold `deploy:write` (`grantsFor` —
|
|
202
|
+
* config's entry for the token's subject); a known identity without it → 403. */
|
|
203
|
+
export function authorizeRestart(
|
|
204
|
+
authorization: string | undefined,
|
|
205
|
+
tokensRaw: string | undefined,
|
|
206
|
+
grantsFor: GrantsLookup,
|
|
207
|
+
): RestartAuth {
|
|
208
|
+
const authn = authenticateRestart(authorization, tokensRaw);
|
|
209
|
+
if (!authn.ok) return authn;
|
|
210
|
+
return authorizeRestartSubject(authn.identity.subject, grantsFor);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Any other `/admin/*` route's whole check: an ingress bearer (401/503) whose
|
|
214
|
+
* actor `http:<subject>` holds `scope` (403) — the restart's rule with the
|
|
215
|
+
* scope as a parameter, so no route invents its own. */
|
|
216
|
+
export function authorizeIngressBearer(
|
|
217
|
+
authorization: string | undefined,
|
|
218
|
+
tokensRaw: string | undefined,
|
|
219
|
+
grantsFor: GrantsLookup,
|
|
220
|
+
scope: string,
|
|
221
|
+
feature: string,
|
|
222
|
+
): RestartAuth {
|
|
223
|
+
const authn = authenticateIngressBearer(authorization, tokensRaw, feature);
|
|
224
|
+
if (!authn.ok) return authn;
|
|
225
|
+
const { identity } = authn;
|
|
226
|
+
if (!hasAction(grantsFor(`http:${identity.subject}`).actions, scope))
|
|
227
|
+
return {
|
|
228
|
+
ok: false,
|
|
229
|
+
status: 403,
|
|
230
|
+
reason: `forbidden: identity "${identity.subject}" holds no ${scope} grant (grants["http:${identity.subject}"] in config.yaml)`,
|
|
231
|
+
};
|
|
232
|
+
return { ok: true, subject: identity.subject };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** The bot's `POST /admin/restart/authorize` answer, as the Worker reads it: 200
|
|
236
|
+
* `{ ok: true, subject }` → allowed; 401 / 403 / 503 `{ ok: false, error }` →
|
|
237
|
+
* relayed as they are; anything else (a bot without the route, a non-JSON body,
|
|
238
|
+
* an unexpected status) → 503, fail-closed — the Worker never restarts on an
|
|
239
|
+
* answer it cannot read. */
|
|
240
|
+
export function parseRestartAuthorization(status: number, text: string): RestartAuth {
|
|
241
|
+
let parsed: unknown;
|
|
242
|
+
try {
|
|
243
|
+
parsed = JSON.parse(text);
|
|
244
|
+
} catch {
|
|
245
|
+
parsed = undefined;
|
|
246
|
+
}
|
|
247
|
+
const body = typeof parsed === "object" && parsed !== null ? (parsed as Record<string, unknown>) : undefined;
|
|
248
|
+
if (status === 200 && body?.ok === true && typeof body.subject === "string" && body.subject !== "")
|
|
249
|
+
return { ok: true, subject: body.subject };
|
|
250
|
+
if ((status === 401 || status === 403 || status === 503) && body?.ok === false && typeof body.error === "string")
|
|
251
|
+
return { ok: false, status, reason: body.error };
|
|
252
|
+
return {
|
|
253
|
+
ok: false,
|
|
254
|
+
status: 503,
|
|
255
|
+
reason: `restart disabled: the bot did not answer the authorization check (HTTP ${status}) — is it running this version?`,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ---- wire shapes ----------------------------------------------------------------------------------
|
|
260
|
+
|
|
261
|
+
export type ParsedRestartRequest = { ok: true; force: boolean } | { ok: false; reason: string };
|
|
262
|
+
|
|
263
|
+
/** The request body: empty, or a JSON object with an optional boolean `force`. */
|
|
264
|
+
export function parseRestartRequest(text: string): ParsedRestartRequest {
|
|
265
|
+
if (text.trim() === "") return { ok: true, force: false };
|
|
266
|
+
let parsed: unknown;
|
|
267
|
+
try {
|
|
268
|
+
parsed = JSON.parse(text);
|
|
269
|
+
} catch {
|
|
270
|
+
return { ok: false, reason: "body is not valid JSON" };
|
|
271
|
+
}
|
|
272
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
|
|
273
|
+
return { ok: false, reason: "body must be a JSON object" };
|
|
274
|
+
const force = (parsed as { force?: unknown }).force;
|
|
275
|
+
if (force !== undefined && typeof force !== "boolean") return { ok: false, reason: "`force` must be a boolean" };
|
|
276
|
+
return { ok: true, force: force === true };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** What the Container DO reports back to the Worker route. */
|
|
280
|
+
export type RestartOutcome =
|
|
281
|
+
/** The container is not running: nothing to stop; the next request starts it with the current env. */
|
|
282
|
+
| { kind: "not-running" }
|
|
283
|
+
| { kind: "refused"; problems: string[] }
|
|
284
|
+
/** SIGTERM sent; `previousStartedAt` is what the OLD container reported (the CLI's baseline). */
|
|
285
|
+
| { kind: "stopping"; forced: boolean; inFlight: number; previousStartedAt: string | undefined };
|
|
286
|
+
|
|
287
|
+
export interface RestartHttpResponse {
|
|
288
|
+
status: number;
|
|
289
|
+
body: Record<string, unknown>;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export function restartResponse(outcome: RestartOutcome): RestartHttpResponse {
|
|
293
|
+
switch (outcome.kind) {
|
|
294
|
+
case "stopping":
|
|
295
|
+
return {
|
|
296
|
+
status: 202,
|
|
297
|
+
body: {
|
|
298
|
+
ok: true,
|
|
299
|
+
stopping: true,
|
|
300
|
+
forced: outcome.forced,
|
|
301
|
+
inFlight: outcome.inFlight,
|
|
302
|
+
previousStartedAt: outcome.previousStartedAt,
|
|
303
|
+
},
|
|
304
|
+
};
|
|
305
|
+
case "refused":
|
|
306
|
+
return { status: 409, body: { ok: false, refused: true, problems: outcome.problems } };
|
|
307
|
+
case "not-running":
|
|
308
|
+
return {
|
|
309
|
+
status: 200,
|
|
310
|
+
body: {
|
|
311
|
+
ok: true,
|
|
312
|
+
stopping: false,
|
|
313
|
+
note: "container not running — nothing to stop; the next request starts it with the current env",
|
|
314
|
+
},
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** The CLI's reading of the route's answer. */
|
|
320
|
+
export type RestartResponseClass =
|
|
321
|
+
| { kind: "stopping"; previousStartedAt: string | undefined }
|
|
322
|
+
| { kind: "not-running" }
|
|
323
|
+
/** 409 — retryable while runs finish. */
|
|
324
|
+
| { kind: "refused"; reason: string }
|
|
325
|
+
| { kind: "unauthorized"; reason: string }
|
|
326
|
+
| { kind: "failed"; reason: string };
|
|
327
|
+
|
|
328
|
+
export function classifyRestartResponse(status: number, text: string): RestartResponseClass {
|
|
329
|
+
const body = parseHealthz(text) as Record<string, unknown> | undefined;
|
|
330
|
+
if (status === 202)
|
|
331
|
+
return {
|
|
332
|
+
kind: "stopping",
|
|
333
|
+
previousStartedAt: typeof body?.previousStartedAt === "string" ? body.previousStartedAt : undefined,
|
|
334
|
+
};
|
|
335
|
+
if (status === 200 && body?.stopping === false) return { kind: "not-running" };
|
|
336
|
+
if (status === 409) {
|
|
337
|
+
const problems = Array.isArray(body?.problems)
|
|
338
|
+
? body.problems.filter((p): p is string => typeof p === "string")
|
|
339
|
+
: [];
|
|
340
|
+
return { kind: "refused", reason: problems[0] ?? "restart refused" };
|
|
341
|
+
}
|
|
342
|
+
if (status === 401 || status === 403)
|
|
343
|
+
return { kind: "unauthorized", reason: `HTTP ${status}: ${text.slice(0, 200)}` };
|
|
344
|
+
return { kind: "failed", reason: `HTTP ${status}: ${text.slice(0, 200)}` };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ---- the plan (what `deploy restart` does, as data) --------------------------------------------
|
|
348
|
+
|
|
349
|
+
export interface RestartOptions {
|
|
350
|
+
/** The only supported target for now: the bot is the one Worker with a long-lived container. */
|
|
351
|
+
only: "bot";
|
|
352
|
+
force: boolean;
|
|
353
|
+
waitMaxMinutes: number;
|
|
354
|
+
pollSeconds: number;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export interface RestartPlan {
|
|
358
|
+
target: "bot";
|
|
359
|
+
adminUrl: string;
|
|
360
|
+
healthUrl: string;
|
|
361
|
+
tokenEnv: string;
|
|
362
|
+
force: boolean;
|
|
363
|
+
/** Budget for waiting out a 409 (runs in flight) before giving up. */
|
|
364
|
+
waitMaxMs: number;
|
|
365
|
+
pollMs: number;
|
|
366
|
+
/** Budget for the new container to answer with a later `startedAt` (drain + cold start). */
|
|
367
|
+
liveDeadlineMs: number;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
export function planRestart(opts: RestartOptions, profile: DeploymentProfile): RestartPlan {
|
|
371
|
+
const urls = profileUrls(profile);
|
|
372
|
+
return {
|
|
373
|
+
target: opts.only,
|
|
374
|
+
adminUrl: urls.botAdminRestartUrl,
|
|
375
|
+
healthUrl: `${urls.publicBaseUrl}/healthz`,
|
|
376
|
+
tokenEnv: RESTART_TOKEN_ENV,
|
|
377
|
+
force: opts.force,
|
|
378
|
+
waitMaxMs: opts.waitMaxMinutes * 60_000,
|
|
379
|
+
pollMs: opts.pollSeconds * 1000,
|
|
380
|
+
liveDeadlineMs: LIVE_GATE_DEADLINE_MS,
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
export function formatRestartPlan(plan: RestartPlan): string {
|
|
385
|
+
const gate = plan.force
|
|
386
|
+
? `preflight FORCED — in-flight runs are drained (SIGTERM), killed only at the drain deadline`
|
|
387
|
+
: `refused while runs are in flight or draining (409) — retry every ${plan.pollMs / 1000}s up to ${plan.waitMaxMs / 60_000} min`;
|
|
388
|
+
return [
|
|
389
|
+
`Restart ${plan.target}: POST ${plan.adminUrl} (bearer from $${plan.tokenEnv}, needs ${RESTART_SCOPE}) — ${gate}`,
|
|
390
|
+
` then wait until ${plan.healthUrl} answers not draining with a later startedAt (up to ${Math.round(plan.liveDeadlineMs / 60_000)} min: drain + cold start)`,
|
|
391
|
+
` no image build: the container restarts on the same build with the Worker's CURRENT secrets`,
|
|
392
|
+
].join("\n");
|
|
393
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Model effort: how hard the model thinks per turn (Anthropic
|
|
2
|
+
// `output_config.effort`; skipped for models without support). A first-class
|
|
3
|
+
// config dimension resolved through the SAME layers as the model ref
|
|
4
|
+
// (docs/reference/specs/routing-and-config.md item 2): request directive > thread-sticky
|
|
5
|
+
// > user scope > channel scope > defaults > the agent definition > the
|
|
6
|
+
// provider's own default. Lower effort = much faster turns; the wall clock is
|
|
7
|
+
// the real budget, so effort is what decides how much of it goes to thinking.
|
|
8
|
+
|
|
9
|
+
export const EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"] as const;
|
|
10
|
+
export type Effort = (typeof EFFORT_LEVELS)[number];
|
|
11
|
+
|
|
12
|
+
export function isEffort(value: unknown): value is Effort {
|
|
13
|
+
return typeof value === "string" && (EFFORT_LEVELS as readonly string[]).includes(value);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The valid levels, for error messages: `low, medium, high, xhigh, max`. */
|
|
17
|
+
export const EFFORT_LEVELS_HINT = EFFORT_LEVELS.join(", ");
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Per-call bash timeout policy (docs/reference/specs/execution.md item 11): ONE clamp and
|
|
2
|
+
// one set of bounds, shared by the tool layer, every executor, and both deploy
|
|
3
|
+
// Workers (which import from src/execution like shellQuote/residentDetach).
|
|
4
|
+
// Deliberately free of node: imports so wrangler can bundle it into Workers.
|
|
5
|
+
|
|
6
|
+
/** Default per-command budget when the caller passes no timeoutMs. */
|
|
7
|
+
export const BASH_TIMEOUT_MS = 5 * 60_000;
|
|
8
|
+
|
|
9
|
+
/** Floor: anything lower is a typo or an attack, not a budget. */
|
|
10
|
+
export const BASH_TIMEOUT_MIN_MS = 1_000;
|
|
11
|
+
|
|
12
|
+
/** Hard ceiling a caller can raise the budget to. Not by itself a guarantee
|
|
13
|
+
* against one command eating a run — 20 minutes is 80% of a 25-minute review
|
|
14
|
+
* — which is what `bashBudgetWithinRun` below is for. */
|
|
15
|
+
export const BASH_TIMEOUT_MAX_MS = 20 * 60_000;
|
|
16
|
+
|
|
17
|
+
/** Wall clock a run keeps back from its last command for the write-up: the
|
|
18
|
+
* runner's deadline forces a final answer, and a command still running at
|
|
19
|
+
* that moment would have been wasted anyway. */
|
|
20
|
+
export const RUN_DEADLINE_RESERVE_MS = 60_000;
|
|
21
|
+
|
|
22
|
+
/** Margin the remote exec clients add to their HTTP wait over the command
|
|
23
|
+
* budget, so the server's own timeout answer (a streamed exit 124) wins the
|
|
24
|
+
* race against the client's transport deadline instead of both firing at the
|
|
25
|
+
* same instant. */
|
|
26
|
+
export const EXEC_CALL_MARGIN_MS = 30_000;
|
|
27
|
+
|
|
28
|
+
/** The documented clamp rule, applied identically bot-side and server-side
|
|
29
|
+
* (a server never trusts the client's number): a finite number is truncated
|
|
30
|
+
* to an integer and clamped into [BASH_TIMEOUT_MIN_MS, BASH_TIMEOUT_MAX_MS]
|
|
31
|
+
* — so 0/negative run at the 1s floor, a 25-minute ask runs at 20 minutes —
|
|
32
|
+
* and anything else (absent, NaN, Infinity, a string) falls back to the
|
|
33
|
+
* 5-minute default. */
|
|
34
|
+
export function clampBashTimeout(requested: unknown): number {
|
|
35
|
+
if (typeof requested !== "number" || !Number.isFinite(requested)) return BASH_TIMEOUT_MS;
|
|
36
|
+
return Math.min(Math.max(Math.trunc(requested), BASH_TIMEOUT_MIN_MS), BASH_TIMEOUT_MAX_MS);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** What a command may actually get when the run's wall clock ends in
|
|
40
|
+
* `remainingMs`: `unchanged` when the wanted budget fits before the reserve,
|
|
41
|
+
* `clipped` to what fits (with the line the model sees so it knows why the
|
|
42
|
+
* command ended early), or `exhausted` when even the 1s floor does not fit —
|
|
43
|
+
* the tool then refuses to start a command that cannot finish, and the model
|
|
44
|
+
* writes up what it has. Without the reserve one first command at the
|
|
45
|
+
* 20-minute ceiling can consume most of a 25-minute run budget and leave
|
|
46
|
+
* nothing for the write-up. */
|
|
47
|
+
export type RunBudget =
|
|
48
|
+
{ kind: "unchanged" } | { kind: "clipped"; timeoutMs: number; note: string } | { kind: "exhausted"; note: string };
|
|
49
|
+
|
|
50
|
+
export function bashBudgetWithinRun(wantedMs: number, remainingMs: number): RunBudget {
|
|
51
|
+
const secs = (ms: number) => Math.max(0, Math.round(ms / 1000));
|
|
52
|
+
const cap = Math.trunc(remainingMs - RUN_DEADLINE_RESERVE_MS);
|
|
53
|
+
if (cap < BASH_TIMEOUT_MIN_MS) {
|
|
54
|
+
return {
|
|
55
|
+
kind: "exhausted",
|
|
56
|
+
note:
|
|
57
|
+
`run budget exhausted — ${secs(remainingMs)}s of wall clock left, inside the ` +
|
|
58
|
+
`${secs(RUN_DEADLINE_RESERVE_MS)}s write-up reserve, so the command was not run; write up what you have now`,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
if (cap >= wantedMs) return { kind: "unchanged" };
|
|
62
|
+
return {
|
|
63
|
+
kind: "clipped",
|
|
64
|
+
timeoutMs: cap,
|
|
65
|
+
note: `[timeout clipped to ${secs(cap)}s — the run's wall clock ends in ${secs(remainingMs)}s]`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The line a timed-out command shows the model: names the limit that fired
|
|
70
|
+
* and the knob that raises it, so the model can self-correct (re-run with a
|
|
71
|
+
* larger timeoutMs, split the command, or background it) instead of guessing
|
|
72
|
+
* at a generic abort. */
|
|
73
|
+
export function bashTimeoutNote(timeoutMs: number): string {
|
|
74
|
+
return (
|
|
75
|
+
`aborted at the ${Math.round(timeoutMs / 1000)}s command timeout ` +
|
|
76
|
+
`(pass the bash tool's timeoutMs for longer commands, max ${BASH_TIMEOUT_MAX_MS} ms)`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// The resident keeps a thread binding after eviction by design
|
|
2
|
+
// (docs/reference/specs/resident-repos.md item 23): the ref stays sticky for the next
|
|
3
|
+
// attach. A load run that attaches fifty synthetic threads therefore leaves
|
|
4
|
+
// fifty evicted rows on the resident's detail page forever. `POST /debug
|
|
5
|
+
// {op: "purge-bindings", prefix}` (item 60) deletes exactly the bindings a
|
|
6
|
+
// harness created and no longer needs. This is its decision, pure and
|
|
7
|
+
// imported by the resident Worker like `residentDiskBudget.ts`.
|
|
8
|
+
|
|
9
|
+
export interface PurgeableBinding {
|
|
10
|
+
threadKey: string;
|
|
11
|
+
evicted?: boolean;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export type PurgeDecision = { ok: true; purge: string[]; keptLive: string[] } | { ok: false; error: string };
|
|
15
|
+
|
|
16
|
+
/** Thread-key namespaces real channels mint (docs/reference/specs/http-ingress.md item 1,
|
|
17
|
+
* slack, mcp, the CLI). A purge is for synthetic keys only. */
|
|
18
|
+
const PRODUCTION_NAMESPACES = new Set(["slack", "http", "mcp", "cli"]);
|
|
19
|
+
|
|
20
|
+
/** A prefix must be a whole namespace (`load:`) or longer (`load:r1:`), never
|
|
21
|
+
* empty, never a bare partial namespace, never a production namespace. */
|
|
22
|
+
export function selectBindingsToPurge(bindings: readonly PurgeableBinding[], prefix: string): PurgeDecision {
|
|
23
|
+
const m = /^([a-z][a-z0-9-]{0,31}):/.exec(prefix);
|
|
24
|
+
if (!m)
|
|
25
|
+
return {
|
|
26
|
+
ok: false,
|
|
27
|
+
error: `prefix must name a whole thread-key namespace, like "load:" (got ${JSON.stringify(prefix)})`,
|
|
28
|
+
};
|
|
29
|
+
if (PRODUCTION_NAMESPACES.has(m[1])) {
|
|
30
|
+
return {
|
|
31
|
+
ok: false,
|
|
32
|
+
error: `prefix ${JSON.stringify(prefix)} is a production namespace; a purge is for synthetic thread keys only`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
const purge: string[] = [];
|
|
36
|
+
const keptLive: string[] = [];
|
|
37
|
+
for (const b of bindings) {
|
|
38
|
+
if (!b.threadKey.startsWith(prefix)) continue;
|
|
39
|
+
if (b.evicted === true) purge.push(b.threadKey);
|
|
40
|
+
else keptLive.push(b.threadKey);
|
|
41
|
+
}
|
|
42
|
+
return { ok: true, purge, keptLive };
|
|
43
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Which way a resident's snapshot bytes travel (docs/reference/specs/resident-repos.md
|
|
2
|
+
* item 61), kept pure so the decision is a unit test and the Worker only
|
|
3
|
+
* reads it.
|
|
4
|
+
*
|
|
5
|
+
* The Sandbox SDK has two transfer modes for `createBackup` / `restoreBackup`:
|
|
6
|
+
* - `localBucket: true` — the Durable Object reads the archive from the R2
|
|
7
|
+
* binding and pumps it to the container over the control RPC (and the
|
|
8
|
+
* reverse on upload). The SDK documents this as the LOCAL-DEVELOPMENT mode
|
|
9
|
+
* ("required for local development where presigned URLs and FUSE are
|
|
10
|
+
* unavailable"). It puts a 128 MB isolate in the data path of every
|
|
11
|
+
* transfer: a checkout restore larger than the isolate's memory (a full
|
|
12
|
+
* checkout runs to gigabytes) resets the isolate mid-transfer, and a
|
|
13
|
+
* resident whose restore was reset sits in `restoring` with no way out but
|
|
14
|
+
* a manual rebuild.
|
|
15
|
+
* - presigned — the DO signs GET/PUT URLs; the container downloads
|
|
16
|
+
* (`downloadBackupParallel`, resumable) and uploads the bytes itself. The
|
|
17
|
+
* DO orchestrates and judges; its memory no longer scales with the archive.
|
|
18
|
+
* This is the mode the SDK's `requirePresignedURLSupport` gates on four env
|
|
19
|
+
* values, named below exactly as it reads them.
|
|
20
|
+
*
|
|
21
|
+
* Fail-closed: any input absent → local mode, with the gap named, so a
|
|
22
|
+
* misprovisioned secret degrades to today's behavior loudly instead of
|
|
23
|
+
* breaking every snapshot. The mode is reported on /healthz for the receipt. */
|
|
24
|
+
|
|
25
|
+
export const PRESIGNED_BACKUP_ENV = [
|
|
26
|
+
"CLOUDFLARE_ACCOUNT_ID",
|
|
27
|
+
"BACKUP_BUCKET_NAME",
|
|
28
|
+
"R2_ACCESS_KEY_ID",
|
|
29
|
+
"R2_SECRET_ACCESS_KEY",
|
|
30
|
+
] as const;
|
|
31
|
+
|
|
32
|
+
export type BackupTransferMode = "presigned" | "local";
|
|
33
|
+
|
|
34
|
+
export interface BackupTransferDecision {
|
|
35
|
+
mode: BackupTransferMode;
|
|
36
|
+
/** What to pass the SDK as `localBucket`. */
|
|
37
|
+
localBucket: boolean;
|
|
38
|
+
/** The presigned inputs that are absent or blank, in PRESIGNED_BACKUP_ENV order; empty in presigned mode. */
|
|
39
|
+
missing: string[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function backupTransferMode(env: Record<string, unknown>): BackupTransferDecision {
|
|
43
|
+
const missing = PRESIGNED_BACKUP_ENV.filter((name) => {
|
|
44
|
+
const v = env[name];
|
|
45
|
+
return typeof v !== "string" || v.trim() === "";
|
|
46
|
+
});
|
|
47
|
+
return missing.length === 0
|
|
48
|
+
? { mode: "presigned", localBucket: false, missing: [] }
|
|
49
|
+
: { mode: "local", localBucket: true, missing: [...missing] };
|
|
50
|
+
}
|