cursedops 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/package.json +2 -2
- package/src/workerDeploy.ts +432 -3
package/README.md
CHANGED
|
@@ -13,6 +13,7 @@ bun add cursedops
|
|
|
13
13
|
| `cursedops/launchd` | installing, replacing and removing a macOS launchd user agent, and the port a LIVE job serves on (`livePort`, 0.5.0) |
|
|
14
14
|
| `cursedops/smoke` | the scaffolding of a deployed smoke — the ledger, the fetch, the DNS hint, the exit code — the one check no app owns (every address of a deployment serving the same built client), and since 0.5.0 its VERDICTS: the origin asked on loopback, the smoke's own environment, a network that lies about DNS, and a version that has settled |
|
|
15
15
|
| `cursedops/worker-deploy` | the sequence that ships a Cloudflare Worker — clean tree, stage first, schema, `--var` stamp, secrets, smoke — and the readers it needs (0.5.0) |
|
|
16
|
+
| `cursedops/worker-deploy` (CPU) | a `wrangler tail` around the traffic a deploy already sends, each request's BILLED CPU charged to its declared route, red on an undeclared, over-budget or uncapped route — and `runWorkerDeploy`'s `stageWalk` + `cpuTail` that wrap the walk and the smoke in it (0.7.0, task 2134, lifted from collections) |
|
|
16
17
|
| `cursedops/worker-secrets` | a Worker holding EXACTLY its deployment's secrets, uploaded over a pipe, read back (0.5.0) |
|
|
17
18
|
| `cursedops/worker-rollback` | a hostname back on its Mac origin: origin first, route second, the route found rather than typed (0.5.0) |
|
|
18
19
|
| `cursedops/edge-fetch` | a request to a deployed Worker as a script must make it — curl pinned past the Mac's negative DNS cache (0.5.0) |
|
|
@@ -418,12 +419,28 @@ most fragile part (a hand-rolled HTTP parser) in two places.
|
|
|
418
419
|
| `edgeFetch` (curl `--resolve` past the negative DNS cache) | **moved** — `createEdgeFetch`; the Access header now THROWS on an incomplete token instead of sending empty headers |
|
|
419
420
|
| the import's literal and row-for-row proof | **moved** — `sqlLiteral`, `rowDigest`, `sameRows`, `rowDifferences`, `wranglerRows` |
|
|
420
421
|
| the steady `/healthz` wait | **moved**, into `cursedops/smoke` as `steadyHealth` |
|
|
422
|
+
| the Worker CPU tail and its judgement (`worker-cpu.ts` + `workerCpu.ts`) | **moved** (0.7.0, task 2134) — `runWorkerCpuTail`, `judgeTailCapture`, `workerBudgets`; `runWorkerDeploy({ stageWalk, cpuTail })` runs the walk and the smoke under it |
|
|
423
|
+
| the Worker BUDGET TABLE (routes, numbers, the unlock's ceiling) | **stays** — what a route may cost is the app |
|
|
421
424
|
| the deployment table (`workerEnvs.ts`: names, database, URLs, secret map, which file) | **stays** — identity, rule 2 |
|
|
422
425
|
| the smoke's CHECKS (gated-route census, health identity fields, shell/CSP) | **stays** — the same argument as the rest of `smoke` |
|
|
423
426
|
| the walk (`workerWalk.ts`) and the stage walk's grant through `apps/auth-stage` | **stays** — what a signed-in owner does is the app |
|
|
424
427
|
| the import's table PLAN (what is skipped, parents first, what is rebuilt: FTS, `art_lookups`) | **stays** — the app's schema |
|
|
425
428
|
| minting a stage's secret FILE (which keys are copied from production, which are throwaway) | **stays** — `throwawaySecret` and `writeSecretsFile` are the mechanism it uses |
|
|
426
429
|
|
|
430
|
+
### The Worker CPU check, in `cursedops/worker-deploy` (0.7.0, task 2134)
|
|
431
|
+
|
|
432
|
+
`collections` wrote it on 2026-09-23 after averaging 489 billed CPU-ms per request with a 15 s
|
|
433
|
+
p99 and every check green; `vault` was measured in the same state the same day (argon2id at
|
|
434
|
+
4,089 CPU-ms per create, 4,458 per unlock, on its stage). The billed number exists only on the
|
|
435
|
+
trace a tail receives, so this tails the Worker around the walk and the smoke, charges each
|
|
436
|
+
request's `cpuTime` to its declared route with `cursedbelt-server/bench`, and fails on an
|
|
437
|
+
undeclared route, one over its budget, or an EXEMPT route past its `noticeAboveMs` (the library
|
|
438
|
+
only notices; a Worker is killed at 30 s of CPU, and a killed unlock reads as a wrong password).
|
|
439
|
+
An exempt route with no ceiling is red on its own. `bench` is passed in (`import * as bench`),
|
|
440
|
+
because `cursedbelt-server` depends on this package. It lives in `worker-deploy` rather than a subpath of its own because a shipped file may not
|
|
441
|
+
import a sibling (`publishShape.test.ts`). An app adopts it with a budget table, a
|
|
442
|
+
ten-line `scripts/worker-cpu.ts`, and `stageWalk` + `cpuTail` on its `runWorkerDeploy` spec.
|
|
443
|
+
|
|
427
444
|
## `cursedops/public-surface`
|
|
428
445
|
|
|
429
446
|
```sh
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedops",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof), and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
|
|
3
|
+
"version": "0.7.1",
|
|
4
|
+
"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke), and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
package/src/workerDeploy.ts
CHANGED
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
* Everything that shells out is behind {@link DeployDeps}, so the suite proves the order and every
|
|
50
50
|
* refusal without wrangler, git or a network.
|
|
51
51
|
*/
|
|
52
|
-
import { spawnSync } from "node:child_process";
|
|
52
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
53
53
|
import { existsSync, readFileSync } from "node:fs";
|
|
54
54
|
|
|
55
55
|
/** The two deployments every Worker app here has. */
|
|
@@ -136,6 +136,18 @@ function readIfThere(path: string): string | null {
|
|
|
136
136
|
return existsSync(path) ? readFileSync(path, "utf8") : null;
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
/**
|
|
140
|
+
* `command` under the app's Worker CPU wrapper, in the argv {@link parseWorkerCpuArgv} reads.
|
|
141
|
+
*
|
|
142
|
+
* 🔴 `--env` is ALWAYS spelled, production included. `bun run <file> -- x` swallows a `--` that
|
|
143
|
+
* comes straight after the file, so `bun run scripts/worker-cpu.ts -- bun run smoke.ts` reached
|
|
144
|
+
* the wrapper with no separator and exited on its usage line — measured on collections'
|
|
145
|
+
* production smoke, 2026-09-23, after the Worker had already shipped (cursedops 0.7.0).
|
|
146
|
+
*/
|
|
147
|
+
export function underCpuTail(wrapper: readonly string[], env: WorkerEnv, command: readonly string[]): string[] {
|
|
148
|
+
return [...wrapper, "--env", env, "--", ...command];
|
|
149
|
+
}
|
|
150
|
+
|
|
139
151
|
/** A refusal an app adds to the sequence: a sentence saying why not, or `null` to proceed. */
|
|
140
152
|
export type Refusal = () => string | null;
|
|
141
153
|
|
|
@@ -154,6 +166,17 @@ export interface WorkerDeploySpec {
|
|
|
154
166
|
build: readonly string[] | null;
|
|
155
167
|
/** Run in order BEFORE a production deploy; any red and production is not touched. Ignored for the stage. */
|
|
156
168
|
stageFirst?: readonly (readonly string[])[];
|
|
169
|
+
/**
|
|
170
|
+
* The signed-in walk of the stage, run after `stageFirst` and before production is touched — and,
|
|
171
|
+
* unlike `stageFirst`, under {@link WorkerDeploySpec.cpuTail} when that is set. Ignored for the stage.
|
|
172
|
+
*/
|
|
173
|
+
stageWalk?: readonly (readonly string[])[];
|
|
174
|
+
/**
|
|
175
|
+
* The app's Worker CPU wrapper (see {@link runWorkerCpuTail}), e.g. `["bun", "run", "scripts/worker-cpu.ts"]`. When
|
|
176
|
+
* set, every `stageWalk` command and the `smoke` run UNDER it (`underCpuTail`), so the traffic a
|
|
177
|
+
* deploy already sends is charged its billed CPU and judged against the app's Worker budgets.
|
|
178
|
+
*/
|
|
179
|
+
cpuTail?: readonly string[];
|
|
157
180
|
/** Before the stage (and before anything is built): the app's own refusals. */
|
|
158
181
|
beforeStage?: readonly Refusal[];
|
|
159
182
|
/** After the build, before anything ships — e.g. "the build output is really there". */
|
|
@@ -227,6 +250,11 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
|
|
|
227
250
|
step(`the stage first — production waits on it: ${command.join(" ")}`);
|
|
228
251
|
if (deps.run(command, {}) !== 0) return stop("stage", `\`${command.join(" ")}\` failed — production was NOT touched.`);
|
|
229
252
|
}
|
|
253
|
+
for (const walk of spec.stageWalk ?? []) {
|
|
254
|
+
const command = spec.cpuTail ? underCpuTail(spec.cpuTail, "stage", walk) : [...walk];
|
|
255
|
+
step(`walk the stage signed in${spec.cpuTail ? ", under a CPU tail" : ""}: ${command.join(" ")}`);
|
|
256
|
+
if (deps.run(command, {}) !== 0) return stop("stage", `\`${command.join(" ")}\` failed — production was NOT touched.`);
|
|
257
|
+
}
|
|
230
258
|
}
|
|
231
259
|
if (spec.build) {
|
|
232
260
|
step("build");
|
|
@@ -255,8 +283,9 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
|
|
|
255
283
|
}
|
|
256
284
|
}
|
|
257
285
|
if (spec.smoke) {
|
|
258
|
-
step(
|
|
259
|
-
|
|
286
|
+
step(`smoke every address this deployment answers on${spec.cpuTail ? ", under a CPU tail" : ""}`);
|
|
287
|
+
const smoke = spec.cpuTail ? underCpuTail(spec.cpuTail, spec.env, spec.smoke) : spec.smoke;
|
|
288
|
+
if (deps.run(smoke, {}) !== 0) {
|
|
260
289
|
return stop(
|
|
261
290
|
"smoke",
|
|
262
291
|
"the Worker is deployed and its smoke FAILED. Nothing was rolled back: `bunx wrangler rollback` puts the previous version back in one command.",
|
|
@@ -267,3 +296,403 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
|
|
|
267
296
|
log(`\n✅ ${detail}`);
|
|
268
297
|
return { code: 0, step: "done", detail, commit, dirty };
|
|
269
298
|
}
|
|
299
|
+
|
|
300
|
+
// ═════════════════════════════════════════════════════════════════════════════════════════════
|
|
301
|
+
/**
|
|
302
|
+
* THE WORKER CPU CHECK (0.7.0, task 2134) — what a Worker's requests REALLY cost, asserted on every deploy: a
|
|
303
|
+
* `wrangler tail` around traffic the deploy already sends (the signed-in stage walk, the smoke),
|
|
304
|
+
* each request's billed `cpuTime` charged to its declared route, and a red for a route that is
|
|
305
|
+
* undeclared, over its budget, or — exempt or not — heading for the platform's 30 s kill.
|
|
306
|
+
*
|
|
307
|
+
* ```ts
|
|
308
|
+
* // scripts/worker-cpu.ts — the whole app side, besides a budget table
|
|
309
|
+
* import * as bench from "cursedbelt-server/bench";
|
|
310
|
+
* import { parseWorkerCpuArgv, runWorkerCpuTail } from "cursedops/worker-deploy";
|
|
311
|
+
*
|
|
312
|
+
* const { env, command } = parseWorkerCpuArgv(process.argv.slice(2));
|
|
313
|
+
* const d = deploymentFor(env);
|
|
314
|
+
* const result = await runWorkerCpuTail({
|
|
315
|
+
* app: "vault", workerName: d.workerName, base: d.previewUrl ?? d.publicUrl, command, cwd: ROOT,
|
|
316
|
+
* credential, fetch: (url) => edgeFetch(url), bench, config: VAULT_WORKER_CPU_BUDGETS,
|
|
317
|
+
* });
|
|
318
|
+
* process.exit(result.code);
|
|
319
|
+
*
|
|
320
|
+
* // scripts/worker-deploy.ts — the sequence wraps the walk and the smoke itself
|
|
321
|
+
* runWorkerDeploy({ …, stageWalk: [["bun", "run", "scripts/worker-stage-walk.ts"]],
|
|
322
|
+
* cpuTail: ["bun", "run", "scripts/worker-cpu.ts"] }, deps);
|
|
323
|
+
* ```
|
|
324
|
+
*
|
|
325
|
+
* ## 🔴 Why it exists
|
|
326
|
+
*
|
|
327
|
+
* `collections` averaged **489 CPU-ms per request on 2026-09-22 with a 15.2 s p99**, and nothing
|
|
328
|
+
* in its gate could see it: its `cpuBudget.spec.ts` benches the Mac, where the same unlock cost
|
|
329
|
+
* ~1.6 s rather than 8.5–15 s, and a Worker handler has no clock to read its own CPU time. The
|
|
330
|
+
* billed number exists only on the trace a tail receives. `vault` was the next app in the same
|
|
331
|
+
* state — pure-JS argon2id at 4,089 CPU-ms per create and 4,458 per unlock on its stage, with
|
|
332
|
+
* every check green (task 2134). Collections wrote this wrapper in `scripts/`; a copy in vault
|
|
333
|
+
* would have been the fourth hand-kept copy of a deploy step this toolkit exists to own.
|
|
334
|
+
*
|
|
335
|
+
* ## Why `bench` is a parameter, not an import
|
|
336
|
+
*
|
|
337
|
+
* The recorder, the tail parser and the assertion are `cursedbelt-server/bench`'s — and
|
|
338
|
+
* `cursedbelt-server` depends on THIS package, so importing it back would be a cycle. The app
|
|
339
|
+
* already depends on it for its proxy table, so it hands the module in (`import * as bench`) and
|
|
340
|
+
* {@link TailBench} names, structurally, the seven functions used. The app's typecheck is what
|
|
341
|
+
* proves the two still agree.
|
|
342
|
+
*
|
|
343
|
+
* ## The decisions it makes about a capture (each one a way it went wrong or could)
|
|
344
|
+
*
|
|
345
|
+
* 1. **A non-API GET is the static tier, `GET *`** — but only when the table DECLARES `GET *`.
|
|
346
|
+
* A Worker that ends in `app.get("*")` serves the shell and every client-side route there;
|
|
347
|
+
* declaring `GET /*` as a pattern instead would swallow an undeclared `GET /api/…`, which is
|
|
348
|
+
* exactly what `requireDeclared` exists to name. So the label is decided here, never for a
|
|
349
|
+
* path under an API prefix, and an app with no `GET *` gets its unknown GETs named instead.
|
|
350
|
+
* 2. **`minSamples: 1`.** A walk makes one request per route; a p99 of one reading is that reading.
|
|
351
|
+
* 3. 🔴 **An exempt route over its `noticeAboveMs` FAILS here**, where the library only notices —
|
|
352
|
+
* and an exempt route with NO `noticeAboveMs` is a red of its own (`exempt-uncapped`). Exempt
|
|
353
|
+
* is from the per-request money budget, never from the platform: an invocation is killed at
|
|
354
|
+
* 30 s of CPU, and a killed unlock reads as a wrong password.
|
|
355
|
+
* 4. **Only this Worker's invocations are charged** (`scriptName`), so the stage's walk cannot be
|
|
356
|
+
* excused by production's traffic on a shared tail, or the reverse.
|
|
357
|
+
* 5. **A tail that never connects is a failure, never an empty pass**; neither is a runtime that
|
|
358
|
+
* stops reporting `cpuTime` (`recordTailTraces` skips it; the empty report is `no-samples`).
|
|
359
|
+
*
|
|
360
|
+
* ## How it knows the tail is listening, and that it has heard everything
|
|
361
|
+
*
|
|
362
|
+
* `wrangler tail --format json` prints nothing until a request arrives, so there is no banner to
|
|
363
|
+
* wait for. It sends `GET /healthz?cpu-tail=<nonce>` until that request comes back on the tail
|
|
364
|
+
* (connected), runs the command, then `…=<nonce>-end` until THAT comes back (drained). The marker
|
|
365
|
+
* requests are real traffic, so `GET /healthz` must be declared like any other route.
|
|
366
|
+
*
|
|
367
|
+
* 🔴 The command's own exit code wins: a walk that failed is reported as the walk failing, and the
|
|
368
|
+
* CPU verdict is still printed but does not mask it.
|
|
369
|
+
*/
|
|
370
|
+
|
|
371
|
+
/** A route's budget as `cursedbelt-server/bench` declares it — structural. */
|
|
372
|
+
export type RouteBudgetLike = number | { cpuMs: number } | { exempt: true; reason: string; noticeAboveMs?: number };
|
|
373
|
+
|
|
374
|
+
/** `CpuBudgetConfig`, structurally. */
|
|
375
|
+
export interface CpuBudgetConfigLike {
|
|
376
|
+
defaultCpuMs?: number;
|
|
377
|
+
routes?: Record<string, RouteBudgetLike>;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** The fields of a `CpuViolation` this reads. */
|
|
381
|
+
export interface CpuViolationLike {
|
|
382
|
+
kind: string;
|
|
383
|
+
key: string;
|
|
384
|
+
fails: boolean;
|
|
385
|
+
message: string;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** `TailRecordResult`, structurally. */
|
|
389
|
+
export interface TailRecordLike {
|
|
390
|
+
recorded: number;
|
|
391
|
+
notFetch: number;
|
|
392
|
+
noCpuTime: number;
|
|
393
|
+
unmatched: number;
|
|
394
|
+
unmatchedPaths: string[];
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* The seven functions of `cursedbelt-server/bench` this uses — pass the module itself
|
|
399
|
+
* (`import * as bench from "cursedbelt-server/bench"`). Method syntax on purpose: the parameters
|
|
400
|
+
* are checked bivariantly, so the real, narrower signatures are assignable.
|
|
401
|
+
*/
|
|
402
|
+
export interface TailBench {
|
|
403
|
+
parseTailLines(text: string): readonly unknown[];
|
|
404
|
+
tailCpuClock(): unknown;
|
|
405
|
+
createCpuRecorder(opts: { clock: never; config?: never }): { report(): unknown };
|
|
406
|
+
recordTailTraces(recorder: never, traces: never, opts: { route?: (method: string, path: string) => string | null; scriptName?: string }): TailRecordLike;
|
|
407
|
+
routeMatcher(keys: readonly string[]): (method: string, path: string) => string | null;
|
|
408
|
+
checkCpuBudgets(report: never, opts: { requireDeclared?: boolean; minSamples?: number }): readonly CpuViolationLike[];
|
|
409
|
+
formatCpuBudgetReport(report: never): string;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** The static tier's label — decision 1. Its key in a table is `GET *`. */
|
|
413
|
+
export const STATIC_TIER = "*";
|
|
414
|
+
const STATIC_TIER_KEY = `GET ${STATIC_TIER}`;
|
|
415
|
+
|
|
416
|
+
/** Paths that are never the static tier unless a table declares them. */
|
|
417
|
+
export const DEFAULT_API_PREFIXES: readonly string[] = ["/api"];
|
|
418
|
+
|
|
419
|
+
const underPrefix = (path: string, prefixes: readonly string[]): boolean =>
|
|
420
|
+
prefixes.some((prefix) => path === prefix || path.startsWith(`${prefix.replace(/\/$/, "")}/`));
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* `(method, path)` → the declared route it is charged to, or `null` (undeclared).
|
|
424
|
+
*
|
|
425
|
+
* 🔴 The `GET *` key itself is NOT given to the matcher: as a pattern it is `.*`, and it would
|
|
426
|
+
* charge an undeclared `GET /api/…` to the static tier — the swallowing decision 1 forbids.
|
|
427
|
+
*/
|
|
428
|
+
export function tierRoute(
|
|
429
|
+
bench: Pick<TailBench, "routeMatcher">,
|
|
430
|
+
config: CpuBudgetConfigLike,
|
|
431
|
+
apiPrefixes: readonly string[] = DEFAULT_API_PREFIXES,
|
|
432
|
+
): (method: string, path: string) => string | null {
|
|
433
|
+
const keys = Object.keys(config.routes ?? {});
|
|
434
|
+
const hasStaticTier = keys.includes(STATIC_TIER_KEY);
|
|
435
|
+
const declared = bench.routeMatcher(keys.filter((key) => !/(^|\s)\*$/.test(key)));
|
|
436
|
+
return (method, path) => {
|
|
437
|
+
const hit = declared(method, path);
|
|
438
|
+
if (hit !== null) return hit;
|
|
439
|
+
return hasStaticTier && method.toUpperCase() === "GET" && !underPrefix(path, apiPrefixes) ? STATIC_TIER : null;
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/** A red this module adds on top of the library's violations. */
|
|
444
|
+
export interface WorkerCpuFailure {
|
|
445
|
+
kind: string;
|
|
446
|
+
key: string;
|
|
447
|
+
message: string;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
export interface TailVerdict {
|
|
451
|
+
ok: boolean;
|
|
452
|
+
/** Every violation that fails this check — decision 3 included. */
|
|
453
|
+
failures: WorkerCpuFailure[];
|
|
454
|
+
/** `CpuBudgetReport` — hand it to `formatCpuBudgetReport`. */
|
|
455
|
+
report: unknown;
|
|
456
|
+
recorded: TailRecordLike;
|
|
457
|
+
/** Traces parsed out of the capture, this Worker's or not. */
|
|
458
|
+
traces: number;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export interface JudgeOpts {
|
|
462
|
+
bench: TailBench;
|
|
463
|
+
/** The Worker's REAL-reading budget table (`noticeAboveMs` on every exempt route). */
|
|
464
|
+
config: CpuBudgetConfigLike;
|
|
465
|
+
/** Only this deployment's invocations are charged — decision 4. */
|
|
466
|
+
scriptName: string;
|
|
467
|
+
/** Paths that are never the static tier. Default {@link DEFAULT_API_PREFIXES}. */
|
|
468
|
+
apiPrefixes?: readonly string[];
|
|
469
|
+
/** Replace the labeller entirely. Default {@link tierRoute}. */
|
|
470
|
+
route?: (method: string, path: string) => string | null;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** Exempt routes with no ceiling — each one a route the 30 s kill could reach with every check green. */
|
|
474
|
+
export function uncappedExemptions(config: CpuBudgetConfigLike): string[] {
|
|
475
|
+
return Object.entries(config.routes ?? {})
|
|
476
|
+
.filter(([, budget]) => typeof budget === "object" && "exempt" in budget && budget.noticeAboveMs === undefined)
|
|
477
|
+
.map(([key]) => key);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** Judge one `wrangler tail --format json` capture. Pure: no Worker, no network. */
|
|
481
|
+
export function judgeTailCapture(text: string, opts: JudgeOpts): TailVerdict {
|
|
482
|
+
const { bench, config } = opts;
|
|
483
|
+
const traces = bench.parseTailLines(text);
|
|
484
|
+
const recorder = bench.createCpuRecorder({ clock: bench.tailCpuClock() as never, config: config as never });
|
|
485
|
+
const route = opts.route ?? tierRoute(bench, config, opts.apiPrefixes);
|
|
486
|
+
const recorded = bench.recordTailTraces(recorder as never, traces as never, { scriptName: opts.scriptName, route });
|
|
487
|
+
const report = recorder.report();
|
|
488
|
+
const failures: WorkerCpuFailure[] = uncappedExemptions(config).map((key) => ({
|
|
489
|
+
kind: "exempt-uncapped",
|
|
490
|
+
key,
|
|
491
|
+
message:
|
|
492
|
+
`${key} is exempt with no noticeAboveMs — on a Worker every route needs a CEILING, because an invocation is ` +
|
|
493
|
+
"killed at 30 s of CPU whatever the table calls it. Give it the ceiling its real readings justify.",
|
|
494
|
+
}));
|
|
495
|
+
for (const v of bench.checkCpuBudgets(report as never, { requireDeclared: true, minSamples: 1 })) {
|
|
496
|
+
if (v.kind === "exempt-notice") {
|
|
497
|
+
failures.push({ kind: v.kind, key: v.key, message: `${v.message} 🔴 On a Worker this is red: an exempt route is still killed at 30 s of CPU.` });
|
|
498
|
+
} else if (v.fails) {
|
|
499
|
+
failures.push({ kind: v.kind, key: v.key, message: v.message });
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
return { ok: failures.length === 0, failures, report, recorded, traces: traces.length };
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/** The verdict in words, for the deploy log. */
|
|
506
|
+
export function formatTailVerdict(verdict: TailVerdict, bench: Pick<TailBench, "formatCpuBudgetReport">, scriptName: string): string {
|
|
507
|
+
const r = verdict.recorded;
|
|
508
|
+
const lines = [
|
|
509
|
+
`Worker CPU for ${scriptName} — ${r.recorded} request(s) charged from ${verdict.traces} trace(s)` +
|
|
510
|
+
(r.notFetch ? `, ${r.notFetch} cron/other skipped` : "") +
|
|
511
|
+
(r.noCpuTime ? `, ${r.noCpuTime} with no cpuTime` : ""),
|
|
512
|
+
bench.formatCpuBudgetReport(verdict.report as never),
|
|
513
|
+
];
|
|
514
|
+
if (r.unmatchedPaths.length > 0) lines.push(`undeclared paths: ${r.unmatchedPaths.join(", ")}`);
|
|
515
|
+
for (const failure of verdict.failures) lines.push(`✗ ${failure.message}`);
|
|
516
|
+
lines.push(verdict.ok ? "✓ every request inside its declared Worker budget" : "🔴 Worker CPU budget FAILED");
|
|
517
|
+
return lines.join("\n");
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* A Worker table derived from a Mac-proxy table, the rule `collections` measured its way to:
|
|
522
|
+
* **`max(floorMs, overProxy × proxy budget)`** per route, plus the Worker-only routes, with each
|
|
523
|
+
* exempt route given its ceiling from `ceilings`. THROWS on an exempt route with no ceiling, and
|
|
524
|
+
* on a ceiling for a route that is not exempt — both are a table that says something untrue.
|
|
525
|
+
*
|
|
526
|
+
* Why a rule rather than forty fitted numbers: the proxy already RANKS the routes, the ratio
|
|
527
|
+
* (3.5–5.3× Mac→workerd, measured three ways on 2026-09-23) carries the ranking to the platform,
|
|
528
|
+
* and the floor is the cold-isolate cost every request pays whatever it does — at a few hundred
|
|
529
|
+
* requests a day almost every request lands on a cold isolate.
|
|
530
|
+
*/
|
|
531
|
+
export function workerBudgets(
|
|
532
|
+
proxy: CpuBudgetConfigLike,
|
|
533
|
+
opts: { floorMs: number; overProxy: number; workerOnly?: Record<string, RouteBudgetLike>; ceilings?: Record<string, number> },
|
|
534
|
+
): { routes: Record<string, RouteBudgetLike> } {
|
|
535
|
+
const routes: Record<string, RouteBudgetLike> = {};
|
|
536
|
+
const ceilings = opts.ceilings ?? {};
|
|
537
|
+
for (const [key, budget] of Object.entries(proxy.routes ?? {})) {
|
|
538
|
+
if (typeof budget === "object" && "exempt" in budget) {
|
|
539
|
+
const ceiling = ceilings[key] ?? budget.noticeAboveMs;
|
|
540
|
+
if (ceiling === undefined) throw new Error(`workerBudgets: ${key} is exempt and has no ceiling — pass ceilings["${key}"].`);
|
|
541
|
+
routes[key] = { ...budget, noticeAboveMs: ceiling };
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
const cpuMs = typeof budget === "number" ? budget : budget.cpuMs;
|
|
545
|
+
routes[key] = Math.max(opts.floorMs, opts.overProxy * cpuMs);
|
|
546
|
+
}
|
|
547
|
+
for (const key of Object.keys(ceilings)) {
|
|
548
|
+
const budget = routes[key];
|
|
549
|
+
if (!(typeof budget === "object" && "exempt" in budget)) throw new Error(`workerBudgets: a ceiling for ${key}, which is not an exempt route.`);
|
|
550
|
+
}
|
|
551
|
+
return { routes: { ...(opts.workerOnly ?? {}), ...routes } };
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* `[--env stage] -- <command…>` → the deployment and the command. THROWS a usage line on a missing
|
|
556
|
+
* command. The `--env` spelling rules are {@link parseWorkerEnv}'s.
|
|
557
|
+
*/
|
|
558
|
+
export function parseWorkerCpuArgv(argv: readonly string[]): { env: WorkerEnv; command: string[] } {
|
|
559
|
+
const split = argv.indexOf("--");
|
|
560
|
+
const own = split >= 0 ? argv.slice(0, split) : argv;
|
|
561
|
+
const command = split >= 0 ? argv.slice(split + 1) : [];
|
|
562
|
+
if (command.length === 0) throw new Error("usage: worker-cpu [--env stage] -- <command…>");
|
|
563
|
+
return { env: parseWorkerEnv(own), command };
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
/** A running `wrangler tail`. */
|
|
568
|
+
export interface TailProcess {
|
|
569
|
+
/** Everything it has printed on stdout so far. */
|
|
570
|
+
captured(): string;
|
|
571
|
+
/** The tail of its stderr, for a failure message. */
|
|
572
|
+
errors(): string;
|
|
573
|
+
exited(): boolean;
|
|
574
|
+
stop(): void;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
export interface WorkerCpuDeps {
|
|
578
|
+
/** Start `wrangler tail <worker> --format json`. */
|
|
579
|
+
startTail: (workerName: string, cwd: string, env: Record<string, string>) => TailProcess;
|
|
580
|
+
/** Run the command, inheriting stdio; its exit code. */
|
|
581
|
+
run: (argv: readonly string[], cwd: string) => number;
|
|
582
|
+
sleep: (ms: number) => Promise<void>;
|
|
583
|
+
now: () => number;
|
|
584
|
+
log: (line: string) => void;
|
|
585
|
+
error: (line: string) => void;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** The real {@link WorkerCpuDeps}: `bunx wrangler@4 tail`, pinned to the major whose JSON shape the parser reads. */
|
|
589
|
+
export function workerCpuDeps(): WorkerCpuDeps {
|
|
590
|
+
return {
|
|
591
|
+
startTail: (workerName, cwd, env) => {
|
|
592
|
+
let out = "";
|
|
593
|
+
let err = "";
|
|
594
|
+
const child = spawn("bunx", ["wrangler@4", "tail", workerName, "--format", "json"], {
|
|
595
|
+
cwd,
|
|
596
|
+
env: { ...process.env, ...env },
|
|
597
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
598
|
+
});
|
|
599
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
600
|
+
out += chunk.toString("utf8");
|
|
601
|
+
});
|
|
602
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
603
|
+
err += chunk.toString("utf8");
|
|
604
|
+
});
|
|
605
|
+
return {
|
|
606
|
+
captured: () => out,
|
|
607
|
+
errors: () => err.trim().slice(-800),
|
|
608
|
+
exited: () => child.exitCode !== null || child.signalCode !== null,
|
|
609
|
+
stop: () => {
|
|
610
|
+
if (child.exitCode === null) child.kill("SIGINT");
|
|
611
|
+
},
|
|
612
|
+
};
|
|
613
|
+
},
|
|
614
|
+
run: (argv, cwd) => spawnSync(argv[0] as string, argv.slice(1), { cwd, stdio: "inherit" }).status ?? 1,
|
|
615
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
616
|
+
now: () => Date.now(),
|
|
617
|
+
log: (line) => console.log(line),
|
|
618
|
+
error: (line) => console.error(line),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
export interface WorkerCpuTailSpec extends Omit<JudgeOpts, "scriptName"> {
|
|
623
|
+
/** For the messages. */
|
|
624
|
+
app: string;
|
|
625
|
+
/** The Worker to tail — also the only `scriptName` charged. */
|
|
626
|
+
workerName: string;
|
|
627
|
+
/** Where the markers are sent: an address of THIS deployment that reaches the Worker. */
|
|
628
|
+
base: string;
|
|
629
|
+
command: readonly string[];
|
|
630
|
+
cwd: string;
|
|
631
|
+
/** The env overlay wrangler runs with — `cloudflareCredential` from `cursedops/worker-deploy`. */
|
|
632
|
+
credential: Record<string, string>;
|
|
633
|
+
/** A GET the app's scripts can make (the stage is behind Access) — `cursedops/edge-fetch`. */
|
|
634
|
+
fetch: (url: string) => Promise<unknown>;
|
|
635
|
+
connectMs?: number;
|
|
636
|
+
drainMs?: number;
|
|
637
|
+
/** How long a marker is watched for before it is sent again. Default 3 s. */
|
|
638
|
+
markerWaitMs?: number;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
export interface WorkerCpuTailResult {
|
|
642
|
+
/** 0 green; the command's own code when it failed; 1 for any CPU or tail failure. */
|
|
643
|
+
code: number;
|
|
644
|
+
/** Where it stopped: `connect`, `command`, `drain`, `budget` or `done`. */
|
|
645
|
+
step: "connect" | "command" | "drain" | "budget" | "done";
|
|
646
|
+
verdict: TailVerdict | null;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/** Run `spec.command` under a tail of `spec.workerName` and judge what it cost. Never exits. */
|
|
650
|
+
export async function runWorkerCpuTail(spec: WorkerCpuTailSpec, deps: WorkerCpuDeps = workerCpuDeps()): Promise<WorkerCpuTailResult> {
|
|
651
|
+
const connectMs = spec.connectMs ?? 60_000;
|
|
652
|
+
const drainMs = spec.drainMs ?? 45_000;
|
|
653
|
+
const markerWaitMs = spec.markerWaitMs ?? 3_000;
|
|
654
|
+
const nonce = `${deps.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
|
|
655
|
+
const tag = `[${spec.app}]`;
|
|
656
|
+
const base = spec.base.replace(/\/$/, "");
|
|
657
|
+
|
|
658
|
+
deps.log(`\n▸ ${tag} Worker CPU: tailing ${spec.workerName} around \`${spec.command.join(" ")}\``);
|
|
659
|
+
const tail = deps.startTail(spec.workerName, spec.cwd, spec.credential);
|
|
660
|
+
|
|
661
|
+
/** Send the marker until the tail has carried it back, or give up. */
|
|
662
|
+
const awaitMarker = async (marker: string, withinMs: number): Promise<boolean> => {
|
|
663
|
+
const deadline = deps.now() + withinMs;
|
|
664
|
+
while (deps.now() < deadline) {
|
|
665
|
+
if (tail.exited()) return false;
|
|
666
|
+
await spec.fetch(`${base}/healthz?cpu-tail=${marker}`).catch(() => null);
|
|
667
|
+
const seenBy = deps.now() + markerWaitMs;
|
|
668
|
+
while (deps.now() < seenBy) {
|
|
669
|
+
// The closing quote: `<nonce>` must not be satisfied by `<nonce>-end`.
|
|
670
|
+
if (tail.captured().includes(`cpu-tail=${marker}"`)) return true;
|
|
671
|
+
await deps.sleep(250);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
return false;
|
|
675
|
+
};
|
|
676
|
+
|
|
677
|
+
if (!(await awaitMarker(nonce, connectMs))) {
|
|
678
|
+
tail.stop();
|
|
679
|
+
deps.error(
|
|
680
|
+
`🔴 ${tag} wrangler tail never delivered a request from ${base} within ${Math.round(connectMs / 1000)}s — nothing could be measured.\n${tail.errors()}`,
|
|
681
|
+
);
|
|
682
|
+
return { code: 1, step: "connect", verdict: null };
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
const ran = deps.run(spec.command, spec.cwd);
|
|
686
|
+
const drained = await awaitMarker(`${nonce}-end`, drainMs);
|
|
687
|
+
// Traces are not strictly ordered; a moment more lets a straggler behind the end marker land.
|
|
688
|
+
await deps.sleep(2_000);
|
|
689
|
+
tail.stop();
|
|
690
|
+
|
|
691
|
+
const verdict = judgeTailCapture(tail.captured(), { ...spec, scriptName: spec.workerName });
|
|
692
|
+
deps.log(`\n${formatTailVerdict(verdict, spec.bench, spec.workerName)}`);
|
|
693
|
+
if (!drained) deps.error(`🔴 ${tag} the end marker never came back — the capture may be missing the command's last requests.`);
|
|
694
|
+
|
|
695
|
+
if (ran !== 0) return { code: ran, step: "command", verdict };
|
|
696
|
+
if (!drained) return { code: 1, step: "drain", verdict };
|
|
697
|
+
return verdict.ok ? { code: 0, step: "done", verdict } : { code: 1, step: "budget", verdict };
|
|
698
|
+
}
|