@coreplane/switchboard 1.251.0 → 1.252.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/config/config.example.yaml +4 -2
- package/dist/assets/deploy/cloudflare/preflight.mjs +19 -21
- package/dist/assets/deploy/cloudflare/worker.ts +6 -3
- package/dist/assets/deploy/cloudflare-memory/worker.ts +77 -12
- package/dist/assets/deploy/cloudflare-resident/memoryGuard.ts +212 -0
- package/dist/assets/deploy/cloudflare-resident/refresh.ts +1 -1
- package/dist/assets/deploy/cloudflare-resident/worker.ts +317 -56
- package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +4 -2
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.d.mts +31 -0
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.mjs +119 -0
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +3 -2
- package/dist/assets/project.json +13 -9
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +5 -5
- package/dist/assets/src/core/budgets.ts +22 -0
- package/dist/assets/src/core/coordinator/contract.ts +42 -0
- package/dist/assets/src/core/coordinator/driver.ts +134 -10
- package/dist/assets/src/core/drain.ts +50 -0
- package/dist/assets/src/core/memory/engine.ts +98 -0
- package/dist/assets/src/core/memory/scorer.ts +12 -4
- package/dist/assets/src/core/memory/types.ts +69 -12
- package/dist/assets/src/core/modelCard.ts +32 -4
- package/dist/assets/src/core/modelPricing.ts +111 -1
- package/dist/assets/src/core/modelProxy/usage.ts +88 -0
- package/dist/assets/src/core/modelRegistry.ts +15 -1
- package/dist/assets/src/core/refusal.ts +4 -7
- package/dist/assets/src/core/reviewVerdict.ts +4 -0
- package/dist/assets/src/core/runEvents.ts +51 -2
- package/dist/assets/src/core/runFriction.ts +7 -2
- package/dist/assets/src/core/runLedger/types.ts +11 -0
- package/dist/assets/src/core/runUsage.ts +67 -13
- package/dist/assets/src/core/schedules.ts +3 -0
- package/dist/assets/src/core/ship/contract.ts +41 -14
- package/dist/assets/src/core/ship/coordinator.ts +380 -53
- package/dist/assets/src/core/ship/renewal.ts +10 -5
- package/dist/assets/src/core/trace/attrs.ts +24 -0
- package/dist/assets/src/core/types.ts +5 -5
- package/dist/assets/src/core/verbosity.ts +48 -0
- package/dist/assets/src/deploy/liveGate.ts +40 -13
- package/dist/assets/src/deploy/restart.ts +11 -12
- package/dist/assets/src/execution/residentDepCache.ts +50 -1
- package/dist/assets/src/execution/residentDepsStore.ts +40 -2
- package/dist/assets/src/execution/residentRefresh.ts +55 -3
- package/dist/assets/src/execution/residentSteps.ts +4 -0
- package/dist/assets/src/execution/sandboxErrors.ts +8 -0
- package/dist/assets/web/dist/.vite/manifest.json +55 -55
- package/dist/assets/web/dist/assets/{DeliveryPage-DUXd-Sl-.js → DeliveryPage-3ELQWM0r.js} +1 -1
- package/dist/assets/web/dist/assets/HomePage-BG_ok-K2.js +2 -0
- package/dist/assets/web/dist/assets/{PendingTurnRow-DDhMhrI7.js → PendingTurnRow-ChCQOLgZ.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentDetailPage-BnEoOnGQ.js → ResidentDetailPage-C9y3nbo8.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-Dxpgf-l-.js → ResidentsIndexPage-i1RG9e7g.js} +1 -1
- package/dist/assets/web/dist/assets/RunFoldRow-D3wVpzBa.js +1 -0
- package/dist/assets/web/dist/assets/{RunRoutePage-9klVWhSF.js → RunRoutePage-B3IirUVi.js} +4 -4
- package/dist/assets/web/dist/assets/RunsIndexPage-DiFmtGaJ.js +1 -0
- package/dist/assets/web/dist/assets/{ScheduledPage-B_GgeJrb.js → ScheduledPage-DvYwM2TE.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-BXX4R113.js → SettingsPage-Bo6yCyXZ.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-BOaw8le9.js → StatusDot-CAfS1AUi.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-DYZZ4l4V.js → Tooltip-tZoum_T-.js} +1 -1
- package/dist/assets/web/dist/assets/{UnitRoutePage-BaSW5Odq.js → UnitRoutePage-BmdOHwNn.js} +1 -1
- package/dist/assets/web/dist/assets/budgets-CbIyPAER.js +1 -0
- package/dist/assets/web/dist/assets/{dist-BCVXeBJ9.js → dist-DfbEpHXR.js} +1 -1
- package/dist/assets/web/dist/assets/indexRow-BT0cPVRw.js +1 -0
- package/dist/assets/web/dist/assets/{main-Dkcbtu3u.js → main-5Gm_1Gv8.js} +2 -2
- package/dist/assets/web/dist/assets/sseReplay-DmyMXfRC.js +11 -0
- package/dist/cli.js +2470 -902
- package/package.json +1 -1
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.sh +0 -68
- package/dist/assets/web/dist/assets/HomePage-mSiqEEcN.js +0 -2
- package/dist/assets/web/dist/assets/RunFoldRow-CSo4-vld.js +0 -1
- package/dist/assets/web/dist/assets/RunsIndexPage-BplMIgaw.js +0 -1
- package/dist/assets/web/dist/assets/budgets-BvWYKPsY.js +0 -1
- package/dist/assets/web/dist/assets/indexRow-Bde9OZxG.js +0 -1
- package/dist/assets/web/dist/assets/sseReplay-DPwdsaok.js +0 -9
|
@@ -325,8 +325,10 @@ workspaceDir: ./workspaces
|
|
|
325
325
|
# # and — when the run is bound to a repo — that repo's scope (repo:owner/name)
|
|
326
326
|
# # plus the requesting user's own scope (user:slack:U…) — a
|
|
327
327
|
# # person's records never surface for anyone else.
|
|
328
|
-
# limit:
|
|
329
|
-
# maxTokens:
|
|
328
|
+
# limit: 32 # max records retrieved/injected per request (default 32)
|
|
329
|
+
# maxTokens: 3000 # hard token budget for the injected block (default 3000)
|
|
330
|
+
# repoWindow: 24 # newest repo facts injected ahead of the keyword hits when
|
|
331
|
+
# # the run is bound to a repo (default 24; 0 disables the window)
|
|
330
332
|
# maxRecordsPerScope: 500
|
|
331
333
|
# # per-scope cap on ACTIVE records (default 500): a write
|
|
332
334
|
# # that would exceed it evicts the least recently used
|
|
@@ -1,23 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Deploy preflight for the bot Worker (docs/reference/specs/slack-channel.md item 8).
|
|
3
3
|
//
|
|
4
|
-
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// next generation
|
|
8
|
-
//
|
|
9
|
-
// `npm run deploy` runs
|
|
10
|
-
// - the bot reports runs in flight (`GET /healthz` `inFlight > 0`): the
|
|
11
|
-
// runner retries every minute up to its budget, then fails by name;
|
|
4
|
+
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM;
|
|
5
|
+
// since the run ledger's handoff (docs/reference/specs/run-history.md item 39) the bot
|
|
6
|
+
// hands every resumable run to the next generation and exits within seconds,
|
|
7
|
+
// and the next generation continues the runs under their own cards — so a
|
|
8
|
+
// deploy no longer waits on runs, and this preflight no longer refuses for
|
|
9
|
+
// them. `npm run deploy` runs it first and refuses only while
|
|
12
10
|
// - the container application is not in a settled state (a rollout is still
|
|
13
11
|
// provisioning/updating — `wrangler containers list --json`): a second
|
|
14
12
|
// rollout on top of one in progress replaces the instance the first put
|
|
15
13
|
// into its graceful drain and kills whatever it was running (two deploys
|
|
16
14
|
// 90 s apart once killed a review at 153 s).
|
|
17
|
-
// It WARNS (never refuses) when the bot
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
15
|
+
// It WARNS (never refuses) when the bot reports runs in flight (`inFlight > 0`
|
|
16
|
+
// — they hand off) or is already draining (`draining: true` — its resumable
|
|
17
|
+
// runs were handed off; a ship pipeline still in flight would be killed), and
|
|
18
|
+
// when /healthz says the reconnect catch-up is failing or the bot token lacks
|
|
19
|
+
// required scopes (`catchUp.error`, `catchUp.missingScopes`).
|
|
21
20
|
//
|
|
22
21
|
// Fail closed: unreachable bot, a body without the JSON shape (a bot whose
|
|
23
22
|
// /healthz answers a bare `ok` is not one this preflight can read), a wrangler failure, or an app
|
|
@@ -41,7 +40,7 @@ export const APP_NAME = "switchboard-switchboardserver";
|
|
|
41
40
|
const SETTLED_APP_STATES = new Set(["active", "ready"]);
|
|
42
41
|
|
|
43
42
|
const HOW_TO_FORCE =
|
|
44
|
-
"to deploy anyway (over
|
|
43
|
+
"to deploy anyway (over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
|
|
45
44
|
|
|
46
45
|
/** GET /healthz. Never throws: `{ok:true,payload}` (parsed JSON, or the raw text when not JSON) or `{ok:false,error}`. */
|
|
47
46
|
export async function fetchHealth(baseUrl, { timeoutMs = 20_000 } = {}) {
|
|
@@ -165,16 +164,15 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
165
164
|
if (!Number.isInteger(p.inFlight) || p.inFlight < 0) {
|
|
166
165
|
problems.push(`bot reports an impossible inFlight=${JSON.stringify(p.inFlight)} (counter bug or old Worker)`);
|
|
167
166
|
} else if (p.inFlight > 0) {
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
`${p.inFlight} run(s) in flight — the rollout would roll the bot container under them (a handoff is a recovery, not a guarantee)`,
|
|
167
|
+
// Not a refusal since the handoff (run-history item 39): SIGTERM hands
|
|
168
|
+
// every resumable run to the next generation, which continues it.
|
|
169
|
+
warnings.push(
|
|
170
|
+
`${p.inFlight} run(s) in flight — handed to the next generation on SIGTERM (run-history item 39); they continue there under their own cards`,
|
|
173
171
|
);
|
|
174
172
|
}
|
|
175
173
|
if (p.draining === true) {
|
|
176
174
|
warnings.push(
|
|
177
|
-
"bot is already draining from a previous deploy —
|
|
175
|
+
"bot is already draining from a previous deploy — its resumable runs are handed off; a ship pipeline still in flight would be killed when this rollout replaces the draining instance",
|
|
178
176
|
);
|
|
179
177
|
}
|
|
180
178
|
}
|
|
@@ -211,7 +209,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
211
209
|
forced: true,
|
|
212
210
|
problems,
|
|
213
211
|
warnings,
|
|
214
|
-
message: `preflight WARNING: deploying by force despite —\n${detail}\n
|
|
212
|
+
message: `preflight WARNING: deploying by force despite —\n${detail}\n a rollout landing on one in progress can disrupt it; in-flight runs hand off regardless (run-history item 39)${warningText}`,
|
|
215
213
|
};
|
|
216
214
|
}
|
|
217
215
|
return {
|
|
@@ -219,7 +217,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
219
217
|
forced: false,
|
|
220
218
|
problems,
|
|
221
219
|
warnings,
|
|
222
|
-
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait
|
|
220
|
+
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait and retry; ${HOW_TO_FORCE}${warningText}`,
|
|
223
221
|
};
|
|
224
222
|
}
|
|
225
223
|
|
|
@@ -197,8 +197,10 @@ export class SwitchboardServer extends Container<Env> {
|
|
|
197
197
|
* and exits, and the NEXT request through `fetch` starts the container again
|
|
198
198
|
* (`startBot`, env computed then). The keep-alive cron GETs /healthz every
|
|
199
199
|
* minute and the CLI's live gate polls it every 15 s, so the next request is
|
|
200
|
-
* never more than seconds away. Refuses (the deploy preflight's rules)
|
|
201
|
-
*
|
|
200
|
+
* never more than seconds away. Refuses (the deploy preflight's rules) only the
|
|
201
|
+
* fail-closed cases — no JSON body, an impossible `inFlight` — unless
|
|
202
|
+
* `force`; runs in flight or a drain under way warn and the stop proceeds
|
|
203
|
+
* (the handoff rule).
|
|
202
204
|
*/
|
|
203
205
|
async restart(opts: { force: boolean }): Promise<RestartOutcome> {
|
|
204
206
|
if (!this.ctx.container?.running) return { kind: "not-running" };
|
|
@@ -265,7 +267,8 @@ export class SwitchboardServer extends Container<Env> {
|
|
|
265
267
|
* `deploy:write` in the bot's config). The Worker authenticates the bearer
|
|
266
268
|
* against the map it holds — an unknown bearer never touches the container —
|
|
267
269
|
* and the Container DO asks the bot for the grant before stopping anything.
|
|
268
|
-
* Body `{ "force": true }` bypasses the
|
|
270
|
+
* Body `{ "force": true }` bypasses the fail-closed refusals (no JSON body,
|
|
271
|
+
* impossible `inFlight`); runs in flight or a drain warn and never refuse. */
|
|
269
272
|
async function handleAdminRestart(request: Request, env: Env): Promise<Response> {
|
|
270
273
|
const json = (status: number, body: Record<string, unknown>) =>
|
|
271
274
|
new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
planEviction,
|
|
8
8
|
planWrite,
|
|
9
9
|
rankRecords,
|
|
10
|
+
rejectionMarkers,
|
|
10
11
|
} from "../../src/core/memory/engine.ts";
|
|
11
12
|
import { tokenize } from "../../src/core/memory/scorer.ts";
|
|
12
13
|
import { FIRING_DETAIL_MAX, isScheduleFiring, type ScheduleFiring } from "../../src/core/schedules.ts";
|
|
@@ -148,6 +149,7 @@ const traceSinks = [workerLogSink((line) => console.log(line))];
|
|
|
148
149
|
// Route surface (JSON in/out; bearer MEMORY_TOKEN on everything but /healthz):
|
|
149
150
|
// POST /retrieve {scopeKey, query, limit} → {records: MemoryRecord[]}
|
|
150
151
|
// POST /write {scopeKey, records: MemoryCandidate[]} → {ok, inserted, deduped, superseded}
|
|
152
|
+
// POST /sweep {scopeKey, dryRun?} → {ok, swept} (+ ids under dryRun — the marked rows, flipped to `swept`)
|
|
151
153
|
// GET /healthz → {ok:true} (deploy wake ping; touches no DO)
|
|
152
154
|
// Scheduled-firing routes (the record behind the /runs Scheduled panel;
|
|
153
155
|
// written by the bot's Worker shim after every cron firing, read by the bot's
|
|
@@ -463,14 +465,19 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
463
465
|
return counts;
|
|
464
466
|
}
|
|
465
467
|
|
|
466
|
-
/** Human view (docs/reference/specs/memory.md
|
|
467
|
-
* bump. With `query`, only rows
|
|
468
|
-
*
|
|
469
|
-
* query with no tokens lists
|
|
470
|
-
|
|
468
|
+
/** Human view and the repository window's read (docs/reference/specs/memory.md items 22, 26): the
|
|
469
|
+
* scope's ACTIVE rows, newest first, no usage bump. With `query`, only rows
|
|
470
|
+
* an FTS token hits (the same quoted-OR MATCH as retrieve, so user text
|
|
471
|
+
* never reaches the FTS parser as syntax); a query with no tokens lists
|
|
472
|
+
* nothing. With `kind`, only rows of that kind (the window lists facts). */
|
|
473
|
+
async list(_scopeKey: string, limit: number, query?: string, kind?: MemoryRecord["kind"]): Promise<MemoryRecord[]> {
|
|
471
474
|
if (query === undefined) {
|
|
472
475
|
return this.sql
|
|
473
|
-
.exec<Row>(
|
|
476
|
+
.exec<Row>(
|
|
477
|
+
`SELECT * FROM records WHERE status = 'active'${kind === undefined ? "" : " AND kind = ?"} ORDER BY seq DESC LIMIT ?`,
|
|
478
|
+
...(kind === undefined ? [] : [kind]),
|
|
479
|
+
limit,
|
|
480
|
+
)
|
|
474
481
|
.toArray()
|
|
475
482
|
.map(toRecord);
|
|
476
483
|
}
|
|
@@ -480,9 +487,10 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
480
487
|
.exec<Row>(
|
|
481
488
|
`SELECT r.* FROM records r
|
|
482
489
|
JOIN records_fts f ON f.id = r.id
|
|
483
|
-
WHERE r.status = 'active' AND records_fts MATCH ?
|
|
490
|
+
WHERE r.status = 'active'${kind === undefined ? "" : " AND r.kind = ?"} AND records_fts MATCH ?
|
|
484
491
|
ORDER BY r.seq DESC
|
|
485
492
|
LIMIT ?`,
|
|
493
|
+
...(kind === undefined ? [] : [kind]),
|
|
486
494
|
match,
|
|
487
495
|
limit,
|
|
488
496
|
)
|
|
@@ -504,6 +512,33 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
504
512
|
return flipped;
|
|
505
513
|
});
|
|
506
514
|
}
|
|
515
|
+
|
|
516
|
+
/** Human control (docs/reference/specs/memory.md item 27): retire every
|
|
517
|
+
* ACTIVE fact whose text the write gate would reject today — the same
|
|
518
|
+
* `rejectionMarkers` the bot's write path runs, imported from the shared
|
|
519
|
+
* engine, so the sweep and the gate can never disagree. Summaries are never
|
|
520
|
+
* gated, so never swept. The scan, the flips and the FTS deletes run in ONE
|
|
521
|
+
* sync transaction (the per-scope cap's atomicity rule): the sweep commits
|
|
522
|
+
* whole or not at all. Idempotent — swept rows are no longer active, so a
|
|
523
|
+
* second call answers 0. Under `dryRun` nothing flips. */
|
|
524
|
+
async sweep(_scopeKey: string, dryRun: boolean): Promise<{ swept: number; ids: string[] }> {
|
|
525
|
+
return this.ctx.storage.transactionSync(() => {
|
|
526
|
+
const ids = this.sql
|
|
527
|
+
.exec<Row>(`SELECT * FROM records WHERE status = 'active' AND kind = 'fact'`)
|
|
528
|
+
.toArray()
|
|
529
|
+
.filter((row) => rejectionMarkers(row.text).length > 0)
|
|
530
|
+
.map((row) => row.id);
|
|
531
|
+
if (!dryRun) {
|
|
532
|
+
for (const id of ids) {
|
|
533
|
+
// Soft delete for the record row, hard delete for its FTS entry —
|
|
534
|
+
// the forget/supersede/evict hygiene rule (§15).
|
|
535
|
+
this.sql.exec(`UPDATE records SET status = 'swept' WHERE id = ? AND status = 'active'`, id);
|
|
536
|
+
this.sql.exec(`DELETE FROM records_fts WHERE id = ?`, id);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
return { swept: ids.length, ids };
|
|
540
|
+
});
|
|
541
|
+
}
|
|
507
542
|
}
|
|
508
543
|
|
|
509
544
|
/** Build the FTS5 MATCH expression for a query: each engine token (`[a-z0-9]+`
|
|
@@ -3121,8 +3156,10 @@ function parseLimit(v: unknown): Validated<number> {
|
|
|
3121
3156
|
return { ok: true, value: v };
|
|
3122
3157
|
}
|
|
3123
3158
|
|
|
3124
|
-
/** `POST /list {scopeKey, limit, query?}`. */
|
|
3125
|
-
function parseList(
|
|
3159
|
+
/** `POST /list {scopeKey, limit, query?, kind?}`. */
|
|
3160
|
+
function parseList(
|
|
3161
|
+
body: unknown,
|
|
3162
|
+
): Validated<{ scopeKey: string; limit: number; query?: string; kind?: MemoryRecord["kind"] }> {
|
|
3126
3163
|
if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
|
|
3127
3164
|
const b = body as Record<string, unknown>;
|
|
3128
3165
|
const scope = parseScopeKey(b.scopeKey);
|
|
@@ -3133,12 +3170,29 @@ function parseList(body: unknown): Validated<{ scopeKey: string; limit: number;
|
|
|
3133
3170
|
if (typeof b.query !== "string") return invalid("query must be a string");
|
|
3134
3171
|
if (b.query.length > MAX_QUERY_CHARS) return invalid(`query must be at most ${MAX_QUERY_CHARS} characters`);
|
|
3135
3172
|
}
|
|
3173
|
+
if (b.kind !== undefined && b.kind !== "fact" && b.kind !== "summary")
|
|
3174
|
+
return invalid('kind must be "fact" or "summary"');
|
|
3136
3175
|
return {
|
|
3137
3176
|
ok: true,
|
|
3138
|
-
value: {
|
|
3177
|
+
value: {
|
|
3178
|
+
scopeKey: scope.value,
|
|
3179
|
+
limit: limit.value,
|
|
3180
|
+
...(typeof b.query === "string" ? { query: b.query } : {}),
|
|
3181
|
+
...(b.kind === "fact" || b.kind === "summary" ? { kind: b.kind } : {}),
|
|
3182
|
+
},
|
|
3139
3183
|
};
|
|
3140
3184
|
}
|
|
3141
3185
|
|
|
3186
|
+
/** `POST /sweep {scopeKey, dryRun?}`. */
|
|
3187
|
+
function parseSweep(body: unknown): Validated<{ scopeKey: string; dryRun: boolean }> {
|
|
3188
|
+
if (!isJsonObject(body)) return invalid("body must be a JSON object");
|
|
3189
|
+
const b = body;
|
|
3190
|
+
const scope = parseScopeKey(b.scopeKey);
|
|
3191
|
+
if (!scope.ok) return scope;
|
|
3192
|
+
if (b.dryRun !== undefined && typeof b.dryRun !== "boolean") return invalid("dryRun must be a boolean");
|
|
3193
|
+
return { ok: true, value: { scopeKey: scope.value, dryRun: b.dryRun === true } };
|
|
3194
|
+
}
|
|
3195
|
+
|
|
3142
3196
|
/** `POST /forget {scopeKey, id}`: the id is an opaque key, same caps as scopeKey. */
|
|
3143
3197
|
function parseForget(body: unknown): Validated<{ scopeKey: string; id: string }> {
|
|
3144
3198
|
if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
|
|
@@ -4329,6 +4383,7 @@ const ROUTES = new Set([
|
|
|
4329
4383
|
"/write",
|
|
4330
4384
|
"/list",
|
|
4331
4385
|
"/forget",
|
|
4386
|
+
"/sweep",
|
|
4332
4387
|
"/schedules/record",
|
|
4333
4388
|
"/schedules/latest",
|
|
4334
4389
|
"/runs/put",
|
|
@@ -4422,8 +4477,8 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
|
|
|
4422
4477
|
if (url.pathname === "/list") {
|
|
4423
4478
|
const parsed = parseList(body);
|
|
4424
4479
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
4425
|
-
const { scopeKey, limit, query } = parsed.value;
|
|
4426
|
-
const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query);
|
|
4480
|
+
const { scopeKey, limit, query, kind } = parsed.value;
|
|
4481
|
+
const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query, kind);
|
|
4427
4482
|
console.log(`[list] ${scopeKey} -> ${records.length} records`);
|
|
4428
4483
|
return json({ records });
|
|
4429
4484
|
}
|
|
@@ -4436,6 +4491,16 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
|
|
|
4436
4491
|
console.log(`[forget] ${scopeKey} ${id} -> ${forgotten}`);
|
|
4437
4492
|
return json({ ok: true, forgotten });
|
|
4438
4493
|
}
|
|
4494
|
+
if (url.pathname === "/sweep") {
|
|
4495
|
+
const parsed = parseSweep(body);
|
|
4496
|
+
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
4497
|
+
const { scopeKey, dryRun } = parsed.value;
|
|
4498
|
+
const out = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).sweep(scopeKey, dryRun);
|
|
4499
|
+
// Observability: scope + count only (ids carry no record text; they ride
|
|
4500
|
+
// the dryRun answer, not the log).
|
|
4501
|
+
console.log(`[sweep] ${scopeKey} -> ${out.swept}${dryRun ? " (dry run)" : ""}`);
|
|
4502
|
+
return json({ ok: true, swept: out.swept, ...(dryRun ? { ids: out.ids } : {}) });
|
|
4503
|
+
}
|
|
4439
4504
|
|
|
4440
4505
|
const parsed = parseWrite(body);
|
|
4441
4506
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// The resident guards its own memory (docs/reference/specs/resident-repos.md
|
|
2
|
+
// item 70) — the PURE half, kept free of the Sandbox SDK and DO storage so it
|
|
3
|
+
// runs under plain-Node vitest (memoryGuard.test.ts) like gc.ts and drain.ts.
|
|
4
|
+
// The Worker owns the one real reader (a single exec of CGROUP_READ_ARGV via
|
|
5
|
+
// its exec choke point) and feeds this module the raw output; tests feed a
|
|
6
|
+
// fake. A container at its cgroup memory cap wedges its own control port and
|
|
7
|
+
// every run on it loses its work, so the resident refuses NEW work by name
|
|
8
|
+
// while it still can: above the soft threshold a new attach is refused like
|
|
9
|
+
// `mirror-busy` (the bot falls back cold or waits, the card says why), above
|
|
10
|
+
// the hard threshold a new exec is refused with the numbers — and a command
|
|
11
|
+
// already running is never touched: the gate sits at each route's start and
|
|
12
|
+
// kills nothing.
|
|
13
|
+
import { MEMORY_PRESSURE_REASON } from "../../src/execution/sandboxErrors.js";
|
|
14
|
+
|
|
15
|
+
export { MEMORY_PRESSURE_REASON };
|
|
16
|
+
|
|
17
|
+
/** Above this percent of the cgroup memory cap a NEW `/attach` is refused —
|
|
18
|
+
* new runs go elsewhere while the ones already here finish. */
|
|
19
|
+
export const MEMORY_SOFT_LIMIT_PCT = 80;
|
|
20
|
+
|
|
21
|
+
/** Above this percent a NEW `/exec` is refused too — the resident answers
|
|
22
|
+
* nothing but the work already in flight, which always runs to completion. */
|
|
23
|
+
export const MEMORY_HARD_LIMIT_PCT = 90;
|
|
24
|
+
|
|
25
|
+
/** One reading of the container's cgroup v2 accounting. */
|
|
26
|
+
export interface MemoryReading {
|
|
27
|
+
/** ISO timestamp the caller supplied (the resident's clock seam). */
|
|
28
|
+
at: string;
|
|
29
|
+
/** memory.current — bytes charged to the cgroup now. */
|
|
30
|
+
usedBytes: number;
|
|
31
|
+
/** memory.max — the cap in bytes, or null when the file reads `max` (no cap). */
|
|
32
|
+
capBytes: number | null;
|
|
33
|
+
/** used/cap rounded to whole percent; null without a cap (nothing to gate on). */
|
|
34
|
+
percent: number | null;
|
|
35
|
+
/** cpu.stat's usage_usec, for the same log line; null when absent. */
|
|
36
|
+
cpuUsageUsec: number | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The reader seam: one call returns the raw output of `CGROUP_READ_ARGV`.
|
|
40
|
+
* A throw of `MemorySampleUnavailable` means "no reading this time" (the
|
|
41
|
+
* container busy or replaced under the probe) and never disables the gate;
|
|
42
|
+
* any other throw is an unreadable cgroup and disables it (one log line). */
|
|
43
|
+
export interface CgroupReader {
|
|
44
|
+
read(): Promise<string>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The one transient escape: the reader could not run at all right now.
|
|
48
|
+
* `invalidates` says the container the last reading came from is gone (a
|
|
49
|
+
* runtime replacement under the probe), so the guard forgets that reading —
|
|
50
|
+
* a dead container's numbers must not gate the work that replaces it. */
|
|
51
|
+
export class MemorySampleUnavailable extends Error {
|
|
52
|
+
constructor(
|
|
53
|
+
message: string,
|
|
54
|
+
readonly invalidates = false,
|
|
55
|
+
) {
|
|
56
|
+
super(message);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The separator `CGROUP_READ_ARGV` prints between the three files. */
|
|
61
|
+
export const CGROUP_FILE_SEPARATOR = ":::";
|
|
62
|
+
|
|
63
|
+
/** One exec, three files: memory.current, memory.max, cpu.stat from the
|
|
64
|
+
* container's own cgroup v2 root. `&&` so a missing file is a non-zero exit
|
|
65
|
+
* (an unreadable cgroup), never a silently short output. */
|
|
66
|
+
export const CGROUP_READ_ARGV: readonly string[] = [
|
|
67
|
+
"sh",
|
|
68
|
+
"-c",
|
|
69
|
+
`cat /sys/fs/cgroup/memory.current && echo ${CGROUP_FILE_SEPARATOR} && ` +
|
|
70
|
+
`cat /sys/fs/cgroup/memory.max && echo ${CGROUP_FILE_SEPARATOR} && cat /sys/fs/cgroup/cpu.stat`,
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
/** Parses one reader output into a reading; throws on anything malformed. */
|
|
74
|
+
export function parseCgroupOutput(raw: string, at: string): MemoryReading {
|
|
75
|
+
const parts = raw.split(CGROUP_FILE_SEPARATOR).map((p) => p.trim());
|
|
76
|
+
if (parts.length !== 3) throw new Error(`expected 3 cgroup sections, got ${parts.length}`);
|
|
77
|
+
const usedBytes = Number(parts[0]);
|
|
78
|
+
if (!Number.isFinite(usedBytes) || usedBytes < 0) throw new Error(`memory.current is not a byte count: ${parts[0]}`);
|
|
79
|
+
let capBytes: number | null = null;
|
|
80
|
+
if (parts[1] !== "max") {
|
|
81
|
+
capBytes = Number(parts[1]);
|
|
82
|
+
if (!Number.isFinite(capBytes) || capBytes <= 0) throw new Error(`memory.max is not a byte count: ${parts[1]}`);
|
|
83
|
+
}
|
|
84
|
+
const usage = /(?:^|\n)usage_usec (\d+)/.exec(parts[2]);
|
|
85
|
+
const cpuUsageUsec = usage ? Number(usage[1]) : null;
|
|
86
|
+
const percent = capBytes === null ? null : Math.round((usedBytes / capBytes) * 100);
|
|
87
|
+
return { at, usedBytes, capBytes, percent, cpuUsageUsec };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The one structured line Workers Logs gets per sample: used bytes, cap
|
|
91
|
+
* bytes, percent (and the cpu counter beside them). */
|
|
92
|
+
export function memoryLogLine(r: MemoryReading): string {
|
|
93
|
+
const cap = r.capBytes === null ? "uncapped" : `${r.capBytes} bytes cap`;
|
|
94
|
+
const pct = r.percent === null ? "" : ` (${r.percent}%)`;
|
|
95
|
+
const cpu = r.cpuUsageUsec === null ? "" : ` — cpu usage_usec ${r.cpuUsageUsec}`;
|
|
96
|
+
return `memory: used ${r.usedBytes} bytes of ${cap}${pct}${cpu}`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Which route asks the gate. */
|
|
100
|
+
export type MemoryGateRoute = "attach" | "exec";
|
|
101
|
+
|
|
102
|
+
/** A refusal with the numbers on it — the message is the whole story. */
|
|
103
|
+
export interface MemoryRefusal {
|
|
104
|
+
reason: typeof MEMORY_PRESSURE_REASON;
|
|
105
|
+
percent: number;
|
|
106
|
+
usedBytes: number;
|
|
107
|
+
capBytes: number;
|
|
108
|
+
message: string;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The pure gate over one reading: below both thresholds nothing changes; at
|
|
112
|
+
* or above the soft one a new attach is refused (queued — the caller waits or
|
|
113
|
+
* falls back, like `mirror-busy`); at or above the hard one a new exec is
|
|
114
|
+
* refused too. No reading, or no cap, gates nothing. */
|
|
115
|
+
export function gateMemory(route: MemoryGateRoute, reading: MemoryReading | null): MemoryRefusal | null {
|
|
116
|
+
if (!reading || reading.percent === null || reading.capBytes === null) return null;
|
|
117
|
+
const { percent, usedBytes, capBytes } = reading;
|
|
118
|
+
const refusal = (message: string): MemoryRefusal => ({
|
|
119
|
+
reason: MEMORY_PRESSURE_REASON,
|
|
120
|
+
percent,
|
|
121
|
+
usedBytes,
|
|
122
|
+
capBytes,
|
|
123
|
+
message,
|
|
124
|
+
});
|
|
125
|
+
if (route === "exec" && percent >= MEMORY_HARD_LIMIT_PCT) {
|
|
126
|
+
return refusal(
|
|
127
|
+
`${MEMORY_PRESSURE_REASON}: resident at ${percent}% of its memory cap (${usedBytes} of ${capBytes} bytes) — ` +
|
|
128
|
+
`a new command is refused above ${MEMORY_HARD_LIMIT_PCT}% while the commands already running finish`,
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
if (route === "attach" && percent >= MEMORY_SOFT_LIMIT_PCT) {
|
|
132
|
+
return refusal(
|
|
133
|
+
`${MEMORY_PRESSURE_REASON}: resident near its memory cap, ${percent}% used (${usedBytes} of ${capBytes} bytes), ` +
|
|
134
|
+
`queued — a new attach is refused above ${MEMORY_SOFT_LIMIT_PCT}% until the runs here release memory`,
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The message of whatever was thrown (local: this module stays dependency-free). */
|
|
141
|
+
const causeText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
|
142
|
+
|
|
143
|
+
/** The guard one resident holds: samples through the reader, keeps the last
|
|
144
|
+
* reading, gates the two routes — and on an unreadable cgroup logs ONCE and
|
|
145
|
+
* disables itself for the incarnation instead of refusing everything. */
|
|
146
|
+
export class MemoryGuard {
|
|
147
|
+
private disabledWhy: string | null = null;
|
|
148
|
+
private last: MemoryReading | null = null;
|
|
149
|
+
|
|
150
|
+
constructor(
|
|
151
|
+
private readonly reader: CgroupReader,
|
|
152
|
+
private readonly log: (line: string) => void,
|
|
153
|
+
) {}
|
|
154
|
+
|
|
155
|
+
/** The last reading taken this incarnation, for `/status` and `/residents`. */
|
|
156
|
+
get lastReading(): MemoryReading | null {
|
|
157
|
+
return this.last;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Whether an unreadable cgroup turned the gate off (the why is logged once). */
|
|
161
|
+
get disabled(): boolean {
|
|
162
|
+
return this.disabledWhy !== null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Forgets the last reading: the container it measured no longer runs (it
|
|
166
|
+
* went inactive, or was replaced under the probe), so nothing gates on it
|
|
167
|
+
* until a fresh sample lands. The gate's disabled state is untouched. */
|
|
168
|
+
invalidate(): void {
|
|
169
|
+
this.last = null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** One reader call, one log line. A `MemorySampleUnavailable` keeps the last
|
|
173
|
+
* reading and the gate as they were — unless it `invalidates` (the runtime
|
|
174
|
+
* was replaced under the probe), which forgets the reading so a stale one
|
|
175
|
+
* never masks the route's own runtime-replaced answer; any other failure
|
|
176
|
+
* disables the gate. */
|
|
177
|
+
async sample(at: string): Promise<MemoryReading | null> {
|
|
178
|
+
if (this.disabledWhy !== null) return null;
|
|
179
|
+
let raw: string;
|
|
180
|
+
try {
|
|
181
|
+
raw = await this.reader.read();
|
|
182
|
+
} catch (err) {
|
|
183
|
+
if (err instanceof MemorySampleUnavailable) {
|
|
184
|
+
if (err.invalidates) this.last = null;
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
this.disable(causeText(err));
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
let reading: MemoryReading;
|
|
191
|
+
try {
|
|
192
|
+
reading = parseCgroupOutput(raw, at);
|
|
193
|
+
} catch (err) {
|
|
194
|
+
this.disable(causeText(err));
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
this.last = reading;
|
|
198
|
+
this.log(memoryLogLine(reading));
|
|
199
|
+
return reading;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The route's verdict over the last reading; a disabled gate refuses nothing. */
|
|
203
|
+
gate(route: MemoryGateRoute): MemoryRefusal | null {
|
|
204
|
+
if (this.disabledWhy !== null) return null;
|
|
205
|
+
return gateMemory(route, this.last);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
private disable(why: string): void {
|
|
209
|
+
this.disabledWhy = why;
|
|
210
|
+
this.log(`memory: cgroup unreadable — the memory gate is disabled for this incarnation (${why})`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -58,7 +58,7 @@ export interface RefreshInstanceAction {
|
|
|
58
58
|
* (`stepTimeoutMs`), so a step timeout and a command timeout agree. */
|
|
59
59
|
const REFRESH_FETCH_STEP_BUDGET_MS = RESTORE_MAX_MS + GIT_NETWORK_TIMEOUT_MS; // a wake's restore, then the fetch
|
|
60
60
|
const REFRESH_INSTALL_STEP_BUDGET_MS = REFRESH_INSTALL_TIMEOUT_MS + DEPS_STEP_OVERHEAD_MS; // the install's own lease
|
|
61
|
-
const REFRESH_BUILD_STEP_BUDGET_MS = GIT_NETWORK_TIMEOUT_MS + REFRESH_BUILD_TIMEOUT_MS; // the
|
|
61
|
+
const REFRESH_BUILD_STEP_BUDGET_MS = 5 * GIT_NETWORK_TIMEOUT_MS + 2 * REFRESH_BUILD_TIMEOUT_MS; // five git-budgeted commands — stage-clear, stage, the staging fetch (under the stage lock), the checkout update and the swap (under the swap lock) — plus the off-lock reset/clean, deps re-link and build
|
|
62
62
|
const REFRESH_SNAPSHOT_STEP_BUDGET_MS = R2_TRANSFER_TIMEOUT_MS + GIT_NETWORK_TIMEOUT_MS; // the archives, then the reclaim pass
|
|
63
63
|
/** The sweep: one cleanliness check per binding idle past an hour (at most the
|
|
64
64
|
* pool's worth, one exec budget each), then the removals under the mirror lock. */
|