@coreplane/switchboard 1.251.0 → 1.252.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/assets/config/config.example.yaml +4 -2
  2. package/dist/assets/deploy/cloudflare/preflight.mjs +19 -21
  3. package/dist/assets/deploy/cloudflare/worker.ts +6 -3
  4. package/dist/assets/deploy/cloudflare-memory/worker.ts +77 -12
  5. package/dist/assets/deploy/cloudflare-resident/memoryGuard.ts +212 -0
  6. package/dist/assets/deploy/cloudflare-resident/refresh.ts +1 -1
  7. package/dist/assets/deploy/cloudflare-resident/worker.ts +317 -56
  8. package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +4 -2
  9. package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.d.mts +31 -0
  10. package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.mjs +119 -0
  11. package/dist/assets/package-lock.json +3 -3
  12. package/dist/assets/package.json +3 -2
  13. package/dist/assets/project.json +13 -9
  14. package/dist/assets/source.json +3 -3
  15. package/dist/assets/src/agents/registry.ts +5 -5
  16. package/dist/assets/src/core/budgets.ts +22 -0
  17. package/dist/assets/src/core/coordinator/contract.ts +42 -0
  18. package/dist/assets/src/core/coordinator/driver.ts +134 -10
  19. package/dist/assets/src/core/drain.ts +50 -0
  20. package/dist/assets/src/core/memory/engine.ts +98 -0
  21. package/dist/assets/src/core/memory/scorer.ts +12 -4
  22. package/dist/assets/src/core/memory/types.ts +69 -12
  23. package/dist/assets/src/core/modelCard.ts +32 -4
  24. package/dist/assets/src/core/modelPricing.ts +111 -1
  25. package/dist/assets/src/core/modelProxy/usage.ts +88 -0
  26. package/dist/assets/src/core/modelRegistry.ts +15 -1
  27. package/dist/assets/src/core/refusal.ts +4 -7
  28. package/dist/assets/src/core/reviewVerdict.ts +4 -0
  29. package/dist/assets/src/core/runEvents.ts +51 -2
  30. package/dist/assets/src/core/runFriction.ts +7 -2
  31. package/dist/assets/src/core/runLedger/types.ts +11 -0
  32. package/dist/assets/src/core/runUsage.ts +67 -13
  33. package/dist/assets/src/core/schedules.ts +3 -0
  34. package/dist/assets/src/core/ship/contract.ts +41 -14
  35. package/dist/assets/src/core/ship/coordinator.ts +380 -53
  36. package/dist/assets/src/core/ship/renewal.ts +10 -5
  37. package/dist/assets/src/core/trace/attrs.ts +24 -0
  38. package/dist/assets/src/core/types.ts +5 -5
  39. package/dist/assets/src/core/verbosity.ts +48 -0
  40. package/dist/assets/src/deploy/liveGate.ts +40 -13
  41. package/dist/assets/src/deploy/restart.ts +11 -12
  42. package/dist/assets/src/execution/residentDepCache.ts +50 -1
  43. package/dist/assets/src/execution/residentDepsStore.ts +40 -2
  44. package/dist/assets/src/execution/residentRefresh.ts +55 -3
  45. package/dist/assets/src/execution/residentSteps.ts +4 -0
  46. package/dist/assets/src/execution/sandboxErrors.ts +8 -0
  47. package/dist/assets/web/dist/.vite/manifest.json +55 -55
  48. package/dist/assets/web/dist/assets/{DeliveryPage-DUXd-Sl-.js → DeliveryPage-3ELQWM0r.js} +1 -1
  49. package/dist/assets/web/dist/assets/HomePage-BG_ok-K2.js +2 -0
  50. package/dist/assets/web/dist/assets/{PendingTurnRow-DDhMhrI7.js → PendingTurnRow-ChCQOLgZ.js} +1 -1
  51. package/dist/assets/web/dist/assets/{ResidentDetailPage-BnEoOnGQ.js → ResidentDetailPage-C9y3nbo8.js} +1 -1
  52. package/dist/assets/web/dist/assets/{ResidentsIndexPage-Dxpgf-l-.js → ResidentsIndexPage-i1RG9e7g.js} +1 -1
  53. package/dist/assets/web/dist/assets/RunFoldRow-D3wVpzBa.js +1 -0
  54. package/dist/assets/web/dist/assets/{RunRoutePage-9klVWhSF.js → RunRoutePage-B3IirUVi.js} +4 -4
  55. package/dist/assets/web/dist/assets/RunsIndexPage-DiFmtGaJ.js +1 -0
  56. package/dist/assets/web/dist/assets/{ScheduledPage-B_GgeJrb.js → ScheduledPage-DvYwM2TE.js} +1 -1
  57. package/dist/assets/web/dist/assets/{SettingsPage-BXX4R113.js → SettingsPage-Bo6yCyXZ.js} +1 -1
  58. package/dist/assets/web/dist/assets/{StatusDot-BOaw8le9.js → StatusDot-CAfS1AUi.js} +1 -1
  59. package/dist/assets/web/dist/assets/{Tooltip-DYZZ4l4V.js → Tooltip-tZoum_T-.js} +1 -1
  60. package/dist/assets/web/dist/assets/{UnitRoutePage-BaSW5Odq.js → UnitRoutePage-BmdOHwNn.js} +1 -1
  61. package/dist/assets/web/dist/assets/budgets-CbIyPAER.js +1 -0
  62. package/dist/assets/web/dist/assets/{dist-BCVXeBJ9.js → dist-DfbEpHXR.js} +1 -1
  63. package/dist/assets/web/dist/assets/indexRow-BT0cPVRw.js +1 -0
  64. package/dist/assets/web/dist/assets/{main-Dkcbtu3u.js → main-5Gm_1Gv8.js} +2 -2
  65. package/dist/assets/web/dist/assets/sseReplay-DmyMXfRC.js +11 -0
  66. package/dist/cli.js +2470 -902
  67. package/package.json +1 -1
  68. package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.sh +0 -68
  69. package/dist/assets/web/dist/assets/HomePage-mSiqEEcN.js +0 -2
  70. package/dist/assets/web/dist/assets/RunFoldRow-CSo4-vld.js +0 -1
  71. package/dist/assets/web/dist/assets/RunsIndexPage-BplMIgaw.js +0 -1
  72. package/dist/assets/web/dist/assets/budgets-BvWYKPsY.js +0 -1
  73. package/dist/assets/web/dist/assets/indexRow-Bde9OZxG.js +0 -1
  74. package/dist/assets/web/dist/assets/sseReplay-DPwdsaok.js +0 -9
@@ -325,8 +325,10 @@ workspaceDir: ./workspaces
325
325
  # # and — when the run is bound to a repo — that repo's scope (repo:owner/name)
326
326
  # # plus the requesting user's own scope (user:slack:U…) — a
327
327
  # # person's records never surface for anyone else.
328
- # limit: 8 # max records retrieved/injected per request (default 8)
329
- # maxTokens: 800 # hard token budget for the injected block (default 800)
328
+ # limit: 32 # max records retrieved/injected per request (default 32)
329
+ # maxTokens: 3000 # hard token budget for the injected block (default 3000)
330
+ # repoWindow: 24 # newest repo facts injected ahead of the keyword hits when
331
+ # # the run is bound to a repo (default 24; 0 disables the window)
330
332
  # maxRecordsPerScope: 500
331
333
  # # per-scope cap on ACTIVE records (default 500): a write
332
334
  # # that would exceed it evicts the least recently used
@@ -1,23 +1,22 @@
1
1
  #!/usr/bin/env node
2
2
  // Deploy preflight for the bot Worker (docs/reference/specs/slack-channel.md item 8).
3
3
  //
4
- // `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM
5
- // and the container is gone at the drain's end, whatever it was still driving:
6
- // the run ledger's handoff (docs/reference/specs/run-history.md item 39) lets the
7
- // next generation resume a run, but a resume is a recovery, not a guarantee —
8
- // one that fails costs the run — so a deploy never rolls over a live run.
9
- // `npm run deploy` runs this first and refuses while
10
- // - the bot reports runs in flight (`GET /healthz` `inFlight > 0`): the
11
- // runner retries every minute up to its budget, then fails by name;
4
+ // `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM;
5
+ // since the run ledger's handoff (docs/reference/specs/run-history.md item 39) the bot
6
+ // hands every resumable run to the next generation and exits within seconds,
7
+ // and the next generation continues the runs under their own cards — so a
8
+ // deploy no longer waits on runs, and this preflight no longer refuses for
9
+ // them. `npm run deploy` runs it first and refuses only while
12
10
  // - the container application is not in a settled state (a rollout is still
13
11
  // provisioning/updating — `wrangler containers list --json`): a second
14
12
  // rollout on top of one in progress replaces the instance the first put
15
13
  // into its graceful drain and kills whatever it was running (two deploys
16
14
  // 90 s apart once killed a review at 153 s).
17
- // It WARNS (never refuses) when the bot is already draining with nothing in
18
- // flight (`draining: true` — the instance exits on its own), and when /healthz
19
- // says the reconnect catch-up is failing or the bot token lacks required
20
- // scopes (`catchUp.error`, `catchUp.missingScopes`).
15
+ // It WARNS (never refuses) when the bot reports runs in flight (`inFlight > 0`
16
+ // — they hand off) or is already draining (`draining: true` — its resumable
17
+ // runs were handed off; a ship pipeline still in flight would be killed), and
18
+ // when /healthz says the reconnect catch-up is failing or the bot token lacks
19
+ // required scopes (`catchUp.error`, `catchUp.missingScopes`).
21
20
  //
22
21
  // Fail closed: unreachable bot, a body without the JSON shape (a bot whose
23
22
  // /healthz answers a bare `ok` is not one this preflight can read), a wrangler failure, or an app
@@ -41,7 +40,7 @@ export const APP_NAME = "switchboard-switchboardserver";
41
40
  const SETTLED_APP_STATES = new Set(["active", "ready"]);
42
41
 
43
42
  const HOW_TO_FORCE =
44
- "to deploy anyway (over the runs in flight — this kills them —, over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
43
+ "to deploy anyway (over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
45
44
 
46
45
  /** GET /healthz. Never throws: `{ok:true,payload}` (parsed JSON, or the raw text when not JSON) or `{ok:false,error}`. */
47
46
  export async function fetchHealth(baseUrl, { timeoutMs = 20_000 } = {}) {
@@ -165,16 +164,15 @@ export function decide({ health, apps }, { force = false } = {}) {
165
164
  if (!Number.isInteger(p.inFlight) || p.inFlight < 0) {
166
165
  problems.push(`bot reports an impossible inFlight=${JSON.stringify(p.inFlight)} (counter bug or old Worker)`);
167
166
  } else if (p.inFlight > 0) {
168
- // A refusal: the rollout rolls the container under these runs. A
169
- // handoff (run-history item 39) may resume them on the next generation,
170
- // but a resume that fails costs the run — the deploy waits instead.
171
- problems.push(
172
- `${p.inFlight} run(s) in flight — the rollout would roll the bot container under them (a handoff is a recovery, not a guarantee)`,
167
+ // Not a refusal since the handoff (run-history item 39): SIGTERM hands
168
+ // every resumable run to the next generation, which continues it.
169
+ warnings.push(
170
+ `${p.inFlight} run(s) in flight — handed to the next generation on SIGTERM (run-history item 39); they continue there under their own cards`,
173
171
  );
174
172
  }
175
173
  if (p.draining === true) {
176
174
  warnings.push(
177
- "bot is already draining from a previous deploy — the draining instance exits on its own; this rollout replaces it at once",
175
+ "bot is already draining from a previous deploy — its resumable runs are handed off; a ship pipeline still in flight would be killed when this rollout replaces the draining instance",
178
176
  );
179
177
  }
180
178
  }
@@ -211,7 +209,7 @@ export function decide({ health, apps }, { force = false } = {}) {
211
209
  forced: true,
212
210
  problems,
213
211
  warnings,
214
- message: `preflight WARNING: deploying by force despite —\n${detail}\n this WILL kill the runs in flight that no resume recovers, and a rollout landing on one in progress can disrupt it${warningText}`,
212
+ message: `preflight WARNING: deploying by force despite —\n${detail}\n a rollout landing on one in progress can disrupt it; in-flight runs hand off regardless (run-history item 39)${warningText}`,
215
213
  };
216
214
  }
217
215
  return {
@@ -219,7 +217,7 @@ export function decide({ health, apps }, { force = false } = {}) {
219
217
  forced: false,
220
218
  problems,
221
219
  warnings,
222
- message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait for them to finish and retry; ${HOW_TO_FORCE}${warningText}`,
220
+ message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait and retry; ${HOW_TO_FORCE}${warningText}`,
223
221
  };
224
222
  }
225
223
 
@@ -197,8 +197,10 @@ export class SwitchboardServer extends Container<Env> {
197
197
  * and exits, and the NEXT request through `fetch` starts the container again
198
198
  * (`startBot`, env computed then). The keep-alive cron GETs /healthz every
199
199
  * minute and the CLI's live gate polls it every 15 s, so the next request is
200
- * never more than seconds away. Refuses (the deploy preflight's rules) while
201
- * runs are in flight or a drain is already under way unless `force`.
200
+ * never more than seconds away. Refuses (the deploy preflight's rules) only the
201
+ * fail-closed cases — no JSON body, an impossible `inFlight` — unless
202
+ * `force`; runs in flight or a drain under way warn and the stop proceeds
203
+ * (the handoff rule).
202
204
  */
203
205
  async restart(opts: { force: boolean }): Promise<RestartOutcome> {
204
206
  if (!this.ctx.container?.running) return { kind: "not-running" };
@@ -265,7 +267,8 @@ export class SwitchboardServer extends Container<Env> {
265
267
  * `deploy:write` in the bot's config). The Worker authenticates the bearer
266
268
  * against the map it holds — an unknown bearer never touches the container —
267
269
  * and the Container DO asks the bot for the grant before stopping anything.
268
- * Body `{ "force": true }` bypasses the in-flight/draining refusal. */
270
+ * Body `{ "force": true }` bypasses the fail-closed refusals (no JSON body,
271
+ * impossible `inFlight`); runs in flight or a drain warn and never refuse. */
269
272
  async function handleAdminRestart(request: Request, env: Env): Promise<Response> {
270
273
  const json = (status: number, body: Record<string, unknown>) =>
271
274
  new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
@@ -7,6 +7,7 @@ import {
7
7
  planEviction,
8
8
  planWrite,
9
9
  rankRecords,
10
+ rejectionMarkers,
10
11
  } from "../../src/core/memory/engine.ts";
11
12
  import { tokenize } from "../../src/core/memory/scorer.ts";
12
13
  import { FIRING_DETAIL_MAX, isScheduleFiring, type ScheduleFiring } from "../../src/core/schedules.ts";
@@ -148,6 +149,7 @@ const traceSinks = [workerLogSink((line) => console.log(line))];
148
149
  // Route surface (JSON in/out; bearer MEMORY_TOKEN on everything but /healthz):
149
150
  // POST /retrieve {scopeKey, query, limit} → {records: MemoryRecord[]}
150
151
  // POST /write {scopeKey, records: MemoryCandidate[]} → {ok, inserted, deduped, superseded}
152
+ // POST /sweep {scopeKey, dryRun?} → {ok, swept} (+ ids under dryRun — the marked rows, flipped to `swept`)
151
153
  // GET /healthz → {ok:true} (deploy wake ping; touches no DO)
152
154
  // Scheduled-firing routes (the record behind the /runs Scheduled panel;
153
155
  // written by the bot's Worker shim after every cron firing, read by the bot's
@@ -463,14 +465,19 @@ export class MemoryDO extends DurableObject<Env> {
463
465
  return counts;
464
466
  }
465
467
 
466
- /** Human view (docs/reference/specs/memory.md item 24): the scope's ACTIVE rows, newest first, no usage
467
- * bump. With `query`, only rows an FTS token hits (the same quoted-OR MATCH
468
- * as retrieve, so user text never reaches the FTS parser as syntax); a
469
- * query with no tokens lists nothing. */
470
- async list(_scopeKey: string, limit: number, query?: string): Promise<MemoryRecord[]> {
468
+ /** Human view and the repository window's read (docs/reference/specs/memory.md items 22, 26): the
469
+ * scope's ACTIVE rows, newest first, no usage bump. With `query`, only rows
470
+ * an FTS token hits (the same quoted-OR MATCH as retrieve, so user text
471
+ * never reaches the FTS parser as syntax); a query with no tokens lists
472
+ * nothing. With `kind`, only rows of that kind (the window lists facts). */
473
+ async list(_scopeKey: string, limit: number, query?: string, kind?: MemoryRecord["kind"]): Promise<MemoryRecord[]> {
471
474
  if (query === undefined) {
472
475
  return this.sql
473
- .exec<Row>(`SELECT * FROM records WHERE status = 'active' ORDER BY seq DESC LIMIT ?`, limit)
476
+ .exec<Row>(
477
+ `SELECT * FROM records WHERE status = 'active'${kind === undefined ? "" : " AND kind = ?"} ORDER BY seq DESC LIMIT ?`,
478
+ ...(kind === undefined ? [] : [kind]),
479
+ limit,
480
+ )
474
481
  .toArray()
475
482
  .map(toRecord);
476
483
  }
@@ -480,9 +487,10 @@ export class MemoryDO extends DurableObject<Env> {
480
487
  .exec<Row>(
481
488
  `SELECT r.* FROM records r
482
489
  JOIN records_fts f ON f.id = r.id
483
- WHERE r.status = 'active' AND records_fts MATCH ?
490
+ WHERE r.status = 'active'${kind === undefined ? "" : " AND r.kind = ?"} AND records_fts MATCH ?
484
491
  ORDER BY r.seq DESC
485
492
  LIMIT ?`,
493
+ ...(kind === undefined ? [] : [kind]),
486
494
  match,
487
495
  limit,
488
496
  )
@@ -504,6 +512,33 @@ export class MemoryDO extends DurableObject<Env> {
504
512
  return flipped;
505
513
  });
506
514
  }
515
+
516
+ /** Human control (docs/reference/specs/memory.md item 27): retire every
517
+ * ACTIVE fact whose text the write gate would reject today — the same
518
+ * `rejectionMarkers` the bot's write path runs, imported from the shared
519
+ * engine, so the sweep and the gate can never disagree. Summaries are never
520
+ * gated, so never swept. The scan, the flips and the FTS deletes run in ONE
521
+ * sync transaction (the per-scope cap's atomicity rule): the sweep commits
522
+ * whole or not at all. Idempotent — swept rows are no longer active, so a
523
+ * second call answers 0. Under `dryRun` nothing flips. */
524
+ async sweep(_scopeKey: string, dryRun: boolean): Promise<{ swept: number; ids: string[] }> {
525
+ return this.ctx.storage.transactionSync(() => {
526
+ const ids = this.sql
527
+ .exec<Row>(`SELECT * FROM records WHERE status = 'active' AND kind = 'fact'`)
528
+ .toArray()
529
+ .filter((row) => rejectionMarkers(row.text).length > 0)
530
+ .map((row) => row.id);
531
+ if (!dryRun) {
532
+ for (const id of ids) {
533
+ // Soft delete for the record row, hard delete for its FTS entry —
534
+ // the forget/supersede/evict hygiene rule (§15).
535
+ this.sql.exec(`UPDATE records SET status = 'swept' WHERE id = ? AND status = 'active'`, id);
536
+ this.sql.exec(`DELETE FROM records_fts WHERE id = ?`, id);
537
+ }
538
+ }
539
+ return { swept: ids.length, ids };
540
+ });
541
+ }
507
542
  }
508
543
 
509
544
  /** Build the FTS5 MATCH expression for a query: each engine token (`[a-z0-9]+`
@@ -3121,8 +3156,10 @@ function parseLimit(v: unknown): Validated<number> {
3121
3156
  return { ok: true, value: v };
3122
3157
  }
3123
3158
 
3124
- /** `POST /list {scopeKey, limit, query?}`. */
3125
- function parseList(body: unknown): Validated<{ scopeKey: string; limit: number; query?: string }> {
3159
+ /** `POST /list {scopeKey, limit, query?, kind?}`. */
3160
+ function parseList(
3161
+ body: unknown,
3162
+ ): Validated<{ scopeKey: string; limit: number; query?: string; kind?: MemoryRecord["kind"] }> {
3126
3163
  if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
3127
3164
  const b = body as Record<string, unknown>;
3128
3165
  const scope = parseScopeKey(b.scopeKey);
@@ -3133,12 +3170,29 @@ function parseList(body: unknown): Validated<{ scopeKey: string; limit: number;
3133
3170
  if (typeof b.query !== "string") return invalid("query must be a string");
3134
3171
  if (b.query.length > MAX_QUERY_CHARS) return invalid(`query must be at most ${MAX_QUERY_CHARS} characters`);
3135
3172
  }
3173
+ if (b.kind !== undefined && b.kind !== "fact" && b.kind !== "summary")
3174
+ return invalid('kind must be "fact" or "summary"');
3136
3175
  return {
3137
3176
  ok: true,
3138
- value: { scopeKey: scope.value, limit: limit.value, ...(typeof b.query === "string" ? { query: b.query } : {}) },
3177
+ value: {
3178
+ scopeKey: scope.value,
3179
+ limit: limit.value,
3180
+ ...(typeof b.query === "string" ? { query: b.query } : {}),
3181
+ ...(b.kind === "fact" || b.kind === "summary" ? { kind: b.kind } : {}),
3182
+ },
3139
3183
  };
3140
3184
  }
3141
3185
 
3186
+ /** `POST /sweep {scopeKey, dryRun?}`. */
3187
+ function parseSweep(body: unknown): Validated<{ scopeKey: string; dryRun: boolean }> {
3188
+ if (!isJsonObject(body)) return invalid("body must be a JSON object");
3189
+ const b = body;
3190
+ const scope = parseScopeKey(b.scopeKey);
3191
+ if (!scope.ok) return scope;
3192
+ if (b.dryRun !== undefined && typeof b.dryRun !== "boolean") return invalid("dryRun must be a boolean");
3193
+ return { ok: true, value: { scopeKey: scope.value, dryRun: b.dryRun === true } };
3194
+ }
3195
+
3142
3196
  /** `POST /forget {scopeKey, id}`: the id is an opaque key, same caps as scopeKey. */
3143
3197
  function parseForget(body: unknown): Validated<{ scopeKey: string; id: string }> {
3144
3198
  if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
@@ -4329,6 +4383,7 @@ const ROUTES = new Set([
4329
4383
  "/write",
4330
4384
  "/list",
4331
4385
  "/forget",
4386
+ "/sweep",
4332
4387
  "/schedules/record",
4333
4388
  "/schedules/latest",
4334
4389
  "/runs/put",
@@ -4422,8 +4477,8 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
4422
4477
  if (url.pathname === "/list") {
4423
4478
  const parsed = parseList(body);
4424
4479
  if (!parsed.ok) return json({ error: parsed.error }, 400);
4425
- const { scopeKey, limit, query } = parsed.value;
4426
- const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query);
4480
+ const { scopeKey, limit, query, kind } = parsed.value;
4481
+ const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query, kind);
4427
4482
  console.log(`[list] ${scopeKey} -> ${records.length} records`);
4428
4483
  return json({ records });
4429
4484
  }
@@ -4436,6 +4491,16 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
4436
4491
  console.log(`[forget] ${scopeKey} ${id} -> ${forgotten}`);
4437
4492
  return json({ ok: true, forgotten });
4438
4493
  }
4494
+ if (url.pathname === "/sweep") {
4495
+ const parsed = parseSweep(body);
4496
+ if (!parsed.ok) return json({ error: parsed.error }, 400);
4497
+ const { scopeKey, dryRun } = parsed.value;
4498
+ const out = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).sweep(scopeKey, dryRun);
4499
+ // Observability: scope + count only (ids carry no record text; they ride
4500
+ // the dryRun answer, not the log).
4501
+ console.log(`[sweep] ${scopeKey} -> ${out.swept}${dryRun ? " (dry run)" : ""}`);
4502
+ return json({ ok: true, swept: out.swept, ...(dryRun ? { ids: out.ids } : {}) });
4503
+ }
4439
4504
 
4440
4505
  const parsed = parseWrite(body);
4441
4506
  if (!parsed.ok) return json({ error: parsed.error }, 400);
@@ -0,0 +1,212 @@
1
+ // The resident guards its own memory (docs/reference/specs/resident-repos.md
2
+ // item 70) — the PURE half, kept free of the Sandbox SDK and DO storage so it
3
+ // runs under plain-Node vitest (memoryGuard.test.ts) like gc.ts and drain.ts.
4
+ // The Worker owns the one real reader (a single exec of CGROUP_READ_ARGV via
5
+ // its exec choke point) and feeds this module the raw output; tests feed a
6
+ // fake. A container at its cgroup memory cap wedges its own control port and
7
+ // every run on it loses its work, so the resident refuses NEW work by name
8
+ // while it still can: above the soft threshold a new attach is refused like
9
+ // `mirror-busy` (the bot falls back cold or waits, the card says why), above
10
+ // the hard threshold a new exec is refused with the numbers — and a command
11
+ // already running is never touched: the gate sits at each route's start and
12
+ // kills nothing.
13
+ import { MEMORY_PRESSURE_REASON } from "../../src/execution/sandboxErrors.js";
14
+
15
+ export { MEMORY_PRESSURE_REASON };
16
+
17
+ /** Above this percent of the cgroup memory cap a NEW `/attach` is refused —
18
+ * new runs go elsewhere while the ones already here finish. */
19
+ export const MEMORY_SOFT_LIMIT_PCT = 80;
20
+
21
+ /** Above this percent a NEW `/exec` is refused too — the resident answers
22
+ * nothing but the work already in flight, which always runs to completion. */
23
+ export const MEMORY_HARD_LIMIT_PCT = 90;
24
+
25
+ /** One reading of the container's cgroup v2 accounting. */
26
+ export interface MemoryReading {
27
+ /** ISO timestamp the caller supplied (the resident's clock seam). */
28
+ at: string;
29
+ /** memory.current — bytes charged to the cgroup now. */
30
+ usedBytes: number;
31
+ /** memory.max — the cap in bytes, or null when the file reads `max` (no cap). */
32
+ capBytes: number | null;
33
+ /** used/cap rounded to whole percent; null without a cap (nothing to gate on). */
34
+ percent: number | null;
35
+ /** cpu.stat's usage_usec, for the same log line; null when absent. */
36
+ cpuUsageUsec: number | null;
37
+ }
38
+
39
+ /** The reader seam: one call returns the raw output of `CGROUP_READ_ARGV`.
40
+ * A throw of `MemorySampleUnavailable` means "no reading this time" (the
41
+ * container busy or replaced under the probe) and never disables the gate;
42
+ * any other throw is an unreadable cgroup and disables it (one log line). */
43
+ export interface CgroupReader {
44
+ read(): Promise<string>;
45
+ }
46
+
47
+ /** The one transient escape: the reader could not run at all right now.
48
+ * `invalidates` says the container the last reading came from is gone (a
49
+ * runtime replacement under the probe), so the guard forgets that reading —
50
+ * a dead container's numbers must not gate the work that replaces it. */
51
+ export class MemorySampleUnavailable extends Error {
52
+ constructor(
53
+ message: string,
54
+ readonly invalidates = false,
55
+ ) {
56
+ super(message);
57
+ }
58
+ }
59
+
60
+ /** The separator `CGROUP_READ_ARGV` prints between the three files. */
61
+ export const CGROUP_FILE_SEPARATOR = ":::";
62
+
63
+ /** One exec, three files: memory.current, memory.max, cpu.stat from the
64
+ * container's own cgroup v2 root. `&&` so a missing file is a non-zero exit
65
+ * (an unreadable cgroup), never a silently short output. */
66
+ export const CGROUP_READ_ARGV: readonly string[] = [
67
+ "sh",
68
+ "-c",
69
+ `cat /sys/fs/cgroup/memory.current && echo ${CGROUP_FILE_SEPARATOR} && ` +
70
+ `cat /sys/fs/cgroup/memory.max && echo ${CGROUP_FILE_SEPARATOR} && cat /sys/fs/cgroup/cpu.stat`,
71
+ ];
72
+
73
+ /** Parses one reader output into a reading; throws on anything malformed. */
74
+ export function parseCgroupOutput(raw: string, at: string): MemoryReading {
75
+ const parts = raw.split(CGROUP_FILE_SEPARATOR).map((p) => p.trim());
76
+ if (parts.length !== 3) throw new Error(`expected 3 cgroup sections, got ${parts.length}`);
77
+ const usedBytes = Number(parts[0]);
78
+ if (!Number.isFinite(usedBytes) || usedBytes < 0) throw new Error(`memory.current is not a byte count: ${parts[0]}`);
79
+ let capBytes: number | null = null;
80
+ if (parts[1] !== "max") {
81
+ capBytes = Number(parts[1]);
82
+ if (!Number.isFinite(capBytes) || capBytes <= 0) throw new Error(`memory.max is not a byte count: ${parts[1]}`);
83
+ }
84
+ const usage = /(?:^|\n)usage_usec (\d+)/.exec(parts[2]);
85
+ const cpuUsageUsec = usage ? Number(usage[1]) : null;
86
+ const percent = capBytes === null ? null : Math.round((usedBytes / capBytes) * 100);
87
+ return { at, usedBytes, capBytes, percent, cpuUsageUsec };
88
+ }
89
+
90
+ /** The one structured line Workers Logs gets per sample: used bytes, cap
91
+ * bytes, percent (and the cpu counter beside them). */
92
+ export function memoryLogLine(r: MemoryReading): string {
93
+ const cap = r.capBytes === null ? "uncapped" : `${r.capBytes} bytes cap`;
94
+ const pct = r.percent === null ? "" : ` (${r.percent}%)`;
95
+ const cpu = r.cpuUsageUsec === null ? "" : ` — cpu usage_usec ${r.cpuUsageUsec}`;
96
+ return `memory: used ${r.usedBytes} bytes of ${cap}${pct}${cpu}`;
97
+ }
98
+
99
+ /** Which route asks the gate. */
100
+ export type MemoryGateRoute = "attach" | "exec";
101
+
102
+ /** A refusal with the numbers on it — the message is the whole story. */
103
+ export interface MemoryRefusal {
104
+ reason: typeof MEMORY_PRESSURE_REASON;
105
+ percent: number;
106
+ usedBytes: number;
107
+ capBytes: number;
108
+ message: string;
109
+ }
110
+
111
+ /** The pure gate over one reading: below both thresholds nothing changes; at
112
+ * or above the soft one a new attach is refused (queued — the caller waits or
113
+ * falls back, like `mirror-busy`); at or above the hard one a new exec is
114
+ * refused too. No reading, or no cap, gates nothing. */
115
+ export function gateMemory(route: MemoryGateRoute, reading: MemoryReading | null): MemoryRefusal | null {
116
+ if (!reading || reading.percent === null || reading.capBytes === null) return null;
117
+ const { percent, usedBytes, capBytes } = reading;
118
+ const refusal = (message: string): MemoryRefusal => ({
119
+ reason: MEMORY_PRESSURE_REASON,
120
+ percent,
121
+ usedBytes,
122
+ capBytes,
123
+ message,
124
+ });
125
+ if (route === "exec" && percent >= MEMORY_HARD_LIMIT_PCT) {
126
+ return refusal(
127
+ `${MEMORY_PRESSURE_REASON}: resident at ${percent}% of its memory cap (${usedBytes} of ${capBytes} bytes) — ` +
128
+ `a new command is refused above ${MEMORY_HARD_LIMIT_PCT}% while the commands already running finish`,
129
+ );
130
+ }
131
+ if (route === "attach" && percent >= MEMORY_SOFT_LIMIT_PCT) {
132
+ return refusal(
133
+ `${MEMORY_PRESSURE_REASON}: resident near its memory cap, ${percent}% used (${usedBytes} of ${capBytes} bytes), ` +
134
+ `queued — a new attach is refused above ${MEMORY_SOFT_LIMIT_PCT}% until the runs here release memory`,
135
+ );
136
+ }
137
+ return null;
138
+ }
139
+
140
+ /** The message of whatever was thrown (local: this module stays dependency-free). */
141
+ const causeText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
142
+
143
+ /** The guard one resident holds: samples through the reader, keeps the last
144
+ * reading, gates the two routes — and on an unreadable cgroup logs ONCE and
145
+ * disables itself for the incarnation instead of refusing everything. */
146
+ export class MemoryGuard {
147
+ private disabledWhy: string | null = null;
148
+ private last: MemoryReading | null = null;
149
+
150
+ constructor(
151
+ private readonly reader: CgroupReader,
152
+ private readonly log: (line: string) => void,
153
+ ) {}
154
+
155
+ /** The last reading taken this incarnation, for `/status` and `/residents`. */
156
+ get lastReading(): MemoryReading | null {
157
+ return this.last;
158
+ }
159
+
160
+ /** Whether an unreadable cgroup turned the gate off (the why is logged once). */
161
+ get disabled(): boolean {
162
+ return this.disabledWhy !== null;
163
+ }
164
+
165
+ /** Forgets the last reading: the container it measured no longer runs (it
166
+ * went inactive, or was replaced under the probe), so nothing gates on it
167
+ * until a fresh sample lands. The gate's disabled state is untouched. */
168
+ invalidate(): void {
169
+ this.last = null;
170
+ }
171
+
172
+ /** One reader call, one log line. A `MemorySampleUnavailable` keeps the last
173
+ * reading and the gate as they were — unless it `invalidates` (the runtime
174
+ * was replaced under the probe), which forgets the reading so a stale one
175
+ * never masks the route's own runtime-replaced answer; any other failure
176
+ * disables the gate. */
177
+ async sample(at: string): Promise<MemoryReading | null> {
178
+ if (this.disabledWhy !== null) return null;
179
+ let raw: string;
180
+ try {
181
+ raw = await this.reader.read();
182
+ } catch (err) {
183
+ if (err instanceof MemorySampleUnavailable) {
184
+ if (err.invalidates) this.last = null;
185
+ return null;
186
+ }
187
+ this.disable(causeText(err));
188
+ return null;
189
+ }
190
+ let reading: MemoryReading;
191
+ try {
192
+ reading = parseCgroupOutput(raw, at);
193
+ } catch (err) {
194
+ this.disable(causeText(err));
195
+ return null;
196
+ }
197
+ this.last = reading;
198
+ this.log(memoryLogLine(reading));
199
+ return reading;
200
+ }
201
+
202
+ /** The route's verdict over the last reading; a disabled gate refuses nothing. */
203
+ gate(route: MemoryGateRoute): MemoryRefusal | null {
204
+ if (this.disabledWhy !== null) return null;
205
+ return gateMemory(route, this.last);
206
+ }
207
+
208
+ private disable(why: string): void {
209
+ this.disabledWhy = why;
210
+ this.log(`memory: cgroup unreadable — the memory gate is disabled for this incarnation (${why})`);
211
+ }
212
+ }
@@ -58,7 +58,7 @@ export interface RefreshInstanceAction {
58
58
  * (`stepTimeoutMs`), so a step timeout and a command timeout agree. */
59
59
  const REFRESH_FETCH_STEP_BUDGET_MS = RESTORE_MAX_MS + GIT_NETWORK_TIMEOUT_MS; // a wake's restore, then the fetch
60
60
  const REFRESH_INSTALL_STEP_BUDGET_MS = REFRESH_INSTALL_TIMEOUT_MS + DEPS_STEP_OVERHEAD_MS; // the install's own lease
61
- const REFRESH_BUILD_STEP_BUDGET_MS = GIT_NETWORK_TIMEOUT_MS + REFRESH_BUILD_TIMEOUT_MS; // the build's mutex lease
61
+ const REFRESH_BUILD_STEP_BUDGET_MS = 5 * GIT_NETWORK_TIMEOUT_MS + 2 * REFRESH_BUILD_TIMEOUT_MS; // five git-budgeted commands — stage-clear, stage, the staging fetch (under the stage lock), the checkout update and the swap (under the swap lock) — plus the off-lock reset/clean, deps re-link and build
62
62
  const REFRESH_SNAPSHOT_STEP_BUDGET_MS = R2_TRANSFER_TIMEOUT_MS + GIT_NETWORK_TIMEOUT_MS; // the archives, then the reclaim pass
63
63
  /** The sweep: one cleanliness check per binding idle past an hour (at most the
64
64
  * pool's worth, one exec budget each), then the removals under the mirror lock. */