@coreplane/switchboard 1.250.0 → 1.252.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/config/config.example.yaml +13 -2
- package/dist/assets/deploy/cloudflare/preflight.mjs +19 -21
- package/dist/assets/deploy/cloudflare/worker.ts +6 -3
- package/dist/assets/deploy/cloudflare-memory/worker.ts +108 -12
- package/dist/assets/deploy/cloudflare-resident/memoryGuard.ts +212 -0
- package/dist/assets/deploy/cloudflare-resident/refresh.ts +1 -1
- package/dist/assets/deploy/cloudflare-resident/worker.ts +317 -56
- package/dist/assets/deploy/cloudflare-sandbox/Dockerfile +4 -2
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.d.mts +31 -0
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.mjs +119 -0
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +3 -2
- package/dist/assets/project.json +13 -9
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +5 -5
- package/dist/assets/src/core/authz/policy.ts +4 -0
- package/dist/assets/src/core/authz/resource.ts +6 -2
- package/dist/assets/src/core/authz/types.ts +2 -0
- package/dist/assets/src/core/budgets.ts +22 -0
- package/dist/assets/src/core/coordinator/contract.ts +42 -0
- package/dist/assets/src/core/coordinator/driver.ts +150 -12
- package/dist/assets/src/core/drain.ts +50 -0
- package/dist/assets/src/core/memory/engine.ts +98 -0
- package/dist/assets/src/core/memory/scorer.ts +12 -4
- package/dist/assets/src/core/memory/types.ts +69 -12
- package/dist/assets/src/core/modelCard.ts +51 -7
- package/dist/assets/src/core/modelPricing.ts +111 -1
- package/dist/assets/src/core/modelProxy/usage.ts +88 -0
- package/dist/assets/src/core/modelRegistry.ts +15 -1
- package/dist/assets/src/core/provider.ts +49 -0
- package/dist/assets/src/core/refusal.ts +6 -6
- package/dist/assets/src/core/reviewVerdict.ts +4 -0
- package/dist/assets/src/core/runEvents.ts +51 -2
- package/dist/assets/src/core/runFriction.ts +7 -2
- package/dist/assets/src/core/runLedger/types.ts +23 -4
- package/dist/assets/src/core/runRecord.ts +16 -0
- package/dist/assets/src/core/runUsage.ts +67 -13
- package/dist/assets/src/core/schedules.ts +3 -0
- package/dist/assets/src/core/ship/contract.ts +45 -5
- package/dist/assets/src/core/ship/coordinator.ts +441 -48
- package/dist/assets/src/core/ship/renewal.ts +10 -5
- package/dist/assets/src/core/trace/attrs.ts +24 -0
- package/dist/assets/src/core/types.ts +5 -5
- package/dist/assets/src/core/verbosity.ts +48 -0
- package/dist/assets/src/deploy/liveGate.ts +40 -13
- package/dist/assets/src/deploy/restart.ts +11 -12
- package/dist/assets/src/execution/residentDepCache.ts +50 -1
- package/dist/assets/src/execution/residentDepsStore.ts +40 -2
- package/dist/assets/src/execution/residentRefresh.ts +55 -3
- package/dist/assets/src/execution/residentSteps.ts +4 -0
- package/dist/assets/src/execution/sandboxErrors.ts +8 -0
- package/dist/assets/web/dist/.vite/manifest.json +58 -52
- package/dist/assets/web/dist/assets/DeliveryPage-3ELQWM0r.js +1 -0
- package/dist/assets/web/dist/assets/{HomePage-AnycA57D.js → HomePage-BG_ok-K2.js} +2 -2
- package/dist/assets/web/dist/assets/{PendingTurnRow-BuRre8it.js → PendingTurnRow-ChCQOLgZ.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentDetailPage-Cb3sFkfj.js → ResidentDetailPage-C9y3nbo8.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-BZ6n6UxF.js → ResidentsIndexPage-i1RG9e7g.js} +1 -1
- package/dist/assets/web/dist/assets/RunFoldRow-D3wVpzBa.js +1 -0
- package/dist/assets/web/dist/assets/RunRoutePage-B3IirUVi.js +9 -0
- package/dist/assets/web/dist/assets/RunsIndexPage-DiFmtGaJ.js +1 -0
- package/dist/assets/web/dist/assets/{ScheduledPage-CBUxbeqN.js → ScheduledPage-DvYwM2TE.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-CBTnZ9Qv.js → SettingsPage-Bo6yCyXZ.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-BnRjWzFN.js → StatusDot-CAfS1AUi.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-Brge0wnd.js → Tooltip-tZoum_T-.js} +1 -1
- package/dist/assets/web/dist/assets/{UnitRoutePage-o6sLju16.js → UnitRoutePage-BmdOHwNn.js} +1 -1
- package/dist/assets/web/dist/assets/budgets-CbIyPAER.js +1 -0
- package/dist/assets/web/dist/assets/{dist-rgAhsmE-.js → dist-DfbEpHXR.js} +1 -1
- package/dist/assets/web/dist/assets/indexRow-BT0cPVRw.js +1 -0
- package/dist/assets/web/dist/assets/{main-CeRuGONy.js → main-5Gm_1Gv8.js} +2 -2
- package/dist/assets/web/dist/assets/sseReplay-DmyMXfRC.js +11 -0
- package/dist/cli.js +3319 -1015
- package/package.json +1 -1
- package/dist/assets/deploy/cloudflare-sandbox/runtime-supervisor.sh +0 -68
- package/dist/assets/web/dist/assets/DeliveryPage-CIfBiINK.js +0 -1
- package/dist/assets/web/dist/assets/RunFoldRow-BRXkjkgO.js +0 -1
- package/dist/assets/web/dist/assets/RunRoutePage-psSMI3fN.js +0 -9
- package/dist/assets/web/dist/assets/RunsIndexPage-68YT_RWt.js +0 -1
- package/dist/assets/web/dist/assets/indexRow-BmK74Vp1.js +0 -1
- package/dist/assets/web/dist/assets/sseReplay-DXC7kGbN.js +0 -9
|
@@ -87,6 +87,14 @@ defaults:
|
|
|
87
87
|
# provider default. Skipped for models without effort support.
|
|
88
88
|
# efforts:
|
|
89
89
|
# coding: medium
|
|
90
|
+
# How much the bot says about its own doing (docs/reference/specs/routing-and-config.md
|
|
91
|
+
# item 28): quiet (default) — only what needs the person: answers, verdicts,
|
|
92
|
+
# refusals, questions, results, the card's progress; verbose — plus every
|
|
93
|
+
# acknowledgement (a follow-up folded into a live run, a plan handed to the
|
|
94
|
+
# runner, the workspace and budget on the card); debug — plus the router's
|
|
95
|
+
# reason and the ledger's word. Same layering as effort: per-channel/user
|
|
96
|
+
# `verbosity` and a per-request `verbosity:<level>` directive override it.
|
|
97
|
+
# verbosity: quiet
|
|
90
98
|
# A boundary caps what ANY run in a scope may have — never grants — on the
|
|
91
99
|
# three axes of a run's profile (docs/reference/specs/routing-and-config.md item 2):
|
|
92
100
|
# maxMinutes (the wall-clock budget; at least 2), maxIdentity (the credential
|
|
@@ -156,6 +164,7 @@ defaults:
|
|
|
156
164
|
# users:
|
|
157
165
|
# slack:U012345:
|
|
158
166
|
# model: openai/gpt-5
|
|
167
|
+
# verbosity: verbose # hear every acknowledgement of what the bot does for you
|
|
159
168
|
# harness:
|
|
160
169
|
# coding: opencode
|
|
161
170
|
|
|
@@ -316,8 +325,10 @@ workspaceDir: ./workspaces
|
|
|
316
325
|
# # and — when the run is bound to a repo — that repo's scope (repo:owner/name)
|
|
317
326
|
# # plus the requesting user's own scope (user:slack:U…) — a
|
|
318
327
|
# # person's records never surface for anyone else.
|
|
319
|
-
# limit:
|
|
320
|
-
# maxTokens:
|
|
328
|
+
# limit: 32 # max records retrieved/injected per request (default 32)
|
|
329
|
+
# maxTokens: 3000 # hard token budget for the injected block (default 3000)
|
|
330
|
+
# repoWindow: 24 # newest repo facts injected ahead of the keyword hits when
|
|
331
|
+
# # the run is bound to a repo (default 24; 0 disables the window)
|
|
321
332
|
# maxRecordsPerScope: 500
|
|
322
333
|
# # per-scope cap on ACTIVE records (default 500): a write
|
|
323
334
|
# # that would exceed it evicts the least recently used
|
|
@@ -1,23 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Deploy preflight for the bot Worker (docs/reference/specs/slack-channel.md item 8).
|
|
3
3
|
//
|
|
4
|
-
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// next generation
|
|
8
|
-
//
|
|
9
|
-
// `npm run deploy` runs
|
|
10
|
-
// - the bot reports runs in flight (`GET /healthz` `inFlight > 0`): the
|
|
11
|
-
// runner retries every minute up to its budget, then fails by name;
|
|
4
|
+
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM;
|
|
5
|
+
// since the run ledger's handoff (docs/reference/specs/run-history.md item 39) the bot
|
|
6
|
+
// hands every resumable run to the next generation and exits within seconds,
|
|
7
|
+
// and the next generation continues the runs under their own cards — so a
|
|
8
|
+
// deploy no longer waits on runs, and this preflight no longer refuses for
|
|
9
|
+
// them. `npm run deploy` runs it first and refuses only while
|
|
12
10
|
// - the container application is not in a settled state (a rollout is still
|
|
13
11
|
// provisioning/updating — `wrangler containers list --json`): a second
|
|
14
12
|
// rollout on top of one in progress replaces the instance the first put
|
|
15
13
|
// into its graceful drain and kills whatever it was running (two deploys
|
|
16
14
|
// 90 s apart once killed a review at 153 s).
|
|
17
|
-
// It WARNS (never refuses) when the bot
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
15
|
+
// It WARNS (never refuses) when the bot reports runs in flight (`inFlight > 0`
|
|
16
|
+
// — they hand off) or is already draining (`draining: true` — its resumable
|
|
17
|
+
// runs were handed off; a ship pipeline still in flight would be killed), and
|
|
18
|
+
// when /healthz says the reconnect catch-up is failing or the bot token lacks
|
|
19
|
+
// required scopes (`catchUp.error`, `catchUp.missingScopes`).
|
|
21
20
|
//
|
|
22
21
|
// Fail closed: unreachable bot, a body without the JSON shape (a bot whose
|
|
23
22
|
// /healthz answers a bare `ok` is not one this preflight can read), a wrangler failure, or an app
|
|
@@ -41,7 +40,7 @@ export const APP_NAME = "switchboard-switchboardserver";
|
|
|
41
40
|
const SETTLED_APP_STATES = new Set(["active", "ready"]);
|
|
42
41
|
|
|
43
42
|
const HOW_TO_FORCE =
|
|
44
|
-
"to deploy anyway (over
|
|
43
|
+
"to deploy anyway (over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
|
|
45
44
|
|
|
46
45
|
/** GET /healthz. Never throws: `{ok:true,payload}` (parsed JSON, or the raw text when not JSON) or `{ok:false,error}`. */
|
|
47
46
|
export async function fetchHealth(baseUrl, { timeoutMs = 20_000 } = {}) {
|
|
@@ -165,16 +164,15 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
165
164
|
if (!Number.isInteger(p.inFlight) || p.inFlight < 0) {
|
|
166
165
|
problems.push(`bot reports an impossible inFlight=${JSON.stringify(p.inFlight)} (counter bug or old Worker)`);
|
|
167
166
|
} else if (p.inFlight > 0) {
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
`${p.inFlight} run(s) in flight — the rollout would roll the bot container under them (a handoff is a recovery, not a guarantee)`,
|
|
167
|
+
// Not a refusal since the handoff (run-history item 39): SIGTERM hands
|
|
168
|
+
// every resumable run to the next generation, which continues it.
|
|
169
|
+
warnings.push(
|
|
170
|
+
`${p.inFlight} run(s) in flight — handed to the next generation on SIGTERM (run-history item 39); they continue there under their own cards`,
|
|
173
171
|
);
|
|
174
172
|
}
|
|
175
173
|
if (p.draining === true) {
|
|
176
174
|
warnings.push(
|
|
177
|
-
"bot is already draining from a previous deploy —
|
|
175
|
+
"bot is already draining from a previous deploy — its resumable runs are handed off; a ship pipeline still in flight would be killed when this rollout replaces the draining instance",
|
|
178
176
|
);
|
|
179
177
|
}
|
|
180
178
|
}
|
|
@@ -211,7 +209,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
211
209
|
forced: true,
|
|
212
210
|
problems,
|
|
213
211
|
warnings,
|
|
214
|
-
message: `preflight WARNING: deploying by force despite —\n${detail}\n
|
|
212
|
+
message: `preflight WARNING: deploying by force despite —\n${detail}\n a rollout landing on one in progress can disrupt it; in-flight runs hand off regardless (run-history item 39)${warningText}`,
|
|
215
213
|
};
|
|
216
214
|
}
|
|
217
215
|
return {
|
|
@@ -219,7 +217,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
219
217
|
forced: false,
|
|
220
218
|
problems,
|
|
221
219
|
warnings,
|
|
222
|
-
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait
|
|
220
|
+
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait and retry; ${HOW_TO_FORCE}${warningText}`,
|
|
223
221
|
};
|
|
224
222
|
}
|
|
225
223
|
|
|
@@ -197,8 +197,10 @@ export class SwitchboardServer extends Container<Env> {
|
|
|
197
197
|
* and exits, and the NEXT request through `fetch` starts the container again
|
|
198
198
|
* (`startBot`, env computed then). The keep-alive cron GETs /healthz every
|
|
199
199
|
* minute and the CLI's live gate polls it every 15 s, so the next request is
|
|
200
|
-
* never more than seconds away. Refuses (the deploy preflight's rules)
|
|
201
|
-
*
|
|
200
|
+
* never more than seconds away. Refuses (the deploy preflight's rules) only the
|
|
201
|
+
* fail-closed cases — no JSON body, an impossible `inFlight` — unless
|
|
202
|
+
* `force`; runs in flight or a drain under way warn and the stop proceeds
|
|
203
|
+
* (the handoff rule).
|
|
202
204
|
*/
|
|
203
205
|
async restart(opts: { force: boolean }): Promise<RestartOutcome> {
|
|
204
206
|
if (!this.ctx.container?.running) return { kind: "not-running" };
|
|
@@ -265,7 +267,8 @@ export class SwitchboardServer extends Container<Env> {
|
|
|
265
267
|
* `deploy:write` in the bot's config). The Worker authenticates the bearer
|
|
266
268
|
* against the map it holds — an unknown bearer never touches the container —
|
|
267
269
|
* and the Container DO asks the bot for the grant before stopping anything.
|
|
268
|
-
* Body `{ "force": true }` bypasses the
|
|
270
|
+
* Body `{ "force": true }` bypasses the fail-closed refusals (no JSON body,
|
|
271
|
+
* impossible `inFlight`); runs in flight or a drain warn and never refuse. */
|
|
269
272
|
async function handleAdminRestart(request: Request, env: Env): Promise<Response> {
|
|
270
273
|
const json = (status: number, body: Record<string, unknown>) =>
|
|
271
274
|
new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
planEviction,
|
|
8
8
|
planWrite,
|
|
9
9
|
rankRecords,
|
|
10
|
+
rejectionMarkers,
|
|
10
11
|
} from "../../src/core/memory/engine.ts";
|
|
11
12
|
import { tokenize } from "../../src/core/memory/scorer.ts";
|
|
12
13
|
import { FIRING_DETAIL_MAX, isScheduleFiring, type ScheduleFiring } from "../../src/core/schedules.ts";
|
|
@@ -148,6 +149,7 @@ const traceSinks = [workerLogSink((line) => console.log(line))];
|
|
|
148
149
|
// Route surface (JSON in/out; bearer MEMORY_TOKEN on everything but /healthz):
|
|
149
150
|
// POST /retrieve {scopeKey, query, limit} → {records: MemoryRecord[]}
|
|
150
151
|
// POST /write {scopeKey, records: MemoryCandidate[]} → {ok, inserted, deduped, superseded}
|
|
152
|
+
// POST /sweep {scopeKey, dryRun?} → {ok, swept} (+ ids under dryRun — the marked rows, flipped to `swept`)
|
|
151
153
|
// GET /healthz → {ok:true} (deploy wake ping; touches no DO)
|
|
152
154
|
// Scheduled-firing routes (the record behind the /runs Scheduled panel;
|
|
153
155
|
// written by the bot's Worker shim after every cron firing, read by the bot's
|
|
@@ -463,14 +465,19 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
463
465
|
return counts;
|
|
464
466
|
}
|
|
465
467
|
|
|
466
|
-
/** Human view (docs/reference/specs/memory.md
|
|
467
|
-
* bump. With `query`, only rows
|
|
468
|
-
*
|
|
469
|
-
* query with no tokens lists
|
|
470
|
-
|
|
468
|
+
/** Human view and the repository window's read (docs/reference/specs/memory.md items 22, 26): the
|
|
469
|
+
* scope's ACTIVE rows, newest first, no usage bump. With `query`, only rows
|
|
470
|
+
* an FTS token hits (the same quoted-OR MATCH as retrieve, so user text
|
|
471
|
+
* never reaches the FTS parser as syntax); a query with no tokens lists
|
|
472
|
+
* nothing. With `kind`, only rows of that kind (the window lists facts). */
|
|
473
|
+
async list(_scopeKey: string, limit: number, query?: string, kind?: MemoryRecord["kind"]): Promise<MemoryRecord[]> {
|
|
471
474
|
if (query === undefined) {
|
|
472
475
|
return this.sql
|
|
473
|
-
.exec<Row>(
|
|
476
|
+
.exec<Row>(
|
|
477
|
+
`SELECT * FROM records WHERE status = 'active'${kind === undefined ? "" : " AND kind = ?"} ORDER BY seq DESC LIMIT ?`,
|
|
478
|
+
...(kind === undefined ? [] : [kind]),
|
|
479
|
+
limit,
|
|
480
|
+
)
|
|
474
481
|
.toArray()
|
|
475
482
|
.map(toRecord);
|
|
476
483
|
}
|
|
@@ -480,9 +487,10 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
480
487
|
.exec<Row>(
|
|
481
488
|
`SELECT r.* FROM records r
|
|
482
489
|
JOIN records_fts f ON f.id = r.id
|
|
483
|
-
WHERE r.status = 'active' AND records_fts MATCH ?
|
|
490
|
+
WHERE r.status = 'active'${kind === undefined ? "" : " AND r.kind = ?"} AND records_fts MATCH ?
|
|
484
491
|
ORDER BY r.seq DESC
|
|
485
492
|
LIMIT ?`,
|
|
493
|
+
...(kind === undefined ? [] : [kind]),
|
|
486
494
|
match,
|
|
487
495
|
limit,
|
|
488
496
|
)
|
|
@@ -504,6 +512,33 @@ export class MemoryDO extends DurableObject<Env> {
|
|
|
504
512
|
return flipped;
|
|
505
513
|
});
|
|
506
514
|
}
|
|
515
|
+
|
|
516
|
+
/** Human control (docs/reference/specs/memory.md item 27): retire every
|
|
517
|
+
* ACTIVE fact whose text the write gate would reject today — the same
|
|
518
|
+
* `rejectionMarkers` the bot's write path runs, imported from the shared
|
|
519
|
+
* engine, so the sweep and the gate can never disagree. Summaries are never
|
|
520
|
+
* gated, so never swept. The scan, the flips and the FTS deletes run in ONE
|
|
521
|
+
* sync transaction (the per-scope cap's atomicity rule): the sweep commits
|
|
522
|
+
* whole or not at all. Idempotent — swept rows are no longer active, so a
|
|
523
|
+
* second call answers 0. Under `dryRun` nothing flips. */
|
|
524
|
+
async sweep(_scopeKey: string, dryRun: boolean): Promise<{ swept: number; ids: string[] }> {
|
|
525
|
+
return this.ctx.storage.transactionSync(() => {
|
|
526
|
+
const ids = this.sql
|
|
527
|
+
.exec<Row>(`SELECT * FROM records WHERE status = 'active' AND kind = 'fact'`)
|
|
528
|
+
.toArray()
|
|
529
|
+
.filter((row) => rejectionMarkers(row.text).length > 0)
|
|
530
|
+
.map((row) => row.id);
|
|
531
|
+
if (!dryRun) {
|
|
532
|
+
for (const id of ids) {
|
|
533
|
+
// Soft delete for the record row, hard delete for its FTS entry —
|
|
534
|
+
// the forget/supersede/evict hygiene rule (§15).
|
|
535
|
+
this.sql.exec(`UPDATE records SET status = 'swept' WHERE id = ? AND status = 'active'`, id);
|
|
536
|
+
this.sql.exec(`DELETE FROM records_fts WHERE id = ?`, id);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
return { swept: ids.length, ids };
|
|
540
|
+
});
|
|
541
|
+
}
|
|
507
542
|
}
|
|
508
543
|
|
|
509
544
|
/** Build the FTS5 MATCH expression for a query: each engine token (`[a-z0-9]+`
|
|
@@ -779,6 +814,27 @@ export class ConfigDO extends DurableObject<Env> {
|
|
|
779
814
|
});
|
|
780
815
|
}
|
|
781
816
|
|
|
817
|
+
/** The thread's pending row when one exists and is inside its ttl, else
|
|
818
|
+
* null. A pure read: expiry is checked here on this object's clock and
|
|
819
|
+
* nothing is deleted — nothing sweeps, and a consume still finds the
|
|
820
|
+
* expired row to name `expired`. */
|
|
821
|
+
async pendingConfirmationByThread(threadKey: string, now: number): Promise<ConfirmationRow | null> {
|
|
822
|
+
const row = this.sql
|
|
823
|
+
.exec<{ id: string; requester: string; expires_at: number; body: string }>(
|
|
824
|
+
`SELECT id, requester, expires_at, body FROM confirmations WHERE thread_key = ?`,
|
|
825
|
+
threadKey,
|
|
826
|
+
)
|
|
827
|
+
.toArray()[0];
|
|
828
|
+
if (!row || row.expires_at <= now) return null;
|
|
829
|
+
return {
|
|
830
|
+
id: row.id,
|
|
831
|
+
threadKey,
|
|
832
|
+
requester: row.requester,
|
|
833
|
+
expiresAt: row.expires_at,
|
|
834
|
+
body: parseStored(row.body, isJsonObject) ?? {},
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
|
|
782
838
|
private readConfirmation(id: string): ConfirmationRow | undefined {
|
|
783
839
|
const row = this.sql
|
|
784
840
|
.exec<{ thread_key: string; requester: string; expires_at: number; body: string }>(
|
|
@@ -1199,6 +1255,7 @@ const CONFIG_ROUTES = new Set([
|
|
|
1199
1255
|
"/config/confirmations/consume",
|
|
1200
1256
|
"/config/confirmations/cancel",
|
|
1201
1257
|
"/config/confirmations/cancel-by-thread",
|
|
1258
|
+
"/config/confirmations/pending-by-thread",
|
|
1202
1259
|
]);
|
|
1203
1260
|
const TICKET_STATES: ReadonlySet<string> = new Set<McpTicketState>(MCP_TICKET_STATES);
|
|
1204
1261
|
/** A confirmation id as the bot mints it (a UUID) — one token, no whitespace, bounded. */
|
|
@@ -1284,6 +1341,15 @@ async function handleConfig(pathname: string, body: unknown, env: Env): Promise<
|
|
|
1284
1341
|
console.log(`[config/confirmations/cancel] ${click.id} ${"ok" in outcome ? "cancelled" : outcome.refused}`);
|
|
1285
1342
|
return json(outcome);
|
|
1286
1343
|
}
|
|
1344
|
+
case "/config/confirmations/pending-by-thread": {
|
|
1345
|
+
if (typeof b.threadKey !== "string" || !b.threadKey) return json({ error: "threadKey required" }, 400);
|
|
1346
|
+
// The stub types this result `never`: workers-types' Serializable rejects
|
|
1347
|
+
// the row's opaque `Record<string, unknown>` body. What arrives is the
|
|
1348
|
+
// object's declared result, so the boundary restates it.
|
|
1349
|
+
const row = (await dO.pendingConfirmationByThread(b.threadKey, systemClock())) as ConfirmationRow | null;
|
|
1350
|
+
console.log(`[config/confirmations/pending-by-thread] ${b.threadKey} ${row === null ? "none" : row.id}`);
|
|
1351
|
+
return json({ row });
|
|
1352
|
+
}
|
|
1287
1353
|
case "/config/confirmations/cancel-by-thread": {
|
|
1288
1354
|
if (typeof b.threadKey !== "string" || !b.threadKey) return json({ error: "threadKey required" }, 400);
|
|
1289
1355
|
if (!Array.isArray(b.actorIds) || !b.actorIds.every((a): a is string => typeof a === "string"))
|
|
@@ -3090,8 +3156,10 @@ function parseLimit(v: unknown): Validated<number> {
|
|
|
3090
3156
|
return { ok: true, value: v };
|
|
3091
3157
|
}
|
|
3092
3158
|
|
|
3093
|
-
/** `POST /list {scopeKey, limit, query?}`. */
|
|
3094
|
-
function parseList(
|
|
3159
|
+
/** `POST /list {scopeKey, limit, query?, kind?}`. */
|
|
3160
|
+
function parseList(
|
|
3161
|
+
body: unknown,
|
|
3162
|
+
): Validated<{ scopeKey: string; limit: number; query?: string; kind?: MemoryRecord["kind"] }> {
|
|
3095
3163
|
if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
|
|
3096
3164
|
const b = body as Record<string, unknown>;
|
|
3097
3165
|
const scope = parseScopeKey(b.scopeKey);
|
|
@@ -3102,12 +3170,29 @@ function parseList(body: unknown): Validated<{ scopeKey: string; limit: number;
|
|
|
3102
3170
|
if (typeof b.query !== "string") return invalid("query must be a string");
|
|
3103
3171
|
if (b.query.length > MAX_QUERY_CHARS) return invalid(`query must be at most ${MAX_QUERY_CHARS} characters`);
|
|
3104
3172
|
}
|
|
3173
|
+
if (b.kind !== undefined && b.kind !== "fact" && b.kind !== "summary")
|
|
3174
|
+
return invalid('kind must be "fact" or "summary"');
|
|
3105
3175
|
return {
|
|
3106
3176
|
ok: true,
|
|
3107
|
-
value: {
|
|
3177
|
+
value: {
|
|
3178
|
+
scopeKey: scope.value,
|
|
3179
|
+
limit: limit.value,
|
|
3180
|
+
...(typeof b.query === "string" ? { query: b.query } : {}),
|
|
3181
|
+
...(b.kind === "fact" || b.kind === "summary" ? { kind: b.kind } : {}),
|
|
3182
|
+
},
|
|
3108
3183
|
};
|
|
3109
3184
|
}
|
|
3110
3185
|
|
|
3186
|
+
/** `POST /sweep {scopeKey, dryRun?}`. */
|
|
3187
|
+
function parseSweep(body: unknown): Validated<{ scopeKey: string; dryRun: boolean }> {
|
|
3188
|
+
if (!isJsonObject(body)) return invalid("body must be a JSON object");
|
|
3189
|
+
const b = body;
|
|
3190
|
+
const scope = parseScopeKey(b.scopeKey);
|
|
3191
|
+
if (!scope.ok) return scope;
|
|
3192
|
+
if (b.dryRun !== undefined && typeof b.dryRun !== "boolean") return invalid("dryRun must be a boolean");
|
|
3193
|
+
return { ok: true, value: { scopeKey: scope.value, dryRun: b.dryRun === true } };
|
|
3194
|
+
}
|
|
3195
|
+
|
|
3111
3196
|
/** `POST /forget {scopeKey, id}`: the id is an opaque key, same caps as scopeKey. */
|
|
3112
3197
|
function parseForget(body: unknown): Validated<{ scopeKey: string; id: string }> {
|
|
3113
3198
|
if (typeof body !== "object" || body === null) return invalid("body must be a JSON object");
|
|
@@ -4298,6 +4383,7 @@ const ROUTES = new Set([
|
|
|
4298
4383
|
"/write",
|
|
4299
4384
|
"/list",
|
|
4300
4385
|
"/forget",
|
|
4386
|
+
"/sweep",
|
|
4301
4387
|
"/schedules/record",
|
|
4302
4388
|
"/schedules/latest",
|
|
4303
4389
|
"/runs/put",
|
|
@@ -4391,8 +4477,8 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
|
|
|
4391
4477
|
if (url.pathname === "/list") {
|
|
4392
4478
|
const parsed = parseList(body);
|
|
4393
4479
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
4394
|
-
const { scopeKey, limit, query } = parsed.value;
|
|
4395
|
-
const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query);
|
|
4480
|
+
const { scopeKey, limit, query, kind } = parsed.value;
|
|
4481
|
+
const records = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).list(scopeKey, limit, query, kind);
|
|
4396
4482
|
console.log(`[list] ${scopeKey} -> ${records.length} records`);
|
|
4397
4483
|
return json({ records });
|
|
4398
4484
|
}
|
|
@@ -4405,6 +4491,16 @@ async function handleRequest(request: Request, env: Env, admission: Admission):
|
|
|
4405
4491
|
console.log(`[forget] ${scopeKey} ${id} -> ${forgotten}`);
|
|
4406
4492
|
return json({ ok: true, forgotten });
|
|
4407
4493
|
}
|
|
4494
|
+
if (url.pathname === "/sweep") {
|
|
4495
|
+
const parsed = parseSweep(body);
|
|
4496
|
+
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
4497
|
+
const { scopeKey, dryRun } = parsed.value;
|
|
4498
|
+
const out = await env.MEMORY.get(env.MEMORY.idFromName(scopeKey)).sweep(scopeKey, dryRun);
|
|
4499
|
+
// Observability: scope + count only (ids carry no record text; they ride
|
|
4500
|
+
// the dryRun answer, not the log).
|
|
4501
|
+
console.log(`[sweep] ${scopeKey} -> ${out.swept}${dryRun ? " (dry run)" : ""}`);
|
|
4502
|
+
return json({ ok: true, swept: out.swept, ...(dryRun ? { ids: out.ids } : {}) });
|
|
4503
|
+
}
|
|
4408
4504
|
|
|
4409
4505
|
const parsed = parseWrite(body);
|
|
4410
4506
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// The resident guards its own memory (docs/reference/specs/resident-repos.md
|
|
2
|
+
// item 70) — the PURE half, kept free of the Sandbox SDK and DO storage so it
|
|
3
|
+
// runs under plain-Node vitest (memoryGuard.test.ts) like gc.ts and drain.ts.
|
|
4
|
+
// The Worker owns the one real reader (a single exec of CGROUP_READ_ARGV via
|
|
5
|
+
// its exec choke point) and feeds this module the raw output; tests feed a
|
|
6
|
+
// fake. A container at its cgroup memory cap wedges its own control port and
|
|
7
|
+
// every run on it loses its work, so the resident refuses NEW work by name
|
|
8
|
+
// while it still can: above the soft threshold a new attach is refused like
|
|
9
|
+
// `mirror-busy` (the bot falls back cold or waits, the card says why), above
|
|
10
|
+
// the hard threshold a new exec is refused with the numbers — and a command
|
|
11
|
+
// already running is never touched: the gate sits at each route's start and
|
|
12
|
+
// kills nothing.
|
|
13
|
+
import { MEMORY_PRESSURE_REASON } from "../../src/execution/sandboxErrors.js";
|
|
14
|
+
|
|
15
|
+
export { MEMORY_PRESSURE_REASON };
|
|
16
|
+
|
|
17
|
+
/** Above this percent of the cgroup memory cap a NEW `/attach` is refused —
|
|
18
|
+
* new runs go elsewhere while the ones already here finish. */
|
|
19
|
+
export const MEMORY_SOFT_LIMIT_PCT = 80;
|
|
20
|
+
|
|
21
|
+
/** Above this percent a NEW `/exec` is refused too — the resident answers
|
|
22
|
+
* nothing but the work already in flight, which always runs to completion. */
|
|
23
|
+
export const MEMORY_HARD_LIMIT_PCT = 90;
|
|
24
|
+
|
|
25
|
+
/** One reading of the container's cgroup v2 accounting. */
|
|
26
|
+
export interface MemoryReading {
|
|
27
|
+
/** ISO timestamp the caller supplied (the resident's clock seam). */
|
|
28
|
+
at: string;
|
|
29
|
+
/** memory.current — bytes charged to the cgroup now. */
|
|
30
|
+
usedBytes: number;
|
|
31
|
+
/** memory.max — the cap in bytes, or null when the file reads `max` (no cap). */
|
|
32
|
+
capBytes: number | null;
|
|
33
|
+
/** used/cap rounded to whole percent; null without a cap (nothing to gate on). */
|
|
34
|
+
percent: number | null;
|
|
35
|
+
/** cpu.stat's usage_usec, for the same log line; null when absent. */
|
|
36
|
+
cpuUsageUsec: number | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The reader seam: one call returns the raw output of `CGROUP_READ_ARGV`.
|
|
40
|
+
* A throw of `MemorySampleUnavailable` means "no reading this time" (the
|
|
41
|
+
* container busy or replaced under the probe) and never disables the gate;
|
|
42
|
+
* any other throw is an unreadable cgroup and disables it (one log line). */
|
|
43
|
+
export interface CgroupReader {
|
|
44
|
+
read(): Promise<string>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The one transient escape: the reader could not run at all right now.
|
|
48
|
+
* `invalidates` says the container the last reading came from is gone (a
|
|
49
|
+
* runtime replacement under the probe), so the guard forgets that reading —
|
|
50
|
+
* a dead container's numbers must not gate the work that replaces it. */
|
|
51
|
+
export class MemorySampleUnavailable extends Error {
|
|
52
|
+
constructor(
|
|
53
|
+
message: string,
|
|
54
|
+
readonly invalidates = false,
|
|
55
|
+
) {
|
|
56
|
+
super(message);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The separator `CGROUP_READ_ARGV` prints between the three files. */
|
|
61
|
+
export const CGROUP_FILE_SEPARATOR = ":::";
|
|
62
|
+
|
|
63
|
+
/** One exec, three files: memory.current, memory.max, cpu.stat from the
|
|
64
|
+
* container's own cgroup v2 root. `&&` so a missing file is a non-zero exit
|
|
65
|
+
* (an unreadable cgroup), never a silently short output. */
|
|
66
|
+
export const CGROUP_READ_ARGV: readonly string[] = [
|
|
67
|
+
"sh",
|
|
68
|
+
"-c",
|
|
69
|
+
`cat /sys/fs/cgroup/memory.current && echo ${CGROUP_FILE_SEPARATOR} && ` +
|
|
70
|
+
`cat /sys/fs/cgroup/memory.max && echo ${CGROUP_FILE_SEPARATOR} && cat /sys/fs/cgroup/cpu.stat`,
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
/** Parses one reader output into a reading; throws on anything malformed. */
|
|
74
|
+
export function parseCgroupOutput(raw: string, at: string): MemoryReading {
|
|
75
|
+
const parts = raw.split(CGROUP_FILE_SEPARATOR).map((p) => p.trim());
|
|
76
|
+
if (parts.length !== 3) throw new Error(`expected 3 cgroup sections, got ${parts.length}`);
|
|
77
|
+
const usedBytes = Number(parts[0]);
|
|
78
|
+
if (!Number.isFinite(usedBytes) || usedBytes < 0) throw new Error(`memory.current is not a byte count: ${parts[0]}`);
|
|
79
|
+
let capBytes: number | null = null;
|
|
80
|
+
if (parts[1] !== "max") {
|
|
81
|
+
capBytes = Number(parts[1]);
|
|
82
|
+
if (!Number.isFinite(capBytes) || capBytes <= 0) throw new Error(`memory.max is not a byte count: ${parts[1]}`);
|
|
83
|
+
}
|
|
84
|
+
const usage = /(?:^|\n)usage_usec (\d+)/.exec(parts[2]);
|
|
85
|
+
const cpuUsageUsec = usage ? Number(usage[1]) : null;
|
|
86
|
+
const percent = capBytes === null ? null : Math.round((usedBytes / capBytes) * 100);
|
|
87
|
+
return { at, usedBytes, capBytes, percent, cpuUsageUsec };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The one structured line Workers Logs gets per sample: used bytes, cap
|
|
91
|
+
* bytes, percent (and the cpu counter beside them). */
|
|
92
|
+
export function memoryLogLine(r: MemoryReading): string {
|
|
93
|
+
const cap = r.capBytes === null ? "uncapped" : `${r.capBytes} bytes cap`;
|
|
94
|
+
const pct = r.percent === null ? "" : ` (${r.percent}%)`;
|
|
95
|
+
const cpu = r.cpuUsageUsec === null ? "" : ` — cpu usage_usec ${r.cpuUsageUsec}`;
|
|
96
|
+
return `memory: used ${r.usedBytes} bytes of ${cap}${pct}${cpu}`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Which route asks the gate. */
|
|
100
|
+
export type MemoryGateRoute = "attach" | "exec";
|
|
101
|
+
|
|
102
|
+
/** A refusal with the numbers on it — the message is the whole story. */
|
|
103
|
+
export interface MemoryRefusal {
|
|
104
|
+
reason: typeof MEMORY_PRESSURE_REASON;
|
|
105
|
+
percent: number;
|
|
106
|
+
usedBytes: number;
|
|
107
|
+
capBytes: number;
|
|
108
|
+
message: string;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The pure gate over one reading: below both thresholds nothing changes; at
|
|
112
|
+
* or above the soft one a new attach is refused (queued — the caller waits or
|
|
113
|
+
* falls back, like `mirror-busy`); at or above the hard one a new exec is
|
|
114
|
+
* refused too. No reading, or no cap, gates nothing. */
|
|
115
|
+
export function gateMemory(route: MemoryGateRoute, reading: MemoryReading | null): MemoryRefusal | null {
|
|
116
|
+
if (!reading || reading.percent === null || reading.capBytes === null) return null;
|
|
117
|
+
const { percent, usedBytes, capBytes } = reading;
|
|
118
|
+
const refusal = (message: string): MemoryRefusal => ({
|
|
119
|
+
reason: MEMORY_PRESSURE_REASON,
|
|
120
|
+
percent,
|
|
121
|
+
usedBytes,
|
|
122
|
+
capBytes,
|
|
123
|
+
message,
|
|
124
|
+
});
|
|
125
|
+
if (route === "exec" && percent >= MEMORY_HARD_LIMIT_PCT) {
|
|
126
|
+
return refusal(
|
|
127
|
+
`${MEMORY_PRESSURE_REASON}: resident at ${percent}% of its memory cap (${usedBytes} of ${capBytes} bytes) — ` +
|
|
128
|
+
`a new command is refused above ${MEMORY_HARD_LIMIT_PCT}% while the commands already running finish`,
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
if (route === "attach" && percent >= MEMORY_SOFT_LIMIT_PCT) {
|
|
132
|
+
return refusal(
|
|
133
|
+
`${MEMORY_PRESSURE_REASON}: resident near its memory cap, ${percent}% used (${usedBytes} of ${capBytes} bytes), ` +
|
|
134
|
+
`queued — a new attach is refused above ${MEMORY_SOFT_LIMIT_PCT}% until the runs here release memory`,
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The message of whatever was thrown (local: this module stays dependency-free). */
|
|
141
|
+
const causeText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
|
142
|
+
|
|
143
|
+
/** The guard one resident holds: samples through the reader, keeps the last
|
|
144
|
+
* reading, gates the two routes — and on an unreadable cgroup logs ONCE and
|
|
145
|
+
* disables itself for the incarnation instead of refusing everything. */
|
|
146
|
+
export class MemoryGuard {
|
|
147
|
+
private disabledWhy: string | null = null;
|
|
148
|
+
private last: MemoryReading | null = null;
|
|
149
|
+
|
|
150
|
+
constructor(
|
|
151
|
+
private readonly reader: CgroupReader,
|
|
152
|
+
private readonly log: (line: string) => void,
|
|
153
|
+
) {}
|
|
154
|
+
|
|
155
|
+
/** The last reading taken this incarnation, for `/status` and `/residents`. */
|
|
156
|
+
get lastReading(): MemoryReading | null {
|
|
157
|
+
return this.last;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Whether an unreadable cgroup turned the gate off (the why is logged once). */
|
|
161
|
+
get disabled(): boolean {
|
|
162
|
+
return this.disabledWhy !== null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Forgets the last reading: the container it measured no longer runs (it
|
|
166
|
+
* went inactive, or was replaced under the probe), so nothing gates on it
|
|
167
|
+
* until a fresh sample lands. The gate's disabled state is untouched. */
|
|
168
|
+
invalidate(): void {
|
|
169
|
+
this.last = null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** One reader call, one log line. A `MemorySampleUnavailable` keeps the last
|
|
173
|
+
* reading and the gate as they were — unless it `invalidates` (the runtime
|
|
174
|
+
* was replaced under the probe), which forgets the reading so a stale one
|
|
175
|
+
* never masks the route's own runtime-replaced answer; any other failure
|
|
176
|
+
* disables the gate. */
|
|
177
|
+
async sample(at: string): Promise<MemoryReading | null> {
|
|
178
|
+
if (this.disabledWhy !== null) return null;
|
|
179
|
+
let raw: string;
|
|
180
|
+
try {
|
|
181
|
+
raw = await this.reader.read();
|
|
182
|
+
} catch (err) {
|
|
183
|
+
if (err instanceof MemorySampleUnavailable) {
|
|
184
|
+
if (err.invalidates) this.last = null;
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
this.disable(causeText(err));
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
let reading: MemoryReading;
|
|
191
|
+
try {
|
|
192
|
+
reading = parseCgroupOutput(raw, at);
|
|
193
|
+
} catch (err) {
|
|
194
|
+
this.disable(causeText(err));
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
this.last = reading;
|
|
198
|
+
this.log(memoryLogLine(reading));
|
|
199
|
+
return reading;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The route's verdict over the last reading; a disabled gate refuses nothing. */
|
|
203
|
+
gate(route: MemoryGateRoute): MemoryRefusal | null {
|
|
204
|
+
if (this.disabledWhy !== null) return null;
|
|
205
|
+
return gateMemory(route, this.last);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
private disable(why: string): void {
|
|
209
|
+
this.disabledWhy = why;
|
|
210
|
+
this.log(`memory: cgroup unreadable — the memory gate is disabled for this incarnation (${why})`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -58,7 +58,7 @@ export interface RefreshInstanceAction {
|
|
|
58
58
|
* (`stepTimeoutMs`), so a step timeout and a command timeout agree. */
|
|
59
59
|
const REFRESH_FETCH_STEP_BUDGET_MS = RESTORE_MAX_MS + GIT_NETWORK_TIMEOUT_MS; // a wake's restore, then the fetch
|
|
60
60
|
const REFRESH_INSTALL_STEP_BUDGET_MS = REFRESH_INSTALL_TIMEOUT_MS + DEPS_STEP_OVERHEAD_MS; // the install's own lease
|
|
61
|
-
const REFRESH_BUILD_STEP_BUDGET_MS = GIT_NETWORK_TIMEOUT_MS + REFRESH_BUILD_TIMEOUT_MS; // the
|
|
61
|
+
const REFRESH_BUILD_STEP_BUDGET_MS = 5 * GIT_NETWORK_TIMEOUT_MS + 2 * REFRESH_BUILD_TIMEOUT_MS; // five git-budgeted commands — stage-clear, stage, the staging fetch (under the stage lock), the checkout update and the swap (under the swap lock) — plus the off-lock reset/clean, deps re-link and build
|
|
62
62
|
const REFRESH_SNAPSHOT_STEP_BUDGET_MS = R2_TRANSFER_TIMEOUT_MS + GIT_NETWORK_TIMEOUT_MS; // the archives, then the reclaim pass
|
|
63
63
|
/** The sweep: one cleanliness check per binding idle past an hour (at most the
|
|
64
64
|
* pool's worth, one exec budget each), then the removals under the mirror lock. */
|