@coreplane/switchboard 1.248.0 → 1.249.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/config/config.example.yaml +44 -18
- package/dist/assets/deploy/cloudflare/preflight.mjs +21 -19
- package/dist/assets/deploy/cloudflare-memory/worker.ts +31 -0
- package/dist/assets/deploy/cloudflare-resident/drain.ts +109 -0
- package/dist/assets/deploy/cloudflare-resident/refresh.ts +5 -3
- package/dist/assets/deploy/cloudflare-resident/threadErr.ts +32 -3
- package/dist/assets/deploy/cloudflare-resident/worker.ts +235 -13
- package/dist/assets/deploy/cloudflare-sandbox/worker.ts +56 -9
- package/dist/assets/deploy/secrets.manifest.json +6 -0
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +4 -4
- package/dist/assets/src/core/budgets.ts +24 -0
- package/dist/assets/src/core/coordinator/contract.ts +4 -3
- package/dist/assets/src/core/coordinator/driver.ts +40 -7
- package/dist/assets/src/core/modelCard.ts +348 -0
- package/dist/assets/src/core/modelPricing.ts +14 -5
- package/dist/assets/src/core/modelRegistry.ts +51 -0
- package/dist/assets/src/core/provider.ts +103 -0
- package/dist/assets/src/core/refusal.ts +181 -0
- package/dist/assets/src/core/runEvents.ts +43 -0
- package/dist/assets/src/core/ship/coordinator.ts +80 -12
- package/dist/assets/src/core/ship/handoff.ts +54 -19
- package/dist/assets/src/core/trace/workerTrace.ts +9 -3
- package/dist/assets/src/core/types.ts +327 -0
- package/dist/assets/src/deploy/liveGate.ts +35 -0
- package/dist/assets/src/deploy/restart.ts +12 -11
- package/dist/assets/src/execution/residentRefresh.ts +26 -1
- package/dist/assets/src/execution/sandboxErrors.ts +122 -4
- package/dist/assets/web/dist/.vite/manifest.json +30 -30
- package/dist/assets/web/dist/assets/DeliveryPage-DF4aQypG.js +1 -0
- package/dist/assets/web/dist/assets/{HomePage-DYxC0izY.js → HomePage-BpQRky8B.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentDetailPage-DLIpWYOc.js → ResidentDetailPage-BIUXyz6K.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-6LipuDjR.js → ResidentsIndexPage-BZymgSAb.js} +1 -1
- package/dist/assets/web/dist/assets/{RunFoldRow-V-iSy64e.js → RunFoldRow-3m4CPRI4.js} +1 -1
- package/dist/assets/web/dist/assets/{RunRoutePage-DUalB1u2.js → RunRoutePage-bgkjkA0p.js} +3 -3
- package/dist/assets/web/dist/assets/{RunsIndexPage-B9Ba1KdD.js → RunsIndexPage-8S944AzB.js} +1 -1
- package/dist/assets/web/dist/assets/{ScheduledPage-KdjLtD_7.js → ScheduledPage-8bBtG9y3.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-IT5l_NaL.js → SettingsPage-DQeNvfaV.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-DBHAl4Il.js → StatusDot-BPE5syBa.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-_LEjptLV.js → Tooltip-DkoeZfTs.js} +1 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-BUzw--Ii.js +1 -0
- package/dist/assets/web/dist/assets/{dist-BcYPGOBL.js → dist-D11y9ZJ4.js} +1 -1
- package/dist/assets/web/dist/assets/{main-DUfSE0dj.js → main-B6LcgNM6.js} +2 -2
- package/dist/cli.js +2475 -1021
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/DeliveryPage-NP4g6bQd.js +0 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-DPBsvGPR.js +0 -1
|
@@ -9,37 +9,63 @@ organization: acme
|
|
|
9
9
|
|
|
10
10
|
providers:
|
|
11
11
|
anthropic:
|
|
12
|
-
|
|
12
|
+
wire: anthropic-messages
|
|
13
13
|
apiKeyEnv: ANTHROPIC_API_KEY
|
|
14
|
-
# OpenAI itself runs on
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
14
|
+
# OpenAI itself runs on its Responses API (record 0052): every first-party
|
|
15
|
+
# OpenAI card in pi's registry is `openai-responses`, OpenAI recommends the
|
|
16
|
+
# wire for new projects, and its chat route loses tool calling with any
|
|
17
|
+
# effort above `none` from GPT-5.4. The wire is a declaration here; the proxy
|
|
18
|
+
# serves the Responses route in a later unit, and until then a run on this
|
|
19
|
+
# block meets the provider's own answer. `type: openai-compatible` keeps
|
|
20
|
+
# loading as `wire: openai-chat` for one release.
|
|
19
21
|
openai:
|
|
20
|
-
|
|
22
|
+
wire: openai-responses
|
|
21
23
|
baseUrl: https://api.openai.com/v1
|
|
22
24
|
apiKeyEnv: OPENAI_API_KEY
|
|
23
25
|
# groq:
|
|
24
|
-
#
|
|
26
|
+
# wire: openai-chat
|
|
25
27
|
# baseUrl: https://api.groq.com/openai/v1
|
|
26
28
|
# apiKeyEnv: GROQ_API_KEY
|
|
27
|
-
# OpenRouter: one key, many vendors' models,
|
|
28
|
-
#
|
|
29
|
-
#
|
|
29
|
+
# OpenRouter: one key, many vendors' models, an aggregator (`vendor: model`
|
|
30
|
+
# reads the vendor off the model id's first segment, so the card is per block
|
|
31
|
+
# AND vendor: a tier the OpenRouter file refuses may be native direct).
|
|
32
|
+
# `catalog: openrouter` names the pi registry file its cards come from; a
|
|
33
|
+
# block named unlike its catalog still reads that file. Any preset's model
|
|
34
|
+
# may be openrouter/<vendor>/<model> — e.g. openrouter/anthropic/claude-sonnet-4:
|
|
30
35
|
# the first slash names this block, the rest is the model id passed through.
|
|
31
36
|
# OPENROUTER_API_KEY is read at the first model call, never at load, so
|
|
32
37
|
# without it every other block runs as today and only a run on this one
|
|
33
|
-
# fails, by that name.
|
|
34
|
-
#
|
|
35
|
-
# rules (prompt-cache markers included); a document becomes a text note.
|
|
36
|
-
# `switchboard init` keeps this block only with --openrouter-key.
|
|
38
|
+
# fails, by that name. `switchboard init` keeps this block only with
|
|
39
|
+
# --openrouter-key.
|
|
37
40
|
openrouter:
|
|
38
|
-
|
|
41
|
+
wire: openai-chat
|
|
42
|
+
vendor: model
|
|
43
|
+
catalog: openrouter
|
|
39
44
|
baseUrl: https://openrouter.ai/api/v1
|
|
40
45
|
apiKeyEnv: OPENROUTER_API_KEY
|
|
46
|
+
# The operator's layer over the registry card: what a run needs that no
|
|
47
|
+
# layer names goes out unvouched (and a note says so on the record) until
|
|
48
|
+
# an override like this pins it. `providers check` (a later unit) reads the
|
|
49
|
+
# provider's own endpoint to fill these in.
|
|
50
|
+
# models:
|
|
51
|
+
# deepseek/deepseek-v4.1-flash:
|
|
52
|
+
# window: 1048576
|
|
53
|
+
# capField: max_tokens
|
|
54
|
+
# levels:
|
|
55
|
+
# high: high
|
|
56
|
+
# xhigh: xhigh
|
|
57
|
+
# inputs:
|
|
58
|
+
# image: true
|
|
59
|
+
# cache: automatic
|
|
60
|
+
# A passthrough is merged into the request body on the wires whose adapter
|
|
61
|
+
# takes one — a vendor-only feature such as OpenRouter's routing, never a
|
|
62
|
+
# control: it is not decided, not noted and not in the matrix, and a wrong
|
|
63
|
+
# one surfaces as the provider's own error on the first turn.
|
|
64
|
+
# passthrough:
|
|
65
|
+
# provider:
|
|
66
|
+
# sort: throughput
|
|
41
67
|
# local:
|
|
42
|
-
#
|
|
68
|
+
# wire: openai-chat
|
|
43
69
|
# baseUrl: http://localhost:11434/v1 # Ollama
|
|
44
70
|
|
|
45
71
|
defaults:
|
|
@@ -236,7 +262,7 @@ execution:
|
|
|
236
262
|
# baseUrl: https://switchboard-resident.<account>.workers.dev
|
|
237
263
|
# tokenEnv: RESIDENT_OPERATOR_TOKEN # operator bearer secret (default)
|
|
238
264
|
# adminTokenEnv: RESIDENT_ADMIN_TOKEN # admin bearer for `repo ...` commands (default)
|
|
239
|
-
# probeTimeoutMs:
|
|
265
|
+
# probeTimeoutMs: 8000 # /status probe deadline; a miss = not warm for that dispatch, never an outage
|
|
240
266
|
|
|
241
267
|
# Where per-conversation agent workspaces are created (local execution only).
|
|
242
268
|
# Under docker compose this directory is a named volume, so checkouts survive a restart.
|
|
@@ -1,22 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Deploy preflight for the bot Worker (docs/reference/specs/slack-channel.md item 8).
|
|
3
3
|
//
|
|
4
|
-
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
4
|
+
// `wrangler deploy` rolls the bot container. Cloudflare's rollout sends SIGTERM
|
|
5
|
+
// and the container is gone at the drain's end, whatever it was still driving:
|
|
6
|
+
// the run ledger's handoff (docs/reference/specs/run-history.md item 39) lets the
|
|
7
|
+
// next generation resume a run, but a resume is a recovery, not a guarantee —
|
|
8
|
+
// one that fails costs the run — so a deploy never rolls over a live run.
|
|
9
|
+
// `npm run deploy` runs this first and refuses while
|
|
10
|
+
// - the bot reports runs in flight (`GET /healthz` `inFlight > 0`): the
|
|
11
|
+
// runner retries every minute up to its budget, then fails by name;
|
|
10
12
|
// - the container application is not in a settled state (a rollout is still
|
|
11
13
|
// provisioning/updating — `wrangler containers list --json`): a second
|
|
12
14
|
// rollout on top of one in progress replaces the instance the first put
|
|
13
15
|
// into its graceful drain and kills whatever it was running (two deploys
|
|
14
16
|
// 90 s apart once killed a review at 153 s).
|
|
15
|
-
// It WARNS (never refuses) when the bot
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
// required scopes (`catchUp.error`, `catchUp.missingScopes`).
|
|
17
|
+
// It WARNS (never refuses) when the bot is already draining with nothing in
|
|
18
|
+
// flight (`draining: true` — the instance exits on its own), and when /healthz
|
|
19
|
+
// says the reconnect catch-up is failing or the bot token lacks required
|
|
20
|
+
// scopes (`catchUp.error`, `catchUp.missingScopes`).
|
|
20
21
|
//
|
|
21
22
|
// Fail closed: unreachable bot, a body without the JSON shape (a bot whose
|
|
22
23
|
// /healthz answers a bare `ok` is not one this preflight can read), a wrangler failure, or an app
|
|
@@ -40,7 +41,7 @@ export const APP_NAME = "switchboard-switchboardserver";
|
|
|
40
41
|
const SETTLED_APP_STATES = new Set(["active", "ready"]);
|
|
41
42
|
|
|
42
43
|
const HOW_TO_FORCE =
|
|
43
|
-
"to deploy anyway (over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
|
|
44
|
+
"to deploy anyway (over the runs in flight — this kills them —, over a rollout in progress, or blind when the bot cannot be consulted): `SWITCHBOARD_DEPLOY_FORCE=1 npm run deploy` (`node preflight.mjs --force` checks alone)";
|
|
44
45
|
|
|
45
46
|
/** GET /healthz. Never throws: `{ok:true,payload}` (parsed JSON, or the raw text when not JSON) or `{ok:false,error}`. */
|
|
46
47
|
export async function fetchHealth(baseUrl, { timeoutMs = 20_000 } = {}) {
|
|
@@ -164,15 +165,16 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
164
165
|
if (!Number.isInteger(p.inFlight) || p.inFlight < 0) {
|
|
165
166
|
problems.push(`bot reports an impossible inFlight=${JSON.stringify(p.inFlight)} (counter bug or old Worker)`);
|
|
166
167
|
} else if (p.inFlight > 0) {
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
|
|
170
|
-
|
|
168
|
+
// A refusal: the rollout rolls the container under these runs. A
|
|
169
|
+
// handoff (run-history item 39) may resume them on the next generation,
|
|
170
|
+
// but a resume that fails costs the run — the deploy waits instead.
|
|
171
|
+
problems.push(
|
|
172
|
+
`${p.inFlight} run(s) in flight — the rollout would roll the bot container under them (a handoff is a recovery, not a guarantee)`,
|
|
171
173
|
);
|
|
172
174
|
}
|
|
173
175
|
if (p.draining === true) {
|
|
174
176
|
warnings.push(
|
|
175
|
-
"bot is already draining from a previous deploy —
|
|
177
|
+
"bot is already draining from a previous deploy — the draining instance exits on its own; this rollout replaces it at once",
|
|
176
178
|
);
|
|
177
179
|
}
|
|
178
180
|
}
|
|
@@ -209,7 +211,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
209
211
|
forced: true,
|
|
210
212
|
problems,
|
|
211
213
|
warnings,
|
|
212
|
-
message: `preflight WARNING: deploying by force despite —\n${detail}\n a rollout landing on one in progress can disrupt it
|
|
214
|
+
message: `preflight WARNING: deploying by force despite —\n${detail}\n this WILL kill the runs in flight that no resume recovers, and a rollout landing on one in progress can disrupt it${warningText}`,
|
|
213
215
|
};
|
|
214
216
|
}
|
|
215
217
|
return {
|
|
@@ -217,7 +219,7 @@ export function decide({ health, apps }, { force = false } = {}) {
|
|
|
217
219
|
forced: false,
|
|
218
220
|
problems,
|
|
219
221
|
warnings,
|
|
220
|
-
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait and retry; ${HOW_TO_FORCE}${warningText}`,
|
|
222
|
+
message: `preflight REFUSED: a Worker deploy rolls the bot container —\n${detail}\n wait for them to finish and retry; ${HOW_TO_FORCE}${warningText}`,
|
|
221
223
|
};
|
|
222
224
|
}
|
|
223
225
|
|
|
@@ -749,6 +749,26 @@ export class ConfigDO extends DurableObject<Env> {
|
|
|
749
749
|
});
|
|
750
750
|
}
|
|
751
751
|
|
|
752
|
+
/** Delete the thread's pending row under the same requester check (record
|
|
753
|
+
* 0054: a typed answer supersedes the button, so a click cannot follow
|
|
754
|
+
* it). At most one row per thread exists (`putConfirmation` replaces); a
|
|
755
|
+
* thread with none is `used`. The body stays opaque here — a row stored
|
|
756
|
+
* before the bot's union gained `kind` cancels the same way. */
|
|
757
|
+
async cancelConfirmationByThread(threadKey: string, actorIds: readonly string[]): Promise<ConfirmationCancelOutcome> {
|
|
758
|
+
return this.ctx.storage.transactionSync(() => {
|
|
759
|
+
const stored = this.sql
|
|
760
|
+
.exec<{ id: string; requester: string }>(
|
|
761
|
+
`SELECT id, requester FROM confirmations WHERE thread_key = ?`,
|
|
762
|
+
threadKey,
|
|
763
|
+
)
|
|
764
|
+
.toArray()[0];
|
|
765
|
+
if (!stored) return { refused: "used" };
|
|
766
|
+
if (!actorIds.includes(stored.requester)) return { refused: "foreign" };
|
|
767
|
+
this.sql.exec(`DELETE FROM confirmations WHERE id = ?`, stored.id);
|
|
768
|
+
return { ok: true };
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
|
|
752
772
|
private readConfirmation(id: string): ConfirmationRow | undefined {
|
|
753
773
|
const row = this.sql
|
|
754
774
|
.exec<{ thread_key: string; requester: string; expires_at: number; body: string }>(
|
|
@@ -1168,6 +1188,7 @@ const CONFIG_ROUTES = new Set([
|
|
|
1168
1188
|
"/config/confirmations/put",
|
|
1169
1189
|
"/config/confirmations/consume",
|
|
1170
1190
|
"/config/confirmations/cancel",
|
|
1191
|
+
"/config/confirmations/cancel-by-thread",
|
|
1171
1192
|
]);
|
|
1172
1193
|
const TICKET_STATES: ReadonlySet<string> = new Set<McpTicketState>(MCP_TICKET_STATES);
|
|
1173
1194
|
/** A confirmation id as the bot mints it (a UUID) — one token, no whitespace, bounded. */
|
|
@@ -1253,6 +1274,16 @@ async function handleConfig(pathname: string, body: unknown, env: Env): Promise<
|
|
|
1253
1274
|
console.log(`[config/confirmations/cancel] ${click.id} ${"ok" in outcome ? "cancelled" : outcome.refused}`);
|
|
1254
1275
|
return json(outcome);
|
|
1255
1276
|
}
|
|
1277
|
+
case "/config/confirmations/cancel-by-thread": {
|
|
1278
|
+
if (typeof b.threadKey !== "string" || !b.threadKey) return json({ error: "threadKey required" }, 400);
|
|
1279
|
+
if (!Array.isArray(b.actorIds) || !b.actorIds.every((a): a is string => typeof a === "string"))
|
|
1280
|
+
return json({ error: "actorIds must be a list of actor ids" }, 400);
|
|
1281
|
+
const outcome = await dO.cancelConfirmationByThread(b.threadKey, b.actorIds);
|
|
1282
|
+
console.log(
|
|
1283
|
+
`[config/confirmations/cancel-by-thread] ${b.threadKey} ${"ok" in outcome ? "cancelled" : outcome.refused}`,
|
|
1284
|
+
);
|
|
1285
|
+
return json(outcome);
|
|
1286
|
+
}
|
|
1256
1287
|
default:
|
|
1257
1288
|
break;
|
|
1258
1289
|
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// The fleet drain (docs/reference/specs/resident-repos.md item 69): one record
|
|
2
|
+
// in the registry Durable Object that closes `POST /attach` to NEW runs while a
|
|
3
|
+
// deploy waits for the runs already in flight to end — a run registered from
|
|
4
|
+
// its attach to its release re-attaches through it (a rolled container, an
|
|
5
|
+
// evicted worktree, a resumed run), so the drain never refuses a run it waits for. Pure over its inputs so
|
|
6
|
+
// it runs under plain Node (drain.test.ts); the Worker holds the storage and
|
|
7
|
+
// the routes, and the deploy runner (src/deploy/residentDrain.ts) the other end.
|
|
8
|
+
//
|
|
9
|
+
// Why a record with an end and not a flag: Durable Object storage survives the
|
|
10
|
+
// isolate swap the deploy performs, so a drain nobody lifted — a runner that
|
|
11
|
+
// died mid-step — would close the fleet for good. Every drain carries `until`;
|
|
12
|
+
// past it the record is nothing, whoever forgot it.
|
|
13
|
+
|
|
14
|
+
import { DRAIN, minutesToMs } from "../../src/core/budgets.js";
|
|
15
|
+
|
|
16
|
+
/** The drain as the registry stores it. */
|
|
17
|
+
export interface DrainRecord {
|
|
18
|
+
/** ISO time the drain began. */
|
|
19
|
+
since: string;
|
|
20
|
+
/** ISO time it ends by itself, whether or not anyone lifts it. */
|
|
21
|
+
until: string;
|
|
22
|
+
/** Who asked (the deploy names its commit). */
|
|
23
|
+
by: string;
|
|
24
|
+
/** Why, in the words the attach refusal repeats. */
|
|
25
|
+
reason: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The longest a drain may run, and the default, from the one clock table
|
|
29
|
+
* (src/core/budgets.ts `DRAIN`): past a coding child's whole lease with room
|
|
30
|
+
* for the deploy itself, and short enough that a forgotten drain is an hour
|
|
31
|
+
* and a half, not a day. */
|
|
32
|
+
export const DRAIN_MAX_MINUTES: number = DRAIN.maxMinutes;
|
|
33
|
+
export const DRAIN_DEFAULT_MINUTES: number = DRAIN.defaultMinutes;
|
|
34
|
+
const REASON_MAX = 200;
|
|
35
|
+
const BY_MAX = 80;
|
|
36
|
+
|
|
37
|
+
export type DrainRequest = { ok: true; record: DrainRecord } | { ok: false; error: string };
|
|
38
|
+
|
|
39
|
+
/** `POST /drain`'s body → the record, or the refusal by name. `minutes` is an
|
|
40
|
+
* integer in [1, DRAIN_MAX_MINUTES] (default DRAIN_DEFAULT_MINUTES); `reason`
|
|
41
|
+
* and `by` are short strings with defaults. A malformed value is refused,
|
|
42
|
+
* never clamped silently: a drain is an operator's act and its words are read
|
|
43
|
+
* back by every run it refuses. */
|
|
44
|
+
export function parseDrainRequest(body: Record<string, unknown>, now: number): DrainRequest {
|
|
45
|
+
let minutes = DRAIN_DEFAULT_MINUTES;
|
|
46
|
+
if (body.minutes !== undefined) {
|
|
47
|
+
if (typeof body.minutes !== "number" || !Number.isInteger(body.minutes))
|
|
48
|
+
return { ok: false, error: "minutes must be an integer number of minutes" };
|
|
49
|
+
if (body.minutes < 1 || body.minutes > DRAIN_MAX_MINUTES)
|
|
50
|
+
return { ok: false, error: `minutes must be between 1 and ${DRAIN_MAX_MINUTES}` };
|
|
51
|
+
minutes = body.minutes;
|
|
52
|
+
}
|
|
53
|
+
const word = (v: unknown, name: string, max: number, fallback: string): string | { error: string } => {
|
|
54
|
+
if (v === undefined) return fallback;
|
|
55
|
+
if (typeof v !== "string" || v.trim() === "") return { error: `${name} must be a non-empty string` };
|
|
56
|
+
const trimmed = v.trim();
|
|
57
|
+
if (trimmed.length > max) return { error: `${name} must be at most ${max} characters` };
|
|
58
|
+
return trimmed;
|
|
59
|
+
};
|
|
60
|
+
const reason = word(body.reason, "reason", REASON_MAX, "a deploy");
|
|
61
|
+
if (typeof reason !== "string") return { ok: false, error: reason.error };
|
|
62
|
+
const by = word(body.by, "by", BY_MAX, "admin");
|
|
63
|
+
if (typeof by !== "string") return { ok: false, error: by.error };
|
|
64
|
+
return {
|
|
65
|
+
ok: true,
|
|
66
|
+
record: {
|
|
67
|
+
since: new Date(now).toISOString(),
|
|
68
|
+
until: new Date(now + minutesToMs(minutes)).toISOString(),
|
|
69
|
+
by,
|
|
70
|
+
reason,
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The drain in force at `now`: the stored record when it is well-formed and
|
|
76
|
+
* its `until` is still ahead; null for none, an expired one, or a record this
|
|
77
|
+
* build cannot read (a shape from another build reads as no drain, never as a
|
|
78
|
+
* closed fleet). */
|
|
79
|
+
export function liveDrain(stored: unknown, now: number): DrainRecord | null {
|
|
80
|
+
if (typeof stored !== "object" || stored === null) return null;
|
|
81
|
+
const r = stored as Record<string, unknown>;
|
|
82
|
+
if (
|
|
83
|
+
typeof r.since !== "string" ||
|
|
84
|
+
typeof r.until !== "string" ||
|
|
85
|
+
typeof r.by !== "string" ||
|
|
86
|
+
typeof r.reason !== "string"
|
|
87
|
+
)
|
|
88
|
+
return null;
|
|
89
|
+
const until = Date.parse(r.until);
|
|
90
|
+
if (!Number.isFinite(until) || until <= now) return null;
|
|
91
|
+
return { since: r.since, until: r.until, by: r.by, reason: r.reason };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The `/attach` answer while the fleet is drained: a 503 whose body carries
|
|
95
|
+
* the record, so the bot waits for `until` at most and the card says why the
|
|
96
|
+
* run has not started. The `error` word `draining:` is the client's key. */
|
|
97
|
+
export function drainRefusal(drain: DrainRecord): {
|
|
98
|
+
error: string;
|
|
99
|
+
status: 503;
|
|
100
|
+
draining: DrainRecord;
|
|
101
|
+
} {
|
|
102
|
+
return {
|
|
103
|
+
error:
|
|
104
|
+
`draining: the resident fleet is closed to new runs for ${drain.reason} (asked by ${drain.by} at ${drain.since}, ` +
|
|
105
|
+
`ends by ${drain.until}) — the run waits at its attach and starts when the fleet reopens`,
|
|
106
|
+
status: 503,
|
|
107
|
+
draining: drain,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
} from "../../src/execution/residentInstanceId.js";
|
|
22
22
|
import { RESTORE_MAX_MS, type RefreshPlan } from "../../src/execution/residentRefresh.js";
|
|
23
23
|
import { residentText } from "../../src/execution/residentText.js";
|
|
24
|
+
import { RUNTIME_BUSY_REASON } from "../../src/execution/sandboxErrors.js";
|
|
24
25
|
import { graftResidentSteps } from "../../src/execution/residentTrace.js";
|
|
25
26
|
import {
|
|
26
27
|
DEFAULT_EXEC_TIMEOUT_MS,
|
|
@@ -288,9 +289,10 @@ export class ResidentRefresh extends WorkflowEntrypoint<Env, RefreshInstancePara
|
|
|
288
289
|
}
|
|
289
290
|
// Housekeeping rides on every instance whatever the cycle's verdict — a
|
|
290
291
|
// parked resident still releases its idle bindings and re-measures, and
|
|
291
|
-
// neither step wakes a slept container — except an offboarded one
|
|
292
|
-
// nothing is left to keep
|
|
293
|
-
|
|
292
|
+
// neither step wakes a slept container — except an offboarded one
|
|
293
|
+
// (nothing is left to keep) and one whose container turned the cycle
|
|
294
|
+
// away (item 68: the sweep and the measure exec on that same container).
|
|
295
|
+
if (cycle.outcome !== "offboarded" && cycle.outcome !== RUNTIME_BUSY_REASON) {
|
|
294
296
|
const swept = await step.do("sweep", { retries, timeout: stepTimeoutMs(REFRESH_SWEEP_STEP_BUDGET_MS) }, () =>
|
|
295
297
|
stub.refreshInstanceSweep({ resource, instance }),
|
|
296
298
|
);
|
|
@@ -11,8 +11,10 @@ import {
|
|
|
11
11
|
isRuntimeUnreachableSignal,
|
|
12
12
|
selfAndCauses,
|
|
13
13
|
} from "../../src/execution/residentRefresh.js";
|
|
14
|
+
import { isRuntimeBusyError, RUNTIME_BUSY_REASON } from "../../src/execution/sandboxErrors.js";
|
|
14
15
|
import type { ResidentLifecycleState } from "../../src/execution/residentState.js";
|
|
15
16
|
import type { ResidentStep } from "../../src/execution/residentStepTrace.js";
|
|
17
|
+
import type { RefusalCause } from "../../src/core/refusal.js";
|
|
16
18
|
|
|
17
19
|
// The cause-chain walker has one home, beside the wording lists; the Worker
|
|
18
20
|
// takes it from here with the rest of the thread data plane's shapes.
|
|
@@ -52,6 +54,12 @@ export interface ThreadErr {
|
|
|
52
54
|
defaultRef?: string;
|
|
53
55
|
state?: ResidentLifecycleState;
|
|
54
56
|
reason?: string;
|
|
57
|
+
/** Why the route refused, in the seam's three classes (record 0054), so the
|
|
58
|
+
* caller reads a field instead of the words: a bad ref or a missing binding
|
|
59
|
+
* is `request`, a repository the App cannot see is `policy`, and the
|
|
60
|
+
* machinery's own failure is `system`. The words stay for the person; the
|
|
61
|
+
* cause is what the bot's command surfaces render by. */
|
|
62
|
+
cause?: RefusalCause;
|
|
55
63
|
}
|
|
56
64
|
|
|
57
65
|
/** Where a replacement or a reset was met: the command's spawn or its collect
|
|
@@ -143,7 +151,7 @@ export class ControlResetError extends Error {
|
|
|
143
151
|
* as an `/exec` script). Gating them would trade a spare re-attach for a read
|
|
144
152
|
* that fails outright while the container starts. */
|
|
145
153
|
export function runtimeReplacedErr(err: RuntimeReplacedError): ThreadErr {
|
|
146
|
-
return { error: err.message, status: 409, reason: "runtime-replaced" };
|
|
154
|
+
return { error: err.message, status: 409, reason: "runtime-replaced", cause: "system" };
|
|
147
155
|
}
|
|
148
156
|
|
|
149
157
|
/** The named ThreadErr a DO code-update reset answers with — its own `reason`
|
|
@@ -152,7 +160,7 @@ export function runtimeReplacedErr(err: RuntimeReplacedError): ThreadErr {
|
|
|
152
160
|
* idempotent op or resolves a write by echo (harness-pi item 16), never the
|
|
153
161
|
* replaced verdict. */
|
|
154
162
|
export function controlResetErr(err: ControlResetError): ThreadErr {
|
|
155
|
-
return { error: err.message, status: 409, reason: "control-reset" };
|
|
163
|
+
return { error: err.message, status: 409, reason: "control-reset", cause: "system" };
|
|
156
164
|
}
|
|
157
165
|
|
|
158
166
|
/** The platform's transient sentences the pinned SDK's own predicate does not
|
|
@@ -280,6 +288,9 @@ export function threadErrBuilders(p: ThrowPredicates): ThreadErrBuilders {
|
|
|
280
288
|
// `isRuntimeUnreachableReason`) — and the remainder sentences.
|
|
281
289
|
for (const link of selfAndCauses(err)) {
|
|
282
290
|
if (isRuntimeUnreachableSignal(link)) return true;
|
|
291
|
+
// The DO's own word for a refused connect (`runtime-busy:`, item 68):
|
|
292
|
+
// the container accepts again in moments, so a re-probe clears it.
|
|
293
|
+
if (isRuntimeBusyError(link)) return true;
|
|
283
294
|
const message = messageOf(link);
|
|
284
295
|
if (isRuntimeUnreachableReason(message) || TRANSIENT_PLATFORM_WORDING.test(message)) return true;
|
|
285
296
|
}
|
|
@@ -291,6 +302,8 @@ export function threadErrBuilders(p: ThrowPredicates): ThreadErrBuilders {
|
|
|
291
302
|
error: prefix ? `${prefix}: ${words}` : words,
|
|
292
303
|
status: 500,
|
|
293
304
|
transient: isTransientPlatformThrow(err, known),
|
|
305
|
+
// A throw no route named is the machinery's own: system.
|
|
306
|
+
cause: "system",
|
|
294
307
|
};
|
|
295
308
|
};
|
|
296
309
|
const threadRejectionErr = (err: unknown, route: ThreadDataRoute): ThreadErr => {
|
|
@@ -299,9 +312,13 @@ export function threadErrBuilders(p: ThrowPredicates): ThreadErrBuilders {
|
|
|
299
312
|
const runtimeReplacement = p.isRuntimeReplacement(err);
|
|
300
313
|
if (runtimeReplacement) {
|
|
301
314
|
const vouched = p.sdkVouchesRuntimeMoved(err);
|
|
302
|
-
if (route === "/exec" && !vouched) return { error: messageOf(err), status: 409 };
|
|
315
|
+
if (route === "/exec" && !vouched) return { error: messageOf(err), status: 409, cause: "system" };
|
|
303
316
|
return runtimeReplacedErr(new RuntimeReplacedError("call", err, vouched));
|
|
304
317
|
}
|
|
318
|
+
// The DO's own word for a refused connect, thrown out of a method before
|
|
319
|
+
// the stub answered (item 68): the same 503 the method answers inside, so
|
|
320
|
+
// the client re-sends on the token wherever the throw was met.
|
|
321
|
+
if (isRuntimeBusyError(err)) return runtimeBusyErr(err instanceof Error ? err : new Error(messageOf(err)));
|
|
305
322
|
// The two verdicts just settled ride into the typed 500, so its walk of the
|
|
306
323
|
// cause chain is for the transient alone.
|
|
307
324
|
return catchAllErr(err, undefined, { controlReset, runtimeReplacement });
|
|
@@ -309,6 +326,17 @@ export function threadErrBuilders(p: ThrowPredicates): ThreadErrBuilders {
|
|
|
309
326
|
return { isTransientPlatformThrow, catchAllErr, threadRejectionErr };
|
|
310
327
|
}
|
|
311
328
|
|
|
329
|
+
/** A container that did not accept the connection
|
|
330
|
+
* (docs/reference/specs/resident-repos.md item 68; execution.md item 28): the
|
|
331
|
+
* platform's accept refusal met at the exec choke point's SPAWN — nothing
|
|
332
|
+
* ran, the worktree is as it was — named with the wait token the client
|
|
333
|
+
* re-sends on, as a 503 like the mirror held. `err` is the DO's typed error
|
|
334
|
+
* (`SandboxRuntimeBusyError`, its message starting with the token), or the
|
|
335
|
+
* same error after the stub boundary. */
|
|
336
|
+
export function runtimeBusyErr(err: Error): ThreadErr {
|
|
337
|
+
return { error: err.message, status: 503, reason: RUNTIME_BUSY_REASON, cause: "system" };
|
|
338
|
+
}
|
|
339
|
+
|
|
312
340
|
/** /exec's failure document, in the item-3 dual shape (`error` beside
|
|
313
341
|
* `stdout: ""`, `stderr`, `exitCode: 127`) so old and new executors both
|
|
314
342
|
* render it. The ThreadErr's fields ride beside the words, as the JSON routes
|
|
@@ -328,6 +356,7 @@ export function execFailureDocument(failure: ThreadErr): object {
|
|
|
328
356
|
...(failure.state ? { state: failure.state } : {}),
|
|
329
357
|
...(typeof failure.stateReason === "string" ? { stateReason: failure.stateReason } : {}),
|
|
330
358
|
...(failure.reason ? { reason: failure.reason } : {}),
|
|
359
|
+
...(failure.cause ? { cause: failure.cause } : {}),
|
|
331
360
|
status: failure.status,
|
|
332
361
|
...(typeof failure.transient === "boolean" ? { transient: failure.transient } : {}),
|
|
333
362
|
stdout: "",
|