@coreplane/switchboard 1.267.3 → 1.268.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/deploy/cloudflare-memory/worker.ts +23 -1
- package/dist/assets/deploy/cloudflare-resident/imageReconcileState.ts +28 -0
- package/dist/assets/deploy/cloudflare-resident/shared.ts +2 -3
- package/dist/assets/deploy/cloudflare-resident/worker.ts +162 -27
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/budgets.ts +4 -0
- package/dist/assets/src/core/coordinator/contract.ts +14 -8
- package/dist/assets/src/core/costs.ts +2 -0
- package/dist/assets/src/core/runEvents.ts +1 -0
- package/dist/assets/src/core/runLedger/types.ts +2 -0
- package/dist/assets/src/core/runRecord.ts +10 -0
- package/dist/assets/src/core/runUsage.ts +53 -0
- package/dist/assets/src/core/shipSpend.ts +52 -0
- package/dist/assets/src/execution/residentPoolSpends.ts +9 -0
- package/dist/assets/src/execution/residentRecreateAdmission.ts +28 -0
- package/dist/assets/web/dist/.vite/manifest.json +31 -31
- package/dist/assets/web/dist/assets/CostsPage-6P9rJ7k_.js +1 -0
- package/dist/assets/web/dist/assets/{HomePage-BQmun9kS.js → HomePage-B4LGwXHO.js} +1 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-jpGfsR5n.js +1 -0
- package/dist/assets/web/dist/assets/ResidentsIndexPage-C4-5nG9M.js +1 -0
- package/dist/assets/web/dist/assets/{RunFoldRow-C1w-mwch.js → RunFoldRow-u4FTh57c.js} +1 -1
- package/dist/assets/web/dist/assets/{RunRoutePage-zWMHTzla.js → RunRoutePage-CxOKb7oC.js} +4 -4
- package/dist/assets/web/dist/assets/{RunsIndexPage-DF16D2Gg.js → RunsIndexPage-CULSlv_d.js} +1 -1
- package/dist/assets/web/dist/assets/{ScheduledPage-CyRs0fXN.js → ScheduledPage-CDjkZuqM.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-BRuMCHSj.js → SettingsPage-BMSomotV.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-BIAQN7iD.js → StatusDot-XxLkmRaK.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-CxiQ7luL.js → Tooltip-qB73RL4b.js} +1 -1
- package/dist/assets/web/dist/assets/{UnitRoutePage-BQB5mAAl.js → UnitRoutePage-DFRWAdOT.js} +1 -1
- package/dist/assets/web/dist/assets/{dist-I2OEmPqz.js → dist-BsLBAkQJ.js} +1 -1
- package/dist/assets/web/dist/assets/{main-DYwKyLY5.js → main-9ynpo780.js} +10 -10
- package/dist/assets/web/dist/assets/main-DSiJ9sHU.css +1 -0
- package/dist/cli.js +385 -226
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/CostsPage-RRggVvIx.js +0 -1
- package/dist/assets/web/dist/assets/ResidentDetailPage-CT6v9INM.js +0 -1
- package/dist/assets/web/dist/assets/ResidentsIndexPage-CWqPFf40.js +0 -1
- package/dist/assets/web/dist/assets/main-DPB0UO1-.css +0 -1
|
@@ -4131,6 +4131,9 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
4131
4131
|
userId: row.user_id,
|
|
4132
4132
|
...(who.userName ? { userName: who.userName } : {}),
|
|
4133
4133
|
...(who.parentRunId ? { parentRunId: who.parentRunId } : {}),
|
|
4134
|
+
...(who.parentInstanceId ? { parentInstanceId: who.parentInstanceId } : {}),
|
|
4135
|
+
...(who.idempotencyKey ? { idempotencyKey: who.idempotencyKey } : {}),
|
|
4136
|
+
...(who.costCapUsd !== undefined ? { costCapUsd: who.costCapUsd } : {}),
|
|
4134
4137
|
threadKey: row.thread_key,
|
|
4135
4138
|
channelId: row.channel_id,
|
|
4136
4139
|
...(row.agent ? { agent: row.agent } : {}),
|
|
@@ -4379,12 +4382,23 @@ function parseUsageJson(raw: string | null): RunUsage | undefined {
|
|
|
4379
4382
|
}
|
|
4380
4383
|
|
|
4381
4384
|
/** The two identity fields the by-user aggregate needs off a summary, read leniently. */
|
|
4382
|
-
function identityOfSummary(raw: string): {
|
|
4385
|
+
function identityOfSummary(raw: string): {
|
|
4386
|
+
userName?: string;
|
|
4387
|
+
parentRunId?: string;
|
|
4388
|
+
parentInstanceId?: string;
|
|
4389
|
+
idempotencyKey?: string;
|
|
4390
|
+
costCapUsd?: number;
|
|
4391
|
+
} {
|
|
4383
4392
|
try {
|
|
4384
4393
|
const s = JSON.parse(raw) as Record<string, unknown>;
|
|
4385
4394
|
return {
|
|
4386
4395
|
...(typeof s.userName === "string" && s.userName ? { userName: s.userName } : {}),
|
|
4387
4396
|
...(typeof s.parentRunId === "string" && s.parentRunId ? { parentRunId: s.parentRunId } : {}),
|
|
4397
|
+
...(typeof s.parentInstanceId === "string" && s.parentInstanceId ? { parentInstanceId: s.parentInstanceId } : {}),
|
|
4398
|
+
...(typeof s.idempotencyKey === "string" && s.idempotencyKey ? { idempotencyKey: s.idempotencyKey } : {}),
|
|
4399
|
+
...(typeof s.costCapUsd === "number" && Number.isFinite(s.costCapUsd) && s.costCapUsd > 0
|
|
4400
|
+
? { costCapUsd: s.costCapUsd }
|
|
4401
|
+
: {}),
|
|
4388
4402
|
};
|
|
4389
4403
|
} catch {
|
|
4390
4404
|
return {};
|
|
@@ -5689,6 +5703,14 @@ function parseClaim(b: Record<string, unknown>): Validated<ClaimRequest> {
|
|
|
5689
5703
|
if (typeof meta.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(meta.idempotencyKey))
|
|
5690
5704
|
return invalid("run.meta.idempotencyKey must be <parentInstanceId>:<step>");
|
|
5691
5705
|
}
|
|
5706
|
+
if (
|
|
5707
|
+
meta.costCapUsd !== undefined &&
|
|
5708
|
+
(meta.parentInstanceId === undefined ||
|
|
5709
|
+
typeof meta.costCapUsd !== "number" ||
|
|
5710
|
+
!Number.isFinite(meta.costCapUsd) ||
|
|
5711
|
+
meta.costCapUsd <= 0)
|
|
5712
|
+
)
|
|
5713
|
+
return invalid("run.meta.costCapUsd must be a positive coordinator cap");
|
|
5692
5714
|
// The restart tag (record 0064) names the predecessor run whose windows the
|
|
5693
5715
|
// claim reuses and rides the `child-resumed` event's reason: a run id or
|
|
5694
5716
|
// absent, never another shape.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** One deploy's durable progress for a resident container. A repeated
|
|
2
|
+
* reconcile may retry a deferred stop or collect an inactive container's
|
|
3
|
+
* report, but must not stop a replacement that is still hydrating. */
|
|
4
|
+
export interface DeployImageReconcileState {
|
|
5
|
+
build: string;
|
|
6
|
+
cycleIssued: boolean;
|
|
7
|
+
cycleStoppedAt?: string;
|
|
8
|
+
containerBeforeCycle?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type DeployImageReconcileAction = "start" | "retry-cycle" | "await-report" | "verified";
|
|
12
|
+
|
|
13
|
+
export function nextDeployImageReconcile(
|
|
14
|
+
build: string,
|
|
15
|
+
stored: unknown,
|
|
16
|
+
reportPending: boolean,
|
|
17
|
+
): DeployImageReconcileAction {
|
|
18
|
+
if (!stored || typeof stored !== "object") return "start";
|
|
19
|
+
const state = stored as Record<string, unknown>;
|
|
20
|
+
if (state.build !== build || typeof state.cycleIssued !== "boolean") return "start";
|
|
21
|
+
if (!reportPending) return "verified";
|
|
22
|
+
return state.cycleIssued ? "await-report" : "retry-cycle";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** A fresh boot after the requested stop can finish hydration and report. */
|
|
26
|
+
export function replacementContainerStarted(before: string | undefined, current: string | undefined): boolean {
|
|
27
|
+
return before !== undefined && current !== undefined && before !== current;
|
|
28
|
+
}
|
|
@@ -30,9 +30,8 @@ export const REFRESH_INTERVAL_S = 600;
|
|
|
30
30
|
* ten-minute buckets since the last instance. */
|
|
31
31
|
export const IDLE_REFRESH_INTERVAL_S = 6 * 60 * 60;
|
|
32
32
|
|
|
33
|
-
/** The
|
|
34
|
-
*
|
|
35
|
-
* step can have to check. */
|
|
33
|
+
/** The image's `worker2`..`worker17` pool: 16 one-use identities per VM
|
|
34
|
+
* generation for thread owners and disposable operations. */
|
|
36
35
|
export const THREAD_POOL_SIZE = 16;
|
|
37
36
|
|
|
38
37
|
/** Exec budgets. Each is a refresh step's own budget too (refresh.ts), so a
|
|
@@ -89,6 +89,11 @@ import {
|
|
|
89
89
|
} from "./legacyCredentials.js";
|
|
90
90
|
import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
|
|
91
91
|
import { KeyedAsyncLock } from "./keyedAsyncLock.js";
|
|
92
|
+
import {
|
|
93
|
+
nextDeployImageReconcile,
|
|
94
|
+
replacementContainerStarted,
|
|
95
|
+
type DeployImageReconcileState,
|
|
96
|
+
} from "./imageReconcileState.js";
|
|
92
97
|
import {
|
|
93
98
|
registeredRunAllowsClaim,
|
|
94
99
|
registeredRunAllowsReattach,
|
|
@@ -179,15 +184,17 @@ import { shellQuote } from "../../src/execution/shellQuote.js";
|
|
|
179
184
|
import { envFromRequest } from "../../src/execution/sandboxEnv.js";
|
|
180
185
|
import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
|
|
181
186
|
import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
|
|
182
|
-
import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
|
|
187
|
+
import { ResidentRecreateAdmission, idleForPoolRecycle } from "../../src/execution/residentRecreateAdmission.js";
|
|
183
188
|
import {
|
|
184
189
|
claimPoolBinding,
|
|
185
190
|
mayRunAsPoolUser,
|
|
191
|
+
ownedPoolUsers,
|
|
186
192
|
parsePoolBindings,
|
|
187
193
|
parseSpentPoolUsers,
|
|
188
194
|
rebuildPoolBindingIndex,
|
|
189
195
|
releasePoolBinding,
|
|
190
196
|
spendPoolUser,
|
|
197
|
+
unavailablePoolUsers,
|
|
191
198
|
} from "../../src/execution/residentPoolSpends.js";
|
|
192
199
|
import {
|
|
193
200
|
recordFiring,
|
|
@@ -591,16 +598,10 @@ const THREADS_DIR = "/workspace/threads";
|
|
|
591
598
|
* at the latest (disk is cache). */
|
|
592
599
|
const OPS_DIR = "/workspace/ops";
|
|
593
600
|
|
|
594
|
-
/**
|
|
595
|
-
*
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
/** Pool of OS users for thread worktrees (worker1 is the build user). Sized
|
|
599
|
-
* for SIMULTANEOUS runs, not for every thread ever seen: a run returns its
|
|
600
|
-
* user via /detach when it ends, so the pool only fills when 16 runs on one
|
|
601
|
-
* repo are genuinely concurrent. Memory, not this list, is the real ceiling
|
|
602
|
-
* — see the instance_type note in wrangler.jsonc. Must match the useradd loop
|
|
603
|
-
* in the Dockerfile. */
|
|
601
|
+
/** Pool of OS users for thread worktrees and disposable ops. A UID stays with
|
|
602
|
+
* its first owner for this VM generation; detach releases the live binding,
|
|
603
|
+
* not the UID spend. Exhaustion may recycle a proven-idle VM. Must match the
|
|
604
|
+
* useradd loop in the Dockerfile. */
|
|
604
605
|
const THREAD_USERS = Array.from({ length: THREAD_POOL_SIZE }, (_, i) => `worker${i + 2}`);
|
|
605
606
|
|
|
606
607
|
/** Force-detach: after killing the thread user's processes, how long
|
|
@@ -1296,7 +1297,7 @@ interface AttachOk {
|
|
|
1296
1297
|
returned?: Returned;
|
|
1297
1298
|
}
|
|
1298
1299
|
|
|
1299
|
-
/** What `POST /detach` answers: whether the
|
|
1300
|
+
/** What `POST /detach` answers: whether the live binding was released, why not, and
|
|
1300
1301
|
* — on a release — what the tree still held (item 16a), now gone with it. */
|
|
1301
1302
|
interface DetachAnswer {
|
|
1302
1303
|
released: boolean;
|
|
@@ -1478,6 +1479,7 @@ const DRAIN_KEY = "drain";
|
|
|
1478
1479
|
* verify this resident's container on the new image; the next reconcile that
|
|
1479
1480
|
* finds it current reports to the registry and clears it. */
|
|
1480
1481
|
const IMAGE_REPORT_PENDING_KEY = "imageReportPending";
|
|
1482
|
+
const IMAGE_RECONCILE_KEY = "imageReconcile";
|
|
1481
1483
|
|
|
1482
1484
|
type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
|
|
1483
1485
|
|
|
@@ -3910,8 +3912,8 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
3910
3912
|
await this.ctx.storage.delete(INFRA_STREAK_KEY);
|
|
3911
3913
|
await this.setResidentState("warm");
|
|
3912
3914
|
// Event-triggered reclamation: the prune above already told the
|
|
3913
|
-
// mirror which branches died; finished refs
|
|
3914
|
-
//
|
|
3915
|
+
// mirror which branches died; finished refs release their worktree and
|
|
3916
|
+
// live binding now, not at the idle TTL. Housekeeping, never a
|
|
3915
3917
|
// lifecycle flip — a failure here is a log line.
|
|
3916
3918
|
try {
|
|
3917
3919
|
const gc = await this.reclaimFinishedRefs(resource, facts.defaultRef, cycle.token);
|
|
@@ -5250,7 +5252,12 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5250
5252
|
}
|
|
5251
5253
|
console.log(`image-stale (${where}): ${stale} — stopping so it restarts on the current image`);
|
|
5252
5254
|
this.swapIncarnation(); // deliberate incarnation swap
|
|
5253
|
-
|
|
5255
|
+
try {
|
|
5256
|
+
await this.stop();
|
|
5257
|
+
} catch (err) {
|
|
5258
|
+
console.log(`image-stale: stop failed: ${errMsg(err)}`);
|
|
5259
|
+
if (where === "deploy") return "deferred";
|
|
5260
|
+
}
|
|
5254
5261
|
// A successful stop proves only that the old process is gone. The pending
|
|
5255
5262
|
// marker deliberately survives it: the replacement's own hydration reports
|
|
5256
5263
|
// after reaching `warm`, so `imageReport: current` can never precede a
|
|
@@ -5306,9 +5313,66 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5306
5313
|
* added afterwards would wait for a report nothing sends (until the drain's
|
|
5307
5314
|
* `until`). Held first, every report finds its hold. */
|
|
5308
5315
|
async reconcileForDeploy(resource: string): Promise<{ result: ImageReconcileResult; verified: boolean }> {
|
|
5309
|
-
|
|
5310
|
-
|
|
5316
|
+
// The deploy's image report and an idle pool recycle both reason about the
|
|
5317
|
+
// same VM. Admit reconciliation as disk work before its first await, so a
|
|
5318
|
+
// recycle sees it as busy; a recycle already holding admission defers this
|
|
5319
|
+
// pass for the deploy's bounded readiness retry.
|
|
5320
|
+
const admitted = await this.withRecreateSafeAdmin(() => this.reconcileForDeployAdmitted(resource));
|
|
5321
|
+
return "error" in admitted ? { result: "deferred", verified: false } : admitted;
|
|
5322
|
+
}
|
|
5323
|
+
|
|
5324
|
+
private async reconcileForDeployAdmitted(
|
|
5325
|
+
resource: string,
|
|
5326
|
+
): Promise<{ result: ImageReconcileResult; verified: boolean }> {
|
|
5327
|
+
const progress = await this.ctx.storage.get<DeployImageReconcileState>(IMAGE_RECONCILE_KEY);
|
|
5328
|
+
const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
|
|
5329
|
+
const action = nextDeployImageReconcile(BUILD.commit, progress, pending !== undefined);
|
|
5330
|
+
if (action === "verified") return { result: "current", verified: true };
|
|
5331
|
+
if (action === "start") {
|
|
5332
|
+
await registryStub(this.env).holdDrainFor([resource]);
|
|
5333
|
+
await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
|
|
5334
|
+
await this.ctx.storage.put(IMAGE_RECONCILE_KEY, { build: BUILD.commit, cycleIssued: false });
|
|
5335
|
+
}
|
|
5336
|
+
if (action === "await-report") {
|
|
5337
|
+
const active = await this.isRuntimeActive().catch(() => null);
|
|
5338
|
+
if (active === false) await this.reportPendingImageCurrent("deploy");
|
|
5339
|
+
else if (active === true) {
|
|
5340
|
+
const [state, facts] = await Promise.all([
|
|
5341
|
+
this.ctx.storage.get<ResidentState>(STATE_KEY),
|
|
5342
|
+
this.ctx.storage.get<RepoFacts>(FACTS_KEY),
|
|
5343
|
+
]);
|
|
5344
|
+
// A replacement can reuse the disk and leave `warm` unchanged; that
|
|
5345
|
+
// hydrate shortcut does not send a report. A different boot after the
|
|
5346
|
+
// successful stop is proof that a replacement started. Finish its
|
|
5347
|
+
// hydration before reporting; an old warm state alone proves nothing.
|
|
5348
|
+
if (replacementContainerStarted(progress?.containerBeforeCycle, await this.containerIdentity())) {
|
|
5349
|
+
await this.ensureHydrated();
|
|
5350
|
+
await this.reportPendingImageCurrent("deploy");
|
|
5351
|
+
} else if (
|
|
5352
|
+
state === "warm" &&
|
|
5353
|
+
facts?.lastRestore?.at &&
|
|
5354
|
+
Date.parse(facts.lastRestore.at) >= Date.parse(progress?.cycleStoppedAt ?? "")
|
|
5355
|
+
) {
|
|
5356
|
+
await this.reportPendingImageCurrent("deploy");
|
|
5357
|
+
}
|
|
5358
|
+
}
|
|
5359
|
+
return {
|
|
5360
|
+
result: active === false ? "inactive" : "deferred",
|
|
5361
|
+
verified: (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) === undefined,
|
|
5362
|
+
};
|
|
5363
|
+
}
|
|
5364
|
+
// Reading the boot ID runs a command and would wake a sleeping resident.
|
|
5365
|
+
// Preserve the inactive path's report-without-start behavior.
|
|
5366
|
+
const activeBeforeCycle = await this.isRuntimeActive().catch(() => null);
|
|
5367
|
+
const containerBeforeCycle = activeBeforeCycle === true ? await this.containerIdentity() : undefined;
|
|
5311
5368
|
const result = await this.reconcileImage("deploy", true);
|
|
5369
|
+
if (result === "restarted")
|
|
5370
|
+
await this.ctx.storage.put(IMAGE_RECONCILE_KEY, {
|
|
5371
|
+
build: BUILD.commit,
|
|
5372
|
+
cycleIssued: true,
|
|
5373
|
+
cycleStoppedAt: new Date(systemClock()).toISOString(),
|
|
5374
|
+
containerBeforeCycle,
|
|
5375
|
+
} satisfies DeployImageReconcileState);
|
|
5312
5376
|
if (result === "deferred") return { result, verified: false };
|
|
5313
5377
|
// Inactivity reports synchronously; a successful stop does not. A
|
|
5314
5378
|
// concurrent fresh start may also have hydrated while this reconcile
|
|
@@ -5615,7 +5679,11 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5615
5679
|
/** Storage bindings, active ops and inspections all reserve a pool user.
|
|
5616
5680
|
* Claim inside this method, before its promise resolves: two concurrent
|
|
5617
5681
|
* callers must not each read the same free UID before either can claim it. */
|
|
5618
|
-
private async findFreePoolUser(
|
|
5682
|
+
private async findFreePoolUser(
|
|
5683
|
+
excludeThreadKey?: string,
|
|
5684
|
+
excluded = new Set<string>(),
|
|
5685
|
+
preferred: readonly string[] = [],
|
|
5686
|
+
): Promise<string | undefined> {
|
|
5619
5687
|
const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
|
|
5620
5688
|
const used = new Set(
|
|
5621
5689
|
[...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
|
|
@@ -5623,7 +5691,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5623
5691
|
for (const u of this.opUsersInUse.keys()) used.add(u);
|
|
5624
5692
|
for (const u of this.poolUsersInspecting) used.add(u);
|
|
5625
5693
|
for (const u of excluded) used.add(u);
|
|
5626
|
-
const user = THREAD_USERS.find((u) => !used.has(u));
|
|
5694
|
+
const user = [...preferred, ...THREAD_USERS].find((u) => !used.has(u));
|
|
5627
5695
|
if (user) this.poolUsersInspecting.add(user);
|
|
5628
5696
|
return user;
|
|
5629
5697
|
}
|
|
@@ -5660,19 +5728,82 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
5660
5728
|
throw new Error("pool-user-stage-scan-invalid");
|
|
5661
5729
|
}
|
|
5662
5730
|
|
|
5731
|
+
/** A full one-use pool can serve the next request by destroying its VM,
|
|
5732
|
+
* never by assigning another owner's UID. The admission mark closes every
|
|
5733
|
+
* other entry before the idle read and stays held through the restore. */
|
|
5734
|
+
private async recycleSpentPoolForAdmission(): Promise<true | ThreadErr> {
|
|
5735
|
+
try {
|
|
5736
|
+
const admission = await this.recreateAdmission.run(async (): Promise<true | ThreadErr> => {
|
|
5737
|
+
const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5738
|
+
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5739
|
+
if (spent.size < THREAD_USERS.length) return true; // another checked recreate already cleared the generation
|
|
5740
|
+
const [status, drain, imagePending, snapshot, registeredRuns, bindings] = await Promise.all([
|
|
5741
|
+
this.getStatus(),
|
|
5742
|
+
this.registry().getDrain(),
|
|
5743
|
+
this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY),
|
|
5744
|
+
this.ctx.storage.get(SNAPSHOT_KEY),
|
|
5745
|
+
this.registeredRunsBeyondOps(),
|
|
5746
|
+
this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX }),
|
|
5747
|
+
]);
|
|
5748
|
+
if (
|
|
5749
|
+
!snapshot ||
|
|
5750
|
+
!idleForPoolRecycle({
|
|
5751
|
+
state: status.state,
|
|
5752
|
+
draining: liveDrain(drain, systemClock()) !== null,
|
|
5753
|
+
imagePending: imagePending !== undefined,
|
|
5754
|
+
inFlight: this.inFlightCount(),
|
|
5755
|
+
refreshAdmissions: this.refreshAdmissionsInFlight,
|
|
5756
|
+
adminWork: this.adminWorkInFlight,
|
|
5757
|
+
hydrating: this.hydration !== null,
|
|
5758
|
+
registeredRuns,
|
|
5759
|
+
liveBindings: [...bindings.values()].filter((binding) => !binding.evicted).length,
|
|
5760
|
+
inspecting: this.poolUsersInspecting.size,
|
|
5761
|
+
})
|
|
5762
|
+
)
|
|
5763
|
+
return {
|
|
5764
|
+
error: "pool-recycle-required: all UIDs spent and the resident is not idle for checked VM recycle",
|
|
5765
|
+
status: 503,
|
|
5766
|
+
};
|
|
5767
|
+
await this.recreateContainer("pool-recycle: all UIDs spent; no other resident work owns this VM");
|
|
5768
|
+
await this.ensureHydrated();
|
|
5769
|
+
const fresh = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5770
|
+
const ready = await this.getStatus();
|
|
5771
|
+
const imageStillPending = (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) !== undefined;
|
|
5772
|
+
if (!fresh || fresh.size !== 0 || ready.state !== "warm" || imageStillPending)
|
|
5773
|
+
return { error: "pool-recycle-failed: the restored VM is not warm, current and empty", status: 503 };
|
|
5774
|
+
return true;
|
|
5775
|
+
});
|
|
5776
|
+
return admission.busy ? this.recreateRefusal() : admission.value;
|
|
5777
|
+
} catch (err) {
|
|
5778
|
+
console.log(`pool-recycle: checked VM recycle failed (${errMsg(err)})`);
|
|
5779
|
+
return { error: "pool-recycle-failed: the checked VM restore did not complete", status: 503 };
|
|
5780
|
+
}
|
|
5781
|
+
}
|
|
5782
|
+
|
|
5663
5783
|
private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
|
|
5664
|
-
|
|
5784
|
+
let spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5665
5785
|
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5666
|
-
|
|
5786
|
+
let excluded = unavailablePoolUsers(spent, owner);
|
|
5667
5787
|
let contaminated = false;
|
|
5788
|
+
let recycled = false;
|
|
5668
5789
|
for (;;) {
|
|
5669
|
-
const user = await this.findFreePoolUser(excludeThreadKey, excluded);
|
|
5670
|
-
if (!user)
|
|
5790
|
+
const user = await this.findFreePoolUser(excludeThreadKey, excluded, ownedPoolUsers(spent, owner));
|
|
5791
|
+
if (!user) {
|
|
5792
|
+
if (!contaminated && excluded.size === THREAD_USERS.length && !recycled) {
|
|
5793
|
+
recycled = true;
|
|
5794
|
+
const result = await this.recycleSpentPoolForAdmission();
|
|
5795
|
+
if (result !== true) return result;
|
|
5796
|
+
spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
5797
|
+
if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
|
|
5798
|
+
excluded = unavailablePoolUsers(spent, owner);
|
|
5799
|
+
continue;
|
|
5800
|
+
}
|
|
5671
5801
|
return contaminated
|
|
5672
5802
|
? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
|
|
5673
5803
|
: excluded.size === THREAD_USERS.length
|
|
5674
5804
|
? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
|
|
5675
5805
|
: { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
|
|
5806
|
+
}
|
|
5676
5807
|
let safe = false;
|
|
5677
5808
|
try {
|
|
5678
5809
|
const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
|
|
@@ -7803,9 +7934,9 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
7803
7934
|
return "evicted";
|
|
7804
7935
|
}
|
|
7805
7936
|
|
|
7806
|
-
/** POST /detach: a run has ended —
|
|
7807
|
-
*
|
|
7808
|
-
*
|
|
7937
|
+
/** POST /detach: a run has ended — release its live binding now instead of
|
|
7938
|
+
* holding it until the TTL sweep. The UID remains spent for this VM
|
|
7939
|
+
* generation. A run starts from a clean tree (item 17), so the tree
|
|
7809
7940
|
* goes whatever it holds: what it held — uncommitted changes, unpushed
|
|
7810
7941
|
* commits — is measured once, as the thread user, and named in the answer
|
|
7811
7942
|
* (`leftBehind`) so the loss is never silent. The one thing that keeps a
|
|
@@ -8787,6 +8918,7 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8787
8918
|
// The runs whose process lives in the container between operator calls
|
|
8788
8919
|
// (item 44): what the deploy preflight must see, added to both counts.
|
|
8789
8920
|
const registeredRuns = await this.registeredRunsBeyondOps();
|
|
8921
|
+
const poolSpends = parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS);
|
|
8790
8922
|
return {
|
|
8791
8923
|
resource: map.get(RESOURCE_KEY) ?? null,
|
|
8792
8924
|
state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
|
|
@@ -8818,7 +8950,10 @@ export class ResidentDO extends Sandbox<Env> {
|
|
|
8818
8950
|
// running container the last deploy could not verify.
|
|
8819
8951
|
imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
|
|
8820
8952
|
recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
|
|
8821
|
-
poolUsersSpent:
|
|
8953
|
+
poolUsersSpent: poolSpends?.size ?? null,
|
|
8954
|
+
// Read-scoped owner receipts explain historical occupancy even after
|
|
8955
|
+
// every live binding and disposable op has ended.
|
|
8956
|
+
poolUserSpends: poolSpends ? [...poolSpends].map(([user, owner]) => ({ user, owner })) : null,
|
|
8822
8957
|
// Item 22: who holds what, as the rows say — the mirror mutex and the
|
|
8823
8958
|
// cycle/hydration leases, each judged against this incarnation.
|
|
8824
8959
|
incarnation: this.incarnation,
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "switchboard",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.268.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "switchboard",
|
|
9
|
-
"version": "1.
|
|
9
|
+
"version": "1.268.0",
|
|
10
10
|
"license": "Apache-2.0",
|
|
11
11
|
"workspaces": [
|
|
12
12
|
"web",
|
|
@@ -21023,7 +21023,7 @@
|
|
|
21023
21023
|
},
|
|
21024
21024
|
"packages/switchboard": {
|
|
21025
21025
|
"name": "@coreplane/switchboard",
|
|
21026
|
-
"version": "1.
|
|
21026
|
+
"version": "1.268.0",
|
|
21027
21027
|
"license": "Apache-2.0",
|
|
21028
21028
|
"dependencies": {
|
|
21029
21029
|
"@earendil-works/pi-ai": "0.85.1",
|
package/dist/assets/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "switchboard",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.268.0",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
|
|
6
6
|
"license": "Apache-2.0",
|
package/dist/assets/source.json
CHANGED
|
@@ -255,6 +255,10 @@ export const PROVIDER_RETRY_BACKOFFS_MS = [5_000, 15_000, 45_000] as const;
|
|
|
255
255
|
* model call itself remains bounded by the run's lease. */
|
|
256
256
|
export const MODEL_STREAM_HEARTBEAT_MS = 15_000;
|
|
257
257
|
|
|
258
|
+
/** A residents page re-reads the registry while open so a deploy's drain and
|
|
259
|
+
* image reports advance even when no run can attach to trigger its feed. */
|
|
260
|
+
export const RESIDENT_LISTING_POLL_MS = 30_000;
|
|
261
|
+
|
|
258
262
|
/** A hosted ship parent's deadline margin past the pipeline's wall clock
|
|
259
263
|
* (record 0060): the row's `state.hosting.until` is the hand-off time plus
|
|
260
264
|
* the instance's `caps.maxMinutes` plus this hour, absorbing the runner's own
|
|
@@ -307,12 +307,13 @@ export interface ExistingPrPublicationBinding {
|
|
|
307
307
|
owner: { instanceId: string; unit: string };
|
|
308
308
|
}
|
|
309
309
|
|
|
310
|
-
/** What a coordinator's spawn stamps on the child's every row:
|
|
311
|
-
*
|
|
312
|
-
* own post-step, the base its pull request targets. */
|
|
310
|
+
/** What a coordinator's spawn stamps on the child's every row: its instance,
|
|
311
|
+
* spawn key and admitted cost cap, plus the base for its PR post-step. */
|
|
313
312
|
export interface CoordinatorTag {
|
|
314
313
|
parentInstanceId: string;
|
|
315
314
|
idempotencyKey: string;
|
|
315
|
+
/** The unit's original dollar limit, fixed at admission; absent means no cost cap. */
|
|
316
|
+
costCapUsd?: number;
|
|
316
317
|
/** The unit's durable write branch, read from its row at child admission. */
|
|
317
318
|
branch?: string;
|
|
318
319
|
/** Workflow transport for a recovered child. Identity and idempotency remain
|
|
@@ -346,15 +347,20 @@ export interface CoordinatorTag {
|
|
|
346
347
|
publication?: ExistingPrPublicationBinding;
|
|
347
348
|
}
|
|
348
349
|
|
|
349
|
-
/** The tag
|
|
350
|
-
*
|
|
351
|
-
* key. The base never rides here: rows and records keep the shape written
|
|
352
|
-
* before it existed. */
|
|
350
|
+
/** The tag's identity and optional cap as flat record fields, or nothing — so
|
|
351
|
+
* a row, summary and record agree. The base never rides here. */
|
|
353
352
|
export function coordinatorFields(tag: CoordinatorTag | undefined): {
|
|
354
353
|
parentInstanceId?: string;
|
|
355
354
|
idempotencyKey?: string;
|
|
355
|
+
costCapUsd?: number;
|
|
356
356
|
} {
|
|
357
|
-
return tag
|
|
357
|
+
return tag
|
|
358
|
+
? {
|
|
359
|
+
parentInstanceId: tag.parentInstanceId,
|
|
360
|
+
idempotencyKey: tag.idempotencyKey,
|
|
361
|
+
...(tag.costCapUsd !== undefined ? { costCapUsd: tag.costCapUsd } : {}),
|
|
362
|
+
}
|
|
363
|
+
: {};
|
|
358
364
|
}
|
|
359
365
|
|
|
360
366
|
/** The parent ship record: what the bot writes at an instance's creation and
|
|
@@ -516,6 +516,8 @@ export interface CostReport {
|
|
|
516
516
|
/** Each biller's daily invoice tie-out (item 4d); absent when no provider block is known
|
|
517
517
|
* (a report built without the wiring) or run history is off. */
|
|
518
518
|
billers?: BillerTieOut[];
|
|
519
|
+
/** Ship children across this bot's runs, independent of the Cloudflare group. */
|
|
520
|
+
shipSpend?: import("./shipSpend.js").ShipSpendReport;
|
|
519
521
|
}
|
|
520
522
|
|
|
521
523
|
/** What the report carries beyond the priced rows: the account behind the
|
|
@@ -974,6 +974,7 @@ export type RunEvent =
|
|
|
974
974
|
type: "coordinator_tag";
|
|
975
975
|
parentInstanceId: string;
|
|
976
976
|
unit?: string;
|
|
977
|
+
costCapUsd?: number;
|
|
977
978
|
branch?: string;
|
|
978
979
|
transportWorkflowId?: string;
|
|
979
980
|
recovery?: import("./coordinator/contract.js").CoordinatorTag["recovery"];
|
|
@@ -99,6 +99,8 @@ export interface LiveRunMeta {
|
|
|
99
99
|
* still sends the parent its event and a retried spawn finds its run. */
|
|
100
100
|
parentInstanceId?: string;
|
|
101
101
|
idempotencyKey?: string;
|
|
102
|
+
/** The original Ship unit cap carried by the claim, available before its tag event is retained. */
|
|
103
|
+
costCapUsd?: number;
|
|
102
104
|
/** Where the run's conversation started (item 52), so a reclaimed run's
|
|
103
105
|
* record still says so: `parent` for a spawned child, `channel` otherwise. */
|
|
104
106
|
seed?: RunSeed;
|
|
@@ -227,6 +227,8 @@ export interface RunRecord {
|
|
|
227
227
|
* item 48), stored at the claim so a retried spawn finds its run. Present
|
|
228
228
|
* exactly when `parentInstanceId` is — both or neither, never one alone. */
|
|
229
229
|
idempotencyKey?: string;
|
|
230
|
+
/** The coordinator unit's original cost cap, carried by each child. */
|
|
231
|
+
costCapUsd?: number;
|
|
230
232
|
/** The plan runner instance this run's hand-off created (record 0051 R2;
|
|
231
233
|
* item 2): the last `ship_handoff` event, folded at the assembly like the
|
|
232
234
|
* coordinator tag. Present only on a ship run whose hand-off succeeded;
|
|
@@ -1195,6 +1197,14 @@ export function isRunRecord(v: unknown): v is RunRecord {
|
|
|
1195
1197
|
(typeof r.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(r.idempotencyKey))
|
|
1196
1198
|
)
|
|
1197
1199
|
return false;
|
|
1200
|
+
if (
|
|
1201
|
+
r.costCapUsd !== undefined &&
|
|
1202
|
+
(r.parentInstanceId === undefined ||
|
|
1203
|
+
typeof r.costCapUsd !== "number" ||
|
|
1204
|
+
!Number.isFinite(r.costCapUsd) ||
|
|
1205
|
+
r.costCapUsd <= 0)
|
|
1206
|
+
)
|
|
1207
|
+
return false;
|
|
1198
1208
|
// The instance a ship run's hand-off created (record 0051 R2; item 2).
|
|
1199
1209
|
if (r.instanceId !== undefined && (typeof r.instanceId !== "string" || !INSTANCE_ID_PATTERN.test(r.instanceId)))
|
|
1200
1210
|
return false;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { RunEvent } from "./runEvents.js";
|
|
2
|
+
import { unitOfIdempotencyKey } from "./coordinator/contract.js";
|
|
2
3
|
|
|
3
4
|
// What a run cost in tokens, and who it belongs to — the data behind the cost
|
|
4
5
|
// dimensions of the costs page (docs/reference/specs/costs.md items 10–10a) and
|
|
@@ -200,6 +201,9 @@ export interface UsageRun {
|
|
|
200
201
|
userName?: string;
|
|
201
202
|
/** A child run is billed to whoever started its parent (run-history item 46). */
|
|
202
203
|
parentRunId?: string;
|
|
204
|
+
parentInstanceId?: string;
|
|
205
|
+
idempotencyKey?: string;
|
|
206
|
+
costCapUsd?: number;
|
|
203
207
|
/** The thread the run ran in and the channel it belongs to (platform-namespaced, invariant 4). */
|
|
204
208
|
threadKey: string;
|
|
205
209
|
channelId: string;
|
|
@@ -258,6 +262,8 @@ export interface UsageRow {
|
|
|
258
262
|
|
|
259
263
|
export interface RunUsageReport {
|
|
260
264
|
rows: UsageRow[];
|
|
265
|
+
/** Individual Ship children retained for the unit-spend trend; old snapshots omit this. */
|
|
266
|
+
shipRuns?: ShipRunUsage[];
|
|
261
267
|
/** Runs in range whose usage is not known yet (written before the field; backfill outstanding). */
|
|
262
268
|
pending: number;
|
|
263
269
|
/** The oldest finish the store still holds, so a page can bound its range to the data. */
|
|
@@ -265,6 +271,33 @@ export interface RunUsageReport {
|
|
|
265
271
|
retentionDays: number;
|
|
266
272
|
}
|
|
267
273
|
|
|
274
|
+
export interface ShipRunUsage {
|
|
275
|
+
id: string;
|
|
276
|
+
unitKey: string;
|
|
277
|
+
finishedAt: number;
|
|
278
|
+
costCapUsd?: number;
|
|
279
|
+
/** Missing while an older record is still being backfilled. */
|
|
280
|
+
usage?: RunUsage;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** The unit identity comes from the coordinator's durable key, never a thread or branch guess. */
|
|
284
|
+
export function shipRunsOf(runs: readonly UsageRun[]): ShipRunUsage[] {
|
|
285
|
+
return runs.flatMap((run) => {
|
|
286
|
+
if (!run.parentInstanceId || !run.idempotencyKey) return [];
|
|
287
|
+
const unit = unitOfIdempotencyKey(run.idempotencyKey);
|
|
288
|
+
if (!unit) return [];
|
|
289
|
+
return [
|
|
290
|
+
{
|
|
291
|
+
id: run.id,
|
|
292
|
+
unitKey: `${run.parentInstanceId}:${unit}`,
|
|
293
|
+
finishedAt: run.finishedAt,
|
|
294
|
+
...(run.costCapUsd !== undefined ? { costCapUsd: run.costCapUsd } : {}),
|
|
295
|
+
...(run.usage !== undefined ? { usage: run.usage } : {}),
|
|
296
|
+
},
|
|
297
|
+
];
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
268
301
|
export const dayOf = (epochMs: number): string => new Date(epochMs).toISOString().slice(0, 10);
|
|
269
302
|
|
|
270
303
|
const identityOf = (who: UsageIdentity): UsageIdentity => ({
|
|
@@ -334,6 +367,7 @@ export function reportOfUsageRows(rows: RunUsageRows): RunUsageReport {
|
|
|
334
367
|
const { rows: cells, pending } = aggregateUsage(rows.runs, (id) => rows.parents[id]);
|
|
335
368
|
return {
|
|
336
369
|
rows: cells,
|
|
370
|
+
shipRuns: shipRunsOf(rows.runs),
|
|
337
371
|
// The store counts what it could not price; the fold sees the same runs without `usage`.
|
|
338
372
|
pending: Math.max(pending, rows.pending),
|
|
339
373
|
...(rows.earliestFinishedAt !== undefined ? { earliestFinishedAt: rows.earliestFinishedAt } : {}),
|
|
@@ -356,6 +390,10 @@ function isUsageRun(v: unknown): v is UsageRun {
|
|
|
356
390
|
typeof r.channelId === "string" &&
|
|
357
391
|
(r.agent === undefined || typeof r.agent === "string") &&
|
|
358
392
|
(r.parentRunId === undefined || typeof r.parentRunId === "string") &&
|
|
393
|
+
(r.parentInstanceId === undefined || typeof r.parentInstanceId === "string") &&
|
|
394
|
+
(r.idempotencyKey === undefined || typeof r.idempotencyKey === "string") &&
|
|
395
|
+
(r.costCapUsd === undefined ||
|
|
396
|
+
(typeof r.costCapUsd === "number" && Number.isFinite(r.costCapUsd) && r.costCapUsd > 0)) &&
|
|
359
397
|
typeof r.startedAt === "number" &&
|
|
360
398
|
typeof r.finishedAt === "number" &&
|
|
361
399
|
(r.usage === undefined || isRunUsage(r.usage))
|
|
@@ -382,6 +420,7 @@ export function isRunUsageReport(v: unknown): v is RunUsageReport {
|
|
|
382
420
|
if (typeof v !== "object" || v === null) return false;
|
|
383
421
|
const r = v as Record<string, unknown>;
|
|
384
422
|
if (!Array.isArray(r.rows) || !hasReportTail(r)) return false;
|
|
423
|
+
if (r.shipRuns !== undefined && (!Array.isArray(r.shipRuns) || !r.shipRuns.every(isShipRunUsage))) return false;
|
|
385
424
|
return r.rows.every(
|
|
386
425
|
(row) =>
|
|
387
426
|
isIdentity(row) &&
|
|
@@ -394,3 +433,17 @@ export function isRunUsageReport(v: unknown): v is RunUsageReport {
|
|
|
394
433
|
isRunUsage((row as UsageRow).usage),
|
|
395
434
|
);
|
|
396
435
|
}
|
|
436
|
+
|
|
437
|
+
function isShipRunUsage(v: unknown): v is ShipRunUsage {
|
|
438
|
+
if (typeof v !== "object" || v === null) return false;
|
|
439
|
+
const r = v as ShipRunUsage;
|
|
440
|
+
return (
|
|
441
|
+
typeof r.id === "string" &&
|
|
442
|
+
typeof r.unitKey === "string" &&
|
|
443
|
+
typeof r.finishedAt === "number" &&
|
|
444
|
+
Number.isFinite(r.finishedAt) &&
|
|
445
|
+
(r.costCapUsd === undefined ||
|
|
446
|
+
(typeof r.costCapUsd === "number" && Number.isFinite(r.costCapUsd) && r.costCapUsd > 0)) &&
|
|
447
|
+
(r.usage === undefined || isRunUsage(r.usage))
|
|
448
|
+
);
|
|
449
|
+
}
|