@coreplane/switchboard 1.267.3 → 1.268.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/assets/deploy/cloudflare-memory/worker.ts +23 -1
  2. package/dist/assets/deploy/cloudflare-resident/imageReconcileState.ts +28 -0
  3. package/dist/assets/deploy/cloudflare-resident/shared.ts +2 -3
  4. package/dist/assets/deploy/cloudflare-resident/worker.ts +162 -27
  5. package/dist/assets/package-lock.json +3 -3
  6. package/dist/assets/package.json +1 -1
  7. package/dist/assets/source.json +3 -3
  8. package/dist/assets/src/core/budgets.ts +4 -0
  9. package/dist/assets/src/core/coordinator/contract.ts +14 -8
  10. package/dist/assets/src/core/costs.ts +2 -0
  11. package/dist/assets/src/core/runEvents.ts +1 -0
  12. package/dist/assets/src/core/runLedger/types.ts +2 -0
  13. package/dist/assets/src/core/runRecord.ts +10 -0
  14. package/dist/assets/src/core/runUsage.ts +53 -0
  15. package/dist/assets/src/core/shipSpend.ts +52 -0
  16. package/dist/assets/src/execution/residentPoolSpends.ts +9 -0
  17. package/dist/assets/src/execution/residentRecreateAdmission.ts +28 -0
  18. package/dist/assets/web/dist/.vite/manifest.json +31 -31
  19. package/dist/assets/web/dist/assets/CostsPage-6P9rJ7k_.js +1 -0
  20. package/dist/assets/web/dist/assets/{HomePage-BQmun9kS.js → HomePage-B4LGwXHO.js} +1 -1
  21. package/dist/assets/web/dist/assets/ResidentDetailPage-jpGfsR5n.js +1 -0
  22. package/dist/assets/web/dist/assets/ResidentsIndexPage-C4-5nG9M.js +1 -0
  23. package/dist/assets/web/dist/assets/{RunFoldRow-C1w-mwch.js → RunFoldRow-u4FTh57c.js} +1 -1
  24. package/dist/assets/web/dist/assets/{RunRoutePage-zWMHTzla.js → RunRoutePage-CxOKb7oC.js} +4 -4
  25. package/dist/assets/web/dist/assets/{RunsIndexPage-DF16D2Gg.js → RunsIndexPage-CULSlv_d.js} +1 -1
  26. package/dist/assets/web/dist/assets/{ScheduledPage-CyRs0fXN.js → ScheduledPage-CDjkZuqM.js} +1 -1
  27. package/dist/assets/web/dist/assets/{SettingsPage-BRuMCHSj.js → SettingsPage-BMSomotV.js} +1 -1
  28. package/dist/assets/web/dist/assets/{StatusDot-BIAQN7iD.js → StatusDot-XxLkmRaK.js} +1 -1
  29. package/dist/assets/web/dist/assets/{Tooltip-CxiQ7luL.js → Tooltip-qB73RL4b.js} +1 -1
  30. package/dist/assets/web/dist/assets/{UnitRoutePage-BQB5mAAl.js → UnitRoutePage-DFRWAdOT.js} +1 -1
  31. package/dist/assets/web/dist/assets/{dist-I2OEmPqz.js → dist-BsLBAkQJ.js} +1 -1
  32. package/dist/assets/web/dist/assets/{main-DYwKyLY5.js → main-9ynpo780.js} +10 -10
  33. package/dist/assets/web/dist/assets/main-DSiJ9sHU.css +1 -0
  34. package/dist/cli.js +385 -226
  35. package/package.json +1 -1
  36. package/dist/assets/web/dist/assets/CostsPage-RRggVvIx.js +0 -1
  37. package/dist/assets/web/dist/assets/ResidentDetailPage-CT6v9INM.js +0 -1
  38. package/dist/assets/web/dist/assets/ResidentsIndexPage-CWqPFf40.js +0 -1
  39. package/dist/assets/web/dist/assets/main-DPB0UO1-.css +0 -1
@@ -4131,6 +4131,9 @@ export class RunHistoryDO extends DurableObject<Env> {
4131
4131
  userId: row.user_id,
4132
4132
  ...(who.userName ? { userName: who.userName } : {}),
4133
4133
  ...(who.parentRunId ? { parentRunId: who.parentRunId } : {}),
4134
+ ...(who.parentInstanceId ? { parentInstanceId: who.parentInstanceId } : {}),
4135
+ ...(who.idempotencyKey ? { idempotencyKey: who.idempotencyKey } : {}),
4136
+ ...(who.costCapUsd !== undefined ? { costCapUsd: who.costCapUsd } : {}),
4134
4137
  threadKey: row.thread_key,
4135
4138
  channelId: row.channel_id,
4136
4139
  ...(row.agent ? { agent: row.agent } : {}),
@@ -4379,12 +4382,23 @@ function parseUsageJson(raw: string | null): RunUsage | undefined {
4379
4382
  }
4380
4383
 
4381
4384
  /** The two identity fields the by-user aggregate needs off a summary, read leniently. */
4382
- function identityOfSummary(raw: string): { userName?: string; parentRunId?: string } {
4385
+ function identityOfSummary(raw: string): {
4386
+ userName?: string;
4387
+ parentRunId?: string;
4388
+ parentInstanceId?: string;
4389
+ idempotencyKey?: string;
4390
+ costCapUsd?: number;
4391
+ } {
4383
4392
  try {
4384
4393
  const s = JSON.parse(raw) as Record<string, unknown>;
4385
4394
  return {
4386
4395
  ...(typeof s.userName === "string" && s.userName ? { userName: s.userName } : {}),
4387
4396
  ...(typeof s.parentRunId === "string" && s.parentRunId ? { parentRunId: s.parentRunId } : {}),
4397
+ ...(typeof s.parentInstanceId === "string" && s.parentInstanceId ? { parentInstanceId: s.parentInstanceId } : {}),
4398
+ ...(typeof s.idempotencyKey === "string" && s.idempotencyKey ? { idempotencyKey: s.idempotencyKey } : {}),
4399
+ ...(typeof s.costCapUsd === "number" && Number.isFinite(s.costCapUsd) && s.costCapUsd > 0
4400
+ ? { costCapUsd: s.costCapUsd }
4401
+ : {}),
4388
4402
  };
4389
4403
  } catch {
4390
4404
  return {};
@@ -5689,6 +5703,14 @@ function parseClaim(b: Record<string, unknown>): Validated<ClaimRequest> {
5689
5703
  if (typeof meta.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(meta.idempotencyKey))
5690
5704
  return invalid("run.meta.idempotencyKey must be <parentInstanceId>:<step>");
5691
5705
  }
5706
+ if (
5707
+ meta.costCapUsd !== undefined &&
5708
+ (meta.parentInstanceId === undefined ||
5709
+ typeof meta.costCapUsd !== "number" ||
5710
+ !Number.isFinite(meta.costCapUsd) ||
5711
+ meta.costCapUsd <= 0)
5712
+ )
5713
+ return invalid("run.meta.costCapUsd must be a positive coordinator cap");
5692
5714
  // The restart tag (record 0064) names the predecessor run whose windows the
5693
5715
  // claim reuses and rides the `child-resumed` event's reason: a run id or
5694
5716
  // absent, never another shape.
@@ -0,0 +1,28 @@
1
+ /** One deploy's durable progress for a resident container. A repeated
2
+ * reconcile may retry a deferred stop or collect an inactive container's
3
+ * report, but must not stop a replacement that is still hydrating. */
4
+ export interface DeployImageReconcileState {
5
+ build: string;
6
+ cycleIssued: boolean;
7
+ cycleStoppedAt?: string;
8
+ containerBeforeCycle?: string;
9
+ }
10
+
11
+ export type DeployImageReconcileAction = "start" | "retry-cycle" | "await-report" | "verified";
12
+
13
+ export function nextDeployImageReconcile(
14
+ build: string,
15
+ stored: unknown,
16
+ reportPending: boolean,
17
+ ): DeployImageReconcileAction {
18
+ if (!stored || typeof stored !== "object") return "start";
19
+ const state = stored as Record<string, unknown>;
20
+ if (state.build !== build || typeof state.cycleIssued !== "boolean") return "start";
21
+ if (!reportPending) return "verified";
22
+ return state.cycleIssued ? "await-report" : "retry-cycle";
23
+ }
24
+
25
+ /** A fresh boot after the requested stop can finish hydration and report. */
26
+ export function replacementContainerStarted(before: string | undefined, current: string | undefined): boolean {
27
+ return before !== undefined && current !== undefined && before !== current;
28
+ }
@@ -30,9 +30,8 @@ export const REFRESH_INTERVAL_S = 600;
30
30
  * ten-minute buckets since the last instance. */
31
31
  export const IDLE_REFRESH_INTERVAL_S = 6 * 60 * 60;
32
32
 
33
- /** The thread user pool the image carries (`worker2`..`worker17`): one OS
34
- * user per attached thread, and the bound on how many bindings one sweep
35
- * step can have to check. */
33
+ /** The image's `worker2`..`worker17` pool: 16 one-use identities per VM
34
+ * generation for thread owners and disposable operations. */
36
35
  export const THREAD_POOL_SIZE = 16;
37
36
 
38
37
  /** Exec budgets. Each is a refresh step's own budget too (refresh.ts), so a
@@ -89,6 +89,11 @@ import {
89
89
  } from "./legacyCredentials.js";
90
90
  import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
91
91
  import { KeyedAsyncLock } from "./keyedAsyncLock.js";
92
+ import {
93
+ nextDeployImageReconcile,
94
+ replacementContainerStarted,
95
+ type DeployImageReconcileState,
96
+ } from "./imageReconcileState.js";
92
97
  import {
93
98
  registeredRunAllowsClaim,
94
99
  registeredRunAllowsReattach,
@@ -179,15 +184,17 @@ import { shellQuote } from "../../src/execution/shellQuote.js";
179
184
  import { envFromRequest } from "../../src/execution/sandboxEnv.js";
180
185
  import { CREDENTIAL_EXPIRY_MARGIN_MS } from "../../src/execution/residentCredentials.js";
181
186
  import { destroyWithPersistentFence } from "../../src/execution/residentDestroyGate.js";
182
- import { ResidentRecreateAdmission } from "../../src/execution/residentRecreateAdmission.js";
187
+ import { ResidentRecreateAdmission, idleForPoolRecycle } from "../../src/execution/residentRecreateAdmission.js";
183
188
  import {
184
189
  claimPoolBinding,
185
190
  mayRunAsPoolUser,
191
+ ownedPoolUsers,
186
192
  parsePoolBindings,
187
193
  parseSpentPoolUsers,
188
194
  rebuildPoolBindingIndex,
189
195
  releasePoolBinding,
190
196
  spendPoolUser,
197
+ unavailablePoolUsers,
191
198
  } from "../../src/execution/residentPoolSpends.js";
192
199
  import {
193
200
  recordFiring,
@@ -591,16 +598,10 @@ const THREADS_DIR = "/workspace/threads";
591
598
  * at the latest (disk is cache). */
592
599
  const OPS_DIR = "/workspace/ops";
593
600
 
594
- /** The thread-user pool. worker1 is the engine's build user; each
595
- * attach allocates one of these to the thread (persisted in the binding)
596
- * and every /exec /read /write for that thread runs privilege-dropped as
597
- * that user. The pool is released by the inactivity sweep. */
598
- /** Pool of OS users for thread worktrees (worker1 is the build user). Sized
599
- * for SIMULTANEOUS runs, not for every thread ever seen: a run returns its
600
- * user via /detach when it ends, so the pool only fills when 16 runs on one
601
- * repo are genuinely concurrent. Memory, not this list, is the real ceiling
602
- * — see the instance_type note in wrangler.jsonc. Must match the useradd loop
603
- * in the Dockerfile. */
601
+ /** Pool of OS users for thread worktrees and disposable ops. A UID stays with
602
+ * its first owner for this VM generation; detach releases the live binding,
603
+ * not the UID spend. Exhaustion may recycle a proven-idle VM. Must match the
604
+ * useradd loop in the Dockerfile. */
604
605
  const THREAD_USERS = Array.from({ length: THREAD_POOL_SIZE }, (_, i) => `worker${i + 2}`);
605
606
 
606
607
  /** Force-detach: after killing the thread user's processes, how long
@@ -1296,7 +1297,7 @@ interface AttachOk {
1296
1297
  returned?: Returned;
1297
1298
  }
1298
1299
 
1299
- /** What `POST /detach` answers: whether the pool user went back, why not, and
1300
+ /** What `POST /detach` answers: whether the live binding was released, why not, and
1300
1301
  * — on a release — what the tree still held (item 16a), now gone with it. */
1301
1302
  interface DetachAnswer {
1302
1303
  released: boolean;
@@ -1478,6 +1479,7 @@ const DRAIN_KEY = "drain";
1478
1479
  * verify this resident's container on the new image; the next reconcile that
1479
1480
  * finds it current reports to the registry and clears it. */
1480
1481
  const IMAGE_REPORT_PENDING_KEY = "imageReportPending";
1482
+ const IMAGE_RECONCILE_KEY = "imageReconcile";
1481
1483
 
1482
1484
  type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
1483
1485
 
@@ -3910,8 +3912,8 @@ export class ResidentDO extends Sandbox<Env> {
3910
3912
  await this.ctx.storage.delete(INFRA_STREAK_KEY);
3911
3913
  await this.setResidentState("warm");
3912
3914
  // Event-triggered reclamation: the prune above already told the
3913
- // mirror which branches died; finished refs give their worktree and
3914
- // pool user back now, not at the idle TTL. Housekeeping, never a
3915
+ // mirror which branches died; finished refs release their worktree and
3916
+ // live binding now, not at the idle TTL. Housekeeping, never a
3915
3917
  // lifecycle flip — a failure here is a log line.
3916
3918
  try {
3917
3919
  const gc = await this.reclaimFinishedRefs(resource, facts.defaultRef, cycle.token);
@@ -5250,7 +5252,12 @@ export class ResidentDO extends Sandbox<Env> {
5250
5252
  }
5251
5253
  console.log(`image-stale (${where}): ${stale} — stopping so it restarts on the current image`);
5252
5254
  this.swapIncarnation(); // deliberate incarnation swap
5253
- await this.stop().catch((err) => console.log(`image-stale: stop failed: ${errMsg(err)}`));
5255
+ try {
5256
+ await this.stop();
5257
+ } catch (err) {
5258
+ console.log(`image-stale: stop failed: ${errMsg(err)}`);
5259
+ if (where === "deploy") return "deferred";
5260
+ }
5254
5261
  // A successful stop proves only that the old process is gone. The pending
5255
5262
  // marker deliberately survives it: the replacement's own hydration reports
5256
5263
  // after reaching `warm`, so `imageReport: current` can never precede a
@@ -5306,9 +5313,66 @@ export class ResidentDO extends Sandbox<Env> {
5306
5313
  * added afterwards would wait for a report nothing sends (until the drain's
5307
5314
  * `until`). Held first, every report finds its hold. */
5308
5315
  async reconcileForDeploy(resource: string): Promise<{ result: ImageReconcileResult; verified: boolean }> {
5309
- await registryStub(this.env).holdDrainFor([resource]);
5310
- await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
5316
+ // The deploy's image report and an idle pool recycle both reason about the
5317
+ // same VM. Admit reconciliation as disk work before its first await, so a
5318
+ // recycle sees it as busy; a recycle already holding admission defers this
5319
+ // pass for the deploy's bounded readiness retry.
5320
+ const admitted = await this.withRecreateSafeAdmin(() => this.reconcileForDeployAdmitted(resource));
5321
+ return "error" in admitted ? { result: "deferred", verified: false } : admitted;
5322
+ }
5323
+
5324
+ private async reconcileForDeployAdmitted(
5325
+ resource: string,
5326
+ ): Promise<{ result: ImageReconcileResult; verified: boolean }> {
5327
+ const progress = await this.ctx.storage.get<DeployImageReconcileState>(IMAGE_RECONCILE_KEY);
5328
+ const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
5329
+ const action = nextDeployImageReconcile(BUILD.commit, progress, pending !== undefined);
5330
+ if (action === "verified") return { result: "current", verified: true };
5331
+ if (action === "start") {
5332
+ await registryStub(this.env).holdDrainFor([resource]);
5333
+ await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
5334
+ await this.ctx.storage.put(IMAGE_RECONCILE_KEY, { build: BUILD.commit, cycleIssued: false });
5335
+ }
5336
+ if (action === "await-report") {
5337
+ const active = await this.isRuntimeActive().catch(() => null);
5338
+ if (active === false) await this.reportPendingImageCurrent("deploy");
5339
+ else if (active === true) {
5340
+ const [state, facts] = await Promise.all([
5341
+ this.ctx.storage.get<ResidentState>(STATE_KEY),
5342
+ this.ctx.storage.get<RepoFacts>(FACTS_KEY),
5343
+ ]);
5344
+ // A replacement can reuse the disk and leave `warm` unchanged; that
5345
+ // hydrate shortcut does not send a report. A different boot after the
5346
+ // successful stop is proof that a replacement started. Finish its
5347
+ // hydration before reporting; an old warm state alone proves nothing.
5348
+ if (replacementContainerStarted(progress?.containerBeforeCycle, await this.containerIdentity())) {
5349
+ await this.ensureHydrated();
5350
+ await this.reportPendingImageCurrent("deploy");
5351
+ } else if (
5352
+ state === "warm" &&
5353
+ facts?.lastRestore?.at &&
5354
+ Date.parse(facts.lastRestore.at) >= Date.parse(progress?.cycleStoppedAt ?? "")
5355
+ ) {
5356
+ await this.reportPendingImageCurrent("deploy");
5357
+ }
5358
+ }
5359
+ return {
5360
+ result: active === false ? "inactive" : "deferred",
5361
+ verified: (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) === undefined,
5362
+ };
5363
+ }
5364
+ // Reading the boot ID runs a command and would wake a sleeping resident.
5365
+ // Preserve the inactive path's report-without-start behavior.
5366
+ const activeBeforeCycle = await this.isRuntimeActive().catch(() => null);
5367
+ const containerBeforeCycle = activeBeforeCycle === true ? await this.containerIdentity() : undefined;
5311
5368
  const result = await this.reconcileImage("deploy", true);
5369
+ if (result === "restarted")
5370
+ await this.ctx.storage.put(IMAGE_RECONCILE_KEY, {
5371
+ build: BUILD.commit,
5372
+ cycleIssued: true,
5373
+ cycleStoppedAt: new Date(systemClock()).toISOString(),
5374
+ containerBeforeCycle,
5375
+ } satisfies DeployImageReconcileState);
5312
5376
  if (result === "deferred") return { result, verified: false };
5313
5377
  // Inactivity reports synchronously; a successful stop does not. A
5314
5378
  // concurrent fresh start may also have hydrated while this reconcile
@@ -5615,7 +5679,11 @@ export class ResidentDO extends Sandbox<Env> {
5615
5679
  /** Storage bindings, active ops and inspections all reserve a pool user.
5616
5680
  * Claim inside this method, before its promise resolves: two concurrent
5617
5681
  * callers must not each read the same free UID before either can claim it. */
5618
- private async findFreePoolUser(excludeThreadKey?: string, excluded = new Set<string>()): Promise<string | undefined> {
5682
+ private async findFreePoolUser(
5683
+ excludeThreadKey?: string,
5684
+ excluded = new Set<string>(),
5685
+ preferred: readonly string[] = [],
5686
+ ): Promise<string | undefined> {
5619
5687
  const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
5620
5688
  const used = new Set(
5621
5689
  [...all.values()].filter((b) => !b.evicted && b.user && b.threadKey !== excludeThreadKey).map((b) => b.user),
@@ -5623,7 +5691,7 @@ export class ResidentDO extends Sandbox<Env> {
5623
5691
  for (const u of this.opUsersInUse.keys()) used.add(u);
5624
5692
  for (const u of this.poolUsersInspecting) used.add(u);
5625
5693
  for (const u of excluded) used.add(u);
5626
- const user = THREAD_USERS.find((u) => !used.has(u));
5694
+ const user = [...preferred, ...THREAD_USERS].find((u) => !used.has(u));
5627
5695
  if (user) this.poolUsersInspecting.add(user);
5628
5696
  return user;
5629
5697
  }
@@ -5660,19 +5728,82 @@ export class ResidentDO extends Sandbox<Env> {
5660
5728
  throw new Error("pool-user-stage-scan-invalid");
5661
5729
  }
5662
5730
 
5731
+ /** A full one-use pool can serve the next request by destroying its VM,
5732
+ * never by assigning another owner's UID. The admission mark closes every
5733
+ * other entry before the idle read and stays held through the restore. */
5734
+ private async recycleSpentPoolForAdmission(): Promise<true | ThreadErr> {
5735
+ try {
5736
+ const admission = await this.recreateAdmission.run(async (): Promise<true | ThreadErr> => {
5737
+ const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5738
+ if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5739
+ if (spent.size < THREAD_USERS.length) return true; // another checked recreate already cleared the generation
5740
+ const [status, drain, imagePending, snapshot, registeredRuns, bindings] = await Promise.all([
5741
+ this.getStatus(),
5742
+ this.registry().getDrain(),
5743
+ this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY),
5744
+ this.ctx.storage.get(SNAPSHOT_KEY),
5745
+ this.registeredRunsBeyondOps(),
5746
+ this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX }),
5747
+ ]);
5748
+ if (
5749
+ !snapshot ||
5750
+ !idleForPoolRecycle({
5751
+ state: status.state,
5752
+ draining: liveDrain(drain, systemClock()) !== null,
5753
+ imagePending: imagePending !== undefined,
5754
+ inFlight: this.inFlightCount(),
5755
+ refreshAdmissions: this.refreshAdmissionsInFlight,
5756
+ adminWork: this.adminWorkInFlight,
5757
+ hydrating: this.hydration !== null,
5758
+ registeredRuns,
5759
+ liveBindings: [...bindings.values()].filter((binding) => !binding.evicted).length,
5760
+ inspecting: this.poolUsersInspecting.size,
5761
+ })
5762
+ )
5763
+ return {
5764
+ error: "pool-recycle-required: all UIDs spent and the resident is not idle for checked VM recycle",
5765
+ status: 503,
5766
+ };
5767
+ await this.recreateContainer("pool-recycle: all UIDs spent; no other resident work owns this VM");
5768
+ await this.ensureHydrated();
5769
+ const fresh = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5770
+ const ready = await this.getStatus();
5771
+ const imageStillPending = (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) !== undefined;
5772
+ if (!fresh || fresh.size !== 0 || ready.state !== "warm" || imageStillPending)
5773
+ return { error: "pool-recycle-failed: the restored VM is not warm, current and empty", status: 503 };
5774
+ return true;
5775
+ });
5776
+ return admission.busy ? this.recreateRefusal() : admission.value;
5777
+ } catch (err) {
5778
+ console.log(`pool-recycle: checked VM recycle failed (${errMsg(err)})`);
5779
+ return { error: "pool-recycle-failed: the checked VM restore did not complete", status: 503 };
5780
+ }
5781
+ }
5782
+
5663
5783
  private async reserveSafePoolUser(owner: string, excludeThreadKey?: string): Promise<string | ThreadErr> {
5664
- const spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5784
+ let spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5665
5785
  if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5666
- const excluded = new Set(spent.keys());
5786
+ let excluded = unavailablePoolUsers(spent, owner);
5667
5787
  let contaminated = false;
5788
+ let recycled = false;
5668
5789
  for (;;) {
5669
- const user = await this.findFreePoolUser(excludeThreadKey, excluded);
5670
- if (!user)
5790
+ const user = await this.findFreePoolUser(excludeThreadKey, excluded, ownedPoolUsers(spent, owner));
5791
+ if (!user) {
5792
+ if (!contaminated && excluded.size === THREAD_USERS.length && !recycled) {
5793
+ recycled = true;
5794
+ const result = await this.recycleSpentPoolForAdmission();
5795
+ if (result !== true) return result;
5796
+ spent = parseSpentPoolUsers(await this.ctx.storage.get<unknown>(SPENT_POOL_USERS_KEY), THREAD_USERS);
5797
+ if (!spent) return { error: "pool-generation-unknown: fresh VM required before UID assignment", status: 503 };
5798
+ excluded = unavailablePoolUsers(spent, owner);
5799
+ continue;
5800
+ }
5671
5801
  return contaminated
5672
5802
  ? { error: "pool-user-contaminated: no clean pool user is available", status: 503 }
5673
5803
  : excluded.size === THREAD_USERS.length
5674
5804
  ? { error: "pool-recycle-required: all UIDs spent on this VM; confirm idle and recreate it", status: 503 }
5675
5805
  : { error: "user-pool-exhausted: no safe pool user is available", status: 429 };
5806
+ }
5676
5807
  let safe = false;
5677
5808
  try {
5678
5809
  const clean = !(await this.poolUserHasOldThreadDir(user, [])) && !(await this.poolUserHasOldStageContent(user));
@@ -7803,9 +7934,9 @@ export class ResidentDO extends Sandbox<Env> {
7803
7934
  return "evicted";
7804
7935
  }
7805
7936
 
7806
- /** POST /detach: a run has ended — give the thread's pool user back now
7807
- * instead of holding it until the TTL sweep (the pool is sized for
7808
- * simultaneous runs). A run starts from a clean tree (item 17), so the tree
7937
+ /** POST /detach: a run has ended — release its live binding now instead of
7938
+ * holding it until the TTL sweep. The UID remains spent for this VM
7939
+ * generation. A run starts from a clean tree (item 17), so the tree
7809
7940
  * goes whatever it holds: what it held — uncommitted changes, unpushed
7810
7941
  * commits — is measured once, as the thread user, and named in the answer
7811
7942
  * (`leftBehind`) so the loss is never silent. The one thing that keeps a
@@ -8787,6 +8918,7 @@ export class ResidentDO extends Sandbox<Env> {
8787
8918
  // The runs whose process lives in the container between operator calls
8788
8919
  // (item 44): what the deploy preflight must see, added to both counts.
8789
8920
  const registeredRuns = await this.registeredRunsBeyondOps();
8921
+ const poolSpends = parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS);
8790
8922
  return {
8791
8923
  resource: map.get(RESOURCE_KEY) ?? null,
8792
8924
  state: this.destroying || map.get(DESTROY_UNCONFIRMED_KEY) ? "down" : (map.get(STATE_KEY) ?? "down"),
@@ -8818,7 +8950,10 @@ export class ResidentDO extends Sandbox<Env> {
8818
8950
  // running container the last deploy could not verify.
8819
8951
  imageReport: map.get(IMAGE_REPORT_PENDING_KEY) !== undefined ? "pending" : "current",
8820
8952
  recreateAdmissionHeld: this.recreateAdmission.pending || map.get(RECREATE_ADMISSION_KEY) === true,
8821
- poolUsersSpent: parseSpentPoolUsers(map.get(SPENT_POOL_USERS_KEY), THREAD_USERS)?.size ?? null,
8953
+ poolUsersSpent: poolSpends?.size ?? null,
8954
+ // Read-scoped owner receipts explain historical occupancy even after
8955
+ // every live binding and disposable op has ended.
8956
+ poolUserSpends: poolSpends ? [...poolSpends].map(([user, owner]) => ({ user, owner })) : null,
8822
8957
  // Item 22: who holds what, as the rows say — the mirror mutex and the
8823
8958
  // cycle/hydration leases, each judged against this incarnation.
8824
8959
  incarnation: this.incarnation,
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "switchboard",
3
- "version": "1.267.3",
3
+ "version": "1.268.0",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "switchboard",
9
- "version": "1.267.3",
9
+ "version": "1.268.0",
10
10
  "license": "Apache-2.0",
11
11
  "workspaces": [
12
12
  "web",
@@ -21023,7 +21023,7 @@
21023
21023
  },
21024
21024
  "packages/switchboard": {
21025
21025
  "name": "@coreplane/switchboard",
21026
- "version": "1.267.3",
21026
+ "version": "1.268.0",
21027
21027
  "license": "Apache-2.0",
21028
21028
  "dependencies": {
21029
21029
  "@earendil-works/pi-ai": "0.85.1",
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "switchboard",
3
- "version": "1.267.3",
3
+ "version": "1.268.0",
4
4
  "private": true,
5
5
  "description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
6
6
  "license": "Apache-2.0",
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.267.3",
3
- "commit": "deb60c5f49012fb663a5797168b9e02d2ba394a7",
4
- "builtAt": "2026-09-29T04:50:34.175Z"
2
+ "version": "1.268.0",
3
+ "commit": "f1bc76a31780740930006bd492101666268596da-dirty",
4
+ "builtAt": "2026-09-29T08:06:43.354Z"
5
5
  }
@@ -255,6 +255,10 @@ export const PROVIDER_RETRY_BACKOFFS_MS = [5_000, 15_000, 45_000] as const;
255
255
  * model call itself remains bounded by the run's lease. */
256
256
  export const MODEL_STREAM_HEARTBEAT_MS = 15_000;
257
257
 
258
+ /** A residents page re-reads the registry while open so a deploy's drain and
259
+ * image reports advance even when no run can attach to trigger its feed. */
260
+ export const RESIDENT_LISTING_POLL_MS = 30_000;
261
+
258
262
  /** A hosted ship parent's deadline margin past the pipeline's wall clock
259
263
  * (record 0060): the row's `state.hosting.until` is the hand-off time plus
260
264
  * the instance's `caps.maxMinutes` plus this hour, absorbing the runner's own
@@ -307,12 +307,13 @@ export interface ExistingPrPublicationBinding {
307
307
  owner: { instanceId: string; unit: string };
308
308
  }
309
309
 
310
- /** What a coordinator's spawn stamps on the child's every row: the instance
311
- * the child belongs to and the key the spawn carried — and, for the child's
312
- * own post-step, the base its pull request targets. */
310
+ /** What a coordinator's spawn stamps on the child's every row: its instance,
311
+ * spawn key and admitted cost cap, plus the base for its PR post-step. */
313
312
  export interface CoordinatorTag {
314
313
  parentInstanceId: string;
315
314
  idempotencyKey: string;
315
+ /** The unit's original dollar limit, fixed at admission; absent means no cost cap. */
316
+ costCapUsd?: number;
316
317
  /** The unit's durable write branch, read from its row at child admission. */
317
318
  branch?: string;
318
319
  /** Workflow transport for a recovered child. Identity and idempotency remain
@@ -346,15 +347,20 @@ export interface CoordinatorTag {
346
347
  publication?: ExistingPrPublicationBinding;
347
348
  }
348
349
 
349
- /** The tag as the two flat record fields, or nothing — so a row, a summary
350
- * and a record spread the same thing and a run with no coordinator carries no
351
- * key. The base never rides here: rows and records keep the shape written
352
- * before it existed. */
350
+ /** The tag's identity and optional cap as flat record fields, or nothing — so
351
+ * a row, summary and record agree. The base never rides here. */
353
352
  export function coordinatorFields(tag: CoordinatorTag | undefined): {
354
353
  parentInstanceId?: string;
355
354
  idempotencyKey?: string;
355
+ costCapUsd?: number;
356
356
  } {
357
- return tag ? { parentInstanceId: tag.parentInstanceId, idempotencyKey: tag.idempotencyKey } : {};
357
+ return tag
358
+ ? {
359
+ parentInstanceId: tag.parentInstanceId,
360
+ idempotencyKey: tag.idempotencyKey,
361
+ ...(tag.costCapUsd !== undefined ? { costCapUsd: tag.costCapUsd } : {}),
362
+ }
363
+ : {};
358
364
  }
359
365
 
360
366
  /** The parent ship record: what the bot writes at an instance's creation and
@@ -516,6 +516,8 @@ export interface CostReport {
516
516
  /** Each biller's daily invoice tie-out (item 4d); absent when no provider block is known
517
517
  * (a report built without the wiring) or run history is off. */
518
518
  billers?: BillerTieOut[];
519
+ /** Ship children across this bot's runs, independent of the Cloudflare group. */
520
+ shipSpend?: import("./shipSpend.js").ShipSpendReport;
519
521
  }
520
522
 
521
523
  /** What the report carries beyond the priced rows: the account behind the
@@ -974,6 +974,7 @@ export type RunEvent =
974
974
  type: "coordinator_tag";
975
975
  parentInstanceId: string;
976
976
  unit?: string;
977
+ costCapUsd?: number;
977
978
  branch?: string;
978
979
  transportWorkflowId?: string;
979
980
  recovery?: import("./coordinator/contract.js").CoordinatorTag["recovery"];
@@ -99,6 +99,8 @@ export interface LiveRunMeta {
99
99
  * still sends the parent its event and a retried spawn finds its run. */
100
100
  parentInstanceId?: string;
101
101
  idempotencyKey?: string;
102
+ /** The original Ship unit cap carried by the claim, available before its tag event is retained. */
103
+ costCapUsd?: number;
102
104
  /** Where the run's conversation started (item 52), so a reclaimed run's
103
105
  * record still says so: `parent` for a spawned child, `channel` otherwise. */
104
106
  seed?: RunSeed;
@@ -227,6 +227,8 @@ export interface RunRecord {
227
227
  * item 48), stored at the claim so a retried spawn finds its run. Present
228
228
  * exactly when `parentInstanceId` is — both or neither, never one alone. */
229
229
  idempotencyKey?: string;
230
+ /** The coordinator unit's original cost cap, carried by each child. */
231
+ costCapUsd?: number;
230
232
  /** The plan runner instance this run's hand-off created (record 0051 R2;
231
233
  * item 2): the last `ship_handoff` event, folded at the assembly like the
232
234
  * coordinator tag. Present only on a ship run whose hand-off succeeded;
@@ -1195,6 +1197,14 @@ export function isRunRecord(v: unknown): v is RunRecord {
1195
1197
  (typeof r.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(r.idempotencyKey))
1196
1198
  )
1197
1199
  return false;
1200
+ if (
1201
+ r.costCapUsd !== undefined &&
1202
+ (r.parentInstanceId === undefined ||
1203
+ typeof r.costCapUsd !== "number" ||
1204
+ !Number.isFinite(r.costCapUsd) ||
1205
+ r.costCapUsd <= 0)
1206
+ )
1207
+ return false;
1198
1208
  // The instance a ship run's hand-off created (record 0051 R2; item 2).
1199
1209
  if (r.instanceId !== undefined && (typeof r.instanceId !== "string" || !INSTANCE_ID_PATTERN.test(r.instanceId)))
1200
1210
  return false;
@@ -1,4 +1,5 @@
1
1
  import type { RunEvent } from "./runEvents.js";
2
+ import { unitOfIdempotencyKey } from "./coordinator/contract.js";
2
3
 
3
4
  // What a run cost in tokens, and who it belongs to — the data behind the cost
4
5
  // dimensions of the costs page (docs/reference/specs/costs.md items 10–10a) and
@@ -200,6 +201,9 @@ export interface UsageRun {
200
201
  userName?: string;
201
202
  /** A child run is billed to whoever started its parent (run-history item 46). */
202
203
  parentRunId?: string;
204
+ parentInstanceId?: string;
205
+ idempotencyKey?: string;
206
+ costCapUsd?: number;
203
207
  /** The thread the run ran in and the channel it belongs to (platform-namespaced, invariant 4). */
204
208
  threadKey: string;
205
209
  channelId: string;
@@ -258,6 +262,8 @@ export interface UsageRow {
258
262
 
259
263
  export interface RunUsageReport {
260
264
  rows: UsageRow[];
265
+ /** Individual Ship children retained for the unit-spend trend; old snapshots omit this. */
266
+ shipRuns?: ShipRunUsage[];
261
267
  /** Runs in range whose usage is not known yet (written before the field; backfill outstanding). */
262
268
  pending: number;
263
269
  /** The oldest finish the store still holds, so a page can bound its range to the data. */
@@ -265,6 +271,33 @@ export interface RunUsageReport {
265
271
  retentionDays: number;
266
272
  }
267
273
 
274
+ export interface ShipRunUsage {
275
+ id: string;
276
+ unitKey: string;
277
+ finishedAt: number;
278
+ costCapUsd?: number;
279
+ /** Missing while an older record is still being backfilled. */
280
+ usage?: RunUsage;
281
+ }
282
+
283
+ /** The unit identity comes from the coordinator's durable key, never a thread or branch guess. */
284
+ export function shipRunsOf(runs: readonly UsageRun[]): ShipRunUsage[] {
285
+ return runs.flatMap((run) => {
286
+ if (!run.parentInstanceId || !run.idempotencyKey) return [];
287
+ const unit = unitOfIdempotencyKey(run.idempotencyKey);
288
+ if (!unit) return [];
289
+ return [
290
+ {
291
+ id: run.id,
292
+ unitKey: `${run.parentInstanceId}:${unit}`,
293
+ finishedAt: run.finishedAt,
294
+ ...(run.costCapUsd !== undefined ? { costCapUsd: run.costCapUsd } : {}),
295
+ ...(run.usage !== undefined ? { usage: run.usage } : {}),
296
+ },
297
+ ];
298
+ });
299
+ }
300
+
268
301
  export const dayOf = (epochMs: number): string => new Date(epochMs).toISOString().slice(0, 10);
269
302
 
270
303
  const identityOf = (who: UsageIdentity): UsageIdentity => ({
@@ -334,6 +367,7 @@ export function reportOfUsageRows(rows: RunUsageRows): RunUsageReport {
334
367
  const { rows: cells, pending } = aggregateUsage(rows.runs, (id) => rows.parents[id]);
335
368
  return {
336
369
  rows: cells,
370
+ shipRuns: shipRunsOf(rows.runs),
337
371
  // The store counts what it could not price; the fold sees the same runs without `usage`.
338
372
  pending: Math.max(pending, rows.pending),
339
373
  ...(rows.earliestFinishedAt !== undefined ? { earliestFinishedAt: rows.earliestFinishedAt } : {}),
@@ -356,6 +390,10 @@ function isUsageRun(v: unknown): v is UsageRun {
356
390
  typeof r.channelId === "string" &&
357
391
  (r.agent === undefined || typeof r.agent === "string") &&
358
392
  (r.parentRunId === undefined || typeof r.parentRunId === "string") &&
393
+ (r.parentInstanceId === undefined || typeof r.parentInstanceId === "string") &&
394
+ (r.idempotencyKey === undefined || typeof r.idempotencyKey === "string") &&
395
+ (r.costCapUsd === undefined ||
396
+ (typeof r.costCapUsd === "number" && Number.isFinite(r.costCapUsd) && r.costCapUsd > 0)) &&
359
397
  typeof r.startedAt === "number" &&
360
398
  typeof r.finishedAt === "number" &&
361
399
  (r.usage === undefined || isRunUsage(r.usage))
@@ -382,6 +420,7 @@ export function isRunUsageReport(v: unknown): v is RunUsageReport {
382
420
  if (typeof v !== "object" || v === null) return false;
383
421
  const r = v as Record<string, unknown>;
384
422
  if (!Array.isArray(r.rows) || !hasReportTail(r)) return false;
423
+ if (r.shipRuns !== undefined && (!Array.isArray(r.shipRuns) || !r.shipRuns.every(isShipRunUsage))) return false;
385
424
  return r.rows.every(
386
425
  (row) =>
387
426
  isIdentity(row) &&
@@ -394,3 +433,17 @@ export function isRunUsageReport(v: unknown): v is RunUsageReport {
394
433
  isRunUsage((row as UsageRow).usage),
395
434
  );
396
435
  }
436
+
437
+ function isShipRunUsage(v: unknown): v is ShipRunUsage {
438
+ if (typeof v !== "object" || v === null) return false;
439
+ const r = v as ShipRunUsage;
440
+ return (
441
+ typeof r.id === "string" &&
442
+ typeof r.unitKey === "string" &&
443
+ typeof r.finishedAt === "number" &&
444
+ Number.isFinite(r.finishedAt) &&
445
+ (r.costCapUsd === undefined ||
446
+ (typeof r.costCapUsd === "number" && Number.isFinite(r.costCapUsd) && r.costCapUsd > 0)) &&
447
+ (r.usage === undefined || isRunUsage(r.usage))
448
+ );
449
+ }