@coreplane/switchboard 1.267.2 → 1.267.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -89,6 +89,13 @@ import {
89
89
  } from "./legacyCredentials.js";
90
90
  import { hasUnexpectedOwnedThreadDir } from "./orphanThreadUsers.js";
91
91
  import { KeyedAsyncLock } from "./keyedAsyncLock.js";
92
+ import {
93
+ registeredRunAllowsClaim,
94
+ registeredRunAllowsReattach,
95
+ registeredRunNeedsProtection,
96
+ registeredRunOwnsRelease,
97
+ } from "./runRegistration.js";
98
+ import { DAY_MS, RUN_REGISTRATION_GRACE_MS } from "../../src/core/budgets.js";
92
99
  import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
93
100
  import { RESIDENT_MIRROR_PERMISSIONS, verifyGithubMintScope } from "../../src/execution/githubMintScope.js";
94
101
  import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
@@ -1194,6 +1201,8 @@ interface ThreadBinding {
1194
1201
  lastAttachAt: string;
1195
1202
  evicted?: boolean;
1196
1203
  evictedAt?: string;
1204
+ /** Owner of the registration removed with this tree, for a delayed push receipt. */
1205
+ lastRunOwner?: { runId?: string; ownerGen?: string; ownerFence?: number };
1197
1206
  /** Why the last eviction happened (the audit trail): `ttl`, `clean-idle`,
1198
1207
  * `detach`, `disk-pressure`, or a reclamation fate — `merged #N` /
1199
1208
  * `closed #N` / `gone`. */
@@ -1764,9 +1773,17 @@ const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
1764
1773
  * a fresh isolate still counts the process that survived in the container. */
1765
1774
  const RUN_REG_KEY_PREFIX = "runReg:";
1766
1775
  const runRegKey = (threadKey: string) => `${RUN_REG_KEY_PREFIX}${threadKey}`;
1776
+ // A thread's highest accepted ledger claim survives registration release and
1777
+ // binding purge: a delayed older /attach must never reopen an evicted tree.
1778
+ const RUN_FENCE_KEY_PREFIX = "runFence:";
1779
+ const runFenceKey = (threadKey: string) => `${RUN_FENCE_KEY_PREFIX}${threadKey}`;
1767
1780
  interface RunRegistration {
1768
1781
  threadKey: string;
1769
1782
  registeredAt: string;
1783
+ deadlineAt?: number;
1784
+ runId?: string;
1785
+ ownerGen?: string;
1786
+ ownerFence?: number;
1770
1787
  }
1771
1788
 
1772
1789
  /** Deterministic per-thread+ref worktree path. Slugs replace anything outside
@@ -4644,23 +4661,21 @@ export class ResidentDO extends Sandbox<Env> {
4644
4661
 
4645
4662
  /** Whether a run may be using this disk: a live binding attached to or
4646
4663
  * used (an exec bumps `lastAttachAt` too, item 21) within IDLE_AFTER_S.
4647
- * The op counter is 0 between a run's tool calls, so this floor is what
4648
- * stands for a run mid-flight. The one predicate behind "the disk may go
4649
- * away": the idle-sleep gate (item 16b — a platform sleep destroys the
4650
- * disk) and the disk-full recycle (item 54) both read it, and neither asks
4651
- * what a tree holds (item 17): a dirty tree no run is using protects
4652
- * nothing — the next attach wipes it. Bindings are storage, so this needs
4653
- * no container. */
4664
+ * The run registration protects the gap between tool calls past this floor.
4665
+ * The idle-sleep gate (item 16b) and disk-full recycle (item 54) both read
4666
+ * the floor and the bounded registrations, never what a tree holds (item
4667
+ * 17): a dirty tree no run is using protects nothing. */
4654
4668
  private recentlyUsed(live: ThreadBinding[]): boolean {
4655
4669
  const recent = systemClock() - IDLE_AFTER_S * 1000;
4656
4670
  return live.some((b) => Date.parse(b.lastAttachAt) >= recent);
4657
4671
  }
4658
4672
 
4659
- /** Idle = no live binding used within the floor and nothing in flight; the
4660
- * sweep's release records what an idle tree held. */
4673
+ /** A live run registration prevents sleep even after an hour without an op;
4674
+ * an expired one cannot keep the container awake. */
4661
4675
  private async isIdle(): Promise<boolean> {
4662
4676
  if (this.recentlyUsed(await this.liveBindings())) return false;
4663
- return this.inFlightCount() === 0;
4677
+ if (this.inFlightCount() > 0) return false;
4678
+ return (await this.registeredRunsBeyondOps()) === 0;
4664
4679
  }
4665
4680
 
4666
4681
  /** Bindings that hold a pool user and a tree on disk (not evicted). */
@@ -4698,10 +4713,10 @@ export class ResidentDO extends Sandbox<Env> {
4698
4713
  * path as a platform sleep. Only when the pure plan allows it: no recycle
4699
4714
  * within the cooldown, nothing in flight (`selfInFlight` excludes the
4700
4715
  * calling refresh cycle from the count) and no live binding used within
4701
- * the idle floor — the idle-sleep gate's own predicate (`recentlyUsed`),
4702
- * because a sleep and a recycle destroy the same disk: it may go away when
4703
- * no run is using it, never for what the trees hold (item 17). A recycle
4704
- * discards every live tree the way an eviction does, so each is measured
4716
+ * the idle floor and no live run registration — the idle-sleep gate's own
4717
+ * predicates, because a sleep and a recycle destroy the same disk: it may
4718
+ * go away when no run is using it, never for what the trees hold (item 17).
4719
+ * A recycle discards every live tree the way an eviction does, so each is measured
4705
4720
  * first, as its thread user, and its binding records what went
4706
4721
  * (`recordRecycledTree`); the plan is decided again after those awaits,
4707
4722
  * right before the stop. A refused recycle is written to
@@ -4710,21 +4725,23 @@ export class ResidentDO extends Sandbox<Env> {
4710
4725
  * whether the container was recycled. */
4711
4726
  private async recoverFromDiskFull(reason: string, selfInFlight: number): Promise<boolean> {
4712
4727
  const lastRecycleAt = await this.ctx.storage.get<number>(DISK_FULL_RECYCLE_KEY);
4713
- const plan = (live: ThreadBinding[]) =>
4714
- planDiskFullRecovery({
4728
+ const plan = async (live: ThreadBinding[]) => {
4729
+ const liveRegistrations = await this.registeredRunsBeyondOps();
4730
+ return planDiskFullRecovery({
4715
4731
  now: systemClock(),
4716
4732
  lastRecycleAt,
4717
4733
  inFlight: this.inFlightCount() - selfInFlight,
4718
- recentlyUsed: this.recentlyUsed(live),
4734
+ recentlyUsed: this.recentlyUsed(live) || liveRegistrations > 0,
4719
4735
  idleFloorS: IDLE_AFTER_S,
4720
4736
  });
4737
+ };
4721
4738
  const kept = async (why: string) => {
4722
4739
  console.log(`disk-full: container kept — ${why}`);
4723
4740
  await this.recordRefreshError(`${reason} — container kept: ${why}`);
4724
4741
  return false;
4725
4742
  };
4726
4743
  const live = await this.liveBindings();
4727
- const first = plan(live);
4744
+ const first = await plan(live);
4728
4745
  if (first.action === "wait") return kept(first.why);
4729
4746
  // What each live tree holds, for its record — never a reason to keep the
4730
4747
  // container. Concurrent: each probe touches only its own tree, one spawn
@@ -4737,7 +4754,7 @@ export class ResidentDO extends Sandbox<Env> {
4737
4754
  : live.map((binding) => [binding, undefined] as const);
4738
4755
  // The measurement awaited (the DO yields at each await): decide again over
4739
4756
  // fresh facts — an attach or an op that landed meanwhile keeps the container.
4740
- const verdict = plan(await this.liveBindings());
4757
+ const verdict = await plan(await this.liveBindings());
4741
4758
  if (verdict.action === "wait") return kept(verdict.why);
4742
4759
  console.log(
4743
4760
  `disk-full: recycling the container — the next cycle restores mirror + checkout from R2 onto an empty disk (${reason})`,
@@ -5153,8 +5170,8 @@ export class ResidentDO extends Sandbox<Env> {
5153
5170
  * run — stop the container so it restarts on the current image (state is DO
5154
5171
  * storage + R2 — the disk is a cache). A deferred restart re-checks on
5155
5172
  * every later refresh cycle until the resident is quiet; a registration
5156
- * whose release never came defers it only until the clean-idle sweep drains
5157
- * that registration. The attach path never stops the container (issue 2101,
5173
+ * whose release never came defers it only through the clean-idle window;
5174
+ * a later sweep removes its binding. The attach path never stops the container (issue 2101,
5158
5175
  * the `stale` answer below); the stop is the refresh cycle's or the
5159
5176
  * deploy's alone. Answers `restarted` when a stop was issued, else why
5160
5177
  * not. */
@@ -5798,23 +5815,37 @@ export class ResidentDO extends Sandbox<Env> {
5798
5815
  traceparent?: string,
5799
5816
  reason: RefHintReason = NO_REF_HINT_REASON,
5800
5817
  githubDoor?: { baseUrl: string; bearer: string },
5818
+ runBudgetMs?: number,
5819
+ runId?: string,
5820
+ ownerGen?: string,
5821
+ ownerFence?: number,
5801
5822
  ): Promise<AttachOk | ThreadErr> {
5802
5823
  // One step trace per attach (docs/reference/specs/tracing.md item 19): every command
5803
5824
  // the attach runs lands on it, and the answer carries it.
5804
5825
  const t0 = systemClock();
5805
5826
  const trace = createStepTrace(t0);
5806
5827
  let res: AttachOk | ThreadErr;
5807
- if (this.recreateAdmission.pending) {
5808
- res = this.recreateRefusal();
5809
- } else {
5810
- this.attachAdmissionsInFlight++;
5811
- try {
5812
- res = await this.stepTrace.run(trace, () =>
5813
- this.attachThreadTraced(threadKey, refHint, readonly, wantSha, reuse, record, t0, reason, githubDoor),
5814
- );
5815
- } finally {
5816
- this.attachAdmissionsInFlight--;
5817
- }
5828
+ this.attachAdmissionsInFlight++;
5829
+ try {
5830
+ res = await this.stepTrace.run(trace, () =>
5831
+ this.attachThreadTraced(
5832
+ threadKey,
5833
+ refHint,
5834
+ readonly,
5835
+ wantSha,
5836
+ reuse,
5837
+ record,
5838
+ t0,
5839
+ reason,
5840
+ githubDoor,
5841
+ runBudgetMs,
5842
+ runId,
5843
+ ownerGen,
5844
+ ownerFence,
5845
+ ),
5846
+ );
5847
+ } finally {
5848
+ this.attachAdmissionsInFlight--;
5818
5849
  }
5819
5850
  // The same steps as the resident's own `resident.attach` root (item 22).
5820
5851
  emitStepRoot("resident.attach", t0, trace.steps(), traceparent, "error" in res ? refusalOutcome(res) : "ok");
@@ -5832,56 +5863,80 @@ export class ResidentDO extends Sandbox<Env> {
5832
5863
  t0: number,
5833
5864
  reason: RefHintReason,
5834
5865
  githubDoor?: { baseUrl: string; bearer: string },
5866
+ runBudgetMs?: number,
5867
+ runId?: string,
5868
+ ownerGen?: string,
5869
+ ownerFence?: number,
5835
5870
  ): Promise<AttachOk | ThreadErr> {
5836
5871
  try {
5837
- if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5838
- await this.ensureHydrated();
5839
- if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5840
- // The fleet drain (item 69): a deploy is waiting for the runs in flight
5841
- // to end, and a NEW run's attach is refused with the record the bot
5842
- // waits on — a real 503 in the streamed document, read by the client as
5843
- // `draining`, never as the platform's transient. A run already in flight
5844
- // — registered from its attach to its release (item 44) — re-attaches
5845
- // through: a rolled container, an evicted worktree, a resumed run are
5846
- // the runs the drain waits FOR, and refusing them would hold the fleet
5847
- // closed on the run it is closed for. Read before the image reconcile so
5848
- // a refused attach never restarts a container.
5849
- const drain = await this.fleetDrain();
5850
- const registered = (await this.ctx.storage.get(runRegKey(threadKey))) !== undefined;
5851
- if (drain && !registered) {
5852
- const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
5853
- return refusal;
5854
- }
5855
- // Item 70: above the soft memory threshold a NEW attach is refused like
5856
- // `mirror-busy` (the bot falls back or waits, the card says why) — after
5857
- // the drain (storage only, cheaper) and before the image reconcile, so a
5858
- // refused attach never restarts a container. A registered run's
5859
- // re-attach passes for the same reason it passes the drain above.
5860
- const memory = await this.memoryGate("attach", registered);
5861
- if (memory) return memory;
5862
- const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
5863
- // An attach never restarts the container (issue 2101): a `stale` verdict
5864
- // refuses the NEW run — it falls back to the seeded sandbox — while a
5865
- // registered run's re-attach passes exactly as it passes the drain and
5866
- // the memory gate; the restart itself is the refresh cycle's or the
5867
- // deploy's.
5868
- if ((await this.reconcileImage("attach")) === "stale" && !registered) {
5869
- const s = await this.getStatus();
5870
- return {
5871
- error:
5872
- "image-stale: the container predates the deploy and restarts on the next quiet refresh; new runs use the fallback sandbox until then",
5873
- status: 503,
5874
- state: s.state,
5875
- stateReason: s.reason,
5876
- reason: "image-stale",
5877
- };
5878
- }
5879
- // From here the attach may hold the mirror lock through clone/install:
5880
- // count it so a concurrent refresh-cycle reconcileImage never stops the
5881
- // container under it (and isIdle never parks the cycle mid-attach).
5882
- this.attachesInFlight++;
5883
- try {
5884
- return await this.threadAttaches.run(threadKey, async () => {
5872
+ return await this.threadAttaches.run(threadKey, async () => {
5873
+ // The attachment fence takes precedence over every 503 gate. Keep the
5874
+ // same thread lock through the gates and registration write so a newer
5875
+ // attach cannot slip between this check and a fallback-eligible refusal.
5876
+ const current = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
5877
+ const accepted = await this.ctx.storage.get<Pick<RunRegistration, "runId" | "ownerGen" | "ownerFence">>(
5878
+ runFenceKey(threadKey),
5879
+ );
5880
+ if (
5881
+ !registeredRunAllowsClaim(current, runId, ownerGen, ownerFence) ||
5882
+ !registeredRunAllowsClaim(accepted, runId, ownerGen, ownerFence)
5883
+ )
5884
+ return { error: "run-registration-mismatch: a newer generation owns the thread", status: 409 };
5885
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5886
+ await this.ensureHydrated();
5887
+ if (await this.recreateAdmission.blocked()) return this.recreateRefusal();
5888
+ // The fleet drain (item 69): a deploy is waiting for the runs in flight
5889
+ // to end, and a NEW run's attach is refused with the record the bot
5890
+ // waits on — a real 503 in the streamed document, read by the client as
5891
+ // `draining`, never as the platform's transient. A run already in flight
5892
+ // — with an owned, live registration (item 44) — re-attaches
5893
+ // through: a rolled container, an evicted worktree, a resumed run are
5894
+ // the runs the drain waits FOR, and refusing them would hold the fleet
5895
+ // closed on the run it is closed for. Read before the image reconcile so
5896
+ // a refused attach never restarts a container.
5897
+ const drain = await this.fleetDrain();
5898
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
5899
+ const registered = registeredRunAllowsReattach(
5900
+ registration,
5901
+ runId,
5902
+ systemClock(),
5903
+ RUN_REGISTRATION_GRACE_MS,
5904
+ ownerGen,
5905
+ ownerFence,
5906
+ );
5907
+ if (drain && !registered) {
5908
+ const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
5909
+ return refusal;
5910
+ }
5911
+ // Item 70: above the soft memory threshold a NEW attach is refused like
5912
+ // `mirror-busy` (the bot falls back or waits, the card says why) — after
5913
+ // the drain (storage only, cheaper) and before the image reconcile, so a
5914
+ // refused attach never restarts a container. An owned, live run's
5915
+ // re-attach passes for the same reason it passes the drain above.
5916
+ const memory = await this.memoryGate("attach", registered);
5917
+ if (memory) return memory;
5918
+ const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
5919
+ // An attach never restarts the container (issue 2101): a `stale` verdict
5920
+ // refuses the NEW run — it falls back to the seeded sandbox — while a
5921
+ // owned, live run's re-attach passes exactly as it passes the drain and
5922
+ // the memory gate; the restart itself is the refresh cycle's or the
5923
+ // deploy's.
5924
+ if ((await this.reconcileImage("attach")) === "stale" && !registered) {
5925
+ const s = await this.getStatus();
5926
+ return {
5927
+ error:
5928
+ "image-stale: the container predates the deploy and restarts on the next quiet refresh; new runs use the fallback sandbox until then",
5929
+ status: 503,
5930
+ state: s.state,
5931
+ stateReason: s.reason,
5932
+ reason: "image-stale",
5933
+ };
5934
+ }
5935
+ // From here the attach may hold the mirror lock through clone/install:
5936
+ // count it so a concurrent refresh-cycle reconcileImage never stops the
5937
+ // container under it (and isIdle never parks the cycle mid-attach).
5938
+ this.attachesInFlight++;
5939
+ try {
5885
5940
  const res = await this.attachThreadBody(
5886
5941
  threadKey,
5887
5942
  refHint,
@@ -5896,12 +5951,12 @@ export class ResidentDO extends Sandbox<Env> {
5896
5951
  );
5897
5952
  // The run this attach opens is now in flight until its release —
5898
5953
  // whatever its op counters read between the bot's calls (item 44).
5899
- if (!("error" in res)) await this.registerRun(threadKey);
5954
+ if (!("error" in res)) await this.registerRun(threadKey, runBudgetMs, runId, ownerGen, ownerFence);
5900
5955
  return res;
5901
- });
5902
- } finally {
5903
- this.attachesInFlight--;
5904
- }
5956
+ } finally {
5957
+ this.attachesInFlight--;
5958
+ }
5959
+ });
5905
5960
  } catch (err) {
5906
5961
  return catchAllErr(err, "attach-failed");
5907
5962
  }
@@ -7736,6 +7791,7 @@ export class ResidentDO extends Sandbox<Env> {
7736
7791
  user: "",
7737
7792
  evicted: true,
7738
7793
  evictedAt: new Date(systemClock()).toISOString(),
7794
+ lastRunOwner: await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey)),
7739
7795
  evictedWhy: why,
7740
7796
  evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
7741
7797
  evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
@@ -7763,65 +7819,77 @@ export class ResidentDO extends Sandbox<Env> {
7763
7819
  force: boolean,
7764
7820
  /** What the ending run pushed (item 16a): remembered on the binding first, whatever the detach then decides. */
7765
7821
  pushed: readonly PushedBranch[] = [],
7822
+ runId?: string,
7823
+ ownerGen?: string,
7824
+ ownerFence?: number,
7766
7825
  ): Promise<DetachAnswer | ThreadErr> {
7767
- if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
7768
- const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7769
- if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
7770
- if (binding.evicted || !binding.user) return { released: false, reason: "already-evicted" };
7771
- if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
7772
- return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
7773
- const plan = planForceDetach({
7774
- force,
7775
- inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
7776
- user: binding.user,
7777
- poolUsers: THREAD_USERS,
7778
- });
7779
- if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
7780
- if (plan.action === "kill") {
7781
- if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
7782
- return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
7783
- console.log(
7784
- `detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
7785
- );
7786
- const left = await this.waitForThreadDrain(threadKey);
7787
- if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
7788
- }
7789
- const active = await this.isRuntimeActive().catch(() => false);
7790
- // What the tree still holds, for the answer and the eviction's record —
7791
- // never a reason to keep it. Not measured on a force release: a read-only
7792
- // tree holds nothing, and a hard stop's tree is whatever the killed
7793
- // command left. A probe that fails names nothing in the answer (never a
7794
- // guess); the record and the log say it could not be measured.
7795
- let tree: EvictedTree | undefined;
7796
- if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
7797
- // Re-check right before removal: the measurement above awaited (the DO
7798
- // yields at each await), so an exec that arrived mid-detach would otherwise
7799
- // have its tree removed under it.
7800
- const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
7801
- if (busyNow > 0)
7802
- return {
7803
- released: false,
7804
- reason: `busy: ${busyNow} operation(s) started during detach — kept`,
7826
+ return this.threadAttaches.run(threadKey, async () => {
7827
+ const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7828
+ if (!binding) return { error: `no-binding: ${threadKey} has never attached to this resident`, status: 404 };
7829
+ if (binding.evicted || !binding.user) {
7830
+ if (pushed.length > 0 && registeredRunOwnsRelease(binding.lastRunOwner, runId, ownerGen, ownerFence))
7831
+ await this.rememberOwnBranches(threadKey, pushed);
7832
+ return { released: false, reason: "already-evicted" };
7833
+ }
7834
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(threadKey));
7835
+ if (!registeredRunOwnsRelease(registration, runId, ownerGen, ownerFence))
7836
+ return { released: false, reason: "run-registration-mismatch: the workspace belongs to another run" };
7837
+ if (pushed.length > 0) await this.rememberOwnBranches(threadKey, pushed);
7838
+ if (!(await this.poolUserOwnerMatches(binding.user, `thread:${threadKey}`)))
7839
+ return { error: "pool-owner-mismatch: detach refused for conflicting UID owner", status: 503 };
7840
+ const plan = planForceDetach({
7841
+ force,
7842
+ inFlight: this.threadOpsInFlight.get(threadKey) ?? 0,
7805
7843
  user: binding.user,
7806
- };
7807
- // Same re-read as the sweep: a re-attach during the measurement means a
7808
- // fresh tree we must not remove from a stale snapshot.
7809
- const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7810
- if (!current || current.evicted) return { released: false, reason: "already-evicted" };
7811
- if (current.lastAttachAt !== binding.lastAttachAt)
7812
- return { released: false, reason: "re-attached during the detach — kept", user: current.user };
7813
- const user = current.user;
7814
- // Same as the sweep: `active` was read before the measurement's awaits; a
7815
- // container that woke meanwhile must get the rm, not an orphaned tree.
7816
- const activeNow = await this.isRuntimeActive().catch(() => true);
7817
- const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
7818
- if (eviction === "cleanup-failed")
7819
- return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
7820
- if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
7821
- // Item 55: the tree is gone; the gauge catches up at the next refresh
7822
- // instance's `measure` step, and the admission's `df` sees the space now.
7823
- const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
7824
- return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
7844
+ poolUsers: THREAD_USERS,
7845
+ });
7846
+ if (plan.action === "refuse") return { released: false, reason: plan.reason, user: binding.user };
7847
+ if (plan.action === "kill") {
7848
+ if (!(await this.killThreadUserProcesses(plan.user, threadKey)))
7849
+ return { error: "pool-owner-mismatch: force detach refused before kill", status: 503 };
7850
+ console.log(
7851
+ `detach: force — killed ${plan.user}'s processes for ${threadKey} (${plan.inFlight} op(s) were in flight)`,
7852
+ );
7853
+ const left = await this.waitForThreadDrain(threadKey);
7854
+ if (left > 0) return { released: false, reason: busyAfterKillReason(left), user: binding.user };
7855
+ }
7856
+ const active = await this.isRuntimeActive().catch(() => false);
7857
+ // What the tree still holds, for the answer and the eviction's record —
7858
+ // never a reason to keep it. Not measured on a force release: a read-only
7859
+ // tree holds nothing, and a hard stop's tree is whatever the killed
7860
+ // command left. A probe that fails names nothing in the answer (never a
7861
+ // guess); the record and the log say it could not be measured.
7862
+ let tree: EvictedTree | undefined;
7863
+ if (!force && active) tree = await this.measureTreeBeforeEviction(binding);
7864
+ // Re-check right before removal: the measurement above awaited (the DO
7865
+ // yields at each await), so an exec that arrived mid-detach would otherwise
7866
+ // have its tree removed under it.
7867
+ const busyNow = this.threadOpsInFlight.get(threadKey) ?? 0;
7868
+ if (busyNow > 0)
7869
+ return {
7870
+ released: false,
7871
+ reason: `busy: ${busyNow} operation(s) started during detach — kept`,
7872
+ user: binding.user,
7873
+ };
7874
+ // Same re-read as the sweep: a re-attach during the measurement means a
7875
+ // fresh tree we must not remove from a stale snapshot.
7876
+ const current = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
7877
+ if (!current || current.evicted) return { released: false, reason: "already-evicted" };
7878
+ if (current.lastAttachAt !== binding.lastAttachAt)
7879
+ return { released: false, reason: "re-attached during the detach — kept", user: current.user };
7880
+ const user = current.user;
7881
+ // Same as the sweep: `active` was read before the measurement's awaits; a
7882
+ // container that woke meanwhile must get the rm, not an orphaned tree.
7883
+ const activeNow = await this.isRuntimeActive().catch(() => true);
7884
+ const eviction = await this.evictBinding(current, activeNow, `detach`, "detach", tree);
7885
+ if (eviction === "cleanup-failed")
7886
+ return { released: false, reason: "thread-cleanup-failed: pool user kept", user };
7887
+ if (eviction === "changed") return { released: false, reason: "re-attached during eviction — kept", user };
7888
+ // Item 55: the tree is gone; the gauge catches up at the next refresh
7889
+ // instance's `measure` step, and the admission's `df` sees the space now.
7890
+ const leftBehind = tree && "leftBehind" in tree ? tree.leftBehind : undefined;
7891
+ return { released: true, user, ...(leftBehind !== undefined ? { leftBehind } : {}) };
7892
+ });
7825
7893
  }
7826
7894
 
7827
7895
  /** Force-detach's kill: end every process owned by the pool user —
@@ -7936,25 +8004,74 @@ export class ResidentDO extends Sandbox<Env> {
7936
8004
  async getInFlightCount(): Promise<number> {
7937
8005
  return this.inFlightCount() + (await this.registeredRunsBeyondOps());
7938
8006
  }
7939
- /** A run holds its worktree from `/attach` to its release (`/detach`, the
7940
- * sweep, disk pressure — every path ends in `evictBinding`, which clears
7941
- * the row). Written by the attach; idempotent, a re-attach refreshes it. */
7942
- private async registerRun(threadKey: string): Promise<void> {
7943
- await this.ctx.storage.put(runRegKey(threadKey), {
7944
- threadKey,
7945
- registeredAt: new Date(systemClock()).toISOString(),
7946
- } satisfies RunRegistration);
8007
+ /** A run's registration protects its worktree until release or the bounded
8008
+ * deadline. `/detach`, the sweep, and disk pressure end in `evictBinding`,
8009
+ * which clears the row; a re-attach refreshes it. */
8010
+ private async registerRun(
8011
+ threadKey: string,
8012
+ runBudgetMs?: number,
8013
+ runId?: string,
8014
+ ownerGen?: string,
8015
+ ownerFence?: number,
8016
+ ): Promise<void> {
8017
+ const now = systemClock();
8018
+ await this.ctx.storage.transaction(async (txn) => {
8019
+ if (ownerFence !== undefined) await txn.put(runFenceKey(threadKey), { runId, ownerGen, ownerFence });
8020
+ await txn.put(runRegKey(threadKey), {
8021
+ threadKey,
8022
+ registeredAt: new Date(now).toISOString(),
8023
+ ...(runBudgetMs !== undefined ? { deadlineAt: now + runBudgetMs } : {}),
8024
+ ...(runId !== undefined ? { runId } : {}),
8025
+ ...(ownerGen !== undefined ? { ownerGen } : {}),
8026
+ ...(ownerFence !== undefined ? { ownerFence } : {}),
8027
+ } satisfies RunRegistration);
8028
+ });
8029
+ }
8030
+ /** The harness's lease begins after attach. Tighten the provisional setup
8031
+ * bound to its actual remaining clock, only for the run that attached. */
8032
+ async updateRunDeadline(
8033
+ threadKey: string,
8034
+ runId: string,
8035
+ remainingMs: number,
8036
+ ownerGen?: string,
8037
+ ownerFence?: number,
8038
+ ): Promise<{ deadlineAt: number } | ThreadErr> {
8039
+ return this.ctx.storage.transaction(async (txn) => {
8040
+ const registration = await txn.get<RunRegistration>(runRegKey(threadKey));
8041
+ if (
8042
+ !registration ||
8043
+ registration.runId !== runId ||
8044
+ registration.ownerGen !== ownerGen ||
8045
+ registration.ownerFence !== ownerFence
8046
+ )
8047
+ return { error: "run-registration-mismatch: this run does not own the thread", status: 409 };
8048
+ const deadlineAt = systemClock() + remainingMs;
8049
+ await txn.put(runRegKey(threadKey), { ...registration, deadlineAt });
8050
+ return { deadlineAt };
8051
+ });
7947
8052
  }
7948
8053
  /** The registrations the op counters do not already see: a thread with an
7949
8054
  * op in flight is counted by `runsInFlightCount`, so its registration is
7950
- * not counted again. Deliberately NOT part of `inFlightCount()`: the
7951
- * container-lifecycle predicates (isIdle, reconcileImage) must not let a
7952
- * stale registration pin a container awake — the clean-idle sweep is what
7953
- * drains a registration whose release never came. */
8055
+ * not counted again. The idle, image-reconcile and disk-recycle gates read
8056
+ * this bounded count alongside their op counters, so a live run protects
8057
+ * its container while a stale registration cannot pin it awake. */
7954
8058
  private async registeredRunsBeyondOps(): Promise<number> {
7955
8059
  const regs = await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX });
8060
+ const cutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
7956
8061
  let n = 0;
7957
- for (const r of regs.values()) if ((this.threadOpsInFlight.get(r.threadKey) ?? 0) === 0) n++;
8062
+ for (const r of regs.values()) {
8063
+ const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(r.threadKey));
8064
+ if (
8065
+ registeredRunNeedsProtection(
8066
+ binding?.lastAttachAt,
8067
+ this.threadOpsInFlight.get(r.threadKey) ?? 0,
8068
+ cutoff,
8069
+ r.deadlineAt === undefined ? undefined : r.deadlineAt + RUN_REGISTRATION_GRACE_MS,
8070
+ systemClock(),
8071
+ )
8072
+ )
8073
+ n++;
8074
+ }
7958
8075
  return n;
7959
8076
  }
7960
8077
  private attachesInFlight = 0;
@@ -7983,6 +8100,15 @@ export class ResidentDO extends Sandbox<Env> {
7983
8100
  const idleCutoff = systemClock() - CLEAN_IDLE_RELEASE_S * 1000;
7984
8101
  for (const binding of all.values()) {
7985
8102
  if (binding.evicted || !binding.user) continue;
8103
+ const registration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
8104
+ if (
8105
+ registration?.deadlineAt !== undefined &&
8106
+ Number.isSafeInteger(registration.deadlineAt) &&
8107
+ systemClock() <= registration.deadlineAt + RUN_REGISTRATION_GRACE_MS
8108
+ ) {
8109
+ kept++;
8110
+ continue;
8111
+ }
7986
8112
  const last = Date.parse(binding.lastAttachAt);
7987
8113
  if (last >= cutoff) {
7988
8114
  // Not past the TTL. Still release it if it has been idle for an hour
@@ -7991,9 +8117,9 @@ export class ResidentDO extends Sandbox<Env> {
7991
8117
  // a clean tree (item 17) — so there is nothing to keep it for. This is
7992
8118
  // what drains the bindings of runs whose release never came (a
7993
8119
  // resident that was sick at the run's end, a run older than
7994
- // `/detach`). A live run is protected by its op in flight, not by its
7995
- // dirt. A slept container has no tree any more anyway (sleep destroys
7996
- // the disk).
8120
+ // `/detach`). A live run is protected by its budget above or an op
8121
+ // in flight, not by its dirt. A slept container has no tree any more
8122
+ // anyway (sleep destroys the disk).
7997
8123
  const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
7998
8124
  if (last >= idleCutoff || busy > 0) {
7999
8125
  kept++;
@@ -8390,13 +8516,30 @@ export class ResidentDO extends Sandbox<Env> {
8390
8516
  fates.set(ref, { fate: looked.fate, detail });
8391
8517
  }),
8392
8518
  );
8393
- // The runs registered on this resident (item 44): a run holds its tree
8394
- // from its attach to its release, and its process lives in the container
8519
+ // The runs registered on this resident (item 44): a live registration
8520
+ // holds its tree through the deadline, and its process lives in the container
8395
8521
  // between the bot's calls — the op counters read 0 while its model thinks.
8522
+ const registrations = new Map(
8523
+ [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map((r) => [
8524
+ r.threadKey,
8525
+ r,
8526
+ ]),
8527
+ );
8528
+ const protectsTree = (binding: ThreadBinding, registration: RunRegistration | undefined): boolean => {
8529
+ if (!registration) return false;
8530
+ const now = systemClock();
8531
+ return registeredRunNeedsProtection(
8532
+ binding.lastAttachAt,
8533
+ 0,
8534
+ now - CLEAN_IDLE_RELEASE_S * 1000,
8535
+ registration.deadlineAt === undefined ? undefined : registration.deadlineAt + RUN_REGISTRATION_GRACE_MS,
8536
+ now,
8537
+ );
8538
+ };
8396
8539
  const registered = new Set(
8397
- [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map(
8398
- (r) => r.threadKey,
8399
- ),
8540
+ live
8541
+ .filter((binding) => protectsTree(binding, registrations.get(binding.threadKey)))
8542
+ .map((binding) => binding.threadKey),
8400
8543
  );
8401
8544
  for (const binding of live) {
8402
8545
  const isDefaultRef = binding.ref === defaultRef;
@@ -8423,7 +8566,8 @@ export class ResidentDO extends Sandbox<Env> {
8423
8566
  continue;
8424
8567
  }
8425
8568
  // A run attached while the tree was measured holds it now (item 44).
8426
- if ((await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey))) !== undefined) {
8569
+ const currentRegistration = await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey));
8570
+ if (protectsTree(current ?? binding, currentRegistration)) {
8427
8571
  kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "run-held" });
8428
8572
  continue;
8429
8573
  }
@@ -9249,6 +9393,7 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
9249
9393
  "/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
9250
9394
  "/status": { scope: "operator", method: "GET" },
9251
9395
  "/attach": { scope: "operator", method: "POST" },
9396
+ "/run-deadline": { scope: "operator", method: "POST" },
9252
9397
  "/detach": { scope: "operator", method: "POST" },
9253
9398
  "/exec": { scope: "operator", method: "POST" },
9254
9399
  "/read": { scope: "operator", method: "POST" },
@@ -9366,6 +9511,8 @@ export default {
9366
9511
  return await handleStatus(env, url);
9367
9512
  case "/attach":
9368
9513
  return await handleAttach(env, body, traceparent);
9514
+ case "/run-deadline":
9515
+ return await handleRunDeadline(env, body);
9369
9516
  case "/detach":
9370
9517
  return await handleDetach(env, body);
9371
9518
  case "/exec":
@@ -9996,6 +10143,27 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
9996
10143
  if ("error" in want) return json({ error: want.error }, 400);
9997
10144
  const reuse = parseReuse(body.reuse);
9998
10145
  if ("error" in reuse) return json({ error: reuse.error }, 400);
10146
+ let runBudgetMs: number | undefined;
10147
+ if (body.runBudgetMs !== undefined) {
10148
+ const value = body.runBudgetMs;
10149
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0 || value > DAY_MS)
10150
+ return json({ error: "invalid run budget" }, 400);
10151
+ runBudgetMs = value;
10152
+ }
10153
+ const runId = body.runId;
10154
+ if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
10155
+ return json({ error: "invalid run id" }, 400);
10156
+ const ownerGen = body.ownerGen;
10157
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10158
+ return json({ error: "invalid run generation" }, 400);
10159
+ const ownerFence = body.ownerFence;
10160
+ if (
10161
+ ownerFence !== undefined &&
10162
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10163
+ )
10164
+ return json({ error: "invalid run fence" }, 400);
10165
+ if (ownerFence !== undefined && (runId === undefined || ownerGen === undefined))
10166
+ return json({ error: "run fence requires a run id and generation" }, 400);
9999
10167
  // Why the hint is what it is (item 16): the thread's own pull request and
10000
10168
  // its head branch — the branch checked against the one ref pattern like
10001
10169
  // every ref, before it can become a git argument — and the bound-by-default flag.
@@ -10050,6 +10218,10 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
10050
10218
  traceparent,
10051
10219
  reason,
10052
10220
  githubDoor,
10221
+ runBudgetMs,
10222
+ runId,
10223
+ ownerGen,
10224
+ ownerFence,
10053
10225
  ),
10054
10226
  ),
10055
10227
  ({ result, levels }) => ({ ...(result as object), ...(levels ? { levels } : {}) }),
@@ -10057,6 +10229,29 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
10057
10229
  );
10058
10230
  }
10059
10231
 
10232
+ async function handleRunDeadline(env: Env, body: Record<string, unknown>): Promise<Response> {
10233
+ const ctx = await resolveThreadRoute(env, body);
10234
+ if (ctx instanceof Response) return ctx;
10235
+ const runId = body.runId;
10236
+ const ownerGen = body.ownerGen;
10237
+ const ownerFence = body.ownerFence;
10238
+ const remainingMs = body.remainingMs;
10239
+ if (typeof runId !== "string" || runId.length === 0 || runId.length > 128)
10240
+ return json({ error: "invalid run id" }, 400);
10241
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10242
+ return json({ error: "invalid run generation" }, 400);
10243
+ if (
10244
+ ownerFence !== undefined &&
10245
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10246
+ )
10247
+ return json({ error: "invalid run fence" }, 400);
10248
+ if (typeof remainingMs !== "number" || !Number.isSafeInteger(remainingMs) || remainingMs <= 0 || remainingMs > DAY_MS)
10249
+ return json({ error: "invalid remaining lease" }, 400);
10250
+ const result = await ctx.stub.updateRunDeadline(ctx.threadKey, runId, remainingMs, ownerGen, ownerFence);
10251
+ if ("error" in result) return threadErrResponse(result);
10252
+ return json(result);
10253
+ }
10254
+
10060
10255
  /** POST /await-restore (docs/reference/specs/execution.md item 27): the bot's
10061
10256
  * one held request while the resident restores. Held server-side by the DO's
10062
10257
  * waiter ledger; the answer streams like /attach (heartbeat whitespace then
@@ -10078,6 +10273,18 @@ async function handleAwaitRestore(env: Env, body: Record<string, unknown>): Prom
10078
10273
  async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Response> {
10079
10274
  const ctx = await resolveThreadRoute(env, body);
10080
10275
  if (ctx instanceof Response) return ctx;
10276
+ const runId = body.runId;
10277
+ if (runId !== undefined && (typeof runId !== "string" || runId.length === 0 || runId.length > 128))
10278
+ return json({ error: "invalid run id" }, 400);
10279
+ const ownerGen = body.ownerGen;
10280
+ if (ownerGen !== undefined && (typeof ownerGen !== "string" || ownerGen.length === 0 || ownerGen.length > 128))
10281
+ return json({ error: "invalid run generation" }, 400);
10282
+ const ownerFence = body.ownerFence;
10283
+ if (
10284
+ ownerFence !== undefined &&
10285
+ (typeof ownerFence !== "number" || !Number.isSafeInteger(ownerFence) || ownerFence <= 0)
10286
+ )
10287
+ return json({ error: "invalid run fence" }, 400);
10081
10288
  // What the run pushed (item 16a), each ref through the one ref pattern
10082
10289
  // before it can be stored or become a git argument.
10083
10290
  const pushed = parsePushed(body.pushed);
@@ -10086,7 +10293,14 @@ async function handleDetach(env: Env, body: Record<string, unknown>): Promise<Re
10086
10293
  const ref = parseRef(entry.ref, "pushed[].ref");
10087
10294
  if ("error" in ref) return json({ error: ref.error }, 400);
10088
10295
  }
10089
- const result = await ctx.stub.detachThread(ctx.threadKey, body.force === true, pushed.pushed);
10296
+ const result = await ctx.stub.detachThread(
10297
+ ctx.threadKey,
10298
+ body.force === true,
10299
+ pushed.pushed,
10300
+ runId,
10301
+ ownerGen,
10302
+ ownerFence,
10303
+ );
10090
10304
  if ("error" in result) return threadErrResponse(result);
10091
10305
  return json(result);
10092
10306
  }