chainlesschain 0.162.185 → 0.162.187

package/src/lib/agent-team/team-runner.js

@@ -42,13 +42,34 @@ export class TeamRunner {
     // Team-wide token/USD/time/task budget (null → unbounded). Consulted before
     // every claim; folded after every settled task.
     this.budget = opts.budget || null;
+    this.budgetForTask =
+      typeof opts.budgetForTask === "function" ? opts.budgetForTask : null;
     // Directed/broadcast messaging between teammates (null → disabled).
     this.mailbox = opts.mailbox || null;
+    // Optional conservative path ownership. Tasks with overlapping scopes are
+    // serialized in a shared workspace; an empty scope means the whole repo.
+    this.scopeLock = opts.scopeLock || null;
+    this.scopeForTask =
+      typeof opts.scopeForTask === "function"
+        ? opts.scopeForTask
+        : (task) => task?.metadata?.scopePaths || [];
+    this.maxScopeScan =
+      Number.isSafeInteger(opts.maxScopeScan) && opts.maxScopeScan > 0
+        ? opts.maxScopeScan
+        : 1024;
     // Optional TelemetryRecorder: emits a `team.task` span per execution (and
     // inherits the recorder's default workflow.run_id/name attributes) so
     // `cc team run --otlp` traces the whole graph. Null → zero cost.
     this.recorder = opts.recorder || null;
+    // Durable state hooks. `beforeTask` runs after lease acquisition but before
+    // the executor (so a claim can be persisted before side effects);
+    // `afterTask` runs after the authoritative registry settlement.
+    this.beforeTask =
+      typeof opts.beforeTask === "function" ? opts.beforeTask : null;
+    this.afterTask =
+      typeof opts.afterTask === "function" ? opts.afterTask : null;
     this._executions = 0;
+    this._reservedExecutions = 0;
     this._inFlight = 0;
     this._maxInFlight = 0;
     // Keys executing in this runner process. An event-loop stall may let a
@@ -57,6 +78,14 @@ export class TeamRunner {
     this._activeKeys = new Set();
     this._members = new Map(); // holder → lifecycle record
     this._budgetStopped = false;
+    this._fatalError = null;
+    // Idle workers wait on graph progress instead of polling the full graph
+    // through setTimeout(0). This matters once a team has many workers but only
+    // a narrow DAG frontier.
+    this._progressWaiters = new Set();
+    this._workerCount = 1;
+    this._claimControllers = new Set();
+    this._wallTimer = null;
   }
 
   _emit(type, extra) {
@@ -131,27 +160,95 @@ export class TeamRunner {
     if (typeof this.runTask !== "function") {
       throw new Error("TeamRunner: opts.runTask is required");
     }
+    const initialScopeStatus = this.scopeLock?.status?.();
+    if (initialScopeStatus?.count > 0) {
+      const error = new Error(
+        "TeamRunner cannot start with orphaned scope ownership",
+      );
+      error.code = "TEAM_SCOPE_LOCK_ORPHANED";
+      throw error;
+    }
     this._emit("run:start", {
       teammates: this.teammates,
       tasks: this.registry.list().length,
     });
-    if (this.budget) this.budget.start();
+    if (this.budget) {
+      this.budget.start();
+      const maxWallMs = Number(this.budget.maxWallMs);
+      if (Number.isFinite(maxWallMs) && maxWallMs > 0) {
+        const exhaustWallBudget = () => {
+          if (!this._budgetStopped) {
+            this._budgetStopped = true;
+            this._emit("run:budget-exhausted", {
+              reason: "max-wall-ms",
+            });
+          }
+          for (const controller of this._claimControllers) {
+            try {
+              controller.abort(new Error("Team wall-clock budget exhausted"));
+            } catch {
+              /* every settlement remains fenced by its lease id */
+            }
+          }
+          this._signalProgress();
+        };
+        const armWallTimer = () => {
+          const elapsedMs = this.budget.status().elapsedMs;
+          const remainingMs = maxWallMs - elapsedMs;
+          if (remainingMs <= 0) {
+            exhaustWallBudget();
+            return;
+          }
+          // Node timers are signed 32-bit. Re-arm long budgets instead of
+          // truncating them, and charge only the active time restored from a
+          // prior process (not the downtime between runs).
+          this._wallTimer = setTimeout(
+            armWallTimer,
+            Math.min(remainingMs, 2_147_483_647),
+          );
+        };
+        armWallTimer();
+      }
+    }
     const workers = [];
-    for (let i = 0; i < this.teammates; i++) {
-      const holder = `teammate-${i + 1}`;
+    const workerCount = Math.min(
+      this.teammates,
+      Math.max(1, this.registry.list().length),
+    );
+    this._workerCount = workerCount;
+    const holders = Array.from(
+      { length: workerCount },
+      (_, index) => `teammate-${index + 1}`,
+    );
+    this.mailbox?.registerRecipients?.(holders);
+    for (const holder of holders) {
       this._setState(holder, "idle");
       workers.push(this._worker(holder));
     }
-    await Promise.all(workers);
+    try {
+      await Promise.all(workers);
+    } finally {
+      if (this._wallTimer) {
+        clearTimeout(this._wallTimer);
+        this._wallTimer = null;
+      }
+    }
+    if (this._fatalError) throw this._fatalError;
     const stats = this.registry.stats();
+    const done = this.registry.allDone();
     const summary = {
-      done: this.registry.allDone(),
+      done,
+      success: done && stats.completed === stats.total,
       executions: this._executions,
       maxConcurrent: this._maxInFlight,
+      requestedTeammates: this.teammates,
+      activeTeammates: workerCount,
       budgetStopped: this._budgetStopped,
       budgetReason: this.budget ? this.budget.reason() : null,
       members: this.members(),
       messages: this.mailbox ? this.mailbox.size() : 0,
+      mailboxStatus: this.mailbox?.status?.() || null,
+      scopeOwnership: this.scopeLock?.status?.() || null,
       stats,
     };
     this._emit("run:end", summary);
@@ -164,11 +261,29 @@ export class TeamRunner {
     // its dep will finish — so a worker that finds nothing claimable yields and
     // re-checks while any work is still in flight.
     for (;;) {
-      if (this._executions >= this.maxTasks) {
+      if (this._fatalError) {
+        this._setState(holder, "shutdown", { reason: "fatal-error" });
+        return;
+      }
+      if (this._executions + this._reservedExecutions >= this.maxTasks) {
         this._emit("run:budget-exhausted", { maxTasks: this.maxTasks });
         this._setState(holder, "shutdown", { reason: "max-tasks" });
         return;
       }
+      const budgetStatus = this.budget?.status?.();
+      if (
+        budgetStatus?.maxTasks != null &&
+        Math.max(budgetStatus.tasks, this._executions) +
+          this._reservedExecutions >=
+          budgetStatus.maxTasks
+      ) {
+        if (!this._budgetStopped) {
+          this._budgetStopped = true;
+          this._emit("run:budget-exhausted", { reason: "max-tasks" });
+        }
+        this._setState(holder, "shutdown", { reason: "max-tasks" });
+        return;
+      }
       // Team budget (token/USD/time/task) — stop CLAIMING once any cap is hit so
       // the team overshoots by at most the tasks already in flight.
       if (this.budget && this.budget.shouldStop()) {
@@ -184,11 +299,21 @@ export class TeamRunner {
       if (!key) {
         // Nothing to claim right now. If peers are still working, wait and
         // retry (their completions may unblock a dependent). Otherwise we're done.
-        if (this._inFlight > 0) {
+        const heldScopes = this.scopeLock?.status?.().count || 0;
+        if (this._inFlight > 0 || this._reservedExecutions > 0) {
           this._setState(holder, "idle");
-          await this._tick();
+          await this._waitForProgress();
           continue;
         }
+        if (heldScopes > 0) {
+          const error = new Error(
+            "TeamRunner found scope ownership without a live local claim",
+          );
+          error.code = "TEAM_SCOPE_LOCK_ORPHANED";
+          this._setFatal(error, { phase: "scope-reconciliation" });
+          this._setState(holder, "shutdown", { reason: "fatal-error" });
+          return;
+        }
         this._setState(holder, "shutdown", { reason: "no-more-work" });
         return; // no claimable + nothing in flight → this worker is finished
       }
@@ -199,12 +324,262 @@ export class TeamRunner {
         await this._tick();
         continue;
       }
-      await this._execute(holder, key, acq.lease);
+      const budgetReservation = this._reserveTaskBudget(
+        key,
+        this.registry.getTask(key),
+        acq.lease,
+      );
+      if (!budgetReservation.ok) {
+        this.registry.release(key, {
+          holder,
+          leaseId: acq.lease.leaseId,
+        });
+        if (budgetReservation.temporary) {
+          this._setState(holder, "idle");
+          await this._waitForProgress();
+          continue;
+        }
+        if (!this._budgetStopped) {
+          this._budgetStopped = true;
+          this._emit("run:budget-exhausted", {
+            reason: budgetReservation.reason,
+          });
+        }
+        this._setState(holder, "shutdown", {
+          reason: budgetReservation.reason,
+        });
+        return;
+      }
+      this._reservedExecutions++;
+      const claim = this._beginClaim(holder, key, acq.lease, budgetReservation);
+      if (!this._acquireScope(holder, key)) {
+        this._abandonClaim(holder, key, claim);
+        if (this._fatalError) {
+          this._setState(holder, "shutdown", { reason: "fatal-error" });
+          return;
+        }
+        this._setState(holder, "idle");
+        await this._waitForProgress();
+        continue;
+      }
+      if (!(await this._prepareTask(holder, key, claim))) {
+        this._abandonClaim(holder, key, claim);
+        return;
+      }
+      if (this._fatalError || claim.lost) {
+        if (claim.lost && !this._fatalError) {
+          const error = new Error(
+            `Lease for task "${key}" was lost while preparing it`,
+          );
+          error.code = "TEAM_TASK_LEASE_LOST";
+          this._setFatal(error, { phase: "before-task", key, holder });
+        }
+        this._abandonClaim(holder, key, claim);
+        this._setState(holder, "shutdown", { reason: "fatal-error" });
+        return;
+      }
+      await this._execute(holder, key, claim);
+    }
+  }
+
+  async _prepareTask(holder, key, claim) {
+    if (!this.beforeTask) return true;
+    try {
+      await this.beforeTask({
+        key,
+        task: this.registry.getTask(key),
+        holder,
+        lease: this.registry.getTask(key)?.lease || claim.lease,
+      });
+      return true;
+    } catch (error) {
+      this._setFatal(error, { phase: "before-task", key, holder });
+      this._setState(holder, "shutdown", { reason: "fatal-error" });
+      return false;
+    }
+  }
+
+  _setFatal(error, extra = {}) {
+    if (this._fatalError) return;
+    const failure =
+      error instanceof Error ? error : new Error(String(error || "team fatal"));
+    this._fatalError = failure;
+    this._emit("run:fatal", {
+      ...extra,
+      error: failure.message,
+    });
+    this._signalProgress();
+  }
+
+  _reserveTaskBudget(key, task, lease) {
+    if (!this.budget?.reserve) return { ok: true, id: null };
+    const requested = this.budgetForTask?.(task, key) || {};
+    if (requested.reserveUsage === false) return { ok: true, id: null };
+    const availableWorkers = Math.max(
+      1,
+      this._workerCount - this._inFlight - this._reservedExecutions,
+    );
+    const readyAfterAcquire =
+      typeof this.registry.claimableCount === "function"
+        ? this.registry.claimableCount()
+        : availableWorkers - 1;
+    const slots = Math.max(
+      1,
+      Math.min(availableWorkers, readyAfterAcquire + 1),
+    );
+    const reservation = this.budget.reserve(lease.leaseId, {
+      maxTokens: requested.maxTokens,
+      maxUsd: requested.maxBudgetUsd ?? requested.maxUsd,
+      slots,
+    });
+    if (!reservation.ok) return reservation;
+    return {
+      ...reservation,
+      maxBudgetUsd: reservation.maxUsd,
+    };
+  }
+
+  _beginClaim(holder, key, lease, budgetReservation = null) {
+    const claim = {
+      holder,
+      key,
+      lease,
+      leaseId: lease?.leaseId,
+      heartbeat: null,
+      lost: false,
+      abortController: new AbortController(),
+      budgetReservation,
+      budgetSettled: false,
+    };
+    // Fence the task immediately after acquire, before any async durable hook.
+    // Otherwise a slow beforeTask can outlive the TTL and a peer can run the
+    // same side effect concurrently.
+    this._activeKeys.add(key);
+    this._claimControllers.add(claim.abortController);
+    const effectiveTtl =
+      this.ttlMs > 0 ? this.ttlMs : this.registry.defaultTtlMs || 60000;
+    const every = Math.max(
+      25,
+      Math.floor(this.renewEveryMs > 0 ? this.renewEveryMs : effectiveTtl / 3),
+    );
+    claim.heartbeat = setInterval(() => {
+      const result = this._renewClaim(claim);
+      if (!result?.ok) clearInterval(claim.heartbeat);
+    }, every);
+    if (typeof claim.heartbeat.unref === "function") claim.heartbeat.unref();
+    return claim;
+  }
+
+  _renewClaim(claim) {
+    if (!claim || claim.lost) {
+      return { ok: false, reason: "lease_lost" };
+    }
+    let result;
+    try {
+      result = this.registry.renew(claim.key, {
+        holder: claim.holder,
+        leaseId: claim.leaseId,
+        ttlMs: this.ttlMs,
+      });
+      // A delayed heartbeat may wake just after expiry. Reacquire creates a new
+      // fencing token and succeeds only if no peer has taken the task.
+      if (!result?.ok && result?.reason === "not_holder_or_expired") {
+        result = this.registry.acquire(claim.key, {
+          holder: claim.holder,
+          ttlMs: this.ttlMs,
+        });
+        if (result?.ok) {
+          claim.lease = result.lease;
+          claim.leaseId = result.lease.leaseId;
+        }
+      } else if (result?.ok) {
+        claim.lease = result.lease;
+      }
+    } catch {
+      result = { ok: false, reason: "renew_failed" };
+    }
+    if (!result?.ok) {
+      claim.lost = true;
+      try {
+        claim.abortController.abort(
+          new Error(`Lease for task "${claim.key}" was lost`),
+        );
+      } catch {
+        /* the fencing token still prevents stale settlement */
+      }
+      this._signalProgress();
+    }
+    return result;
+  }
+
+  _endClaim(claim) {
+    if (!claim) return;
+    if (claim.heartbeat) clearInterval(claim.heartbeat);
+    this._activeKeys.delete(claim.key);
+    this._claimControllers.delete(claim.abortController);
+  }
+
+  _abandonClaim(holder, key, claim) {
+    this.registry.release(key, {
+      holder,
+      leaseId: claim?.leaseId,
+    });
+    this._releaseScope(holder, key);
+    if (claim?.budgetReservation?.id) {
+      this.budget?.releaseReservation?.(claim.budgetReservation.id);
+    }
+    this._endClaim(claim);
+    this._reservedExecutions = Math.max(0, this._reservedExecutions - 1);
+    this._signalProgress();
+  }
+
+  async _notifySettlement(settlement) {
+    if (!this.afterTask) return;
+    try {
+      await this.afterTask(settlement);
+    } catch (error) {
+      this._setFatal(error, {
+        phase: "after-task",
+        key: settlement.key,
+        holder: settlement.holder,
+      });
     }
   }
 
   /** Pick a claimable task key for this holder (highest priority first). */
   _nextFor() {
+    // A whole-workspace owner blocks every other scope, so avoid walking a
+    // large ready queue only to rediscover the same conflict for each task.
+    if (
+      this.scopeLock
+        ?.status?.()
+        ?.locks?.some((candidate) => candidate.workspace === true)
+    ) {
+      return null;
+    }
+    if (typeof this.registry.nextClaimable === "function") {
+      const excluded = new Set(this._activeKeys);
+      for (let scanned = 0; scanned < this.maxScopeScan; scanned++) {
+        const key = this.registry.nextClaimable({ excludeKeys: excluded });
+        if (!key || !this.scopeLock) return key;
+        const task = this.registry.getTask(key);
+        const check = this.scopeLock.canAcquire(
+          key,
+          this.scopeForTask(task, key),
+        );
+        if (check.ok) return key;
+        if (check.code !== "TEAM_SCOPE_LOCK_SCOPE_CONFLICT") {
+          const error = new Error(
+            check.message || `invalid task scope ownership for "${key}"`,
+          );
+          error.code = check.code || "TEAM_SCOPE_LOCK_INVALID_SCOPES";
+          this._setFatal(error, { phase: "scope-validation", key });
+          return null;
+        }
+        excluded.add(key);
+      }
+      return null;
+    }
     const claimable = this.registry.claimable();
     if (claimable.length === 0) return null;
     // Stable: prefer higher priority, else insertion order (claimable order).
@@ -223,55 +598,82 @@ export class TeamRunner {
     return best;
   }
 
-  async _execute(holder, key, lease) {
+  _acquireScope(holder, key) {
+    if (!this.scopeLock) return true;
     const task = this.registry.getTask(key);
+    const result = this.scopeLock.acquire(key, this.scopeForTask(task, key));
+    if (!result.ok) {
+      if (result.code !== "TEAM_SCOPE_LOCK_SCOPE_CONFLICT") {
+        const error = new Error(
+          result.message || `invalid task scope ownership for "${key}"`,
+        );
+        error.code = result.code || "TEAM_SCOPE_LOCK_INVALID_SCOPES";
+        this._setFatal(error, {
+          phase: "scope-acquire",
+          key,
+          holder,
+        });
+      }
+      return false;
+    }
+    this._emit("task:scope-acquired", {
+      key,
+      holder,
+      scopes: result.scopes,
+      workspace: result.scopes.length === 0,
+    });
+    return true;
+  }
+
+  _releaseScope(holder, key) {
+    if (!this.scopeLock) return;
+    const released = this.scopeLock.release(key);
+    if (released?.ok) {
+      this._emit("task:scope-released", {
+        key,
+        holder,
+        scopes: released.scopes,
+        workspace: released.workspace,
+      });
+    }
+  }
+
+  async _execute(holder, key, claim) {
+    const task = this.registry.getTask(key);
+    this._reservedExecutions--;
     this._executions++;
     this._inFlight++;
-    this._activeKeys.add(key);
     this._maxInFlight = Math.max(this._maxInFlight, this._inFlight);
     this._setState(holder, "running", { key });
     this._emit("task:claimed", { key, holder, attempts: task.attempts });
-    const renew = () => {
-      const result = this.registry.renew(key, {
-        holder,
-        ttlMs: this.ttlMs,
-      });
-      // A delayed timer may wake just after expiry. Reacquire is safe and
-      // optimistic: it succeeds only if no external runner has taken the
-      // lease; a real takeover remains authoritative and stops the heartbeat.
-      return !result?.ok && result?.reason === "not_holder_or_expired"
-        ? this.registry.acquire(key, { holder, ttlMs: this.ttlMs })
-        : result;
-    };
-    // Heartbeat: the built-in executors (shell / agent / worktree) never call
-    // `renew` themselves, so before this ANY task outliving the lease TTL was
-    // indistinguishable from a crash — a peer could steal it and the SAME task
-    // would run twice concurrently. The runner owns the TTL, so it renews here
-    // for every executor. Stops itself once the lease is lost (renew rejected —
-    // keeping a lost lease alive would be lying to the registry).
-    const effectiveTtl =
-      this.ttlMs > 0 ? this.ttlMs : this.registry.defaultTtlMs || 60000;
-    const every = Math.max(
-      25,
-      Math.floor(this.renewEveryMs > 0 ? this.renewEveryMs : effectiveTtl / 3),
-    );
-    const heartbeat = setInterval(() => {
-      try {
-        const r = renew();
-        if (!r || r.ok !== true) clearInterval(heartbeat);
-      } catch {
-        clearInterval(heartbeat);
-      }
-    }, every);
-    // Never hold the process open just for a heartbeat.
-    if (typeof heartbeat.unref === "function") heartbeat.unref();
+    const renew = () => this._renewClaim(claim);
     // A teammate-scoped messaging handle: post to a peer / broadcast, and read
     // its own inbox (direct messages + unseen broadcasts).
     const inbox = this.mailbox ? this.mailbox.drain(holder) : [];
-    const sendMessage = (to, body, subject = null) =>
-      this.mailbox
-        ? this.mailbox.send({ from: holder, to, subject, body })
-        : null;
+    const sendMessage = (to, body, subject = null) => {
+      if (!this.mailbox) return null;
+      let message;
+      try {
+        message = this.mailbox.send({ from: holder, to, subject, body });
+      } catch (error) {
+        this._emit("mailbox:backpressure", {
+          holder,
+          code: error?.code || "TEAM_MAILBOX_SEND_FAILED",
+          status: this.mailbox.status?.() || null,
+        });
+        throw error;
+      }
+      const pressure = this.mailbox.pressure?.();
+      if (pressure && pressure.level !== "normal") {
+        this._emit("mailbox:pressure", {
+          holder,
+          level: pressure.level,
+          ratio: pressure.ratio,
+          messages: this.mailbox.size(),
+        });
+      }
+      return message;
+    };
     const startedAt = this._now();
     const span = this.recorder
       ? this.recorder.startSpan("team.task", {
@@ -290,8 +692,9 @@ export class TeamRunner {
         sendMessage,
         mailbox: this.mailbox,
         budget: this.budget,
+        budgetReservation: claim.budgetReservation,
+        signal: claim.abortController.signal,
       });
-      const done = this.registry.complete(key, { holder, result });
       // Fold usage/cost into the team budget regardless — the task DID execute
       // and consumed resources. `--exec` shell tasks carry no usage → only the
       // task-count / wall-clock dimensions move.
@@ -299,12 +702,28 @@ export class TeamRunner {
         this.budget.record(
           {
             usage: result?.usage || null,
+            usageRecords: result?.usageRecords || null,
             provider: result?.provider,
             model: result?.model,
+            reservationId: claim.budgetReservation?.id,
           },
           this._now(),
         );
+        claim.budgetSettled = true;
+        if (this.budget.reason() === "unpriced-usage") {
+          const error = new Error(
+            "Team USD budget cannot account for unpriced remote usage",
+          );
+          error.code = "TEAM_BUDGET_UNPRICED_USAGE";
+          error.retryable = false;
+          throw error;
+        }
       }
+      const done = this.registry.complete(key, {
+        holder,
+        leaseId: claim.leaseId,
+        result,
+      });
       if (done.ok) {
         if (span) span.end();
         this._setState(holder, "completed-task");
@@ -313,6 +732,12 @@ export class TeamRunner {
           holder,
           ms: this._now() - startedAt,
         });
+        await this._notifySettlement({
+          key,
+          holder,
+          status: "completed",
+          result,
+        });
       } else {
         if (span) {
           span.setAttribute("team.completion_discarded", true);
@@ -330,6 +755,12 @@ export class TeamRunner {
           reason: done.reason,
           ms: this._now() - startedAt,
         });
+        await this._notifySettlement({
+          key,
+          holder,
+          status: "completion-discarded",
+          reason: done.reason,
+        });
       }
     } catch (err) {
       if (span) {
@@ -338,28 +769,81 @@ export class TeamRunner {
       }
       // A failed task still consumed a task-count slot (and any wall-clock) — fold
       // it so a doomed retry loop can't dodge the budget.
-      if (this.budget) this.budget.record({ usage: null }, this._now());
+      if (this.budget && !claim.budgetSettled) {
+        this.budget.record(
+          {
+            usage: err?.usage || null,
+            usageRecords: err?.usageRecords || null,
+            provider: err?.provider,
+            model: err?.model,
+            reservationId: claim.budgetReservation?.id,
+          },
+          this._now(),
+        );
+        claim.budgetSettled = true;
+      }
       const outcome = this.registry.fail(key, {
         holder,
+        leaseId: claim.leaseId,
         error: err?.message || String(err),
+        retryable: err?.retryable !== false,
       });
-      this._setState(holder, "failed-task", {
-        error: err?.message || String(err),
-      });
-      this._emit("task:failed", {
-        key,
-        holder,
-        error: err?.message || String(err),
-        retry: outcome?.retry === true,
-        attempts: outcome?.attempts,
-      });
+      if (outcome?.ok) {
+        this._setState(holder, "failed-task", {
+          error: err?.message || String(err),
+        });
+        this._emit("task:failed", {
+          key,
+          holder,
+          error: err?.message || String(err),
+          retry: outcome.retry === true,
+          attempts: outcome.attempts,
+        });
+        await this._notifySettlement({
+          key,
+          holder,
+          status: outcome.retry === true ? "pending" : "failed",
+          error: err?.message || String(err),
+          retry: outcome.retry === true,
+          attempts: outcome.attempts,
+        });
+      } else {
+        this._emit("task:failure-discarded", {
+          key,
+          holder,
+          reason: outcome?.reason || "lease_lost",
+          error: err?.message || String(err),
+          ms: this._now() - startedAt,
+        });
+        await this._notifySettlement({
+          key,
+          holder,
+          status: "failure-discarded",
+          reason: outcome?.reason || "lease_lost",
+        });
+      }
     } finally {
-      clearInterval(heartbeat);
-      this._activeKeys.delete(key);
+      if (!claim.budgetSettled && claim.budgetReservation?.id) {
+        this.budget?.releaseReservation?.(claim.budgetReservation.id);
+      }
+      this._releaseScope(holder, key);
+      this._endClaim(claim);
       this._inFlight--;
+      this._signalProgress();
     }
   }
 
+  _waitForProgress() {
+    return new Promise((resolve) => this._progressWaiters.add(resolve));
+  }
+
+  _signalProgress() {
+    if (this._progressWaiters.size === 0) return;
+    const waiters = Array.from(this._progressWaiters);
+    this._progressWaiters.clear();
+    for (const resolve of waiters) resolve();
+  }
+
   /** Yield to the event loop so peer completions land before re-checking. */
   _tick() {
     return new Promise((resolve) => setTimeout(resolve, 0));