deepline 0.1.230 → 0.1.232

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -89,20 +89,8 @@ export interface RateStateCooldown {
 }
 
 interface LeasedBlock {
-  /**
-   * Concurrency reservations, one per locally-cached permit. Only rules with a
-   * `maxConcurrency` mint lease tokens; each id maps to a durable slot that must
-   * be released (or TTL-expired) so the shared concurrency count drains.
-   */
-  leaseIds: string[];
-  /**
-   * Pure-rate (RPS-only) permits with no concurrency slot behind them. The
-   * server decremented the provider window by this count already, so drawing
-   * one locally is a no-op that needs no release token. Tracked separately from
-   * `leaseIds` so a pure-RPS block (the common `defaultPacingForTool` shape)
-   * actually amortizes its whole grant across acquires instead of burning it.
-   */
-  windowPermits: number;
+  permits: Array<[scheduledAtMs: number, leaseId: string | null]>;
+  rateScopeToken: string | null;
   expiresAt: number;
   rulesKey: string;
   rules: PacingRule[];
@@ -110,6 +98,7 @@ interface LeasedBlock {
 
 interface PendingRelease {
   bucketId: string;
+  rateScopeToken: string | null;
   rules: PacingRule[];
   leaseIds: string[];
   flushing: boolean;
@@ -156,6 +145,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly waiters = new Map<string, number>();
   /** Last-known server cooldown per bucket (surfaced for PROVIDER_EXHAUSTED). */
   private readonly cooldowns = new Map<string, RateStateCooldown>();
+  private readonly effectiveRps = new Map<string, number>();
+  private readonly pendingSuccesses = new Map<string, number>();
   private pendingReleaseFailure: Error | null = null;
 
   constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
@@ -220,6 +211,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
   async acquire(input: {
     bucketId: string;
+    rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
   }): Promise<PacingPermit> {
@@ -249,13 +241,29 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           : new Error('Rate-state acquire aborted.');
       }
       const localPermit = this.drawFromBlock(bucketId, rulesKey);
-      if (localPermit) return localPermit;
+      if (localPermit) {
+        const waitMs = localPermit.scheduledAtMs - this.now();
+        if (waitMs > 0) await this.sleep(waitMs);
+        if (signal?.aborted) {
+          localPermit.permit.release();
+          throw signal.reason instanceof Error
+            ? signal.reason
+            : new Error('Rate-state acquire aborted.');
+        }
+        return localPermit.permit;
+      }
       // Count this acquirer as waiting on the refill so the single-flight refill
       // can size its block to the live wave. Decremented once a block lands (or
       // the refill errors) — see the finally in `refillBlock`.
       this.waiters.set(bucketId, (this.waiters.get(bucketId) ?? 0) + 1);
       try {
-        await this.refillBlock(bucketId, ordered, rulesKey, signal);
+        await this.refillBlock(
+          bucketId,
+          input.rateScopeToken?.trim() || null,
+          ordered,
+          rulesKey,
+          signal,
+        );
       } finally {
         const remaining = (this.waiters.get(bucketId) ?? 1) - 1;
         if (remaining > 0) this.waiters.set(bucketId, remaining);
@@ -279,7 +287,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   ): number {
     const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
     if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
+    const configuredRps = Math.min(
+      ...rules.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000),
+    );
+    const horizonBudget = Math.max(
+      1,
+      Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
+    );
     const windowBudget = Math.min(
+      horizonBudget,
       ...rules
         .map((rule) => rule.requestsPerWindow)
         .filter((value) => Number.isFinite(value) && value > 0),
@@ -353,6 +369,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    */
   private refillBlock(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
     signal?: AbortSignal,
@@ -361,6 +378,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     if (existing) return existing;
     const promise = this.performRefill(
       bucketId,
+      rateScopeToken,
       ordered,
       rulesKey,
       signal,
@@ -375,6 +393,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
   private async performRefill(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
     signal?: AbortSignal,
@@ -397,19 +416,45 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         if (signal?.aborted) return;
       }
       try {
+        const observedSuccesses = this.pendingSuccesses.get(bucketId) ?? 0;
         const response = await this.port.acquire({
           bucketId,
+          rateScopeToken,
           // Stamp adaptiveMaxRps on default-pacing rules so the DB row's ceiling
           // matches the old in-memory AIMD range.
           rules: withAdaptiveMaxRps(ordered),
           requested: this.requestedForBucket(bucketId, ordered),
+          observedSuccesses,
           schedulerSchema: this.schedulerSchema,
         });
+        if (observedSuccesses > 0) {
+          const remaining = Math.max(
+            0,
+            (this.pendingSuccesses.get(bucketId) ?? 0) - observedSuccesses,
+          );
+          if (remaining > 0) this.pendingSuccesses.set(bucketId, remaining);
+          else this.pendingSuccesses.delete(bucketId);
+        }
         this.recordCooldown(bucketId, response);
+        if (
+          Number.isFinite(response.effectiveRequestsPerSecond) &&
+          Number(response.effectiveRequestsPerSecond) > 0
+        ) {
+          this.effectiveRps.set(
+            bucketId,
+            Number(response.effectiveRequestsPerSecond),
+          );
+        }
         if (response.granted > 0) {
           // Merge the whole grant into the block and let acquirers re-draw. No
           // permit is handed out here, so nothing is double-drawn.
-          this.mergeGrantedBlock(bucketId, ordered, rulesKey, response);
+          this.mergeGrantedBlock(
+            bucketId,
+            rateScopeToken,
+            ordered,
+            rulesKey,
+            response,
+          );
           return;
         }
         // Live path: the server told us to wait past the threshold. Fail fast
@@ -456,9 +501,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    */
   private mergeGrantedBlock(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
-    response: { granted: number; leaseIds?: unknown },
+    response: {
+      granted: number;
+      leaseIds?: unknown;
+      scheduledAtMs?: unknown;
+    },
   ): void {
     const hasConcurrency = ordered.some((rule) => rule.maxConcurrency != null);
     const leaseIds = Array.isArray(response.leaseIds)
@@ -477,38 +527,71 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
       );
     }
+    // Rolling deploy compatibility: an older gateway returns no schedule. It
+    // has already debited the block under the legacy token-bucket contract, so
+    // keep those permits immediately usable until every gateway serves the new
+    // globally reserved timestamps.
+    const scheduledAtMs = Array.isArray(response.scheduledAtMs)
+      ? response.scheduledAtMs.map(Number)
+      : Array.from({ length: response.granted }, () => this.now());
+    if (
+      scheduledAtMs.length !== response.granted ||
+      scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
+    ) {
+      throw new Error(
+        `Rate-state store granted ${response.granted} permits for ${bucketId} with ${scheduledAtMs.length} dispatch timestamps.`,
+      );
+    }
     this.mergeBlock(
       bucketId,
-      hasConcurrency ? leaseIds : [],
-      hasConcurrency ? 0 : response.granted,
+      rateScopeToken,
+      scheduledAtMs.map(
+        (scheduledAtMs, index): [number, string | null] => [
+          scheduledAtMs,
+          hasConcurrency ? (leaseIds[index] ?? null) : null,
+        ],
+      ),
       rulesKey,
       ordered,
     );
   }
 
-  penalize(input: { bucketId: string; cooldownMs: number }): void {
+  async penalize(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+    cooldownMs: number;
+  }): Promise<void> {
     if (input.cooldownMs <= 0) return;
     this.drainBlock(input.bucketId);
-    void this.port
-      .penalize({
+    try {
+      await this.port.penalize({
         bucketId: input.bucketId,
+        rateScopeToken: input.rateScopeToken?.trim() || null,
         cooldownMs: input.cooldownMs,
         schedulerSchema: this.schedulerSchema,
-      })
-      .catch((error) => {
-        const normalized = normalizeError(error);
-        this.pendingReleaseFailure = new Error(
-          `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
-        );
-        this.onStoreFailure({
-          bucketId: input.bucketId,
-          error: normalized.message,
-        });
       });
+    } catch (error) {
+      const normalized = normalizeError(error);
+      this.onStoreFailure({
+        bucketId: input.bucketId,
+        error: normalized.message,
+      });
+      throw new Error(
+        `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
+      );
+    }
+  }
+
+  observeSuccess(input: { bucketId: string }): void {
+    this.pendingSuccesses.set(
+      input.bucketId,
+      Math.min(1_000_000, (this.pendingSuccesses.get(input.bucketId) ?? 0) + 1),
+    );
   }
 
   private permitFor(
     bucketId: string,
+    rateScopeToken: string | null,
     rules: PacingRule[],
     leaseId: string | null,
   ): PacingPermit {
@@ -520,35 +603,49 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       release: () => {
         if (released) return;
         released = true;
-        this.releaseReserved(bucketId, rules, leaseId ? [leaseId] : []);
+        this.releaseReserved(
+          bucketId,
+          rateScopeToken,
+          rules,
+          leaseId ? [leaseId] : [],
+        );
       },
     };
   }
 
   private mergeBlock(
     bucketId: string,
-    leaseIds: string[],
-    windowPermits: number,
+    rateScopeToken: string | null,
+    permits: LeasedBlock['permits'],
     rulesKey: string,
     rules: PacingRule[],
   ): void {
-    const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
+    const lastScheduledAtMs = permits.at(-1)?.[0] ?? this.now();
+    const freshExpiresAt =
+      Math.max(this.now(), lastScheduledAtMs) + leasedBlockTtlMs(rules);
     const existing = this.blocks.get(bucketId);
     if (
       existing &&
       existing.rulesKey === rulesKey &&
       existing.expiresAt > this.now()
     ) {
-      existing.leaseIds.push(...leaseIds);
-      existing.windowPermits += windowPermits;
-      existing.expiresAt = Math.min(existing.expiresAt, freshExpiresAt);
+      existing.permits.push(...permits);
+      existing.permits.sort((a, b) => a[0] - b[0]);
+      existing.expiresAt = Math.max(existing.expiresAt, freshExpiresAt);
       return;
     }
     if (existing)
-      this.releaseReserved(bucketId, existing.rules, existing.leaseIds);
+      this.releaseReserved(
+        bucketId,
+        existing.rateScopeToken,
+        existing.rules,
+        existing.permits.flatMap(([, leaseId]) =>
+          leaseId ? [leaseId] : [],
+        ),
+      );
     this.blocks.set(bucketId, {
-      leaseIds,
-      windowPermits,
+      permits,
+      rateScopeToken,
       expiresAt: freshExpiresAt,
       rulesKey,
       rules,
@@ -558,7 +655,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private drawFromBlock(
     bucketId: string,
     rulesKey: string,
-  ): PacingPermit | null {
+  ): { permit: PacingPermit; scheduledAtMs: number } | null {
     const block = this.blocks.get(bucketId);
     if (!block) return null;
     if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
@@ -567,30 +664,50 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       // decremented the provider window for them, and the rule-scoped TTL is
       // capped so at most one block's worth is stranded per bucket/window).
       // Concurrency leases must be released so the shared slot count drains.
-      this.releaseReserved(bucketId, block.rules, block.leaseIds);
+      this.releaseReserved(
+        bucketId,
+        block.rateScopeToken,
+        block.rules,
+        block.permits.flatMap(([, leaseId]) =>
+          leaseId ? [leaseId] : [],
+        ),
+      );
       return null;
     }
-    const hasConcurrency = block.rules.some(
-      (rule) => rule.maxConcurrency != null,
-    );
-    // Concurrency blocks draw a lease token (release semantics); pure-rate
-    // blocks draw an untracked window permit that the server already accounted.
-    const leaseId = hasConcurrency ? (block.leaseIds.pop() ?? null) : null;
-    if (!hasConcurrency && block.windowPermits > 0) block.windowPermits -= 1;
-    if (block.leaseIds.length === 0 && block.windowPermits === 0)
+    const scheduled = block.permits.shift();
+    if (!scheduled) {
       this.blocks.delete(bucketId);
-    return this.permitFor(bucketId, block.rules, leaseId);
+      return null;
+    }
+    if (block.permits.length === 0) this.blocks.delete(bucketId);
+    return {
+      permit: this.permitFor(
+        bucketId,
+        block.rateScopeToken,
+        block.rules,
+        scheduled[1],
+      ),
+      scheduledAtMs: scheduled[0],
+    };
   }
 
   private drainBlock(bucketId: string): void {
     const block = this.blocks.get(bucketId);
     if (!block) return;
     this.blocks.delete(bucketId);
-    this.releaseReserved(bucketId, block.rules, block.leaseIds);
+    this.releaseReserved(
+      bucketId,
+      block.rateScopeToken,
+      block.rules,
+      block.permits.flatMap(([, leaseId]) =>
+        leaseId ? [leaseId] : [],
+      ),
+    );
   }
 
   private releaseReserved(
     bucketId: string,
+    rateScopeToken: string | null,
     rules: readonly PacingRule[],
     leaseIds: readonly string[],
   ): void {
@@ -603,6 +720,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     const key = `${bucketId}\0${rulesSignature(ordered)}`;
     const pending = this.pendingReleases.get(key) ?? {
       bucketId,
+      rateScopeToken,
       rules: ordered,
       leaseIds: [],
       flushing: false,
@@ -623,6 +741,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       try {
         await this.port.release({
           bucketId: pending.bucketId,
+          rateScopeToken: pending.rateScopeToken,
           rules: pending.rules,
           leaseIds,
           schedulerSchema: this.schedulerSchema,

package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts

@@ -167,22 +167,26 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
     }
   }
 
-  penalize(input: { bucketId: string; cooldownMs: number }): void {
+  async penalize(input: {
+    bucketId: string;
+    cooldownMs: number;
+  }): Promise<void> {
     if (input.cooldownMs <= 0) return;
     // Drop any cached block for this bucket so the cooldown takes effect on the
     // very next acquire instead of being masked by already-leased permits.
     this.blocks.delete(input.bucketId);
-    void this.port
-      .ratePenalize({
+    try {
+      await this.port.ratePenalize({
         bucketId: input.bucketId,
         cooldownMs: input.cooldownMs,
-      })
-      .catch((error) => {
-        this.onDegraded({
-          bucketId: input.bucketId,
-          error: error instanceof Error ? error.message : String(error),
-        });
       });
+    } catch (error) {
+      this.onDegraded({
+        bucketId: input.bucketId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
   }
 
   /**

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -70,6 +70,14 @@ export type PacingResolver = (
   toolId: string,
 ) => Promise<{ provider: string; rules: PacingRule[] } | null>;
 
+export type RateScopeResolver = (
+  toolId: string,
+  provider: string,
+) =>
+  | Promise<{ bucketId: string; token: string } | null>
+  | { bucketId: string; token: string }
+  | null;
+
 export function defaultPacingForTool(
   toolId: string,
   policy: ResolvedExecutionPolicy,
@@ -96,12 +104,9 @@ export interface PlayExecutionGovernor {
   /** Block until a map-row slot is free. */
   acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
   /**
-   * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
-   * permit are free, then charge the tool-call budget and return a lease. Order:
-   * concurrency slot → provider pace → tool budget (charged last so a
-   * failed/aborted acquire never consumes budget). A run over tool budget still
-   * acquires and holds a slot + pacing permit before the breach is detected; the
-   * breach surfaces only once the call is otherwise cleared to run.
+   * Block until the per-provider pacer permit and then a global
+   * tool-concurrency slot are free, charge the tool-call budget, and return a
+   * lease. Parked future permits never consume scarce execution slots.
    */
   acquireToolSlot(
     toolId: string,
@@ -132,12 +137,19 @@ export interface PlayExecutionGovernor {
   /** Feed a provider's Retry-After back into the shared pacer. */
   reportProviderBackpressure(input: {
     provider: string;
+    /**
+     * The tool whose provider call received the 429. Credential-scoped rate
+     * buckets are resolved per tool, so delayed feedback must retain this
+     * identity instead of selecting the latest scope for the provider.
+     */
+    toolId?: string;
     retryAfterMs: number;
-  }): void;
+  }): Promise<void>;
 
   /** Feed clean provider calls back into adaptive admission. */
   observeProviderSuccess(input: {
     provider: string;
+    toolId?: string;
     latencyMs?: number | null;
   }): void;
   /** Feed success back using the same tool-to-provider resolution as acquire. */
@@ -155,6 +167,7 @@ interface GovernorInput {
   rateState: RateStateBackend;
   budgetState?: BudgetStateBackend;
   resolvePacing: PacingResolver;
+  resolveRateScope?: RateScopeResolver;
   adaptiveAdmission?: RuntimeAdaptiveAdmission;
   resume?: GovernanceSnapshot;
 }
@@ -246,6 +259,7 @@ export function createPlayExecutionGovernor(
     });
   const budgetState = input.budgetState ?? new InMemoryBudgetStateBackend();
   const providerByTool = new Map<string, string>();
+  const scopeByTool = new Map<string, [string, string]>();
 
   // When the rate-state backend owns pacing authoritatively (the Absurd/Node
   // app-runtime Postgres pacer runs the whole token bucket + AIMD in the row),
@@ -256,7 +270,20 @@ export function createPlayExecutionGovernor(
   // undefined and keep the in-memory adaptive admission (byte-identical).
   const dbAuthoritativePacing = input.rateState.kind === 'app_runtime_postgres';
 
-  const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
+  const orgBucket = (provider: string) => `${input.scope.orgId}:${provider}`;
+
+  async function resolveScope(toolId: string, provider: string) {
+    const resolved = await input.resolveRateScope?.(toolId, provider);
+    const scope = [
+      resolved?.bucketId ?? orgBucket(provider),
+      resolved?.token ?? '',
+    ] as [string, string];
+    scopeByTool.set(toolId, scope);
+    return scope;
+  }
+
+  const scopeFor = (toolId: string | undefined, provider: string) =>
+    scopeByTool.get(toolId ?? '') ?? ([orgBucket(provider), ''] as [string, string]);
 
   async function resolveAdaptivePacing(toolId: string): Promise<{
     provider: string;
@@ -348,23 +375,26 @@ export function createPlayExecutionGovernor(
 
     acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
     async acquireToolSlot(toolId, opts) {
-      // 1. global tool-concurrency slot.
-      const slot = await toolSlots.acquire(opts?.signal);
-      // 2. per-(org,provider) pacing. The provider comes from the pacing
+      // 1. Per-provider pacing. The provider comes from the pacing
       //    resolver, so callers only need the toolId. No rules → no pacing.
-      let permit: { release(): void };
+      const pacing = await resolveAdaptivePacing(toolId);
+      const rateScope = await resolveScope(toolId, pacing.provider);
+      const permit = await input.rateState.acquire({
+        bucketId: rateScope[0],
+        rateScopeToken: rateScope[1],
+        rules: pacing.rules,
+        signal: opts?.signal,
+      });
+      // 2. Acquire a scarce execution slot only after the provider permit is
+      // due. If the slot wait aborts, refund any concurrency-bearing permit.
+      let slot: WorkLease;
       try {
-        const pacing = await resolveAdaptivePacing(toolId);
-        permit = await input.rateState.acquire({
-          bucketId: bucketId(pacing.provider),
-          rules: pacing.rules,
-          signal: opts?.signal,
-        });
+        slot = await toolSlots.acquire(opts?.signal);
       } catch (error) {
-        slot.release();
+        permit.release();
         throw error;
       }
-      // 3. charge the budget only once the call is actually cleared to run, so a
+      // 3. Charge the budget only once the call is actually cleared to run, so a
       // failed/aborted acquisition never permanently consumes tool budget.
       try {
         await chargeBudget('toolCall');
@@ -409,7 +439,7 @@ export function createPlayExecutionGovernor(
     resolveRowConcurrency: (requested) =>
       resolveRowConcurrency(policy, requested),
 
-    reportProviderBackpressure(bp) {
+    async reportProviderBackpressure(bp) {
       // DB-authoritative pacer: route backpressure ONLY to the row via
       // penalize. The in-memory adaptive admission must not also halve, or the
       // provider would be throttled twice (once in the row, once in memory).
@@ -420,13 +450,22 @@ export function createPlayExecutionGovernor(
           retryAfterMs: bp.retryAfterMs,
         });
       }
-      input.rateState.penalize({
-        bucketId: bucketId(bp.provider),
+      const rateScope = scopeFor(bp.toolId, bp.provider);
+      await input.rateState.penalize({
+        bucketId: rateScope[0],
+        ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
         cooldownMs: bp.retryAfterMs,
       });
     },
 
     observeProviderSuccess(success) {
+      if (dbAuthoritativePacing) {
+        const rateScope = scopeFor(success.toolId, success.provider);
+        input.rateState.observeSuccess?.({
+          bucketId: rateScope[0],
+        });
+        return;
+      }
       adaptiveAdmission.observeProviderSuccess({
         orgId: input.scope.orgId,
         provider: success.provider,
@@ -438,9 +477,9 @@ export function createPlayExecutionGovernor(
       const provider =
         providerByTool.get(success.toolId) ??
         defaultPacingForTool(success.toolId, policy).provider;
-      adaptiveAdmission.observeProviderSuccess({
-        orgId: input.scope.orgId,
+      governor.observeProviderSuccess({
         provider,
+        toolId: success.toolId,
         latencyMs: success.latencyMs,
       });
     },
@@ -507,8 +546,8 @@ function createInlineChildGovernor(
       root.suggestedParallelism(toolId, fallback),
     chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
     resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
-    reportProviderBackpressure: (input) =>
-      root.reportProviderBackpressure(input),
+    reportProviderBackpressure: async (input) =>
+      await root.reportProviderBackpressure(input),
     observeProviderSuccess: (input) => root.observeProviderSuccess(input),
     observeToolSuccess: (input) => root.observeToolSuccess(input),
     async forkInlineChild(input) {

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -86,6 +86,7 @@ export interface RateStateBackend {
    */
   acquire(input: {
     bucketId: string;
+    rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
   }): Promise<PacingPermit>;
@@ -94,7 +95,17 @@ export interface RateStateBackend {
    * Feed a server-observed Retry-After back so future acquires for this bucket
    * back off. Advisory and idempotent; never un-charges an in-flight call.
    */
-  penalize(input: { bucketId: string; cooldownMs: number }): void;
+  penalize(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+    cooldownMs: number;
+  }): void | Promise<void>;
+
+  /** Record confirmed provider success for adaptive recovery. */
+  observeSuccess?(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+  }): void;
 }
 
 const NOOP_PERMIT: PacingPermit = { release() {} };