deepline 0.1.231 → 0.1.232

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -1454,7 +1454,7 @@ async function executeTool(
     providerIdempotencyKey?: string | null;
   },
   workflowStep?: WorkflowStep,
-  onProviderBackpressure?: (retryAfterMs: number) => void,
+  onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
   onRetryAttempt?: () => void | Promise<void>,
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
@@ -1500,7 +1500,7 @@ async function executeToolWithLifecycle(
   },
   workflowStep: WorkflowStep | undefined,
   callbacks: WorkerCtxCallbacks | undefined,
-  onProviderBackpressure?: (retryAfterMs: number) => void,
+  onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
   onRetryAttempt?: () => void | Promise<void>,
   transientHttpRetrySafe = false,
   abortSignal?: AbortSignal,
@@ -1690,7 +1690,7 @@ async function callToolDirect(
     executionAuthScopeDigest?: string | null;
     providerIdempotencyKey?: string | null;
   },
-  onProviderBackpressure?: (retryAfterMs: number) => void,
+  onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
   // Invoked once per in-process retry attempt (429 / retryable 5xx / synthetic
   // transient) so the Governor charges chargeBudget('retry') per attempt — the
   // same runaway guard the cjs runner applies (context.ts charges retry on each
@@ -1877,7 +1877,7 @@ async function callToolDirect(
     if (failure.backpressureDelayMs !== null) {
       // Feed the provider's backpressure into the shared pacer even on the
       // final attempt so the (org, provider) bucket backs off across isolates.
-      onProviderBackpressure?.(failure.backpressureDelayMs);
+      await onProviderBackpressure?.(failure.backpressureDelayMs);
     }
     if (!failure.shouldRetry) {
       throw lastError;
@@ -2616,7 +2616,7 @@ function isWorkerFetchResponse(value: unknown): value is WorkerFetchResponse {
 async function postRuntimeEgressFetch(
   req: RunRequest,
   payload: RuntimeEgressFetchPayload,
-  onProviderBackpressure?: (retryAfterMs: number) => void,
+  onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>,
   onRetryAttempt?: () => void | Promise<void>,
 ): Promise<WorkerFetchResponse> {
   let lastError: Error | null = null;
@@ -2667,7 +2667,7 @@ async function postRuntimeEgressFetch(
       Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
         ? Math.ceil(retryAfterSeconds * 1000)
         : 1_000;
-    onProviderBackpressure?.(retryAfterMs);
+    await onProviderBackpressure?.(retryAfterMs);
     if (attempt >= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS) {
       throw lastError;
     }
@@ -3776,7 +3776,7 @@ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
   // (RuntimeToolAuthScopeDigestResolver), whose `invalidate` member is
   // REQUIRED — the AUTH_SCOPE_CHANGED re-claim wiring below calls it directly,
   // and a resolver without eviction support must be a type error.
-  return createRuntimeToolAuthScopeDigestResolver({
+  const digestResolver = createRuntimeToolAuthScopeDigestResolver({
     missingToolIdMessage: 'Tool auth scope needs tool id.',
     fetchDigest: async (normalized) => {
       const runScopedPath =
@@ -3807,12 +3807,19 @@ function createWorkerToolAuthScopeDigestResolver(req: RunRequest) {
       return digest;
     },
   });
+  return digestResolver;
 }
 
 function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
   const cache = new Map<
     string,
-    Promise<(ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null>
+    Promise<
+      | (ResolvedPacingPolicy & {
+          retrySafeTransientHttp: boolean;
+          r: { bucketId: string; token: string } | null;
+        })
+      | null
+    >
   >();
   return (toolId: string) => {
     const normalized = String(toolId || '').trim();
@@ -3840,6 +3847,7 @@ function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
         provider?: unknown;
         queueHints?: unknown;
         retry?: unknown;
+        r?: { bucketId: string; token: string };
       } | null;
       if (!body) return null;
       const pacing = pacingPolicyFromUnknownQueueHints(body.queueHints);
@@ -3857,6 +3865,7 @@ function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
               : '',
           rules: [],
         }),
+        r: body.r ?? null,
         retrySafeTransientHttp: retry.retrySafeTransientHttp === true,
       };
     })();
@@ -3883,11 +3892,13 @@ function resumeGovernanceFromRequest(req: RunRequest): GovernanceSnapshot {
   };
 }
 
-function createGovernorForRun(req: RunRequest): {
+function createGovernorForRun(
+  req: RunRequest,
+  resolvePacing: WorkerPacingResolver,
+): {
   governor: PlayExecutionGovernor;
   resolvePacing: WorkerPacingResolver;
 } {
-  const resolvePacing = createWorkerPacingResolver(req);
   const governor = createPlayExecutionGovernor({
     adapter: 'esm_workers',
     scope: {
@@ -3906,6 +3917,8 @@ function createGovernorForRun(req: RunRequest): {
         }),
     ),
     resolvePacing,
+    resolveRateScope: async (toolId) =>
+      (await resolvePacing(toolId))?.r ?? null,
     resume: resumeGovernanceFromRequest(req),
   });
   return { governor, resolvePacing };
@@ -3944,12 +3957,15 @@ function createMinimalWorkerCtx(
   leasedSheetNamespaces?: Set<string>,
   receiptSalvage?: ReceiptSalvageBuffer,
 ): unknown {
-  const { governor, resolvePacing: resolveToolPacing } =
-    createGovernorForRun(req);
-  const resolveToolActionCacheVersion =
-    createWorkerToolActionCacheVersionResolver(req);
+  const resolveToolPacing = createWorkerPacingResolver(req);
+  const { governor } = createGovernorForRun(req, resolveToolPacing);
   const resolveToolAuthScopeDigest =
     createWorkerToolAuthScopeDigestResolver(req);
+  const resolveToolActionCacheVersion =
+    createWorkerToolActionCacheVersionResolver(req);
+  // Play-call depth/count/per-parent budgets, child-play concurrency, and the
+  // lineage snapshot are owned by the Governor (createGovernorForRun above).
+  // The worker keeps only substrate mechanism here.
   const stepCallCounts: Record<string, number> = {};
   const secretRedactor = createSecretRedactionContext();
 
@@ -4004,7 +4020,7 @@ function createMinimalWorkerCtx(
     input: Record<string, unknown>;
     workflowStep?: unknown;
     callbacks?: WorkerCtxCallbacks;
-    onProviderBackpressure?: (retryAfterMs: number) => void;
+    onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
     onRetryAttempt?: () => void | Promise<void>;
     transientHttpRetrySafe?: boolean;
     durableCallReceiptKey?: string | null;

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts

@@ -73,7 +73,11 @@ import type { WorkerWorkBudgetMeter } from './work-budget';
 export type WorkerPacingResolver = (
   toolId: string,
 ) => Promise<
-  (ResolvedPacingPolicy & { retrySafeTransientHttp: boolean }) | null
+  | (ResolvedPacingPolicy & {
+      retrySafeTransientHttp: boolean;
+      r: { bucketId: string; token: string } | null;
+    })
+  | null
 >;
 
 export type WorkerToolActionCacheVersionResolver = (
@@ -111,7 +115,7 @@ export type WorkerToolDispatchExecuteTool = (input: {
   input: Record<string, unknown>;
   workflowStep?: WorkflowStepLike;
   callbacks?: WorkerToolDispatchCallbacks;
-  onProviderBackpressure?: (retryAfterMs: number) => void;
+  onProviderBackpressure?: (retryAfterMs: number) => void | Promise<void>;
   onRetryAttempt?: () => void | Promise<void>;
   transientHttpRetrySafe?: boolean;
   durableCallReceiptKey?: string | null;
@@ -561,17 +565,19 @@ export class WorkerToolBatchScheduler {
    * isolates. Provider comes from the same pacing resolver the Governor uses
    * (the worker has no local catalog), so callers pass only the toolId.
    */
-  private reportBackpressure(toolId: string, retryAfterMs: number): void {
+  private async reportBackpressure(
+    toolId: string,
+    retryAfterMs: number,
+  ): Promise<void> {
     if (retryAfterMs <= 0) return;
-    void (async () => {
-      const pacing = await this.options.resolvePacing(toolId).catch(() => null);
-      if (pacing?.provider) {
-        this.options.governor.reportProviderBackpressure({
-          provider: pacing.provider,
-          retryAfterMs,
-        });
-      }
-    })();
+    const pacing = await this.options.resolvePacing(toolId).catch(() => null);
+    if (pacing?.provider) {
+      await this.options.governor.reportProviderBackpressure({
+        provider: pacing.provider,
+        toolId,
+        retryAfterMs,
+      });
+    }
   }
 
   async execute(

package/dist/bundling-sources/sdk/src/release.ts

@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: '0.1.231',
+  version: '0.1.232',
   apiContract: '2026-06-dataset-handle-results-hard-cutover',
   supportPolicy: {
-    latest: '0.1.231',
+    latest: '0.1.232',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.219',
     commandMinimumSupported: [

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -57,10 +57,12 @@ export {
 import {
   createDefaultGovernanceSnapshot,
   createPlayExecutionGovernor,
+  defaultPacingForTool,
   type GovernanceSnapshot,
   type PacingResolver,
   type PlayExecutionGovernor,
 } from './governor/governor';
+import { resolveExecutionPolicy } from './governor/policy';
 import {
   createRuntimeResourceGovernor,
   type RuntimeResourceGovernor,
@@ -1142,12 +1144,20 @@ function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
  */
 function createPacingResolver(
   getToolQueueHints: ContextOptions['getToolQueueHints'],
+  getToolProvider: ContextOptions['getToolProvider'],
 ): PacingResolver {
   return async (toolId: string) => {
     const builtin = pacingPolicyForTool(toolId, []);
     if (builtin) return builtin;
     const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
-    return pacingPolicyForTool(toolId, hints);
+    const declared = pacingPolicyForTool(toolId, hints);
+    if (declared) return declared;
+    const provider = (await getToolProvider?.(toolId))?.trim();
+    if (!provider) return null;
+    return {
+      ...defaultPacingForTool(toolId, resolveExecutionPolicy('cjs_node20')),
+      provider,
+    };
   };
 }
 
@@ -1375,7 +1385,11 @@ export class PlayContextImpl {
         },
         rateState: options.rateState ?? new InMemoryRateStateBackend(),
         budgetState: options.budgetState,
-        resolvePacing: createPacingResolver(options.getToolQueueHints),
+        resolvePacing: createPacingResolver(
+          options.getToolQueueHints,
+          options.getToolProvider,
+        ),
+        resolveRateScope: options.getToolRateScope,
         // A root run keeps depth 0 (its first child play is depth 1) but still
         // seeds its own play id into the ancestry/currentPlayId so the cycle guard
         // catches a child re-invoking the root. createDefaultGovernanceSnapshot
@@ -8533,8 +8547,9 @@ export class PlayContextImpl {
       ? await this.#options.getToolQueueHints(toolId)
       : [];
     const provider = hints[0]?.provider?.trim() || `tool:${toolId}`;
-    this.resourceGovernor.reportProviderBackpressure({
+    await this.resourceGovernor.reportProviderBackpressure({
       provider,
+      toolId,
       retryAfterMs,
     });
   }

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -641,12 +641,20 @@ export interface ContextOptions {
   runtimeSheetBackedMapDatasets?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
+  getToolProvider?: (toolId: string) => Promise<string | null>;
   getToolRetryPolicy?: (toolId: string) => Promise<{
     retrySafeTransientHttp?: boolean;
     requiresExecutionFence?: boolean;
   } | null>;
   getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
   getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
+  getToolRateScope?: (
+    toolId: string,
+    provider: string,
+  ) =>
+    | Promise<{ bucketId: string; token: string } | null>
+    | { bucketId: string; token: string }
+    | null;
   invalidateToolAuthScopeDigest?: (toolId: string) => void;
   getToolTargetGetters?: (
     toolId: string,

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -89,20 +89,8 @@ export interface RateStateCooldown {
 }
 
 interface LeasedBlock {
-  /**
-   * Concurrency reservations, one per locally-cached permit. Only rules with a
-   * `maxConcurrency` mint lease tokens; each id maps to a durable slot that must
-   * be released (or TTL-expired) so the shared concurrency count drains.
-   */
-  leaseIds: string[];
-  /**
-   * Pure-rate (RPS-only) permits with no concurrency slot behind them. The
-   * server decremented the provider window by this count already, so drawing
-   * one locally is a no-op that needs no release token. Tracked separately from
-   * `leaseIds` so a pure-RPS block (the common `defaultPacingForTool` shape)
-   * actually amortizes its whole grant across acquires instead of burning it.
-   */
-  windowPermits: number;
+  permits: Array<[scheduledAtMs: number, leaseId: string | null]>;
+  rateScopeToken: string | null;
   expiresAt: number;
   rulesKey: string;
   rules: PacingRule[];
@@ -110,6 +98,7 @@ interface LeasedBlock {
 
 interface PendingRelease {
   bucketId: string;
+  rateScopeToken: string | null;
   rules: PacingRule[];
   leaseIds: string[];
   flushing: boolean;
@@ -156,6 +145,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly waiters = new Map<string, number>();
   /** Last-known server cooldown per bucket (surfaced for PROVIDER_EXHAUSTED). */
   private readonly cooldowns = new Map<string, RateStateCooldown>();
+  private readonly effectiveRps = new Map<string, number>();
+  private readonly pendingSuccesses = new Map<string, number>();
   private pendingReleaseFailure: Error | null = null;
 
   constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
@@ -220,6 +211,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
   async acquire(input: {
     bucketId: string;
+    rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
   }): Promise<PacingPermit> {
@@ -249,13 +241,29 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           : new Error('Rate-state acquire aborted.');
       }
       const localPermit = this.drawFromBlock(bucketId, rulesKey);
-      if (localPermit) return localPermit;
+      if (localPermit) {
+        const waitMs = localPermit.scheduledAtMs - this.now();
+        if (waitMs > 0) await this.sleep(waitMs);
+        if (signal?.aborted) {
+          localPermit.permit.release();
+          throw signal.reason instanceof Error
+            ? signal.reason
+            : new Error('Rate-state acquire aborted.');
+        }
+        return localPermit.permit;
+      }
       // Count this acquirer as waiting on the refill so the single-flight refill
       // can size its block to the live wave. Decremented once a block lands (or
       // the refill errors) — see the finally in `refillBlock`.
       this.waiters.set(bucketId, (this.waiters.get(bucketId) ?? 0) + 1);
       try {
-        await this.refillBlock(bucketId, ordered, rulesKey, signal);
+        await this.refillBlock(
+          bucketId,
+          input.rateScopeToken?.trim() || null,
+          ordered,
+          rulesKey,
+          signal,
+        );
       } finally {
         const remaining = (this.waiters.get(bucketId) ?? 1) - 1;
         if (remaining > 0) this.waiters.set(bucketId, remaining);
@@ -279,7 +287,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   ): number {
     const floor = APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE;
     if (rules.some((rule) => rule.maxConcurrency != null)) return floor;
+    const configuredRps = Math.min(
+      ...rules.map((rule) => (rule.requestsPerWindow / rule.windowMs) * 1_000),
+    );
+    const horizonBudget = Math.max(
+      1,
+      Math.ceil(this.effectiveRps.get(bucketId) ?? configuredRps),
+    );
     const windowBudget = Math.min(
+      horizonBudget,
       ...rules
         .map((rule) => rule.requestsPerWindow)
         .filter((value) => Number.isFinite(value) && value > 0),
@@ -353,6 +369,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    */
   private refillBlock(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
     signal?: AbortSignal,
@@ -361,6 +378,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     if (existing) return existing;
     const promise = this.performRefill(
       bucketId,
+      rateScopeToken,
       ordered,
       rulesKey,
       signal,
@@ -375,6 +393,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
 
   private async performRefill(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
     signal?: AbortSignal,
@@ -397,19 +416,45 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         if (signal?.aborted) return;
       }
       try {
+        const observedSuccesses = this.pendingSuccesses.get(bucketId) ?? 0;
         const response = await this.port.acquire({
           bucketId,
+          rateScopeToken,
           // Stamp adaptiveMaxRps on default-pacing rules so the DB row's ceiling
           // matches the old in-memory AIMD range.
           rules: withAdaptiveMaxRps(ordered),
           requested: this.requestedForBucket(bucketId, ordered),
+          observedSuccesses,
           schedulerSchema: this.schedulerSchema,
         });
+        if (observedSuccesses > 0) {
+          const remaining = Math.max(
+            0,
+            (this.pendingSuccesses.get(bucketId) ?? 0) - observedSuccesses,
+          );
+          if (remaining > 0) this.pendingSuccesses.set(bucketId, remaining);
+          else this.pendingSuccesses.delete(bucketId);
+        }
         this.recordCooldown(bucketId, response);
+        if (
+          Number.isFinite(response.effectiveRequestsPerSecond) &&
+          Number(response.effectiveRequestsPerSecond) > 0
+        ) {
+          this.effectiveRps.set(
+            bucketId,
+            Number(response.effectiveRequestsPerSecond),
+          );
+        }
         if (response.granted > 0) {
           // Merge the whole grant into the block and let acquirers re-draw. No
           // permit is handed out here, so nothing is double-drawn.
-          this.mergeGrantedBlock(bucketId, ordered, rulesKey, response);
+          this.mergeGrantedBlock(
+            bucketId,
+            rateScopeToken,
+            ordered,
+            rulesKey,
+            response,
+          );
           return;
         }
         // Live path: the server told us to wait past the threshold. Fail fast
@@ -456,9 +501,14 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
    */
   private mergeGrantedBlock(
     bucketId: string,
+    rateScopeToken: string | null,
     ordered: PacingRule[],
     rulesKey: string,
-    response: { granted: number; leaseIds?: unknown },
+    response: {
+      granted: number;
+      leaseIds?: unknown;
+      scheduledAtMs?: unknown;
+    },
   ): void {
     const hasConcurrency = ordered.some((rule) => rule.maxConcurrency != null);
     const leaseIds = Array.isArray(response.leaseIds)
@@ -477,38 +527,71 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
       );
     }
+    // Rolling deploy compatibility: an older gateway returns no schedule. It
+    // has already debited the block under the legacy token-bucket contract, so
+    // keep those permits immediately usable until every gateway serves the new
+    // globally reserved timestamps.
+    const scheduledAtMs = Array.isArray(response.scheduledAtMs)
+      ? response.scheduledAtMs.map(Number)
+      : Array.from({ length: response.granted }, () => this.now());
+    if (
+      scheduledAtMs.length !== response.granted ||
+      scheduledAtMs.some((value) => !Number.isFinite(value) || value < 0)
+    ) {
+      throw new Error(
+        `Rate-state store granted ${response.granted} permits for ${bucketId} with ${scheduledAtMs.length} dispatch timestamps.`,
+      );
+    }
     this.mergeBlock(
       bucketId,
-      hasConcurrency ? leaseIds : [],
-      hasConcurrency ? 0 : response.granted,
+      rateScopeToken,
+      scheduledAtMs.map(
+        (scheduledAtMs, index): [number, string | null] => [
+          scheduledAtMs,
+          hasConcurrency ? (leaseIds[index] ?? null) : null,
+        ],
+      ),
       rulesKey,
       ordered,
     );
   }
 
-  penalize(input: { bucketId: string; cooldownMs: number }): void {
+  async penalize(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+    cooldownMs: number;
+  }): Promise<void> {
     if (input.cooldownMs <= 0) return;
     this.drainBlock(input.bucketId);
-    void this.port
-      .penalize({
+    try {
+      await this.port.penalize({
         bucketId: input.bucketId,
+        rateScopeToken: input.rateScopeToken?.trim() || null,
         cooldownMs: input.cooldownMs,
         schedulerSchema: this.schedulerSchema,
-      })
-      .catch((error) => {
-        const normalized = normalizeError(error);
-        this.pendingReleaseFailure = new Error(
-          `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
-        );
-        this.onStoreFailure({
-          bucketId: input.bucketId,
-          error: normalized.message,
-        });
       });
+    } catch (error) {
+      const normalized = normalizeError(error);
+      this.onStoreFailure({
+        bucketId: input.bucketId,
+        error: normalized.message,
+      });
+      throw new Error(
+        `Rate-state penalize failed for ${input.bucketId}: ${normalized.message}`,
+      );
+    }
+  }
+
+  observeSuccess(input: { bucketId: string }): void {
+    this.pendingSuccesses.set(
+      input.bucketId,
+      Math.min(1_000_000, (this.pendingSuccesses.get(input.bucketId) ?? 0) + 1),
+    );
   }
 
   private permitFor(
     bucketId: string,
+    rateScopeToken: string | null,
     rules: PacingRule[],
     leaseId: string | null,
   ): PacingPermit {
@@ -520,35 +603,49 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       release: () => {
         if (released) return;
         released = true;
-        this.releaseReserved(bucketId, rules, leaseId ? [leaseId] : []);
+        this.releaseReserved(
+          bucketId,
+          rateScopeToken,
+          rules,
+          leaseId ? [leaseId] : [],
+        );
       },
     };
   }
 
   private mergeBlock(
     bucketId: string,
-    leaseIds: string[],
-    windowPermits: number,
+    rateScopeToken: string | null,
+    permits: LeasedBlock['permits'],
     rulesKey: string,
     rules: PacingRule[],
   ): void {
-    const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
+    const lastScheduledAtMs = permits.at(-1)?.[0] ?? this.now();
+    const freshExpiresAt =
+      Math.max(this.now(), lastScheduledAtMs) + leasedBlockTtlMs(rules);
     const existing = this.blocks.get(bucketId);
     if (
       existing &&
       existing.rulesKey === rulesKey &&
       existing.expiresAt > this.now()
     ) {
-      existing.leaseIds.push(...leaseIds);
-      existing.windowPermits += windowPermits;
-      existing.expiresAt = Math.min(existing.expiresAt, freshExpiresAt);
+      existing.permits.push(...permits);
+      existing.permits.sort((a, b) => a[0] - b[0]);
+      existing.expiresAt = Math.max(existing.expiresAt, freshExpiresAt);
       return;
     }
     if (existing)
-      this.releaseReserved(bucketId, existing.rules, existing.leaseIds);
+      this.releaseReserved(
+        bucketId,
+        existing.rateScopeToken,
+        existing.rules,
+        existing.permits.flatMap(([, leaseId]) =>
+          leaseId ? [leaseId] : [],
+        ),
+      );
     this.blocks.set(bucketId, {
-      leaseIds,
-      windowPermits,
+      permits,
+      rateScopeToken,
       expiresAt: freshExpiresAt,
       rulesKey,
       rules,
@@ -558,7 +655,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private drawFromBlock(
     bucketId: string,
     rulesKey: string,
-  ): PacingPermit | null {
+  ): { permit: PacingPermit; scheduledAtMs: number } | null {
     const block = this.blocks.get(bucketId);
     if (!block) return null;
     if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
@@ -567,30 +664,50 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       // decremented the provider window for them, and the rule-scoped TTL is
       // capped so at most one block's worth is stranded per bucket/window).
       // Concurrency leases must be released so the shared slot count drains.
-      this.releaseReserved(bucketId, block.rules, block.leaseIds);
+      this.releaseReserved(
+        bucketId,
+        block.rateScopeToken,
+        block.rules,
+        block.permits.flatMap(([, leaseId]) =>
+          leaseId ? [leaseId] : [],
+        ),
+      );
       return null;
     }
-    const hasConcurrency = block.rules.some(
-      (rule) => rule.maxConcurrency != null,
-    );
-    // Concurrency blocks draw a lease token (release semantics); pure-rate
-    // blocks draw an untracked window permit that the server already accounted.
-    const leaseId = hasConcurrency ? (block.leaseIds.pop() ?? null) : null;
-    if (!hasConcurrency && block.windowPermits > 0) block.windowPermits -= 1;
-    if (block.leaseIds.length === 0 && block.windowPermits === 0)
+    const scheduled = block.permits.shift();
+    if (!scheduled) {
       this.blocks.delete(bucketId);
-    return this.permitFor(bucketId, block.rules, leaseId);
+      return null;
+    }
+    if (block.permits.length === 0) this.blocks.delete(bucketId);
+    return {
+      permit: this.permitFor(
+        bucketId,
+        block.rateScopeToken,
+        block.rules,
+        scheduled[1],
+      ),
+      scheduledAtMs: scheduled[0],
+    };
   }
 
   private drainBlock(bucketId: string): void {
     const block = this.blocks.get(bucketId);
     if (!block) return;
     this.blocks.delete(bucketId);
-    this.releaseReserved(bucketId, block.rules, block.leaseIds);
+    this.releaseReserved(
+      bucketId,
+      block.rateScopeToken,
+      block.rules,
+      block.permits.flatMap(([, leaseId]) =>
+        leaseId ? [leaseId] : [],
+      ),
+    );
   }
 
   private releaseReserved(
     bucketId: string,
+    rateScopeToken: string | null,
     rules: readonly PacingRule[],
     leaseIds: readonly string[],
   ): void {
@@ -603,6 +720,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     const key = `${bucketId}\0${rulesSignature(ordered)}`;
     const pending = this.pendingReleases.get(key) ?? {
       bucketId,
+      rateScopeToken,
       rules: ordered,
       leaseIds: [],
       flushing: false,
@@ -623,6 +741,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       try {
         await this.port.release({
           bucketId: pending.bucketId,
+          rateScopeToken: pending.rateScopeToken,
           rules: pending.rules,
           leaseIds,
           schedulerSchema: this.schedulerSchema,

package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts

@@ -167,22 +167,26 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
     }
   }
 
-  penalize(input: { bucketId: string; cooldownMs: number }): void {
+  async penalize(input: {
+    bucketId: string;
+    cooldownMs: number;
+  }): Promise<void> {
     if (input.cooldownMs <= 0) return;
     // Drop any cached block for this bucket so the cooldown takes effect on the
     // very next acquire instead of being masked by already-leased permits.
     this.blocks.delete(input.bucketId);
-    void this.port
-      .ratePenalize({
+    try {
+      await this.port.ratePenalize({
         bucketId: input.bucketId,
         cooldownMs: input.cooldownMs,
-      })
-      .catch((error) => {
-        this.onDegraded({
-          bucketId: input.bucketId,
-          error: error instanceof Error ? error.message : String(error),
-        });
       });
+    } catch (error) {
+      this.onDegraded({
+        bucketId: input.bucketId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
   }
 
   /**

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -70,6 +70,14 @@ export type PacingResolver = (
   toolId: string,
 ) => Promise<{ provider: string; rules: PacingRule[] } | null>;
 
+export type RateScopeResolver = (
+  toolId: string,
+  provider: string,
+) =>
+  | Promise<{ bucketId: string; token: string } | null>
+  | { bucketId: string; token: string }
+  | null;
+
 export function defaultPacingForTool(
   toolId: string,
   policy: ResolvedExecutionPolicy,
@@ -96,12 +104,9 @@ export interface PlayExecutionGovernor {
   /** Block until a map-row slot is free. */
   acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
   /**
-   * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
-   * permit are free, then charge the tool-call budget and return a lease. Order:
-   * concurrency slot → provider pace → tool budget (charged last so a
-   * failed/aborted acquire never consumes budget). A run over tool budget still
-   * acquires and holds a slot + pacing permit before the breach is detected; the
-   * breach surfaces only once the call is otherwise cleared to run.
+   * Block until the per-provider pacer permit and then a global
+   * tool-concurrency slot are free, charge the tool-call budget, and return a
+   * lease. Parked future permits never consume scarce execution slots.
    */
   acquireToolSlot(
     toolId: string,
@@ -132,12 +137,19 @@ export interface PlayExecutionGovernor {
   /** Feed a provider's Retry-After back into the shared pacer. */
   reportProviderBackpressure(input: {
     provider: string;
+    /**
+     * The tool whose provider call received the 429. Credential-scoped rate
+     * buckets are resolved per tool, so delayed feedback must retain this
+     * identity instead of selecting the latest scope for the provider.
+     */
+    toolId?: string;
     retryAfterMs: number;
-  }): void;
+  }): Promise<void>;
 
   /** Feed clean provider calls back into adaptive admission. */
   observeProviderSuccess(input: {
     provider: string;
+    toolId?: string;
     latencyMs?: number | null;
   }): void;
   /** Feed success back using the same tool-to-provider resolution as acquire. */
@@ -155,6 +167,7 @@ interface GovernorInput {
   rateState: RateStateBackend;
   budgetState?: BudgetStateBackend;
   resolvePacing: PacingResolver;
+  resolveRateScope?: RateScopeResolver;
   adaptiveAdmission?: RuntimeAdaptiveAdmission;
   resume?: GovernanceSnapshot;
 }
@@ -246,6 +259,7 @@ export function createPlayExecutionGovernor(
     });
   const budgetState = input.budgetState ?? new InMemoryBudgetStateBackend();
   const providerByTool = new Map<string, string>();
+  const scopeByTool = new Map<string, [string, string]>();
 
   // When the rate-state backend owns pacing authoritatively (the Absurd/Node
   // app-runtime Postgres pacer runs the whole token bucket + AIMD in the row),
@@ -256,7 +270,20 @@ export function createPlayExecutionGovernor(
   // undefined and keep the in-memory adaptive admission (byte-identical).
   const dbAuthoritativePacing = input.rateState.kind === 'app_runtime_postgres';
 
-  const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
+  const orgBucket = (provider: string) => `${input.scope.orgId}:${provider}`;
+
+  async function resolveScope(toolId: string, provider: string) {
+    const resolved = await input.resolveRateScope?.(toolId, provider);
+    const scope = [
+      resolved?.bucketId ?? orgBucket(provider),
+      resolved?.token ?? '',
+    ] as [string, string];
+    scopeByTool.set(toolId, scope);
+    return scope;
+  }
+
+  const scopeFor = (toolId: string | undefined, provider: string) =>
+    scopeByTool.get(toolId ?? '') ?? ([orgBucket(provider), ''] as [string, string]);
 
   async function resolveAdaptivePacing(toolId: string): Promise<{
     provider: string;
@@ -348,23 +375,26 @@ export function createPlayExecutionGovernor(
 
     acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
     async acquireToolSlot(toolId, opts) {
-      // 1. global tool-concurrency slot.
-      const slot = await toolSlots.acquire(opts?.signal);
-      // 2. per-(org,provider) pacing. The provider comes from the pacing
+      // 1. Per-provider pacing. The provider comes from the pacing
       //    resolver, so callers only need the toolId. No rules → no pacing.
-      let permit: { release(): void };
+      const pacing = await resolveAdaptivePacing(toolId);
+      const rateScope = await resolveScope(toolId, pacing.provider);
+      const permit = await input.rateState.acquire({
+        bucketId: rateScope[0],
+        rateScopeToken: rateScope[1],
+        rules: pacing.rules,
+        signal: opts?.signal,
+      });
+      // 2. Acquire a scarce execution slot only after the provider permit is
+      // due. If the slot wait aborts, refund any concurrency-bearing permit.
+      let slot: WorkLease;
       try {
-        const pacing = await resolveAdaptivePacing(toolId);
-        permit = await input.rateState.acquire({
-          bucketId: bucketId(pacing.provider),
-          rules: pacing.rules,
-          signal: opts?.signal,
-        });
+        slot = await toolSlots.acquire(opts?.signal);
       } catch (error) {
-        slot.release();
+        permit.release();
         throw error;
       }
-      // 3. charge the budget only once the call is actually cleared to run, so a
+      // 3. Charge the budget only once the call is actually cleared to run, so a
       // failed/aborted acquisition never permanently consumes tool budget.
       try {
         await chargeBudget('toolCall');
@@ -409,7 +439,7 @@ export function createPlayExecutionGovernor(
     resolveRowConcurrency: (requested) =>
       resolveRowConcurrency(policy, requested),
 
-    reportProviderBackpressure(bp) {
+    async reportProviderBackpressure(bp) {
       // DB-authoritative pacer: route backpressure ONLY to the row via
       // penalize. The in-memory adaptive admission must not also halve, or the
       // provider would be throttled twice (once in the row, once in memory).
@@ -420,13 +450,22 @@ export function createPlayExecutionGovernor(
           retryAfterMs: bp.retryAfterMs,
         });
       }
-      input.rateState.penalize({
-        bucketId: bucketId(bp.provider),
+      const rateScope = scopeFor(bp.toolId, bp.provider);
+      await input.rateState.penalize({
+        bucketId: rateScope[0],
+        ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
         cooldownMs: bp.retryAfterMs,
       });
     },
 
     observeProviderSuccess(success) {
+      if (dbAuthoritativePacing) {
+        const rateScope = scopeFor(success.toolId, success.provider);
+        input.rateState.observeSuccess?.({
+          bucketId: rateScope[0],
+        });
+        return;
+      }
       adaptiveAdmission.observeProviderSuccess({
         orgId: input.scope.orgId,
         provider: success.provider,
@@ -438,9 +477,9 @@ export function createPlayExecutionGovernor(
       const provider =
         providerByTool.get(success.toolId) ??
         defaultPacingForTool(success.toolId, policy).provider;
-      adaptiveAdmission.observeProviderSuccess({
-        orgId: input.scope.orgId,
+      governor.observeProviderSuccess({
         provider,
+        toolId: success.toolId,
         latencyMs: success.latencyMs,
       });
     },
@@ -507,8 +546,8 @@ function createInlineChildGovernor(
       root.suggestedParallelism(toolId, fallback),
     chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
     resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
-    reportProviderBackpressure: (input) =>
-      root.reportProviderBackpressure(input),
+    reportProviderBackpressure: async (input) =>
+      await root.reportProviderBackpressure(input),
     observeProviderSuccess: (input) => root.observeProviderSuccess(input),
     observeToolSuccess: (input) => root.observeToolSuccess(input),
     async forkInlineChild(input) {

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -86,6 +86,7 @@ export interface RateStateBackend {
    */
   acquire(input: {
     bucketId: string;
+    rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
   }): Promise<PacingPermit>;
@@ -94,7 +95,17 @@ export interface RateStateBackend {
    * Feed a server-observed Retry-After back so future acquires for this bucket
    * back off. Advisory and idempotent; never un-charges an in-flight call.
    */
-  penalize(input: { bucketId: string; cooldownMs: number }): void;
+  penalize(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+    cooldownMs: number;
+  }): void | Promise<void>;
+
+  /** Record confirmed provider success for adaptive recovery. */
+  observeSuccess?(input: {
+    bucketId: string;
+    rateScopeToken?: string | null;
+  }): void;
 }
 
 const NOOP_PERMIT: PacingPermit = { release() {} };

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -25,14 +25,24 @@ export type PlayRunnerRateStateBackendConfig =
 
 export type PlayRunnerRateStateAcquireInput = {
   bucketId: string;
+  rateScopeToken?: string | null;
   rules: PacingRule[];
   requested: number;
+  /** Confirmed provider successes since this runner's previous reservation. */
+  observedSuccesses?: number;
   schedulerSchema?: string | null;
 };
 
 export type PlayRunnerRateStateAcquireResult = {
   granted: number;
   waitMs: number;
+  /**
+   * One globally reserved dispatch timestamp per granted permit. The runtime
+   * may cache the block, but must not dispatch a permit before its timestamp.
+   */
+  scheduledAtMs?: number[];
+  /** Live shared rate after adaptive backpressure/recovery. */
+  effectiveRequestsPerSecond?: number;
   /** One independently-expiring concurrency reservation per granted permit. */
   leaseIds?: string[];
   /**
@@ -50,6 +60,7 @@ export type PlayRunnerRateStateAcquireResult = {
 
 export type PlayRunnerRateStateReleaseInput = {
   bucketId: string;
+  rateScopeToken?: string | null;
   rules: PacingRule[];
   leaseIds: string[];
   schedulerSchema?: string | null;
@@ -57,6 +68,7 @@ export type PlayRunnerRateStateReleaseInput = {
 
 export type PlayRunnerRateStatePenalizeInput = {
   bucketId: string;
+  rateScopeToken?: string | null;
   cooldownMs: number;
   schedulerSchema?: string | null;
 };

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -67,8 +67,9 @@ export interface RuntimeResourceGovernor {
   resolveRowConcurrency(requested?: number): number;
   reportProviderBackpressure(input: {
     provider: string;
+    toolId?: string;
     retryAfterMs: number;
-  }): void;
+  }): Promise<void>;
   observe(input: RuntimeResourceObservation): void;
   snapshot(): RuntimeResourceSnapshot;
 }
@@ -308,8 +309,8 @@ export function createRuntimeResourceGovernor(input: {
       return input.executionGovernor.resolveRowConcurrency(requested);
     },
 
-    reportProviderBackpressure(backpressure) {
-      input.executionGovernor.reportProviderBackpressure(backpressure);
+    async reportProviderBackpressure(backpressure) {
+      await input.executionGovernor.reportProviderBackpressure(backpressure);
       observe({
         providerResourceKey: backpressure.provider,
         provider429: true,

package/dist/cli/index.js

@@ -625,10 +625,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.231",
+  version: "0.1.232",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.231",
+    latest: "0.1.232",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/cli/index.mjs

@@ -610,10 +610,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.231",
+  version: "0.1.232",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.231",
+    latest: "0.1.232",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/index.js

@@ -424,10 +424,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.231",
+  version: "0.1.232",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.231",
+    latest: "0.1.232",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/dist/index.mjs

@@ -354,10 +354,10 @@ var SDK_RELEASE = {
   // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
   // 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
   // automatically without blocking their current command.
-  version: "0.1.231",
+  version: "0.1.232",
   apiContract: "2026-06-dataset-handle-results-hard-cutover",
   supportPolicy: {
-    latest: "0.1.231",
+    latest: "0.1.232",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.219",
     commandMinimumSupported: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.231",
+  "version": "0.1.232",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {