deepline 0.1.79 → 0.1.81

package/dist/repo/shared_libs/play-runtime/governor/governor.ts

@@ -0,0 +1,376 @@
+/**
+ * Play Execution Governor — the deep module that owns execution policy.
+ *
+ * Both runner Adapters (`cjs_node20`, `esm_workers`) gate their work through one
+ * Governor instance per run-attempt so the substrates cannot diverge on
+ * concurrency, budgets, or pacing. Adapters keep only substrate mechanism (how
+ * to resolve a row, batch a tool call, or submit a child); the Governor owns the
+ * "may I, and how many at once" policy. See ADR 0007 + CONTEXT.md.
+ *
+ * Surface (small, by design):
+ *   - acquireRowSlot / acquireChildPlaySlot / acquireToolSlot → blocking leases
+ *   - chargeBudget                                            → throws on breach
+ *   - forkChild                                               → child lineage snapshot
+ *   - resolveRowConcurrency / reportProviderBackpressure / snapshot
+ */
+import {
+  type AdapterId,
+  type ResolvedExecutionPolicy,
+  resolveExecutionPolicy,
+  resolveRowConcurrency,
+} from './policy';
+import {
+  noopPacingPermit,
+  type PacingRule,
+  type RateStateBackend,
+} from './rate-state-backend';
+
+export interface WorkLease {
+  /** Free the slot / pacing permit. Idempotent. MUST be called in a finally. */
+  release(): void;
+}
+
+export type BudgetKind =
+  | 'playCall'
+  | 'toolCall'
+  | 'retry'
+  | 'descendant'
+  | 'waterfallStep';
+
+/** Counters that accumulate down the dispatch lineage; thread via the snapshot. */
+export interface GovernanceSnapshot {
+  rootRunId: string;
+  currentRunId: string;
+  currentPlayId: string;
+  ancestryPlayIds: string[];
+  ancestryRunIds: string[];
+  callDepth: number;
+  playCallCount: number;
+  toolCallCount: number;
+  retryCount: number;
+  descendantCount: number;
+  waterfallStepExecutions: number;
+  /** Direct child plays launched, keyed by parent play id. */
+  parentChildCalls: Record<string, number>;
+}
+
+export class GovernorBudgetError extends Error {
+  constructor(
+    readonly budget: BudgetKind | 'playDepth' | 'childPerParent',
+    readonly observed: number,
+    readonly limit: number,
+  ) {
+    const message =
+      budget === 'playDepth'
+        ? `Play-call depth exceeded (${observed}/${limit}).`
+        : `Play execution ${budget} budget exceeded (${observed}/${limit}).`;
+    super(message);
+    this.name = 'GovernorBudgetError';
+  }
+}
+
+/** Maps a toolId to its provider + resolved pacing rules (from rate-limit defs). */
+export type PacingResolver = (
+  toolId: string,
+) => Promise<{ provider: string; rules: PacingRule[] } | null>;
+
+export interface PlayExecutionGovernor {
+  readonly adapter: AdapterId;
+  readonly policy: ResolvedExecutionPolicy;
+
+  /** Block until a map-row slot is free. */
+  acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
+  /** Block until a child-play slot is free. */
+  acquireChildPlaySlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
+  /**
+   * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
+   * permit are free, then charge the tool-call budget and return a lease. Order:
+   * concurrency slot → provider pace → tool budget (charged last so a
+   * failed/aborted acquire never consumes budget). A run over tool budget still
+   * acquires and holds a slot + pacing permit before the breach is detected; the
+   * breach surfaces only once the call is otherwise cleared to run.
+   */
+  acquireToolSlot(
+    toolId: string,
+    opts?: { signal?: AbortSignal },
+  ): Promise<WorkLease>;
+
+  /**
+   * Suggested batch parallelism for a tool: the provider's own rate hints
+   * tightened to the policy's suggested ceiling. No hints → the fallback.
+   */
+  suggestedParallelism(toolId: string, fallback: number): Promise<number>;
+
+  /** Increment a monotonic budget counter; throws GovernorBudgetError on breach. */
+  chargeBudget(kind: BudgetKind, amount?: number): void;
+
+  /**
+   * Reserve depth + per-parent + descendant budget for a child play and return
+   * the snapshot to thread into the child run so budgets accumulate across the
+   * lineage (and across isolates on `esm_workers`). Throws on breach.
+   *
+   * Unlike {@link acquireToolSlot} (which charges last so an aborted acquire
+   * never consumes budget), child-lineage counters are charged here at fork
+   * time, BEFORE the caller acquires a child-play slot. A slot acquire that then
+   * fails (e.g. abort) does NOT refund these counters. This is intentional and
+   * safe given the 100k child caps: the charge reserves lineage-global capacity
+   * for a launch the caller has committed to, and forkChild must return the
+   * threaded snapshot synchronously, so the charge cannot be deferred behind the
+   * async slot acquire.
+   */
+  forkChild(input: {
+    childPlayName: string;
+    childRunId: string;
+  }): GovernanceSnapshot;
+
+  /** Effective row concurrency: explicit request clamped to [1, rowMax], else default. */
+  resolveRowConcurrency(requested?: number): number;
+
+  /** Feed a provider's Retry-After back into the shared pacer. */
+  reportProviderBackpressure(input: {
+    provider: string;
+    retryAfterMs: number;
+  }): void;
+
+  snapshot(): GovernanceSnapshot;
+}
+
+interface GovernorInput {
+  adapter: AdapterId;
+  scope: { orgId: string; rootRunId: string };
+  rateState: RateStateBackend;
+  resolvePacing: PacingResolver;
+  resume?: GovernanceSnapshot;
+}
+
+class Semaphore {
+  private inFlight = 0;
+  private readonly waiters: Array<() => void> = [];
+  constructor(private readonly limit: number) {}
+
+  async acquire(signal?: AbortSignal): Promise<WorkLease> {
+    // Fail fast on an already-aborted signal: the parked-promise abort listener
+    // below registers with { once: true } and never fires for a signal that was
+    // aborted before we parked, so without this check a full pool would block
+    // the waiter until a slot frees (or forever if it never drains).
+    if (signal?.aborted) {
+      throw signal.reason instanceof Error
+        ? signal.reason
+        : new Error('Slot acquire aborted.');
+    }
+    while (this.inFlight >= this.limit) {
+      await new Promise<void>((resolve, reject) => {
+        const onResolve = () => {
+          signal?.removeEventListener('abort', onAbort);
+          resolve();
+        };
+        const onAbort = () => {
+          const idx = this.waiters.indexOf(onResolve);
+          if (idx >= 0) this.waiters.splice(idx, 1);
+          reject(
+            signal?.reason instanceof Error
+              ? signal.reason
+              : new Error('Slot acquire aborted.'),
+          );
+        };
+        this.waiters.push(onResolve);
+        signal?.addEventListener('abort', onAbort, { once: true });
+      });
+      if (signal?.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Slot acquire aborted.');
+      }
+    }
+    this.inFlight += 1;
+    let released = false;
+    return {
+      release: () => {
+        if (released) return;
+        released = true;
+        this.inFlight = Math.max(0, this.inFlight - 1);
+        this.waiters.shift()?.();
+      },
+    };
+  }
+}
+
+export function createDefaultGovernanceSnapshot(scope: {
+  orgId: string;
+  rootRunId: string;
+  rootPlayId?: string;
+}): GovernanceSnapshot {
+  return {
+    rootRunId: scope.rootRunId,
+    currentRunId: scope.rootRunId,
+    currentPlayId: scope.rootPlayId ?? scope.rootRunId,
+    ancestryPlayIds: scope.rootPlayId ? [scope.rootPlayId] : [],
+    ancestryRunIds: [scope.rootRunId],
+    callDepth: scope.rootPlayId ? 1 : 0,
+    playCallCount: 0,
+    toolCallCount: 0,
+    retryCount: 0,
+    descendantCount: 0,
+    waterfallStepExecutions: 0,
+    parentChildCalls: {},
+  };
+}
+
+export function createPlayExecutionGovernor(
+  input: GovernorInput,
+): PlayExecutionGovernor {
+  const policy = resolveExecutionPolicy(input.adapter);
+  const state: GovernanceSnapshot =
+    input.resume ?? createDefaultGovernanceSnapshot(input.scope);
+
+  const rowSlots = new Semaphore(policy.concurrency.rowMax);
+  const childPlaySlots = new Semaphore(policy.concurrency.childPlay);
+  const toolSlots = new Semaphore(policy.concurrency.toolCalls);
+
+  const bucketId = (provider: string) => `${input.scope.orgId}:${provider}`;
+
+  function chargeBudget(kind: BudgetKind, amount = 1): void {
+    switch (kind) {
+      case 'playCall':
+        state.playCallCount += amount;
+        if (state.playCallCount > policy.budgets.maxPlayCallCount)
+          throw new GovernorBudgetError('playCall', state.playCallCount, policy.budgets.maxPlayCallCount);
+        return;
+      case 'toolCall':
+        state.toolCallCount += amount;
+        if (state.toolCallCount > policy.budgets.maxToolCallCount)
+          throw new GovernorBudgetError('toolCall', state.toolCallCount, policy.budgets.maxToolCallCount);
+        return;
+      case 'retry':
+        state.retryCount += amount;
+        if (state.retryCount > policy.budgets.maxRetryCount)
+          throw new GovernorBudgetError('retry', state.retryCount, policy.budgets.maxRetryCount);
+        return;
+      case 'descendant':
+        state.descendantCount += amount;
+        if (state.descendantCount > policy.budgets.maxDescendants)
+          throw new GovernorBudgetError('descendant', state.descendantCount, policy.budgets.maxDescendants);
+        return;
+      case 'waterfallStep':
+        state.waterfallStepExecutions += amount;
+        if (state.waterfallStepExecutions > policy.budgets.maxWaterfallStepExecutions)
+          throw new GovernorBudgetError('waterfallStep', state.waterfallStepExecutions, policy.budgets.maxWaterfallStepExecutions);
+        return;
+    }
+  }
+
+  return {
+    adapter: input.adapter,
+    policy,
+
+    acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
+    acquireChildPlaySlot: (opts) => childPlaySlots.acquire(opts?.signal),
+
+    async acquireToolSlot(toolId, opts) {
+      // 1. global tool-concurrency slot.
+      const slot = await toolSlots.acquire(opts?.signal);
+      // 2. per-(org,provider) pacing. The provider comes from the pacing
+      //    resolver, so callers only need the toolId. No rules → no pacing.
+      let permit: { release(): void };
+      try {
+        const pacing = await input.resolvePacing(toolId);
+        permit =
+          pacing && pacing.rules.length > 0
+            ? await input.rateState.acquire({
+                bucketId: bucketId(pacing.provider),
+                rules: pacing.rules,
+                signal: opts?.signal,
+              })
+            : noopPacingPermit();
+      } catch (error) {
+        slot.release();
+        throw error;
+      }
+      // 3. charge the budget only once the call is actually cleared to run, so a
+      // failed/aborted acquisition never permanently consumes tool budget.
+      try {
+        chargeBudget('toolCall');
+      } catch (error) {
+        permit.release();
+        slot.release();
+        throw error;
+      }
+      let released = false;
+      return {
+        release: () => {
+          if (released) return;
+          released = true;
+          permit.release();
+          slot.release();
+        },
+      };
+    },
+
+    async suggestedParallelism(toolId, fallback) {
+      const pacing = await input.resolvePacing(toolId);
+      if (!pacing || pacing.rules.length === 0) return fallback;
+      const limits = pacing.rules.flatMap((rule) =>
+        rule.maxConcurrency != null
+          ? [rule.requestsPerWindow, rule.maxConcurrency]
+          : [rule.requestsPerWindow],
+      );
+      return Math.max(
+        1,
+        Math.min(policy.pacing.suggestedMaxParallelism, ...limits),
+      );
+    },
+
+    chargeBudget,
+
+    forkChild(childInput) {
+      if (state.ancestryPlayIds.includes(childInput.childPlayName)) {
+        const chain = [...state.ancestryPlayIds, childInput.childPlayName].join(' -> ');
+        throw new Error(`Recursive play graph detected: ${chain}`);
+      }
+      const nextDepth = state.callDepth + 1;
+      if (nextDepth > policy.budgets.maxPlayCallDepth)
+        throw new GovernorBudgetError('playDepth', nextDepth, policy.budgets.maxPlayCallDepth);
+      const parentKey = state.currentPlayId;
+      const nextParent = (state.parentChildCalls[parentKey] ?? 0) + 1;
+      if (nextParent > policy.budgets.maxChildPlayCallsPerParent)
+        throw new GovernorBudgetError('childPerParent', nextParent, policy.budgets.maxChildPlayCallsPerParent);
+      // Charge the run-wide play/descendant budgets on the parent. Charged at
+      // fork time (not after the caller's child-play slot acquire) and never
+      // refunded if that acquire fails — see the forkChild interface doc.
+      chargeBudget('playCall');
+      chargeBudget('descendant');
+      state.parentChildCalls[parentKey] = nextParent;
+      // Child seeds from the parent's accumulated counters → lineage-global budget.
+      return {
+        rootRunId: state.rootRunId,
+        currentRunId: childInput.childRunId,
+        currentPlayId: childInput.childPlayName,
+        ancestryPlayIds: [...state.ancestryPlayIds, childInput.childPlayName],
+        ancestryRunIds: [...state.ancestryRunIds, childInput.childRunId],
+        callDepth: nextDepth,
+        playCallCount: state.playCallCount,
+        toolCallCount: state.toolCallCount,
+        retryCount: state.retryCount,
+        descendantCount: state.descendantCount,
+        waterfallStepExecutions: state.waterfallStepExecutions,
+        parentChildCalls: {},
+      };
+    },
+
+    resolveRowConcurrency: (requested) => resolveRowConcurrency(policy, requested),
+
+    reportProviderBackpressure(bp) {
+      input.rateState.penalize({
+        bucketId: bucketId(bp.provider),
+        cooldownMs: bp.retryAfterMs,
+      });
+    },
+
+    snapshot: () => ({
+      ...state,
+      ancestryPlayIds: [...state.ancestryPlayIds],
+      ancestryRunIds: [...state.ancestryRunIds],
+      parentChildCalls: { ...state.parentChildCalls },
+    }),
+  };
+}

package/dist/repo/shared_libs/play-runtime/governor/policy.ts

@@ -0,0 +1,179 @@
+/**
+ * Play Execution Governor — the single policy table.
+ *
+ * This is the ONE source of truth for every concurrency, budget, and pacing
+ * number in the play runtime. Both runner Adapters (`cjs_node20` in-process and
+ * `esm_workers` cloud) resolve their limits from here, so the two substrates
+ * cannot drift. See ADR 0007 and the "Play Execution Governor" entry in
+ * CONTEXT.md.
+ *
+ * Tuning philosophy (product decision):
+ *  - Caps are GENEROUS and tuned for fast execution. They are runaway /
+ *    anti-starvation guards, NOT per-workload throttles. A legitimate large run
+ *    should never hit them.
+ *  - The real-time bounds on a run are (a) the wall-clock runtime cap and
+ *    (b) per-provider rate pacing. The counters below only stop pathological
+ *    recursion / fan-out.
+ *  - Every value is an EXPLICIT, finite cap. Nothing here is unbounded — there
+ *    is no `null`, no "off", no implicit infinity. Unaccounted resource use
+ *    must be impossible.
+ *
+ * The docs catalog (`src/lib/plays/limits-catalog.ts`) and its generated public
+ * + internal pages source their numbers from this table.
+ */
+
+/** Concurrency ceilings — how much may run at once. */
+export interface ExecutionConcurrencyPolicy {
+  /** Map rows resolving their fields concurrently when no `concurrency` is given. */
+  readonly rowDefault: number;
+  /** Hard ceiling for an explicit map `concurrency` value; larger is clamped. */
+  readonly rowMax: number;
+  /** Concurrently in-flight child plays (`ctx.runPlay`). Excess launches block. */
+  readonly childPlay: number;
+  /** Global backstop on concurrently in-flight tool calls across all providers. */
+  readonly toolCalls: number;
+}
+
+/**
+ * Per-run budgets — total attempts allowed before a run is treated as runaway.
+ * Accumulated down the dispatch lineage (a child seeds from the parent), so the
+ * budget is global across the nested-play tree, not per-worker.
+ */
+export interface ExecutionBudgetPolicy {
+  /** Max nesting depth of `ctx.runPlay` chains. Deeper is almost certainly a cycle. */
+  readonly maxPlayCallDepth: number;
+  /** Max total `ctx.runPlay` calls along a lineage. */
+  readonly maxPlayCallCount: number;
+  /** Max direct child plays one play may launch. */
+  readonly maxChildPlayCallsPerParent: number;
+  /** Max nested-play descendants created during a run. */
+  readonly maxDescendants: number;
+  /** Max total tool calls in a run. */
+  readonly maxToolCallCount: number;
+  /** Max total retries across all steps/tools. */
+  readonly maxRetryCount: number;
+  /** Max total waterfall-step executions in a run. */
+  readonly maxWaterfallStepExecutions: number;
+}
+
+/**
+ * Per-provider rate pacing — the real outbound throughput governor.
+ *
+ * SUBSTRATE NOTE: the per-provider request RATE (`requestsPerWindow`/`windowMs`
+ * from a provider's `PacingRule`s) is enforced on BOTH substrates. A rule's
+ * optional `maxConcurrency` (simultaneous-in-flight cap) is enforced only on
+ * `cjs_node20`; on `esm_workers` it is intentionally excluded from the pacing
+ * contract because a fanned-out run cannot guarantee the per-isolate release
+ * signal an in-flight count needs, so only the org-wide tool-concurrency
+ * backstop applies there. See PacingRule.maxConcurrency in rate-state-backend.ts.
+ */
+export interface ExecutionPacingPolicy {
+  /** RPS applied to a provider that declares no explicit rate limit. */
+  readonly defaultProviderRequestsPerSecond: number;
+  /** Parallelism the scheduler suggests before a provider's own hints tighten it. */
+  readonly suggestedMaxParallelism: number;
+}
+
+export interface ResolvedExecutionPolicy {
+  readonly concurrency: ExecutionConcurrencyPolicy;
+  readonly budgets: ExecutionBudgetPolicy;
+  readonly pacing: ExecutionPacingPolicy;
+}
+
+/**
+ * The shared default policy. Both substrates use this verbatim unless an entry
+ * in {@link ADAPTER_POLICY_OVERRIDES} forces a documented difference.
+ */
+export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
+  concurrency: {
+    // Map row concurrency is platform-controlled — customers cannot set it — so
+    // every map runs at the ceiling: high enough that per-provider pacing (not
+    // the row pool) is the bottleneck even for multi-step waterfall rows. Pure-JS
+    // maps use a separate fast path and are not bound by this. Kept == rowMax so
+    // the default and the cap are the same single platform value.
+    rowDefault: 2_000,
+    // Hard cap. Above this, more in-flight rows only park memory — outbound is
+    // already bounded by tool concurrency + per-provider pacing.
+    rowMax: 2_000,
+    // Concurrent child-play launches. Generous; each child is a real launch, so
+    // this is the one value most likely to need a documented esm_workers
+    // override if isolate pressure shows up in E2E.
+    childPlay: 32,
+    // Global all-provider backstop. Per-provider pacing is the real limit; this
+    // just stops a single run from opening an absurd number of sockets at once.
+    toolCalls: 256,
+  },
+  budgets: {
+    // Runaway guards, not workload limits. A 5,000-row map calling several tools
+    // per row is normal and must fit comfortably under these.
+    maxPlayCallDepth: 8,
+    maxPlayCallCount: 100_000,
+    maxChildPlayCallsPerParent: 100_000,
+    maxDescendants: 100_000,
+    maxToolCallCount: 5_000_000,
+    maxRetryCount: 100_000,
+    maxWaterfallStepExecutions: 5_000_000,
+  },
+  pacing: {
+    // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
+    defaultProviderRequestsPerSecond: 30,
+    suggestedMaxParallelism: 50,
+  },
+};
+
+export type AdapterId = 'cjs_node20' | 'esm_workers';
+
+/**
+ * One level deep on purpose: each policy section is a flat record of numbers, so
+ * an override is `{ section: { key: value } }`. The merge in
+ * {@link resolveExecutionPolicy} is one level; this type matches it exactly so
+ * it can never advertise nested-override support the merge doesn't implement.
+ */
+type PolicyOverride = {
+  [S in keyof ResolvedExecutionPolicy]?: Partial<ResolvedExecutionPolicy[S]>;
+};
+
+/**
+ * The ONLY sanctioned per-substrate divergence. Empty by design — both
+ * substrates run the shared policy. Every entry added here MUST carry a
+ * one-line comment citing the substrate constraint that forces it (e.g. isolate
+ * CPU/memory). CI may assert this map stays small. This is the anti-drift seam:
+ * differences are explicit and justified, never accidental.
+ */
+export const ADAPTER_POLICY_OVERRIDES: Record<AdapterId, PolicyOverride> = {
+  cjs_node20: {},
+  esm_workers: {},
+};
+
+/** Merge the shared policy with any documented Adapter override. */
+export function resolveExecutionPolicy(
+  adapter: AdapterId,
+): ResolvedExecutionPolicy {
+  const override = ADAPTER_POLICY_OVERRIDES[adapter];
+  return {
+    concurrency: {
+      ...SHARED_EXECUTION_POLICY.concurrency,
+      ...override.concurrency,
+    },
+    budgets: { ...SHARED_EXECUTION_POLICY.budgets, ...override.budgets },
+    pacing: { ...SHARED_EXECUTION_POLICY.pacing, ...override.pacing },
+  };
+}
+
+/**
+ * Effective row concurrency for a map: an explicit positive request clamped to
+ * `[1, rowMax]`, otherwise the default. Single helper, used by both substrates.
+ */
+export function resolveRowConcurrency(
+  policy: ResolvedExecutionPolicy,
+  requested?: number,
+): number {
+  if (
+    typeof requested === 'number' &&
+    Number.isFinite(requested) &&
+    requested > 0
+  ) {
+    return Math.min(Math.floor(requested), policy.concurrency.rowMax);
+  }
+  return policy.concurrency.rowDefault;
+}

package/dist/repo/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -0,0 +1,87 @@
+/**
+ * Rate State Backend — the one swappable seam of the Play Execution Governor.
+ *
+ * Per-`(org, provider)` rate windows are the only execution state that cannot be
+ * process-local: on `esm_workers` a run fans child plays across isolates, so the
+ * window must be shared. Everything else the Governor owns (budgets, concurrency
+ * slots) stays Governor-local and threads down the lineage via the snapshot.
+ *
+ * Adapters: an in-memory backend for the single-process `cjs_node20` runner, and
+ * a shared backend (Run Coordination Cache Durable Object, or server-side Redis)
+ * for `esm_workers`. See ADR 0007 and CONTEXT.md (Rate State Backend).
+ */
+
+/** A single resolved rate rule for a bucket (from rate-limit-definitions). */
+export interface PacingRule {
+  readonly ruleId: string;
+  readonly requestsPerWindow: number;
+  readonly windowMs: number;
+  /**
+   * Optional simultaneous-in-flight cap for this rule.
+   *
+   * SUBSTRATE NOTE: `maxConcurrency` is enforced ONLY on `cjs_node20` (the
+   * single-process {@link RateStateBackend} can hold a reliable in-flight count
+   * and release it on permit.release). On `esm_workers` it is intentionally NOT
+   * enforced: a run fans across V8 isolates and a dying isolate cannot guarantee
+   * the release signal a leased in-flight count would require, so the
+   * coordinator DO debits `requestsPerWindow` only. The Governor's global
+   * tool-concurrency semaphore is the cross-isolate in-flight backstop there.
+   * `requestsPerWindow`/`windowMs` (the request-RATE governor) ARE enforced on
+   * both substrates. See coordinator-rate-state-backend.ts and dedup-do.ts
+   * (computeRateAcquire) for the workers side.
+   */
+  readonly maxConcurrency: number | null;
+}
+
+/**
+ * Per-tool queue-hint metadata produced by the runtime tool catalog
+ * (`src/lib/plays/runtime-tool-metadata.ts`) and surfaced to the runtime via
+ * `ContextOptions.getToolQueueHints`. It is the raw provider rate-limit metadata
+ * that the runtime maps into one {@link PacingRule} per hint before handing it
+ * to the Governor's pacing resolver. The `provider` field identifies the pacing
+ * bucket; `bucketId`/`operation` are descriptive and used for logging/grouping.
+ */
+export interface PlayQueueHint {
+  bucketId: string;
+  provider: string;
+  operation: string;
+  ruleId: string;
+  requestsPerWindow: number;
+  windowMs: number;
+  maxConcurrency: number | null;
+}
+
+/** Handle returned by acquire(); release frees any concurrency held by the rules. */
+export interface PacingPermit {
+  release(): void;
+}
+
+export interface RateStateBackend {
+  /**
+   * Block until one outbound call is permitted for `bucketId` under all `rules`
+   * (request windows always; per-rule `maxConcurrency` only on the in-memory
+   * `cjs_node20` backend — see {@link PacingRule.maxConcurrency}), then debit and
+   * return a permit. `bucketId` is `${orgId}:${provider}` so the window is global
+   * per (org, provider). Resolves immediately when `rules` is empty (provider has
+   * no configured limit — pacing is a no-op, the global tool-concurrency backstop
+   * still applies).
+   */
+  acquire(input: {
+    bucketId: string;
+    rules: readonly PacingRule[];
+    signal?: AbortSignal;
+  }): Promise<PacingPermit>;
+
+  /**
+   * Feed a server-observed Retry-After back so future acquires for this bucket
+   * back off. Advisory and idempotent; never un-charges an in-flight call.
+   */
+  penalize(input: { bucketId: string; cooldownMs: number }): void;
+}
+
+const NOOP_PERMIT: PacingPermit = { release() {} };
+
+/** Permit used when a bucket has no rules — nothing to debit or release. */
+export function noopPacingPermit(): PacingPermit {
+  return NOOP_PERMIT;
+}

package/dist/repo/shared_libs/play-runtime/run-failure.ts

@@ -38,6 +38,18 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause: CLOUDFLARE_DURABLE_OBJECT_CODE_UPDATED_ERROR,
     };
   }
+  const playDepthBudgetMatch = cause.match(
+    /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
+  );
+  if (playDepthBudgetMatch) {
+    return {
+      code: 'PLAY_CALL_DEPTH_EXCEEDED',
+      phase: 'runtime',
+      message: `Play-call depth exceeded (${playDepthBudgetMatch[1]}/${playDepthBudgetMatch[2]}).`,
+      retryable: false,
+      cause,
+    };
+  }
   return {
     code: 'RUN_FAILED',
     phase: 'runtime',

package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts

@@ -35,6 +35,30 @@ export type PlayCallGovernanceSnapshot = {
   key: string;
   ancestryPlayIds: string[];
   callDepth: number;
+  /**
+   * Cumulative lineage-global budget counters consumed by ancestors at the
+   * moment this child was launched. The child seeds its own counters from these
+   * so the corresponding budgets (`maxPlayCallCount`, `maxToolCallCount`,
+   * `maxRetryCount`, `maxDescendants`, `maxWaterfallStepExecutions`) accumulate
+   * down the dispatch lineage instead of resetting to 0 in each worker isolate —
+   * matching the cjs forkChild path, which threads all of them. The Governor
+   * documents these budgets as global across the nested-play tree, not
+   * per-worker (see policy.ts / rate-state-backend.ts); threading them here is
+   * what makes that true on `esm_workers`.
+   *
+   * `descendantCount` is load-bearing for fan-out: forkChild charges
+   * `descendant` on every child launch, so without threading it a deep/wide tree
+   * would reset descendant accounting at each isolate and never converge on the
+   * lineage-global cap.
+   *
+   * All optional and fail-safe: if absent (older callers / dropped in transit)
+   * the child falls back to 0, i.e. prior behavior.
+   */
+  playCallCount?: number;
+  toolCallCount?: number;
+  retryCount?: number;
+  descendantCount?: number;
+  waterfallStepExecutions?: number;
 };
 
 export type PlaySchedulerSubmitInput = {