deepline 0.3.183 → 0.3.185

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -21,6 +21,7 @@ import {
   resolveRowConcurrency,
 } from './policy';
 import {
+  DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
   type PacingPermit,
   type PacingRule,
   type ProviderConcurrencyGrant,
@@ -50,6 +51,8 @@ export interface ProviderPermit extends WorkLease {
   readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
   /** Effective rate-controller reading for this physical provider permit. */
   readonly pacingGrant?: ProviderPacingGrant | null;
+  /** Signed shared-scope proof consumed by the execution gateway after queueing. */
+  readonly gatewayDispatchProof?: PacingPermit['gatewayDispatchProof'];
   /**
    * Return an acquired provider permit before an outbound request started.
    * Unlike `release`, this returns the rate reservation as well as any
@@ -64,6 +67,8 @@ export interface ProviderPermit extends WorkLease {
    * backend's lease block when that block is still live; otherwise refunds.
    */
   park(): Promise<void>;
+  /** Recheck shared provider state immediately before network dispatch. */
+  validate?(): Promise<boolean>;
 }
 
 export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
@@ -174,7 +179,7 @@ export interface PlayExecutionGovernor {
    */
   acquireProviderPermit(
     toolId: string,
-    opts?: { signal?: AbortSignal },
+    opts?: { signal?: AbortSignal; yieldProviderHolds?: boolean },
   ): Promise<ProviderPermit>;
 
   /**
@@ -226,6 +231,8 @@ export interface PlayExecutionGovernor {
      */
     toolId?: string;
     retryAfterMs: number;
+    /** Absolute provider retry deadline from the observed response. */
+    retryAtMs?: number;
     /** Provider residence for the rejected physical request, when observed. */
     latencyMs?: number | null;
     /** Server-authored context for attributing this physical provider 429. */
@@ -434,6 +441,15 @@ export function createPlayExecutionGovernor(
       provider,
       concurrencyGrant: rateLease.concurrencyGrant ?? null,
       pacingGrant: rateLease.pacingGrant ?? null,
+      gatewayDispatchProof: rateLease.gatewayDispatchProof ?? null,
+      ...(rateLease.validate
+        ? {
+            validate: async () => {
+              if (settled) return false;
+              return await rateLease.validate!();
+            },
+          }
+        : {}),
       release() {
         if (settled) return;
         settled = true;
@@ -487,16 +503,37 @@ export function createPlayExecutionGovernor(
       // Passthrough: hand the DB-backed pacer the declared rules verbatim (or
       // the default-pacing shape when undeclared). No in-memory admission — the
       // row is authoritative.
-      const pacing = declaredPacing ?? defaultPacingForTool(toolId, policy);
+      const fallback = defaultPacingForTool(toolId, policy);
+      const pacing = declaredPacing ?? {
+        ...fallback,
+        provider: resolvedPacing?.provider ?? fallback.provider,
+      };
+      if (!declaredPacing) {
+        // bucketId is the authenticated quota scope. Unknown tools sharing a
+        // credential must not each mint an independent copy of the default rate.
+        // Start its one shared lane at one probe/sec; the DB-owned AIMD quickly
+        // ramps on sustained success and reacts to Retry-After if the provider
+        // tells us the quota is lower.
+        pacing.rules = pacing.rules.map((rule) => ({
+          ...rule,
+          ruleId: 'default:provider-scope',
+          requestsPerWindow: DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
+          windowMs: 1_000,
+        }));
+      }
       providerByTool.set(toolId, pacing.provider);
       pacingRulesByTool.set(toolId, pacing.rules);
       return pacing;
     }
+    const fallback = defaultPacingForTool(toolId, policy);
     const pacing = adaptiveAdmission.resolvePacing({
       orgId: input.scope.orgId,
       toolId,
       declaredPacing,
-      fallbackPacing: defaultPacingForTool(toolId, policy),
+      fallbackPacing: {
+        ...fallback,
+        provider: resolvedPacing?.provider ?? fallback.provider,
+      },
     });
     providerByTool.set(toolId, pacing.provider);
     pacingRulesByTool.set(toolId, pacing.rules);
@@ -623,8 +660,13 @@ export function createPlayExecutionGovernor(
           rateScopeToken: rateScope[1],
           rules: pacing.rules,
           signal: opts?.signal,
+          yieldProviderHolds: opts?.yieldProviderHolds,
         });
-        return providerPermitFrom(pacing.provider, rateLease, localConcurrencySlot);
+        return providerPermitFrom(
+          pacing.provider,
+          rateLease,
+          localConcurrencySlot,
+        );
       } catch (error) {
         localConcurrencySlot?.release();
         throw error;
@@ -727,6 +769,7 @@ export function createPlayExecutionGovernor(
         bucketId: rateScope[0],
         ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
         cooldownMs: bp.retryAfterMs,
+        ...(bp.retryAtMs != null ? { retryAtMs: bp.retryAtMs } : {}),
         ...(rules ? { rules } : {}),
         ...(bp.latencyMs != null ? { latencyMs: bp.latencyMs } : {}),
         ...(bp.concurrencyGrant

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -188,7 +188,9 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
     maxWaterfallStepExecutions: 5_000_000,
   },
   pacing: {
-    // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
+    // Keep the generic fallback rate. The shared Postgres backend separately
+    // cold-starts unknown provider scopes at one probe/sec.
+    // Declared providers (rate-limit-definitions.ts) keep their own rates.
     defaultProviderRequestsPerSecond: 10,
     // This is the scheduler's resident-call envelope, not an RPS target.
     // Keep it aligned with the explicit run ceiling so an RPS-only provider

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -14,6 +14,12 @@
  */
 export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
 
+/** Cold-start rate for an unknown provider bucket shared across a fleet key. */
+export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 1;
+
+/** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
+export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;
+
 /** A single resolved rate rule for a bucket (from rate-limit-definitions). */
 export interface PacingRule {
   readonly ruleId: string;
@@ -83,10 +89,18 @@ export interface PacingPermit {
    * Absent on backends that keep no local block; callers use `cancel` then.
    */
   park?(): Promise<void>;
+  /** Recheck the durable provider-scope hold immediately before dispatch. */
+  validate?(): Promise<boolean>;
   /** Server-authored context for attributing this request's provider 429. */
   readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
   /** Effective shared rate for every app-runtime provider permit. */
   readonly pacingGrant?: ProviderPacingGrant | null;
+  /** Signed shared-scope proof for revalidation after the execution-gateway queue. */
+  readonly gatewayDispatchProof?: {
+    readonly bucketId: string;
+    readonly rateScopeToken: string;
+    readonly retryAtMsAtGrant: number;
+  } | null;
 }
 
 /**
@@ -118,6 +132,8 @@ export interface RateStateBackend {
     rateScopeToken?: string | null;
     rules: readonly PacingRule[];
     signal?: AbortSignal;
+    /** Managed Plays hand a shared provider hold to durable row suspension. */
+    yieldProviderHolds?: boolean;
   }): Promise<PacingPermit>;
 
   /**
@@ -140,6 +156,8 @@ export interface RateStateBackend {
     bucketId: string;
     rateScopeToken?: string | null;
     cooldownMs: number;
+    /** Absolute upstream retry deadline, stable across transport retries. */
+    retryAtMs?: number;
     /** Resolved provider rules for the rejected physical attempt. */
     rules?: readonly PacingRule[];
     /** Provider residence measured at the HTTP boundary. */

package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts

@@ -15,6 +15,9 @@ export function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
           suspendedInputRows: frame.suspendedInputRows.map((row) => ({
             inputIndex: row.inputIndex,
             data: { ...row.data },
+            ...(row.authoredInputDigest
+              ? { authoredInputDigest: row.authoredInputDigest }
+              : {}),
           })),
         }
       : {}),

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -79,6 +79,8 @@ export type PlayRunnerRateStateConcurrencyGrant = {
 export type PlayRunnerRateStateAcquireResult = {
   granted: number;
   waitMs: number;
+  /** This receiver serves rate_state_validate. Absent on pre-expansion readers. */
+  scopeValidationSupported?: boolean;
   /**
    * Gateway clock used to produce `scheduledAtMs`. Runners translate the
    * schedule into relative delays so cross-machine clock skew cannot compress
@@ -108,8 +110,8 @@ export type PlayRunnerRateStateAcquireResult = {
    */
   coolUntilMs?: number;
   /**
-   * The most recent server Retry-After stored verbatim (uncapped) as an absolute
-   * epoch-ms deadline. Purely observational — never gates a grant on its own.
+   * Most recent provider Retry-After deadline, surfaced for diagnostics. The
+   * authoritative admission gate is the durable provider-scope hold row.
    */
   claimedRetryAtMs?: number;
 };
@@ -128,6 +130,12 @@ export type PlayRunnerRateStatePenalizeInput = {
   bucketId: string;
   rateScopeToken?: string | null;
   cooldownMs: number;
+  /** Absolute provider retry deadline captured with the observed response. */
+  retryAtMs?: number;
+  /** Unflushed confirmed successes, sharing acquire's replay-safe identity. */
+  observedSuccesses?: number;
+  concurrencyObservations?: PlayRunnerRateStateConcurrencyObservation[];
+  concurrencyObservationBatch?: PlayRunnerRateStateConcurrencyObservationBatch;
   /**
    * The same resolved rules that admitted the attempt. Optional during the
    * rolling deploy window: old runners retain rate-only backpressure handling.
@@ -140,6 +148,18 @@ export type PlayRunnerRateStatePenalizeInput = {
   schedulerSchema?: string | null;
 };
 
+export type PlayRunnerRateStateValidateInput = {
+  bucketId: string;
+  rateScopeToken?: string | null;
+  schedulerSchema?: string | null;
+};
+
+export type PlayRunnerRateStateValidateResult = {
+  allowed: boolean;
+  serverNowMs: number;
+  retryAtMs: number;
+};
+
 export type PlayRunnerBudgetCharge = {
   key: string;
   amount: number;

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -711,6 +711,7 @@ export interface RuntimeResourceGovernor {
   acquireProviderPermit(input: {
     toolId: string;
     signal?: AbortSignal;
+    yieldProviderHolds?: boolean;
   }): Promise<RuntimeProviderPermit>;
   /**
    * Non-blocking provider admission from capacity this process already holds.
@@ -748,6 +749,7 @@ export interface RuntimeResourceGovernor {
     provider: string;
     toolId?: string;
     retryAfterMs: number;
+    retryAtMs?: number;
     /** Provider residence for the rejected physical request, when observed. */
     latencyMs?: number | null;
     /** Server-authored context for attributing this physical provider 429. */
@@ -1585,7 +1587,10 @@ export function createRuntimeResourceGovernor(input: {
       const startedAt = Date.now();
       const lease = await input.executionGovernor.acquireProviderPermit(
         providerInput.toolId,
-        { signal: providerInput.signal },
+        {
+          signal: providerInput.signal,
+          yieldProviderHolds: providerInput.yieldProviderHolds,
+        },
       );
       observe({
         providerResourceKey: lease.provider,

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -241,15 +241,17 @@ export function providerExhaustedMessage(input: {
 }
 
 /**
- * Thrown by the DB-authoritative pacer (`app_runtime_postgres` rate-state
- * backend) when a provider's hold exceeds the tolerable wait
- * (PROVIDER_EXHAUSTED_MAX_WAIT_MS). The call is skipped BEFORE any tool
- * execution request, so nothing is dispatched and nothing is billed. Row-failure
- * isolation still applies: this is a row/step outcome, not a run-fatal error.
+ * Control signal from the DB-authoritative pacer when a provider hold exceeds
+ * the in-process wait threshold. The next provider call is not dispatched.
+ * Managed durable Plays translate this into a persisted run/row wait and resume
+ * after the deadline; callers without durable boundaries receive a repairable
+ * provider-exhausted outcome. This signal is not a terminal row failure in the
+ * managed durable path.
  */
 export class ProviderExhaustedError extends Error {
   readonly code = PROVIDER_EXHAUSTED_CODE;
   readonly provider: string;
+  readonly publicDetails: ToolExecutionError['publicDetails'];
   /** ISO 8601 retry deadline (from the server Retry-After / cooldown). */
   readonly retryAt: string;
   constructor(input: { provider: string; retryAtMs: number; cause?: unknown }) {
@@ -264,6 +266,10 @@ export class ProviderExhaustedError extends Error {
     );
     this.name = 'ProviderExhaustedError';
     this.provider = input.provider;
+    this.publicDetails =
+      input.cause instanceof ToolExecutionError
+        ? input.cause.publicDetails
+        : null;
     this.retryAt = Number.isFinite(Number(input.retryAtMs))
       ? new Date(Number(input.retryAtMs)).toISOString()
       : new Date(0).toISOString();

package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts

@@ -85,6 +85,8 @@ export function createRuntimeFlow<Input>(input: {
 export type RuntimeDispatchProviderPermit = RuntimeFlowLease & {
   cancel(): Promise<void>;
   park?(): Promise<void>;
+  /** Recheck a shared provider hold at the physical dispatch boundary. */
+  validate?(): Promise<boolean>;
 };
 
 export type RuntimeDispatchFlowInput = {
@@ -92,6 +94,11 @@ export type RuntimeDispatchFlowInput = {
   signal?: AbortSignal;
   /** Fixtures consume provider capacity but acquire app transport after delay. */
   requiresTransportPermit: boolean;
+  /**
+   * Defer shared-hold validation until the request callback reaches its actual
+   * outbound send. The callback must call the supplied boundary exactly once.
+   */
+  deferProviderPermitValidation?: boolean;
   /** Emits the request-side transport wait observed by the caller's metrics. */
   onTransportAdmitted?: (waitMs: number) => void;
 };
@@ -114,12 +121,18 @@ type RuntimeDispatchPermit<
  */
 export type RuntimeDispatchScope = {
   /**
-   * Run the one physical request. `beforeRelease` executes after the response
-   * is known but while the provider lease is still valid, so a caller can
-   * durably record response-bound feedback that the lease authorizes.
+   * Run the one physical request. Deferred-validation callers must invoke
+   * `beforeProviderDispatch` immediately before the outbound send, after any
+   * async setup/queueing. `beforeRelease` executes after the response is known
+   * but while the provider lease is still valid, so a caller can durably
+   * record response-bound feedback that the lease authorizes.
    */
   request<Output>(
-    work: () => Output | Promise<Output>,
+    work: (
+      beforeProviderDispatch: (
+        onPermitInvalidated?: () => void,
+      ) => boolean | Promise<boolean>,
+    ) => Output | Promise<Output>,
     beforeRelease?: () => void | Promise<void>,
   ): Promise<Output>;
 };
@@ -130,10 +143,12 @@ export type RuntimeDispatchFlow<
   /**
    * Admit and run exactly one physical provider request.
    *
-   * Work that exits before `request` returns the provider permit through its
-   * cancellation path. Once `request` starts, the permit is released after
-   * the request body is consumed, even if that body cannot be verified. This
-   * preserves the ambiguity boundary while making resource lifetime private.
+   * Work must call `beforeProviderDispatch` at the actual outbound-send edge.
+   * That is where shared Retry-After state is checked: not before async
+   * request preparation. Work that exits before that call returns the provider
+   * permit through its cancellation path. Once the send starts, the permit is
+   * released after the response body is consumed, even if it cannot be
+   * verified.
    */
   run<Output>(
     input: RuntimeDispatchFlowInput,
@@ -264,8 +279,10 @@ export function createRuntimeDispatchFlow<
       admission: RuntimeDispatchFlowInput,
       work: (scope: RuntimeDispatchScope) => Output | Promise<Output>,
     ): Promise<Output> {
-      const permit = await admit(admission);
+      let permit = await admit(admission);
       let transportLease = permit.integrationRequestLease;
+      let requiresTransportPermitForReacquire =
+        admission.requiresTransportPermit;
       let requestStarted = false;
       let requestUsed = false;
       let scopeOpen = true;
@@ -284,7 +301,11 @@ export function createRuntimeDispatchFlow<
       try {
         const output = await work({
           request: <RequestOutput>(
-            request: () => RequestOutput | Promise<RequestOutput>,
+            request: (
+              beforeProviderDispatch: (
+                onPermitInvalidated?: () => void,
+              ) => boolean | Promise<boolean>,
+            ) => RequestOutput | Promise<RequestOutput>,
             beforeRelease?: () => void | Promise<void>,
           ): Promise<RequestOutput> => {
             if (!scopeOpen) {
@@ -301,10 +322,69 @@ export function createRuntimeDispatchFlow<
             }
             requestUsed = true;
             requestPromise = (async () => {
+              let dispatchStarted = false;
+              const beforeProviderDispatch = (
+                onPermitInvalidated?: () => void,
+              ): boolean | Promise<boolean> => {
+                if (dispatchStarted) {
+                  throw new Error(
+                    'Runtime dispatch flow permits exactly one physical provider dispatch.',
+                  );
+                }
+                if (!permit.providerPermit.validate) {
+                  // Preserve the zero-round-trip in-memory path: deferred
+                  // callers still cross the exact send boundary, but do not
+                  // pay an artificial microtask when there is nothing shared
+                  // to revalidate.
+                  dispatchStarted = true;
+                  requestStarted = true;
+                  return false;
+                }
+                return validateAndStartProviderDispatch(onPermitInvalidated);
+              };
+              const validateAndStartProviderDispatch = async (
+                onPermitInvalidated?: () => void,
+              ): Promise<boolean> => {
+                let replaced = false;
+                while (
+                  permit.providerPermit.validate &&
+                  !(await permit.providerPermit.validate())
+                ) {
+                  requestStarted = false;
+                  onPermitInvalidated?.();
+                  await releasePermit();
+                  permit = await admit({
+                    ...admission,
+                    requiresTransportPermit:
+                      requiresTransportPermitForReacquire,
+                  });
+                  transportLease = permit.integrationRequestLease;
+                  requestStarted = false;
+                  permitReleased = false;
+                  replaced = true;
+                }
+                dispatchStarted = true;
+                if (admission.deferProviderPermitValidation) {
+                  requestStarted = true;
+                }
+                return replaced;
+              };
               try {
+                if (!admission.deferProviderPermitValidation) {
+                  // Preserve the ordinary API: its request callback begins
+                  // the physical send immediately. Callers with asynchronous
+                  // preparation opt into deferred validation and invoke this
+                  // same boundary at the final outbound edge.
+                  if (permit.providerPermit.validate) {
+                    await beforeProviderDispatch();
+                  } else {
+                    dispatchStarted = true;
+                  }
+                }
                 if (transportLease) {
                   admission.onTransportAdmitted?.(0);
                 } else {
+                  requiresTransportPermitForReacquire = true;
                   const transportStartedAt = Date.now();
                   transportLease = await input.waitForTransportSlot({
                     signal: admission.signal,
@@ -313,8 +393,18 @@ export function createRuntimeDispatchFlow<
                     Date.now() - transportStartedAt,
                   );
                 }
-                requestStarted = true;
-                return await request();
+                if (!admission.deferProviderPermitValidation) {
+                  // Preserve the cancellation boundary for a permit whose
+                  // transport wait fails before the request callback begins.
+                  requestStarted = true;
+                }
+                const result = await request(beforeProviderDispatch);
+                if (!dispatchStarted) {
+                  throw new Error(
+                    'Runtime dispatch request completed without crossing its provider-dispatch boundary.',
+                  );
+                }
+                return result;
               } finally {
                 // Feedback is response-bound, but it is not allowed to strand
                 // the capacity that made the response possible. Preserve the

package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts

@@ -47,6 +47,7 @@ export const RUNTIME_OPERATION_CAPABILITIES = {
   rate_state_acquire: 'rate.acquire',
   rate_state_release: 'rate.acquire',
   rate_state_penalize: 'rate.acquire',
+  rate_state_validate: 'rate.acquire',
   get_runtime_traffic_policy: 'rate.acquire',
   governor_budget_charge: 'budget.charge',
   resolve_secret: 'secret.resolve',
@@ -69,6 +70,13 @@ export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
         kind: 'equivalent-fallback',
         action: 'complete_runtime_step_receipts',
       },
+      // Permit validation cannot safely fall back to a cached ticket: an old
+      // receiver would ignore the durable provider Retry-After hold. New
+      // workers therefore fail closed with this typed compatibility result.
+      rate_state_validate: {
+        kind: 'feature-unavailable',
+        code: 'RATE_STATE_VALIDATE_UNAVAILABLE',
+      },
     },
   });