deepline 0.3.322 → 0.3.324

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -26,6 +26,7 @@ import {
   type PacingRule,
   type ProviderConcurrencyGrant,
   type ProviderPacingGrant,
+  type ProviderPermitValidationOptions,
   type RateStateBackend,
 } from './rate-state-backend';
 import {
@@ -37,6 +38,7 @@ import {
   InMemoryBudgetStateBackend,
   type BudgetStateBackend,
 } from './budget-state-backend';
+import { PROVIDER_CONCURRENCY_SAFETY_LIMIT } from '../provider-capacity/provider-concurrency-policy';
 
 export interface WorkLease {
   /** Free the slot / pacing permit. Idempotent. MUST be called in a finally. */
@@ -68,7 +70,10 @@ export interface ProviderPermit extends WorkLease {
    */
   park(): Promise<void>;
   /** Recheck shared provider state immediately before network dispatch. */
-  validate?(): Promise<boolean>;
+  validate?(
+    signal?: AbortSignal,
+    options?: ProviderPermitValidationOptions,
+  ): Promise<boolean>;
 }
 
 export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
@@ -179,7 +184,11 @@ export interface PlayExecutionGovernor {
    */
   acquireProviderPermit(
     toolId: string,
-    opts?: { signal?: AbortSignal; yieldProviderHolds?: boolean },
+    opts?: {
+      signal?: AbortSignal;
+      yieldProviderHolds?: boolean;
+      retryPriority?: boolean;
+    },
   ): Promise<ProviderPermit>;
 
   /**
@@ -444,9 +453,12 @@ export function createPlayExecutionGovernor(
       gatewayDispatchProof: rateLease.gatewayDispatchProof ?? null,
       ...(rateLease.validate
         ? {
-            validate: async () => {
+            validate: async (
+              signal?: AbortSignal,
+              options?: ProviderPermitValidationOptions,
+            ) => {
               if (settled) return false;
-              return await rateLease.validate!();
+              return await rateLease.validate!(signal, options);
             },
           }
         : {}),
@@ -519,6 +531,7 @@ export function createPlayExecutionGovernor(
           ruleId: 'default:provider-scope',
           requestsPerWindow: DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS,
           windowMs: 1_000,
+          maxConcurrency: PROVIDER_CONCURRENCY_SAFETY_LIMIT,
         }));
       }
       providerByTool.set(toolId, pacing.provider);
@@ -661,6 +674,7 @@ export function createPlayExecutionGovernor(
           rules: pacing.rules,
           signal: opts?.signal,
           yieldProviderHolds: opts?.yieldProviderHolds,
+          ...(opts?.retryPriority ? { retryPriority: true } : {}),
         });
         return providerPermitFrom(
           pacing.provider,

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -17,6 +17,10 @@ export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
 /** Cold-start rate for an unknown provider bucket shared across a fleet key. */
 export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 10;
 
+/** Recover quickly to useful seed capacity while leaving quota headroom. */
+export const DEFAULT_UNKNOWN_PROVIDER_FAST_RECOVERY_RPS =
+  DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS * 0.9;
+
 /** Learned-rate floor, independent of the unknown provider's starting guess. */
 export const DEFAULT_UNKNOWN_PROVIDER_MIN_RPS = 2;
 
@@ -93,7 +97,10 @@ export interface PacingPermit {
    */
   park?(): Promise<void>;
   /** Recheck the durable provider-scope hold immediately before dispatch. */
-  validate?(): Promise<boolean>;
+  validate?(
+    signal?: AbortSignal,
+    options?: ProviderPermitValidationOptions,
+  ): Promise<boolean>;
   /** Server-authored context for attributing this request's provider 429. */
   readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
   /** Effective shared rate for every app-runtime provider permit. */
@@ -103,9 +110,15 @@ export interface PacingPermit {
     readonly bucketId: string;
     readonly rateScopeToken: string;
     readonly retryAtMsAtGrant: number;
+    readonly dispatchValidationSupported?: boolean;
   } | null;
 }
 
+export type ProviderPermitValidationOptions = {
+  /** The request carries this permit's signed proof to the execution gateway. */
+  dispatchValidationAtGateway?: boolean;
+};
+
 /**
  * Discriminates the backend so the Governor can adapt its policy to it. Only
  * `app_runtime_postgres` (the Absurd/Node substrate's DB-backed pacer) is
@@ -137,6 +150,8 @@ export interface RateStateBackend {
     signal?: AbortSignal;
     /** Managed Plays hand a shared provider hold to durable row suspension. */
     yieldProviderHolds?: boolean;
+    /** Prefer an already-started logical call over fresh permit waiters. */
+    retryPriority?: boolean;
   }): Promise<PacingPermit>;
 
   /**

package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts

@@ -503,9 +503,13 @@ export class MapExecutionProgress {
     });
   }
 
-  resolverStarted(): void {
+  get activeResolvers(): number {
+    return this.active;
+  }
+
+  resolverStarted(input: { retry?: boolean } = {}): void {
     this.tracksActivity = true;
-    this.started += 1;
+    if (!input.retry) this.started += 1;
     this.active += 1;
   }
 

package/dist/bundling-sources/shared_libs/play-runtime/play-http-request.ts

@@ -291,7 +291,7 @@ export async function sendPlayHttpRequest(input: {
                     try {
                       signal.throwIfAborted();
                       const valid =
-                        !acquired.validate || (await acquired.validate());
+                        !acquired.validate || (await acquired.validate(signal));
                       signal.throwIfAborted();
                       if (valid) return acquired;
                     } catch (error) {

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -88,6 +88,8 @@ export type PlayRunnerRateStateAcquireResult = {
   waitMs: number;
   /** This receiver serves rate_state_validate. Absent on pre-expansion readers. */
   scopeValidationSupported?: boolean;
+  /** This gateway rechecks signed provider permits on integration execute. */
+  dispatchValidationSupported?: boolean;
   /**
    * Gateway clock used to produce `scheduledAtMs`. Runners translate the
    * schedule into relative delays so cross-machine clock skew cannot compress

package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts

@@ -7,6 +7,8 @@ export type Gradient2ConcurrencyControllerOptions = Readonly<{
   rttTolerance: number;
   /** Bounded cold-start probe; one or below preserves ordinary Gradient2. */
   startupProbeMultiplier?: number;
+  /** Reprobe undeclared capacity after an entire healthy completion window. */
+  recoveryProbeMultiplier?: number;
 }>;
 
 export type Gradient2Observation = Readonly<{
@@ -63,6 +65,7 @@ export class Gradient2ConcurrencyController {
   readonly #longWindowSmoothing: number;
   readonly #rttTolerance: number;
   readonly #startupProbeMultiplier: number;
+  readonly #recoveryProbeMultiplier: number;
   #limit: number;
   #shortRttMs: number | null = null;
   #longRttMs: number | null = null;
@@ -95,6 +98,13 @@ export class Gradient2ConcurrencyController {
       1,
       positive(options.startupProbeMultiplier ?? 1, 'Startup probe multiplier'),
     );
+    this.#recoveryProbeMultiplier = Math.max(
+      1,
+      positive(
+        options.recoveryProbeMultiplier ?? 1,
+        'Recovery probe multiplier',
+      ),
+    );
     this.#windowStartedAtLimit = this.#limit;
     if (checkpoint) {
       this.#limit = clamp(
@@ -171,13 +181,13 @@ export class Gradient2ConcurrencyController {
       0.5,
       1,
     );
-    const probeHealthy =
-      !this.#hasObservedThrottle &&
-      this.#startupProbeMultiplier > 1 &&
-      gradient >= 1 - this.#smoothing;
+    const probeMultiplier = this.#hasObservedThrottle
+      ? this.#recoveryProbeMultiplier
+      : this.#startupProbeMultiplier;
+    const probeHealthy = probeMultiplier > 1 && gradient >= 1 - this.#smoothing;
     if (probeHealthy) {
       this.#limit = clamp(
-        this.#limit * this.#startupProbeMultiplier,
+        this.#limit * probeMultiplier,
         this.#minimumLimit,
         this.#maximumLimit,
       );

package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts

@@ -30,6 +30,7 @@ export const PROVIDER_CAPACITY_POLICY = Object.freeze({
     longWindowSmoothing: 0.03,
     rttTolerance: 1.25,
     startupProbeMultiplier: 2,
+    unknownProviderRecoveryProbeMultiplier: 2,
   }),
 } as const);
 

package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts

@@ -15,16 +15,21 @@ export const PROVIDER_CONCURRENCY_CONTROLLER_TUNING = Object.freeze({
 
 /**
  * The one Gradient2 policy used by Postgres enforcement, shadow evaluation,
- * and simulations. Callers choose only the server-authored initial and maximum
- * limits; algorithm tuning must not drift by deployment surface.
+ * and simulations. Callers choose the server-authored limits and whether the
+ * scope is undeclared; algorithm tuning must not drift by deployment surface.
  */
 export function providerConcurrencyControllerOptions(input: {
   initialLimit: number;
   maximumLimit?: number;
+  unknownProvider?: boolean;
 }): Gradient2ConcurrencyControllerOptions {
   return Object.freeze({
     ...PROVIDER_CONCURRENCY_CONTROLLER_TUNING,
     initialLimit: input.initialLimit,
     maximumLimit: input.maximumLimit ?? PROVIDER_CONCURRENCY_SAFETY_LIMIT,
+    recoveryProbeMultiplier: input.unknownProvider
+      ? PROVIDER_CAPACITY_POLICY.concurrency
+          .unknownProviderRecoveryProbeMultiplier
+      : 1,
   });
 }

package/dist/bundling-sources/shared_libs/play-runtime/provider-row-retry-queue.ts

@@ -0,0 +1,65 @@
+export type ProviderRowRetry = {
+  rowIndex: number;
+  provider: string;
+  retryAtMs: number;
+  readyAtMs: number;
+  /** Ready work may run, but this row must not wait in an otherwise idle map. */
+  residentWaitBudgetExhausted?: boolean;
+};
+
+/** Deadlines belong to rows; a long hold must not hide another ready row. */
+export class ProviderRowRetryQueue {
+  private readonly rows: ProviderRowRetry[] = [];
+
+  get size(): number {
+    return this.rows.length;
+  }
+
+  peek(): ProviderRowRetry | undefined {
+    return this.rows[0];
+  }
+
+  add(row: ProviderRowRetry): void {
+    let index = this.rows.length;
+    this.rows.push(row);
+    while (index > 0) {
+      const parentIndex = Math.floor((index - 1) / 2);
+      const parent = this.rows[parentIndex]!;
+      if (!this.precedes(row, parent)) break;
+      this.rows[index] = parent;
+      index = parentIndex;
+    }
+    this.rows[index] = row;
+  }
+
+  takeReady(nowMs: number): ProviderRowRetry | undefined {
+    const first = this.peek();
+    if (!first || first.readyAtMs > nowMs) return undefined;
+    const last = this.rows.pop()!;
+    if (this.rows.length === 0) return first;
+    let index = 0;
+    while (true) {
+      const left = index * 2 + 1;
+      if (left >= this.rows.length) break;
+      const right = left + 1;
+      const childIndex =
+        right < this.rows.length &&
+        this.precedes(this.rows[right]!, this.rows[left]!)
+          ? right
+          : left;
+      const child = this.rows[childIndex]!;
+      if (!this.precedes(child, last)) break;
+      this.rows[index] = child;
+      index = childIndex;
+    }
+    this.rows[index] = last;
+    return first;
+  }
+
+  private precedes(left: ProviderRowRetry, right: ProviderRowRetry): boolean {
+    return (
+      left.readyAtMs < right.readyAtMs ||
+      (left.readyAtMs === right.readyAtMs && left.rowIndex < right.rowIndex)
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -770,6 +770,7 @@ export interface RuntimeResourceGovernor {
     toolId: string;
     signal?: AbortSignal;
     yieldProviderHolds?: boolean;
+    retryPriority?: boolean;
   }): Promise<RuntimeProviderPermit>;
   /**
    * Non-blocking provider admission from capacity this process already holds.
@@ -1746,6 +1747,7 @@ export function createRuntimeResourceGovernor(input: {
         {
           signal: providerInput.signal,
           yieldProviderHolds: providerInput.yieldProviderHolds,
+          ...(providerInput.retryPriority ? { retryPriority: true } : {}),
         },
       );
       observe({

package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts

@@ -8,6 +8,8 @@
  * inventing a second runtime scheduler.
  */
 
+import type { ProviderPermitValidationOptions } from './governor/rate-state-backend';
+
 export type RuntimeFlowLease = {
   release(): void;
 };
@@ -86,7 +88,10 @@ export type RuntimeDispatchProviderPermit = RuntimeFlowLease & {
   cancel(): Promise<void>;
   park?(): Promise<void>;
   /** Recheck a shared provider hold at the physical dispatch boundary. */
-  validate?(): Promise<boolean>;
+  validate?(
+    signal?: AbortSignal,
+    options?: ProviderPermitValidationOptions,
+  ): Promise<boolean>;
 };
 
 export type RuntimeDispatchFlowInput = {
@@ -99,6 +104,8 @@ export type RuntimeDispatchFlowInput = {
    * outbound send. The callback must call the supplied boundary exactly once.
    */
   deferProviderPermitValidation?: boolean;
+  /** The request forwards its signed permit to the execution gateway. */
+  providerDispatchAtGateway?: boolean;
   /** Emits the request-side transport wait observed by the caller's metrics. */
   onTransportAdmitted?: (waitMs: number) => void;
 };
@@ -133,7 +140,15 @@ export type RuntimeDispatchScope = {
         onPermitInvalidated?: () => void,
       ) => boolean | Promise<boolean>,
     ) => Output | Promise<Output>,
-    beforeRelease?: () => void | Promise<void>,
+    /**
+     * Persist response feedback while the lease is live. Return
+     * `not_dispatched` only for a definite rejection before provider work;
+     * this refunds the reservation. Ambiguous outcomes retain release semantics.
+     */
+    beforeRelease?: () =>
+      | void
+      | 'not_dispatched'
+      | Promise<void | 'not_dispatched'>,
   ): Promise<Output>;
 };
 
@@ -306,7 +321,10 @@ export function createRuntimeDispatchFlow<
                 onPermitInvalidated?: () => void,
               ) => boolean | Promise<boolean>,
             ) => RequestOutput | Promise<RequestOutput>,
-            beforeRelease?: () => void | Promise<void>,
+            beforeRelease?: () =>
+              | void
+              | 'not_dispatched'
+              | Promise<void | 'not_dispatched'>,
           ): Promise<RequestOutput> => {
             if (!scopeOpen) {
               return Promise.reject(
@@ -348,7 +366,11 @@ export function createRuntimeDispatchFlow<
                 let replaced = false;
                 while (
                   permit.providerPermit.validate &&
-                  !(await permit.providerPermit.validate())
+                  !(await (admission.providerDispatchAtGateway
+                    ? permit.providerPermit.validate(admission.signal, {
+                        dispatchValidationAtGateway: true,
+                      })
+                    : permit.providerPermit.validate(admission.signal)))
                 ) {
                   requestStarted = false;
                   onPermitInvalidated?.();
@@ -412,7 +434,14 @@ export function createRuntimeDispatchFlow<
                 // always release all leases even when feedback persistence
                 // fails.
                 try {
-                  if (requestStarted) await beforeRelease?.();
+                  if (
+                    requestStarted &&
+                    (await beforeRelease?.()) === 'not_dispatched'
+                  ) {
+                    // A definite receiver rejection returns the reservation.
+                    // Ambiguous exchanges retain ordinary release semantics.
+                    requestStarted = false;
+                  }
                 } finally {
                   await releasePermit();
                 }

package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts

@@ -15,6 +15,7 @@ export type RuntimeMapRowFeedSnapshot = {
 };
 
 export const RUNTIME_MAP_ROW_FEED_MAX_WAIT_MS = 30 * 60 * 1000;
+const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;
 
 export class RuntimeMapRowFeedStalledError extends Error {
   readonly snapshot: RuntimeMapRowFeedSnapshot;
@@ -51,7 +52,11 @@ export class RuntimeMapRowFeed {
   #closed = false;
   #failure: unknown = null;
   #failed = false;
-  readonly #rowWaiters = new Map<Waiter<boolean>, number>();
+  // A null index waits for pool activity/deadlines independently of source
+  // closure; an indexed waiter still means "more source rows".
+  readonly #rowWaiters = new Map<Waiter<boolean>, number | null>();
+  #workerWakeTimer: ReturnType<typeof setTimeout> | null = null;
+  #workerWakeAtMs: number | null = null;
   #drainWaiter: DrainWaiter | null = null;
   #onAppended: ((fromIndex: number, toIndex: number) => void) | null = null;
   readonly #maxWaitMs: number;
@@ -89,6 +94,7 @@ export class RuntimeMapRowFeed {
   close(): void {
     if (this.#closed || this.#failed) return;
     this.#closed = true;
+    if (!this.#hasWorkerWakeWaiters()) this.#clearWorkerWake();
     this.#wakeRowWaiters();
     this.#settleDrainWaiter();
   }
@@ -97,6 +103,7 @@ export class RuntimeMapRowFeed {
     if (this.#failed) return;
     this.#failed = true;
     this.#failure = error;
+    this.#clearWorkerWake();
     for (const [waiter] of this.#rowWaiters) {
       clearTimeout(waiter.timer);
       waiter.reject(error);
@@ -117,6 +124,38 @@ export class RuntimeMapRowFeed {
     return this.#closed;
   }
 
+  /** Recheck a provider retry deadline while the source prepares its next page. */
+  wakeWorkersAt(wakeAtMs: number): void {
+    if (!Number.isFinite(wakeAtMs)) {
+      throw new Error('Runtime map worker wake deadline must be finite.');
+    }
+    if (this.#failed || (this.#closed && !this.#hasWorkerWakeWaiters())) return;
+    if (this.#workerWakeAtMs !== null && this.#workerWakeAtMs <= wakeAtMs)
+      return;
+    this.#clearWorkerWake();
+    this.#workerWakeAtMs = wakeAtMs;
+    this.#scheduleWorkerWake();
+  }
+
+  #scheduleWorkerWake(): void {
+    const wakeAtMs = this.#workerWakeAtMs;
+    if (wakeAtMs === null) return;
+    const waitMs = Math.min(
+      MAX_TIMER_DELAY_MS,
+      Math.max(0, wakeAtMs - Date.now()),
+    );
+    this.#workerWakeTimer = setTimeout(() => {
+      this.#workerWakeTimer = null;
+      if (Date.now() < wakeAtMs) {
+        this.#scheduleWorkerWake();
+        return;
+      }
+      this.#workerWakeAtMs = null;
+      this.wakeWorkers();
+    }, waitMs);
+    this.#workerWakeTimer.unref?.();
+  }
+
   markDispatched(count = 1): void {
     if (!Number.isInteger(count) || count < 0) {
       throw new Error(
@@ -144,28 +183,77 @@ export class RuntimeMapRowFeed {
   }
 
   /**
-   * Resolves true once row `index` has been appended, false once the feed is
-   * closed with no such row. Rejects with the feed's failure.
+   * Resolves true once row `index` has been appended or an explicitly scheduled
+   * worker wake requests a recheck, false once the feed is closed with no such
+   * row. Rejects with the feed's failure.
    */
   waitForRows(index: number): Promise<boolean> {
     if (this.#failed) return Promise.reject(this.#failure);
     if (index < this.#appended) return Promise.resolve(true);
     if (this.#closed) return Promise.resolve(false);
+    return this.#waitForRowsOrWake(index);
+  }
+
+  /** A retry can become ready while siblings work after the source closes. */
+  waitForWorkerWake(wakeAtMs: number): Promise<boolean> {
+    if (!Number.isFinite(wakeAtMs)) {
+      return Promise.reject(
+        new Error('Runtime map worker wake deadline must be finite.'),
+      );
+    }
+    if (this.#failed) return Promise.reject(this.#failure);
+    if (wakeAtMs <= Date.now()) return Promise.resolve(true);
+    // A known provider hold is bounded by its own deadline, rather than the
+    // source's idle diagnostic. Leave indexed source waits on their usual bound.
+    const waiting = this.#waitForRowsOrWake(
+      null,
+      Math.max(this.#maxWaitMs, wakeAtMs - Date.now() + 1),
+    );
+    this.wakeWorkersAt(wakeAtMs);
+    return waiting;
+  }
+
+  /** Recheck retry ownership when the last active resolver returns its lease. */
+  wakeWorkers(): void {
+    this.#clearWorkerWake();
+    for (const [waiter] of this.#rowWaiters) {
+      clearTimeout(waiter.timer);
+      waiter.resolve(true);
+    }
+    this.#rowWaiters.clear();
+  }
+
+  #waitForRowsOrWake(
+    index: number | null,
+    maxWaitMs = this.#maxWaitMs,
+  ): Promise<boolean> {
     return new Promise<boolean>((resolve, reject) => {
       const startedAt = Date.now();
+      const expiresAtMs = startedAt + maxWaitMs;
+      const onTimeout = () => {
+        const remainingMs = expiresAtMs - Date.now();
+        if (remainingMs > 0) {
+          waiter.timer = setTimeout(
+            onTimeout,
+            Math.min(MAX_TIMER_DELAY_MS, remainingMs),
+          );
+          waiter.timer.unref?.();
+          return;
+        }
+        this.#rowWaiters.delete(waiter);
+        if (this.#rowWaiters.size === 0) this.#clearWorkerWake();
+        reject(
+          new RuntimeMapRowFeedStalledError({
+            waiter: 'worker',
+            waitMs: Date.now() - startedAt,
+            snapshot: this.snapshot(),
+          }),
+        );
+      };
       const waiter: Waiter<boolean> = {
         resolve,
         reject,
-        timer: setTimeout(() => {
-          this.#rowWaiters.delete(waiter);
-          reject(
-            new RuntimeMapRowFeedStalledError({
-              waiter: 'worker',
-              waitMs: Date.now() - startedAt,
-              snapshot: this.snapshot(),
-            }),
-          );
-        }, this.#maxWaitMs),
+        timer: setTimeout(onTimeout, Math.min(MAX_TIMER_DELAY_MS, maxWaitMs)),
       };
       waiter.timer.unref?.();
       this.#rowWaiters.set(waiter, index);
@@ -216,13 +304,26 @@ export class RuntimeMapRowFeed {
     };
   }
 
+  #clearWorkerWake(): void {
+    if (this.#workerWakeTimer) clearTimeout(this.#workerWakeTimer);
+    this.#workerWakeTimer = null;
+    this.#workerWakeAtMs = null;
+  }
+
+  #hasWorkerWakeWaiters(): boolean {
+    for (const index of this.#rowWaiters.values()) {
+      if (index === null) return true;
+    }
+    return false;
+  }
+
   #wakeRowWaiters(): void {
     for (const [waiter, index] of this.#rowWaiters) {
-      if (index < this.#appended) {
+      if (index !== null && index < this.#appended) {
         this.#rowWaiters.delete(waiter);
         clearTimeout(waiter.timer);
         waiter.resolve(true);
-      } else if (this.#closed) {
+      } else if (this.#closed && index !== null) {
         this.#rowWaiters.delete(waiter);
         clearTimeout(waiter.timer);
         waiter.resolve(false);