deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts

@@ -1,3 +1,4 @@
+import { capacityShedRetryAfterMs } from '../app-runtime-api';
 import type {
   PlayRunnerBudgetCharge,
   PlayRunnerBudgetChargeResult,
@@ -11,12 +12,29 @@ const RESERVATION_BLOCKS: Record<string, number> = {
   waterfallStep: 256,
 };
 
+/**
+ * Longest one charge waits out load shedding, measured from when it was
+ * requested so that time queued behind an earlier shed charge counts. A shed
+ * rate-state acquire waits longer (`RATE_ACQUIRE_MAX_WAIT_MS`), and both add to
+ * one tool call's time outside its `timeoutMs`. Past this a shed is treated
+ * like any other failure and surfaces, because the run cannot prove it is
+ * inside its budget.
+ */
+export const BUDGET_CHARGE_SHED_WAIT_BUDGET_MS = 120_000;
+
 export type BudgetStateChargePort = (input: {
   rootRunId: string;
   reservationId: string;
   charges: readonly PlayRunnerBudgetCharge[];
 }) => Promise<PlayRunnerBudgetChargeResult>;
 
+export interface BlockReservingBudgetStateOptions {
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  /** Source in [0,1) for the shed wait's jitter. Tests pin it. */
+  jitter?: () => number;
+}
+
 /**
  * Reserves durable budget in coarse blocks while charging each local operation
  * exactly once. The durable port remains the authority across sandboxes and
@@ -25,14 +43,29 @@ export type BudgetStateChargePort = (input: {
 export class BlockReservingBudgetStateBackend implements BudgetStateBackend {
   private readonly remaining = new Map<string, number>();
   private chargeTail: Promise<void> = Promise.resolve();
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly jitter: () => number;
 
-  constructor(private readonly chargePort: BudgetStateChargePort) {}
+  constructor(
+    private readonly chargePort: BudgetStateChargePort,
+    options: BlockReservingBudgetStateOptions = {},
+  ) {
+    this.now = options.now ?? (() => Date.now());
+    this.sleep =
+      options.sleep ??
+      ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.jitter = options.jitter ?? Math.random;
+  }
 
   charge(input: {
     rootRunId: string;
     charges: readonly PlayRunnerBudgetCharge[];
   }): Promise<PlayRunnerBudgetChargeResult> {
-    const result = this.chargeTail.then(() => this.chargeSerial(input));
+    const requestedAtMs = this.now();
+    const result = this.chargeTail.then(() =>
+      this.chargeSerial(input, requestedAtMs),
+    );
     this.chargeTail = result.then(
       () => undefined,
       () => undefined,
@@ -40,10 +73,13 @@ export class BlockReservingBudgetStateBackend implements BudgetStateBackend {
     return result;
   }
 
-  private async chargeSerial(input: {
-    rootRunId: string;
-    charges: readonly PlayRunnerBudgetCharge[];
-  }): Promise<PlayRunnerBudgetChargeResult> {
+  private async chargeSerial(
+    input: {
+      rootRunId: string;
+      charges: readonly PlayRunnerBudgetCharge[];
+    },
+    requestedAtMs: number,
+  ): Promise<PlayRunnerBudgetChargeResult> {
     const reservations: PlayRunnerBudgetCharge[] = [];
     const exactReservations: PlayRunnerBudgetCharge[] = [];
     const previousRemaining = new Map<string, number | undefined>();
@@ -84,14 +120,33 @@ export class BlockReservingBudgetStateBackend implements BudgetStateBackend {
       charges: readonly PlayRunnerBudgetCharge[],
       reservationId: string,
     ) => {
-      try {
-        return await request(charges, reservationId);
-      } catch (error) {
-        const normalized = normalizeBudgetError(error);
-        if (normalized instanceof BudgetStateLimitError) throw normalized;
-        return await request(charges, reservationId).catch((retryError) => {
-          throw normalizeBudgetError(retryError);
-        });
+      let resent = false;
+      while (true) {
+        try {
+          return await request(charges, reservationId);
+        } catch (error) {
+          const normalized = normalizeBudgetError(error);
+          if (normalized instanceof BudgetStateLimitError) throw normalized;
+          const shedRetryAfterMs = capacityShedRetryAfterMs(error);
+          if (shedRetryAfterMs !== null) {
+            // Back-pressure from a working store: wait as acquire does, then
+            // re-send. The reservation id is what makes this safe, not the
+            // shed. An earlier delivery of this request may have committed
+            // with its response lost, and the store replays that id instead
+            // of charging it again.
+            const waitMs =
+              shedRetryAfterMs + Math.round(this.jitter() * shedRetryAfterMs);
+            if (
+              this.now() - requestedAtMs + waitMs <
+              BUDGET_CHARGE_SHED_WAIT_BUDGET_MS
+            ) {
+              await this.sleep(waitMs);
+              continue;
+            }
+          }
+          if (resent) throw normalized;
+          resent = true;
+        }
       }
     };
     try {

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -163,6 +163,12 @@ export interface PlayExecutionGovernor {
    */
   tryAcquireToolExecutionSlot(): WorkLease | null;
 
+  /** Process-local live resource signal; never part of a stored launch. */
+  registerExternalExecutionPressure?(input: {
+    read(): boolean;
+    subscribe(changed: () => void): () => void;
+  }): () => void;
+
   /**
    * Admit one brief runtime-to-app HTTP fetch/body exchange. Long provider or
    * fixture residence happens before this edge and must not occupy this slot.
@@ -279,7 +285,27 @@ interface GovernorInput {
 class Semaphore {
   private inFlight = 0;
   private readonly waiters: Array<(reserved: boolean) => void> = [];
-  constructor(private readonly limit: number) {}
+  constructor(
+    private readonly limit: number,
+    private readonly readLimit?: () => number,
+  ) {}
+
+  private capacity(): number {
+    if (!this.readLimit) return this.limit;
+    const capacity = this.readLimit();
+    return Number.isSafeInteger(capacity) && capacity > 0
+      ? Math.min(this.limit, capacity)
+      : this.limit;
+  }
+
+  /** Re-read policy before reserving each FIFO handoff. */
+  refresh(): void {
+    while (this.waiters.length > 0 && this.inFlight < this.capacity()) {
+      const next = this.waiters.shift()!;
+      this.inFlight += 1;
+      next(true);
+    }
+  }
 
   private lease(): WorkLease {
     this.inFlight += 1;
@@ -294,19 +320,18 @@ class Semaphore {
         if (released) return;
         released = true;
         this.inFlight = Math.max(0, this.inFlight - 1);
-        const next = this.waiters.shift();
-        if (next) {
-          // Reserve before waking. A synchronous tryAcquire between this
-          // release and the promise continuation must not overtake FIFO work.
-          this.inFlight += 1;
-          next(true);
-        }
+        // Reserve before waking, with the current pressure verdict. A decrease
+        // drains existing work instead of revoking its acquired identity.
+        this.refresh();
       },
     };
   }
 
   tryAcquire(): WorkLease | null {
-    return this.inFlight >= this.limit ? null : this.lease();
+    this.refresh();
+    return this.waiters.length > 0 || this.inFlight >= this.capacity()
+      ? null
+      : this.lease();
   }
 
   async acquire(signal?: AbortSignal): Promise<WorkLease> {
@@ -319,7 +344,8 @@ class Semaphore {
         ? signal.reason
         : new Error('Slot acquire aborted.');
     }
-    while (this.inFlight >= this.limit) {
+    this.refresh();
+    while (this.waiters.length > 0 || this.inFlight >= this.capacity()) {
       const reserved = await new Promise<boolean>((resolve, reject) => {
         const onResolve = (wasReserved: boolean) => {
           signal?.removeEventListener('abort', onAbort);
@@ -328,6 +354,7 @@ class Semaphore {
         const onAbort = () => {
           const idx = this.waiters.indexOf(onResolve);
           if (idx >= 0) this.waiters.splice(idx, 1);
+          this.refresh();
           reject(
             signal?.reason instanceof Error
               ? signal.reason
@@ -336,6 +363,8 @@ class Semaphore {
         };
         this.waiters.push(onResolve);
         signal?.addEventListener('abort', onAbort, { once: true });
+        if (signal?.aborted) onAbort();
+        else this.refresh();
       });
       if (signal?.aborted) {
         // A release may have handed this waiter capacity immediately before
@@ -394,7 +423,17 @@ export function createPlayExecutionGovernor(
     input.resume ?? createDefaultGovernanceSnapshot(input.scope);
 
   const rowSlots = new Semaphore(policy.concurrency.rowMax);
-  const toolSlots = new Semaphore(policy.concurrency.toolCalls);
+  const externalPressureReaders = new Set<() => boolean>();
+  const readExternalCapacity = (): number => {
+    for (const read of externalPressureReaders) {
+      if (read()) return 1;
+    }
+    return policy.concurrency.toolCalls;
+  };
+  const toolSlots = new Semaphore(
+    policy.concurrency.toolCalls,
+    readExternalCapacity,
+  );
   const integrationRequestSlots = new Semaphore(
     policy.concurrency.integrationRequests,
   );
@@ -507,6 +546,7 @@ export function createPlayExecutionGovernor(
   async function resolveAdaptivePacing(toolId: string): Promise<{
     provider: string;
     rules: PacingRule[];
+    refreshRules?: () => readonly PacingRule[];
   }> {
     const resolvedPacing = await input.resolvePacing(toolId);
     const declaredPacing =
@@ -539,18 +579,30 @@ export function createPlayExecutionGovernor(
       return pacing;
     }
     const fallback = defaultPacingForTool(toolId, policy);
-    const pacing = adaptiveAdmission.resolvePacing({
-      orgId: input.scope.orgId,
-      toolId,
-      declaredPacing,
-      fallbackPacing: {
-        ...fallback,
-        provider: resolvedPacing?.provider ?? fallback.provider,
-      },
-    });
-    providerByTool.set(toolId, pacing.provider);
-    pacingRulesByTool.set(toolId, pacing.rules);
-    return pacing;
+    const resolveCurrentPacing = () => {
+      const pacing = adaptiveAdmission.resolvePacing({
+        orgId: input.scope.orgId,
+        toolId,
+        declaredPacing,
+        fallbackPacing: {
+          ...fallback,
+          provider: resolvedPacing?.provider ?? fallback.provider,
+        },
+      });
+      providerByTool.set(toolId, pacing.provider);
+      pacingRulesByTool.set(toolId, pacing.rules);
+      return pacing;
+    };
+    const pacing = resolveCurrentPacing();
+    return {
+      ...pacing,
+      // An unhinted call can wait across several learned-rate changes. Keep
+      // the existing controller authoritative when that waiter is reconsidered,
+      // rather than pinning its initial cold-start guess for the whole wait.
+      ...(!declaredPacing
+        ? { refreshRules: () => resolveCurrentPacing().rules }
+        : {}),
+    };
   }
 
   const budgetLimit = (kind: BudgetKind): number => {
@@ -623,6 +675,21 @@ export function createPlayExecutionGovernor(
 
     tryAcquireToolExecutionSlot: () => toolSlots.tryAcquire(),
 
+    registerExternalExecutionPressure(pressureInput) {
+      const read = () => pressureInput.read();
+      externalPressureReaders.add(read);
+      const unsubscribe = pressureInput.subscribe(() => toolSlots.refresh());
+      toolSlots.refresh();
+      let removed = false;
+      return () => {
+        if (removed) return;
+        removed = true;
+        unsubscribe();
+        externalPressureReaders.delete(read);
+        toolSlots.refresh();
+      };
+    },
+
     tryAcquireIntegrationRequestSlot: () =>
       integrationRequestSlots.tryAcquire(),
 
@@ -672,6 +739,7 @@ export function createPlayExecutionGovernor(
           bucketId: rateScope[0],
           rateScopeToken: rateScope[1],
           rules: pacing.rules,
+          ...(pacing.refreshRules ? { refreshRules: pacing.refreshRules } : {}),
           signal: opts?.signal,
           yieldProviderHolds: opts?.yieldProviderHolds,
           ...(opts?.retryPriority ? { retryPriority: true } : {}),
@@ -883,6 +951,8 @@ function createInlineChildGovernor(
     acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
     acquireToolExecutionSlot: (opts) => root.acquireToolExecutionSlot(opts),
     tryAcquireToolExecutionSlot: () => root.tryAcquireToolExecutionSlot(),
+    registerExternalExecutionPressure: (input) =>
+      root.registerExternalExecutionPressure?.(input) ?? (() => {}),
     tryAcquireIntegrationRequestSlot: () =>
       root.tryAcquireIntegrationRequestSlot(),
     acquireIntegrationRequestSlot: (opts) =>

package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts

@@ -45,15 +45,16 @@ export class InMemoryRateStateBackend implements RateStateBackend {
   async acquire(input: {
     bucketId: string;
     rules: readonly PacingRule[];
+    refreshRules?: () => readonly PacingRule[];
     signal?: AbortSignal;
   }): Promise<PacingPermit> {
     const { bucketId, rules, signal } = input;
     if (rules.length === 0) {
       return noopPacingPermit();
     }
-    // Stable order to avoid deadlock between concurrent acquirers on shared rules.
-    const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
-
+    const fixedOrdered = input.refreshRules
+      ? null
+      : [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
     while (true) {
       if (signal?.aborted) {
         throw signal.reason instanceof Error
@@ -61,6 +62,13 @@ export class InMemoryRateStateBackend implements RateStateBackend {
           : new Error('Rate-state acquire aborted.');
       }
       const decision = await this.withLock(() => {
+        // Re-read the existing adaptive policy after every wait, under the
+        // same admission lock as the debit. Declared rules have no refresher.
+        const ordered =
+          fixedOrdered ??
+          [...input.refreshRules!()].sort((a, b) =>
+            a.ruleId.localeCompare(b.ruleId),
+          );
         const now = this.now();
         let waitMs = 0;
 
@@ -92,10 +100,13 @@ export class InMemoryRateStateBackend implements RateStateBackend {
           state.startedInWindow += 1;
           if (rule.maxConcurrency != null) state.activeCount += 1;
         }
-        return { acquired: true, waitMs: 0 } as const;
+        return { acquired: true, waitMs: 0, rules: ordered } as const;
       });
 
       if (decision.acquired) {
+        // Release/cancel must use the rules actually debited at this grant,
+        // even if the learned policy changes again while the call is resident.
+        const ordered = decision.rules;
         let settled = false;
         const releaseConcurrency = async (refundRateReservation: boolean) => {
           await this.withLock(() => {

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -86,6 +86,8 @@ export const DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS = 64;
 // remain guarded independently by `integrationRequests` below.
 export const MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS = 1_000;
 export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
+/** Automatic rows keep a conservative window when the heap cannot be read. */
+export const DEFAULT_AUTOMATIC_UNMEASURED_ROW_CONCURRENCY = 20;
 // Dispatch groups are only a scheduler envelope. The external-call semaphore
 // remains the resource bound, so do not impose a lower hidden ceiling here.
 // This lets an explicit run-level call limit describe the actual concurrency
@@ -144,19 +146,16 @@ export function resolveMaxConcurrentRows(
  */
 export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
   concurrency: {
-    // Bound the complete live row promise graph, not only outbound calls.
-    // Cached provider payloads can expand far beyond their encoded receipt size
-    // while Node parses and delivers result lists. Fifty I/O rows overlap slow
-    // providers; heap, resident-memory, event-loop and byte admission still
-    // tighten this ceiling for larger response shapes. Pending rows remain
-    // compact indexes. Pure-compute maps use a separate fast path.
-    rowDefault: 50,
-    // Internal callers can request a wider window, but the default remains the
-    // safe standard-sandbox resident set. The input-byte estimator may reduce
-    // either value further for large source rows.
-    rowMax: 1_000,
-    // Global logical-call backstop. The default is intentionally close to the
-    // standard row cohort so slow small responses do not serialize most rows.
+    // The default is an adaptive ceiling, not an opening wave. The map's
+    // existing byte, heap, RSS and event-loop admission chooses the live width;
+    // an unmeasured automatic map uses the named conservative fallback above.
+    // Provider residence and provider pacing have their own unchanged limits.
+    rowDefault: MAX_CONFIGURABLE_CONCURRENT_ROWS,
+    // Explicit customer settings remain ceilings and are never widened.
+    rowMax: MAX_CONFIGURABLE_CONCURRENT_ROWS,
+    // Logical external-call backstop, independent of the adaptive row window.
+    // Rows waiting for this capacity still count toward memory admission.
+
     // Runs may request a value through the validated 1..1000 launch contract;
     // provider pacing and the runner-owned hard ceiling still apply.
     toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
@@ -247,6 +246,15 @@ export function resolveExecutionPolicy(
   };
 }
 
+/** The canonical authored-width test; other values use automatic admission. */
+export function hasExplicitRowConcurrency(
+  requested: unknown,
+): requested is number {
+  return (
+    typeof requested === 'number' && Number.isFinite(requested) && requested > 0
+  );
+}
+
 /**
  * Effective row concurrency for a map: an explicit positive request clamped to
  * `[1, rowMax]`, otherwise the default. Single helper, used by both substrates.
@@ -257,11 +265,7 @@ export function resolveRowConcurrency(
   runMaximum?: number | null,
 ): number {
   const normalizedRunMaximum = resolveMaxConcurrentRows(runMaximum);
-  if (
-    typeof requested === 'number' &&
-    Number.isFinite(requested) &&
-    requested > 0
-  ) {
+  if (hasExplicitRowConcurrency(requested)) {
     return Math.min(
       Math.floor(requested),
       normalizedRunMaximum ?? policy.concurrency.rowMax,

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -147,6 +147,11 @@ export interface RateStateBackend {
     bucketId: string;
     rateScopeToken?: string | null;
     rules: readonly PacingRule[];
+    /**
+     * Local undeclared-provider policy, refreshed at the admission decision.
+     * Absent for declared quotas and DB-authoritative pacing. Never serialized.
+     */
+    refreshRules?: () => readonly PacingRule[];
     signal?: AbortSignal;
     /** Managed Plays hand a shared provider hold to durable row suspension. */
     yieldProviderHolds?: boolean;

package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts

@@ -73,6 +73,8 @@ export const PLAY_LAUNCH_ENVELOPE_FIELD_ROLES: {
   runLedgerExecutorToken: 'detail',
   runtimeAuthority: 'detail',
   runtimeApiBaseUrl: 'detail',
+  arrayInputTerminalRetirementSupported: 'detail',
+  arrayInputTerminalProjectionOrigin: 'detail',
   runtimeTokenOrigin: 'detail',
   baseUrl: 'detail',
   coordinatorUrl: 'detail',