deepline 0.3.443 → 0.3.445

package/dist/bundling-sources/sdk/src/release.ts

@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.443',
+  version: '0.3.445',
   updateSummary: {
     introducedIn: '0.3.90',
     summary:

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -1102,6 +1102,27 @@ function appRuntimeRetryDelayMs(
   return appRuntimeRetryDelaysForAction(action)[attempt - 1] ?? 1_000;
 }
 
+const APP_RUNTIME_RETRY_AFTER_MIN_SPREAD_MS = 1_000;
+
+/**
+ * Every caller shed at once gets the same retry-after. Spreading above it,
+ * instead of taking max() with a smaller jittered step, keeps them from
+ * retrying in lockstep.
+ */
+function appRuntimeRetrySleepMs(
+  ladderDelayMs: number,
+  retryAfterMs: number | null | undefined,
+): number {
+  const jitteredMs = applyRetryJitter(ladderDelayMs);
+  const floorMs = retryAfterMs ?? 0;
+  if (floorMs <= jitteredMs) return jitteredMs;
+  const spreadMs = Math.min(
+    floorMs,
+    Math.max(ladderDelayMs, APP_RUNTIME_RETRY_AFTER_MIN_SPREAD_MS),
+  );
+  return floorMs + Math.round(Math.random() * spreadMs);
+}
+
 function appRuntimeRetryAfterMs(
   response: Response,
   body: string,
@@ -1292,9 +1313,9 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
       error: input.error,
     })
   ) {
-    const retryDelayMs = Math.max(
-      applyRetryJitter(appRuntimeRetryDelayMs(input.action, input.attempt)),
-      input.retryAfterMs ?? 0,
+    const retryDelayMs = appRuntimeRetrySleepMs(
+      appRuntimeRetryDelayMs(input.action, input.attempt),
+      input.retryAfterMs,
     );
     recordAppRuntimeRetryFailure({
       telemetry: input.telemetry,
@@ -1860,9 +1881,9 @@ async function postAppRuntimeApi<TResponse>(
         contentType: response.headers.get('content-type'),
       })
     ) {
-      const retryDelayMs = Math.max(
-        applyRetryJitter(appRuntimeRetryDelayMs(body.action, attempt)),
-        appRuntimeRetryAfterMs(response, responseText) ?? 0,
+      const retryDelayMs = appRuntimeRetrySleepMs(
+        appRuntimeRetryDelayMs(body.action, attempt),
+        appRuntimeRetryAfterMs(response, responseText),
       );
       recordAppRuntimeRetryFailure({
         telemetry: retryTelemetry,

package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts

@@ -1,5 +1,9 @@
 export type GatewayPostgresLane = 'ordinary' | 'commit';
 
+export type GatewayPostgresAdmissionTenant = string | null;
+export const GATEWAY_POSTGRES_MACHINE_TENANT: GatewayPostgresAdmissionTenant =
+  null;
+
 export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
 export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
 export const GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS = 8;
@@ -16,8 +20,10 @@ export const GATEWAY_POSTGRES_MAX_WAITING_PER_LANE = 256;
 export type GatewayPostgresAdmissionSnapshot = {
   active: Record<GatewayPostgresLane, number> & { total: number };
   waiting: Record<GatewayPostgresLane, number>;
+  waitingTenants: Record<GatewayPostgresLane, number>;
   cancelled: number;
   overloaded: number;
+  fairShareEvictions: number;
   commitPriorityGrants: number;
 };
 
@@ -30,7 +36,7 @@ export class GatewayPostgresAdmissionCancelledError extends Error {
 
 /**
  * Load shed at the admission boundary. It is only ever thrown before a waiter
- * is enqueued, so no pooled client, statement, or provider call exists for the
+ * is granted, so no pooled client, statement, or provider call exists for the
  * rejected request. Callers must map it to a retryable back-pressure response.
  */
 export class GatewayPostgresAdmissionOverloadedError extends Error {
@@ -38,24 +44,31 @@ export class GatewayPostgresAdmissionOverloadedError extends Error {
   readonly lane: GatewayPostgresLane;
   readonly waiting: number;
   readonly limit: number;
+  readonly reason: 'queue_full' | 'fair_share_evicted';
 
   constructor(input: {
     lane: GatewayPostgresLane;
     waiting: number;
     limit: number;
+    reason?: 'queue_full' | 'fair_share_evicted';
   }) {
+    const reason = input.reason ?? 'queue_full';
     super(
-      `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}).`,
+      reason === 'queue_full'
+        ? `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}).`
+        : `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}); this waiter yielded its place to an org with a shorter queue.`,
     );
     this.name = 'GatewayPostgresAdmissionOverloadedError';
     this.lane = input.lane;
     this.waiting = input.waiting;
     this.limit = input.limit;
+    this.reason = reason;
   }
 }
 
 type Waiter = {
   lane: GatewayPostgresLane;
+  tenant: GatewayPostgresAdmissionTenant;
   signal: AbortSignal | null;
   onAbort: (() => void) | null;
   resolve: (release: () => void) => void;
@@ -63,15 +76,32 @@ type Waiter = {
   settled: boolean;
 };
 
+type LaneQueue = {
+  ring: Map<GatewayPostgresAdmissionTenant, Waiter[]>;
+  size: number;
+};
+
+/**
+ * Fair across orgs within each lane, so one org's burst cannot shed or starve
+ * every other org on this Machine.
+ */
 export class GatewayPostgresAdmission {
-  readonly #waiting: Record<GatewayPostgresLane, Waiter[]> = {
-    ordinary: [],
-    commit: [],
+  readonly #waiting: Record<GatewayPostgresLane, LaneQueue> = {
+    ordinary: { ring: new Map(), size: 0 },
+    commit: { ring: new Map(), size: 0 },
+  };
+  readonly #activeByTenant: Record<
+    GatewayPostgresLane,
+    Map<GatewayPostgresAdmissionTenant, number>
+  > = {
+    ordinary: new Map(),
+    commit: new Map(),
   };
   #ordinaryActive = 0;
   #commitActive = 0;
   #cancelled = 0;
   #overloaded = 0;
+  #fairShareEvictions = 0;
   #commitPriorityGrants = 0;
   #consecutiveCommitPriorityGrants = 0;
 
@@ -95,29 +125,27 @@ export class GatewayPostgresAdmission {
 
   async acquire(
     lane: GatewayPostgresLane,
-    options: { signal?: AbortSignal } = {},
+    options: {
+      signal?: AbortSignal;
+      tenant?: GatewayPostgresAdmissionTenant;
+    } = {},
   ): Promise<() => void> {
+    const tenant = options.tenant ?? GATEWAY_POSTGRES_MACHINE_TENANT;
     if (options.signal?.aborted) {
       this.#cancelled += 1;
       throw new GatewayPostgresAdmissionCancelledError();
     }
-    if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
-      return this.#grant(lane);
+    if (this.#canGrant(lane) && this.#waiting.commit.size === 0) {
+      return this.#grant(lane, tenant);
     }
-    const waiting = this.#waiting[lane].length;
-    const limit = this.#maxWaitingPerLane();
-    if (waiting >= limit) {
-      this.#overloaded += 1;
-      throw new GatewayPostgresAdmissionOverloadedError({
-        lane,
-        waiting,
-        limit,
-      });
+    if (this.#waiting[lane].size >= this.#maxWaitingPerLane()) {
+      this.#shedForFullLane(lane, tenant);
     }
 
     return await new Promise<() => void>((resolve, reject) => {
       const waiter: Waiter = {
         lane,
+        tenant,
         signal: options.signal ?? null,
         onAbort: null,
         resolve,
@@ -130,7 +158,7 @@ export class GatewayPostgresAdmission {
         reject(new GatewayPostgresAdmissionCancelledError());
       };
       waiter.signal?.addEventListener('abort', waiter.onAbort, { once: true });
-      this.#waiting[lane].push(waiter);
+      this.#enqueue(waiter);
       if (waiter.signal?.aborted) {
         waiter.onAbort();
         return;
@@ -147,11 +175,16 @@ export class GatewayPostgresAdmission {
         total: this.#activeTotal(),
       },
       waiting: {
-        ordinary: this.#waiting.ordinary.length,
-        commit: this.#waiting.commit.length,
+        ordinary: this.#waiting.ordinary.size,
+        commit: this.#waiting.commit.size,
+      },
+      waitingTenants: {
+        ordinary: this.#waiting.ordinary.ring.size,
+        commit: this.#waiting.commit.ring.size,
       },
       cancelled: this.#cancelled,
       overloaded: this.#overloaded,
+      fairShareEvictions: this.#fairShareEvictions,
       commitPriorityGrants: this.#commitPriorityGrants,
     };
   }
@@ -183,15 +216,23 @@ export class GatewayPostgresAdmission {
     );
   }
 
-  #grant(lane: GatewayPostgresLane): () => void {
+  #grant(
+    lane: GatewayPostgresLane,
+    tenant: GatewayPostgresAdmissionTenant,
+  ): () => void {
     if (lane === 'ordinary') this.#ordinaryActive += 1;
     else this.#commitActive += 1;
+    const activeByTenant = this.#activeByTenant[lane];
+    activeByTenant.set(tenant, (activeByTenant.get(tenant) ?? 0) + 1);
     let released = false;
     return () => {
       if (released) return;
       released = true;
       if (lane === 'ordinary') this.#ordinaryActive -= 1;
       else this.#commitActive -= 1;
+      const remaining = (activeByTenant.get(tenant) ?? 1) - 1;
+      if (remaining > 0) activeByTenant.set(tenant, remaining);
+      else activeByTenant.delete(tenant);
       this.#drain();
     };
   }
@@ -199,19 +240,19 @@ export class GatewayPostgresAdmission {
   #drain(): void {
     while (this.#activeTotal() < this.#maxTotalActive()) {
       const ordinaryCanRun =
-        this.#waiting.ordinary.length > 0 &&
+        this.#waiting.ordinary.size > 0 &&
         this.#ordinaryActive < this.#maxOrdinaryActive();
       if (
         ordinaryCanRun &&
-        this.#waiting.commit.length > 0 &&
+        this.#waiting.commit.size > 0 &&
         this.#consecutiveCommitPriorityGrants >=
           GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS
       ) {
         this.#consecutiveCommitPriorityGrants = 0;
-        this.#resolve(this.#waiting.ordinary.shift()!);
+        this.#resolve(this.#dequeueLeastActiveTenant('ordinary')!);
         continue;
       }
-      const commit = this.#waiting.commit.shift();
+      const commit = this.#dequeueLeastActiveTenant('commit');
       if (commit) {
         if (ordinaryCanRun) {
           this.#commitPriorityGrants += 1;
@@ -224,29 +265,110 @@ export class GatewayPostgresAdmission {
       }
       if (ordinaryCanRun) {
         this.#consecutiveCommitPriorityGrants = 0;
-        this.#resolve(this.#waiting.ordinary.shift()!);
+        this.#resolve(this.#dequeueLeastActiveTenant('ordinary')!);
         continue;
       }
-      if (this.#waiting.ordinary.length === 0) {
+      if (this.#waiting.ordinary.size === 0) {
         this.#consecutiveCommitPriorityGrants = 0;
       }
       return;
     }
   }
 
+  #enqueue(waiter: Waiter): void {
+    const queue = this.#waiting[waiter.lane];
+    const tenantQueue = queue.ring.get(waiter.tenant);
+    if (tenantQueue) tenantQueue.push(waiter);
+    else queue.ring.set(waiter.tenant, [waiter]);
+    queue.size += 1;
+  }
+
+  #shedForFullLane(
+    lane: GatewayPostgresLane,
+    tenant: GatewayPostgresAdmissionTenant,
+  ): void {
+    const queue = this.#waiting[lane];
+    const waiting = queue.size;
+    const limit = this.#maxWaitingPerLane();
+    const longest = this.#longestTenantQueue(queue);
+    const own = queue.ring.get(tenant)?.length ?? 0;
+    this.#overloaded += 1;
+    if (!longest || own >= longest.length) {
+      throw new GatewayPostgresAdmissionOverloadedError({
+        lane,
+        waiting,
+        limit,
+      });
+    }
+    const evicted = longest[longest.length - 1]!;
+    this.#remove(evicted);
+    this.#fairShareEvictions += 1;
+    evicted.reject(
+      new GatewayPostgresAdmissionOverloadedError({
+        lane,
+        waiting,
+        limit,
+        reason: 'fair_share_evicted',
+      }),
+    );
+  }
+
+  #dequeueLeastActiveTenant(lane: GatewayPostgresLane): Waiter | undefined {
+    const queue = this.#waiting[lane];
+    const activeByTenant = this.#activeByTenant[lane];
+    let chosen: GatewayPostgresAdmissionTenant | undefined;
+    let chosenActive = Infinity;
+    for (const tenant of queue.ring.keys()) {
+      const active = activeByTenant.get(tenant) ?? 0;
+      if (active < chosenActive) {
+        chosen = tenant;
+        chosenActive = active;
+        if (active === 0) break;
+      }
+    }
+    if (chosen === undefined) return undefined;
+    const tenantQueue = queue.ring.get(chosen)!;
+    const waiter = tenantQueue.shift()!;
+    queue.size -= 1;
+    this.#rotateToBack(queue, chosen, tenantQueue);
+    return waiter;
+  }
+
+  #rotateToBack(
+    queue: LaneQueue,
+    tenant: GatewayPostgresAdmissionTenant,
+    tenantQueue: Waiter[],
+  ): void {
+    queue.ring.delete(tenant);
+    if (tenantQueue.length > 0) queue.ring.set(tenant, tenantQueue);
+  }
+
+  #longestTenantQueue(queue: LaneQueue): Waiter[] | null {
+    let longest: Waiter[] | null = null;
+    for (const tenantQueue of queue.ring.values()) {
+      if (!longest || tenantQueue.length > longest.length) {
+        longest = tenantQueue;
+      }
+    }
+    return longest;
+  }
+
   #resolve(waiter: Waiter): void {
     waiter.settled = true;
     this.#cleanup(waiter);
-    waiter.resolve(this.#grant(waiter.lane));
+    waiter.resolve(this.#grant(waiter.lane, waiter.tenant));
   }
 
   #remove(waiter: Waiter): boolean {
     if (waiter.settled) return false;
     const queue = this.#waiting[waiter.lane];
-    const index = queue.indexOf(waiter);
-    if (index < 0) return false;
+    const tenantQueue = queue.ring.get(waiter.tenant);
+    const index = tenantQueue ? tenantQueue.indexOf(waiter) : -1;
+    if (!tenantQueue || index < 0) return false;
     waiter.settled = true;
-    queue.splice(index, 1);
+    tenantQueue.splice(index, 1);
+    if (tenantQueue.length === 0) queue.ring.delete(waiter.tenant);
+    queue.size -= 1;
     this.#cleanup(waiter);
     return true;
   }

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -1,5 +1,7 @@
 import {
   acquireRateStateViaAppRuntime,
+  AppRuntimeApiResponseError,
+  isAppRuntimeApiCapacityError,
   penalizeRateStateViaAppRuntime,
   releaseRateStateViaAppRuntime,
   RateStateValidationUnavailableError,
@@ -141,6 +143,20 @@ function sleepWithAbort(
  */
 const RATE_BLOCK_HORIZON_MS = 1_000;
 const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
+const CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS = 1_000;
+
+function capacityShedRetryAfterMs(error: unknown): number | null {
+  const shed =
+    isAppRuntimeApiCapacityError(error) ||
+    (error instanceof AppRuntimeApiResponseError &&
+      error.retryable &&
+      error.status === 429);
+  if (!shed) return null;
+  const retryAfterMs = (error as AppRuntimeApiResponseError).retryAfterMs;
+  return retryAfterMs && retryAfterMs > 0
+    ? retryAfterMs
+    : CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS;
+}
 /**
  * Jitter fraction applied to a server-supplied `waitMs` so a synchronized wave
  * of buckets that all hit granted=0 at the same instant re-probe at slightly
@@ -1140,6 +1156,22 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
         // A provider hold is not a store failure. Preserve its deadline for
         // the caller's durable suspension or non-durable error path.
         if (error instanceof ProviderExhaustedError) throw error;
+        // A capacity shed is back-pressure from a working store. Wait as for
+        // a zero grant, then return so the acquirer re-checks its wait budget.
+        const shedRetryAfterMs = capacityShedRetryAfterMs(error);
+        if (shedRetryAfterMs !== null) {
+          this.onStoreFailure({
+            bucketId,
+            error: normalizeError(error).message,
+          });
+          await this.sleep(
+            Math.min(
+              shedRetryAfterMs + Math.round(this.jitter() * shedRetryAfterMs),
+              MAX_ACQUIRE_SLEEP_MS,
+            ),
+          );
+          return;
+        }
         // Fail closed. A degraded rate-state store must not grant unlimited
         // provider calls; after this bounded retry ladder the run fails loudly.
         consecutiveFailures += 1;

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -14,8 +14,15 @@ import {
   RuntimeSandboxProvider,
 } from './runtime-sandbox-operation';
 
-/** The safe baseline when an organization has no current override. */
-export const DEFAULT_PLAY_SANDBOX_CONCURRENCY = 4;
+/**
+ * The baseline when an organization has no current override. It sits above
+ * every ordinary organization's observed peak (at most 46 live sandboxes in the
+ * week before 2026-10-29) and bounds a burst: on 2026-10-29 one organization
+ * held 2,838 live sandboxes polling the receipt gateway. Keep in sync with the
+ * Convex control plane's DEFAULT_PLAY_SANDBOX_CONCURRENCY in
+ * packages/convex/auth.ts.
+ */
+export const DEFAULT_PLAY_SANDBOX_CONCURRENCY = 50;
 export const MAX_PLAY_SANDBOX_CONCURRENCY = 250;
 /** A crashed worker cannot retain capacity forever. */
 export const RUNTIME_SANDBOX_CAPACITY_LEASE_MS = 50 * 60 * 1_000;

package/dist/cli/{auth-CP2CAI7J.mjs → auth-QNWYRLML.mjs}

@@ -13,13 +13,13 @@ import {
   reportSetupCompleted,
   resolvePendingAuthClaimForResume,
   statusBannerLine
-} from "./chunk-MZLZNISS.mjs";
-import "./chunk-6NUGMJQA.mjs";
-import "./chunk-4A45A2MH.mjs";
+} from "./chunk-C54S4QPU.mjs";
+import "./chunk-3OBHZADJ.mjs";
+import "./chunk-ZRWX23RQ.mjs";
 import "./chunk-4QF2K7TM.mjs";
 import "./chunk-DLIRSHV6.mjs";
 import "./chunk-HBL4PM5Q.mjs";
-import "./chunk-DOKTNYMQ.mjs";
+import "./chunk-6LVOC2HD.mjs";
 import "./chunk-WZYKRD7T.mjs";
 import "./chunk-KRD6H754.mjs";
 export {

package/dist/cli/auth-ZA4FRBSX.js

@@ -0,0 +1,40 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+var _chunkFC7LS3Y5js = require('./chunk-FC7LS3Y5.js');
+require('./chunk-5JJ6JYTL.js');
+require('./chunk-TABMEHIP.js');
+require('./chunk-ALJJRZNJ.js');
+require('./chunk-XNYIZMNQ.js');
+require('./chunk-DIAWYWT5.js');
+require('./chunk-EI2NN6HP.js');
+require('./chunk-OG64RBAC.js');
+require('./chunk-COGTRVWW.js');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+exports.EXIT_WORKSPACE_MISMATCH = _chunkFC7LS3Y5js.EXIT_WORKSPACE_MISMATCH; exports.authStatusLine = _chunkFC7LS3Y5js.authStatusLine; exports.clearPendingClaim = _chunkFC7LS3Y5js.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkFC7LS3Y5js.clearSatisfiedPendingClaim; exports.handleRegister = _chunkFC7LS3Y5js.handleRegister; exports.handleStatus = _chunkFC7LS3Y5js.handleStatus; exports.handleWait = _chunkFC7LS3Y5js.handleWait; exports.httpJson = _chunkFC7LS3Y5js.httpJson; exports.readPendingAuthClaim = _chunkFC7LS3Y5js.readPendingAuthClaim; exports.readStatusBanner = _chunkFC7LS3Y5js.readStatusBanner; exports.registerAuthCommands = _chunkFC7LS3Y5js.registerAuthCommands; exports.reportSetupCompleted = _chunkFC7LS3Y5js.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkFC7LS3Y5js.resolvePendingAuthClaimForResume; exports.statusBannerLine = _chunkFC7LS3Y5js.statusBannerLine;