deepline 0.3.63 → 0.3.65

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -16,10 +16,17 @@ export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_MAX_ATTEMPTS =
   TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS.length + 1;
 export const TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS = 8;
 export const TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS = 2;
+// A Deepline async-capacity denial means the provider has not been called.
+// Treat it as queueing, not a failed physical provider attempt. The enclosing
+// tool deadline remains the hard bound; this prevents a normal one-minute
+// queue wait from being compressed into eight five-second retries.
+export const TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_MAX_ATTEMPTS = 65;
 export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
 export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
+export const TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS = 30_000;
+export const TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_DELAY_MAX_MS = 5 * 60_000;
 export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
 export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
 export const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE =
@@ -50,6 +57,7 @@ export type ToolExecuteHttpRetryDecision = {
   attemptCap: number;
   reason:
     | 'rate_limit'
+    | 'concurrency_backpressure'
     | 'idempotency_in_progress'
     | 'gateway_invocation_in_progress'
     | 'customer_db_storage_unavailable'
@@ -131,6 +139,19 @@ function isCustomerDbStorageUnavailableResponse(input: {
   );
 }
 
+function isConcurrencyBackpressureResponse(input: {
+  status: number;
+  bodyText: string;
+}): boolean {
+  if (input.status !== 429) return false;
+  const body = parseJsonObject(input.bodyText);
+  return (
+    body?.code === 'RATE_LIMIT' &&
+    body.rate_limit_kind === 'concurrency' &&
+    body.failure_origin === 'deepline_rate_limit'
+  );
+}
+
 function idempotencyInProgressRetryDelayMs(attempt: number): number {
   return (
     TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -164,6 +185,7 @@ function decideToolExecuteHttpRetry(input: {
   idempotencyInProgress?: boolean;
   gatewayInvocationInProgress?: boolean;
   customerDbStorageUnavailable?: boolean;
+  concurrencyBackpressure?: boolean;
   hardBillingFailure?: boolean;
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
@@ -177,6 +199,13 @@ function decideToolExecuteHttpRetry(input: {
     };
   }
   if (input.status === 429) {
+    if (input.concurrencyBackpressure) {
+      return {
+        retryable: true,
+        attemptCap: TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_MAX_ATTEMPTS,
+        reason: 'concurrency_backpressure',
+      };
+    }
     if (!input.hasRetryAfterHeader) {
       return {
         retryable: true,
@@ -241,6 +270,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
     number
   > = {
     rate_limit: 0,
+    concurrency_backpressure: 0,
     idempotency_in_progress: 0,
     gateway_invocation_in_progress: 0,
     customer_db_storage_unavailable: 0,
@@ -267,6 +297,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
           status: input.status,
           bodyText: input.bodyText ?? '',
         }),
+        concurrencyBackpressure: isConcurrencyBackpressureResponse({
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
         hasRetryAfterHeader: true,
         transientHttpRetrySafe: input.transientHttpRetrySafe === true,
       });
@@ -312,11 +346,13 @@ export function classifyToolExecuteHttpFailure(input: {
     isGatewayInvocationInProgressResponse(input);
   const customerDbStorageUnavailable =
     isCustomerDbStorageUnavailableResponse(input);
+  const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
   const initialRetryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
     gatewayInvocationInProgress,
     customerDbStorageUnavailable,
+    concurrencyBackpressure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
   });
@@ -348,6 +384,7 @@ export function classifyToolExecuteHttpFailure(input: {
     idempotencyInProgress,
     gatewayInvocationInProgress,
     customerDbStorageUnavailable,
+    concurrencyBackpressure,
     hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
@@ -366,23 +403,31 @@ export function classifyToolExecuteHttpFailure(input: {
     error.receiptFailureKind = 'repairable';
   }
   const retryDelayMs =
-    input.status === 429
+    retryDecision.reason === 'concurrency_backpressure'
       ? Math.min(
-          TOOL_EXECUTE_RETRY_DELAY_MAX_MS,
+          TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_DELAY_MAX_MS,
           Math.max(
             retryAfterMs,
             TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
           ),
         )
-      : retryDecision.reason === 'idempotency_in_progress' &&
-          !hasRetryAfterHeader
-        ? idempotencyInProgressRetryDelayMs(input.attempt)
-        : retryDecision.reason === 'gateway_invocation_in_progress' &&
+      : input.status === 429
+        ? Math.min(
+            TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
+            Math.max(
+              retryAfterMs,
+              TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
+            ),
+          )
+        : retryDecision.reason === 'idempotency_in_progress' &&
             !hasRetryAfterHeader
           ? idempotencyInProgressRetryDelayMs(input.attempt)
-          : retryAfterMs > 0
-            ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
-            : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+          : retryDecision.reason === 'gateway_invocation_in_progress' &&
+              !hasRetryAfterHeader
+            ? idempotencyInProgressRetryDelayMs(input.attempt)
+            : retryAfterMs > 0
+              ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
+              : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
   return {
     ...retryDecision,
     error,
@@ -401,6 +446,8 @@ export function classifyToolExecuteHttpFailure(input: {
           : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
         : null,
     chargeRetryBudget:
-      shouldRetry && retryDecision.reason !== 'gateway_invocation_in_progress',
+      shouldRetry &&
+      retryDecision.reason !== 'gateway_invocation_in_progress' &&
+      retryDecision.reason !== 'concurrency_backpressure',
   };
 }