deepline 0.1.287 → 0.1.288

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.287',
+  version: '0.1.288',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -309,6 +309,10 @@ const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
 const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
 const TOOL_SCALAR_COALESCE_WINDOW_MS = 5;
 const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
+// The receipt gateway fully buffers the integration response before sending
+// headers to the runner. Once those headers arrive, a long body stall is a
+// broken gateway-to-runner connection, not slow provider work.
+export const TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS = 30_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // Every runtime-API tool fetch needs a client-side deadline. Without one, a
@@ -352,6 +356,35 @@ class ToolExecuteResponseBodyTransportError extends Error {
   }
 }
 
+class ToolExecuteResponseBodyTimeoutError extends Error {
+  constructor(toolId: string) {
+    super(
+      `Tool ${toolId} response body was not delivered within ${TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS}ms after response headers.`,
+    );
+    this.name = 'ToolExecuteResponseBodyTimeoutError';
+  }
+}
+
+async function readToolExecuteResponseBody<T>(input: {
+  toolId: string;
+  abortController: AbortController | null;
+  read: () => Promise<T>;
+}): Promise<T> {
+  let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timeoutHandle = setTimeout(() => {
+      const error = new ToolExecuteResponseBodyTimeoutError(input.toolId);
+      input.abortController?.abort(error);
+      reject(error);
+    }, TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS);
+  });
+  try {
+    return await Promise.race([input.read(), timeout]);
+  } finally {
+    if (timeoutHandle) clearTimeout(timeoutHandle);
+  }
+}
+
 class ToolExecuteInvalidJsonError extends Error {
   readonly cause: unknown;
 
@@ -1289,6 +1322,7 @@ export class PlayContextImpl {
     createSecretRedactionContext();
   private mapInvocationIndex = 0;
   private readonly stepCallIndexByKey = new Map<string, number>();
+  private readonly toolCallIndexByKey = new Map<string, number>();
   /**
    * Parent-level inline-child aggregates. Maintained only by the single-writer
    * progress path (never per child) so concurrent fan-out cannot contend. See
@@ -3281,7 +3315,9 @@ export class PlayContextImpl {
     const executionNonce =
       typeof wrapped.job_id === 'string' && wrapped.job_id.trim()
         ? wrapped.job_id.trim()
-        : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+        : stableDigest(
+            `${this.currentRunId}:${toolId}:${stableStringify(originalRequestInput)}:${datasetLimit}`,
+          );
     const datasetId = `tool-list:${sha256Hex(
       `${toolId}:${this.currentRunId}:${sql}:${datasetLimit}:${executionNonce}`,
     )}`;
@@ -3294,6 +3330,10 @@ export class PlayContextImpl {
         toolId,
         originalRequestInput,
         {
+          durableCallReceiptKey: `${buildDurableToolReceiptPrefix({
+            orgId: this.#options.orgId ?? 'unknown-org',
+            toolId,
+          })}${stableDigest(`${datasetId}:${offset}:${limit}`)}`,
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId),
           customerDbDataset: {
             limit: datasetLimit,
@@ -6461,6 +6501,14 @@ export class PlayContextImpl {
       toolId,
       requestInput: input,
     });
+    const store = rowContext.getStore();
+    let logicalCallId: string | null = null;
+    if (!store) {
+      const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:workflow:${normalizedKey}:${toolRequestIdentity}`;
+      const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
+      this.toolCallIndexByKey.set(callIndexKey, callIndex + 1);
+      logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`);
+    }
     let executionAuthScopeDigest =
       (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
     const eventWaitHandler =
@@ -6476,7 +6524,6 @@ export class PlayContextImpl {
       staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
     });
     const checkpointCacheKeys = [durableCacheKey];
-    const store = rowContext.getStore();
 
     const executeTool = async (context?: {
       leaseId?: string | null;
@@ -6531,18 +6578,28 @@ export class PlayContextImpl {
             : `Calling tool: ${toolId}`,
         );
         const directReceiptLeaseId = context?.leaseId?.trim() || null;
+        if (!logicalCallId) {
+          throw new Error(
+            'Direct tool execution requires a stable logical call identity.',
+          );
+        }
+        const physicalDirectKey = cacheableToolResult
+          ? directCacheKey
+          : `${buildDurableToolReceiptPrefix({
+              orgId: this.#options.orgId ?? 'unknown-org',
+              toolId,
+            })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
         const execution = await this.callToolExecutionAPI(toolId, input, {
-          ...(cacheableToolResult
-            ? {
-                durableCallReceiptKey: directCacheKey,
-                executionAuthScopeDigest,
-                providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({
-                  cacheKey: directCacheKey,
-                  force: toolCachePolicy.force,
-                  leaseId: directReceiptLeaseId,
-                }),
-              }
-            : {}),
+          durableCallReceiptKey: physicalDirectKey,
+          executionAuthScopeDigest,
+          providerIdempotencyKey: cacheableToolResult
+            ? this.providerIdempotencyKeyForToolCall({
+                cacheKey: directCacheKey,
+                force: toolCachePolicy.force,
+                leaseId: directReceiptLeaseId,
+                logicalCallId,
+              })
+            : physicalDirectKey,
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
           ...(directReceiptLeaseId &&
           (this.#options.markRuntimeStepReceiptRunning ||
@@ -6692,7 +6749,7 @@ export class PlayContextImpl {
     ) {
       try {
         const toolRetryPolicy = await this.#options
-          .getToolRetryPolicy?.(toolId)
+          .getToolRetryPolicy?.(toolId, input)
           .catch(() => null);
         return await this.executeWithRuntimeReceipt(
           'tool',
@@ -8547,11 +8604,17 @@ export class PlayContextImpl {
     cacheKey: string;
     force?: boolean;
     leaseId?: string | null;
+    logicalCallId?: string;
   }): string {
-    const fallbackAttemptId =
-      input.force === true && !input.leaseId
-        ? `${this.currentReceiptOwnerRunId}:${crypto.randomUUID()}`
-        : null;
+    let fallbackAttemptId: string | null = null;
+    if (input.force === true && !input.leaseId) {
+      if (!input.logicalCallId) {
+        throw new Error(
+          'Forced tool execution without a receipt lease requires a stable logical call identity.',
+        );
+      }
+      fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
+    }
     return buildDurableToolProviderIdempotencyKey({
       receiptKey: input.cacheKey,
       force: input.force,
@@ -8570,7 +8633,13 @@ export class PlayContextImpl {
         'executorToken and baseUrl are required for tool API calls (cloud execution only)',
       );
     }
-    const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
+    const requestsDurableInvocationFence =
+      this.#options.requestDurableInvocationFence === true ||
+      this.#options.durableInvocationFence === true;
+    const executeSuffix = requestsDurableInvocationFence
+      ? 'execute-fenced-v1'
+      : 'execute';
+    const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/${executeSuffix}`;
     const timeoutMs = resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs);
     const provider = toolId.split(/[._]/)[0]?.trim() || 'provider';
     const activityId = `provider:${toolId}`;
@@ -8607,12 +8676,89 @@ export class PlayContextImpl {
         async (span) => {
           const httpFailureAttempts =
             createToolExecuteHttpFailureAttemptTracker();
+          // Snapshot the caller-controlled payload exactly once. The replay
+          // decision and every physical attempt must refer to the same bytes,
+          // even if the original object has getters or is later mutated.
+          const toolInputSnapshot = JSON.parse(JSON.stringify(input)) as Record<
+            string,
+            unknown
+          >;
           const retryPolicy = await this.#options
-            .getToolRetryPolicy?.(toolId)
+            .getToolRetryPolicy?.(toolId, toolInputSnapshot)
             .catch(() => null);
           const retrySafeTransientHttp =
             retryPolicy?.retrySafeTransientHttp === true;
+          const invocationOrgId = this.#options.orgId?.trim() ?? '';
+          const durableCallReceiptKey =
+            options?.durableCallReceiptKey?.trim() ||
+            (requestsDurableInvocationFence && invocationOrgId
+              ? `${buildDurableToolReceiptPrefix({
+                  orgId: invocationOrgId,
+                  toolId,
+                })}${stableDigest(
+                  `transport:${this.currentRunId}:${toolId}:${crypto.randomUUID()}`,
+                )}`
+              : null);
+          const executionAuthScopeDigest = options?.executionAuthScopeDigest
+            ? options.executionAuthScopeDigest.trim() || null
+            : ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
+          const providerIdempotencyKey =
+            options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
+          // Correlation identity is stable across every transport retry,
+          // including calls without a durable receipt.
+          const deeplineRequestId = providerIdempotencyKey
+            ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
+            : `ctx-tool-${crypto.randomUUID()}`;
+          const serializedRequestBody = (invocationAttempt: number) =>
+            JSON.stringify({
+              payload: toolInputSnapshot,
+              metadata: {
+                parent_run_id: this.#options.runId,
+                invocation_attempt: invocationAttempt,
+                ...(requestsDurableInvocationFence
+                  ? { invocation_fence_version: 1 }
+                  : {}),
+                ...(durableCallReceiptKey
+                  ? {
+                      durable_call_receipt_key: durableCallReceiptKey,
+                      ...(executionAuthScopeDigest
+                        ? {
+                            execution_auth_scope_digest:
+                              executionAuthScopeDigest,
+                          }
+                        : {}),
+                    }
+                  : {}),
+                ...(options?.customerDbDataset
+                  ? {
+                      query_result_dataset: {
+                        limit: options.customerDbDataset.limit,
+                        offset: options.customerDbDataset.offset,
+                        page_size: options.customerDbDataset.pageSize,
+                        total_rows: options.customerDbDataset.totalRows,
+                      },
+                      ...(isCustomerDbDatasetTool(toolId)
+                        ? {
+                            customer_db_dataset: {
+                              limit: options.customerDbDataset.limit,
+                              offset: options.customerDbDataset.offset,
+                              page_size: options.customerDbDataset.pageSize,
+                              total_rows: options.customerDbDataset.totalRows,
+                            },
+                          }
+                        : {}),
+                    }
+                  : {}),
+                ...(providerIdempotencyKey
+                  ? { provider_idempotency_key: providerIdempotencyKey }
+                  : {}),
+              },
+              ...(this.#options.integrationMode
+                ? { integration_mode: this.#options.integrationMode }
+                : {}),
+            });
           let transportAttempt = 0;
+          let invocationAttempt = 0;
           const retryToolTransportFailure = async (input: {
             error: unknown;
             elapsedMs: number;
@@ -8662,23 +8808,7 @@ export class PlayContextImpl {
             let responseErrorText: string | null = null;
             let providerCallStartedAt: number | null = null;
             let providerCallElapsedMs: number | null = null;
-            const durableCallReceiptKey =
-              options?.durableCallReceiptKey?.trim() || null;
-            const executionAuthScopeDigest =
-              durableCallReceiptKey && options?.executionAuthScopeDigest
-                ? options.executionAuthScopeDigest.trim() || null
-                : durableCallReceiptKey
-                  ? ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ??
-                    null)
-                  : null;
-            const providerIdempotencyKey =
-              options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
-            // Every gateway request needs a correlation ID. When the call has
-            // an idempotency key, keep its stable ID across retries; otherwise
-            // give this individual attempt a fresh ID for gateway pairing.
-            const deeplineRequestId = providerIdempotencyKey
-              ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
-              : `ctx-tool-${crypto.randomUUID()}`;
+            let fetchDispatched = false;
             // Receipt ownership is a liveness contract, not a one-time check.
             // Keep it alive for the whole provider HTTP request. The cadence
             // comes from the store-issued expiry because a remote runner may
@@ -8747,6 +8877,7 @@ export class PlayContextImpl {
                 const integrationFetchStartedAt = Date.now();
                 providerCallStartedAt = integrationFetchStartedAt;
                 try {
+                  fetchDispatched = true;
                   response = await fetch(url, {
                     method: 'POST',
                     signal: abortController?.signal,
@@ -8765,59 +8896,23 @@ export class PlayContextImpl {
                           }
                         : {}),
                       ...protectionHeaders,
-                    },
-                    body: JSON.stringify({
-                      payload: input,
-                      metadata: {
-                        parent_run_id: this.#options.runId,
-                        ...(durableCallReceiptKey
-                          ? {
-                              durable_call_receipt_key: durableCallReceiptKey,
-                              ...(executionAuthScopeDigest
-                                ? {
-                                    execution_auth_scope_digest:
-                                      executionAuthScopeDigest,
-                                  }
-                                : {}),
-                            }
-                          : {}),
-                        ...(options?.customerDbDataset
-                          ? {
-                              query_result_dataset: {
-                                limit: options.customerDbDataset.limit,
-                                offset: options.customerDbDataset.offset,
-                                page_size: options.customerDbDataset.pageSize,
-                                total_rows: options.customerDbDataset.totalRows,
-                              },
-                              ...(isCustomerDbDatasetTool(toolId)
-                                ? {
-                                    customer_db_dataset: {
-                                      limit: options.customerDbDataset.limit,
-                                      offset: options.customerDbDataset.offset,
-                                      page_size:
-                                        options.customerDbDataset.pageSize,
-                                      total_rows:
-                                        options.customerDbDataset.totalRows,
-                                    },
-                                  }
-                                : {}),
-                            }
-                          : {}),
-                        ...(providerIdempotencyKey
-                          ? { provider_idempotency_key: providerIdempotencyKey }
-                          : {}),
-                      },
-                      ...(this.#options.integrationMode
-                        ? { integration_mode: this.#options.integrationMode }
+                      ...(this.#options.runtimeTestFaultHeader
+                        ? {
+                            'x-deepline-test-fault':
+                              this.#options.runtimeTestFaultHeader,
+                          }
                         : {}),
-                    }),
+                    },
+                    body: serializedRequestBody(invocationAttempt),
                   });
                   if (response.ok) {
                     try {
-                      responseData = (await response.json()) as Record<
-                        string,
-                        unknown
-                      >;
+                      responseData = await readToolExecuteResponseBody({
+                        toolId,
+                        abortController,
+                        read: () =>
+                          response!.json() as Promise<Record<string, unknown>>,
+                      });
                     } catch (error) {
                       if (error instanceof SyntaxError) {
                         throw new ToolExecuteInvalidJsonError(error);
@@ -8826,7 +8921,11 @@ export class PlayContextImpl {
                     }
                   } else {
                     try {
-                      responseErrorText = await response.text();
+                      responseErrorText = await readToolExecuteResponseBody({
+                        toolId,
+                        abortController,
+                        read: () => response!.text(),
+                      });
                     } catch (error) {
                       throw new ToolExecuteResponseBodyTransportError(error);
                     }
@@ -8887,11 +8986,39 @@ export class PlayContextImpl {
                   402,
                 );
               }
-              const responseBodyReplaySafe =
-                !(error instanceof ToolExecuteResponseBodyTransportError) ||
-                retrySafeTransientHttp ||
-                response?.status === 429;
-              if (!responseBodyReplaySafe) {
+              const ambiguousDispatchedFailure =
+                fetchDispatched &&
+                (response === null ||
+                  error instanceof ToolExecuteResponseBodyTransportError);
+              const hasDurableInvocationIdentity = Boolean(
+                durableCallReceiptKey &&
+                providerIdempotencyKey &&
+                executionAuthScopeDigest,
+              );
+              // A post-header body deadline aborts the fetch controller solely
+              // to stop consuming that response. It must not also cancel the
+              // independent gateway health check that establishes whether the
+              // committed response can be replayed safely. Preserve genuine
+              // caller/runtime cancellation by dropping the signal only when
+              // this exact body deadline caused the abort.
+              const durableFenceVerificationSignal =
+                abortController?.signal.aborted === true &&
+                abortController.signal.reason instanceof
+                  ToolExecuteResponseBodyTimeoutError
+                  ? undefined
+                  : abortController?.signal;
+              const durableInvocationFenceVerified =
+                hasDurableInvocationIdentity &&
+                requestsDurableInvocationFence &&
+                (this.#options.durableInvocationFence === true ||
+                  (await this.#options
+                    .verifyDurableInvocationFence?.(
+                      durableFenceVerificationSignal,
+                    )
+                    .catch(() => false)) === true);
+              const transportReplaySafe =
+                !ambiguousDispatchedFailure || durableInvocationFenceVerified;
+              if (!transportReplaySafe) {
                 const diagnostic = describeTransportError(transportError);
                 this.log(
                   `[runtime.transport_failure] ${JSON.stringify({
@@ -8905,13 +9032,17 @@ export class PlayContextImpl {
                         : Date.now() - providerCallStartedAt,
                     request_id: deeplineRequestId,
                     aborted: abortController?.signal.aborted === true,
-                    response_headers_received: true,
+                    response_headers_received: response !== null,
                     retry_safe: false,
                     error: diagnostic,
                   })}`,
                 );
+                const failureBoundary =
+                  error instanceof ToolExecuteResponseBodyTransportError
+                    ? 'response body transport failed after response headers'
+                    : 'request transport failed after dispatch before response headers';
                 throw new ToolHttpError(
-                  `Tool ${toolId} response body transport failed after response headers; the ambiguous call was not retried because this operation is not declared retry-safe: ${diagnostic.message ?? 'unknown transport error'}`,
+                  `Tool ${toolId} ${failureBoundary}; the ambiguous call was not retried because this runtime has no durable invocation fence: ${diagnostic.message ?? 'unknown transport error'}`,
                   null,
                   0,
                   'repairable',
@@ -9004,6 +9135,9 @@ export class PlayContextImpl {
                 );
               }
               if (failure.shouldRetry) {
+                if (failure.reason !== 'gateway_invocation_in_progress') {
+                  invocationAttempt += 1;
+                }
                 if (failure.chargeRetryBudget) {
                   await this.governor.chargeBudget('retry');
                 }

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -512,6 +512,15 @@ export interface ContextOptions {
   /** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
   executorToken?: string;
   baseUrl?: string;
+  /**
+   * The integration base URL persists each keyed execute response before
+   * exposing it. Only runtime adapters that route through that gateway may
+   * enable ambiguous transport replay.
+   */
+  durableInvocationFence?: boolean;
+  requestDurableInvocationFence?: boolean;
+  verifyDurableInvocationFence?: (signal?: AbortSignal) => Promise<boolean>;
+  runtimeTestFaultHeader?: string | null;
   /**
    * Runtime-sheet transport selected by the runner. Daytona sandboxes use the
    * execution gateway and must never fall back to minting direct DB sessions.
@@ -644,7 +653,10 @@ export interface ContextOptions {
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolProvider?: (toolId: string) => Promise<string | null>;
-  getToolRetryPolicy?: (toolId: string) => Promise<{
+  getToolRetryPolicy?: (
+    toolId: string,
+    input: Record<string, unknown>,
+  ) => Promise<{
     retrySafeTransientHttp?: boolean;
     requiresExecutionFence?: boolean;
   } | null>;

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -6,6 +6,7 @@ export type RuntimeTestFaultName =
   | 'receipt_complete_write_fail'
   | 'receipt_fail_write_fail'
   | 'worker_receipt_complete_write_fail'
+  | 'invocation_response_delivery_abort'
   /**
    * Holds an already checked-out receipt-gateway scheduler client for the
    * supplied bounded millisecond value. Preview/CI only: this creates a
@@ -70,6 +71,7 @@ const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
   'receipt_complete_write_fail',
   'receipt_fail_write_fail',
   'worker_receipt_complete_write_fail',
+  'invocation_response_delivery_abort',
   'receipt_gateway_hold_ms',
 ]);
 

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -21,6 +21,8 @@ export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
 export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
 export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
+export const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE =
+  'CUSTOMER_DB_STORAGE_UNAVAILABLE';
 /**
  * A provider/action-local outcome contract emits this only when a provider has
  * explicitly said that the same idempotency key is still executing. It is not
@@ -28,6 +30,9 @@ export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
  */
 export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE =
   'UPSTREAM_IDEMPOTENCY_IN_PROGRESS';
+export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE =
+  'GATEWAY_INVOCATION_IN_PROGRESS';
+export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS = 65;
 
 export class ToolExecuteAuthScopeChangedError extends Error {
   readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
@@ -45,6 +50,8 @@ export type ToolExecuteHttpRetryDecision = {
   reason:
     | 'rate_limit'
     | 'idempotency_in_progress'
+    | 'gateway_invocation_in_progress'
+    | 'customer_db_storage_unavailable'
     | 'retry_safe_transient_5xx'
     | 'unsafe_transient_5xx'
     | 'hard_billing_error'
@@ -94,6 +101,35 @@ function isIdempotencyInProgressResponse(input: {
   );
 }
 
+function isGatewayInvocationInProgressResponse(input: {
+  status: number;
+  bodyText: string;
+}): boolean {
+  return (
+    input.status === 409 &&
+    parseJsonObject(input.bodyText)?.code ===
+      TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE
+  );
+}
+
+function isCustomerDbStorageUnavailableResponse(input: {
+  toolId: string;
+  status: number;
+  bodyText: string;
+}): boolean {
+  if (
+    input.status !== 503 ||
+    (input.toolId !== 'query_customer_db' &&
+      input.toolId !== 'customer_db_query_customer_db')
+  ) {
+    return false;
+  }
+  return (
+    parseJsonObject(input.bodyText)?.code ===
+    TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE
+  );
+}
+
 function idempotencyInProgressRetryDelayMs(attempt: number): number {
   return (
     TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -125,6 +161,8 @@ export function parseToolExecuteAuthScopeChangedError(input: {
 function decideToolExecuteHttpRetry(input: {
   status: number;
   idempotencyInProgress?: boolean;
+  gatewayInvocationInProgress?: boolean;
+  customerDbStorageUnavailable?: boolean;
   hardBillingFailure?: boolean;
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
@@ -151,6 +189,13 @@ function decideToolExecuteHttpRetry(input: {
       reason: 'rate_limit',
     };
   }
+  if (input.gatewayInvocationInProgress) {
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS,
+      reason: 'gateway_invocation_in_progress',
+    };
+  }
   if (input.idempotencyInProgress) {
     return {
       retryable: true,
@@ -158,6 +203,16 @@ function decideToolExecuteHttpRetry(input: {
       reason: 'idempotency_in_progress',
     };
   }
+  // This typed response is emitted for a PostgreSQL permission denial. The
+  // failed statement cannot commit, so it is safe to retry even when the SQL
+  // itself is a mutation. Do not generalize this to arbitrary 503 responses.
+  if (input.customerDbStorageUnavailable) {
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+      reason: 'customer_db_storage_unavailable',
+    };
+  }
   if (input.status >= 500 && input.status < 600) {
     if (!input.transientHttpRetrySafe) {
       return {
@@ -186,6 +241,8 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
   > = {
     rate_limit: 0,
     idempotency_in_progress: 0,
+    gateway_invocation_in_progress: 0,
+    customer_db_storage_unavailable: 0,
     retry_safe_transient_5xx: 0,
     unsafe_transient_5xx: 0,
     hard_billing_error: 0,
@@ -200,6 +257,15 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
           status: input.status,
           bodyText: input.bodyText ?? '',
         }),
+        gatewayInvocationInProgress: isGatewayInvocationInProgressResponse({
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
+        customerDbStorageUnavailable: isCustomerDbStorageUnavailableResponse({
+          toolId: input.toolId,
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
         hasRetryAfterHeader: true,
         transientHttpRetrySafe: input.transientHttpRetrySafe === true,
       });
@@ -240,9 +306,15 @@ export function classifyToolExecuteHttpFailure(input: {
     typeof input.retryAfterHeader === 'string' &&
     input.retryAfterHeader.trim().length > 0;
   const idempotencyInProgress = isIdempotencyInProgressResponse(input);
+  const gatewayInvocationInProgress =
+    isGatewayInvocationInProgressResponse(input);
+  const customerDbStorageUnavailable =
+    isCustomerDbStorageUnavailableResponse(input);
   const initialRetryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
+    gatewayInvocationInProgress,
+    customerDbStorageUnavailable,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
   });
@@ -264,6 +336,8 @@ export function classifyToolExecuteHttpFailure(input: {
   const retryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
+    gatewayInvocationInProgress,
+    customerDbStorageUnavailable,
     hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
@@ -295,6 +369,9 @@ export function classifyToolExecuteHttpFailure(input: {
           ),
         )
       : retryDecision.reason === 'idempotency_in_progress' &&
+            !hasRetryAfterHeader
+        ? idempotencyInProgressRetryDelayMs(input.attempt)
+        : retryDecision.reason === 'gateway_invocation_in_progress' &&
           !hasRetryAfterHeader
         ? idempotencyInProgressRetryDelayMs(input.attempt)
         : retryAfterMs > 0
@@ -317,6 +394,8 @@ export function classifyToolExecuteHttpFailure(input: {
           ? retryAfterMs
           : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
         : null,
-    chargeRetryBudget: shouldRetry,
+    chargeRetryBudget:
+      shouldRetry &&
+      retryDecision.reason !== 'gateway_invocation_in_progress',
   };
 }

package/dist/cli/index.js

@@ -718,7 +718,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.288",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -703,7 +703,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.288",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.js

@@ -438,7 +438,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.288",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -367,7 +367,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.288",
   contracts: {
     api: {
       name: "sdk-http-api",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.287",
+  "version": "0.1.288",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {