deepline 0.1.287 → 0.1.289

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.287',
+  version: '0.1.289',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -309,6 +309,10 @@ const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
 const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
 const TOOL_SCALAR_COALESCE_WINDOW_MS = 5;
 const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
+// The receipt gateway fully buffers the integration response before sending
+// headers to the runner. Once those headers arrive, a long body stall is a
+// broken gateway-to-runner connection, not slow provider work.
+export const TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS = 30_000;
 const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // Every runtime-API tool fetch needs a client-side deadline. Without one, a
@@ -352,6 +356,35 @@ class ToolExecuteResponseBodyTransportError extends Error {
   }
 }
 
+class ToolExecuteResponseBodyTimeoutError extends Error {
+  constructor(toolId: string) {
+    super(
+      `Tool ${toolId} response body was not delivered within ${TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS}ms after response headers.`,
+    );
+    this.name = 'ToolExecuteResponseBodyTimeoutError';
+  }
+}
+
+async function readToolExecuteResponseBody<T>(input: {
+  toolId: string;
+  abortController: AbortController | null;
+  read: () => Promise<T>;
+}): Promise<T> {
+  let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timeoutHandle = setTimeout(() => {
+      const error = new ToolExecuteResponseBodyTimeoutError(input.toolId);
+      input.abortController?.abort(error);
+      reject(error);
+    }, TOOL_EXECUTE_RESPONSE_BODY_TIMEOUT_MS);
+  });
+  try {
+    return await Promise.race([input.read(), timeout]);
+  } finally {
+    if (timeoutHandle) clearTimeout(timeoutHandle);
+  }
+}
+
 class ToolExecuteInvalidJsonError extends Error {
   readonly cause: unknown;
 
@@ -1289,6 +1322,7 @@ export class PlayContextImpl {
     createSecretRedactionContext();
   private mapInvocationIndex = 0;
   private readonly stepCallIndexByKey = new Map<string, number>();
+  private readonly toolCallIndexByKey = new Map<string, number>();
   /**
    * Parent-level inline-child aggregates. Maintained only by the single-writer
    * progress path (never per child) so concurrent fan-out cannot contend. See
@@ -3281,7 +3315,9 @@ export class PlayContextImpl {
     const executionNonce =
       typeof wrapped.job_id === 'string' && wrapped.job_id.trim()
         ? wrapped.job_id.trim()
-        : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+        : stableDigest(
+            `${this.currentRunId}:${toolId}:${stableStringify(originalRequestInput)}:${datasetLimit}`,
+          );
     const datasetId = `tool-list:${sha256Hex(
       `${toolId}:${this.currentRunId}:${sql}:${datasetLimit}:${executionNonce}`,
     )}`;
@@ -3294,6 +3330,10 @@ export class PlayContextImpl {
         toolId,
         originalRequestInput,
         {
+          durableCallReceiptKey: `${buildDurableToolReceiptPrefix({
+            orgId: this.#options.orgId ?? 'unknown-org',
+            toolId,
+          })}${stableDigest(`${datasetId}:${offset}:${limit}`)}`,
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId),
           customerDbDataset: {
             limit: datasetLimit,
@@ -6461,6 +6501,14 @@ export class PlayContextImpl {
       toolId,
       requestInput: input,
     });
+    const store = rowContext.getStore();
+    let logicalCallId: string | null = null;
+    if (!store) {
+      const callIndexKey = `${this.currentExecutionScope.receipt.namespace}:workflow:${normalizedKey}:${toolRequestIdentity}`;
+      const callIndex = this.toolCallIndexByKey.get(callIndexKey) ?? 0;
+      this.toolCallIndexByKey.set(callIndexKey, callIndex + 1);
+      logicalCallId = stableDigest(`${callIndexKey}:${callIndex}`);
+    }
     let executionAuthScopeDigest =
       (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
     const eventWaitHandler =
@@ -6476,7 +6524,6 @@ export class PlayContextImpl {
       staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
     });
     const checkpointCacheKeys = [durableCacheKey];
-    const store = rowContext.getStore();
 
     const executeTool = async (context?: {
       leaseId?: string | null;
@@ -6531,18 +6578,28 @@ export class PlayContextImpl {
             : `Calling tool: ${toolId}`,
         );
         const directReceiptLeaseId = context?.leaseId?.trim() || null;
+        if (!logicalCallId) {
+          throw new Error(
+            'Direct tool execution requires a stable logical call identity.',
+          );
+        }
+        const physicalDirectKey = cacheableToolResult
+          ? directCacheKey
+          : `${buildDurableToolReceiptPrefix({
+              orgId: this.#options.orgId ?? 'unknown-org',
+              toolId,
+            })}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
         const execution = await this.callToolExecutionAPI(toolId, input, {
-          ...(cacheableToolResult
-            ? {
-                durableCallReceiptKey: directCacheKey,
-                executionAuthScopeDigest,
-                providerIdempotencyKey: this.providerIdempotencyKeyForToolCall({
-                  cacheKey: directCacheKey,
-                  force: toolCachePolicy.force,
-                  leaseId: directReceiptLeaseId,
-                }),
-              }
-            : {}),
+          durableCallReceiptKey: physicalDirectKey,
+          executionAuthScopeDigest,
+          providerIdempotencyKey: cacheableToolResult
+            ? this.providerIdempotencyKeyForToolCall({
+                cacheKey: directCacheKey,
+                force: toolCachePolicy.force,
+                leaseId: directReceiptLeaseId,
+                logicalCallId,
+              })
+            : physicalDirectKey,
           timeoutMs: resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs),
           ...(directReceiptLeaseId &&
           (this.#options.markRuntimeStepReceiptRunning ||
@@ -6692,7 +6749,7 @@ export class PlayContextImpl {
     ) {
       try {
         const toolRetryPolicy = await this.#options
-          .getToolRetryPolicy?.(toolId)
+          .getToolRetryPolicy?.(toolId, input)
           .catch(() => null);
         return await this.executeWithRuntimeReceipt(
           'tool',
@@ -8547,11 +8604,17 @@ export class PlayContextImpl {
     cacheKey: string;
     force?: boolean;
     leaseId?: string | null;
+    logicalCallId?: string;
   }): string {
-    const fallbackAttemptId =
-      input.force === true && !input.leaseId
-        ? `${this.currentReceiptOwnerRunId}:${crypto.randomUUID()}`
-        : null;
+    let fallbackAttemptId: string | null = null;
+    if (input.force === true && !input.leaseId) {
+      if (!input.logicalCallId) {
+        throw new Error(
+          'Forced tool execution without a receipt lease requires a stable logical call identity.',
+        );
+      }
+      fallbackAttemptId = `${this.currentRunId}:${input.logicalCallId}`;
+    }
     return buildDurableToolProviderIdempotencyKey({
       receiptKey: input.cacheKey,
       force: input.force,
@@ -8570,7 +8633,13 @@ export class PlayContextImpl {
         'executorToken and baseUrl are required for tool API calls (cloud execution only)',
       );
     }
-    const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
+    const requestsDurableInvocationFence =
+      this.#options.requestDurableInvocationFence === true ||
+      this.#options.durableInvocationFence === true;
+    const executeSuffix = requestsDurableInvocationFence
+      ? 'execute-fenced-v1'
+      : 'execute';
+    const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/${executeSuffix}`;
     const timeoutMs = resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs);
     const provider = toolId.split(/[._]/)[0]?.trim() || 'provider';
     const activityId = `provider:${toolId}`;
@@ -8607,12 +8676,89 @@ export class PlayContextImpl {
         async (span) => {
           const httpFailureAttempts =
             createToolExecuteHttpFailureAttemptTracker();
+          // Snapshot the caller-controlled payload exactly once. The replay
+          // decision and every physical attempt must refer to the same bytes,
+          // even if the original object has getters or is later mutated.
+          const toolInputSnapshot = JSON.parse(JSON.stringify(input)) as Record<
+            string,
+            unknown
+          >;
           const retryPolicy = await this.#options
-            .getToolRetryPolicy?.(toolId)
+            .getToolRetryPolicy?.(toolId, toolInputSnapshot)
             .catch(() => null);
           const retrySafeTransientHttp =
             retryPolicy?.retrySafeTransientHttp === true;
+          const invocationOrgId = this.#options.orgId?.trim() ?? '';
+          const durableCallReceiptKey =
+            options?.durableCallReceiptKey?.trim() ||
+            (requestsDurableInvocationFence && invocationOrgId
+              ? `${buildDurableToolReceiptPrefix({
+                  orgId: invocationOrgId,
+                  toolId,
+                })}${stableDigest(
+                  `transport:${this.currentRunId}:${toolId}:${crypto.randomUUID()}`,
+                )}`
+              : null);
+          const executionAuthScopeDigest = options?.executionAuthScopeDigest
+            ? options.executionAuthScopeDigest.trim() || null
+            : ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
+          const providerIdempotencyKey =
+            options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
+          // Correlation identity is stable across every transport retry,
+          // including calls without a durable receipt.
+          const deeplineRequestId = providerIdempotencyKey
+            ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
+            : `ctx-tool-${crypto.randomUUID()}`;
+          const serializedRequestBody = (invocationAttempt: number) =>
+            JSON.stringify({
+              payload: toolInputSnapshot,
+              metadata: {
+                parent_run_id: this.#options.runId,
+                invocation_attempt: invocationAttempt,
+                ...(requestsDurableInvocationFence
+                  ? { invocation_fence_version: 1 }
+                  : {}),
+                ...(durableCallReceiptKey
+                  ? {
+                      durable_call_receipt_key: durableCallReceiptKey,
+                      ...(executionAuthScopeDigest
+                        ? {
+                            execution_auth_scope_digest:
+                              executionAuthScopeDigest,
+                          }
+                        : {}),
+                    }
+                  : {}),
+                ...(options?.customerDbDataset
+                  ? {
+                      query_result_dataset: {
+                        limit: options.customerDbDataset.limit,
+                        offset: options.customerDbDataset.offset,
+                        page_size: options.customerDbDataset.pageSize,
+                        total_rows: options.customerDbDataset.totalRows,
+                      },
+                      ...(isCustomerDbDatasetTool(toolId)
+                        ? {
+                            customer_db_dataset: {
+                              limit: options.customerDbDataset.limit,
+                              offset: options.customerDbDataset.offset,
+                              page_size: options.customerDbDataset.pageSize,
+                              total_rows: options.customerDbDataset.totalRows,
+                            },
+                          }
+                        : {}),
+                    }
+                  : {}),
+                ...(providerIdempotencyKey
+                  ? { provider_idempotency_key: providerIdempotencyKey }
+                  : {}),
+              },
+              ...(this.#options.integrationMode
+                ? { integration_mode: this.#options.integrationMode }
+                : {}),
+            });
           let transportAttempt = 0;
+          let invocationAttempt = 0;
           const retryToolTransportFailure = async (input: {
             error: unknown;
             elapsedMs: number;
@@ -8662,23 +8808,7 @@ export class PlayContextImpl {
             let responseErrorText: string | null = null;
             let providerCallStartedAt: number | null = null;
             let providerCallElapsedMs: number | null = null;
-            const durableCallReceiptKey =
-              options?.durableCallReceiptKey?.trim() || null;
-            const executionAuthScopeDigest =
-              durableCallReceiptKey && options?.executionAuthScopeDigest
-                ? options.executionAuthScopeDigest.trim() || null
-                : durableCallReceiptKey
-                  ? ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ??
-                    null)
-                  : null;
-            const providerIdempotencyKey =
-              options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
-            // Every gateway request needs a correlation ID. When the call has
-            // an idempotency key, keep its stable ID across retries; otherwise
-            // give this individual attempt a fresh ID for gateway pairing.
-            const deeplineRequestId = providerIdempotencyKey
-              ? `ctx-tool-${stableDigest(providerIdempotencyKey).slice(0, 32)}`
-              : `ctx-tool-${crypto.randomUUID()}`;
+            let fetchDispatched = false;
             // Receipt ownership is a liveness contract, not a one-time check.
             // Keep it alive for the whole provider HTTP request. The cadence
             // comes from the store-issued expiry because a remote runner may
@@ -8747,6 +8877,7 @@ export class PlayContextImpl {
                 const integrationFetchStartedAt = Date.now();
                 providerCallStartedAt = integrationFetchStartedAt;
                 try {
+                  fetchDispatched = true;
                   response = await fetch(url, {
                     method: 'POST',
                     signal: abortController?.signal,
@@ -8765,59 +8896,23 @@ export class PlayContextImpl {
                           }
                         : {}),
                       ...protectionHeaders,
-                    },
-                    body: JSON.stringify({
-                      payload: input,
-                      metadata: {
-                        parent_run_id: this.#options.runId,
-                        ...(durableCallReceiptKey
-                          ? {
-                              durable_call_receipt_key: durableCallReceiptKey,
-                              ...(executionAuthScopeDigest
-                                ? {
-                                    execution_auth_scope_digest:
-                                      executionAuthScopeDigest,
-                                  }
-                                : {}),
-                            }
-                          : {}),
-                        ...(options?.customerDbDataset
-                          ? {
-                              query_result_dataset: {
-                                limit: options.customerDbDataset.limit,
-                                offset: options.customerDbDataset.offset,
-                                page_size: options.customerDbDataset.pageSize,
-                                total_rows: options.customerDbDataset.totalRows,
-                              },
-                              ...(isCustomerDbDatasetTool(toolId)
-                                ? {
-                                    customer_db_dataset: {
-                                      limit: options.customerDbDataset.limit,
-                                      offset: options.customerDbDataset.offset,
-                                      page_size:
-                                        options.customerDbDataset.pageSize,
-                                      total_rows:
-                                        options.customerDbDataset.totalRows,
-                                    },
-                                  }
-                                : {}),
-                            }
-                          : {}),
-                        ...(providerIdempotencyKey
-                          ? { provider_idempotency_key: providerIdempotencyKey }
-                          : {}),
-                      },
-                      ...(this.#options.integrationMode
-                        ? { integration_mode: this.#options.integrationMode }
+                      ...(this.#options.runtimeTestFaultHeader
+                        ? {
+                            'x-deepline-test-fault':
+                              this.#options.runtimeTestFaultHeader,
+                          }
                         : {}),
-                    }),
+                    },
+                    body: serializedRequestBody(invocationAttempt),
                   });
                   if (response.ok) {
                     try {
-                      responseData = (await response.json()) as Record<
-                        string,
-                        unknown
-                      >;
+                      responseData = await readToolExecuteResponseBody({
+                        toolId,
+                        abortController,
+                        read: () =>
+                          response!.json() as Promise<Record<string, unknown>>,
+                      });
                     } catch (error) {
                       if (error instanceof SyntaxError) {
                         throw new ToolExecuteInvalidJsonError(error);
@@ -8826,7 +8921,11 @@ export class PlayContextImpl {
                     }
                   } else {
                     try {
-                      responseErrorText = await response.text();
+                      responseErrorText = await readToolExecuteResponseBody({
+                        toolId,
+                        abortController,
+                        read: () => response!.text(),
+                      });
                     } catch (error) {
                       throw new ToolExecuteResponseBodyTransportError(error);
                     }
@@ -8887,11 +8986,39 @@ export class PlayContextImpl {
                   402,
                 );
               }
-              const responseBodyReplaySafe =
-                !(error instanceof ToolExecuteResponseBodyTransportError) ||
-                retrySafeTransientHttp ||
-                response?.status === 429;
-              if (!responseBodyReplaySafe) {
+              const ambiguousDispatchedFailure =
+                fetchDispatched &&
+                (response === null ||
+                  error instanceof ToolExecuteResponseBodyTransportError);
+              const hasDurableInvocationIdentity = Boolean(
+                durableCallReceiptKey &&
+                providerIdempotencyKey &&
+                executionAuthScopeDigest,
+              );
+              // A post-header body deadline aborts the fetch controller solely
+              // to stop consuming that response. It must not also cancel the
+              // independent gateway health check that establishes whether the
+              // committed response can be replayed safely. Preserve genuine
+              // caller/runtime cancellation by dropping the signal only when
+              // this exact body deadline caused the abort.
+              const durableFenceVerificationSignal =
+                abortController?.signal.aborted === true &&
+                abortController.signal.reason instanceof
+                  ToolExecuteResponseBodyTimeoutError
+                  ? undefined
+                  : abortController?.signal;
+              const durableInvocationFenceVerified =
+                hasDurableInvocationIdentity &&
+                requestsDurableInvocationFence &&
+                (this.#options.durableInvocationFence === true ||
+                  (await this.#options
+                    .verifyDurableInvocationFence?.(
+                      durableFenceVerificationSignal,
+                    )
+                    .catch(() => false)) === true);
+              const transportReplaySafe =
+                !ambiguousDispatchedFailure || durableInvocationFenceVerified;
+              if (!transportReplaySafe) {
                 const diagnostic = describeTransportError(transportError);
                 this.log(
                   `[runtime.transport_failure] ${JSON.stringify({
@@ -8905,13 +9032,17 @@ export class PlayContextImpl {
                         : Date.now() - providerCallStartedAt,
                     request_id: deeplineRequestId,
                     aborted: abortController?.signal.aborted === true,
-                    response_headers_received: true,
+                    response_headers_received: response !== null,
                     retry_safe: false,
                     error: diagnostic,
                   })}`,
                 );
+                const failureBoundary =
+                  error instanceof ToolExecuteResponseBodyTransportError
+                    ? 'response body transport failed after response headers'
+                    : 'request transport failed after dispatch before response headers';
                 throw new ToolHttpError(
-                  `Tool ${toolId} response body transport failed after response headers; the ambiguous call was not retried because this operation is not declared retry-safe: ${diagnostic.message ?? 'unknown transport error'}`,
+                  `Tool ${toolId} ${failureBoundary}; the ambiguous call was not retried because this runtime has no durable invocation fence: ${diagnostic.message ?? 'unknown transport error'}`,
                   null,
                   0,
                   'repairable',
@@ -9004,6 +9135,9 @@ export class PlayContextImpl {
                 );
               }
               if (failure.shouldRetry) {
+                if (failure.reason !== 'gateway_invocation_in_progress') {
+                  invocationAttempt += 1;
+                }
                 if (failure.chargeRetryBudget) {
                   await this.governor.chargeBudget('retry');
                 }

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -512,6 +512,15 @@ export interface ContextOptions {
   /** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
   executorToken?: string;
   baseUrl?: string;
+  /**
+   * The integration base URL persists each keyed execute response before
+   * exposing it. Only runtime adapters that route through that gateway may
+   * enable ambiguous transport replay.
+   */
+  durableInvocationFence?: boolean;
+  requestDurableInvocationFence?: boolean;
+  verifyDurableInvocationFence?: (signal?: AbortSignal) => Promise<boolean>;
+  runtimeTestFaultHeader?: string | null;
   /**
    * Runtime-sheet transport selected by the runner. Daytona sandboxes use the
    * execution gateway and must never fall back to minting direct DB sessions.
@@ -644,7 +653,10 @@ export interface ContextOptions {
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolProvider?: (toolId: string) => Promise<string | null>;
-  getToolRetryPolicy?: (toolId: string) => Promise<{
+  getToolRetryPolicy?: (
+    toolId: string,
+    input: Record<string, unknown>,
+  ) => Promise<{
     retrySafeTransientHttp?: boolean;
     requiresExecutionFence?: boolean;
   } | null>;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -27,7 +27,8 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
 
 export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
 
-export type DaytonaClient = Pick<Daytona, 'create'>;
+export type DaytonaClient = Pick<Daytona, 'create'> &
+  Partial<Pick<Daytona, 'get'>>;
 export type DaytonaSandbox = Awaited<ReturnType<DaytonaClient['create']>>;
 export type DaytonaExecutionContext = PlayRunnerExecutionConfig['context'];
 export type DaytonaStageEmitter = (
@@ -37,6 +38,7 @@ export type DaytonaStageEmitter = (
 
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
+  daytonaOrganizationId: string;
   billingStartedAt: number;
   billingEndedAt?: number;
 };
@@ -56,6 +58,22 @@ type DaytonaCreateResult = {
   attemptElapsedMs: number;
 };
 
+async function rejectAcquiredSandbox(
+  sandbox: DaytonaSandbox,
+  reason: string,
+): Promise<never> {
+  try {
+    await sandbox.delete(30);
+  } catch (error) {
+    const cleanupError = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `${reason} Defensive deletion of Daytona sandbox ${sandbox.id} also failed: ${cleanupError}`,
+      { cause: error },
+    );
+  }
+  throw new Error(reason);
+}
+
 function normalizeLabelValue(value: string | null | undefined): string | null {
   const trimmed = value?.trim();
   return trimmed ? trimmed.slice(0, 63) : null;
@@ -233,7 +251,15 @@ async function createRetriedOneShotDaytonaSandbox(input: {
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       errors.push(message);
+      input.emitStage('create:attempt_failed', {
+        attempt,
+        elapsedMs: Date.now() - input.startedAt,
+        attemptElapsedMs: Date.now() - attemptStartedAt,
+        error: message,
+      });
       console.warn('[play-runner.daytona.create_attempt_failed]', {
+        workflowId: input.context.workflowId ?? null,
+        runId: input.context.runId ?? null,
         attempt,
         error: message,
       });
@@ -265,11 +291,46 @@ async function acquireOneShotDaytonaSandbox(input: {
     granted.diskGiB !== DAYTONA_SANDBOX_DISK_GIB ||
     granted.gpu !== DAYTONA_SANDBOX_GPU
   ) {
-    await result.sandbox.delete(30).catch(() => undefined);
-    throw new Error(
+    await rejectAcquiredSandbox(
+      result.sandbox,
       `Daytona sandbox resource boundary mismatch: expected cpu=${DAYTONA_SANDBOX_CPU} memoryGiB=${DAYTONA_SANDBOX_MEMORY_GIB} diskGiB=${DAYTONA_SANDBOX_DISK_GIB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
     );
   }
+  const configuredOrganizationId =
+    process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+  const observedOrganizationId = result.sandbox.organizationId?.trim() || null;
+  if (
+    configuredOrganizationId &&
+    observedOrganizationId &&
+    configuredOrganizationId !== observedOrganizationId
+  ) {
+    await rejectAcquiredSandbox(
+      result.sandbox,
+      'Daytona sandbox organization routing mismatch. Refusing to run customer code in a sandbox whose observed organization differs from the configured organization.',
+    );
+  }
+  let lookupOrganizationId: string | null = null;
+  if (
+    !observedOrganizationId &&
+    !configuredOrganizationId &&
+    input.daytona.get
+  ) {
+    try {
+      const lookedUpSandbox = await input.daytona.get(result.sandbox.id);
+      lookupOrganizationId = lookedUpSandbox.organizationId?.trim() || null;
+    } catch {
+      // The failure below is intentionally about the invariant, not the
+      // provider response. The newly created sandbox is still deleted.
+    }
+  }
+  const daytonaOrganizationId =
+    observedOrganizationId ?? configuredOrganizationId ?? lookupOrganizationId;
+  if (!daytonaOrganizationId) {
+    return await rejectAcquiredSandbox(
+      result.sandbox,
+      'Daytona sandbox organization routing identity is missing. Refusing to run customer code without a durable cleanup routing domain.',
+    );
+  }
   const billingStartedAt = Date.now();
   const sandbox = result.sandbox;
   input.emitStage('create:done', {
@@ -281,7 +342,7 @@ async function acquireOneShotDaytonaSandbox(input: {
     memoryGiB: granted.memoryGiB,
     diskGiB: granted.diskGiB,
   });
-  return { sandbox, billingStartedAt };
+  return { sandbox, daytonaOrganizationId, billingStartedAt };
 }
 
 export function createOneShotDaytonaSandboxLifecycle(input: {

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -61,6 +61,7 @@ export type StagedDaytonaPayload = {
   command: string;
   outputPath: string;
   exitCodePath: string;
+  runtimeCompletedPath: string;
   progressEventPath: string;
 };
 
@@ -557,6 +558,7 @@ export async function stageDaytonaRunnerPayload(input: {
     command,
     outputPath,
     exitCodePath,
+    runtimeCompletedPath,
     progressEventPath,
   };
 }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -289,27 +289,161 @@ export async function inspectDetachedDaytonaRunner(input: {
   }
 }
 
-export async function deleteDaytonaSandboxById(input: {
+/** Read the runner's exact customer-code completion fence before terminal GC. */
+export async function readDetachedDaytonaRuntimeCompletion(input: {
+  sandboxId: string;
+  runtimeCompletedPath: string;
+}): Promise<number | null> {
+  try {
+    const { clientOptions } = loadDaytonaRequiredConfig();
+    const sandbox = (await daytonaSdkClientFactory
+      .createFull(clientOptions)
+      .get(input.sandboxId)) as DaytonaSandbox;
+    const marker = JSON.parse(
+      (await sandbox.fs.downloadFile(input.runtimeCompletedPath, 5)).toString(
+        'utf-8',
+      ),
+    ) as { at?: unknown };
+    return typeof marker.at === 'number' && Number.isFinite(marker.at)
+      ? marker.at
+      : null;
+  } catch (error) {
+    console.warn(
+      '[play-runner.daytona.runtime_completion_marker_unavailable]',
+      {
+        sandboxId: input.sandboxId,
+        // Path is generated per attempt and contains no customer data.
+        runtimeCompletedPath: input.runtimeCompletedPath,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    );
+    return null;
+  }
+}
+
+export type DaytonaSandboxDeleteOutcome =
+  | {
+      kind: 'deleted' | 'already_absent';
+      organizationId: string | null;
+    }
+  | {
+      kind: 'timed_out' | 'rate_limited' | 'failed';
+      organizationId: string | null;
+      code: string;
+      detail: string;
+    };
+
+export async function deleteDaytonaSandboxByIdWithOutcome(input: {
   sandboxId: string;
   timeoutSeconds?: number;
-}): Promise<boolean> {
+  expectedOrganizationId?: string | null;
+  allowUnscopedAlreadyAbsent?: boolean;
+}): Promise<DaytonaSandboxDeleteOutcome> {
   const sandboxId = input.sandboxId?.trim();
-  if (!sandboxId) return false;
+  const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
+  if (!sandboxId) {
+    return {
+      kind: 'failed',
+      organizationId: expectedOrganizationId,
+      code: 'invalid_sandbox_id',
+      detail: 'Sandbox ID is required.',
+    };
+  }
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
     const daytona = daytonaSdkClientFactory.createFull(clientOptions);
     const sandbox = await daytona.get(sandboxId);
+    const observedOrganizationId = sandbox.organizationId?.trim() || null;
+    if (
+      expectedOrganizationId &&
+      observedOrganizationId !== expectedOrganizationId
+    ) {
+      return {
+        kind: 'failed',
+        organizationId: observedOrganizationId,
+        code: 'wrong_routing_domain',
+        detail: 'Sandbox belongs to a different Daytona organization.',
+      };
+    }
     await daytona.delete(sandbox, input.timeoutSeconds ?? 30);
-    return true;
+    return {
+      kind: 'deleted',
+      organizationId: observedOrganizationId,
+    };
   } catch (error) {
+    const failure = describeDaytonaLookupFailure(error);
+    // Cleanup is an idempotent "ensure absent" operation. Daytona returning
+    // not-found means another cleanup owner already satisfied the obligation.
+    // The durable expected organization came from the sandbox returned by the
+    // same organization-scoped credential at creation time. When an explicit
+    // worker organization is configured it must still match that evidence;
+    // deployments which rely only on the provider-returned organization retain
+    // that durable creation-domain proof. Legacy eager cleanup retains its
+    // previous unscoped behavior through the explicit compatibility option.
+    if (failure.httpStatus === 404) {
+      const configuredOrganizationId =
+        process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+      if (
+        !input.allowUnscopedAlreadyAbsent &&
+        (!expectedOrganizationId ||
+          (configuredOrganizationId &&
+            configuredOrganizationId !== expectedOrganizationId))
+      ) {
+        return {
+          kind: 'failed',
+          organizationId: configuredOrganizationId,
+          code: expectedOrganizationId
+            ? 'wrong_routing_domain'
+            : 'missing_routing_domain',
+          detail:
+            'Daytona returned not-found without an exact creation-domain match.',
+        };
+      }
+      console.info('[play-runner.daytona.reclaim_sandbox_already_absent]', {
+        sandboxId,
+      });
+      return {
+        kind: 'already_absent',
+        organizationId:
+          configuredOrganizationId ?? expectedOrganizationId ?? null,
+      };
+    }
     console.warn('[play-runner.daytona.reclaim_sandbox_delete_failed]', {
       sandboxId,
-      error: error instanceof Error ? error.message : String(error),
+      failure,
     });
-    return false;
+    const timedOut =
+      failure.httpStatus === 408 ||
+      /(?:timeout|timed out|ETIMEDOUT)/i.test(
+        `${failure.errorCode ?? ''} ${failure.detail}`,
+      );
+    return {
+      kind:
+        failure.httpStatus === 429
+          ? 'rate_limited'
+          : timedOut
+            ? 'timed_out'
+            : 'failed',
+      organizationId: expectedOrganizationId,
+      code:
+        failure.errorCode ??
+        (failure.httpStatus ? `http_${failure.httpStatus}` : 'delete_failed'),
+      detail: failure.detail,
+    };
   }
 }
 
+export async function deleteDaytonaSandboxById(input: {
+  sandboxId: string;
+  timeoutSeconds?: number;
+}): Promise<boolean> {
+  const outcome = await deleteDaytonaSandboxByIdWithOutcome({
+    ...input,
+    allowUnscopedAlreadyAbsent: true,
+  });
+  return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
+}
+
 function formatDaytonaExecutionError(error: unknown): string {
   const message = formatDaytonaError(error);
   return isConfiguredDaytonaRuntimeLimit(error)
@@ -630,6 +764,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     let cancellationCleanupStarted = false;
     let activeAcquiredResource: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     } | null = null;
@@ -639,6 +774,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     const runtimeResourceReportErrors = new Set<unknown>();
     const reportRuntimeResource = async (acquired: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     }) => {
@@ -658,6 +794,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
               ? 'preview'
               : 'production',
+          daytonaOrganizationId: acquired.daytonaOrganizationId,
           billingStartedAt: acquired.billingStartedAt,
           billingEndedAt,
           cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
@@ -673,6 +810,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     };
     const reportRetiringRuntimeResource = async (acquired: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     }) => {
@@ -929,6 +1067,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               cmdId: start.cmdId,
               outputPath: stagedPayload.outputPath,
               exitCodePath: stagedPayload.exitCodePath,
+              runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
               startedAtMs: Date.now(),
               ceilingMs: DAYTONA_DETACHED_CEILING_SECONDS * 1_000,
             },
@@ -988,8 +1127,28 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       // Resource persistence belongs to the scheduler control plane. Preserve
       // its typed capacity/fence errors so Absurd can defer or fence the
       // attempt; converting them into a runner failure would terminally fail a
-      // play that never began executing customer code.
-      if (runtimeResourceReportErrors.has(error)) throw error;
+      // play that never began executing customer code. The sandbox exists
+      // before that durable callback can succeed, so synchronously delete it
+      // before returning the control-plane error. The old fire-and-forget
+      // cleanup path was skipped by this rethrow and leaked the acquisition.
+      if (runtimeResourceReportErrors.has(error)) {
+        const sandbox = sandboxCleanup.currentSandbox();
+        if (sandbox) {
+          try {
+            await sandbox.delete(30);
+            console.info(
+              '[play-runner.daytona.resource_report_failure_cleanup_done]',
+              { sandboxId: sandbox.id },
+            );
+          } catch (cleanupError) {
+            throw new AggregateError(
+              [error, cleanupError],
+              `Failed to persist or delete acquired Daytona sandbox ${sandbox.id}.`,
+            );
+          }
+        }
+        throw error;
+      }
       emitDaytonaStage(callbacks, config.context, 'execute:error', {
         sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
         error: formatDaytonaError(error),

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -13,6 +13,8 @@ export type PlayRunnerRuntimeResource = {
   kind: 'daytona_sandbox';
   sandboxId: string;
   daytonaEnvironment?: 'preview' | 'production';
+  /** Stable, non-secret provider ownership domain returned by Daytona. */
+  daytonaOrganizationId?: string;
   billingStartedAt: number;
   billingEndedAt?: number | null;
   terminalReason?: RuntimeResourceTerminalReason | null;

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -39,6 +39,8 @@ export type PlayExecutionSuspension =
        * timeout-wake salvage verification. */
       outputPath: string;
       exitCodePath: string;
+      /** Exact customer-code completion fence inside the Daytona sandbox. */
+      runtimeCompletedPath?: string;
       startedAtMs: number;
       /** Overall run ceiling; the park timeout. */
       ceilingMs: number;

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -6,6 +6,7 @@ export type RuntimeTestFaultName =
   | 'receipt_complete_write_fail'
   | 'receipt_fail_write_fail'
   | 'worker_receipt_complete_write_fail'
+  | 'invocation_response_delivery_abort'
   /**
    * Holds an already checked-out receipt-gateway scheduler client for the
    * supplied bounded millisecond value. Preview/CI only: this creates a
@@ -70,6 +71,7 @@ const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
   'receipt_complete_write_fail',
   'receipt_fail_write_fail',
   'worker_receipt_complete_write_fail',
+  'invocation_response_delivery_abort',
   'receipt_gateway_hold_ms',
 ]);
 

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -21,6 +21,8 @@ export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
 export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
 export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
+export const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE =
+  'CUSTOMER_DB_STORAGE_UNAVAILABLE';
 /**
  * A provider/action-local outcome contract emits this only when a provider has
  * explicitly said that the same idempotency key is still executing. It is not
@@ -28,6 +30,9 @@ export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
  */
 export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE =
   'UPSTREAM_IDEMPOTENCY_IN_PROGRESS';
+export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE =
+  'GATEWAY_INVOCATION_IN_PROGRESS';
+export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS = 65;
 
 export class ToolExecuteAuthScopeChangedError extends Error {
   readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
@@ -45,6 +50,8 @@ export type ToolExecuteHttpRetryDecision = {
   reason:
     | 'rate_limit'
     | 'idempotency_in_progress'
+    | 'gateway_invocation_in_progress'
+    | 'customer_db_storage_unavailable'
     | 'retry_safe_transient_5xx'
     | 'unsafe_transient_5xx'
     | 'hard_billing_error'
@@ -94,6 +101,35 @@ function isIdempotencyInProgressResponse(input: {
   );
 }
 
+function isGatewayInvocationInProgressResponse(input: {
+  status: number;
+  bodyText: string;
+}): boolean {
+  return (
+    input.status === 409 &&
+    parseJsonObject(input.bodyText)?.code ===
+      TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE
+  );
+}
+
+function isCustomerDbStorageUnavailableResponse(input: {
+  toolId: string;
+  status: number;
+  bodyText: string;
+}): boolean {
+  if (
+    input.status !== 503 ||
+    (input.toolId !== 'query_customer_db' &&
+      input.toolId !== 'customer_db_query_customer_db')
+  ) {
+    return false;
+  }
+  return (
+    parseJsonObject(input.bodyText)?.code ===
+    TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_CODE
+  );
+}
+
 function idempotencyInProgressRetryDelayMs(attempt: number): number {
   return (
     TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -125,6 +161,8 @@ export function parseToolExecuteAuthScopeChangedError(input: {
 function decideToolExecuteHttpRetry(input: {
   status: number;
   idempotencyInProgress?: boolean;
+  gatewayInvocationInProgress?: boolean;
+  customerDbStorageUnavailable?: boolean;
   hardBillingFailure?: boolean;
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
@@ -151,6 +189,13 @@ function decideToolExecuteHttpRetry(input: {
       reason: 'rate_limit',
     };
   }
+  if (input.gatewayInvocationInProgress) {
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS,
+      reason: 'gateway_invocation_in_progress',
+    };
+  }
   if (input.idempotencyInProgress) {
     return {
       retryable: true,
@@ -158,6 +203,16 @@ function decideToolExecuteHttpRetry(input: {
       reason: 'idempotency_in_progress',
     };
   }
+  // This typed response is emitted for a PostgreSQL permission denial. The
+  // failed statement cannot commit, so it is safe to retry even when the SQL
+  // itself is a mutation. Do not generalize this to arbitrary 503 responses.
+  if (input.customerDbStorageUnavailable) {
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+      reason: 'customer_db_storage_unavailable',
+    };
+  }
   if (input.status >= 500 && input.status < 600) {
     if (!input.transientHttpRetrySafe) {
       return {
@@ -186,6 +241,8 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
   > = {
     rate_limit: 0,
     idempotency_in_progress: 0,
+    gateway_invocation_in_progress: 0,
+    customer_db_storage_unavailable: 0,
     retry_safe_transient_5xx: 0,
     unsafe_transient_5xx: 0,
     hard_billing_error: 0,
@@ -200,6 +257,15 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
           status: input.status,
           bodyText: input.bodyText ?? '',
         }),
+        gatewayInvocationInProgress: isGatewayInvocationInProgressResponse({
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
+        customerDbStorageUnavailable: isCustomerDbStorageUnavailableResponse({
+          toolId: input.toolId,
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
         hasRetryAfterHeader: true,
         transientHttpRetrySafe: input.transientHttpRetrySafe === true,
       });
@@ -240,9 +306,15 @@ export function classifyToolExecuteHttpFailure(input: {
     typeof input.retryAfterHeader === 'string' &&
     input.retryAfterHeader.trim().length > 0;
   const idempotencyInProgress = isIdempotencyInProgressResponse(input);
+  const gatewayInvocationInProgress =
+    isGatewayInvocationInProgressResponse(input);
+  const customerDbStorageUnavailable =
+    isCustomerDbStorageUnavailableResponse(input);
   const initialRetryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
+    gatewayInvocationInProgress,
+    customerDbStorageUnavailable,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
   });
@@ -264,6 +336,8 @@ export function classifyToolExecuteHttpFailure(input: {
   const retryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
+    gatewayInvocationInProgress,
+    customerDbStorageUnavailable,
     hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
@@ -295,6 +369,9 @@ export function classifyToolExecuteHttpFailure(input: {
           ),
         )
       : retryDecision.reason === 'idempotency_in_progress' &&
+            !hasRetryAfterHeader
+        ? idempotencyInProgressRetryDelayMs(input.attempt)
+        : retryDecision.reason === 'gateway_invocation_in_progress' &&
           !hasRetryAfterHeader
         ? idempotencyInProgressRetryDelayMs(input.attempt)
         : retryAfterMs > 0
@@ -317,6 +394,8 @@ export function classifyToolExecuteHttpFailure(input: {
           ? retryAfterMs
           : TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
         : null,
-    chargeRetryBudget: shouldRetry,
+    chargeRetryBudget:
+      shouldRetry &&
+      retryDecision.reason !== 'gateway_invocation_in_progress',
   };
 }

package/dist/cli/index.js

@@ -718,7 +718,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -703,7 +703,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.js

@@ -438,7 +438,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -367,7 +367,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.287",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.287",
+  "version": "0.1.289",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {