deepline 0.2.30 → 0.2.32

package/dist/bundling-sources/sdk/src/client.ts

@@ -2155,6 +2155,9 @@ export class DeeplineClient {
         // defaults to absurd; callers normally omit this field.
         ...(request.profile ? { profile: request.profile } : {}),
         ...(integrationMode ? { integrationMode } : {}),
+        ...(request.fixtureBehavior
+          ? { fixtureBehavior: request.fixtureBehavior }
+          : {}),
         ...(runtime ? { runtime } : {}),
         ...(testPolicyOverrides ? { testPolicyOverrides } : {}),
       },
@@ -2225,6 +2228,9 @@ export class DeeplineClient {
         : {}),
       ...(request.profile ? { profile: request.profile } : {}),
       ...(integrationMode ? { integrationMode } : {}),
+      ...(request.fixtureBehavior
+        ? { fixtureBehavior: request.fixtureBehavior }
+        : {}),
       ...(runtime ? { runtime } : {}),
       ...(testPolicyOverrides ? { testPolicyOverrides } : {}),
     };

package/dist/bundling-sources/sdk/src/release.ts

@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: '0.2.30',
+  version: '0.2.32',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/sdk/src/types.ts

@@ -1394,6 +1394,8 @@ export interface StartPlayRunRequest {
   profile?: string;
   /** Optional per-run provider execution mode for eval/smoke runs. */
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Fixture-only provider response timing and outcome simulation. */
+  fixtureBehavior?: import('../../shared_libs/play-runtime/fixture-behavior').FixtureBehavior;
   /** Internal runtime estate selection. The app host remains unchanged. */
   runtime?: PlayRuntimeSelection;
   /** Internal/dev-only runtime policy overrides for black-box durability tests. */

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -992,6 +992,9 @@ async function postAppRuntimeApi<TResponse>(
           : AbortSignal.timeout(requestTimeoutMs),
       });
     } catch (error) {
+      if (context.signal?.aborted) {
+        throw context.signal.reason ?? error;
+      }
       if (
         attempt < maxAttempts &&
         isRetryableAppRuntimeFetchError({ action: body.action, error })

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -35,6 +35,12 @@ import {
   runtimeLeaseHeartbeatIntervalFromExpiry,
 } from './lease-policy';
 import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-supervisor';
+import {
+  shouldRouteFixtureProvider,
+  shouldRouteFixtureToolId,
+  validateFixtureBehavior,
+  waitForFixtureResponseDelay,
+} from './fixture-behavior';
 import { dispatchBoundedSettled } from './bounded-dispatch';
 import type { PlayQueueHint } from './governor/rate-state-backend';
 import type { MapRowOutcome } from './durability-store';
@@ -639,6 +645,24 @@ function isDeeplineDeveloperTunnelOrigin502(input: {
   }
 }
 
+function isUnmarkedExecutionGateway502(input: {
+  url: string;
+  status: number;
+  responseHeaders: Headers;
+}): boolean {
+  if (
+    input.status !== 502 ||
+    input.responseHeaders.has('x-deepline-request-id')
+  ) {
+    return false;
+  }
+  try {
+    return new URL(input.url).pathname.endsWith('/execute-fenced-v1');
+  } catch {
+    return false;
+  }
+}
+
 function loadSafeFetch(): Promise<SafeFetchModule> {
   safeFetchModule ??= import('@shared_libs/security/safe-fetch');
   return safeFetchModule;
@@ -1622,6 +1646,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
   };
 
   constructor(options: ContextOptions) {
+    const fixtureBehavior = validateFixtureBehavior(options.fixtureBehavior);
+    if (fixtureBehavior.ok === false) {
+      throw new Error(fixtureBehavior.error);
+    }
+    if (
+      fixtureBehavior.behavior !== null &&
+      options.integrationMode !== 'fixture'
+    ) {
+      throw new Error(
+        'fixtureBehavior is only valid when integrationMode is fixture.',
+      );
+    }
+    options = {
+      ...options,
+      fixtureBehavior: fixtureBehavior.behavior,
+    };
     this.#options = options;
     this.checkpoint = options.checkpoint ?? emptyCheckpoint();
     this.durableMappedToolResultsBackedByReceipts = Boolean(
@@ -6720,6 +6760,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     }
   }
 
+  private fixtureProviderPacingDisabled(): boolean {
+    return (
+      this.#options.integrationMode === 'fixture' &&
+      this.#options.enforceFixtureProviderPacing !== true
+    );
+  }
+
   private toolDispatchLane(request: ToolCallRequest): {
     key: string;
     readyCount: number;
@@ -8983,15 +9030,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                 getPayload: (request: ToolCallRequest) => request.input,
               });
 
+              const batchParallelismCeiling =
+                this.governor.policy.pacing.workerToolBatchDefaultParallelism;
               const batchSize =
-                compiledBatches.length > 0
+                compiledBatches.length > 0 &&
+                !this.fixtureProviderPacingDisabled()
                   ? await this.resourceGovernor.suggestedToolParallelism(
                       compiledBatches[0]!.batchOperation,
-                      this.governor.policy.pacing
-                        .workerToolBatchDefaultParallelism,
+                      batchParallelismCeiling,
                     )
-                  : this.governor.policy.pacing
-                      .workerToolBatchDefaultParallelism;
+                  : batchParallelismCeiling;
               await executeChunkedRequests({
                 requests: compiledBatches,
                 batchSize,
@@ -9182,11 +9230,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               // each in-flight call, so this only trims the launch burst.
               const toolCallConcurrencyCeiling =
                 this.governor.policy.concurrency.toolCalls;
-              const shapedToolParallelism =
-                await this.resourceGovernor.suggestedToolParallelism(
-                  toolId,
-                  toolCallConcurrencyCeiling,
-                );
+              const shapedToolParallelism = this.fixtureProviderPacingDisabled()
+                ? toolCallConcurrencyCeiling
+                : await this.resourceGovernor.suggestedToolParallelism(
+                    toolId,
+                    toolCallConcurrencyCeiling,
+                  );
               const dispatchWidth = Math.min(
                 toolCallConcurrencyCeiling,
                 Math.max(1, shapedToolParallelism),
@@ -9563,6 +9612,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             });
           let transportAttempt = 0;
           let invocationAttempt = 0;
+          let physicalAttempt = 0;
           const retryToolTransportFailure = async (input: {
             error: unknown;
             elapsedMs: number;
@@ -9626,11 +9676,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           };
 
           while (true) {
+            physicalAttempt += 1;
             let response: Response | null = null;
             let responseData: Record<string, unknown> | null = null;
             let responseErrorText: string | null = null;
             let providerCallStartedAt: number | null = null;
             let providerCallElapsedMs: number | null = null;
+            let integrationFetchStartedAt: number | null = null;
             let fetchDispatched = false;
             // Receipt ownership is a liveness contract, not a one-time check.
             // Keep it alive for the whole provider HTTP request. The cadence
@@ -9641,15 +9693,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               timeoutMs || hasReceiptHeartbeat ? new AbortController() : null;
             const receiptHeartbeat = options?.heartbeatReceipt;
             let heartbeatFailure: unknown = null;
-            const timeoutHandle = timeoutMs
-              ? setTimeout(() => {
-                  abortController?.abort(
-                    new Error(
-                      `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
-                    ),
-                  );
-                }, timeoutMs)
-              : null;
+            let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
             try {
               const ownershipStartedAt = Date.now();
               await options?.beforeProviderCall?.();
@@ -9705,7 +9749,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   fixtureExecution &&
                   this.#options.enforceFixtureProviderPacing === true;
                 const fixtureOnlyExecution =
-                  fixtureExecution && !enforceFixtureProviderPacing;
+                  this.fixtureProviderPacingDisabled();
                 if (
                   fixtureOnlyExecution &&
                   !this.fixtureProviderPacingBypassLogged
@@ -9730,69 +9774,181 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       toolId,
                       signal: abortController?.signal,
                     });
-                const integrationFetchStartedAt = Date.now();
-                providerCallStartedAt = integrationFetchStartedAt;
                 try {
-                  fetchDispatched = true;
-                  response = await fetch(url, {
-                    method: 'POST',
-                    signal: abortController?.signal,
-                    headers: {
-                      'Content-Type': 'application/json',
-                      Authorization: `Bearer ${this.#options.executorToken}`,
-                      [EXECUTE_RESPONSE_CONTRACT_HEADER]:
-                        V2_EXECUTE_RESPONSE_CONTRACT,
-                      [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
-                      [EXECUTE_TOOL_METADATA_HEADER]: 'true',
-                      [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
-                        toolErrorSchemaVersion,
-                      ),
-                      'x-deepline-request-id': deeplineRequestId,
-                      ...(providerIdempotencyKey
-                        ? {
-                            'x-deepline-idempotency-key':
-                              providerIdempotencyKey,
-                          }
-                        : {}),
-                      ...protectionHeaders,
-                      ...(this.#options.runtimeTestFaultHeader
-                        ? {
-                            'x-deepline-test-fault':
-                              this.#options.runtimeTestFaultHeader,
-                          }
-                        : {}),
-                    },
-                    body: serializedRequestBody(invocationAttempt),
-                  });
-                  if (response.ok) {
-                    try {
-                      responseData = await readToolExecuteResponseBody({
-                        toolId,
-                        abortController,
-                        read: () =>
-                          response!.json() as Promise<Record<string, unknown>>,
-                      });
-                    } catch (error) {
-                      if (error instanceof SyntaxError) {
-                        throw new ToolExecuteInvalidJsonError(error);
+                  // Provider admission is our queue, not provider execution.
+                  // Start the tool deadline only after admission so a busy
+                  // runtime cannot consume the remote-call budget while this
+                  // attempt is still waiting for permission to leave.
+                  if (
+                    timeoutMs &&
+                    abortController &&
+                    !abortController.signal.aborted
+                  ) {
+                    timeoutHandle = setTimeout(() => {
+                      abortController.abort(
+                        new Error(
+                          `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
+                        ),
+                      );
+                    }, timeoutMs);
+                  }
+                  const fixtureBehavior = this.#options.fixtureBehavior;
+                  if (
+                    fixtureExecution &&
+                    fixtureBehavior &&
+                    shouldRouteFixtureToolId(toolId)
+                  ) {
+                    const canonicalProvider =
+                      (await this.#options.getToolProvider?.(toolId))?.trim() ||
+                      provider;
+                    if (shouldRouteFixtureProvider(canonicalProvider)) {
+                      if (!durableCallReceiptKey) {
+                        throw new Error(
+                          'Configured fixture response timing requires a durable call receipt key.',
+                        );
+                      }
+                      const canonicalOperation =
+                        (
+                          await this.#options.getToolOperation?.(toolId)
+                        )?.trim() || toolId;
+                      const stableRequestKey = `${canonicalProvider}:${canonicalOperation}:${durableCallReceiptKey}`;
+                      const behaviorDigest = stableDigest(
+                        JSON.stringify(fixtureBehavior),
+                      );
+                      const requestKeyDigest = stableDigest(stableRequestKey);
+                      let selected: {
+                        delayMs: number;
+                        sampleIndex: number;
+                      } | null = null;
+                      const delayStartedAt = Date.now();
+                      providerCallStartedAt = delayStartedAt;
+                      try {
+                        selected = await waitForFixtureResponseDelay({
+                          behavior: fixtureBehavior,
+                          stableRequestKey,
+                          signal: abortController?.signal,
+                          onSelected: (value) => {
+                            selected = value;
+                            this.log(
+                              `[fixture.response_delay] ${JSON.stringify({
+                                outcome: 'scheduled',
+                                provider: canonicalProvider,
+                                operation: canonicalOperation,
+                                behavior_digest: `sha256_${behaviorDigest}`,
+                                request_key_digest: `sha256_${requestKeyDigest}`,
+                                delay_ms: value.delayMs,
+                                sample_index: value.sampleIndex,
+                                physical_attempt: physicalAttempt,
+                              })}`,
+                            );
+                          },
+                        });
+                      } catch (error) {
+                        this.log(
+                          `[fixture.response_delay] ${JSON.stringify({
+                            outcome: 'aborted',
+                            provider: canonicalProvider,
+                            operation: canonicalOperation,
+                            behavior_digest: `sha256_${behaviorDigest}`,
+                            request_key_digest: `sha256_${requestKeyDigest}`,
+                            delay_ms: selected?.delayMs ?? null,
+                            sample_index: selected?.sampleIndex ?? null,
+                            physical_attempt: physicalAttempt,
+                            elapsed_ms: Date.now() - delayStartedAt,
+                          })}`,
+                        );
+                        throw error;
                       }
-                      throw new ToolExecuteResponseBodyTransportError(error);
+                      this.log(
+                        `[fixture.response_delay] ${JSON.stringify({
+                          outcome: 'completed',
+                          provider: canonicalProvider,
+                          operation: canonicalOperation,
+                          behavior_digest: `sha256_${behaviorDigest}`,
+                          request_key_digest: `sha256_${requestKeyDigest}`,
+                          delay_ms: selected.delayMs,
+                          sample_index: selected.sampleIndex,
+                          physical_attempt: physicalAttempt,
+                          elapsed_ms: Date.now() - delayStartedAt,
+                        })}`,
+                      );
                     }
-                  } else {
-                    try {
-                      responseErrorText = await readToolExecuteResponseBody({
-                        toolId,
-                        abortController,
-                        read: () => response!.text(),
-                      });
-                    } catch (error) {
-                      throw new ToolExecuteResponseBodyTransportError(error);
+                  }
+                  const integrationRequestLease =
+                    await this.governor.acquireIntegrationRequestSlot({
+                      signal: abortController?.signal,
+                    });
+                  try {
+                    integrationFetchStartedAt = Date.now();
+                    providerCallStartedAt ??= integrationFetchStartedAt;
+                    fetchDispatched = true;
+                    response = await fetch(url, {
+                      method: 'POST',
+                      signal: abortController?.signal,
+                      headers: {
+                        'Content-Type': 'application/json',
+                        Authorization: `Bearer ${this.#options.executorToken}`,
+                        [EXECUTE_RESPONSE_CONTRACT_HEADER]:
+                          V2_EXECUTE_RESPONSE_CONTRACT,
+                        [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
+                        [EXECUTE_TOOL_METADATA_HEADER]: 'true',
+                        [TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
+                          toolErrorSchemaVersion,
+                        ),
+                        'x-deepline-request-id': deeplineRequestId,
+                        ...(providerIdempotencyKey
+                          ? {
+                              'x-deepline-idempotency-key':
+                                providerIdempotencyKey,
+                            }
+                          : {}),
+                        ...protectionHeaders,
+                        ...(this.#options.runtimeTestFaultHeader
+                          ? {
+                              'x-deepline-test-fault':
+                                this.#options.runtimeTestFaultHeader,
+                            }
+                          : {}),
+                      },
+                      body: serializedRequestBody(invocationAttempt),
+                    });
+                    if (response.ok) {
+                      try {
+                        responseData = await readToolExecuteResponseBody({
+                          toolId,
+                          abortController,
+                          read: () =>
+                            response!.json() as Promise<
+                              Record<string, unknown>
+                            >,
+                        });
+                      } catch (error) {
+                        if (error instanceof SyntaxError) {
+                          throw new ToolExecuteInvalidJsonError(error);
+                        }
+                        throw new ToolExecuteResponseBodyTransportError(error);
+                      }
+                    } else {
+                      try {
+                        responseErrorText = await readToolExecuteResponseBody({
+                          toolId,
+                          abortController,
+                          read: () => response!.text(),
+                        });
+                      } catch (error) {
+                        throw new ToolExecuteResponseBodyTransportError(error);
+                      }
                     }
+                  } finally {
+                    integrationRequestLease.release();
                   }
                 } finally {
                   providerPermit.release();
                 }
-                if (runtimeReceiptReadTraceEnabled) {
+                if (
+                  runtimeReceiptReadTraceEnabled &&
+                  integrationFetchStartedAt !== null
+                ) {
                   this.log(
                     `[perf] tool call id=${toolId} phase=integration_fetch_body elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
                   );
@@ -9877,8 +10033,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       durableFenceVerificationSignal,
                     )
                     .catch(() => false)) === true);
+              // Fixture execution has no provider-side effect. Replaying the
+              // same stable request after our own transport fails is always
+              // safe, even when this runner lacks a live-provider fence.
+              const fixtureReplaySafe =
+                this.#options.integrationMode === 'fixture';
               const transportReplaySafe =
-                !ambiguousDispatchedFailure || durableInvocationFenceVerified;
+                !ambiguousDispatchedFailure ||
+                durableInvocationFenceVerified ||
+                fixtureReplaySafe;
               if (!transportReplaySafe) {
                 const diagnostic = describeTransportError(transportError);
                 this.log(
@@ -9946,16 +10109,25 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
             if (!response.ok) {
               const text = responseErrorText ?? '';
-              if (
+              const developerTunnelOrigin502 =
                 isDeeplineDeveloperTunnelOrigin502({
                   url,
                   status: response.status,
                   bodyText: text,
-                })
-              ) {
+                });
+              const unmarkedExecutionGateway502 = isUnmarkedExecutionGateway502(
+                {
+                  url,
+                  status: response.status,
+                  responseHeaders: response.headers,
+                },
+              );
+              if (developerTunnelOrigin502 || unmarkedExecutionGateway502) {
                 await retryToolTransportFailure({
                   error: new Error(
-                    'the Deepline development tunnel returned a branded 502 before reaching the app origin',
+                    developerTunnelOrigin502
+                      ? 'the Deepline development tunnel returned a branded 502 before reaching the app origin'
+                      : 'the execution gateway ingress returned an unmarked 502 before reaching the durable invocation handler',
                   ),
                   elapsedMs: providerCallElapsedMs ?? 0,
                   requestId: deeplineRequestId,

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -42,6 +42,7 @@ import type {
   ToolExecutionErrorSchemaVersion,
   ToolExecutionFailureV1,
 } from '../tool-execution-error';
+import type { FixtureBehavior } from './fixture-behavior';
 
 export interface RowState {
   results: Map<string, unknown>;
@@ -512,6 +513,8 @@ export interface ContextOptions {
   vercelProtectionBypassToken?: string | null;
   /** Optional per-run integration execution mode for provider calls. */
   integrationMode?: 'live' | 'eval_stub' | 'fixture';
+  /** Internal fixture-only simulation of provider response residence. */
+  fixtureBehavior?: FixtureBehavior | null;
   /** Preview/dev test seam that applies provider pacing to fixture responses. */
   enforceFixtureProviderPacing?: boolean;
   orgId?: string;
@@ -641,6 +644,7 @@ export interface ContextOptions {
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolProvider?: (toolId: string) => Promise<string | null>;
+  getToolOperation?: (toolId: string) => Promise<string | null>;
   getToolRetryPolicy?: (
     toolId: string,
     input: Record<string, unknown>,

package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts

@@ -1,3 +1,8 @@
+import {
+  validateFixtureBehavior,
+  type FixtureBehavior,
+} from './fixture-behavior';
+
 export const RUNTIME_EXECUTION_CAPABILITIES = [
   'artifact.read',
   'file.read',
@@ -37,6 +42,7 @@ export type RuntimeAuthorityDescriptor = {
   allowedSecrets: string[];
   maxCreditsPerRun?: number | null;
   integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
+  fixtureBehavior?: FixtureBehavior | null;
   synthetic?: boolean | null;
   actorUserId?: string | null;
   actorEmail?: string | null;
@@ -91,11 +97,15 @@ export function requireRuntimeAuthorityDescriptor(
   const capabilities = normalizeRuntimeExecutionCapabilities(
     descriptor.capabilities,
   );
+  const fixtureBehavior = validateFixtureBehavior(descriptor.fixtureBehavior);
   if (
     requiredStrings.some(
       (entry) => typeof entry !== 'string' || entry.trim().length === 0,
     ) ||
-    capabilities.length === 0
+    capabilities.length === 0 ||
+    !fixtureBehavior.ok ||
+    (fixtureBehavior.behavior !== null &&
+      descriptor.integrationMode !== 'fixture')
   ) {
     throw new Error(
       'Durable play launch contains an invalid runtimeAuthority descriptor.',
@@ -114,6 +124,7 @@ export function requireRuntimeAuthorityDescriptor(
       : [],
     maxCreditsPerRun: descriptor.maxCreditsPerRun ?? null,
     integrationMode: descriptor.integrationMode ?? null,
+    fixtureBehavior: fixtureBehavior.ok ? fixtureBehavior.behavior : null,
     synthetic: descriptor.synthetic === true,
     actorUserId: descriptor.actorUserId ?? null,
     actorEmail: descriptor.actorEmail ?? null,