deepline 0.2.55 → 0.2.56

package/dist/bundling-sources/sdk/src/client.ts

@@ -2172,6 +2172,14 @@ export class DeeplineClient {
           : {}),
         ...(request.force ? { force: true } : {}),
         ...(forceToolRefresh ? { forceToolRefresh: true } : {}),
+        ...(typeof request.maxConcurrentExternalCalls === 'number'
+          ? {
+              maxConcurrentExternalCalls: request.maxConcurrentExternalCalls,
+            }
+          : {}),
+        ...(typeof request.maxConcurrentRows === 'number'
+          ? { maxConcurrentRows: request.maxConcurrentRows }
+          : {}),
         ...(typeof request.waitForCompletionMs === 'number'
           ? { waitForCompletionMs: request.waitForCompletionMs }
           : {}),
@@ -2247,6 +2255,12 @@ export class DeeplineClient {
         : {}),
       ...(request.force ? { force: true } : {}),
       ...(forceToolRefresh ? { forceToolRefresh: true } : {}),
+      ...(typeof request.maxConcurrentExternalCalls === 'number'
+        ? { maxConcurrentExternalCalls: request.maxConcurrentExternalCalls }
+        : {}),
+      ...(typeof request.maxConcurrentRows === 'number'
+        ? { maxConcurrentRows: request.maxConcurrentRows }
+        : {}),
       ...(typeof request.waitForCompletionMs === 'number'
         ? { waitForCompletionMs: request.waitForCompletionMs }
         : {}),

package/dist/bundling-sources/sdk/src/http.ts

@@ -652,10 +652,28 @@ export class HttpClient {
         if (error instanceof AuthError || error instanceof DeeplineError) {
           throw error;
         }
-        lastError = error instanceof Error ? error : new Error(String(error));
+        const normalized =
+          error instanceof Error ? error : new Error(String(error));
+        if (isAbortLikeError(normalized) && options?.signal?.aborted) {
+          throw new DeeplineError(
+            `Stream from ${this.config.baseUrl} was aborted by the caller.`,
+            undefined,
+            'ABORTED',
+          );
+        }
+        lastError = normalized;
       }
     }
 
+    if (lastError && isAbortLikeError(lastError)) {
+      throw new DeeplineError(
+        withCoworkNetworkHint(
+          `Unable to stream from ${this.config.baseUrl}. The remote stream was interrupted.`,
+        ),
+        undefined,
+        'PLAY_STREAM_NETWORK_ABORTED',
+      );
+    }
     throw new DeeplineError(
       withCoworkNetworkHint(
         lastError?.message

package/dist/bundling-sources/sdk/src/release.ts

@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: '0.2.55',
+  version: '0.2.56',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/sdk/src/stream-reconnect.ts

@@ -30,6 +30,12 @@ export function streamReconnectDelayMs(attempt: number): number {
 }
 
 export function isTransientPlayStreamError(error: unknown): boolean {
+  if (
+    error instanceof DeeplineError &&
+    error.code === 'PLAY_STREAM_NETWORK_ABORTED'
+  ) {
+    return true;
+  }
   if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
     // Server-shaped errors with a definite status code are NOT transient by
     // pattern — only network-level failures are. 5xx counts as transient

package/dist/bundling-sources/sdk/src/types.ts

@@ -1488,6 +1488,13 @@ export interface StartPlayRunRequest {
   force?: boolean;
   /** Explicit cache-bypass flag for durable dataset and tool-call reuse. */
   forceToolRefresh?: boolean;
+  /**
+   * Per-run ceiling for concurrently resident provider-tool executions and
+   * direct ctx.fetch calls. The server validates the supported range.
+   */
+  maxConcurrentExternalCalls?: number;
+  /** Run-wide default and ceiling for live dataset-map row resolvers. */
+  maxConcurrentRows?: number;
   /** Optionally let the start request wait briefly and return a terminal result. */
   waitForCompletionMs?: number;
   /**

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -1,4 +1,5 @@
 import { createWriteStream } from 'node:fs';
+import { randomUUID } from 'node:crypto';
 import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
@@ -39,13 +40,18 @@ import type {
   PlayRunnerRateStateReleaseInput,
 } from '@shared_libs/play-runtime/protocol';
 import {
+  PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER,
   PLAY_RUNTIME_CONTRACT,
   PLAY_RUNTIME_CONTRACT_HEADER,
 } from '@shared_libs/play-runtime/runtime-contract';
 import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
-import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
+import {
+  PLAY_RUNTIME_TEST_FAULT_HEADER,
+  recognizedRuntimeTestFaultCount,
+} from '@shared_libs/play-runtime/test-runtime-seams';
 import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
 import type { RuntimeReceiptAction } from '@shared_libs/play-runtime/runtime-actions';
+import { RUNTIME_CAPACITY_POLICY } from '@shared_libs/play-runtime/runtime-capacity-policy';
 
 export type StoredPlayArtifactPayload = {
   sourceCode: string;
@@ -244,7 +250,8 @@ function applyRetryJitter(delayMs: number): number {
   const half = delayMs / 2;
   return Math.round(half + Math.random() * half);
 }
-const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+const APP_RUNTIME_API_DEFAULT_REQUEST_TIMEOUT_MS =
+  RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
 const APP_RUNTIME_RECEIPT_RETRY_TELEMETRY_TAG =
   '[perf][worker.receipt_api.transport]';
 const RUN_STATUS_LEDGER_SNAPSHOT_CACHE_LIMIT = 1_000;
@@ -254,10 +261,103 @@ const runStatusLedgerSnapshots = new Map<string, PlayRunLedgerSnapshot>();
 // ingestion skips overlap within that same attempt.
 const runLogChannelSentCounts = new Map<string, number>();
 const runStatusUpdateChains = new Map<string, Promise<void>>();
+const receiptClaimResponseTimeoutFaults = new Map<string, number>();
+const receiptClaimQueryHoldFaults = new Set<string>();
+const RECEIPT_CLAIM_RESPONSE_TIMEOUT_FAULT_LIMIT = 1_000;
 const APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES = 128 * 1024;
 const APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES = 512 * 1024;
 const appRuntimeTextEncoder = new TextEncoder();
 
+function shouldSuppressBulkClaimResponse(input: {
+  body: RuntimeApiRequest;
+  runtimeTestFaultHeader?: string | null;
+}): boolean {
+  if (
+    input.body.action !== 'claim_runtime_step_receipts' ||
+    input.body.keys.length < 128
+  ) {
+    return false;
+  }
+  const requested = recognizedRuntimeTestFaultCount(
+    input.runtimeTestFaultHeader,
+    'receipt_claim_response_timeout',
+  );
+  if (requested <= 0) return false;
+  const key = `${input.body.runId}:${input.body.runAttempt ?? 0}`;
+  const consumed = receiptClaimResponseTimeoutFaults.get(key) ?? 0;
+  if (consumed >= requested) return false;
+  receiptClaimResponseTimeoutFaults.set(key, consumed + 1);
+  while (
+    receiptClaimResponseTimeoutFaults.size >
+    RECEIPT_CLAIM_RESPONSE_TIMEOUT_FAULT_LIMIT
+  ) {
+    const oldest = receiptClaimResponseTimeoutFaults.keys().next().value;
+    if (typeof oldest !== 'string') break;
+    receiptClaimResponseTimeoutFaults.delete(oldest);
+  }
+  return true;
+}
+
+function runtimeTestFaultHeaderForRequest(input: {
+  body: RuntimeApiRequest;
+  runtimeTestFaultHeader?: string | null;
+}): string | null {
+  const header = input.runtimeTestFaultHeader?.trim();
+  if (!header) return null;
+  if (
+    input.body.action !== 'claim_runtime_step_receipts' ||
+    input.body.keys.length < 128 ||
+    recognizedRuntimeTestFaultCount(
+      header,
+      'receipt_claim_query_hold_once_ms',
+    ) <= 0
+  ) {
+    return header;
+  }
+
+  const key = `${input.body.runId}:${input.body.runAttempt ?? 0}`;
+  if (!receiptClaimQueryHoldFaults.has(key)) {
+    receiptClaimQueryHoldFaults.add(key);
+    while (
+      receiptClaimQueryHoldFaults.size >
+      RECEIPT_CLAIM_RESPONSE_TIMEOUT_FAULT_LIMIT
+    ) {
+      const oldest = receiptClaimQueryHoldFaults.values().next().value;
+      if (typeof oldest !== 'string') break;
+      receiptClaimQueryHoldFaults.delete(oldest);
+    }
+    return header;
+  }
+
+  const filtered = header
+    .split(',')
+    .map((part) => part.trim())
+    .filter(
+      (part) =>
+        part &&
+        part.split(':', 1)[0]?.trim() !== 'receipt_claim_query_hold_once_ms',
+    )
+    .join(',');
+  return filtered || null;
+}
+
+async function suppressBulkClaimResponseUntilTimeout(
+  signal: AbortSignal,
+): Promise<never> {
+  if (signal.aborted) {
+    throw signal.reason ?? new Error('Runtime receipt request was aborted.');
+  }
+  return await new Promise<never>((_resolve, reject) => {
+    const onAbort = () => {
+      signal.removeEventListener('abort', onAbort);
+      reject(
+        signal.reason ?? new Error('Runtime receipt request was aborted.'),
+      );
+    };
+    signal.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
 function splitRunEventForAppRuntime(
   event: PlayRunLedgerEvent,
 ): PlayRunLedgerEvent[] {
@@ -633,6 +733,7 @@ type AppRuntimeRetryFailureKind =
 
 type AppRuntimeRetryFailure = {
   attempt: number;
+  transportAttemptId: string;
   failureKind: AppRuntimeRetryFailureKind;
   attemptElapsedMs: number;
   retryDelayMs: number;
@@ -674,6 +775,7 @@ function appRuntimeTransportFailureKind(
 function recordAppRuntimeRetryFailure(input: {
   telemetry: AppRuntimeRetryTelemetry;
   attempt: number;
+  transportAttemptId: string;
   failureKind: AppRuntimeRetryFailureKind;
   attemptStartedAt: number;
   retryDelayMs: number;
@@ -685,6 +787,7 @@ function recordAppRuntimeRetryFailure(input: {
   if (input.telemetry.failures.length < 8) {
     input.telemetry.failures.push({
       attempt: input.attempt,
+      transportAttemptId: input.transportAttemptId,
       failureKind: input.failureKind,
       attemptElapsedMs: Date.now() - input.attemptStartedAt,
       retryDelayMs: input.retryDelayMs,
@@ -734,6 +837,7 @@ function emitAppRuntimeRetrySummary(input: {
 async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   action: RuntimeApiRequest['action'];
   attempt: number;
+  transportAttemptId: string;
   maxAttempts: number;
   error: unknown;
   telemetry: AppRuntimeRetryTelemetry;
@@ -746,6 +850,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
     recordAppRuntimeRetryFailure({
       telemetry: input.telemetry,
       attempt: input.attempt,
+      transportAttemptId: input.transportAttemptId,
       failureKind: 'response_body_error',
       attemptStartedAt: input.attemptStartedAt,
       retryDelayMs: 0,
@@ -775,6 +880,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
     recordAppRuntimeRetryFailure({
       telemetry: input.telemetry,
       attempt: input.attempt,
+      transportAttemptId: input.transportAttemptId,
       failureKind: 'response_body_timeout',
       attemptStartedAt: input.attemptStartedAt,
       retryDelayMs,
@@ -786,6 +892,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   recordAppRuntimeRetryFailure({
     telemetry: input.telemetry,
     attempt: input.attempt,
+    transportAttemptId: input.transportAttemptId,
     failureKind: 'response_body_timeout',
     attemptStartedAt: input.attemptStartedAt,
     retryDelayMs: 0,
@@ -802,6 +909,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   throw new AppRuntimeApiTransportError({
     action: input.action,
     attempts: input.attempt,
+    transportAttemptId: input.transportAttemptId,
     cause: input.error,
     boundaryLabel: input.boundaryLabel,
   });
@@ -818,22 +926,28 @@ function runtimeApiBoundaryLabel(
 export class AppRuntimeApiTransportError extends Error {
   readonly action: RuntimeApiRequest['action'];
   readonly attempts: number;
+  readonly transportAttemptId: string | null;
 
   constructor(input: {
     action: RuntimeApiRequest['action'];
     attempts: number;
+    transportAttemptId?: string | null;
     cause: unknown;
     boundaryLabel?: string;
   }) {
     const causeMessage =
       input.cause instanceof Error ? input.cause.message : String(input.cause);
     super(
-      `${input.boundaryLabel ?? 'App runtime API'} transport exhausted action=${input.action} attempts=${input.attempts}: ${causeMessage}`,
+      `${input.boundaryLabel ?? 'App runtime API'} transport exhausted action=${input.action} attempts=${input.attempts}: ${causeMessage}` +
+        (input.transportAttemptId
+          ? ` (transport_attempt=${input.transportAttemptId})`
+          : ''),
       { cause: input.cause },
     );
     this.name = 'AppRuntimeApiTransportError';
     this.action = input.action;
     this.attempts = input.attempts;
+    this.transportAttemptId = input.transportAttemptId ?? null;
   }
 }
 
@@ -874,6 +988,7 @@ export class AppRuntimeApiResponseError extends Error {
 
 const APP_RUNTIME_CAPACITY_ERROR_CODES = new Set([
   'receipt_db_admission_backpressure',
+  'runtime_receipt_claim_timeout',
   'scheduler_capacity',
   'runtime_postgres_admission_backpressure',
   'RUNTIME_SCHEDULER_SATURATED',
@@ -970,7 +1085,15 @@ async function postAppRuntimeApi<TResponse>(
 
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     const attemptStartedAt = Date.now();
+    const transportAttemptId = randomUUID();
+    const requestSignal = context.signal
+      ? AbortSignal.any([context.signal, AbortSignal.timeout(requestTimeoutMs)])
+      : AbortSignal.timeout(requestTimeoutMs);
     let response: Response;
+    const runtimeTestFaultHeader = runtimeTestFaultHeaderForRequest({
+      body,
+      runtimeTestFaultHeader: context.runtimeTestFaultHeader,
+    });
     try {
       response = await runtimeFetch(resolveAppRuntimeApiUrl(context), {
         method: 'POST',
@@ -978,21 +1101,16 @@ async function postAppRuntimeApi<TResponse>(
           'content-type': 'application/json',
           authorization: `Bearer ${token}`,
           [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+          [PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER]: transportAttemptId,
           ...vercelHeaders,
-          ...(context.runtimeTestFaultHeader?.trim()
+          ...(runtimeTestFaultHeader
             ? {
-                [PLAY_RUNTIME_TEST_FAULT_HEADER]:
-                  context.runtimeTestFaultHeader.trim(),
+                [PLAY_RUNTIME_TEST_FAULT_HEADER]: runtimeTestFaultHeader,
               }
             : {}),
         },
         body: JSON.stringify(body),
-        signal: context.signal
-          ? AbortSignal.any([
-              context.signal,
-              AbortSignal.timeout(requestTimeoutMs),
-            ])
-          : AbortSignal.timeout(requestTimeoutMs),
+        signal: requestSignal,
       });
     } catch (error) {
       if (context.signal?.aborted) {
@@ -1008,6 +1126,7 @@ async function postAppRuntimeApi<TResponse>(
         recordAppRuntimeRetryFailure({
           telemetry: retryTelemetry,
           attempt,
+          transportAttemptId,
           failureKind: appRuntimeTransportFailureKind(error),
           attemptStartedAt,
           retryDelayMs,
@@ -1018,6 +1137,7 @@ async function postAppRuntimeApi<TResponse>(
       recordAppRuntimeRetryFailure({
         telemetry: retryTelemetry,
         attempt,
+        transportAttemptId,
         failureKind: appRuntimeTransportFailureKind(error),
         attemptStartedAt,
         retryDelayMs: 0,
@@ -1033,6 +1153,7 @@ async function postAppRuntimeApi<TResponse>(
       throw new AppRuntimeApiTransportError({
         action: body.action,
         attempts: attempt,
+        transportAttemptId,
         cause: error,
         boundaryLabel,
       });
@@ -1040,6 +1161,14 @@ async function postAppRuntimeApi<TResponse>(
     if (response.ok) {
       try {
         const parsed = (await response.json()) as TResponse;
+        if (
+          shouldSuppressBulkClaimResponse({
+            body,
+            runtimeTestFaultHeader: context.runtimeTestFaultHeader,
+          })
+        ) {
+          await suppressBulkClaimResponseUntilTimeout(requestSignal);
+        }
         emitAppRuntimeRetrySummary({
           telemetry: retryTelemetry,
           action: body.action,
@@ -1053,6 +1182,7 @@ async function postAppRuntimeApi<TResponse>(
         await retryAppRuntimeBodyTimeoutOrThrow({
           action: body.action,
           attempt,
+          transportAttemptId,
           maxAttempts,
           error,
           telemetry: retryTelemetry,
@@ -1074,6 +1204,7 @@ async function postAppRuntimeApi<TResponse>(
         recordAppRuntimeRetryFailure({
           telemetry: retryTelemetry,
           attempt,
+          transportAttemptId,
           failureKind: 'response_body_timeout',
           attemptStartedAt,
           retryDelayMs: 0,
@@ -1095,6 +1226,7 @@ async function postAppRuntimeApi<TResponse>(
       await retryAppRuntimeBodyTimeoutOrThrow({
         action: body.action,
         attempt,
+        transportAttemptId,
         maxAttempts,
         error,
         telemetry: retryTelemetry,
@@ -1135,6 +1267,7 @@ async function postAppRuntimeApi<TResponse>(
       recordAppRuntimeRetryFailure({
         telemetry: retryTelemetry,
         attempt,
+        transportAttemptId,
         failureKind: 'retryable_http',
         attemptStartedAt,
         retryDelayMs,
@@ -1146,6 +1279,7 @@ async function postAppRuntimeApi<TResponse>(
     recordAppRuntimeRetryFailure({
       telemetry: retryTelemetry,
       attempt,
+      transportAttemptId,
       failureKind: isRetryableAppRuntimeResponse({
         action: body.action,
         status: response.status,

package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts

@@ -52,8 +52,7 @@ function planExecutionChunks<TRequest>(
   for (const request of requests) {
     if (
       current.length > 0 &&
-      (current.length >= maxUnits ||
-        (weightCap != null && weight >= weightCap))
+      (current.length >= maxUnits || (weightCap != null && weight >= weightCap))
     ) {
       chunks.push(current);
       current = [];
@@ -91,6 +90,8 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
   onChunkComplete?: (
     results: Array<ChunkExecutionResult<TRequest, TResult>>,
   ) => void | Promise<void>;
+  /** Keep settled entries for the returned aggregate. Defaults to true. */
+  retainResults?: boolean;
 }): Promise<Array<ChunkExecutionResult<TRequest, TResult>>> {
   const results: Array<ChunkExecutionResult<TRequest, TResult>> = [];
 
@@ -105,7 +106,9 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
     const notify = async (
       entry: ChunkExecutionResult<TRequest, TResult>,
     ): Promise<void> => {
-      results.push(entry);
+      if (input.retainResults !== false) {
+        results.push(entry);
+      }
       notifyChain = notifyChain.then(
         async () => await input.onChunkComplete?.([entry]),
       );